diff --git a/VERSION b/VERSION index 7fe52d36..b213cbdc 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v2.2.1 +v2.3.0-dev diff --git a/config.yaml b/config.yaml index f061e25c..a54de891 100644 --- a/config.yaml +++ b/config.yaml @@ -13,7 +13,7 @@ logging: data_inspection.inspector: debug: false data_analysis.detector: - debug: true + debug: false pipeline: acceleration: @@ -40,7 +40,7 @@ pipeline: modules: log_storage.logserver: executor: thread - max_workers: 4 + max_workers: 1 log_collection.collector: executor: thread max_workers: 1 @@ -59,7 +59,7 @@ pipeline: max_workers: 1 data_inspection.inspector: executor: thread - max_workers: 2 + max_workers: 1 instances: dga_inspector: max_workers: 1 @@ -72,7 +72,7 @@ pipeline: RF-dga_detector: max_workers: 1 domainator: - max_workers: 2 + max_workers: 1 domainator_attributor: max_workers: 1 domainator_attributor_behaviour: @@ -89,7 +89,7 @@ pipeline: max_workers: 1 monitoring.agent: executor: thread - max_workers: 2 + max_workers: 1 log_storage: logserver: input_file: "/opt/file.txt" @@ -274,6 +274,14 @@ pipeline: external_kafka_topic: "hamstring_alerts" plugins: [] monitoring: + kafka_consumer: + # Monitoring produces several records per pipeline message. Consume them + # in larger blocks so schema decoding, ClickHouse writes, and offset + # commits are amortized across the batch. + batch_size: 5000 + # Match the ClickHouse partial-batch window. A shorter timeout forces + # sparse tables into many small synchronous inserts. + timeout_ms: 2000 clickhouse_connector: batch_size: 50 # do not set higher batch_timeout: 2.0 @@ -282,7 +290,7 @@ pipeline: zeek: sensors: zeek: - static_analysis: false + static_analysis: true protocols: - dns interfaces: @@ -304,9 +312,13 @@ environment: node_ip: 127.0.0.1 kafka_consumer: # Allow long-running detector batches without Kafka forcing a group rebalance. - # Default librdkafka value is 300000 ms (5 minutes), which can be too short - # for model inference plus downstream alert/monitoring writes. - max_poll_interval_ms: 1800000 + # Default librdkafka value is 300000 ms (5 minutes) + max_poll_interval_ms: 300000 + kafka_transaction_batch: + # Each stage processes this many Kafka records before atomically publishing + # all outputs and committing the corresponding source offsets. + size: 250 + timeout_ms: 50 kafka_topics: replication_factor: 3 auto_expand_partitions: true diff --git a/docker/create_tables/zz_monitoring_rollups.sql b/docker/create_tables/zz_monitoring_rollups.sql index a6782a7a..38e08c86 100644 --- a/docker/create_tables/zz_monitoring_rollups.sql +++ b/docker/create_tables/zz_monitoring_rollups.sql @@ -6,7 +6,10 @@ CREATE TABLE IF NOT EXISTS alerts_1m ( ENGINE = AggregatingMergeTree PARTITION BY toYYYYMM(time_bucket) ORDER BY (time_bucket, src_ip) -TTL toDateTime(time_bucket) + INTERVAL 1 DAY; +TTL toDateTime(time_bucket) + INTERVAL 7 DAY; + +ALTER TABLE alerts_1m +MODIFY TTL toDateTime(time_bucket) + INTERVAL 7 DAY; CREATE MATERIALIZED VIEW IF NOT EXISTS alerts_1m_mv @@ -31,7 +34,10 @@ CREATE TABLE IF NOT EXISTS fill_levels_1m ( ENGINE = AggregatingMergeTree PARTITION BY toYYYYMM(time_bucket) ORDER BY (stage, entry_type, time_bucket) -TTL toDateTime(time_bucket) + INTERVAL 1 DAY; +TTL toDateTime(time_bucket) + INTERVAL 7 DAY; + +ALTER TABLE fill_levels_1m +MODIFY TTL toDateTime(time_bucket) + INTERVAL 7 DAY; CREATE MATERIALIZED VIEW IF NOT EXISTS fill_levels_1m_mv TO fill_levels_1m diff --git a/docker/create_tables/zzz_monitoring_history.sql b/docker/create_tables/zzz_monitoring_history.sql new file mode 100644 index 00000000..4ef24cbc --- /dev/null +++ b/docker/create_tables/zzz_monitoring_history.sql @@ -0,0 +1,649 @@ +-- Multi-resolution monitoring history. +-- +-- Detailed source events remain short-lived. Aggregate tables retain one-minute +-- data for seven days, fifteen-minute data for thirty days, and hourly data for +-- ninety days. The history views use non-overlapping time windows so queries do +-- not double-count buckets available at more than one resolution. + +CREATE TABLE IF NOT EXISTS alerts_15m ( + time_bucket DateTime NOT NULL, + src_ip String NOT NULL, + alert_count SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(time_bucket) +ORDER BY (time_bucket, src_ip) +TTL toDateTime(time_bucket) + INTERVAL 30 DAY; + +ALTER TABLE alerts_15m +MODIFY TTL toDateTime(time_bucket) + INTERVAL 30 DAY; + +INSERT INTO alerts_15m +SELECT + toStartOfInterval(alert_timestamp, INTERVAL 15 MINUTE) AS time_bucket, + src_ip, + count() AS alert_count +FROM alerts +WHERE alert_timestamp >= now() - INTERVAL 30 DAY + AND (SELECT count() FROM alerts_15m) = 0 +GROUP BY time_bucket, src_ip; + +CREATE MATERIALIZED VIEW IF NOT EXISTS alerts_15m_mv +TO alerts_15m +AS +SELECT + toStartOfInterval(alert_timestamp, INTERVAL 15 MINUTE) AS time_bucket, + src_ip, + count() AS alert_count +FROM alerts +GROUP BY time_bucket, src_ip; + +CREATE TABLE IF NOT EXISTS alerts_1h ( + time_bucket DateTime NOT NULL, + src_ip String NOT NULL, + alert_count SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(time_bucket) +ORDER BY (time_bucket, src_ip) +TTL toDateTime(time_bucket) + INTERVAL 90 DAY; + +ALTER TABLE alerts_1h +MODIFY TTL toDateTime(time_bucket) + INTERVAL 90 DAY; + +INSERT INTO alerts_1h +SELECT + toStartOfHour(alert_timestamp) AS time_bucket, + src_ip, + count() AS alert_count +FROM alerts +WHERE alert_timestamp >= now() - INTERVAL 90 DAY + AND (SELECT count() FROM alerts_1h) = 0 +GROUP BY time_bucket, src_ip; + +CREATE MATERIALIZED VIEW IF NOT EXISTS alerts_1h_mv +TO alerts_1h +AS +SELECT + toStartOfHour(alert_timestamp) AS time_bucket, + src_ip, + count() AS alert_count +FROM alerts +GROUP BY time_bucket, src_ip; + +CREATE OR REPLACE VIEW alerts_history AS +SELECT time_bucket, src_ip, alert_count +FROM alerts_1m +WHERE time_bucket >= now() - INTERVAL 7 DAY +UNION ALL +SELECT time_bucket, src_ip, alert_count +FROM alerts_15m +WHERE time_bucket >= now() - INTERVAL 30 DAY + AND time_bucket < now() - INTERVAL 7 DAY +UNION ALL +SELECT time_bucket, src_ip, alert_count +FROM alerts_1h +WHERE time_bucket >= now() - INTERVAL 90 DAY + AND time_bucket < now() - INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS fill_levels_15m ( + time_bucket DateTime NOT NULL, + stage LowCardinality(String) NOT NULL, + entry_type LowCardinality(String) NOT NULL, + min_count SimpleAggregateFunction(min, UInt32), + avg_count AggregateFunction(avg, UInt32), + max_count SimpleAggregateFunction(max, UInt32), + median_count AggregateFunction(quantileTDigest(0.5), UInt32) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(time_bucket) +ORDER BY (stage, entry_type, time_bucket) +TTL toDateTime(time_bucket) + INTERVAL 30 DAY; + +ALTER TABLE fill_levels_15m +MODIFY TTL toDateTime(time_bucket) + INTERVAL 30 DAY; + +INSERT INTO fill_levels_15m +SELECT + toStartOfInterval(timestamp, INTERVAL 15 MINUTE) AS time_bucket, + stage, + entry_type, + min(entry_count) AS min_count, + avgState(entry_count) AS avg_count, + max(entry_count) AS max_count, + quantileTDigestState(0.5)(entry_count) AS median_count +FROM fill_levels +WHERE timestamp >= now() - INTERVAL 30 DAY + AND (SELECT count() FROM fill_levels_15m) = 0 +GROUP BY time_bucket, stage, entry_type; + +CREATE MATERIALIZED VIEW IF NOT EXISTS fill_levels_15m_mv +TO fill_levels_15m +AS +SELECT + toStartOfInterval(timestamp, INTERVAL 15 MINUTE) AS time_bucket, + stage, + entry_type, + min(entry_count) AS min_count, + avgState(entry_count) AS avg_count, + max(entry_count) AS max_count, + quantileTDigestState(0.5)(entry_count) AS median_count +FROM fill_levels +GROUP BY time_bucket, stage, entry_type; + +CREATE TABLE IF NOT EXISTS fill_levels_1h ( + time_bucket DateTime NOT NULL, + stage LowCardinality(String) NOT NULL, + entry_type LowCardinality(String) NOT NULL, + min_count SimpleAggregateFunction(min, UInt32), + avg_count AggregateFunction(avg, UInt32), + max_count SimpleAggregateFunction(max, UInt32), + median_count AggregateFunction(quantileTDigest(0.5), UInt32) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(time_bucket) +ORDER BY (stage, entry_type, time_bucket) +TTL toDateTime(time_bucket) + INTERVAL 90 DAY; + +ALTER TABLE fill_levels_1h +MODIFY TTL toDateTime(time_bucket) + INTERVAL 90 DAY; + +INSERT INTO fill_levels_1h +SELECT + toStartOfHour(timestamp) AS time_bucket, + stage, + entry_type, + min(entry_count) AS min_count, + avgState(entry_count) AS avg_count, + max(entry_count) AS max_count, + quantileTDigestState(0.5)(entry_count) AS median_count +FROM fill_levels +WHERE timestamp >= now() - INTERVAL 90 DAY + AND (SELECT count() FROM fill_levels_1h) = 0 +GROUP BY time_bucket, stage, entry_type; + +CREATE MATERIALIZED VIEW IF NOT EXISTS fill_levels_1h_mv +TO fill_levels_1h +AS +SELECT + toStartOfHour(timestamp) AS time_bucket, + stage, + entry_type, + min(entry_count) AS min_count, + avgState(entry_count) AS avg_count, + max(entry_count) AS max_count, + quantileTDigestState(0.5)(entry_count) AS median_count +FROM fill_levels +GROUP BY time_bucket, stage, entry_type; + +CREATE OR REPLACE VIEW fill_levels_history AS +SELECT + time_bucket, + stage, + entry_type, + min_count, + avg_count, + max_count, + median_count +FROM fill_levels_1m +WHERE time_bucket >= now() - INTERVAL 7 DAY +UNION ALL +SELECT + time_bucket, + stage, + entry_type, + min_count, + avg_count, + max_count, + median_count +FROM fill_levels_15m +WHERE time_bucket >= now() - INTERVAL 30 DAY + AND time_bucket < now() - INTERVAL 7 DAY +UNION ALL +SELECT + time_bucket, + stage, + entry_type, + min_count, + avg_count, + max_count, + median_count +FROM fill_levels_1h +WHERE time_bucket >= now() - INTERVAL 90 DAY + AND time_bucket < now() - INTERVAL 30 DAY; + +-- Latency values are first completed per source entity. This avoids calculating +-- a duration from start/end events that arrived in different insert blocks. + +CREATE OR REPLACE VIEW server_log_current_latency_values AS +SELECT + sl.message_id AS message_id, + toDate(slt.event_timestamp) AS event_date, + sl.timestamp_in AS start_timestamp, + slt.event_timestamp AS end_timestamp, + dateDiff('microsecond', sl.timestamp_in, slt.event_timestamp) AS latency_us +FROM server_logs sl +INNER JOIN server_logs_timestamps slt ON sl.message_id = slt.message_id +WHERE slt.event = 'timestamp_out' + AND slt.event_timestamp > sl.timestamp_in; + +CREATE OR REPLACE VIEW logline_stage_current_latency_values AS +SELECT + stage, + logline_id, + min(event_date) AS event_date, + minMerge(start_timestamp) AS start_timestamp, + maxMerge(end_timestamp) AS end_timestamp, + dateDiff('microsecond', start_timestamp, end_timestamp) AS latency_us +FROM logline_stage_latencies +GROUP BY stage, logline_id +HAVING start_timestamp > toDateTime64(0, 6) + AND end_timestamp > start_timestamp; + +CREATE OR REPLACE VIEW batch_stage_current_latency_values AS +SELECT + stage, + instance_name, + batch_id, + min(event_date) AS event_date, + minMerge(start_timestamp) AS start_timestamp, + maxMerge(end_timestamp) AS end_timestamp, + dateDiff('microsecond', start_timestamp, end_timestamp) AS latency_us +FROM batch_stage_latencies +GROUP BY stage, instance_name, batch_id +HAVING start_timestamp > toDateTime64(0, 6) + AND end_timestamp > start_timestamp; + +CREATE OR REPLACE VIEW suspicious_batch_stage_current_latency_values AS +SELECT + stage, + instance_name, + suspicious_batch_id, + min(event_date) AS event_date, + minMerge(start_timestamp) AS start_timestamp, + maxMerge(end_timestamp) AS end_timestamp, + dateDiff('microsecond', start_timestamp, end_timestamp) AS latency_us +FROM suspicious_batch_stage_latencies +GROUP BY stage, instance_name, suspicious_batch_id +HAVING start_timestamp > toDateTime64(0, 6) + AND end_timestamp > start_timestamp; + +CREATE OR REPLACE VIEW pipeline_roundtrip_current_latency_values AS +SELECT + message_id, + toDate(terminal_timestamp) AS event_date, + start_timestamp, + terminal_timestamp AS end_timestamp, + dateDiff('microsecond', start_timestamp, terminal_timestamp) AS latency_us +FROM ( + SELECT + sl.message_id AS message_id, + min(sl.timestamp_in) AS start_timestamp, + max(terminal.terminal_timestamp) AS terminal_timestamp + FROM server_logs sl + INNER JOIN ( + SELECT + sltl.message_id AS message_id, + max(lt.timestamp) AS terminal_timestamp + FROM server_log_to_logline sltl + INNER JOIN logline_timestamps lt ON sltl.logline_id = lt.logline_id + WHERE lt.is_active = false + GROUP BY sltl.message_id + UNION ALL + SELECT + message_id, + max(timestamp) AS terminal_timestamp + FROM server_log_terminal_events + GROUP BY message_id + ) AS terminal ON sl.message_id = terminal.message_id + GROUP BY sl.message_id +) +WHERE terminal_timestamp > start_timestamp; + +CREATE OR REPLACE VIEW pipeline_latency_current_values AS +SELECT + 'server_log' AS family, + 'log_storage.logserver' AS stage, + end_timestamp, + toFloat64(latency_us) AS latency_us +FROM server_log_current_latency_values +UNION ALL +SELECT + 'logline' AS family, + stage, + end_timestamp, + toFloat64(latency_us) AS latency_us +FROM logline_stage_current_latency_values +UNION ALL +SELECT + 'batch' AS family, + stage, + end_timestamp, + toFloat64(latency_us) AS latency_us +FROM batch_stage_current_latency_values +UNION ALL +SELECT + 'suspicious_batch' AS family, + stage, + end_timestamp, + toFloat64(latency_us) AS latency_us +FROM suspicious_batch_stage_current_latency_values +UNION ALL +SELECT + 'roundtrip' AS family, + 'pipeline.roundtrip' AS stage, + end_timestamp, + toFloat64(latency_us) AS latency_us +FROM pipeline_roundtrip_current_latency_values +UNION ALL +SELECT + 'transport' AS family, + 'transport.batch_handler_to_prefilter' AS stage, + bt2.timestamp AS end_timestamp, + toFloat64(dateDiff('microsecond', bt1.timestamp, bt2.timestamp)) AS latency_us +FROM batch_tree bt1 +INNER JOIN batch_tree bt2 ON bt1.batch_row_id = bt2.parent_batch_row_id +WHERE bt1.stage = 'log_collection.batch_handler' + AND bt1.status = 'completed' + AND bt2.stage = 'log_filtering.prefilter' + AND bt2.status = 'in_process' + AND bt2.timestamp > bt1.timestamp +UNION ALL +SELECT + 'transport' AS family, + 'transport.prefilter_to_inspector' AS stage, + bt2.timestamp AS end_timestamp, + toFloat64(dateDiff('microsecond', bt1.timestamp, bt2.timestamp)) AS latency_us +FROM batch_tree bt1 +INNER JOIN batch_tree bt2 ON bt1.batch_row_id = bt2.parent_batch_row_id +WHERE bt1.stage = 'log_filtering.prefilter' + AND bt1.status = 'finished' + AND bt2.stage = 'data_inspection.inspector' + AND bt2.status = 'in_process' + AND bt2.timestamp > bt1.timestamp +UNION ALL +SELECT + 'transport' AS family, + 'transport.inspector_to_detector' AS stage, + bt2.timestamp AS end_timestamp, + toFloat64(dateDiff('microsecond', bt1.timestamp, bt2.timestamp)) AS latency_us +FROM batch_tree bt1 +INNER JOIN batch_tree bt2 ON bt1.batch_row_id = bt2.parent_batch_row_id +WHERE bt1.stage = 'data_inspection.inspector' + AND bt1.status = 'finished' + AND bt2.stage = 'data_analysis.detector' + AND bt2.status = 'in_process' + AND bt2.timestamp > bt1.timestamp; + +CREATE TABLE IF NOT EXISTS pipeline_latency_1m ( + time_bucket DateTime NOT NULL, + family LowCardinality(String) NOT NULL, + stage LowCardinality(String) NOT NULL, + sample_count UInt64 NOT NULL, + min_latency_us Float64 NOT NULL, + avg_latency_us Float64 NOT NULL, + p50_latency_us Float64 NOT NULL, + p95_latency_us Float64 NOT NULL, + p99_latency_us Float64 NOT NULL, + max_latency_us Float64 NOT NULL, + snapshot_version DateTime64(3) NOT NULL +) +ENGINE = ReplacingMergeTree(snapshot_version) +PARTITION BY toYYYYMM(time_bucket) +ORDER BY (family, stage, time_bucket) +TTL toDateTime(time_bucket) + INTERVAL 7 DAY; + +ALTER TABLE pipeline_latency_1m +MODIFY TTL toDateTime(time_bucket) + INTERVAL 7 DAY; + +CREATE MATERIALIZED VIEW IF NOT EXISTS pipeline_latency_1m_refresh +REFRESH EVERY 1 MINUTE APPEND TO pipeline_latency_1m +AS +SELECT + toStartOfMinute(end_timestamp) AS time_bucket, + family, + stage, + count() AS sample_count, + min(latency_us) AS min_latency_us, + avg(latency_us) AS avg_latency_us, + quantileTDigest(0.5)(latency_us) AS p50_latency_us, + quantileTDigest(0.95)(latency_us) AS p95_latency_us, + quantileTDigest(0.99)(latency_us) AS p99_latency_us, + max(latency_us) AS max_latency_us, + now64(3) AS snapshot_version +FROM pipeline_latency_current_values +WHERE end_timestamp >= now() - INTERVAL 2 HOUR + AND end_timestamp < toStartOfMinute(now()) +GROUP BY time_bucket, family, stage; + +CREATE TABLE IF NOT EXISTS pipeline_latency_15m ( + time_bucket DateTime NOT NULL, + family LowCardinality(String) NOT NULL, + stage LowCardinality(String) NOT NULL, + sample_count UInt64 NOT NULL, + min_latency_us Float64 NOT NULL, + avg_latency_us Float64 NOT NULL, + p50_latency_us Float64 NOT NULL, + p95_latency_us Float64 NOT NULL, + p99_latency_us Float64 NOT NULL, + max_latency_us Float64 NOT NULL, + snapshot_version DateTime64(3) NOT NULL +) +ENGINE = ReplacingMergeTree(snapshot_version) +PARTITION BY toYYYYMM(time_bucket) +ORDER BY (family, stage, time_bucket) +TTL toDateTime(time_bucket) + INTERVAL 30 DAY; + +ALTER TABLE pipeline_latency_15m +MODIFY TTL toDateTime(time_bucket) + INTERVAL 30 DAY; + +CREATE MATERIALIZED VIEW IF NOT EXISTS pipeline_latency_15m_refresh +REFRESH EVERY 5 MINUTE APPEND TO pipeline_latency_15m +AS +SELECT + toStartOfInterval(end_timestamp, INTERVAL 15 MINUTE) AS time_bucket, + family, + stage, + count() AS sample_count, + min(latency_us) AS min_latency_us, + avg(latency_us) AS avg_latency_us, + quantileTDigest(0.5)(latency_us) AS p50_latency_us, + quantileTDigest(0.95)(latency_us) AS p95_latency_us, + quantileTDigest(0.99)(latency_us) AS p99_latency_us, + max(latency_us) AS max_latency_us, + now64(3) AS snapshot_version +FROM pipeline_latency_current_values +WHERE end_timestamp >= now() - INTERVAL 6 HOUR + AND end_timestamp < toStartOfInterval(now(), INTERVAL 15 MINUTE) +GROUP BY time_bucket, family, stage; + +CREATE TABLE IF NOT EXISTS pipeline_latency_1h ( + time_bucket DateTime NOT NULL, + family LowCardinality(String) NOT NULL, + stage LowCardinality(String) NOT NULL, + sample_count UInt64 NOT NULL, + min_latency_us Float64 NOT NULL, + avg_latency_us Float64 NOT NULL, + p50_latency_us Float64 NOT NULL, + p95_latency_us Float64 NOT NULL, + p99_latency_us Float64 NOT NULL, + max_latency_us Float64 NOT NULL, + snapshot_version DateTime64(3) NOT NULL +) +ENGINE = ReplacingMergeTree(snapshot_version) +PARTITION BY toYYYYMM(time_bucket) +ORDER BY (family, stage, time_bucket) +TTL toDateTime(time_bucket) + INTERVAL 90 DAY; + +ALTER TABLE pipeline_latency_1h +MODIFY TTL toDateTime(time_bucket) + INTERVAL 90 DAY; + +CREATE MATERIALIZED VIEW IF NOT EXISTS pipeline_latency_1h_refresh +REFRESH EVERY 15 MINUTE APPEND TO pipeline_latency_1h +AS +SELECT + toStartOfHour(end_timestamp) AS time_bucket, + family, + stage, + count() AS sample_count, + min(latency_us) AS min_latency_us, + avg(latency_us) AS avg_latency_us, + quantileTDigest(0.5)(latency_us) AS p50_latency_us, + quantileTDigest(0.95)(latency_us) AS p95_latency_us, + quantileTDigest(0.99)(latency_us) AS p99_latency_us, + max(latency_us) AS max_latency_us, + now64(3) AS snapshot_version +FROM pipeline_latency_current_values +WHERE end_timestamp >= now() - INTERVAL 12 HOUR + AND end_timestamp < toStartOfHour(now()) +GROUP BY time_bucket, family, stage; + +CREATE OR REPLACE VIEW pipeline_latency_rollup_history AS +SELECT * EXCEPT snapshot_version +FROM pipeline_latency_1m FINAL +WHERE time_bucket >= now() - INTERVAL 7 DAY + AND time_bucket < now() - INTERVAL 1 DAY +UNION ALL +SELECT * EXCEPT snapshot_version +FROM pipeline_latency_15m FINAL +WHERE time_bucket >= now() - INTERVAL 30 DAY + AND time_bucket < now() - INTERVAL 7 DAY +UNION ALL +SELECT * EXCEPT snapshot_version +FROM pipeline_latency_1h FINAL +WHERE time_bucket >= now() - INTERVAL 90 DAY + AND time_bucket < now() - INTERVAL 30 DAY; + +CREATE OR REPLACE VIEW pipeline_latency_values AS +SELECT family, stage, end_timestamp, latency_us +FROM pipeline_latency_current_values +WHERE end_timestamp >= now() - INTERVAL 1 DAY +UNION ALL +SELECT + family, + stage, + toDateTime64(time_bucket, 6) AS end_timestamp, + p50_latency_us AS latency_us +FROM pipeline_latency_rollup_history; + +-- Keep the existing dashboard-facing schemas. Recent ranges contain exact +-- per-entity durations; older ranges contain one representative p50 row per +-- retained time bucket. + +CREATE OR REPLACE VIEW server_log_latency_values AS +SELECT + message_id, + event_date, + start_timestamp, + end_timestamp, + toFloat64(latency_us) AS latency_us +FROM server_log_current_latency_values +WHERE end_timestamp >= now() - INTERVAL 1 DAY +UNION ALL +SELECT + toUUID('00000000-0000-0000-0000-000000000000') AS message_id, + toDate(time_bucket) AS event_date, + toDateTime64(time_bucket, 6) AS start_timestamp, + toDateTime64(time_bucket, 6) AS end_timestamp, + p50_latency_us AS latency_us +FROM pipeline_latency_rollup_history +WHERE family = 'server_log'; + +CREATE OR REPLACE VIEW logline_stage_latency_values AS +SELECT + stage, + logline_id, + event_date, + start_timestamp, + end_timestamp, + toFloat64(latency_us) AS latency_us +FROM logline_stage_current_latency_values +WHERE end_timestamp >= now() - INTERVAL 1 DAY +UNION ALL +SELECT + stage, + toUUID('00000000-0000-0000-0000-000000000000') AS logline_id, + toDate(time_bucket) AS event_date, + toDateTime64(time_bucket, 6) AS start_timestamp, + toDateTime64(time_bucket, 6) AS end_timestamp, + p50_latency_us AS latency_us +FROM pipeline_latency_rollup_history +WHERE family = 'logline'; + +CREATE OR REPLACE VIEW batch_stage_latency_values AS +SELECT + stage, + instance_name, + batch_id, + event_date, + start_timestamp, + end_timestamp, + toFloat64(latency_us) AS latency_us +FROM batch_stage_current_latency_values +WHERE end_timestamp >= now() - INTERVAL 1 DAY +UNION ALL +SELECT + stage, + 'historical_rollup' AS instance_name, + toUUID('00000000-0000-0000-0000-000000000000') AS batch_id, + toDate(time_bucket) AS event_date, + toDateTime64(time_bucket, 6) AS start_timestamp, + toDateTime64(time_bucket, 6) AS end_timestamp, + p50_latency_us AS latency_us +FROM pipeline_latency_rollup_history +WHERE family = 'batch'; + +CREATE OR REPLACE VIEW suspicious_batch_stage_latency_values AS +SELECT + stage, + instance_name, + suspicious_batch_id, + event_date, + start_timestamp, + end_timestamp, + toFloat64(latency_us) AS latency_us +FROM suspicious_batch_stage_current_latency_values +WHERE end_timestamp >= now() - INTERVAL 1 DAY +UNION ALL +SELECT + stage, + 'historical_rollup' AS instance_name, + toUUID('00000000-0000-0000-0000-000000000000') AS suspicious_batch_id, + toDate(time_bucket) AS event_date, + toDateTime64(time_bucket, 6) AS start_timestamp, + toDateTime64(time_bucket, 6) AS end_timestamp, + p50_latency_us AS latency_us +FROM pipeline_latency_rollup_history +WHERE family = 'suspicious_batch'; + +CREATE OR REPLACE VIEW pipeline_transport_latency_values AS +SELECT stage, end_timestamp, latency_us +FROM pipeline_latency_current_values +WHERE family = 'transport' + AND end_timestamp >= now() - INTERVAL 1 DAY +UNION ALL +SELECT + stage, + toDateTime64(time_bucket, 6) AS end_timestamp, + p50_latency_us AS latency_us +FROM pipeline_latency_rollup_history +WHERE family = 'transport'; + +CREATE OR REPLACE VIEW pipeline_roundtrip_latency_values AS +SELECT + message_id, + event_date, + start_timestamp, + end_timestamp, + toFloat64(latency_us) AS latency_us +FROM pipeline_roundtrip_current_latency_values +WHERE end_timestamp >= now() - INTERVAL 1 DAY +UNION ALL +SELECT + toUUID('00000000-0000-0000-0000-000000000000') AS message_id, + toDate(time_bucket) AS event_date, + toDateTime64(time_bucket, 6) AS start_timestamp, + toDateTime64(time_bucket, 6) AS end_timestamp, + p50_latency_us AS latency_us +FROM pipeline_latency_rollup_history +WHERE family = 'roundtrip'; diff --git a/docker/data/test_pcaps/ctu-sample.pcap b/docker/data/test_pcaps/ctu-sample.pcap deleted file mode 100755 index b9b8d19c..00000000 Binary files a/docker/data/test_pcaps/ctu-sample.pcap and /dev/null differ diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index fe962042..840d0e8e 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -149,7 +149,7 @@ services: network_mode: host environment: - CONTAINER_NAME=zeek - - HAMSTRING_ZEEK_KAFKA_WAIT_INTERVAL_SECONDS=${HAMSTRING_ZEEK_KAFKA_WAIT_INTERVAL_SECONDS:-5} + - HAMSTRING_ZEEK_KAFKA_WAIT_INTERVAL_SECONDS=${HAMSTRING_ZEEK_KAFKA_WAIT_INTERVAL_SECONDS:-15} volumes: - ../config.yaml:/opt/config.yaml - ../data/test_pcaps/:/opt/static_files diff --git a/docker/docker-compose/prod/docker-compose.pipeline.yml b/docker/docker-compose/prod/docker-compose.pipeline.yml index 0793f239..33e65d35 100644 --- a/docker/docker-compose/prod/docker-compose.pipeline.yml +++ b/docker/docker-compose/prod/docker-compose.pipeline.yml @@ -1,6 +1,6 @@ services: logserver: - image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-logserver:${HAMSTRING_LOGSERVER_IMAGE_TAG:-v2.2.1}" + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-logserver:${HAMSTRING_LOGSERVER_IMAGE_TAG:-v2.3.0}" restart: "unless-stopped" networks: hamstring: @@ -10,7 +10,7 @@ services: - GROUP_ID=log_storage - HAMSTRING_WAIT_FOR=kafka logcollector: - image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-logcollector:${HAMSTRING_LOGCOLLECTOR_IMAGE_TAG:-v2.2.1}" + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-logcollector:${HAMSTRING_LOGCOLLECTOR_IMAGE_TAG:-v2.3.0}" restart: "unless-stopped" volumes: - ../../../config.yaml:/app/config.yaml @@ -21,7 +21,7 @@ services: - HAMSTRING_WAIT_FOR=kafka prefilter: - image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-prefilter:${HAMSTRING_PREFILTER_IMAGE_TAG:-v2.2.1}" + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-prefilter:${HAMSTRING_PREFILTER_IMAGE_TAG:-v2.3.0}" restart: "unless-stopped" volumes: - ../../../config.yaml:/app/config.yaml @@ -35,7 +35,7 @@ services: - HAMSTRING_WAIT_FOR=kafka inspector: - image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-inspector:${HAMSTRING_INSPECTOR_IMAGE_TAG:-v2.2.1}" + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-inspector:${HAMSTRING_INSPECTOR_IMAGE_TAG:-v2.3.0}" restart: "unless-stopped" volumes: - ../../../config.yaml:/app/config.yaml @@ -57,7 +57,7 @@ services: - HAMSTRING_WAIT_FOR=kafka detector: - image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-detector:${HAMSTRING_DETECTOR_IMAGE_TAG:-v2.2.1}" + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-detector:${HAMSTRING_DETECTOR_IMAGE_TAG:-v2.3.0}" restart: "unless-stopped" volumes: - ../../../config.yaml:/app/config.yaml @@ -71,7 +71,7 @@ services: - HAMSTRING_WAIT_FOR=kafka detector-gpu: - image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-detector:${HAMSTRING_DETECTOR_IMAGE_TAG:-v2.2.1}" + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-detector:${HAMSTRING_DETECTOR_IMAGE_TAG:-v2.3.0}" restart: "unless-stopped" volumes: - ../../../config.yaml:/app/config.yaml @@ -93,7 +93,7 @@ services: - NVIDIA_DRIVER_CAPABILITIES=compute,utility alerter: - image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-alerter:${HAMSTRING_ALERTER_IMAGE_TAG:-v2.2.1}" + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-alerter:${HAMSTRING_ALERTER_IMAGE_TAG:-v2.3.0}" restart: "unless-stopped" volumes: - ../../../config.yaml:/app/config.yaml diff --git a/docker/docker_swarm/docker-compose.swarm.yml b/docker/docker_swarm/docker-compose.swarm.yml index da95e781..50099209 100644 --- a/docker/docker_swarm/docker-compose.swarm.yml +++ b/docker/docker_swarm/docker-compose.swarm.yml @@ -206,7 +206,7 @@ services: condition: any monitoring_agent: - image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-monitoring:${HAMSTRING_MONITORING_IMAGE_TAG:-v2.2.1}" + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-monitoring:${HAMSTRING_MONITORING_IMAGE_TAG:-v2.3.0}" command: ["src/monitoring/monitoring_agent.py"] networks: - hamstring @@ -226,7 +226,7 @@ services: condition: any logserver: - image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-logserver:${HAMSTRING_LOGSERVER_IMAGE_TAG:-v2.2.1}" + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-logserver:${HAMSTRING_LOGSERVER_IMAGE_TAG:-v2.3.0}" command: ["src/logserver/server.py"] networks: - hamstring @@ -247,7 +247,7 @@ services: condition: any logcollector: - image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-logcollector:${HAMSTRING_LOGCOLLECTOR_IMAGE_TAG:-v2.2.1}" + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-logcollector:${HAMSTRING_LOGCOLLECTOR_IMAGE_TAG:-v2.3.0}" command: ["src/logcollector/collector.py"] networks: - hamstring @@ -267,7 +267,7 @@ services: condition: any prefilter: - image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-prefilter:${HAMSTRING_PREFILTER_IMAGE_TAG:-v2.2.1}" + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-prefilter:${HAMSTRING_PREFILTER_IMAGE_TAG:-v2.3.0}" command: ["src/prefilter/prefilter.py"] networks: - hamstring @@ -287,7 +287,7 @@ services: condition: any inspector: - image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-inspector:${HAMSTRING_INSPECTOR_IMAGE_TAG:-v2.2.1}" + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-inspector:${HAMSTRING_INSPECTOR_IMAGE_TAG:-v2.3.0}" command: ["src/inspector/inspector.py"] networks: - hamstring @@ -315,7 +315,7 @@ services: condition: any detector: - image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-detector:${HAMSTRING_DETECTOR_IMAGE_TAG:-v2.2.1}" + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-detector:${HAMSTRING_DETECTOR_IMAGE_TAG:-v2.3.0}" command: ["src/detector/detector.py"] networks: - hamstring @@ -344,7 +344,7 @@ services: condition: any alerter: - image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-alerter:${HAMSTRING_ALERTER_IMAGE_TAG:-v2.2.1}" + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-alerter:${HAMSTRING_ALERTER_IMAGE_TAG:-v2.3.0}" command: ["src/alerter/alerter.py"] networks: - hamstring diff --git a/docker/grafana-provisioning/dashboards/alerts.json b/docker/grafana-provisioning/dashboards/alerts.json index 436382c9..cd7b3a6f 100644 --- a/docker/grafana-provisioning/dashboards/alerts.json +++ b/docker/grafana-provisioning/dashboards/alerts.json @@ -132,7 +132,7 @@ }, "pluginVersion": "4.5.1", "queryType": "table", - "rawSql": "SELECT\n toStartOf${Granularity}(time_bucket) AS time_bucket,\n sum(alert_count) AS \"count\"\nFROM alerts_1m\nWHERE time_bucket >= toStartOf${Granularity}(toDateTime($__fromTime))\n AND time_bucket <= toStartOf${Granularity}(toDateTime($__toTime))\nGROUP BY time_bucket\nORDER BY time_bucket\nWITH FILL\nFROM toStartOf${Granularity}(toDateTime($__fromTime))\nTO toStartOf${Granularity}(toDateTime($__toTime))\nSTEP toInterval${Granularity}(1);", + "rawSql": "SELECT\n toStartOf${Granularity}(time_bucket) AS time_bucket,\n sum(alert_count) AS \"count\"\nFROM alerts_history\nWHERE time_bucket >= toStartOf${Granularity}(toDateTime($__fromTime))\n AND time_bucket <= toStartOf${Granularity}(toDateTime($__toTime))\nGROUP BY time_bucket\nORDER BY time_bucket\nWITH FILL\nFROM toStartOf${Granularity}(toDateTime($__fromTime))\nTO toStartOf${Granularity}(toDateTime($__toTime))\nSTEP toInterval${Granularity}(1);", "refId": "B" } ], @@ -351,7 +351,7 @@ }, "pluginVersion": "4.6.0", "queryType": "table", - "rawSql": "SELECT\n alert_month AS time,\n sumIf(alert_count, day = 1) AS `1`,\n sumIf(alert_count, day = 2) AS `2`,\n sumIf(alert_count, day = 3) AS `3`,\n sumIf(alert_count, day = 4) AS `4`,\n sumIf(alert_count, day = 5) AS `5`,\n sumIf(alert_count, day = 6) AS `6`,\n sumIf(alert_count, day = 7) AS `7`,\n sumIf(alert_count, day = 8) AS `8`,\n sumIf(alert_count, day = 9) AS `9`,\n sumIf(alert_count, day = 10) AS `10`,\n sumIf(alert_count, day = 11) AS `11`,\n sumIf(alert_count, day = 12) AS `12`,\n sumIf(alert_count, day = 13) AS `13`,\n sumIf(alert_count, day = 14) AS `14`,\n sumIf(alert_count, day = 15) AS `15`,\n sumIf(alert_count, day = 16) AS `16`,\n sumIf(alert_count, day = 17) AS `17`,\n sumIf(alert_count, day = 18) AS `18`,\n sumIf(alert_count, day = 19) AS `19`,\n sumIf(alert_count, day = 20) AS `20`,\n sumIf(alert_count, day = 21) AS `21`,\n sumIf(alert_count, day = 22) AS `22`,\n sumIf(alert_count, day = 23) AS `23`,\n sumIf(alert_count, day = 24) AS `24`,\n sumIf(alert_count, day = 25) AS `25`,\n sumIf(alert_count, day = 26) AS `26`,\n sumIf(alert_count, day = 27) AS `27`,\n sumIf(alert_count, day = 28) AS `28`,\n sumIf(alert_count, day = 29) AS `29`,\n sumIf(alert_count, day = 30) AS `30`,\n sumIf(alert_count, day = 31) AS `31`\nFROM (\n SELECT\n toStartOfMonth(time_bucket) AS alert_month,\n toDayOfMonth(time_bucket) AS day,\n sum(alert_count) AS alert_count\n FROM alerts_1m\n WHERE time_bucket >= toStartOfMonth(now() - INTERVAL 1 YEAR)\n AND time_bucket < toStartOfMonth(now())\n GROUP BY alert_month, day\n) AS subquery\nGROUP BY alert_month\nORDER BY alert_month\nWITH FILL\nFROM toStartOfMonth(now() - INTERVAL 1 YEAR)\nTO toStartOfMonth(now())\nSTEP toIntervalMonth(1);", + "rawSql": "SELECT\n alert_month AS time,\n sumIf(alert_count, day = 1) AS `1`,\n sumIf(alert_count, day = 2) AS `2`,\n sumIf(alert_count, day = 3) AS `3`,\n sumIf(alert_count, day = 4) AS `4`,\n sumIf(alert_count, day = 5) AS `5`,\n sumIf(alert_count, day = 6) AS `6`,\n sumIf(alert_count, day = 7) AS `7`,\n sumIf(alert_count, day = 8) AS `8`,\n sumIf(alert_count, day = 9) AS `9`,\n sumIf(alert_count, day = 10) AS `10`,\n sumIf(alert_count, day = 11) AS `11`,\n sumIf(alert_count, day = 12) AS `12`,\n sumIf(alert_count, day = 13) AS `13`,\n sumIf(alert_count, day = 14) AS `14`,\n sumIf(alert_count, day = 15) AS `15`,\n sumIf(alert_count, day = 16) AS `16`,\n sumIf(alert_count, day = 17) AS `17`,\n sumIf(alert_count, day = 18) AS `18`,\n sumIf(alert_count, day = 19) AS `19`,\n sumIf(alert_count, day = 20) AS `20`,\n sumIf(alert_count, day = 21) AS `21`,\n sumIf(alert_count, day = 22) AS `22`,\n sumIf(alert_count, day = 23) AS `23`,\n sumIf(alert_count, day = 24) AS `24`,\n sumIf(alert_count, day = 25) AS `25`,\n sumIf(alert_count, day = 26) AS `26`,\n sumIf(alert_count, day = 27) AS `27`,\n sumIf(alert_count, day = 28) AS `28`,\n sumIf(alert_count, day = 29) AS `29`,\n sumIf(alert_count, day = 30) AS `30`,\n sumIf(alert_count, day = 31) AS `31`\nFROM (\n SELECT\n toStartOfMonth(time_bucket) AS alert_month,\n toDayOfMonth(time_bucket) AS day,\n sum(alert_count) AS alert_count\n FROM alerts_history\n WHERE time_bucket >= toStartOfMonth(now() - INTERVAL 90 DAY)\n AND time_bucket < toStartOfMonth(now())\n GROUP BY alert_month, day\n) AS subquery\nGROUP BY alert_month\nORDER BY alert_month\nWITH FILL\nFROM toStartOfMonth(now() - INTERVAL 90 DAY)\nTO toStartOfMonth(now())\nSTEP toIntervalMonth(1);", "refId": "A" } ], @@ -447,7 +447,7 @@ }, "pluginVersion": "4.6.0", "queryType": "table", - "rawSql": "SELECT\n alert_day AS time,\n sumIf(alert_count, hour = 0) AS `0`,\n sumIf(alert_count, hour = 1) AS `1`,\n sumIf(alert_count, hour = 2) AS `2`,\n sumIf(alert_count, hour = 3) AS `3`,\n sumIf(alert_count, hour = 4) AS `4`,\n sumIf(alert_count, hour = 5) AS `5`,\n sumIf(alert_count, hour = 6) AS `6`,\n sumIf(alert_count, hour = 7) AS `7`,\n sumIf(alert_count, hour = 8) AS `8`,\n sumIf(alert_count, hour = 9) AS `9`,\n sumIf(alert_count, hour = 10) AS `10`,\n sumIf(alert_count, hour = 11) AS `11`,\n sumIf(alert_count, hour = 12) AS `12`,\n sumIf(alert_count, hour = 13) AS `13`,\n sumIf(alert_count, hour = 14) AS `14`,\n sumIf(alert_count, hour = 15) AS `15`,\n sumIf(alert_count, hour = 16) AS `16`,\n sumIf(alert_count, hour = 17) AS `17`,\n sumIf(alert_count, hour = 18) AS `18`,\n sumIf(alert_count, hour = 19) AS `19`,\n sumIf(alert_count, hour = 20) AS `20`,\n sumIf(alert_count, hour = 21) AS `21`,\n sumIf(alert_count, hour = 22) AS `22`,\n sumIf(alert_count, hour = 23) AS `23`\nFROM (\n SELECT\n toStartOfDay(time_bucket) AS alert_day,\n toHour(time_bucket) AS hour,\n sum(alert_count) AS alert_count\n FROM alerts_1m\n WHERE time_bucket >= toStartOfMonth(now() - INTERVAL 1 MONTH)\n AND time_bucket < toStartOfMonth(now())\n GROUP BY alert_day, hour\n) AS subquery\nGROUP BY alert_day\nORDER BY alert_day\nWITH FILL\nFROM toDateTime(toStartOfMonth(now() - INTERVAL 1 MONTH))\nTO toDateTime(toStartOfMonth(now()))\nSTEP toIntervalDay(1);", + "rawSql": "SELECT\n alert_day AS time,\n sumIf(alert_count, hour = 0) AS `0`,\n sumIf(alert_count, hour = 1) AS `1`,\n sumIf(alert_count, hour = 2) AS `2`,\n sumIf(alert_count, hour = 3) AS `3`,\n sumIf(alert_count, hour = 4) AS `4`,\n sumIf(alert_count, hour = 5) AS `5`,\n sumIf(alert_count, hour = 6) AS `6`,\n sumIf(alert_count, hour = 7) AS `7`,\n sumIf(alert_count, hour = 8) AS `8`,\n sumIf(alert_count, hour = 9) AS `9`,\n sumIf(alert_count, hour = 10) AS `10`,\n sumIf(alert_count, hour = 11) AS `11`,\n sumIf(alert_count, hour = 12) AS `12`,\n sumIf(alert_count, hour = 13) AS `13`,\n sumIf(alert_count, hour = 14) AS `14`,\n sumIf(alert_count, hour = 15) AS `15`,\n sumIf(alert_count, hour = 16) AS `16`,\n sumIf(alert_count, hour = 17) AS `17`,\n sumIf(alert_count, hour = 18) AS `18`,\n sumIf(alert_count, hour = 19) AS `19`,\n sumIf(alert_count, hour = 20) AS `20`,\n sumIf(alert_count, hour = 21) AS `21`,\n sumIf(alert_count, hour = 22) AS `22`,\n sumIf(alert_count, hour = 23) AS `23`\nFROM (\n SELECT\n toStartOfDay(time_bucket) AS alert_day,\n toHour(time_bucket) AS hour,\n sum(alert_count) AS alert_count\n FROM alerts_history\n WHERE time_bucket >= toStartOfMonth(now() - INTERVAL 1 MONTH)\n AND time_bucket < toStartOfMonth(now())\n GROUP BY alert_day, hour\n) AS subquery\nGROUP BY alert_day\nORDER BY alert_day\nWITH FILL\nFROM toDateTime(toStartOfMonth(now() - INTERVAL 1 MONTH))\nTO toDateTime(toStartOfMonth(now()))\nSTEP toIntervalDay(1);", "refId": "A" } ], @@ -543,7 +543,7 @@ }, "pluginVersion": "4.6.0", "queryType": "table", - "rawSql": "SELECT\n alert_hour AS time,\n sumIf(alert_count, quarter = 0) AS `0`,\n sumIf(alert_count, quarter = 1) AS `15`,\n sumIf(alert_count, quarter = 2) AS `30`,\n sumIf(alert_count, quarter = 3) AS `45`\nFROM (\n SELECT\n toStartOfHour(time_bucket) AS alert_hour,\n intDiv(toMinute(time_bucket), 15) AS quarter,\n sum(alert_count) AS alert_count\n FROM alerts_1m\n WHERE time_bucket >= toStartOfHour(now() - INTERVAL 1 DAY)\n AND time_bucket <= now()\n GROUP BY alert_hour, quarter\n) AS subquery\nGROUP BY alert_hour\nORDER BY alert_hour\nWITH FILL\nFROM toStartOfHour(now() - INTERVAL 1 DAY)\nTO toStartOfHour(now())\nSTEP toIntervalHour(1);", + "rawSql": "SELECT\n alert_hour AS time,\n sumIf(alert_count, quarter = 0) AS `0`,\n sumIf(alert_count, quarter = 1) AS `15`,\n sumIf(alert_count, quarter = 2) AS `30`,\n sumIf(alert_count, quarter = 3) AS `45`\nFROM (\n SELECT\n toStartOfHour(time_bucket) AS alert_hour,\n intDiv(toMinute(time_bucket), 15) AS quarter,\n sum(alert_count) AS alert_count\n FROM alerts_history\n WHERE time_bucket >= toStartOfHour(now() - INTERVAL 1 DAY)\n AND time_bucket <= now()\n GROUP BY alert_hour, quarter\n) AS subquery\nGROUP BY alert_hour\nORDER BY alert_hour\nWITH FILL\nFROM toStartOfHour(now() - INTERVAL 1 DAY)\nTO toStartOfHour(now())\nSTEP toIntervalHour(1);", "refId": "A" } ], diff --git a/docker/grafana-provisioning/dashboards/latencies.json b/docker/grafana-provisioning/dashboards/latencies.json index 206efa9d..36db9f1d 100644 --- a/docker/grafana-provisioning/dashboards/latencies.json +++ b/docker/grafana-provisioning/dashboards/latencies.json @@ -82,7 +82,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [] }, @@ -199,7 +199,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [ { @@ -324,7 +324,7 @@ "displayName": "${__field.column}", "fieldMinMax": false, "mappings": [], - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [ { @@ -442,7 +442,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [ { @@ -563,7 +563,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [ { @@ -633,7 +633,7 @@ }, "pluginVersion": "4.10.1", "queryType": "table", - "rawSql": "SELECT 'Including transport and wait time' AS name, sum(median) AS median\nFROM (\nSELECT 'LogServer' AS name, if(isFinite(median(latency_us)), median(latency_us), 0) AS median\nFROM (\n SELECT\n sl.message_id AS message_id,\n toDate(slt.event_timestamp) AS event_date,\n sl.timestamp_in AS start_timestamp,\n slt.event_timestamp AS end_timestamp,\n dateDiff('microsecond', sl.timestamp_in, slt.event_timestamp) AS latency_us\n FROM server_logs sl\n INNER JOIN server_logs_timestamps slt ON sl.message_id = slt.message_id\n WHERE slt.event = 'timestamp_out'\n AND slt.event_timestamp > sl.timestamp_in\n) AS server_log_latency_values\nWHERE event_date >= toDate($__fromTime) AND event_date <= toDate($__toTime)\n AND end_timestamp >= $__fromTime AND end_timestamp <= $__toTime\n\nUNION ALL\n\nSELECT 'LogCollection' AS name, if(isFinite(median(latency_us)), median(latency_us), 0) AS median\nFROM logline_stage_latency_values\nWHERE stage = 'log_collection.collector'\n AND event_date >= toDate($__fromTime) AND event_date <= toDate($__toTime)\n AND end_timestamp >= $__fromTime AND end_timestamp <= $__toTime\n\nUNION ALL\n\nSELECT 'BatchHandler' AS name, if(isFinite(median(latency_us)), median(latency_us), 0) AS median\nFROM logline_stage_latency_values\nWHERE stage = 'log_collection.batch_handler'\n AND event_date >= toDate($__fromTime) AND event_date <= toDate($__toTime)\n AND end_timestamp >= $__fromTime AND end_timestamp <= $__toTime\n\nUNION ALL\n\nSELECT 'Prefilter' AS name, if(isFinite(median(latency_us)), median(latency_us), 0) AS median\nFROM batch_stage_latency_values\nWHERE stage = 'log_filtering.prefilter'\n AND event_date >= toDate($__fromTime) AND event_date <= toDate($__toTime)\n AND end_timestamp >= $__fromTime AND end_timestamp <= $__toTime\n\nUNION ALL\n\nSELECT 'Inspector' AS name, if(isFinite(median(latency_us)), median(latency_us), 0) AS median\nFROM batch_stage_latency_values\nWHERE stage = 'data_inspection.inspector'\n AND event_date >= toDate($__fromTime) AND event_date <= toDate($__toTime)\n AND end_timestamp >= $__fromTime AND end_timestamp <= $__toTime\n\nUNION ALL\n\nSELECT 'Detector' AS name, if(isFinite(median(latency_us)), median(latency_us), 0) AS median\nFROM suspicious_batch_stage_latency_values\nWHERE stage = 'data_analysis.detector'\n AND event_date >= toDate($__fromTime) AND event_date <= toDate($__toTime)\n AND end_timestamp >= $__fromTime AND end_timestamp <= $__toTime\n\nUNION ALL\n\nSELECT 'Between BatchHandler and Prefilter' AS name, if(isFinite(median(value)), median(value), 0) AS median\nFROM (\n SELECT dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\n FROM batch_tree bt1\n INNER JOIN batch_tree bt2\n ON bt1.batch_row_id = bt2.parent_batch_row_id\n WHERE bt1.stage = 'log_collection.batch_handler'\n AND bt1.status = 'completed'\n AND bt2.stage = 'log_filtering.prefilter'\n AND bt2.status = 'in_process'\n AND bt1.timestamp >= $__fromTime\n AND bt2.timestamp <= $__toTime\n)\n\nUNION ALL\n\nSELECT 'Between Prefilter and Inspector' AS name, if(isFinite(median(value)), median(value), 0) AS median\nFROM (\n SELECT dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\n FROM batch_tree bt1\n INNER JOIN batch_tree bt2\n ON bt1.batch_row_id = bt2.parent_batch_row_id\n WHERE bt1.stage = 'log_filtering.prefilter'\n AND bt1.status = 'finished'\n AND bt2.stage = 'data_inspection.inspector'\n AND bt2.status = 'in_process'\n AND bt1.timestamp >= $__fromTime\n AND bt2.timestamp <= $__toTime\n)\n\nUNION ALL\n\nSELECT 'Between Inspector and Detector' AS name, if(isFinite(median(value)), median(value), 0) AS median\nFROM (\n SELECT dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\n FROM batch_tree bt1\n INNER JOIN batch_tree bt2\n ON bt1.batch_row_id = bt2.parent_batch_row_id\n WHERE bt1.stage = 'data_inspection.inspector'\n AND bt1.status = 'finished'\n AND bt2.stage = 'data_analysis.detector'\n AND bt2.status = 'in_process'\n AND bt1.timestamp >= $__fromTime\n AND bt2.timestamp <= $__toTime\n AND dateDiff('microsecond', bt1.timestamp, bt2.timestamp) > 0\n)\n)\n\nUNION ALL\n\nSELECT 'Excluding transport and wait time' AS name, sum(median) AS median\nFROM (\nSELECT 'LogServer' AS name, if(isFinite(median(latency_us)), median(latency_us), 0) AS median\nFROM (\n SELECT\n sl.message_id AS message_id,\n toDate(slt.event_timestamp) AS event_date,\n sl.timestamp_in AS start_timestamp,\n slt.event_timestamp AS end_timestamp,\n dateDiff('microsecond', sl.timestamp_in, slt.event_timestamp) AS latency_us\n FROM server_logs sl\n INNER JOIN server_logs_timestamps slt ON sl.message_id = slt.message_id\n WHERE slt.event = 'timestamp_out'\n AND slt.event_timestamp > sl.timestamp_in\n) AS server_log_latency_values\nWHERE event_date >= toDate($__fromTime) AND event_date <= toDate($__toTime)\n AND end_timestamp >= $__fromTime AND end_timestamp <= $__toTime\n\nUNION ALL\n\nSELECT 'LogCollection' AS name, if(isFinite(median(latency_us)), median(latency_us), 0) AS median\nFROM logline_stage_latency_values\nWHERE stage = 'log_collection.collector'\n AND event_date >= toDate($__fromTime) AND event_date <= toDate($__toTime)\n AND end_timestamp >= $__fromTime AND end_timestamp <= $__toTime\n\nUNION ALL\n\nSELECT 'BatchHandler' AS name, if(isFinite(median(latency_us)), median(latency_us), 0) AS median\nFROM logline_stage_latency_values\nWHERE stage = 'log_collection.batch_handler'\n AND event_date >= toDate($__fromTime) AND event_date <= toDate($__toTime)\n AND end_timestamp >= $__fromTime AND end_timestamp <= $__toTime\n\nUNION ALL\n\nSELECT 'Prefilter' AS name, if(isFinite(median(latency_us)), median(latency_us), 0) AS median\nFROM batch_stage_latency_values\nWHERE stage = 'log_filtering.prefilter'\n AND event_date >= toDate($__fromTime) AND event_date <= toDate($__toTime)\n AND end_timestamp >= $__fromTime AND end_timestamp <= $__toTime\n\nUNION ALL\n\nSELECT 'Inspector' AS name, if(isFinite(median(latency_us)), median(latency_us), 0) AS median\nFROM batch_stage_latency_values\nWHERE stage = 'data_inspection.inspector'\n AND event_date >= toDate($__fromTime) AND event_date <= toDate($__toTime)\n AND end_timestamp >= $__fromTime AND end_timestamp <= $__toTime\n\nUNION ALL\n\nSELECT 'Detector' AS name, if(isFinite(median(latency_us)), median(latency_us), 0) AS median\nFROM suspicious_batch_stage_latency_values\nWHERE stage = 'data_analysis.detector'\n AND event_date >= toDate($__fromTime) AND event_date <= toDate($__toTime)\n AND end_timestamp >= $__fromTime AND end_timestamp <= $__toTime\n);", + "rawSql": "SELECT\n 'Including transport and wait time' AS name,\n sum(stage_median) AS median\nFROM (\n SELECT stage, median(latency_us) AS stage_median\n FROM pipeline_latency_values\n WHERE family IN ('server_log', 'logline', 'batch', 'suspicious_batch', 'transport')\n AND end_timestamp >= $__fromTime\n AND end_timestamp <= $__toTime\n GROUP BY stage\n)\nUNION ALL\nSELECT\n 'Excluding transport and wait time' AS name,\n sum(stage_median) AS median\nFROM (\n SELECT stage, median(latency_us) AS stage_median\n FROM pipeline_latency_values\n WHERE family IN ('server_log', 'logline', 'batch', 'suspicious_batch')\n AND end_timestamp >= $__fromTime\n AND end_timestamp <= $__toTime\n GROUP BY stage\n);", "refId": "A" } ], @@ -690,7 +690,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [] }, @@ -739,7 +739,7 @@ }, "pluginVersion": "4.10.1", "queryType": "table", - "rawSql": "SELECT if(isFinite(mapped.average_latency_us), mapped.average_latency_us, if(isFinite(fallback.average_latency_us), fallback.average_latency_us, 0)) AS \"Average alerting request roundtrip latency\"\nFROM (\n SELECT avg(latency_us) AS average_latency_us\n FROM (\n SELECT\n sl.message_id AS message_id,\n sltl.logline_id AS logline_id,\n a.suspicious_batch_id AS suspicious_batch_id,\n dateDiff('microsecond', min(sl.timestamp_in), max(ste.timestamp)) AS latency_us\n FROM alerts a\n INNER JOIN suspicious_batches_to_batch sbtb\n ON a.suspicious_batch_id = sbtb.suspicious_batch_id\n INNER JOIN logline_to_batches ltb\n ON sbtb.batch_id = ltb.batch_id\n INNER JOIN loglines ll\n ON ltb.logline_id = ll.logline_id\n AND ll.src_ip = a.src_ip\n INNER JOIN server_log_to_logline sltl\n ON ltb.logline_id = sltl.logline_id\n INNER JOIN server_logs sl\n ON sltl.message_id = sl.message_id\n INNER JOIN server_log_terminal_events ste\n ON sl.message_id = ste.message_id\n AND ste.stage = 'pipeline.alerter'\n AND ste.status = 'processed'\n WHERE toDate(ste.timestamp) >= toDate($__fromTime)\n AND toDate(ste.timestamp) <= toDate($__toTime)\n AND ste.timestamp >= $__fromTime\n AND ste.timestamp <= $__toTime\n AND sl.timestamp_in <= ste.timestamp\n GROUP BY sl.message_id, sltl.logline_id, a.suspicious_batch_id\n )\n WHERE latency_us > 0\n) mapped\nCROSS JOIN (\n SELECT avg(latency_us) AS average_latency_us\n FROM (\n SELECT\n ltb.logline_id AS logline_id,\n a.suspicious_batch_id AS suspicious_batch_id,\n dateDiff('microsecond', min(lt.timestamp), min(a.alert_timestamp)) + any(server_latency_us) AS latency_us\n FROM alerts a\n INNER JOIN suspicious_batches_to_batch sbtb\n ON a.suspicious_batch_id = sbtb.suspicious_batch_id\n INNER JOIN logline_to_batches ltb\n ON sbtb.batch_id = ltb.batch_id\n INNER JOIN loglines ll\n ON ltb.logline_id = ll.logline_id\n AND ll.src_ip = a.src_ip\n INNER JOIN logline_timestamps lt\n ON ltb.logline_id = lt.logline_id\n CROSS JOIN (\n SELECT if(isFinite(avg(latency_us)), avg(latency_us), 0) AS server_latency_us\n FROM server_log_latency_values\n WHERE event_date >= toDate($__fromTime)\n AND event_date <= toDate($__toTime)\n AND end_timestamp >= $__fromTime\n AND end_timestamp <= $__toTime\n ) server_latency\n WHERE lt.stage = 'log_collection.collector'\n AND lt.status = 'in_process'\n AND toDate(a.alert_timestamp) >= toDate($__fromTime)\n AND toDate(a.alert_timestamp) <= toDate($__toTime)\n AND a.alert_timestamp >= $__fromTime\n AND a.alert_timestamp <= $__toTime\n AND lt.timestamp <= a.alert_timestamp\n GROUP BY ltb.logline_id, a.suspicious_batch_id\n )\n WHERE latency_us > 0\n) fallback;\n", + "rawSql": "WITH alert_message_ids AS (\n SELECT\n toUUIDOrNull(JSONExtractString(server_message_id)) AS message_id\n FROM alerts\n ARRAY JOIN JSONExtractArrayRaw(result) AS warning\n ARRAY JOIN JSONExtractArrayRaw(warning, 'server_message_ids') AS server_message_id\n WHERE alert_timestamp >= $__fromTime\n AND alert_timestamp <= $__toTime\n)\nSELECT avg(dateDiff('microsecond', sl.timestamp_in, ste.timestamp)) AS \"Average alerting request roundtrip latency\"\nFROM alert_message_ids ami\nINNER JOIN server_logs sl\n ON ami.message_id = sl.message_id\nINNER JOIN server_log_terminal_events ste\n ON ami.message_id = ste.message_id\n AND ste.stage = 'pipeline.alerter'\n AND ste.status = 'processed'\nWHERE ami.message_id IS NOT NULL\n AND ste.timestamp >= $__fromTime\n AND ste.timestamp <= $__toTime\n AND ste.timestamp > sl.timestamp_in;\n", "refId": "alerting_request_roundtrip_latency" } ], @@ -780,7 +780,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [] }, @@ -830,7 +830,7 @@ }, "pluginVersion": "4.10.1", "queryType": "table", - "rawSql": "SELECT if(isFinite(mapped.average_latency_us), mapped.average_latency_us, if(isFinite(fallback.average_latency_us), fallback.average_latency_us, 0)) AS \"Average all logevents roundtrip latency\"\nFROM (\n SELECT avg(latency_us) AS average_latency_us\n FROM (\n SELECT\n message_id,\n dateDiff('microsecond', start_timestamp, terminal_timestamp) AS latency_us\n FROM (\n SELECT\n sl.message_id AS message_id,\n min(sl.timestamp_in) AS start_timestamp,\n max(terminal.terminal_timestamp) AS terminal_timestamp\n FROM server_logs sl\n INNER JOIN (\n SELECT\n sltl.message_id AS message_id,\n max(lt.timestamp) AS terminal_timestamp\n FROM server_log_to_logline sltl\n INNER JOIN logline_timestamps lt\n ON sltl.logline_id = lt.logline_id\n WHERE lt.is_active = false\n GROUP BY sltl.message_id\n\n UNION ALL\n\n SELECT\n message_id,\n max(timestamp) AS terminal_timestamp\n FROM server_log_terminal_events\n GROUP BY message_id\n ) terminal\n ON sl.message_id = terminal.message_id\n GROUP BY sl.message_id\n )\n WHERE terminal_timestamp >= start_timestamp\n AND toDate(terminal_timestamp) >= toDate($__fromTime)\n AND toDate(terminal_timestamp) <= toDate($__toTime)\n AND terminal_timestamp >= $__fromTime\n AND terminal_timestamp <= $__toTime\n )\n WHERE latency_us > 0\n) mapped\nCROSS JOIN (\n SELECT avg(latency_us) AS average_latency_us\n FROM (\n SELECT\n dateDiff('microsecond', collector_started_at, terminal_timestamp) + server_latency_us AS latency_us\n FROM (\n SELECT\n lt.logline_id AS logline_id,\n minIf(lt.timestamp, lt.stage = 'log_collection.collector' AND lt.status = 'in_process') AS collector_started_at,\n maxIf(lt.timestamp, lt.is_active = false) AS terminal_timestamp,\n any(server_latency_us) AS server_latency_us\n FROM logline_timestamps lt\n CROSS JOIN (\n SELECT if(isFinite(avg(latency_us)), avg(latency_us), 0) AS server_latency_us\n FROM server_log_latency_values\n WHERE event_date >= toDate($__fromTime)\n AND event_date <= toDate($__toTime)\n AND end_timestamp >= $__fromTime\n AND end_timestamp <= $__toTime\n ) server_latency\n GROUP BY lt.logline_id\n )\n WHERE collector_started_at > toDateTime64(0, 6)\n AND terminal_timestamp >= collector_started_at\n AND toDate(terminal_timestamp) >= toDate($__fromTime)\n AND toDate(terminal_timestamp) <= toDate($__toTime)\n AND terminal_timestamp >= $__fromTime\n AND terminal_timestamp <= $__toTime\n\n UNION ALL\n\n SELECT\n dateDiff('microsecond', timestamp_in, timestamp_failed) + server_latency_us AS latency_us\n FROM failed_loglines\n CROSS JOIN (\n SELECT if(isFinite(avg(latency_us)), avg(latency_us), 0) AS server_latency_us\n FROM server_log_latency_values\n WHERE event_date >= toDate($__fromTime)\n AND event_date <= toDate($__toTime)\n AND end_timestamp >= $__fromTime\n AND end_timestamp <= $__toTime\n ) server_latency\n WHERE timestamp_failed >= timestamp_in\n AND toDate(timestamp_failed) >= toDate($__fromTime)\n AND toDate(timestamp_failed) <= toDate($__toTime)\n AND timestamp_failed >= $__fromTime\n AND timestamp_failed <= $__toTime\n )\n WHERE latency_us > 0\n) fallback;\n", + "rawSql": "SELECT\n if(isFinite(avg(latency_us)), avg(latency_us), 0) AS \"Average all logevents roundtrip latency\"\nFROM pipeline_roundtrip_latency_values\nWHERE event_date >= toDate($__fromTime)\n AND event_date <= toDate($__toTime)\n AND end_timestamp >= $__fromTime\n AND end_timestamp <= $__toTime;", "refId": "all_logevents_roundtrip_latency" } ], @@ -911,7 +911,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [ { @@ -1061,7 +1061,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [] }, @@ -1178,7 +1178,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [ { @@ -1328,7 +1328,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [] }, @@ -1408,7 +1408,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [] }, @@ -1487,7 +1487,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [] }, @@ -1604,7 +1604,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [ { @@ -1754,7 +1754,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [] }, @@ -1871,7 +1871,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [ { @@ -2021,7 +2021,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [] }, @@ -2101,7 +2101,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [] }, @@ -2180,7 +2180,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [] }, @@ -2296,7 +2296,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [ { @@ -2446,7 +2446,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [] }, @@ -2563,7 +2563,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [ { @@ -2713,7 +2713,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [] }, @@ -2793,7 +2793,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [] }, @@ -2872,7 +2872,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [] }, @@ -3001,7 +3001,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [ { @@ -3098,7 +3098,7 @@ }, "pluginVersion": "4.10.1", "queryType": "timeseries", - "rawSql": "SELECT\n bt2.timestamp AS time,\n dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\nFROM batch_tree bt1\nINNER JOIN batch_tree bt2\n ON bt1.batch_row_id = bt2.parent_batch_row_id\nWHERE bt1.stage = 'log_collection.batch_handler'\n AND bt1.status = 'completed'\n AND bt2.stage = 'log_filtering.prefilter'\n AND bt2.status = 'in_process'\n AND bt1.timestamp >= $__fromTime\n AND bt2.timestamp <= $__toTime\nORDER BY time ASC;", + "rawSql": "SELECT\n end_timestamp AS time,\n latency_us AS value\nFROM pipeline_transport_latency_values\nWHERE stage = 'transport.batch_handler_to_prefilter'\n AND end_timestamp >= $__fromTime\n AND end_timestamp <= $__toTime\nORDER BY time ASC;", "refId": "Latency" }, { @@ -3121,7 +3121,7 @@ }, "pluginVersion": "4.10.1", "queryType": "table", - "rawSql": "SELECT\n toStartOfMinute(time) AS time_bucket,\n if(isFinite(median(value)), median(value), 0) AS value\nFROM (\nSELECT\n bt2.timestamp AS time,\n dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\nFROM batch_tree bt1\nINNER JOIN batch_tree bt2\n ON bt1.batch_row_id = bt2.parent_batch_row_id\nWHERE bt1.stage = 'log_collection.batch_handler'\n AND bt1.status = 'completed'\n AND bt2.stage = 'log_filtering.prefilter'\n AND bt2.status = 'in_process'\n AND bt1.timestamp >= $__fromTime\n AND bt2.timestamp <= $__toTime\n)\nGROUP BY time_bucket\nORDER BY time_bucket;", + "rawSql": "SELECT\n toStartOfMinute(end_timestamp) AS time_bucket,\n if(isFinite(median(latency_us)), median(latency_us), 0) AS value\nFROM pipeline_transport_latency_values\nWHERE stage = 'transport.batch_handler_to_prefilter'\n AND end_timestamp >= $__fromTime\n AND end_timestamp <= $__toTime\nGROUP BY time_bucket\nORDER BY time_bucket;", "refId": "Median" } ], @@ -3188,7 +3188,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [ { @@ -3283,7 +3283,7 @@ }, "pluginVersion": "4.10.1", "queryType": "timeseries", - "rawSql": "SELECT\n bt2.timestamp AS time,\n dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\nFROM batch_tree bt1\nINNER JOIN batch_tree bt2\n ON bt1.batch_row_id = bt2.parent_batch_row_id\nWHERE bt1.stage = 'log_filtering.prefilter'\n AND bt1.status = 'finished'\n AND bt2.stage = 'data_inspection.inspector'\n AND bt2.status = 'in_process'\n AND bt1.timestamp >= $__fromTime\n AND bt2.timestamp <= $__toTime\nORDER BY time ASC;", + "rawSql": "SELECT\n end_timestamp AS time,\n latency_us AS value\nFROM pipeline_transport_latency_values\nWHERE stage = 'transport.prefilter_to_inspector'\n AND end_timestamp >= $__fromTime\n AND end_timestamp <= $__toTime\nORDER BY time ASC;", "refId": "Latency" }, { @@ -3306,7 +3306,7 @@ }, "pluginVersion": "4.10.1", "queryType": "table", - "rawSql": "SELECT\n toStartOfMinute(time) AS time_bucket,\n if(isFinite(median(value)), median(value), 0) AS value\nFROM (\nSELECT\n bt2.timestamp AS time,\n dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\nFROM batch_tree bt1\nINNER JOIN batch_tree bt2\n ON bt1.batch_row_id = bt2.parent_batch_row_id\nWHERE bt1.stage = 'log_filtering.prefilter'\n AND bt1.status = 'finished'\n AND bt2.stage = 'data_inspection.inspector'\n AND bt2.status = 'in_process'\n AND bt1.timestamp >= $__fromTime\n AND bt2.timestamp <= $__toTime\n)\nGROUP BY time_bucket\nORDER BY time_bucket;", + "rawSql": "SELECT\n toStartOfMinute(end_timestamp) AS time_bucket,\n if(isFinite(median(latency_us)), median(latency_us), 0) AS value\nFROM pipeline_transport_latency_values\nWHERE stage = 'transport.prefilter_to_inspector'\n AND end_timestamp >= $__fromTime\n AND end_timestamp <= $__toTime\nGROUP BY time_bucket\nORDER BY time_bucket;", "refId": "Median" } ], @@ -3373,7 +3373,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [ { @@ -3468,7 +3468,7 @@ }, "pluginVersion": "4.10.1", "queryType": "timeseries", - "rawSql": "SELECT\n bt2.timestamp AS time,\n dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\nFROM batch_tree bt1\nINNER JOIN batch_tree bt2\n ON bt1.batch_row_id = bt2.parent_batch_row_id\nWHERE bt1.stage = 'data_inspection.inspector'\n AND bt1.status = 'finished'\n AND bt2.stage = 'data_analysis.detector'\n AND bt2.status = 'in_process'\n AND bt1.timestamp >= $__fromTime\n AND bt2.timestamp <= $__toTime\n AND dateDiff('microsecond', bt1.timestamp, bt2.timestamp) > 0\nORDER BY time ASC;", + "rawSql": "SELECT\n end_timestamp AS time,\n latency_us AS value\nFROM pipeline_transport_latency_values\nWHERE stage = 'transport.inspector_to_detector'\n AND end_timestamp >= $__fromTime\n AND end_timestamp <= $__toTime\nORDER BY time ASC;", "refId": "Latency" }, { @@ -3491,7 +3491,7 @@ }, "pluginVersion": "4.10.1", "queryType": "table", - "rawSql": "SELECT\n toStartOfMinute(time) AS time_bucket,\n if(isFinite(median(value)), median(value), 0) AS value\nFROM (\nSELECT\n bt2.timestamp AS time,\n dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\nFROM batch_tree bt1\nINNER JOIN batch_tree bt2\n ON bt1.batch_row_id = bt2.parent_batch_row_id\nWHERE bt1.stage = 'data_inspection.inspector'\n AND bt1.status = 'finished'\n AND bt2.stage = 'data_analysis.detector'\n AND bt2.status = 'in_process'\n AND bt1.timestamp >= $__fromTime\n AND bt2.timestamp <= $__toTime\n AND dateDiff('microsecond', bt1.timestamp, bt2.timestamp) > 0\n)\nGROUP BY time_bucket\nORDER BY time_bucket;", + "rawSql": "SELECT\n toStartOfMinute(end_timestamp) AS time_bucket,\n if(isFinite(median(latency_us)), median(latency_us), 0) AS value\nFROM pipeline_transport_latency_values\nWHERE stage = 'transport.inspector_to_detector'\n AND end_timestamp >= $__fromTime\n AND end_timestamp <= $__toTime\nGROUP BY time_bucket\nORDER BY time_bucket;", "refId": "Median" } ], @@ -3520,7 +3520,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [ { @@ -3585,7 +3585,7 @@ }, "pluginVersion": "4.10.1", "queryType": "table", - "rawSql": "SELECT\n min(value) AS \"Minimum\",\n if(isFinite(median(value)), median(value), 0) AS \"Median\",\n if(isFinite(avg(value)), avg(value), 0) AS \"Average\",\n max(value) AS \"Maximum\"\nFROM (\nSELECT\n bt2.timestamp AS time,\n dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\nFROM batch_tree bt1\nINNER JOIN batch_tree bt2\n ON bt1.batch_row_id = bt2.parent_batch_row_id\nWHERE bt1.stage = 'log_collection.batch_handler'\n AND bt1.status = 'completed'\n AND bt2.stage = 'log_filtering.prefilter'\n AND bt2.status = 'in_process'\n AND bt1.timestamp >= $__fromTime\n AND bt2.timestamp <= $__toTime\n);", + "rawSql": "SELECT\n min(latency_us) AS \"Minimum\",\n if(isFinite(median(latency_us)), median(latency_us), 0) AS \"Median\",\n if(isFinite(avg(latency_us)), avg(latency_us), 0) AS \"Average\",\n max(latency_us) AS \"Maximum\"\nFROM pipeline_transport_latency_values\nWHERE stage = 'transport.batch_handler_to_prefilter'\n AND end_timestamp >= $__fromTime\n AND end_timestamp <= $__toTime;", "refId": "A" } ], @@ -3613,7 +3613,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [ { @@ -3678,7 +3678,7 @@ }, "pluginVersion": "4.10.1", "queryType": "table", - "rawSql": "SELECT\n min(value) AS \"Minimum\",\n if(isFinite(median(value)), median(value), 0) AS \"Median\",\n if(isFinite(avg(value)), avg(value), 0) AS \"Average\",\n max(value) AS \"Maximum\"\nFROM (\nSELECT\n bt2.timestamp AS time,\n dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\nFROM batch_tree bt1\nINNER JOIN batch_tree bt2\n ON bt1.batch_row_id = bt2.parent_batch_row_id\nWHERE bt1.stage = 'log_filtering.prefilter'\n AND bt1.status = 'finished'\n AND bt2.stage = 'data_inspection.inspector'\n AND bt2.status = 'in_process'\n AND bt1.timestamp >= $__fromTime\n AND bt2.timestamp <= $__toTime\n);", + "rawSql": "SELECT\n min(latency_us) AS \"Minimum\",\n if(isFinite(median(latency_us)), median(latency_us), 0) AS \"Median\",\n if(isFinite(avg(latency_us)), avg(latency_us), 0) AS \"Average\",\n max(latency_us) AS \"Maximum\"\nFROM pipeline_transport_latency_values\nWHERE stage = 'transport.prefilter_to_inspector'\n AND end_timestamp >= $__fromTime\n AND end_timestamp <= $__toTime;", "refId": "A" } ], @@ -3706,7 +3706,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [ { @@ -3771,7 +3771,7 @@ }, "pluginVersion": "4.10.1", "queryType": "table", - "rawSql": "SELECT\n min(value) AS \"Minimum\",\n if(isFinite(median(value)), median(value), 0) AS \"Median\",\n if(isFinite(avg(value)), avg(value), 0) AS \"Average\",\n max(value) AS \"Maximum\"\nFROM (\nSELECT\n bt2.timestamp AS time,\n dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\nFROM batch_tree bt1\nINNER JOIN batch_tree bt2\n ON bt1.batch_row_id = bt2.parent_batch_row_id\nWHERE bt1.stage = 'data_inspection.inspector'\n AND bt1.status = 'finished'\n AND bt2.stage = 'data_analysis.detector'\n AND bt2.status = 'in_process'\n AND bt1.timestamp >= $__fromTime\n AND bt2.timestamp <= $__toTime\n AND dateDiff('microsecond', bt1.timestamp, bt2.timestamp) > 0\n);", + "rawSql": "SELECT\n min(latency_us) AS \"Minimum\",\n if(isFinite(median(latency_us)), median(latency_us), 0) AS \"Median\",\n if(isFinite(avg(latency_us)), avg(latency_us), 0) AS \"Average\",\n max(latency_us) AS \"Maximum\"\nFROM pipeline_transport_latency_values\nWHERE stage = 'transport.inspector_to_detector'\n AND end_timestamp >= $__fromTime\n AND end_timestamp <= $__toTime;", "refId": "A" } ], diff --git a/docker/grafana-provisioning/dashboards/log_volumes.json b/docker/grafana-provisioning/dashboards/log_volumes.json index d73dcf3e..c767c8ee 100644 --- a/docker/grafana-provisioning/dashboards/log_volumes.json +++ b/docker/grafana-provisioning/dashboards/log_volumes.json @@ -556,7 +556,7 @@ }, "pluginVersion": "4.5.1", "queryType": "table", - "rawSql": "SELECT\n time_bucket AS timestamp,\n avgMerge(avg_count) AS \" \"\nFROM fill_levels_1m\nWHERE stage = 'log_filtering.prefilter'\n AND entry_type = 'total_loglines'\n AND 'Prefilter' IN (${include_modules:csv})\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\nGROUP BY time_bucket\nORDER BY time_bucket ASC;", + "rawSql": "SELECT\n time_bucket AS timestamp,\n avgMerge(avg_count) AS \" \"\nFROM fill_levels_history\nWHERE stage = 'log_filtering.prefilter'\n AND entry_type = 'total_loglines'\n AND 'Prefilter' IN (${include_modules:csv})\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\nGROUP BY time_bucket\nORDER BY time_bucket ASC;", "refId": "Prefilter" }, { @@ -579,7 +579,7 @@ }, "pluginVersion": "4.5.1", "queryType": "table", - "rawSql": "SELECT\n time_bucket AS timestamp,\n avgMerge(avg_count) AS \" \"\nFROM fill_levels_1m\nWHERE stage = 'data_inspection.inspector'\n AND entry_type = 'total_loglines'\n AND 'Inspector' IN (${include_modules:csv})\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\nGROUP BY time_bucket\nORDER BY time_bucket ASC;", + "rawSql": "SELECT\n time_bucket AS timestamp,\n avgMerge(avg_count) AS \" \"\nFROM fill_levels_history\nWHERE stage = 'data_inspection.inspector'\n AND entry_type = 'total_loglines'\n AND 'Inspector' IN (${include_modules:csv})\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\nGROUP BY time_bucket\nORDER BY time_bucket ASC;", "refId": "Inspector" }, { @@ -602,7 +602,7 @@ }, "pluginVersion": "4.5.1", "queryType": "table", - "rawSql": "SELECT\n time_bucket AS timestamp,\n avgMerge(avg_count) AS \" \"\nFROM fill_levels_1m\nWHERE stage = 'data_analysis.detector'\n AND entry_type = 'total_loglines'\n AND 'Detector' IN (${include_modules:csv})\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\nGROUP BY time_bucket\nORDER BY time_bucket ASC;", + "rawSql": "SELECT\n time_bucket AS timestamp,\n avgMerge(avg_count) AS \" \"\nFROM fill_levels_history\nWHERE stage = 'data_analysis.detector'\n AND entry_type = 'total_loglines'\n AND 'Detector' IN (${include_modules:csv})\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\nGROUP BY time_bucket\nORDER BY time_bucket ASC;", "refId": "Detector" }, { @@ -625,7 +625,7 @@ }, "pluginVersion": "4.5.1", "queryType": "table", - "rawSql": "SELECT\n time_bucket AS timestamp,\n avgMerge(avg_count) AS \" \"\nFROM fill_levels_1m\nWHERE stage = 'log_collection.batch_handler'\n AND entry_type = 'total_loglines_in_batches'\n AND 'BatchHandler' IN (${include_modules:csv})\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\nGROUP BY time_bucket\nORDER BY time_bucket ASC;", + "rawSql": "SELECT\n time_bucket AS timestamp,\n avgMerge(avg_count) AS \" \"\nFROM fill_levels_history\nWHERE stage = 'log_collection.batch_handler'\n AND entry_type = 'total_loglines_in_batches'\n AND 'BatchHandler' IN (${include_modules:csv})\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\nGROUP BY time_bucket\nORDER BY time_bucket ASC;", "refId": "BatchHandler" } ], @@ -728,7 +728,7 @@ }, "pluginVersion": "4.10.2", "queryType": "table", - "rawSql": "SELECT *\nFROM (\n SELECT 'BatchHandler' AS name, quantileTDigestMerge(0.5)(median_count) AS \"Median\", 3 AS sort_order\n FROM fill_levels_1m\n WHERE stage = 'log_collection.batch_handler' AND entry_type = 'total_loglines_in_batches'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\n\n UNION ALL\n\n SELECT 'Prefilter' AS name, quantileTDigestMerge(0.5)(median_count) AS \"Median\", 4 AS sort_order\n FROM fill_levels_1m\n WHERE stage = 'log_filtering.prefilter' AND entry_type = 'total_loglines'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\n\n UNION ALL\n\n SELECT 'Inspector' AS name, quantileTDigestMerge(0.5)(median_count) AS \"Median\", 5 AS sort_order\n FROM fill_levels_1m\n WHERE stage = 'data_inspection.inspector' AND entry_type = 'total_loglines'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\n\n UNION ALL\n\n SELECT 'Detector' AS name, quantileTDigestMerge(0.5)(median_count) AS \"Median\", 6 AS sort_order\n FROM fill_levels_1m\n WHERE stage = 'data_analysis.detector' AND entry_type = 'total_loglines'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\n)\nWHERE name IN (${include_modules:csv})\nORDER BY sort_order ASC;", + "rawSql": "SELECT *\nFROM (\n SELECT 'BatchHandler' AS name, quantileTDigestMerge(0.5)(median_count) AS \"Median\", 3 AS sort_order\n FROM fill_levels_history\n WHERE stage = 'log_collection.batch_handler' AND entry_type = 'total_loglines_in_batches'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\n\n UNION ALL\n\n SELECT 'Prefilter' AS name, quantileTDigestMerge(0.5)(median_count) AS \"Median\", 4 AS sort_order\n FROM fill_levels_history\n WHERE stage = 'log_filtering.prefilter' AND entry_type = 'total_loglines'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\n\n UNION ALL\n\n SELECT 'Inspector' AS name, quantileTDigestMerge(0.5)(median_count) AS \"Median\", 5 AS sort_order\n FROM fill_levels_history\n WHERE stage = 'data_inspection.inspector' AND entry_type = 'total_loglines'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\n\n UNION ALL\n\n SELECT 'Detector' AS name, quantileTDigestMerge(0.5)(median_count) AS \"Median\", 6 AS sort_order\n FROM fill_levels_history\n WHERE stage = 'data_analysis.detector' AND entry_type = 'total_loglines'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\n)\nWHERE name IN (${include_modules:csv})\nORDER BY sort_order ASC;", "refId": "Fill States" } ], @@ -873,7 +873,7 @@ }, "pluginVersion": "4.5.1", "queryType": "table", - "rawSql": "SELECT\n time_bucket AS timestamp,\n avgMerge(avg_count) AS \" \"\nFROM fill_levels_1m\nWHERE stage = 'log_collection.batch_handler'\n AND entry_type = 'total_loglines_in_batches'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\nGROUP BY time_bucket\nORDER BY time_bucket ASC;", + "rawSql": "SELECT\n time_bucket AS timestamp,\n avgMerge(avg_count) AS \" \"\nFROM fill_levels_history\nWHERE stage = 'log_collection.batch_handler'\n AND entry_type = 'total_loglines_in_batches'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\nGROUP BY time_bucket\nORDER BY time_bucket ASC;", "refId": "Batch" }, { @@ -896,7 +896,7 @@ }, "pluginVersion": "4.5.1", "queryType": "table", - "rawSql": "SELECT\n time_bucket AS timestamp,\n avgMerge(avg_count) AS \" \"\nFROM fill_levels_1m\nWHERE stage = 'log_collection.batch_handler'\n AND entry_type = 'total_loglines_in_buffer'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\nGROUP BY time_bucket\nORDER BY time_bucket ASC;", + "rawSql": "SELECT\n time_bucket AS timestamp,\n avgMerge(avg_count) AS \" \"\nFROM fill_levels_history\nWHERE stage = 'log_collection.batch_handler'\n AND entry_type = 'total_loglines_in_buffer'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\nGROUP BY time_bucket\nORDER BY time_bucket ASC;", "refId": "Buffer" } ], @@ -976,7 +976,7 @@ }, "pluginVersion": "4.5.1", "queryType": "table", - "rawSql": "SELECT quantileTDigestMerge(0.5)(median_count) AS \"Batch Median\"\nFROM fill_levels_1m\nWHERE stage = 'log_collection.batch_handler'\n AND entry_type = 'total_loglines_in_batches'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime;", + "rawSql": "SELECT quantileTDigestMerge(0.5)(median_count) AS \"Batch Median\"\nFROM fill_levels_history\nWHERE stage = 'log_collection.batch_handler'\n AND entry_type = 'total_loglines_in_batches'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime;", "refId": "Batch" } ], @@ -1057,7 +1057,7 @@ }, "pluginVersion": "4.5.1", "queryType": "table", - "rawSql": "SELECT quantileTDigestMerge(0.5)(median_count) AS \"Buffer Median\"\nFROM fill_levels_1m\nWHERE stage = 'log_collection.batch_handler'\n AND entry_type = 'total_loglines_in_buffer'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime;", + "rawSql": "SELECT quantileTDigestMerge(0.5)(median_count) AS \"Buffer Median\"\nFROM fill_levels_history\nWHERE stage = 'log_collection.batch_handler'\n AND entry_type = 'total_loglines_in_buffer'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime;", "refId": "Buffer" } ], @@ -1168,7 +1168,7 @@ }, "pluginVersion": "4.5.1", "queryType": "table", - "rawSql": "SELECT\n time_bucket AS timestamp,\n avgMerge(avg_count) AS \" \"\nFROM fill_levels_1m\nWHERE stage = 'data_inspection.inspector'\n AND entry_type = 'total_loglines'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\nGROUP BY time_bucket\nORDER BY time_bucket ASC;", + "rawSql": "SELECT\n time_bucket AS timestamp,\n avgMerge(avg_count) AS \" \"\nFROM fill_levels_history\nWHERE stage = 'data_inspection.inspector'\n AND entry_type = 'total_loglines'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\nGROUP BY time_bucket\nORDER BY time_bucket ASC;", "refId": "Batch" } ], @@ -1248,7 +1248,7 @@ }, "pluginVersion": "4.5.1", "queryType": "table", - "rawSql": "SELECT quantileTDigestMerge(0.5)(median_count) AS \"Median\"\nFROM fill_levels_1m\nWHERE stage = 'data_inspection.inspector'\n AND entry_type = 'total_loglines'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime;", + "rawSql": "SELECT quantileTDigestMerge(0.5)(median_count) AS \"Median\"\nFROM fill_levels_history\nWHERE stage = 'data_inspection.inspector'\n AND entry_type = 'total_loglines'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime;", "refId": "A" } ], @@ -1343,7 +1343,7 @@ }, "pluginVersion": "4.5.1", "queryType": "table", - "rawSql": "SELECT\n min(min_count) AS \"Minimum\",\n avgMerge(avg_count) AS \"Average\",\n max(max_count) AS \"Maximum\"\nFROM fill_levels_1m\nWHERE stage = 'log_collection.batch_handler'\n AND entry_type = 'total_loglines_in_batches'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime;", + "rawSql": "SELECT\n min(min_count) AS \"Minimum\",\n avgMerge(avg_count) AS \"Average\",\n max(max_count) AS \"Maximum\"\nFROM fill_levels_history\nWHERE stage = 'log_collection.batch_handler'\n AND entry_type = 'total_loglines_in_batches'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime;", "refId": "Batch" }, { @@ -1366,7 +1366,7 @@ }, "pluginVersion": "4.5.1", "queryType": "table", - "rawSql": "SELECT\n min(min_count) AS \"Minimum\",\n avgMerge(avg_count) AS \"Average\",\n max(max_count) AS \"Maximum\"\nFROM fill_levels_1m\nWHERE stage = 'log_collection.batch_handler'\n AND entry_type = 'total_loglines_in_buffer'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime;", + "rawSql": "SELECT\n min(min_count) AS \"Minimum\",\n avgMerge(avg_count) AS \"Average\",\n max(max_count) AS \"Maximum\"\nFROM fill_levels_history\nWHERE stage = 'log_collection.batch_handler'\n AND entry_type = 'total_loglines_in_buffer'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime;", "refId": "Buffer" } ], @@ -1460,7 +1460,7 @@ }, "pluginVersion": "4.5.1", "queryType": "table", - "rawSql": "SELECT\n min(min_count) AS \"Minimum\",\n avgMerge(avg_count) AS \"Average\",\n max(max_count) AS \"Maximum\"\nFROM fill_levels_1m\nWHERE stage = 'data_inspection.inspector'\n AND entry_type = 'total_loglines'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime;", + "rawSql": "SELECT\n min(min_count) AS \"Minimum\",\n avgMerge(avg_count) AS \"Average\",\n max(max_count) AS \"Maximum\"\nFROM fill_levels_history\nWHERE stage = 'data_inspection.inspector'\n AND entry_type = 'total_loglines'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime;", "refId": "A" } ], @@ -1571,7 +1571,7 @@ }, "pluginVersion": "4.5.1", "queryType": "table", - "rawSql": "SELECT\n time_bucket AS timestamp,\n avgMerge(avg_count) AS \" \"\nFROM fill_levels_1m\nWHERE stage = 'log_filtering.prefilter'\n AND entry_type = 'total_loglines'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\nGROUP BY time_bucket\nORDER BY time_bucket ASC;", + "rawSql": "SELECT\n time_bucket AS timestamp,\n avgMerge(avg_count) AS \" \"\nFROM fill_levels_history\nWHERE stage = 'log_filtering.prefilter'\n AND entry_type = 'total_loglines'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\nGROUP BY time_bucket\nORDER BY time_bucket ASC;", "refId": "Batch" } ], @@ -1651,7 +1651,7 @@ }, "pluginVersion": "4.5.1", "queryType": "table", - "rawSql": "SELECT quantileTDigestMerge(0.5)(median_count) AS \"Median\"\nFROM fill_levels_1m\nWHERE stage = 'log_filtering.prefilter'\n AND entry_type = 'total_loglines'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime;", + "rawSql": "SELECT quantileTDigestMerge(0.5)(median_count) AS \"Median\"\nFROM fill_levels_history\nWHERE stage = 'log_filtering.prefilter'\n AND entry_type = 'total_loglines'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime;", "refId": "A" } ], @@ -1763,7 +1763,7 @@ }, "pluginVersion": "4.5.1", "queryType": "table", - "rawSql": "SELECT\n time_bucket AS timestamp,\n avgMerge(avg_count) AS \" \"\nFROM fill_levels_1m\nWHERE stage = 'data_analysis.detector'\n AND entry_type = 'total_loglines'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\nGROUP BY time_bucket\nORDER BY time_bucket ASC;", + "rawSql": "SELECT\n time_bucket AS timestamp,\n avgMerge(avg_count) AS \" \"\nFROM fill_levels_history\nWHERE stage = 'data_analysis.detector'\n AND entry_type = 'total_loglines'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\nGROUP BY time_bucket\nORDER BY time_bucket ASC;", "refId": "Batch" } ], @@ -1843,7 +1843,7 @@ }, "pluginVersion": "4.5.1", "queryType": "table", - "rawSql": "SELECT quantileTDigestMerge(0.5)(median_count) AS \"Median\"\nFROM fill_levels_1m\nWHERE stage = 'data_analysis.detector'\n AND entry_type = 'total_loglines'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime;", + "rawSql": "SELECT quantileTDigestMerge(0.5)(median_count) AS \"Median\"\nFROM fill_levels_history\nWHERE stage = 'data_analysis.detector'\n AND entry_type = 'total_loglines'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime;", "refId": "A" } ], @@ -1938,7 +1938,7 @@ }, "pluginVersion": "4.5.1", "queryType": "table", - "rawSql": "SELECT\n min(min_count) AS \"Minimum\",\n avgMerge(avg_count) AS \"Average\",\n max(max_count) AS \"Maximum\"\nFROM fill_levels_1m\nWHERE stage = 'log_filtering.prefilter'\n AND entry_type = 'total_loglines'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime;", + "rawSql": "SELECT\n min(min_count) AS \"Minimum\",\n avgMerge(avg_count) AS \"Average\",\n max(max_count) AS \"Maximum\"\nFROM fill_levels_history\nWHERE stage = 'log_filtering.prefilter'\n AND entry_type = 'total_loglines'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime;", "refId": "A" } ], @@ -2032,7 +2032,7 @@ }, "pluginVersion": "4.5.1", "queryType": "table", - "rawSql": "SELECT\n min(min_count) AS \"Minimum\",\n avgMerge(avg_count) AS \"Average\",\n max(max_count) AS \"Maximum\"\nFROM fill_levels_1m\nWHERE stage = 'data_analysis.detector'\n AND entry_type = 'total_loglines'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime;", + "rawSql": "SELECT\n min(min_count) AS \"Minimum\",\n avgMerge(avg_count) AS \"Average\",\n max(max_count) AS \"Maximum\"\nFROM fill_levels_history\nWHERE stage = 'data_analysis.detector'\n AND entry_type = 'total_loglines'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime;", "refId": "A" } ], diff --git a/docker/grafana-provisioning/dashboards/overview.json b/docker/grafana-provisioning/dashboards/overview.json index af78d5c6..b7264456 100644 --- a/docker/grafana-provisioning/dashboards/overview.json +++ b/docker/grafana-provisioning/dashboards/overview.json @@ -82,7 +82,7 @@ "color": "rgba(255,0,255,0.7)" }, "filterValues": { - "le": 1e-09 + "le": 1E-9 }, "legend": { "show": false @@ -123,7 +123,7 @@ }, "pluginVersion": "4.7.0", "queryType": "table", - "rawSql": "SELECT\n alert_hour AS time,\n sumIf(alert_count, quarter = 0) AS `0`,\n sumIf(alert_count, quarter = 1) AS `15`,\n sumIf(alert_count, quarter = 2) AS `30`,\n sumIf(alert_count, quarter = 3) AS `45`\nFROM (\n SELECT\n toStartOfHour(time_bucket) AS alert_hour,\n intDiv(toMinute(time_bucket), 15) AS quarter,\n sum(alert_count) AS alert_count\n FROM alerts_1m\n WHERE time_bucket >= toStartOfHour(now() - INTERVAL 1 DAY)\n AND time_bucket <= now()\n GROUP BY alert_hour, quarter\n) AS subquery\nGROUP BY alert_hour\nORDER BY alert_hour\nWITH FILL\nFROM toStartOfHour(now() - INTERVAL 1 DAY)\nTO toStartOfHour(now())\nSTEP toIntervalHour(1);", + "rawSql": "SELECT\n alert_hour AS time,\n sumIf(alert_count, quarter = 0) AS `0`,\n sumIf(alert_count, quarter = 1) AS `15`,\n sumIf(alert_count, quarter = 2) AS `30`,\n sumIf(alert_count, quarter = 3) AS `45`\nFROM (\n SELECT\n toStartOfHour(time_bucket) AS alert_hour,\n intDiv(toMinute(time_bucket), 15) AS quarter,\n sum(alert_count) AS alert_count\n FROM alerts_history\n WHERE time_bucket >= toStartOfHour(now() - INTERVAL 1 DAY)\n AND time_bucket <= now()\n GROUP BY alert_hour, quarter\n) AS subquery\nGROUP BY alert_hour\nORDER BY alert_hour\nWITH FILL\nFROM toStartOfHour(now() - INTERVAL 1 DAY)\nTO toStartOfHour(now())\nSTEP toIntervalHour(1);", "refId": "A" } ], @@ -366,7 +366,7 @@ }, "pluginVersion": "4.5.1", "queryType": "table", - "rawSql": "SELECT\n time_bucket AS timestamp,\n avgMerge(avg_count) AS \" \"\nFROM fill_levels_1m\nWHERE stage = 'log_filtering.prefilter'\n AND entry_type = 'total_loglines'\n AND 'Prefilter' IN (${include_modules:csv})\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\nGROUP BY time_bucket\nORDER BY time_bucket ASC;", + "rawSql": "SELECT\n time_bucket AS timestamp,\n avgMerge(avg_count) AS \" \"\nFROM fill_levels_history\nWHERE stage = 'log_filtering.prefilter'\n AND entry_type = 'total_loglines'\n AND 'Prefilter' IN (${include_modules:csv})\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\nGROUP BY time_bucket\nORDER BY time_bucket ASC;", "refId": "Prefilter" }, { @@ -389,7 +389,7 @@ }, "pluginVersion": "4.5.1", "queryType": "table", - "rawSql": "SELECT\n time_bucket AS timestamp,\n avgMerge(avg_count) AS \" \"\nFROM fill_levels_1m\nWHERE stage = 'data_inspection.inspector'\n AND entry_type = 'total_loglines'\n AND 'Inspector' IN (${include_modules:csv})\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\nGROUP BY time_bucket\nORDER BY time_bucket ASC;", + "rawSql": "SELECT\n time_bucket AS timestamp,\n avgMerge(avg_count) AS \" \"\nFROM fill_levels_history\nWHERE stage = 'data_inspection.inspector'\n AND entry_type = 'total_loglines'\n AND 'Inspector' IN (${include_modules:csv})\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\nGROUP BY time_bucket\nORDER BY time_bucket ASC;", "refId": "Inspector" }, { @@ -412,7 +412,7 @@ }, "pluginVersion": "4.5.1", "queryType": "table", - "rawSql": "SELECT\n time_bucket AS timestamp,\n avgMerge(avg_count) AS \" \"\nFROM fill_levels_1m\nWHERE stage = 'data_analysis.detector'\n AND entry_type = 'total_loglines'\n AND 'Detector' IN (${include_modules:csv})\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\nGROUP BY time_bucket\nORDER BY time_bucket ASC;", + "rawSql": "SELECT\n time_bucket AS timestamp,\n avgMerge(avg_count) AS \" \"\nFROM fill_levels_history\nWHERE stage = 'data_analysis.detector'\n AND entry_type = 'total_loglines'\n AND 'Detector' IN (${include_modules:csv})\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\nGROUP BY time_bucket\nORDER BY time_bucket ASC;", "refId": "Detector" }, { @@ -435,7 +435,7 @@ }, "pluginVersion": "4.5.1", "queryType": "table", - "rawSql": "SELECT\n time_bucket AS timestamp,\n avgMerge(avg_count) AS \" \"\nFROM fill_levels_1m\nWHERE stage = 'log_collection.batch_handler'\n AND entry_type = 'total_loglines_in_batches'\n AND 'BatchHandler' IN (${include_modules:csv})\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\nGROUP BY time_bucket\nORDER BY time_bucket ASC;", + "rawSql": "SELECT\n time_bucket AS timestamp,\n avgMerge(avg_count) AS \" \"\nFROM fill_levels_history\nWHERE stage = 'log_collection.batch_handler'\n AND entry_type = 'total_loglines_in_batches'\n AND 'BatchHandler' IN (${include_modules:csv})\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\nGROUP BY time_bucket\nORDER BY time_bucket ASC;", "refId": "BatchHandler" } ], @@ -538,7 +538,7 @@ }, "pluginVersion": "4.10.2", "queryType": "table", - "rawSql": "SELECT *\nFROM (\n SELECT 'BatchHandler' AS name, quantileTDigestMerge(0.5)(median_count) AS \"Median\", 3 AS sort_order\n FROM fill_levels_1m\n WHERE stage = 'log_collection.batch_handler' AND entry_type = 'total_loglines_in_batches'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\n\n UNION ALL\n\n SELECT 'Prefilter' AS name, quantileTDigestMerge(0.5)(median_count) AS \"Median\", 4 AS sort_order\n FROM fill_levels_1m\n WHERE stage = 'log_filtering.prefilter' AND entry_type = 'total_loglines'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\n\n UNION ALL\n\n SELECT 'Inspector' AS name, quantileTDigestMerge(0.5)(median_count) AS \"Median\", 5 AS sort_order\n FROM fill_levels_1m\n WHERE stage = 'data_inspection.inspector' AND entry_type = 'total_loglines'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\n\n UNION ALL\n\n SELECT 'Detector' AS name, quantileTDigestMerge(0.5)(median_count) AS \"Median\", 6 AS sort_order\n FROM fill_levels_1m\n WHERE stage = 'data_analysis.detector' AND entry_type = 'total_loglines'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\n)\nWHERE name IN (${include_modules:csv})\nORDER BY sort_order ASC;", + "rawSql": "SELECT *\nFROM (\n SELECT 'BatchHandler' AS name, quantileTDigestMerge(0.5)(median_count) AS \"Median\", 3 AS sort_order\n FROM fill_levels_history\n WHERE stage = 'log_collection.batch_handler' AND entry_type = 'total_loglines_in_batches'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\n\n UNION ALL\n\n SELECT 'Prefilter' AS name, quantileTDigestMerge(0.5)(median_count) AS \"Median\", 4 AS sort_order\n FROM fill_levels_history\n WHERE stage = 'log_filtering.prefilter' AND entry_type = 'total_loglines'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\n\n UNION ALL\n\n SELECT 'Inspector' AS name, quantileTDigestMerge(0.5)(median_count) AS \"Median\", 5 AS sort_order\n FROM fill_levels_history\n WHERE stage = 'data_inspection.inspector' AND entry_type = 'total_loglines'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\n\n UNION ALL\n\n SELECT 'Detector' AS name, quantileTDigestMerge(0.5)(median_count) AS \"Median\", 6 AS sort_order\n FROM fill_levels_history\n WHERE stage = 'data_analysis.detector' AND entry_type = 'total_loglines'\n AND time_bucket >= $__fromTime AND time_bucket <= $__toTime\n)\nWHERE name IN (${include_modules:csv})\nORDER BY sort_order ASC;", "refId": "Fill States" } ], @@ -590,7 +590,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [ { @@ -718,7 +718,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [ { @@ -839,7 +839,7 @@ } ] }, - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [ { @@ -909,7 +909,7 @@ }, "pluginVersion": "4.6.0", "queryType": "table", - "rawSql": "SELECT 'Including transport and wait time' AS name, sum(median) AS median\nFROM (\nSELECT 'LogServer' AS name, if(isFinite(median(latency_us)), median(latency_us), 0) AS median\nFROM (\n SELECT\n sl.message_id AS message_id,\n toDate(slt.event_timestamp) AS event_date,\n sl.timestamp_in AS start_timestamp,\n slt.event_timestamp AS end_timestamp,\n dateDiff('microsecond', sl.timestamp_in, slt.event_timestamp) AS latency_us\n FROM server_logs sl\n INNER JOIN server_logs_timestamps slt ON sl.message_id = slt.message_id\n WHERE slt.event = 'timestamp_out'\n AND slt.event_timestamp > sl.timestamp_in\n) AS server_log_latency_values\nWHERE event_date >= toDate($__fromTime) AND event_date <= toDate($__toTime)\n AND end_timestamp >= $__fromTime AND end_timestamp <= $__toTime\n\nUNION ALL\n\nSELECT 'LogCollection' AS name, if(isFinite(median(latency_us)), median(latency_us), 0) AS median\nFROM logline_stage_latency_values\nWHERE stage = 'log_collection.collector'\n AND event_date >= toDate($__fromTime) AND event_date <= toDate($__toTime)\n AND end_timestamp >= $__fromTime AND end_timestamp <= $__toTime\n\nUNION ALL\n\nSELECT 'BatchHandler' AS name, if(isFinite(median(latency_us)), median(latency_us), 0) AS median\nFROM logline_stage_latency_values\nWHERE stage = 'log_collection.batch_handler'\n AND event_date >= toDate($__fromTime) AND event_date <= toDate($__toTime)\n AND end_timestamp >= $__fromTime AND end_timestamp <= $__toTime\n\nUNION ALL\n\nSELECT 'Prefilter' AS name, if(isFinite(median(latency_us)), median(latency_us), 0) AS median\nFROM batch_stage_latency_values\nWHERE stage = 'log_filtering.prefilter'\n AND event_date >= toDate($__fromTime) AND event_date <= toDate($__toTime)\n AND end_timestamp >= $__fromTime AND end_timestamp <= $__toTime\n\nUNION ALL\n\nSELECT 'Inspector' AS name, if(isFinite(median(latency_us)), median(latency_us), 0) AS median\nFROM batch_stage_latency_values\nWHERE stage = 'data_inspection.inspector'\n AND event_date >= toDate($__fromTime) AND event_date <= toDate($__toTime)\n AND end_timestamp >= $__fromTime AND end_timestamp <= $__toTime\n\nUNION ALL\n\nSELECT 'Detector' AS name, if(isFinite(median(latency_us)), median(latency_us), 0) AS median\nFROM suspicious_batch_stage_latency_values\nWHERE stage = 'data_analysis.detector'\n AND event_date >= toDate($__fromTime) AND event_date <= toDate($__toTime)\n AND end_timestamp >= $__fromTime AND end_timestamp <= $__toTime\n\nUNION ALL\n\nSELECT 'Between BatchHandler and Prefilter' AS name, if(isFinite(median(value)), median(value), 0) AS median\nFROM (\n SELECT dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\n FROM batch_tree bt1\n INNER JOIN batch_tree bt2\n ON bt1.batch_row_id = bt2.parent_batch_row_id\n WHERE bt1.stage = 'log_collection.batch_handler'\n AND bt1.status = 'completed'\n AND bt2.stage = 'log_filtering.prefilter'\n AND bt2.status = 'in_process'\n AND bt1.timestamp >= $__fromTime\n AND bt2.timestamp <= $__toTime\n)\n\nUNION ALL\n\nSELECT 'Between Prefilter and Inspector' AS name, if(isFinite(median(value)), median(value), 0) AS median\nFROM (\n SELECT dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\n FROM batch_tree bt1\n INNER JOIN batch_tree bt2\n ON bt1.batch_row_id = bt2.parent_batch_row_id\n WHERE bt1.stage = 'log_filtering.prefilter'\n AND bt1.status = 'finished'\n AND bt2.stage = 'data_inspection.inspector'\n AND bt2.status = 'in_process'\n AND bt1.timestamp >= $__fromTime\n AND bt2.timestamp <= $__toTime\n)\n\nUNION ALL\n\nSELECT 'Between Inspector and Detector' AS name, if(isFinite(median(value)), median(value), 0) AS median\nFROM (\n SELECT dateDiff('microsecond', bt1.timestamp, bt2.timestamp) AS value\n FROM batch_tree bt1\n INNER JOIN batch_tree bt2\n ON bt1.batch_row_id = bt2.parent_batch_row_id\n WHERE bt1.stage = 'data_inspection.inspector'\n AND bt1.status = 'finished'\n AND bt2.stage = 'data_analysis.detector'\n AND bt2.status = 'in_process'\n AND bt1.timestamp >= $__fromTime\n AND bt2.timestamp <= $__toTime\n AND dateDiff('microsecond', bt1.timestamp, bt2.timestamp) > 0\n)\n)\n\nUNION ALL\n\nSELECT 'Excluding transport and wait time' AS name, sum(median) AS median\nFROM (\nSELECT 'LogServer' AS name, if(isFinite(median(latency_us)), median(latency_us), 0) AS median\nFROM (\n SELECT\n sl.message_id AS message_id,\n toDate(slt.event_timestamp) AS event_date,\n sl.timestamp_in AS start_timestamp,\n slt.event_timestamp AS end_timestamp,\n dateDiff('microsecond', sl.timestamp_in, slt.event_timestamp) AS latency_us\n FROM server_logs sl\n INNER JOIN server_logs_timestamps slt ON sl.message_id = slt.message_id\n WHERE slt.event = 'timestamp_out'\n AND slt.event_timestamp > sl.timestamp_in\n) AS server_log_latency_values\nWHERE event_date >= toDate($__fromTime) AND event_date <= toDate($__toTime)\n AND end_timestamp >= $__fromTime AND end_timestamp <= $__toTime\n\nUNION ALL\n\nSELECT 'LogCollection' AS name, if(isFinite(median(latency_us)), median(latency_us), 0) AS median\nFROM logline_stage_latency_values\nWHERE stage = 'log_collection.collector'\n AND event_date >= toDate($__fromTime) AND event_date <= toDate($__toTime)\n AND end_timestamp >= $__fromTime AND end_timestamp <= $__toTime\n\nUNION ALL\n\nSELECT 'BatchHandler' AS name, if(isFinite(median(latency_us)), median(latency_us), 0) AS median\nFROM logline_stage_latency_values\nWHERE stage = 'log_collection.batch_handler'\n AND event_date >= toDate($__fromTime) AND event_date <= toDate($__toTime)\n AND end_timestamp >= $__fromTime AND end_timestamp <= $__toTime\n\nUNION ALL\n\nSELECT 'Prefilter' AS name, if(isFinite(median(latency_us)), median(latency_us), 0) AS median\nFROM batch_stage_latency_values\nWHERE stage = 'log_filtering.prefilter'\n AND event_date >= toDate($__fromTime) AND event_date <= toDate($__toTime)\n AND end_timestamp >= $__fromTime AND end_timestamp <= $__toTime\n\nUNION ALL\n\nSELECT 'Inspector' AS name, if(isFinite(median(latency_us)), median(latency_us), 0) AS median\nFROM batch_stage_latency_values\nWHERE stage = 'data_inspection.inspector'\n AND event_date >= toDate($__fromTime) AND event_date <= toDate($__toTime)\n AND end_timestamp >= $__fromTime AND end_timestamp <= $__toTime\n\nUNION ALL\n\nSELECT 'Detector' AS name, if(isFinite(median(latency_us)), median(latency_us), 0) AS median\nFROM suspicious_batch_stage_latency_values\nWHERE stage = 'data_analysis.detector'\n AND event_date >= toDate($__fromTime) AND event_date <= toDate($__toTime)\n AND end_timestamp >= $__fromTime AND end_timestamp <= $__toTime\n);", + "rawSql": "SELECT\n 'Including transport and wait time' AS name,\n sum(stage_median) AS median\nFROM (\n SELECT stage, median(latency_us) AS stage_median\n FROM pipeline_latency_values\n WHERE family IN ('server_log', 'logline', 'batch', 'suspicious_batch', 'transport')\n AND end_timestamp >= $__fromTime\n AND end_timestamp <= $__toTime\n GROUP BY stage\n)\nUNION ALL\nSELECT\n 'Excluding transport and wait time' AS name,\n sum(stage_median) AS median\nFROM (\n SELECT stage, median(latency_us) AS stage_median\n FROM pipeline_latency_values\n WHERE family IN ('server_log', 'logline', 'batch', 'suspicious_batch')\n AND end_timestamp >= $__fromTime\n AND end_timestamp <= $__toTime\n GROUP BY stage\n);", "refId": "A" } ], diff --git a/docs/monitoring.rst b/docs/monitoring.rst index 9b2c27c2..0c9d8e0f 100644 --- a/docs/monitoring.rst +++ b/docs/monitoring.rst @@ -49,15 +49,29 @@ All Kafka values can be overridden at startup, for example: ``KAFKA_LOG_RETENTION_BYTES`` is a limit **per partition replica**, rather than a per-broker or per-volume limit. Increasing the partition count, replication factor, or this value increases the possible disk usage accordingly. -ClickHouse monitoring tables have a one-day TTL (alerts have a 60-day TTL). TTL deletion happens in background -merges; the supplied ClickHouse configuration checks for TTL work every 15 minutes. Its server logs are also rotated -at 100 MiB with three retained files. - -The new retention settings apply after restarting Kafka and ClickHouse. They reclaim data as the brokers close new -segments and ClickHouse performs TTL merges; they do not remove a stopped stack's volume files immediately. If this is -a disposable local environment and its historical data is not needed, stop the stack and remove its volumes with -``docker compose -f docker/docker-compose.yml down -v`` before starting it again. Do not remove production volumes as -a retention operation. +ClickHouse keeps detailed monitoring events and per-message latency state for one day (raw alerts are retained for +60 days). It then preserves compact monitoring history at progressively lower resolution: + +* one-minute aggregates for seven days; +* fifteen-minute aggregates for 30 days; +* hourly aggregates for 90 days. + +Alert counts and fill-level states are aggregated as rows arrive. Completed latency values are snapshotted by +ClickHouse refreshable materialized views; their historical rows retain the sample count and minimum, average, p50, +p95, p99, and maximum latency without retaining message, logline, or batch identifiers. Grafana uses unified history +views, so selecting an older time range automatically uses the appropriate resolution. Recent latency data remains +exact, while latency older than one day is represented by its bucketed p50 value in existing dashboard panels. + +TTL deletion happens in background merges; the supplied ClickHouse configuration checks for TTL work every 15 +minutes. ClickHouse server logs are rotated at 100 MiB with three retained files. Increasing a raw-table TTL has a +much larger storage impact than increasing an aggregate-table TTL. + +The retention schema is reconciled when the monitoring agent starts. New aggregate tables are backfilled once from +whatever source data is still available; data already removed by an older TTL cannot be recovered. Retention changes +reclaim data as Kafka closes segments and ClickHouse performs TTL merges, and therefore do not remove a stopped +stack's volume files immediately. If this is a disposable local environment and its historical data is not needed, +stop the stack and remove its volumes with ``docker compose -f docker/docker-compose.yml down -v`` before starting it +again. Do not remove production volumes as a retention operation. `Datatest` mode --------------- diff --git a/src/alerter/alerter.py b/src/alerter/alerter.py index 38f27886..d0cd40ec 100644 --- a/src/alerter/alerter.py +++ b/src/alerter/alerter.py @@ -17,7 +17,7 @@ run_thread_worker_pool, start_pipeline_worker_replicas, ) -from src.base.kafka_handler import ( +from src.base.kafka import ( ExactlyOnceKafkaConsumeHandler, ExactlyOnceKafkaProduceHandler, KafkaMessageFetchException, @@ -71,6 +71,7 @@ def __init__(self, alerter_config, consume_topic) -> None: self.key = None self.kafka_consume_handler = ExactlyOnceKafkaConsumeHandler(self.consume_topic) + self.kafka_produce_handler = None self.server_log_terminal_events = ClickHouseKafkaSender( "server_log_terminal_events" ) @@ -193,14 +194,14 @@ def _extract_rotated_log_date(self, log_path: Path) -> datetime.date | None: except ValueError: return None - def get_and_fill_data(self) -> None: + def get_and_fill_data(self, source_message=None) -> None: if self.alert_data: logger.warning( "Alerter is busy: Not consuming new messages. Wait for the Alerter to finish the current workload." ) return - key, data = self.kafka_consume_handler.consume_as_json() + key, data = self.kafka_consume_handler.consume_as_json(source_message) if data: self.alert_data = data self.key = key @@ -311,16 +312,28 @@ def bootstrap_alerter_instance(self): logger.info(f"Starting {self.name} Alerter") while True: try: - self.get_and_fill_data() - if self.alert_data: - server_message_ids = self._extract_server_message_ids() - # 1. Process specific action - self.process_alert() - # 2. Executing Base Logging Actions - self._log_to_file_action() - self._log_to_kafka_action() - self._record_alerter_terminal_events(server_message_ids) - self.kafka_consume_handler.commit() + source_messages = self.kafka_consume_handler.consume_batch() + if not source_messages: + continue + if self.kafka_produce_handler is None: + self.kafka_produce_handler = ExactlyOnceKafkaProduceHandler() + + with self.kafka_produce_handler.transaction_batch( + self.kafka_consume_handler, source_messages + ): + for source_message in source_messages: + try: + self.get_and_fill_data(source_message) + if self.alert_data: + server_message_ids = self._extract_server_message_ids() + # 1. Process specific action + self.process_alert() + # 2. Executing Base Logging Actions + self._log_to_file_action() + self._log_to_kafka_action() + self._record_alerter_terminal_events(server_message_ids) + finally: + self.clear_data() except KafkaMessageFetchException as e: logger.debug(e) @@ -334,8 +347,6 @@ def bootstrap_alerter_instance(self): break except Exception as e: logger.error(f"Unexpected error: {e}") - finally: - self.clear_data() async def start(self): loop = asyncio.get_running_loop() diff --git a/src/base/clickhouse_kafka_sender.py b/src/base/clickhouse_kafka_sender.py index 56145edb..41e47a0f 100644 --- a/src/base/clickhouse_kafka_sender.py +++ b/src/base/clickhouse_kafka_sender.py @@ -10,7 +10,10 @@ sys.path.append(os.getcwd()) from src.base.data_classes.clickhouse_connectors import TABLE_NAME_TO_TYPE -from src.base.kafka_handler import SimpleKafkaProduceHandler +from src.base.kafka import ( + BufferedKafkaProduceHandler, + SimpleKafkaProduceHandler, +) from src.base.log_config import get_logger logger = get_logger() @@ -25,13 +28,16 @@ class ClickHouseKafkaSender: """ @staticmethod - def create_shared_producer() -> SimpleKafkaProduceHandler: - return SimpleKafkaProduceHandler() + def create_shared_producer() -> BufferedKafkaProduceHandler: + """Create the non-blocking producer shared by one pipeline worker.""" + return BufferedKafkaProduceHandler() def __init__( self, table_name: str, - kafka_producer: SimpleKafkaProduceHandler | None = None, + kafka_producer: ( + BufferedKafkaProduceHandler | SimpleKafkaProduceHandler | None + ) = None, ): """ Args: @@ -41,7 +47,10 @@ def __init__( KeyError: If the specified table name is not found in TABLE_NAME_TO_TYPE mapping. """ self.table_name = table_name - self.kafka_producer = kafka_producer or SimpleKafkaProduceHandler() + # Monitoring has much higher fan-out than the pipeline's durable data + # path. Use the buffered producer so an acknowledgement for one + # monitoring row does not stall consumption of the next logline. + self.kafka_producer = kafka_producer or BufferedKafkaProduceHandler() self.data_schema = marshmallow_dataclass.class_schema( TABLE_NAME_TO_TYPE.get(table_name) )() diff --git a/src/base/kafka/__init__.py b/src/base/kafka/__init__.py new file mode 100644 index 00000000..3ec781dc --- /dev/null +++ b/src/base/kafka/__init__.py @@ -0,0 +1,42 @@ +"""Public Kafka implementation package.""" + +from src.base.kafka.client import KafkaHandler +from src.base.kafka.consumer import ( + ExactlyOnceKafkaConsumeHandler, + KafkaConsumeHandler, + SimpleKafkaConsumeHandler, +) +from src.base.kafka.errors import ( + KafkaMessageFetchException, + TooManyFailedAttemptsError, +) +from src.base.kafka.producer import ( + BufferedKafkaProduceHandler, + ExactlyOnceKafkaProduceHandler, + KafkaProduceHandler, + SimpleKafkaProduceHandler, +) +from src.base.kafka.records import ConsumedKafkaMessage, KafkaProduceRecord +from src.base.kafka.topics import ( + KafkaTopicManager, + build_consumer_group_id, + ensure_topics, +) + +__all__ = [ + "BufferedKafkaProduceHandler", + "ConsumedKafkaMessage", + "ExactlyOnceKafkaConsumeHandler", + "ExactlyOnceKafkaProduceHandler", + "KafkaConsumeHandler", + "KafkaHandler", + "KafkaMessageFetchException", + "KafkaProduceHandler", + "KafkaProduceRecord", + "KafkaTopicManager", + "SimpleKafkaConsumeHandler", + "SimpleKafkaProduceHandler", + "TooManyFailedAttemptsError", + "build_consumer_group_id", + "ensure_topics", +] diff --git a/src/base/kafka/client.py b/src/base/kafka/client.py new file mode 100644 index 00000000..68f88db8 --- /dev/null +++ b/src/base/kafka/client.py @@ -0,0 +1,8 @@ +"""Common base type for Kafka client wrappers.""" + + +class KafkaHandler: + """Base class shared by producer and consumer wrappers.""" + + def __init__(self) -> None: + self.consumer = None diff --git a/src/base/kafka/config.py b/src/base/kafka/config.py new file mode 100644 index 00000000..2988c56e --- /dev/null +++ b/src/base/kafka/config.py @@ -0,0 +1,60 @@ +"""Process-level Kafka configuration shared by all Kafka clients.""" + +import os + +from src.base.retry import load_retry_settings +from src.base.utils import setup_config + + +HOSTNAME = os.getenv("HOSTNAME", "default_tid") +CONSUMER_GROUP_ID = os.getenv("GROUP_ID", "default_gid") +NUMBER_OF_INSTANCES = int(os.getenv("NUMBER_OF_INSTANCES", 1)) + +CONFIG = setup_config() +RETRY_SETTINGS = load_retry_settings(CONFIG) + +KAFKA_BROKERS = CONFIG["environment"]["kafka_brokers"] +KAFKA_CONSUMER_CONFIG = CONFIG["environment"].get("kafka_consumer", {}) +KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS = int( + KAFKA_CONSUMER_CONFIG.get("max_poll_interval_ms", 1_800_000) +) + +KAFKA_TRANSACTION_BATCH_CONFIG = CONFIG["environment"].get( + "kafka_transaction_batch", {} +) +KAFKA_TRANSACTION_BATCH_SIZE = int( + os.getenv( + "KAFKA_TRANSACTION_BATCH_SIZE", + KAFKA_TRANSACTION_BATCH_CONFIG.get("size", 100), + ) +) +KAFKA_TRANSACTION_BATCH_TIMEOUT_MS = int( + os.getenv( + "KAFKA_TRANSACTION_BATCH_TIMEOUT_MS", + KAFKA_TRANSACTION_BATCH_CONFIG.get("timeout_ms", 50), + ) +) + +KAFKA_TOPIC_CONFIG = CONFIG["environment"].get("kafka_topics", {}) +KAFKA_TOPIC_DEFAULT_PARTITIONS = int(os.getenv("KAFKA_TOPIC_PARTITIONS", 12)) +KAFKA_TOPIC_REPLICATION_FACTOR = int( + os.getenv( + "KAFKA_TOPIC_REPLICATION_FACTOR", + KAFKA_TOPIC_CONFIG.get("replication_factor", len(KAFKA_BROKERS) or 1), + ) +) +KAFKA_TOPIC_AUTO_EXPAND_PARTITIONS = KAFKA_TOPIC_CONFIG.get( + "auto_expand_partitions", True +) +KAFKA_TOPIC_STAGE_CONFIG = KAFKA_TOPIC_CONFIG.get("stages", {}) +KAFKA_TOPIC_EXACT_CONFIG = KAFKA_TOPIC_CONFIG.get("topics", {}) +KAFKA_PIPELINE_TOPIC_PREFIXES = ( + CONFIG["environment"].get("kafka_topics_prefix", {}).get("pipeline", {}) +) + + +def bootstrap_servers() -> str: + """Return the configured brokers in confluent-kafka format.""" + return ",".join( + f"{broker['hostname']}:{broker['internal_port']}" for broker in KAFKA_BROKERS + ) diff --git a/src/base/kafka/consumer.py b/src/base/kafka/consumer.py new file mode 100644 index 00000000..60a1d103 --- /dev/null +++ b/src/base/kafka/consumer.py @@ -0,0 +1,317 @@ +"""Kafka consumer hierarchy and offset management.""" + +import time +from abc import abstractmethod +from collections.abc import Sequence +from typing import Optional + +from confluent_kafka import Consumer, KafkaError, KafkaException, TopicPartition +from confluent_kafka.admin import AdminClient + +from src.base.kafka import config as kafka_config +from src.base.kafka.client import KafkaHandler +from src.base.kafka.errors import ( + KafkaMessageFetchException, + TooManyFailedAttemptsError, +) +from src.base.kafka.records import ConsumedKafkaMessage +from src.base.kafka.resilience import is_retriable_kafka_error +from src.base.kafka.serialization import KafkaSerializationMixin +from src.base.kafka.topics import ( + build_consumer_group_id, + ensure_topics, + normalize_topics, + topic_partition_count, +) +from src.base.log_config import get_logger +from src.base.retry import retry_forever + +logger = get_logger() + + +class KafkaConsumeHandler(KafkaSerializationMixin, KafkaHandler): + """Common connection, batching, decoding, and offset behavior.""" + + def __init__(self, topics: str | list[str]) -> None: + super().__init__() + self._last_consumed_message = None + self.topics = normalize_topics(topics) + self.brokers = kafka_config.bootstrap_servers() + self.conf = self._build_consumer_conf() + self._connect_consumer() + + def _build_consumer_conf(self) -> dict: + return { + "bootstrap.servers": self.brokers, + "group.id": build_consumer_group_id(self.topics), + "enable.auto.commit": False, + "auto.offset.reset": "earliest", + "enable.partition.eof": True, + "max.poll.interval.ms": kafka_config.KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS, + } + + def _connect_consumer(self) -> None: + def connect(): + consumer = Consumer(self.conf) + admin_client = AdminClient({"bootstrap.servers": self.brokers}) + target_partitions_by_topic = ensure_topics(admin_client, self.topics) + if not self._all_topics_created( + self.topics, target_partitions_by_topic, consumer + ): + try: + consumer.close() + except Exception: + pass + raise TooManyFailedAttemptsError("Not all topics were created.") + consumer.subscribe(self.topics) + return consumer + + self.consumer = retry_forever( + connect, + f"Kafka consumer setup for {self.topics}", + kafka_config.RETRY_SETTINGS, + ) + + def _reset_consumer(self) -> None: + try: + if self.consumer: + self.consumer.close() + except Exception as exception: + logger.warning( + "Ignoring Kafka consumer close failure during reconnect: %s", + exception, + ) + self._last_consumed_message = None + self._connect_consumer() + + def commit( + self, + consumed_messages: Sequence[ConsumedKafkaMessage] | None = None, + ) -> None: + """Commit an explicit batch or the latest record from ``consume``.""" + if not self.consumer: + return + + if consumed_messages is not None: + if not consumed_messages: + return + retry_forever( + lambda: self.consumer.commit( + offsets=self.offsets_for(consumed_messages), + asynchronous=False, + ), + "Kafka consumer batch offset commit", + kafka_config.RETRY_SETTINGS, + retryable=(KafkaException, RuntimeError, OSError), + ) + self._last_consumed_message = None + return + + if self._last_consumed_message is not None: + retry_forever( + lambda: self.consumer.commit(self._last_consumed_message), + "Kafka consumer offset commit", + kafka_config.RETRY_SETTINGS, + retryable=(KafkaException, RuntimeError, OSError), + ) + self._last_consumed_message = None + + def consume_batch( + self, + max_messages: int | None = None, + timeout_ms: int | None = None, + ) -> list[ConsumedKafkaMessage]: + """Fetch a bounded group of records without committing offsets.""" + batch_size = max( + 1, + ( + kafka_config.KAFKA_TRANSACTION_BATCH_SIZE + if max_messages is None + else int(max_messages) + ), + ) + batch_timeout_ms = max( + 0, + ( + kafka_config.KAFKA_TRANSACTION_BATCH_TIMEOUT_MS + if timeout_ms is None + else int(timeout_ms) + ), + ) + deadline = time.monotonic() + batch_timeout_ms / 1000 + records = [] + + while len(records) < batch_size: + timeout = max(0.0, deadline - time.monotonic()) + try: + messages = self.consumer.consume( + num_messages=batch_size - len(records), + timeout=timeout, + ) + except (KafkaException, RuntimeError, OSError) as exception: + logger.warning( + "Kafka consumer batch fetch failed, reconnecting: %s", + exception, + ) + self._reset_consumer() + return [] + + for message in messages or []: + record = self._record_from_message(message) + if record is not None: + records.append(record) + if not messages or time.monotonic() >= deadline: + break + + if records: + self._last_consumed_message = records[-1].raw_message + return records + + @staticmethod + def offsets_for( + consumed_messages: Sequence[ConsumedKafkaMessage], + ) -> list[TopicPartition]: + """Return the highest processed next offset for each source partition.""" + offsets_by_partition: dict[tuple[str, int], int] = {} + for message in consumed_messages: + partition_key = (message.topic, message.partition) + offsets_by_partition[partition_key] = max( + offsets_by_partition.get(partition_key, 0), message.offset + 1 + ) + return [ + TopicPartition(topic, partition, offset) + for (topic, partition), offset in offsets_by_partition.items() + ] + + def group_metadata(self): + return self.consumer.consumer_group_metadata() + + @abstractmethod + def consume(self, *args, **kwargs): + raise NotImplementedError + + def _all_topics_created( + self, + topics: list[str], + min_partitions: int | dict[str, int] = 1, + consumer=None, + ) -> bool: + number_of_retries_left = 30 + all_topics_created = False + consumer = consumer or self.consumer + while not all_topics_created: + assigned_topics = retry_forever( + lambda: consumer.list_topics(timeout=10), + "Kafka topic visibility check", + kafka_config.RETRY_SETTINGS, + retryable=(KafkaException, RuntimeError, OSError), + ) + all_topics_created = True + for topic in topics: + partition_count = topic_partition_count(assigned_topics, topic) + required_partitions = ( + min_partitions.get(topic, 1) + if isinstance(min_partitions, dict) + else min_partitions + ) + if partition_count is None or partition_count < required_partitions: + all_topics_created = False + + if not all_topics_created: + number_of_retries_left -= 1 + if number_of_retries_left <= 0: + return False + time.sleep(0.5) + return True + + def _poll_message(self): + while True: + try: + message = self.consumer.poll(timeout=1.0) + except (KafkaException, RuntimeError, OSError) as exception: + logger.warning( + "Kafka consumer poll failed, reconnecting: %s", exception + ) + self._reset_consumer() + continue + + if message is None: + return None + if message.error(): + if message.error().code() == KafkaError._PARTITION_EOF: + return None + if is_retriable_kafka_error(message.error()): + logger.warning( + "Kafka consumer error is retriable, reconnecting: %s", + message.error(), + ) + self._reset_consumer() + return None + return message + + @staticmethod + def _record_from_message(message) -> ConsumedKafkaMessage | None: + if message is None: + return None + error = message.error() + if error is not None: + if error.code() == KafkaError._PARTITION_EOF: + return None + if is_retriable_kafka_error(error): + logger.warning("Kafka consumer received retriable error: %s", error) + return None + raise KafkaMessageFetchException(f"Kafka consumer error: {error}") + + return ConsumedKafkaMessage( + key=message.key().decode("utf-8") if message.key() else None, + value=message.value().decode("utf-8") if message.value() else None, + topic=message.topic(), + partition=message.partition(), + offset=message.offset(), + raw_message=message, + ) + + def _consume_single(self, shutdown_message: str): + empty_data_retrieved = False + try: + while True: + message = self._poll_message() + if message is None: + if not empty_data_retrieved: + logger.info("Waiting for messages...") + empty_data_retrieved = True + continue + if message.error(): + logger.error("Consumer error: %s", message.error()) + raise ValueError("Message is invalid") + + key = message.key().decode("utf-8") if message.key() else None + value = message.value().decode("utf-8") if message.value() else None + topic = message.topic() if message.topic() else None + self._last_consumed_message = message + return key, value, topic + except KeyboardInterrupt: + logger.info(shutdown_message) + + def __del__(self) -> None: + if getattr(self, "consumer", None): + self.consumer.close() + + +class SimpleKafkaConsumeHandler(KafkaConsumeHandler): + """Consumer without transactional read isolation.""" + + def consume(self) -> tuple[Optional[str], Optional[str], Optional[str]]: + return self._consume_single("Stopping KafkaConsumeHandler...") + + +class ExactlyOnceKafkaConsumeHandler(KafkaConsumeHandler): + """Consumer that exposes only committed transactional records.""" + + def _build_consumer_conf(self) -> dict: + conf = super()._build_consumer_conf() + conf["isolation.level"] = "read_committed" + return conf + + def consume(self) -> tuple[Optional[str], Optional[str], Optional[str]]: + return self._consume_single("Shutting down KafkaConsumeHandler...") diff --git a/src/base/kafka/errors.py b/src/base/kafka/errors.py new file mode 100644 index 00000000..8b8af9a7 --- /dev/null +++ b/src/base/kafka/errors.py @@ -0,0 +1,9 @@ +"""Kafka wrapper exceptions.""" + + +class TooManyFailedAttemptsError(Exception): + """Raised when required Kafka topics never become available.""" + + +class KafkaMessageFetchException(Exception): + """Raised when Kafka returns a permanent consumer error.""" diff --git a/src/base/kafka/producer.py b/src/base/kafka/producer.py new file mode 100644 index 00000000..4ea0ae9e --- /dev/null +++ b/src/base/kafka/producer.py @@ -0,0 +1,308 @@ +"""Kafka producer hierarchy for simple, buffered, and EoS delivery.""" + +import time +import uuid +from abc import abstractmethod +from collections.abc import Iterator, Sequence +from contextlib import contextmanager +from typing import TYPE_CHECKING, Callable + +from confluent_kafka import KafkaException, Producer + +from src.base.kafka import config as kafka_config +from src.base.kafka.client import KafkaHandler +from src.base.kafka.records import ConsumedKafkaMessage, KafkaProduceRecord +from src.base.kafka.resilience import ( + is_retriable_kafka_error, + is_retriable_kafka_exception, +) +from src.base.log_config import get_logger +from src.base.retry import retry_forever +from src.base.utils import kafka_delivery_report + +if TYPE_CHECKING: + from src.base.kafka.consumer import KafkaConsumeHandler + +logger = get_logger() + + +class KafkaProduceHandler(KafkaHandler): + """Common lifecycle and retry behavior for Kafka producers.""" + + def __init__(self, conf): + super().__init__() + self.conf = conf + self.producer = self._new_producer() + + def _new_producer(self): + return retry_forever( + lambda: Producer(self.conf), + "Kafka producer creation", + kafka_config.RETRY_SETTINGS, + ) + + def _reset_producer(self) -> None: + try: + if self.producer: + self.producer.flush(5) + except Exception as exception: + logger.warning( + "Ignoring Kafka producer flush failure during reconnect: %s", + exception, + ) + self.producer = self._new_producer() + + def _with_producer_retry( + self, description: str, operation: Callable[[], None] + ) -> None: + def attempt(): + try: + operation() + except Exception as exception: + if not is_retriable_kafka_exception(exception): + raise + logger.warning( + "%s failed, recreating Kafka producer: %s", + description, + exception, + ) + self._reset_producer() + raise + + retry_forever( + attempt, + description, + kafka_config.RETRY_SETTINGS, + retryable=(KafkaException, BufferError, RuntimeError, OSError), + ) + + @abstractmethod + def produce(self, *args, **kwargs): + raise NotImplementedError + + def __del__(self) -> None: + if getattr(self, "producer", None): + self.producer.flush() + + +class SimpleKafkaProduceHandler(KafkaProduceHandler): + """Synchronous Kafka producer without transactional semantics.""" + + def __init__(self): + self.brokers = kafka_config.bootstrap_servers() + super().__init__( + { + "bootstrap.servers": self.brokers, + "enable.idempotence": False, + "acks": "1", + "message.max.bytes": 1_000_000_000, + } + ) + + def produce(self, topic: str, data: str, key: None | str = None) -> None: + if not data: + return + + def operation(): + delivery_errors = [] + + def delivery_callback(error, message): + kafka_delivery_report(error, message) + if error: + delivery_errors.append(error) + + self.producer.flush() + self.producer.produce( + topic=topic, + key=key, + value=data, + callback=delivery_callback, + ) + self.producer.flush() + if delivery_errors: + delivery_error = delivery_errors[0] + if is_retriable_kafka_error(delivery_error): + raise KafkaException(delivery_error) + raise ValueError(f"Kafka delivery failed: {delivery_error}") + + self._with_producer_retry(f"Kafka produce to {topic}", operation) + + +class BufferedKafkaProduceHandler(SimpleKafkaProduceHandler): + """Asynchronous producer with bounded local backpressure.""" + + _QUEUE_POLL_TIMEOUT_SECONDS = 0.1 + + def __init__(self): + self.brokers = kafka_config.bootstrap_servers() + KafkaProduceHandler.__init__( + self, + { + "bootstrap.servers": self.brokers, + "enable.idempotence": False, + "acks": "1", + "message.max.bytes": 1_000_000_000, + "linger.ms": 10, + "batch.num.messages": 1000, + "queue.buffering.max.messages": 10000, + }, + ) + + def produce(self, topic: str, data: str, key: None | str = None) -> None: + if not data: + return + + def delivery_callback(error, message): + kafka_delivery_report(error, message) + + def operation(): + queue_was_full = False + while True: + self.producer.poll(0) + try: + self.producer.produce( + topic=topic, + key=key, + value=data, + callback=delivery_callback, + ) + return + except BufferError: + if not queue_was_full: + logger.warning( + "Kafka telemetry producer queue is full; " + "waiting for delivery reports." + ) + queue_was_full = True + self.producer.poll(self._QUEUE_POLL_TIMEOUT_SECONDS) + + self._with_producer_retry(f"Buffered Kafka produce to {topic}", operation) + + +class ExactlyOnceKafkaProduceHandler(KafkaProduceHandler): + """Transactional producer that atomically commits outputs and input offsets.""" + + def __init__(self): + self._transaction_records: list[KafkaProduceRecord] | None = None + self.brokers = kafka_config.bootstrap_servers() + super().__init__( + { + "bootstrap.servers": self.brokers, + "transactional.id": f"{kafka_config.HOSTNAME}-{uuid.uuid4()}", + "enable.idempotence": True, + "message.max.bytes": 1_000_000_000, + } + ) + self._init_transactions_with_retry() + + def _reset_producer(self) -> None: + super()._reset_producer() + self._init_transactions_with_retry() + + def _init_transactions_with_retry(self) -> None: + retry_forever( + lambda: self.producer.init_transactions(), + "Kafka transactional producer initialization", + kafka_config.RETRY_SETTINGS, + ) + + def produce(self, topic: str, data: str, key: None | str = None) -> None: + if not data: + return + + record = KafkaProduceRecord(topic=topic, data=data, key=key) + if self._transaction_records is not None: + self._transaction_records.append(record) + return + self._run_transaction([record]) + + @contextmanager + def transaction_batch( + self, + consumer: "KafkaConsumeHandler", + consumed_messages: Sequence[ConsumedKafkaMessage], + ) -> Iterator[None]: + """Collect outputs and commit them with source offsets on clean exit.""" + if self._transaction_records is not None: + raise RuntimeError("Kafka transaction batches cannot be nested.") + if not consumed_messages: + raise ValueError("A Kafka transaction batch requires source messages.") + + self._transaction_records = [] + try: + yield + records = self._transaction_records + except Exception: + self._transaction_records = None + raise + else: + self._transaction_records = None + self._run_transaction( + records, + consumer=consumer, + consumed_messages=consumed_messages, + ) + + def _run_transaction( + self, + records: Sequence[KafkaProduceRecord], + consumer: "KafkaConsumeHandler | None" = None, + consumed_messages: Sequence[ConsumedKafkaMessage] = (), + ) -> None: + def operation(): + self.producer.begin_transaction() + try: + for record in records: + if not record.data: + continue + self.producer.produce( + topic=record.topic, + key=record.key, + value=record.data, + callback=kafka_delivery_report, + ) + if consumer is not None: + self.producer.send_offsets_to_transaction( + consumer.offsets_for(consumed_messages), + consumer.group_metadata(), + ) + self.commit_transaction_with_retry() + except Exception as exception: + logger.info("Aborting Kafka transaction.") + try: + self.producer.abort_transaction() + except Exception as abort_exception: + logger.warning( + "Kafka transaction abort failed: %s", abort_exception + ) + logger.error("Transaction aborted.") + logger.error(exception) + raise + + self._with_producer_retry("Kafka transaction", operation) + + def commit_transaction_with_retry( + self, max_retries: int = 3, retry_interval_ms: int = 1000 + ) -> None: + committed = False + retry_count = 0 + while not committed and retry_count < max_retries: + try: + self.producer.commit_transaction() + committed = True + except KafkaException as exception: + if ( + "Conflicting commit_transaction API call is already in progress" + in str(exception) + ): + retry_count += 1 + logger.debug( + "Conflicting commit_transaction API call is already in " + "progress: Retrying" + ) + time.sleep(retry_interval_ms / 1000.0) + else: + raise + + if not committed: + raise RuntimeError("Failed to commit transaction after retries.") diff --git a/src/base/kafka/records.py b/src/base/kafka/records.py new file mode 100644 index 00000000..d33134ff --- /dev/null +++ b/src/base/kafka/records.py @@ -0,0 +1,24 @@ +"""Value objects passed between Kafka consumers and producers.""" + +from dataclasses import dataclass, field + + +@dataclass(frozen=True) +class ConsumedKafkaMessage: + """Decoded Kafka record plus the source offset required for EoS.""" + + key: str | None + value: str | None + topic: str + partition: int + offset: int + raw_message: object | None = field(default=None, repr=False, compare=False) + + +@dataclass(frozen=True) +class KafkaProduceRecord: + """One output record queued for a Kafka transaction.""" + + topic: str + data: str + key: str | None = None diff --git a/src/base/kafka/resilience.py b/src/base/kafka/resilience.py new file mode 100644 index 00000000..518f0504 --- /dev/null +++ b/src/base/kafka/resilience.py @@ -0,0 +1,27 @@ +"""Kafka-specific transient error classification.""" + +from confluent_kafka import KafkaError, KafkaException + + +def is_retriable_kafka_exception(exception: Exception) -> bool: + return isinstance(exception, (KafkaException, BufferError, RuntimeError, OSError)) + + +def is_retriable_kafka_error(error) -> bool: + retriable = getattr(error, "retriable", None) + if callable(retriable) and retriable(): + return True + + retriable_codes = { + getattr(KafkaError, name) + for name in ( + "_ALL_BROKERS_DOWN", + "_TRANSPORT", + "_TIMED_OUT", + "_MSG_TIMED_OUT", + "_RESOLVE", + "_WAIT_COORD", + ) + if hasattr(KafkaError, name) + } + return hasattr(error, "code") and error.code() in retriable_codes diff --git a/src/base/kafka/serialization.py b/src/base/kafka/serialization.py new file mode 100644 index 00000000..b92a5a04 --- /dev/null +++ b/src/base/kafka/serialization.py @@ -0,0 +1,72 @@ +"""Deserialization behavior shared by Kafka consumer implementations.""" + +import json +from typing import Optional + +import marshmallow_dataclass + +from src.base.data_classes.batch import Batch +from src.base.kafka.records import ConsumedKafkaMessage + + +class KafkaSerializationMixin: + """Decode consumed Kafka values while leaving transport logic separate.""" + + def consume_as_json( + self, source_message: ConsumedKafkaMessage | None = None + ) -> tuple[Optional[str], dict]: + if source_message is None: + key, value, _topic = self.consume() + else: + key, value = source_message.key, source_message.value + + if not key and not value: + return None, {} + + try: + decoded_data = json.loads(value) + except Exception as exception: + raise ValueError("Unknown data format") from exception + if not isinstance(decoded_data, dict): + raise ValueError("Unknown data format") + return key, decoded_data + + @staticmethod + def _is_dicts(obj): + return isinstance(obj, list) and all(isinstance(item, dict) for item in obj) + + @staticmethod + def _decode_batch_data(data): + if data is None: + return [] + if not isinstance(data, list): + raise ValueError("Batch data must be a list.") + + decoded_data = [] + for item in data: + if isinstance(item, str): + decoded_data.append(json.loads(item)) + elif isinstance(item, (dict, list)): + decoded_data.append(item) + else: + raise ValueError("Batch data contains unsupported item type.") + return decoded_data + + def consume_as_object( + self, source_message: ConsumedKafkaMessage | None = None + ) -> tuple[None | str, Batch]: + if source_message is None: + key, value, _topic = self.consume() + else: + key, value = source_message.key, source_message.value + + if not key and not value: + return None, {} + + decoded_data: dict = json.loads(value) + decoded_data["data"] = self._decode_batch_data(decoded_data.get("data")) + batch_schema = marshmallow_dataclass.class_schema(Batch)() + batch = batch_schema.load(decoded_data) + if isinstance(batch, Batch): + return key, batch + raise ValueError("Unknown data format.") diff --git a/src/base/kafka/topics.py b/src/base/kafka/topics.py new file mode 100644 index 00000000..258b7e36 --- /dev/null +++ b/src/base/kafka/topics.py @@ -0,0 +1,267 @@ +"""Kafka topic configuration, provisioning, and consumer-group naming.""" + +import os + +from confluent_kafka import KafkaError, KafkaException +from confluent_kafka.admin import AdminClient, NewPartitions, NewTopic + +from src.base.kafka import config as kafka_config +from src.base.log_config import get_logger +from src.base.retry import retry_forever + +logger = get_logger() + + +def normalize_topics(topics: str | list[str]) -> list[str]: + if isinstance(topics, str): + return [topics] + return topics + + +def _as_bool(value) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, str): + return value.lower() in {"1", "true", "yes", "on"} + return bool(value) + + +def _topic_config(topic: str | None) -> dict: + if topic is None: + return {} + + exact_config = kafka_config.KAFKA_TOPIC_EXACT_CONFIG.get(topic) + if exact_config is not None: + return exact_config + + matched_stage = None + matched_prefix_length = -1 + for stage_name, topic_prefix in kafka_config.KAFKA_PIPELINE_TOPIC_PREFIXES.items(): + if not topic_prefix: + continue + if topic == topic_prefix or topic.startswith(f"{topic_prefix}-"): + if len(topic_prefix) > matched_prefix_length: + matched_stage = stage_name + matched_prefix_length = len(topic_prefix) + + if matched_stage is None: + return {} + return kafka_config.KAFKA_TOPIC_STAGE_CONFIG.get(matched_stage, {}) + + +def _runtime_min_topic_partitions() -> int: + try: + return int(os.getenv("KAFKA_TOPIC_MIN_PARTITIONS", "1")) + except ValueError: + return 1 + + +def _desired_topic_partitions( + topic: str | None = None, override: int | None = None +) -> int: + topic_config = _topic_config(topic) + configured_partitions = override + if configured_partitions is None: + configured_partitions = topic_config.get( + "partitions", kafka_config.KAFKA_TOPIC_DEFAULT_PARTITIONS + ) + return max( + 1, + kafka_config.NUMBER_OF_INSTANCES, + _runtime_min_topic_partitions(), + int(configured_partitions), + ) + + +def _topic_replication_factor( + topic: str | None = None, override: int | None = None +) -> int: + broker_count = max(1, len(kafka_config.KAFKA_BROKERS)) + topic_config = _topic_config(topic) + configured_replication_factor = override + if configured_replication_factor is None: + configured_replication_factor = topic_config.get( + "replication_factor", kafka_config.KAFKA_TOPIC_REPLICATION_FACTOR + ) + configured_replication_factor = max(1, int(configured_replication_factor)) + return min(configured_replication_factor, broker_count) + + +def topic_partition_count(cluster_metadata, topic: str) -> int | None: + topics_metadata = getattr(cluster_metadata, "topics", {}) + if isinstance(topics_metadata, dict): + topic_metadata = topics_metadata.get(topic) + if topic_metadata is None: + return None + partitions = getattr(topic_metadata, "partitions", None) + return 1 if partitions is None else len(partitions) + return 1 if topic in topics_metadata else None + + +class KafkaTopicManager: + """Reconcile a set of topics with the configured partition policy.""" + + def __init__(self, admin_client: AdminClient) -> None: + self.admin_client = admin_client + + def ensure( + self, + topics: str | list[str], + target_partitions: int | None = None, + replication_factor: int | None = None, + auto_expand_partitions: bool | None = None, + ) -> dict[str, int]: + normalized_topics = normalize_topics(topics) + target_by_topic = { + topic: _desired_topic_partitions(topic, target_partitions) + for topic in normalized_topics + } + replication_by_topic = { + topic: _topic_replication_factor(topic, replication_factor) + for topic in normalized_topics + } + should_expand = ( + _as_bool(kafka_config.KAFKA_TOPIC_AUTO_EXPAND_PARTITIONS) + if auto_expand_partitions is None + else _as_bool(auto_expand_partitions) + ) + + cluster_metadata = retry_forever( + lambda: self.admin_client.list_topics(timeout=10), + "Kafka metadata lookup", + kafka_config.RETRY_SETTINGS, + ) + topics_metadata = getattr(cluster_metadata, "topics", {}) + existing_topics = ( + set(topics_metadata.keys()) + if isinstance(topics_metadata, dict) + else set(topics_metadata) + ) + missing_topics = [ + topic for topic in normalized_topics if topic not in existing_topics + ] + + if missing_topics: + logger.info("Creating Kafka topics %s.", missing_topics) + retry_forever( + lambda: self._wait_for_admin_futures( + self.admin_client.create_topics( + [ + NewTopic( + topic, + target_by_topic[topic], + replication_by_topic[topic], + ) + for topic in missing_topics + ] + ), + "create topic", + ), + f"Kafka topic creation for {missing_topics}", + kafka_config.RETRY_SETTINGS, + ) + + if not should_expand: + return target_by_topic + + cluster_metadata = retry_forever( + lambda: self.admin_client.list_topics(timeout=10), + "Kafka metadata lookup after topic creation", + kafka_config.RETRY_SETTINGS, + ) + topics_to_expand = [] + for topic in normalized_topics: + current_partition_count = topic_partition_count(cluster_metadata, topic) + if current_partition_count is None: + continue + target = target_by_topic[topic] + if current_partition_count < target: + logger.info( + "Expanding Kafka topic '%s' from %d to %d partition(s).", + topic, + current_partition_count, + target, + ) + topics_to_expand.append(NewPartitions(topic, target)) + + if topics_to_expand: + retry_forever( + lambda: self._wait_for_admin_futures( + self.admin_client.create_partitions(topics_to_expand), + "expand partitions", + ), + "Kafka partition expansion for " + f"{[str(topic) for topic in topics_to_expand]}", + kafka_config.RETRY_SETTINGS, + ) + + return target_by_topic + + @staticmethod + def _wait_for_admin_futures(futures: dict, operation: str) -> None: + for topic, future in futures.items(): + try: + future.result() + except KafkaException as exception: + if operation == "create topic" and _is_topic_already_created(exception): + logger.info("Kafka topic '%s' already exists.", topic) + continue + if ( + operation == "expand partitions" + and _is_partition_count_already_satisfied(exception) + ): + logger.info( + "Kafka topic '%s' already has enough partitions.", topic + ) + continue + raise + + +def _is_topic_already_created(exception: Exception) -> bool: + kafka_error = exception.args[0] if getattr(exception, "args", None) else None + topic_already_exists_code = getattr(KafkaError, "TOPIC_ALREADY_EXISTS", None) + if ( + topic_already_exists_code is not None + and hasattr(kafka_error, "code") + and kafka_error.code() == topic_already_exists_code + ): + return True + return "already exists" in str(exception).lower() + + +def _is_partition_count_already_satisfied(exception: Exception) -> bool: + message = str(exception).lower() + return "already has" in message or "smaller than current" in message + + +def ensure_topics( + admin_client: AdminClient, + topics: str | list[str], + target_partitions: int | None = None, + replication_factor: int | None = None, + auto_expand_partitions: bool | None = None, +) -> dict[str, int]: + """Preserve the existing functional API around the topic manager.""" + return KafkaTopicManager(admin_client).ensure( + topics, + target_partitions, + replication_factor, + auto_expand_partitions, + ) + + +def _sanitize_consumer_group_part(value: str) -> str: + return "".join( + character if character.isalnum() or character in "._-" else "_" + for character in value + ) + + +def build_consumer_group_id(topics: str | list[str]) -> str: + normalized_topics = sorted(normalize_topics(topics)) + topic_suffix = "__".join( + _sanitize_consumer_group_part(topic) for topic in normalized_topics + ) + if not topic_suffix: + return kafka_config.CONSUMER_GROUP_ID + return f"{kafka_config.CONSUMER_GROUP_ID}.{topic_suffix}" diff --git a/src/base/kafka_handler.py b/src/base/kafka_handler.py deleted file mode 100644 index f906a991..00000000 --- a/src/base/kafka_handler.py +++ /dev/null @@ -1,1112 +0,0 @@ -""" -The Write-Exactly-Once-Semantics used by the :class:`KafkaHandler` is shown by -https://github.com/confluentinc/confluent-kafka-python/blob/master/examples/eos-transactions.py, -parts of which are similar to the code in this module. -""" - -import ast -import json -import os -import sys -import time -import uuid -from abc import abstractmethod -from typing import Callable, Optional - -import marshmallow_dataclass -from confluent_kafka import ( - Consumer, - KafkaError, - KafkaException, - Producer, -) -from confluent_kafka.admin import AdminClient, NewPartitions, NewTopic - -sys.path.append(os.getcwd()) -from src.base.data_classes.batch import Batch -from src.base.log_config import get_logger -from src.base.retry import retry_forever -from src.base.utils import kafka_delivery_report, setup_config - -logger = get_logger() - -HOSTNAME = os.getenv("HOSTNAME", "default_tid") -CONSUMER_GROUP_ID = os.getenv("GROUP_ID", "default_gid") -NUMBER_OF_INSTANCES = int(os.getenv("NUMBER_OF_INSTANCES", 1)) - -config = setup_config() -KAFKA_BROKERS = config["environment"]["kafka_brokers"] -KAFKA_CONSUMER_CONFIG = config["environment"].get("kafka_consumer", {}) -KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS = int( - KAFKA_CONSUMER_CONFIG.get("max_poll_interval_ms", 1800000) -) -KAFKA_TOPIC_CONFIG = config["environment"].get("kafka_topics", {}) -KAFKA_TOPIC_DEFAULT_PARTITIONS = int(os.getenv("KAFKA_TOPIC_PARTITIONS", 12)) -KAFKA_TOPIC_REPLICATION_FACTOR = int( - os.getenv( - "KAFKA_TOPIC_REPLICATION_FACTOR", - KAFKA_TOPIC_CONFIG.get("replication_factor", len(KAFKA_BROKERS) or 1), - ) -) -KAFKA_TOPIC_AUTO_EXPAND_PARTITIONS = KAFKA_TOPIC_CONFIG.get( - "auto_expand_partitions", True -) -KAFKA_TOPIC_STAGE_CONFIG = KAFKA_TOPIC_CONFIG.get("stages", {}) -KAFKA_TOPIC_EXACT_CONFIG = KAFKA_TOPIC_CONFIG.get("topics", {}) -KAFKA_PIPELINE_TOPIC_PREFIXES = ( - config["environment"].get("kafka_topics_prefix", {}).get("pipeline", {}) -) - - -def _normalize_topics(topics: str | list[str]) -> list[str]: - if isinstance(topics, str): - return [topics] - return topics - - -def _as_bool(value) -> bool: - if isinstance(value, bool): - return value - if isinstance(value, str): - return value.lower() in {"1", "true", "yes", "on"} - return bool(value) - - -def _topic_config(topic: str | None) -> dict: - if topic is None: - return {} - - exact_config = KAFKA_TOPIC_EXACT_CONFIG.get(topic) - if exact_config is not None: - return exact_config - - matched_stage = None - matched_prefix_length = -1 - for stage_name, topic_prefix in KAFKA_PIPELINE_TOPIC_PREFIXES.items(): - if not topic_prefix: - continue - if topic == topic_prefix or topic.startswith(f"{topic_prefix}-"): - if len(topic_prefix) > matched_prefix_length: - matched_stage = stage_name - matched_prefix_length = len(topic_prefix) - - if matched_stage is None: - return {} - - return KAFKA_TOPIC_STAGE_CONFIG.get(matched_stage, {}) - - -def _desired_topic_partitions( - topic: str | None = None, override: int | None = None -) -> int: - topic_config = _topic_config(topic) - configured_partitions = override - if configured_partitions is None: - configured_partitions = topic_config.get( - "partitions", KAFKA_TOPIC_DEFAULT_PARTITIONS - ) - return max( - 1, - NUMBER_OF_INSTANCES, - _runtime_min_topic_partitions(), - int(configured_partitions), - ) - - -def _runtime_min_topic_partitions() -> int: - try: - return int(os.getenv("KAFKA_TOPIC_MIN_PARTITIONS", "1")) - except ValueError: - return 1 - - -def _topic_replication_factor( - topic: str | None = None, override: int | None = None -) -> int: - broker_count = max(1, len(KAFKA_BROKERS)) - topic_config = _topic_config(topic) - configured_replication_factor = override - if configured_replication_factor is None: - configured_replication_factor = topic_config.get( - "replication_factor", KAFKA_TOPIC_REPLICATION_FACTOR - ) - configured_replication_factor = max(1, int(configured_replication_factor)) - return min(configured_replication_factor, broker_count) - - -def _topic_partition_count(cluster_metadata, topic: str) -> int | None: - topics_metadata = getattr(cluster_metadata, "topics", {}) - - if isinstance(topics_metadata, dict): - topic_metadata = topics_metadata.get(topic) - if topic_metadata is None: - return None - - partitions = getattr(topic_metadata, "partitions", None) - if partitions is None: - return 1 - return len(partitions) - - if topic in topics_metadata: - return 1 - - return None - - -def _is_topic_already_created(exception: Exception) -> bool: - kafka_error = exception.args[0] if getattr(exception, "args", None) else None - topic_already_exists_code = getattr(KafkaError, "TOPIC_ALREADY_EXISTS", None) - if ( - topic_already_exists_code is not None - and hasattr(kafka_error, "code") - and kafka_error.code() == topic_already_exists_code - ): - return True - - return "already exists" in str(exception).lower() - - -def _is_partition_count_already_satisfied(exception: Exception) -> bool: - message = str(exception).lower() - return "already has" in message or "smaller than current" in message - - -def _wait_for_admin_futures(futures: dict, operation: str) -> None: - for topic, future in futures.items(): - try: - future.result() - except KafkaException as exception: - if operation == "create topic" and _is_topic_already_created(exception): - logger.info("Kafka topic '%s' already exists.", topic) - continue - if ( - operation == "expand partitions" - and _is_partition_count_already_satisfied(exception) - ): - logger.info("Kafka topic '%s' already has enough partitions.", topic) - continue - raise - - -def _is_retriable_kafka_exception(exception: Exception) -> bool: - if isinstance(exception, (KafkaException, BufferError, RuntimeError, OSError)): - return True - return False - - -def _is_retriable_kafka_error(error) -> bool: - retriable = getattr(error, "retriable", None) - if callable(retriable) and retriable(): - return True - - retriable_codes = { - getattr(KafkaError, name) - for name in ( - "_ALL_BROKERS_DOWN", - "_TRANSPORT", - "_TIMED_OUT", - "_MSG_TIMED_OUT", - "_RESOLVE", - "_WAIT_COORD", - ) - if hasattr(KafkaError, name) - } - return hasattr(error, "code") and error.code() in retriable_codes - - -def ensure_topics( - admin_client: AdminClient, - topics: str | list[str], - target_partitions: int | None = None, - replication_factor: int | None = None, - auto_expand_partitions: bool | None = None, -) -> dict[str, int]: - normalized_topics = _normalize_topics(topics) - target_partitions_by_topic = { - topic: _desired_topic_partitions(topic, target_partitions) - for topic in normalized_topics - } - replication_factor_by_topic = { - topic: _topic_replication_factor(topic, replication_factor) - for topic in normalized_topics - } - auto_expand_partitions = ( - _as_bool(KAFKA_TOPIC_AUTO_EXPAND_PARTITIONS) - if auto_expand_partitions is None - else _as_bool(auto_expand_partitions) - ) - - cluster_metadata = retry_forever( - lambda: admin_client.list_topics(timeout=10), - "Kafka metadata lookup", - ) - topics_metadata = getattr(cluster_metadata, "topics", {}) - existing_topics = ( - set(topics_metadata.keys()) - if isinstance(topics_metadata, dict) - else set(topics_metadata) - ) - missing_topics = [ - topic for topic in normalized_topics if topic not in existing_topics - ] - - if missing_topics: - logger.info( - "Creating Kafka topics %s.", - missing_topics, - ) - retry_forever( - lambda: _wait_for_admin_futures( - admin_client.create_topics( - [ - NewTopic( - topic, - target_partitions_by_topic[topic], - replication_factor_by_topic[topic], - ) - for topic in missing_topics - ] - ), - "create topic", - ), - f"Kafka topic creation for {missing_topics}", - ) - - if not auto_expand_partitions: - return target_partitions_by_topic - - cluster_metadata = retry_forever( - lambda: admin_client.list_topics(timeout=10), - "Kafka metadata lookup after topic creation", - ) - topics_to_expand = [] - for topic in normalized_topics: - current_partition_count = _topic_partition_count(cluster_metadata, topic) - if current_partition_count is None: - continue - target_partitions = target_partitions_by_topic[topic] - if current_partition_count < target_partitions: - logger.info( - "Expanding Kafka topic '%s' from %d to %d partition(s).", - topic, - current_partition_count, - target_partitions, - ) - topics_to_expand.append(NewPartitions(topic, target_partitions)) - - if topics_to_expand: - retry_forever( - lambda: _wait_for_admin_futures( - admin_client.create_partitions(topics_to_expand), - "expand partitions", - ), - f"Kafka partition expansion for {[str(topic) for topic in topics_to_expand]}", - ) - - return target_partitions_by_topic - - -def _sanitize_consumer_group_part(value: str) -> str: - return "".join( - character if character.isalnum() or character in "._-" else "_" - for character in value - ) - - -def build_consumer_group_id(topics: str | list[str]) -> str: - normalized_topics = sorted(_normalize_topics(topics)) - topic_suffix = "__".join( - _sanitize_consumer_group_part(topic) for topic in normalized_topics - ) - if not topic_suffix: - return CONSUMER_GROUP_ID - return f"{CONSUMER_GROUP_ID}.{topic_suffix}" - - -class TooManyFailedAttemptsError(Exception): - """Exception raised when operations exceed the maximum number of retry attempts - - This exception is typically raised during Kafka topic creation or connection - establishment when the maximum number of retry attempts has been exceeded. - """ - - pass - - -class KafkaMessageFetchException(Exception): - """Exception raised when Kafka message consumption fails - - This exception is raised when there are errors during the process of fetching - or consuming messages from Kafka topics, including network issues, timeout - errors, or malformed message data. - """ - - pass - - -class KafkaHandler: - """Base class for all Kafka wrappers and handlers - - Provides common initialization and configuration setup for Kafka producers - and consumers. This abstract base class establishes the foundation for - specific Kafka handling implementations. - """ - - def __init__(self) -> None: - """ - Sets up the initial configuration and initializes the consumer attribute - to None. Specific implementations should override this method to establish - their respective Kafka clients. - """ - self.consumer = None - - -class KafkaProduceHandler(KafkaHandler): - """Abstract base class for Kafka Producer wrappers - - Extends KafkaHandler to provide producer-specific functionality. This class - establishes the interface for Kafka message production with different - semantic guarantees (simple vs exactly-once). - """ - - def __init__(self, conf): - """ - Args: - conf (dict): Configuration dictionary for the Kafka producer. - Should contain broker settings and producer-specific options. - """ - super().__init__() - self.conf = conf - self.producer = self._new_producer() - - def _new_producer(self): - return retry_forever( - lambda: Producer(self.conf), - "Kafka producer creation", - ) - - def _reset_producer(self) -> None: - try: - if self.producer: - self.producer.flush(5) - except Exception as exception: - logger.warning( - "Ignoring Kafka producer flush failure during reconnect: %s", exception - ) - self.producer = self._new_producer() - - def _with_producer_retry( - self, description: str, operation: Callable[[], None] - ) -> None: - def attempt(): - try: - operation() - except Exception as exception: - if not _is_retriable_kafka_exception(exception): - raise - logger.warning( - "%s failed, recreating Kafka producer: %s", description, exception - ) - self._reset_producer() - raise - - retry_forever( - attempt, - description, - retryable=(KafkaException, BufferError, RuntimeError, OSError), - ) - - @abstractmethod - def produce(self, *args, **kwargs): - """Abstract method for producing messages to Kafka topics - - Encodes the given data for transport and sends it to the specified topic. - Implementations must define the specific behavior for message production. - - Args: - *args: Variable arguments depending on implementation. - **kwargs: Keyword arguments depending on implementation. - - Raises: - NotImplementedError: This method must be implemented by subclasses. - """ - raise NotImplementedError - - def __del__(self) -> None: - """Cleanup method called when the object is destroyed - - Ensures that all pending messages are flushed before the producer - is destroyed, preventing message loss. - """ - if self.producer: - self.producer.flush() - - -class SimpleKafkaProduceHandler(KafkaProduceHandler): - """Simple Kafka Producer wrapper without Write-Exactly-Once semantics - - Provides basic message production capabilities with at-least-once delivery - guarantees. This implementation prioritizes simplicity and performance over - strict consistency guarantees. - """ - - def __init__(self): - """ - Sets up a Kafka producer with standard configuration for simple message - production without transactional guarantees. Broker addresses are - automatically configured from the global KAFKA_BROKERS setting. - """ - self.brokers = ",".join( - [ - f"{broker['hostname']}:{broker['internal_port']}" - for broker in KAFKA_BROKERS - ] - ) - - conf = { - "bootstrap.servers": self.brokers, - "enable.idempotence": False, - "acks": "1", - "message.max.bytes": 1000000000, - } - - super().__init__(conf) - - def produce(self, topic: str, data: str, key: None | str = None) -> None: - """Produce a message to the specified Kafka topic. - - Encodes and sends the provided data to the specified topic. The producer - is flushed before sending to ensure message delivery. Empty data is - silently ignored. - - Args: - topic (str): Target Kafka topic name. - data (str): Message data to send (ignored if empty). - key (str, optional): Optional message key for partitioning. - Default: None. - - Raises: - KafkaException: If message production fails. - BufferError: If the producer's message buffer is full. - """ - if not data: - return - - def operation(): - delivery_errors = [] - - def delivery_callback(err, msg): - kafka_delivery_report(err, msg) - if err: - delivery_errors.append(err) - - self.producer.flush() - self.producer.produce( - topic=topic, - key=key, - value=data, - callback=delivery_callback, - ) - self.producer.flush() - if delivery_errors: - delivery_error = delivery_errors[0] - if _is_retriable_kafka_error(delivery_error): - raise KafkaException(delivery_error) - raise ValueError(f"Kafka delivery failed: {delivery_error}") - - self._with_producer_retry(f"Kafka produce to {topic}", operation) - - -class ExactlyOnceKafkaProduceHandler(KafkaProduceHandler): - """Kafka Producer wrapper with Write-Exactly-Once semantics - - Provides transactional message production with exactly-once delivery - guarantees. This implementation ensures that messages are delivered - exactly once, even in the presence of failures and retries. - - Configuration: - - transactional.id: Set to HOSTNAME for unique transaction identification - - enable.idempotence: True (required for exactly-once semantics) - - Note: - Each instance must have a unique transactional.id to avoid conflicts. - """ - - def __init__(self): - """ - Sets up a Kafka producer with transactional capabilities for exactly-once - semantics. The producer is initialized with transactions enabled and - configured with a unique transactional ID based on the hostname. - - Raises: - KafkaException: If transaction initialization fails. - """ - self.brokers = ",".join( - [ - f"{broker['hostname']}:{broker['internal_port']}" - for broker in KAFKA_BROKERS - ] - ) - - conf = { - "bootstrap.servers": self.brokers, - "transactional.id": f"{HOSTNAME}-{uuid.uuid4()}", - "enable.idempotence": True, - "message.max.bytes": 1000000000, - } - - super().__init__(conf) - self._init_transactions_with_retry() - - def _reset_producer(self) -> None: - super()._reset_producer() - self._init_transactions_with_retry() - - def _init_transactions_with_retry(self) -> None: - retry_forever( - lambda: self.producer.init_transactions(), - "Kafka transactional producer initialization", - ) - - def produce(self, topic: str, data: str, key: None | str = None) -> None: - """Produce a message to the specified Kafka topic with exactly-once semantics. - - Sends the provided data within a Kafka transaction to ensure exactly-once - delivery. The transaction is automatically committed on success or aborted - on failure. Empty data is silently ignored. - - Args: - topic (str): Target Kafka topic name. - data (str): Message data to send (ignored if empty). - key (str, optional): Optional message key for partitioning. - Default: None. - - Raises: - KafkaException: If message production or transaction handling fails. - RuntimeError: If transaction commit fails after retries. - """ - if not data: - return - - def operation(): - self.producer.flush() - self.producer.begin_transaction() - - try: - self.producer.produce( - topic=topic, - key=key, - value=data, - callback=kafka_delivery_report, - ) - self.commit_transaction_with_retry() - except Exception as e: - logger.info(f"aborted for topic {topic}") - try: - self.producer.abort_transaction() - except Exception as abort_exception: - logger.warning( - "Kafka transaction abort failed: %s", abort_exception - ) - logger.error("Transaction aborted.") - logger.error(e) - raise - - self._with_producer_retry(f"Kafka transactional produce to {topic}", operation) - - def commit_transaction_with_retry( - self, max_retries: int = 3, retry_interval_ms: int = 1000 - ) -> None: - """Commit a Kafka transaction with automatic retry logic. - - Attempts to commit the current transaction with built-in retry mechanism - for handling transient failures. If committing fails due to conflicting - API calls, the method will retry after the specified interval. - - Args: - max_retries (int): Maximum number of commit retry attempts. Default: 3. - retry_interval_ms (int): Time to wait between retries in milliseconds. - Default: 1000. - - Raises: - KafkaException: If transaction commit fails for reasons other than - conflicting API calls. - RuntimeError: If transaction commit fails after all retry attempts. - """ - committed = False - retry_count = 0 - - while not committed and retry_count < max_retries: - try: - self.producer.commit_transaction() - committed = True - except KafkaException as e: - if ( - "Conflicting commit_transaction API call is already in progress" - in str(e) - ): - retry_count += 1 - logger.debug( - "Conflicting commit_transaction API call is already in progress: Retrying" - ) - time.sleep(retry_interval_ms / 1000.0) - else: - raise e - - if not committed: - raise RuntimeError("Failed to commit transaction after retries.") - - -class KafkaConsumeHandler(KafkaHandler): - """Abstract base class for Kafka Consumer wrappers - - Provides common functionality for Kafka message consumption including - topic creation, subscription management, and consumer configuration. - All consumer implementations should extend this class. - - Attributes: - brokers (str): Comma-separated list of Kafka broker addresses. - consumer (Consumer): Confluent Kafka Consumer instance. - """ - - def __init__(self, topics: str | list[str]) -> None: - """ - Creates a Kafka consumer, ensures the specified topics exist, and - subscribes to them. Topics are automatically created if they don't exist. - - Args: - topics (str | list[str]): Topic name(s) to subscribe to. - Can be a single topic string or list of topics. - - Raises: - TooManyFailedAttemptsError: If topic creation fails after retries. - KafkaException: If consumer creation or subscription fails. - """ - super().__init__() - self._last_consumed_message = None - - if isinstance(topics, str): - topics = [topics] - self.topics = topics - - # get brokers - self.brokers = ",".join( - [ - f"{broker['hostname']}:{broker['internal_port']}" - for broker in KAFKA_BROKERS - ] - ) - self.conf = self._build_consumer_conf() - self._connect_consumer() - - def _build_consumer_conf(self) -> dict: - return { - "bootstrap.servers": self.brokers, - "group.id": build_consumer_group_id(self.topics), - "enable.auto.commit": False, - "auto.offset.reset": "earliest", - "enable.partition.eof": True, - "max.poll.interval.ms": KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS, - } - - def _connect_consumer(self) -> None: - def connect(): - consumer = Consumer(self.conf) - admin_client = AdminClient( - { - "bootstrap.servers": self.brokers, - } - ) - target_partitions_by_topic = ensure_topics(admin_client, self.topics) - - if not self._all_topics_created( - self.topics, target_partitions_by_topic, consumer - ): - try: - consumer.close() - except Exception: - pass - raise TooManyFailedAttemptsError("Not all topics were created.") - - consumer.subscribe(self.topics) - return consumer - - self.consumer = retry_forever( - connect, f"Kafka consumer setup for {self.topics}" - ) - - def _reset_consumer(self) -> None: - try: - if self.consumer: - self.consumer.close() - except Exception as exception: - logger.warning( - "Ignoring Kafka consumer close failure during reconnect: %s", exception - ) - self._last_consumed_message = None - self._connect_consumer() - - def commit(self) -> None: - """Commit the last message returned by ``consume``.""" - if self.consumer and self._last_consumed_message is not None: - retry_forever( - lambda: self.consumer.commit(self._last_consumed_message), - "Kafka consumer offset commit", - retryable=(KafkaException, RuntimeError, OSError), - ) - self._last_consumed_message = None - - @abstractmethod - def consume(self, *args, **kwargs): - """Abstract method for consuming messages from Kafka topics - - Implementations must define the specific behavior for message consumption, - including how to handle message polling, error handling, and data decoding. - - Args: - *args: Variable arguments depending on implementation. - **kwargs: Keyword arguments depending on implementation. - - Raises: - NotImplementedError: This method must be implemented by subclasses. - """ - raise NotImplementedError - - def consume_as_json(self) -> tuple[Optional[str], dict]: - """Consume messages and return them in JSON format. - - Consumes available messages from subscribed topics, decodes the data, - and returns the contents as a JSON dictionary. This method blocks - until a message is available. - - Returns: - tuple[Optional[str], dict]: A tuple containing: - - Message key (str or None) - - Message value as dictionary (empty dict if no message) - - Raises: - ValueError: If the message data format is invalid or cannot be parsed. - """ - key, value, topic = self.consume() - - if not key and not value: - return None, {} - - try: - eval_data = json.loads(value) - - if isinstance(eval_data, dict): - return key, eval_data - else: - raise - except Exception: - raise ValueError("Unknown data format") - - def _all_topics_created( - self, - topics: list[str], - min_partitions: int | dict[str, int] = 1, - consumer=None, - ) -> bool: - """Verify that all specified topics have been created successfully. - - Polls the Kafka cluster to check if each topic in the provided list - has been created. Retries for a maximum duration if topics are not - immediately available. - - Args: - topics (list[str]): List of topic names to verify. - - Returns: - bool: True if all topics are created, False if timeout exceeded. - """ - number_of_retries_left = 30 - all_topics_created = False - consumer = consumer or self.consumer - while not all_topics_created: # try for 15 seconds - assigned_topics = retry_forever( - lambda: consumer.list_topics(timeout=10), - "Kafka topic visibility check", - retryable=(KafkaException, RuntimeError, OSError), - ) - - all_topics_created = True - for topic in topics: - partition_count = _topic_partition_count(assigned_topics, topic) - required_partitions = ( - min_partitions.get(topic, 1) - if isinstance(min_partitions, dict) - else min_partitions - ) - if partition_count is None or partition_count < required_partitions: - all_topics_created = False - - if not all_topics_created: - number_of_retries_left -= 1 - - if not number_of_retries_left > 0: - return False - - time.sleep(0.5) - - return True - - def _poll_message(self): - while True: - try: - msg = self.consumer.poll(timeout=1.0) - except (KafkaException, RuntimeError, OSError) as exception: - logger.warning( - "Kafka consumer poll failed, reconnecting: %s", exception - ) - self._reset_consumer() - continue - - if msg is None: - return None - - if msg.error(): - if msg.error().code() == KafkaError._PARTITION_EOF: - return None - if _is_retriable_kafka_error(msg.error()): - logger.warning( - "Kafka consumer error is retriable, reconnecting: %s", - msg.error(), - ) - self._reset_consumer() - return None - - return msg - - def __del__(self) -> None: - """Cleanup method called when the object is destroyed - - Properly closes the Kafka consumer connection to release resources - and ensure graceful shutdown. - """ - if self.consumer: - self.consumer.close() - - @staticmethod - def _is_dicts(obj): - return isinstance(obj, list) and all(isinstance(item, dict) for item in obj) - - @staticmethod - def _decode_batch_data(data): - if data is None: - return [] - if not isinstance(data, list): - raise ValueError("Batch data must be a list.") - - decoded_data = [] - for item in data: - if isinstance(item, str): - decoded_data.append(json.loads(item)) - elif isinstance(item, (dict, list)): - decoded_data.append(item) - else: - raise ValueError("Batch data contains unsupported item type.") - return decoded_data - - def consume_as_object(self) -> tuple[None | str, Batch]: - """ - Consumes available messages on the specified topic. Decodes the data and converts it to a Batch - object. Returns the Batch object. - - Returns: - Consumed data as Batch object - - Raises: - ValueError: Invalid data format - """ - key, value, topic = self.consume() - if not key and not value: - # TODO: Change return value to fit the type, maybe switch to raise - return None, {} - eval_data: dict = json.loads(value) - eval_data["data"] = self._decode_batch_data(eval_data.get("data")) - batch_schema = marshmallow_dataclass.class_schema(Batch)() - eval_data: Batch = batch_schema.load(eval_data) - if isinstance(eval_data, Batch): - return key, eval_data - else: - raise ValueError("Unknown data format.") - - -class SimpleKafkaConsumeHandler(KafkaConsumeHandler): - """Simple Kafka Consumer wrapper without Write-Exactly-Once semantics - - Provides basic message consumption capabilities with at-least-once delivery - semantics. Messages are not automatically committed, allowing for manual - offset management by the application. - """ - - def __init__(self, topics: str | list[str]) -> None: - """ - Args: - topics (str | list[str]): Topic name(s) to subscribe to. - """ - super().__init__(topics) - - def consume(self) -> tuple[Optional[str], Optional[str], Optional[str]]: - """ - Consume messages from subscribed Kafka topics. - - Polls for available messages and decodes them. This method blocks - until a message is available or a keyboard interrupt is received. - The consumer does not automatically commit offsets. - - Returns: - tuple[Optional[str], Optional[str], Optional[str]]: A tuple containing: - - Message key (str or None) - - Message value (str or None) - - Topic name (str or None) - - Returns (None, None, None) if no valid message is retrieved. - - Raises: - ValueError: If the received message is invalid. - KeyboardInterrupt: If consumption is interrupted by user. - KafkaException: If message commit fails. - """ - empty_data_retrieved = False - - try: - while True: - msg = self._poll_message() - - if msg is None: - if not empty_data_retrieved: - logger.info("Waiting for messages...") - - empty_data_retrieved = True - continue - if msg.error(): - logger.error(f"Consumer error: {msg.error()}") - raise ValueError("Message is invalid") - - # unpack message - key = msg.key().decode("utf-8") if msg.key() else None - value = msg.value().decode("utf-8") if msg.value() else None - topic = msg.topic() if msg.topic() else None - self._last_consumed_message = msg - return key, value, topic - except KeyboardInterrupt: - logger.info("Stopping KafkaConsumeHandler...") - - -class ExactlyOnceKafkaConsumeHandler(KafkaConsumeHandler): - """Kafka Consumer wrapper with Write-Exactly-Once semantics - - Provides message consumption with exactly-once processing guarantees. - Messages are automatically committed after successful processing to - ensure each message is processed exactly once. - """ - - def __init__(self, topics: str | list[str]) -> None: - """ - Args: - topics (str | list[str]): Topic name(s) to subscribe to. - """ - super().__init__(topics) - - def consume(self) -> tuple[Optional[str], Optional[str], Optional[str]]: - """ - Consume messages from subscribed Kafka topics with exactly-once semantics. - - Polls for available messages, decodes them, and automatically commits - the message offset after successful processing. This ensures each - message is processed exactly once. - - Returns: - tuple[Optional[str], Optional[str], Optional[str]]: A tuple containing: - - Message key (str or None) - - Message value (str or None) - - Topic name (str or None) - - Returns (None, None, None) if no valid message is retrieved. - - Raises: - ValueError: If the received message is invalid. - KeyboardInterrupt: If consumption is interrupted by user. - KafkaException: If message commit fails. - """ - empty_data_retrieved = False - - try: - while True: - msg = self._poll_message() - - if msg is None: - if not empty_data_retrieved: - logger.info("Waiting for messages...") - - empty_data_retrieved = True - continue - - if msg.error(): - logger.error(f"Consumer error: {msg.error()}") - raise ValueError("Message is invalid") - - # unpack message - key = msg.key().decode("utf-8") if msg.key() else None - value = msg.value().decode("utf-8") if msg.value() else None - topic = msg.topic() if msg.topic() else None - self._last_consumed_message = msg - - return key, value, topic - except KeyboardInterrupt: - logger.info("Shutting down KafkaConsumeHandler...") - - # @staticmethod - # def _is_dicts(obj): - # """Check if the provided object is a list containing only dictionaries. - - # Args: - # obj: Object to check. - - # Returns: - # bool: True if obj is a list of dictionaries, False otherwise. - # """ - # return isinstance(obj, list) and all(isinstance(item, dict) for item in obj) - - # def consume_as_object(self) -> tuple[Optional[str], Batch]: - # """ - # Consume messages and return them as Batch objects. - - # Consumes available messages from subscribed topics, decodes the data, - # and converts it to a structured Batch object using marshmallow schema - # validation. This method provides type-safe message consumption. - - # Returns: - # tuple[Optional[str], Batch]: A tuple containing: - # - Message key (str or None). - # - Batch object containing the deserialized message data. - - # Raises: - # ValueError: If the message data format is invalid or cannot be - # converted to a Batch object. - # marshmallow.ValidationError: If data doesn't conform to Batch schema. - # """ - # key, value, topic = self.consume() - - # if not key and not value: - # # TODO: Change return value to fit the type, maybe switch to raise - # return None, {} - - # eval_data: dict = ast.literal_eval(value) - - # if self._is_dicts(eval_data.get("data")): - # eval_data["data"] = eval_data.get("data") - # else: - # eval_data["data"] = [ - # ast.literal_eval(item) for item in eval_data.get("data") - # ] - - # batch_schema = marshmallow_dataclass.class_schema(Batch)() - # eval_data: Batch = batch_schema.load(eval_data) - - # if isinstance(eval_data, Batch): - # return key, eval_data - # else: - # raise ValueError("Unknown data format.") diff --git a/src/base/retry.py b/src/base/retry.py index c10affab..aadaf1c7 100644 --- a/src/base/retry.py +++ b/src/base/retry.py @@ -1,10 +1,10 @@ import os import random import time +from dataclasses import dataclass from typing import Any, Callable, TypeVar from src.base.log_config import get_logger -from src.base.utils import setup_config logger = get_logger("base.retry") @@ -19,32 +19,47 @@ } -def resilience_config() -> dict[str, Any]: - config = setup_config() - if not isinstance(config, dict): - return dict(_DEFAULT_CONFIG) - retry_config = config.get("pipeline", {}).get("resilience", {}).get("retry", {}) - if not isinstance(retry_config, dict): - retry_config = {} +@dataclass(frozen=True) +class RetrySettings: + """Validated retry configuration loaded once during module initialization.""" + + initial_delay_seconds: float + max_delay_seconds: float + backoff_multiplier: float + jitter_seconds: float + log_every_attempts: int + + +def load_retry_settings(config: dict[str, Any]) -> RetrySettings: + """Build immutable retry settings from an already-loaded application config.""" + retry_config = ( + config.get("pipeline", {}).get("resilience", {}).get("retry", {}) + if isinstance(config, dict) + else {} + ) merged = dict(_DEFAULT_CONFIG) - merged.update(retry_config) - return merged + if isinstance(retry_config, dict): + merged.update(retry_config) + + initial_delay = max(0.01, _float_setting(merged, "initial_delay_seconds")) + return RetrySettings( + initial_delay_seconds=initial_delay, + max_delay_seconds=max( + initial_delay, _float_setting(merged, "max_delay_seconds") + ), + backoff_multiplier=max(1.0, _float_setting(merged, "backoff_multiplier")), + jitter_seconds=max(0.0, _float_setting(merged, "jitter_seconds")), + log_every_attempts=max(1, _int_setting(merged, "log_every_attempts")), + ) def retry_forever( operation: Callable[[], T], description: str, - retry_config: dict[str, Any] | None = None, + settings: RetrySettings, retryable: tuple[type[BaseException], ...] = (Exception,), ) -> T: - config = retry_config if retry_config is not None else resilience_config() - initial_delay = _float_setting(config, "initial_delay_seconds") - max_delay = _float_setting(config, "max_delay_seconds") - multiplier = max(1.0, _float_setting(config, "backoff_multiplier")) - jitter = max(0.0, _float_setting(config, "jitter_seconds")) - log_every = max(1, _int_setting(config, "log_every_attempts")) - - delay = initial_delay + delay = settings.initial_delay_seconds attempt = 0 while True: @@ -52,7 +67,7 @@ def retry_forever( return operation() except retryable as exception: attempt += 1 - if attempt == 1 or attempt % log_every == 0: + if attempt == 1 or attempt % settings.log_every_attempts == 0: logger.warning( "%s failed on attempt %d: %s. Retrying in %.1fs.", description, @@ -60,9 +75,16 @@ def retry_forever( exception, delay, ) - sleep_for = delay + (random.uniform(0, jitter) if jitter else 0) + sleep_for = delay + ( + random.uniform(0, settings.jitter_seconds) + if settings.jitter_seconds + else 0 + ) time.sleep(sleep_for) - delay = min(max_delay, delay * multiplier) + delay = min( + settings.max_delay_seconds, + delay * settings.backoff_multiplier, + ) def _float_setting(config: dict[str, Any], key: str) -> float: diff --git a/src/detector/detector.py b/src/detector/detector.py index 287ca6e3..5edb4c73 100644 --- a/src/detector/detector.py +++ b/src/detector/detector.py @@ -22,7 +22,7 @@ apply_model_acceleration, resolve_acceleration_config, ) -from src.base.kafka_handler import ( +from src.base.kafka import ( ExactlyOnceKafkaConsumeHandler, ExactlyOnceKafkaProduceHandler, KafkaMessageFetchException, @@ -267,7 +267,7 @@ def __init__( ) ) - def get_and_fill_data(self) -> None: + def get_and_fill_data(self, source_message=None) -> None: """ Consume data from Kafka and store it for processing. @@ -284,7 +284,7 @@ def get_and_fill_data(self) -> None: "current workload." ) return - key, data = self.kafka_consume_handler.consume_as_object() + key, data = self.kafka_consume_handler.consume_as_object(source_message) if data.data: self.parent_row_id = data.batch_tree_row_id self.suspicious_batch_id = data.batch_id @@ -849,13 +849,25 @@ def bootstrap_detector_instance(self): """ while True: try: - logger.debug("Before getting and filling data") - self.get_and_fill_data() - logger.debug("Inspect Data") - self.detect() - logger.debug("Send warnings") - self.send_warning() - self.kafka_consume_handler.commit() + source_messages = self.kafka_consume_handler.consume_batch() + if not source_messages: + continue + if self.kafka_produce_handler is None: + self.kafka_produce_handler = ExactlyOnceKafkaProduceHandler() + + with self.kafka_produce_handler.transaction_batch( + self.kafka_consume_handler, source_messages + ): + for source_message in source_messages: + try: + logger.debug("Before getting and filling data") + self.get_and_fill_data(source_message) + logger.debug("Inspect Data") + self.detect() + logger.debug("Send warnings") + self.send_warning() + finally: + self.clear_data() except KafkaMessageFetchException as e: # pragma: no cover logger.debug(e) except IOError as e: @@ -866,8 +878,6 @@ def bootstrap_detector_instance(self): except KeyboardInterrupt: logger.info("Closing down Detector...") break - finally: - self.clear_data() async def start(self): # pragma: no cover """ diff --git a/src/inspector/inspector.py b/src/inspector/inspector.py index 4cdabdd9..7d6cb88e 100644 --- a/src/inspector/inspector.py +++ b/src/inspector/inspector.py @@ -19,7 +19,7 @@ generate_collisions_resistant_uuid, ) from src.base.acceleration import resolve_acceleration_config -from src.base.kafka_handler import ( +from src.base.kafka import ( ExactlyOnceKafkaConsumeHandler, ExactlyOnceKafkaProduceHandler, KafkaMessageFetchException, @@ -148,7 +148,7 @@ def __init__(self, consume_topic, produce_topics, config) -> None: ) ) - def get_and_fill_data(self) -> None: + def get_and_fill_data(self, source_message=None) -> None: """Consumes data from Kafka and stores it for processing. Fetches batch data from the configured Kafka topic and stores it in internal data structures. @@ -162,7 +162,7 @@ def get_and_fill_data(self) -> None: ) return - key, data = self.kafka_consume_handler.consume_as_object() + key, data = self.kafka_consume_handler.consume_as_object(source_message) if data: self.parent_row_id = data.batch_tree_row_id self.batch_id = data.batch_id @@ -383,10 +383,20 @@ def bootstrap_inspection_process(self): logger.info(f"Starting {self.name}") while True: try: - self.get_and_fill_data() - self.inspect() - self.send_data() - self.kafka_consume_handler.commit() + source_messages = self.kafka_consume_handler.consume_batch() + if not source_messages: + continue + + with self.kafka_produce_handler.transaction_batch( + self.kafka_consume_handler, source_messages + ): + for source_message in source_messages: + try: + self.get_and_fill_data(source_message) + self.inspect() + self.send_data() + finally: + self.clear_data() except KafkaMessageFetchException as e: # pragma: no cover logger.debug(e) except IOError as e: @@ -397,8 +407,6 @@ def bootstrap_inspection_process(self): except KeyboardInterrupt: logger.info(f" {self.consume_topic} Closing down Inspector...") break - finally: - self.clear_data() async def start(self): # pragma: no cover """ diff --git a/src/logcollector/batch_handler.py b/src/logcollector/batch_handler.py index b5ed112a..d84668cd 100644 --- a/src/logcollector/batch_handler.py +++ b/src/logcollector/batch_handler.py @@ -10,7 +10,7 @@ sys.path.append(os.getcwd()) from src.base.data_classes.batch import Batch from src.base.clickhouse_kafka_sender import ClickHouseKafkaSender -from src.base.kafka_handler import ExactlyOnceKafkaProduceHandler +from src.base.kafka import ExactlyOnceKafkaProduceHandler from src.base.utils import ( setup_config, get_batch_configuration, @@ -451,10 +451,12 @@ def __init__( produce_topics, collector_name, monitoring_kafka_producer=None, + use_timer=True, ): self.topics = produce_topics self.batch_configuration = get_batch_configuration(collector_name) self.timer = None + self.use_timer = use_timer self.monitoring_kafka_producer = ( monitoring_kafka_producer or ClickHouseKafkaSender.create_shared_producer() ) @@ -522,9 +524,16 @@ def add_message(self, key: str, message: str) -> None: f"Full batch: Successfully sent batch messages for subnet_id {key}.\n" f" ⤷ {number_of_messages_for_key} messages sent." ) - elif not self.timer: # First time setting the timer + elif self.use_timer and not self.timer: # First time setting the timer self._reset_timer() + def flush(self) -> None: + """Send every accumulated batch without scheduling another timer.""" + if self.timer: + self.timer.cancel() + self.timer = None + self._send_all_batches(reset_timer=False) + def _send_all_batches(self, reset_timer: bool = True) -> None: """Dispatches all available batches to the Kafka queue. diff --git a/src/logcollector/collector.py b/src/logcollector/collector.py index de287ace..029a92cb 100644 --- a/src/logcollector/collector.py +++ b/src/logcollector/collector.py @@ -8,7 +8,7 @@ sys.path.append(os.getcwd()) from src.base.clickhouse_kafka_sender import ClickHouseKafkaSender -from src.base.kafka_handler import ( +from src.base.kafka import ( ExactlyOnceKafkaConsumeHandler, ) from src.base.logline_handler import LoglineHandler @@ -76,6 +76,7 @@ def __init__( produce_topics=produce_topics, collector_name=collector_name, monitoring_kafka_producer=self.monitoring_kafka_producer, + use_timer=False, ) self.logline_handler = LoglineHandler(validation_config) @@ -142,10 +143,28 @@ def fetch(self) -> None: """ while True: - key, value, topic = self.kafka_consume_handler.consume() - logger.debug(f"From Kafka: '{value}'") - self.send(datetime.datetime.now(), value, server_message_id=key) - self.kafka_consume_handler.commit() + source_messages = self.kafka_consume_handler.consume_batch( + max_messages=self.batch_configuration["batch_size"], + timeout_ms=int(self.batch_configuration["batch_timeout"] * 1000), + ) + if not source_messages: + continue + + with self.batch_handler.kafka_produce_handler.transaction_batch( + self.kafka_consume_handler, source_messages + ): + for source_message in source_messages: + if source_message.value is None: + raise ValueError( + "LogCollector received a Kafka record without data." + ) + logger.debug(f"From Kafka: '{source_message.value}'") + self.send( + datetime.datetime.now(), + source_message.value, + server_message_id=source_message.key, + ) + self.batch_handler.flush() def send( self, diff --git a/src/logserver/server.py b/src/logserver/server.py index 66c7755a..27e54d7f 100644 --- a/src/logserver/server.py +++ b/src/logserver/server.py @@ -7,7 +7,7 @@ import aiofiles sys.path.append(os.getcwd()) -from src.base.kafka_handler import ( +from src.base.kafka import ( ExactlyOnceKafkaConsumeHandler, ExactlyOnceKafkaProduceHandler, ) @@ -114,20 +114,31 @@ def fetch_from_kafka(self) -> None: its timestamp ("timestamp_in") is logged. """ while True: - key, value, topic = self.kafka_consume_handler.consume() - logger.debug(f"From Kafka: '{value}'") - - message_id = uuid.uuid4() - self.server_logs.insert( - dict( - message_id=message_id, - timestamp_in=datetime.datetime.now(), - message_text=value, - ) - ) - - self.send(message_id, value) - self.kafka_consume_handler.commit() + source_messages = self.kafka_consume_handler.consume_batch() + if not source_messages: + continue + + with self.kafka_produce_handler.transaction_batch( + self.kafka_consume_handler, source_messages + ): + for source_message in source_messages: + value = source_message.value + if value is None: + raise ValueError( + "LogServer received a Kafka record without data." + ) + logger.debug(f"From Kafka: '{value}'") + + message_id = uuid.uuid4() + self.server_logs.insert( + dict( + message_id=message_id, + timestamp_in=datetime.datetime.now(), + message_text=value, + ) + ) + + self.send(message_id, value) def build_logserver_worker(consume_topic, produce_topics, worker_id=None): diff --git a/src/monitoring/clickhouse_batch_sender.py b/src/monitoring/clickhouse_batch_sender.py index c4c21c46..2d263fec 100644 --- a/src/monitoring/clickhouse_batch_sender.py +++ b/src/monitoring/clickhouse_batch_sender.py @@ -10,12 +10,13 @@ sys.path.append(os.getcwd()) from src.base.log_config import get_logger -from src.base.retry import retry_forever +from src.base.retry import load_retry_settings, retry_forever from src.base.utils import setup_config logger = get_logger() CONFIG = setup_config() +RETRY_SETTINGS = load_retry_settings(CONFIG) CLICKHOUSE_HOSTNAME = CONFIG["environment"]["monitoring"]["clickhouse_server"][ "hostname" ] @@ -91,7 +92,8 @@ class ClickHouseBatchSender: insertion with automatic schema validation for all monitored tables. """ - def __init__(self): + def __init__(self, use_timer: bool = True): + self.use_timer = use_timer self.tables = { "server_logs": Table( "server_logs", @@ -239,6 +241,7 @@ def _connect_client(self): return retry_forever( create_clickhouse_client, "ClickHouse client connection", + RETRY_SETTINGS, ) def _reset_client(self) -> None: @@ -278,7 +281,7 @@ def add(self, table_name: str, data: dict[str, Any]): if len(self.batch.get(table_name)) >= self.max_batch_size: self.insert(table_name) - if not self.timer: + if self.use_timer and not self.timer: self._start_timer() def insert(self, table_name: str): @@ -312,9 +315,15 @@ def insert_batch(): raise retry_forever( - insert_batch, f"ClickHouse insert for table '{table_name}'" + insert_batch, + f"ClickHouse insert for table '{table_name}'", + RETRY_SETTINGS, + ) + logger.debug( + "Inserted %d monitoring row(s) into %s.", + len(pending_rows), + table_name, ) - logger.debug(f"Inserted {table_name=},{pending_rows=},{column_names=}") self.batch[table_name] = [] def insert_all(self): diff --git a/src/monitoring/monitoring_agent.py b/src/monitoring/monitoring_agent.py index 059fd938..0e648ea9 100644 --- a/src/monitoring/monitoring_agent.py +++ b/src/monitoring/monitoring_agent.py @@ -7,22 +7,28 @@ sys.path.append(os.getcwd()) from src.monitoring.clickhouse_batch_sender import * -from src.base.kafka_handler import SimpleKafkaConsumeHandler +from src.base.kafka import SimpleKafkaConsumeHandler from src.base.data_classes.clickhouse_connectors import TABLE_NAME_TO_TYPE from src.base.log_config import get_logger from src.base.utils import setup_config -from src.base.execution import create_pipeline_executor -from src.base.retry import retry_forever +from src.base.execution import ( + run_thread_worker_pool, + start_pipeline_worker_replicas, +) +from src.base.retry import load_retry_settings, retry_forever logger = get_logger() module_name = "monitoring.agent" CONFIG = setup_config() +RETRY_SETTINGS = load_retry_settings(CONFIG) CREATE_TABLES_DIRECTORY = "docker/create_tables" # TODO: Get from config CLICKHOUSE_HOSTNAME = CONFIG["environment"]["monitoring"]["clickhouse_server"][ "hostname" ] -RETRY_CONFIG = CONFIG.get("pipeline", {}).get("resilience", {}).get("retry", {}) +MONITORING_CONSUMER_CONFIG = CONFIG["pipeline"]["monitoring"]["kafka_consumer"] +MONITORING_CONSUMER_BATCH_SIZE = max(1, int(MONITORING_CONSUMER_CONFIG["batch_size"])) +MONITORING_CONSUMER_TIMEOUT_MS = max(0, int(MONITORING_CONSUMER_CONFIG["timeout_ms"])) def prepare_all_tables(): @@ -53,7 +59,7 @@ def _iter_statements(sql_content: str): with retry_forever( create_clickhouse_client, "ClickHouse table preparation connection", - retry_config=RETRY_CONFIG, + RETRY_SETTINGS, ) as client: for statement in _iter_statements(sql_content): try: @@ -71,11 +77,12 @@ class MonitoringAgent: the batch sender for persistent storage. """ - def __init__(self): + def __init__(self, worker_id: str = "default"): """ Sets up consumption from all ClickHouse-related Kafka topics and initializes the batch sender for efficient data insertion. """ + self.worker_id = worker_id self.table_names = [ "server_logs", "server_logs_timestamps", @@ -95,9 +102,17 @@ def __init__(self): self.topics = [f"clickhouse_{table_name}" for table_name in self.table_names] self.kafka_consumer = SimpleKafkaConsumeHandler(self.topics) - self.batch_sender = ClickHouseBatchSender() - - async def start(self): + # This worker explicitly flushes before committing its Kafka offsets. + # A timer would race with record decoding and create smaller inserts. + self.batch_sender = ClickHouseBatchSender(use_timer=False) + self.data_schemas = { + table_name: marshmallow_dataclass.class_schema( + TABLE_NAME_TO_TYPE[table_name] + )() + for table_name in self.table_names + } + + def run(self) -> None: """Starts the monitoring agent to consume and process data continuously. Runs an infinite loop to consume messages from Kafka topics, deserialize @@ -108,41 +123,84 @@ async def start(self): KeyboardInterrupt: When the agent is manually stopped. Exception: For any other processing errors (logged as warnings). """ - loop = asyncio.get_running_loop() - executor = create_pipeline_executor(CONFIG, module_name) - try: while True: try: - key, value, topic = await loop.run_in_executor( - executor, self.kafka_consumer.consume + source_records = self.kafka_consumer.consume_batch( + MONITORING_CONSUMER_BATCH_SIZE, + MONITORING_CONSUMER_TIMEOUT_MS, ) - logger.debug(f"From Kafka: {value}") - - table_name = topic.replace("clickhouse_", "") - data_schema = marshmallow_dataclass.class_schema( - TABLE_NAME_TO_TYPE.get(table_name) - )() - data = data_schema.loads(value) + if not source_records: + continue - self.batch_sender.add(table_name, asdict(data)) + logger.debug( + "Monitoring worker %s fetched %d Kafka record(s).", + self.worker_id, + len(source_records), + ) + for source_record in source_records: + try: + table_name = source_record.topic.removeprefix("clickhouse_") + data = self.data_schemas[table_name].loads( + source_record.value + ) + self.batch_sender.add(table_name, asdict(data)) + except Exception as exception: + logger.warning( + "Discarding invalid monitoring record at %s[%d] " + "offset %d: %s", + source_record.topic, + source_record.partition, + source_record.offset, + exception, + ) + + self.batch_sender.insert_all() + self.kafka_consumer.commit(source_records) except KeyboardInterrupt: logger.info("Stopped MonitoringAgent.") break except Exception as e: logger.warning(e) finally: - executor.shutdown(wait=False, cancel_futures=True) + self.batch_sender.insert_all() + + +def build_monitoring_worker(worker_id: str) -> MonitoringAgent: + """Create one independently consumable monitoring worker.""" + return MonitoringAgent(worker_id=worker_id) + + +def run_monitoring_worker_process(process_index: int, threads_per_process: int) -> None: + """Run all monitoring threads assigned to one process.""" + run_thread_worker_pool( + worker_factory=build_monitoring_worker, + target_name="run", + module_name=module_name, + instance_name=None, + process_index=process_index, + threads_per_process=threads_per_process, + ) + + +async def start_monitoring_workers() -> None: + """Start every configured monitoring consumer replica.""" + await start_pipeline_worker_replicas( + config=CONFIG, + module_name=module_name, + instance_name=None, + worker_factory=build_monitoring_worker, + target_name="run", + process_entrypoint=run_monitoring_worker_process, + ) def main(): - """Creates the :class:`MonitoringAgent` instance and starts it. + """Start all configured :class:`MonitoringAgent` workers. - Entry point for the monitoring agent that initializes and runs - the asynchronous monitoring process. + Each worker owns an independent Kafka consumer and ClickHouse batch sender. """ - clickhouse_consumer = MonitoringAgent() - asyncio.run(clickhouse_consumer.start()) + asyncio.run(start_monitoring_workers()) if __name__ == "__main__": # pragma: no cover diff --git a/src/prefilter/prefilter.py b/src/prefilter/prefilter.py index f8e7b3b7..7edee651 100644 --- a/src/prefilter/prefilter.py +++ b/src/prefilter/prefilter.py @@ -10,7 +10,7 @@ from src.base.clickhouse_kafka_sender import ClickHouseKafkaSender from src.base.data_classes.batch import Batch from src.base.logline_handler import LoglineHandler -from src.base.kafka_handler import ( +from src.base.kafka import ( ExactlyOnceKafkaProduceHandler, ExactlyOnceKafkaConsumeHandler, KafkaMessageFetchException, @@ -108,7 +108,7 @@ def __init__( ) ) - def get_and_fill_data(self) -> None: + def get_and_fill_data(self, source_message=None) -> None: """Retrieves and processes new data from Kafka. This method: @@ -124,7 +124,7 @@ def get_and_fill_data(self) -> None: """ self.clear_data() # clear in case we already have data stored - key, data = self.kafka_consume_handler.consume_as_object() + key, data = self.kafka_consume_handler.consume_as_object(source_message) self.subnet_id = key if data.data: self.parent_row_id = data.batch_tree_row_id @@ -315,10 +315,20 @@ def bootstrap_prefiltering_process(self): """ while True: - self.get_and_fill_data() - self.check_data_relevance_using_rules() - self.send_filtered_data() - self.kafka_consume_handler.commit() + source_messages = self.kafka_consume_handler.consume_batch() + if not source_messages: + continue + + with self.kafka_produce_handler.transaction_batch( + self.kafka_consume_handler, source_messages + ): + for source_message in source_messages: + try: + self.get_and_fill_data(source_message) + self.check_data_relevance_using_rules() + self.send_filtered_data() + finally: + self.clear_data() async def start(self): # pragma: no cover """Starts the ``Prefilter`` processing loop. diff --git a/tests/clickhouse/test_clickhouse_batch_sender.py b/tests/clickhouse/test_clickhouse_batch_sender.py index c24f1964..dc2c3d63 100644 --- a/tests/clickhouse/test_clickhouse_batch_sender.py +++ b/tests/clickhouse/test_clickhouse_batch_sender.py @@ -156,6 +156,21 @@ def test_timer_already_started(self): # Assert mock_start_timer.assert_not_called() + def test_timer_can_be_disabled_for_explicit_batch_management(self): + with patch("src.monitoring.clickhouse_batch_sender.clickhouse_connect"): + sut = ClickHouseBatchSender(use_timer=False) + test_table_name = "test_table" + sut.tables = {test_table_name: Table(test_table_name, {})} + sut.batch = {test_table_name: []} + + with ( + patch("src.monitoring.clickhouse_batch_sender.Table.verify"), + patch.object(sut, "_start_timer") as mock_start_timer, + ): + sut.add(test_table_name, {"value": 1}) + + mock_start_timer.assert_not_called() + def test_max_batch_size_reached(self): # Arrange test_table_name = "test_table" diff --git a/tests/clickhouse/test_clickhouse_kafka_sender.py b/tests/clickhouse/test_clickhouse_kafka_sender.py index d19a44dd..b4fde5ef 100644 --- a/tests/clickhouse/test_clickhouse_kafka_sender.py +++ b/tests/clickhouse/test_clickhouse_kafka_sender.py @@ -6,7 +6,7 @@ class TestInit(unittest.TestCase): @patch("src.base.clickhouse_kafka_sender.marshmallow_dataclass") - @patch("src.base.clickhouse_kafka_sender.SimpleKafkaProduceHandler") + @patch("src.base.clickhouse_kafka_sender.BufferedKafkaProduceHandler") def test_init(self, mock_produce_handler, mock_marshmallow): # Arrange table_name = "test_table" @@ -22,7 +22,7 @@ def test_init(self, mock_produce_handler, mock_marshmallow): mock_produce_handler.assert_called_once() @patch("src.base.clickhouse_kafka_sender.marshmallow_dataclass") - @patch("src.base.clickhouse_kafka_sender.SimpleKafkaProduceHandler") + @patch("src.base.clickhouse_kafka_sender.BufferedKafkaProduceHandler") def test_init_uses_provided_producer(self, mock_produce_handler, mock_marshmallow): # Arrange table_name = "test_table" @@ -39,7 +39,7 @@ def test_init_uses_provided_producer(self, mock_produce_handler, mock_marshmallo class TestInsert(unittest.TestCase): @patch("src.base.clickhouse_kafka_sender.marshmallow_dataclass") - @patch("src.base.clickhouse_kafka_sender.SimpleKafkaProduceHandler") + @patch("src.base.clickhouse_kafka_sender.BufferedKafkaProduceHandler") def test_insert(self, mock_produce_handler, mock_marshmallow): # Arrange mock_produce_handler_instance = mock_produce_handler diff --git a/tests/clickhouse/test_monitoring_history.py b/tests/clickhouse/test_monitoring_history.py new file mode 100644 index 00000000..8f2a4dbe --- /dev/null +++ b/tests/clickhouse/test_monitoring_history.py @@ -0,0 +1,72 @@ +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +CREATE_TABLES = ROOT / "docker" / "create_tables" +DASHBOARDS = ROOT / "docker" / "grafana-provisioning" / "dashboards" + + +class TestMonitoringHistorySchema(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.rollups = (CREATE_TABLES / "zz_monitoring_rollups.sql").read_text() + cls.history = (CREATE_TABLES / "zzz_monitoring_history.sql").read_text() + + def test_uses_three_non_overlapping_retention_tiers(self): + self.assertIn("INTERVAL 7 DAY", self.rollups) + self.assertIn("INTERVAL 30 DAY", self.history) + self.assertIn("INTERVAL 90 DAY", self.history) + self.assertIn("CREATE OR REPLACE VIEW alerts_history", self.history) + self.assertIn("CREATE OR REPLACE VIEW fill_levels_history", self.history) + + def test_latency_snapshots_are_refreshable_and_replaceable(self): + for resolution in ("1m", "15m", "1h"): + self.assertIn( + f"CREATE TABLE IF NOT EXISTS pipeline_latency_{resolution}", + self.history, + ) + self.assertIn( + f"CREATE MATERIALIZED VIEW IF NOT EXISTS " + f"pipeline_latency_{resolution}_refresh", + self.history, + ) + self.assertEqual(3, self.history.count("ENGINE = ReplacingMergeTree")) + self.assertEqual(3, self.history.count(" APPEND TO pipeline_latency_")) + + def test_latency_history_keeps_distribution_summary(self): + for column in ( + "sample_count", + "min_latency_us", + "avg_latency_us", + "p50_latency_us", + "p95_latency_us", + "p99_latency_us", + "max_latency_us", + ): + self.assertIn(column, self.history) + + +class TestHistoricalDashboardQueries(unittest.TestCase): + def test_alert_and_fill_dashboards_use_history_views(self): + alerts = (DASHBOARDS / "alerts.json").read_text() + overview = (DASHBOARDS / "overview.json").read_text() + log_volumes = (DASHBOARDS / "log_volumes.json").read_text() + + self.assertNotIn("FROM alerts_1m", alerts + overview) + self.assertIn("FROM alerts_history", alerts + overview) + self.assertNotIn("FROM fill_levels_1m", log_volumes) + self.assertIn("FROM fill_levels_history", log_volumes) + + def test_latency_dashboard_uses_historical_latency_views(self): + latencies = (DASHBOARDS / "latencies.json").read_text() + overview = (DASHBOARDS / "overview.json").read_text() + + self.assertNotIn("FROM batch_tree bt1", latencies + overview) + self.assertIn("FROM pipeline_latency_values", latencies + overview) + self.assertIn("FROM pipeline_transport_latency_values", latencies) + self.assertIn("FROM pipeline_roundtrip_latency_values", latencies) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/detector/test_detector.py b/tests/detector/test_detector.py index e74aa250..494d84a3 100644 --- a/tests/detector/test_detector.py +++ b/tests/detector/test_detector.py @@ -14,7 +14,7 @@ build_detector_consume_topic, build_downstream_detector_topics, ) -from src.base.kafka_handler import KafkaMessageFetchException +from src.base.kafka import KafkaMessageFetchException MINIMAL_DETECTOR_CONFIG = { "name": "test-detector", @@ -955,6 +955,8 @@ def test_bootstrap_normal_execution( consume_topic="test_topic", detector_config=MINIMAL_DETECTOR_CONFIG ) sut.kafka_consume_handler = mock_kafka_consume_handler_instance + sut.kafka_consume_handler.consume_batch.return_value = [MagicMock()] + sut.kafka_produce_handler = MagicMock() # Mock the methods called in the loop with patch.object(sut, "get_and_fill_data") as mock_get_data, patch.object( @@ -990,6 +992,8 @@ def test_bootstrap_graceful_shutdown( sut = TestDetector( consume_topic="test_topic", detector_config=MINIMAL_DETECTOR_CONFIG ) + sut.kafka_consume_handler.consume_batch.return_value = [MagicMock()] + sut.kafka_produce_handler = MagicMock() # Mock methods and raise KeyboardInterrupt with patch.object(sut, "get_and_fill_data"), patch.object( diff --git a/tests/inspector/test_inspector.py b/tests/inspector/test_inspector.py index 4234e72b..b02cb3ab 100644 --- a/tests/inspector/test_inspector.py +++ b/tests/inspector/test_inspector.py @@ -7,7 +7,7 @@ import numpy as np from streamad.model import ZScoreDetector, RShashDetector -from src.base.kafka_handler import ( +from src.base.kafka import ( KafkaMessageFetchException, ) from src.base.data_classes.batch import Batch @@ -451,7 +451,6 @@ def test_inspect_multiple_models_configured( produce_topics=["produce_topic_1"], config=config, ) - # Mock data sut.messages = [{"test": "data"}] @@ -543,6 +542,7 @@ def test_bootstrap_normal_execution( produce_topics=["produce_topic_1"], config=config, ) + sut.kafka_consume_handler.consume_batch.return_value = [MagicMock()] # Mock data so send_data works sut.messages = [{"src_ip": "192.168.0.1", "logline_id": "test_id"}] @@ -603,6 +603,7 @@ def test_bootstrap_kafka_exception_handling( produce_topics=["produce_topic_1"], config=config, ) + sut.kafka_consume_handler.consume_batch.return_value = [MagicMock()] # Mock data so send_data works sut.messages = [{"src_ip": "192.168.0.1", "logline_id": "test_id"}] diff --git a/tests/kafka/test_exactly_once_kafka_consume_handler.py b/tests/kafka/test_exactly_once_kafka_consume_handler.py index d5958513..1a8215bc 100644 --- a/tests/kafka/test_exactly_once_kafka_consume_handler.py +++ b/tests/kafka/test_exactly_once_kafka_consume_handler.py @@ -8,14 +8,15 @@ from confluent_kafka import KafkaException, KafkaError from src.base.data_classes.batch import Batch -from src.base.kafka_handler import ExactlyOnceKafkaConsumeHandler +from src.base.kafka.config import KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS +from src.base.kafka import ExactlyOnceKafkaConsumeHandler CONSUMER_GROUP_ID = "default_gid.test_topic" class TestInit(unittest.TestCase): @patch( - "src.base.kafka_handler.KAFKA_BROKERS", + "src.base.kafka.config.KAFKA_BROKERS", [ { "hostname": "127.0.0.1", @@ -32,11 +33,11 @@ class TestInit(unittest.TestCase): ], ) @patch( - "src.base.kafka_handler.KafkaConsumeHandler._all_topics_created", + "src.base.kafka.consumer.KafkaConsumeHandler._all_topics_created", return_value=True, ) - @patch("src.base.kafka_handler.AdminClient") - @patch("src.base.kafka_handler.Consumer") + @patch("src.base.kafka.consumer.AdminClient") + @patch("src.base.kafka.consumer.Consumer") def test_init(self, mock_consumer, mock_admin_client, mock_all_topics_created): mock_consumer_instance = Mock() mock_consumer.return_value = mock_consumer_instance @@ -47,7 +48,8 @@ def test_init(self, mock_consumer, mock_admin_client, mock_all_topics_created): "enable.auto.commit": False, "auto.offset.reset": "earliest", "enable.partition.eof": True, - "max.poll.interval.ms": 1800000, + "max.poll.interval.ms": KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS, + "isolation.level": "read_committed", } sut = ExactlyOnceKafkaConsumeHandler(topics="test_topic") @@ -58,7 +60,7 @@ def test_init(self, mock_consumer, mock_admin_client, mock_all_topics_created): mock_consumer_instance.subscribe.assert_called_once() @patch( - "src.base.kafka_handler.KAFKA_BROKERS", + "src.base.kafka.config.KAFKA_BROKERS", [ { "hostname": "127.0.0.1", @@ -75,12 +77,12 @@ def test_init(self, mock_consumer, mock_admin_client, mock_all_topics_created): ], ) @patch( - "src.base.kafka_handler.KafkaConsumeHandler._all_topics_created", + "src.base.kafka.consumer.KafkaConsumeHandler._all_topics_created", return_value=True, ) @patch("src.base.retry.time.sleep", return_value=None) - @patch("src.base.kafka_handler.AdminClient") - @patch("src.base.kafka_handler.Consumer") + @patch("src.base.kafka.consumer.AdminClient") + @patch("src.base.kafka.consumer.Consumer") def test_init_retries_until_subscribe_succeeds( self, mock_consumer, mock_admin_client, mock_sleep, mock_all_topics_created ): @@ -93,7 +95,8 @@ def test_init_retries_until_subscribe_succeeds( "enable.auto.commit": False, "auto.offset.reset": "earliest", "enable.partition.eof": True, - "max.poll.interval.ms": 1800000, + "max.poll.interval.ms": KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS, + "isolation.level": "read_committed", } mock_consumer_instance.subscribe.side_effect = [KafkaException(), None] @@ -108,9 +111,9 @@ def test_init_retries_until_subscribe_succeeds( class TestConsume(unittest.TestCase): - @patch("src.base.kafka_handler.CONSUMER_GROUP_ID", "test_group_id") + @patch("src.base.kafka.config.CONSUMER_GROUP_ID", "test_group_id") @patch( - "src.base.kafka_handler.KAFKA_BROKERS", + "src.base.kafka.config.KAFKA_BROKERS", [ { "hostname": "127.0.0.1", @@ -126,12 +129,12 @@ class TestConsume(unittest.TestCase): }, ], ) - @patch("src.base.kafka_handler.Consumer") + @patch("src.base.kafka.consumer.Consumer") @patch( - "src.base.kafka_handler.KafkaConsumeHandler._all_topics_created", + "src.base.kafka.consumer.KafkaConsumeHandler._all_topics_created", return_value=True, ) - @patch("src.base.kafka_handler.AdminClient") + @patch("src.base.kafka.consumer.AdminClient") def setUp(self, mock_admin_client, mock_all_topics_created, mock_consumer): self.mock_consumer = mock_consumer self.topics = ["test_topic_1", "test_topic_2"] @@ -213,9 +216,9 @@ def test_consumer_raises_keyboard_interrupt(self): class TestDel(unittest.TestCase): - @patch("src.base.kafka_handler.CONSUMER_GROUP_ID", "test_group_id") + @patch("src.base.kafka.config.CONSUMER_GROUP_ID", "test_group_id") @patch( - "src.base.kafka_handler.KAFKA_BROKERS", + "src.base.kafka.config.KAFKA_BROKERS", [ { "hostname": "127.0.0.1", @@ -231,12 +234,12 @@ class TestDel(unittest.TestCase): }, ], ) - @patch("src.base.kafka_handler.Consumer") + @patch("src.base.kafka.consumer.Consumer") @patch( - "src.base.kafka_handler.KafkaConsumeHandler._all_topics_created", + "src.base.kafka.consumer.KafkaConsumeHandler._all_topics_created", return_value=True, ) - @patch("src.base.kafka_handler.AdminClient") + @patch("src.base.kafka.consumer.AdminClient") def test_del_with_existing_consumer( self, mock_admin_client, mock_all_topics_created, mock_consumer ): @@ -253,9 +256,9 @@ def test_del_with_existing_consumer( # Assert mock_consumer_instance.close.assert_called_once() - @patch("src.base.kafka_handler.CONSUMER_GROUP_ID", "test_group_id") + @patch("src.base.kafka.config.CONSUMER_GROUP_ID", "test_group_id") @patch( - "src.base.kafka_handler.KAFKA_BROKERS", + "src.base.kafka.config.KAFKA_BROKERS", [ { "hostname": "127.0.0.1", @@ -271,12 +274,12 @@ def test_del_with_existing_consumer( }, ], ) - @patch("src.base.kafka_handler.Consumer") + @patch("src.base.kafka.consumer.Consumer") @patch( - "src.base.kafka_handler.KafkaConsumeHandler._all_topics_created", + "src.base.kafka.consumer.KafkaConsumeHandler._all_topics_created", return_value=True, ) - @patch("src.base.kafka_handler.AdminClient") + @patch("src.base.kafka.consumer.AdminClient") def test_del_with_existing_consumer( self, mock_admin_client, mock_all_topics_created, mock_consumer ): @@ -295,9 +298,9 @@ def test_del_with_existing_consumer( class TestDict(unittest.TestCase): - @patch("src.base.kafka_handler.CONSUMER_GROUP_ID", "test_group_id") + @patch("src.base.kafka.config.CONSUMER_GROUP_ID", "test_group_id") @patch( - "src.base.kafka_handler.KAFKA_BROKERS", + "src.base.kafka.config.KAFKA_BROKERS", [ { "hostname": "127.0.0.1", @@ -313,12 +316,12 @@ class TestDict(unittest.TestCase): }, ], ) - @patch("src.base.kafka_handler.Consumer") + @patch("src.base.kafka.consumer.Consumer") @patch( - "src.base.kafka_handler.KafkaConsumeHandler._all_topics_created", + "src.base.kafka.consumer.KafkaConsumeHandler._all_topics_created", return_value=True, ) - @patch("src.base.kafka_handler.AdminClient") + @patch("src.base.kafka.consumer.AdminClient") def test_dict(self, mock_admin_client, mock_all_topics_created, mock_consumer): mock_consumer_instance = Mock() mock_consumer.return_value = mock_consumer_instance @@ -328,9 +331,9 @@ def test_dict(self, mock_admin_client, mock_all_topics_created, mock_consumer): class TestConsumeAsObject(unittest.TestCase): - @patch("src.base.kafka_handler.CONSUMER_GROUP_ID", "test_group_id") + @patch("src.base.kafka.config.CONSUMER_GROUP_ID", "test_group_id") @patch( - "src.base.kafka_handler.KAFKA_BROKERS", + "src.base.kafka.config.KAFKA_BROKERS", [ { "hostname": "127.0.0.1", @@ -347,17 +350,17 @@ class TestConsumeAsObject(unittest.TestCase): ], ) @patch( - "src.base.kafka_handler.KafkaConsumeHandler._all_topics_created", + "src.base.kafka.consumer.KafkaConsumeHandler._all_topics_created", return_value=True, ) - @patch("src.base.kafka_handler.AdminClient") - @patch("src.base.kafka_handler.Consumer") + @patch("src.base.kafka.consumer.AdminClient") + @patch("src.base.kafka.consumer.Consumer") def setUp(self, mock_consumer, mock_admin_client, mock_all_topics_created): self.sut = ExactlyOnceKafkaConsumeHandler(topics="test_topic") def test_consume_as_object_no_key_no_value(self): with patch( - "src.base.kafka_handler.ExactlyOnceKafkaConsumeHandler.consume" + "src.base.kafka.consumer.ExactlyOnceKafkaConsumeHandler.consume" ) as mock_consume: mock_consume.return_value = [None, None, None] @@ -380,7 +383,7 @@ def test_consume_as_object_valid_data(self): topic = "test_topic" with patch( - "src.base.kafka_handler.ExactlyOnceKafkaConsumeHandler.consume" + "src.base.kafka.consumer.ExactlyOnceKafkaConsumeHandler.consume" ) as mock_consume: mock_consume.return_value = [key, value, topic] @@ -407,7 +410,7 @@ def test_consume_as_object_valid_data_with_inner_strings(self): topic = "test_topic" with patch( - "src.base.kafka_handler.ExactlyOnceKafkaConsumeHandler.consume" + "src.base.kafka.consumer.ExactlyOnceKafkaConsumeHandler.consume" ) as mock_consume: mock_consume.return_value = [key, value, topic] @@ -436,7 +439,7 @@ def test_consume_as_object_valid_data_with_nested_request_windows(self): topic = "test_topic" with patch( - "src.base.kafka_handler.ExactlyOnceKafkaConsumeHandler.consume" + "src.base.kafka.consumer.ExactlyOnceKafkaConsumeHandler.consume" ) as mock_consume: mock_consume.return_value = [key, value, topic] @@ -454,14 +457,14 @@ def test_consume_as_object_invalid_data(self): topic = "test_topic" with patch( - "src.base.kafka_handler.ExactlyOnceKafkaConsumeHandler.consume" + "src.base.kafka.consumer.ExactlyOnceKafkaConsumeHandler.consume" ) as mock_consume: mock_consume.return_value = [key, value, topic] with self.assertRaises(ValueError): self.sut.consume_as_object() - @patch("src.base.kafka_handler.marshmallow_dataclass.class_schema") + @patch("src.base.kafka.serialization.marshmallow_dataclass.class_schema") def test_consume_as_object_invalid_batch(self, mock_schema): key = "valid_key" value = json.dumps({"data": [{"field1": "value1", "field2": "value2"}]}) @@ -473,7 +476,7 @@ def test_consume_as_object_invalid_batch(self, mock_schema): mock_schema_instance.load.return_value = None with patch( - "src.base.kafka_handler.ExactlyOnceKafkaConsumeHandler.consume" + "src.base.kafka.consumer.ExactlyOnceKafkaConsumeHandler.consume" ) as mock_consume: mock_consume.return_value = [key, value, topic] diff --git a/tests/kafka/test_exactly_once_kafka_produce_handler.py b/tests/kafka/test_exactly_once_kafka_produce_handler.py index 3a30988c..8631b099 100644 --- a/tests/kafka/test_exactly_once_kafka_produce_handler.py +++ b/tests/kafka/test_exactly_once_kafka_produce_handler.py @@ -3,13 +3,13 @@ from confluent_kafka import KafkaException -from src.base.kafka_handler import ExactlyOnceKafkaProduceHandler +from src.base.kafka import ExactlyOnceKafkaProduceHandler class TestInit(unittest.TestCase): - @patch("src.base.kafka_handler.HOSTNAME", "test_transactional_id") + @patch("src.base.kafka.config.HOSTNAME", "test_transactional_id") @patch( - "src.base.kafka_handler.KAFKA_BROKERS", + "src.base.kafka.config.KAFKA_BROKERS", [ { "hostname": "127.0.0.1", @@ -25,8 +25,8 @@ class TestInit(unittest.TestCase): }, ], ) - @patch("src.base.kafka_handler.uuid") - @patch("src.base.kafka_handler.Producer") + @patch("src.base.kafka.producer.uuid") + @patch("src.base.kafka.producer.Producer") def test_init(self, mock_producer, mock_uuid): mock_producer_instance = MagicMock() mock_producer.return_value = mock_producer_instance @@ -47,11 +47,11 @@ def test_init(self, mock_producer, mock_uuid): mock_producer_instance.init_transactions.assert_called_once() @patch("src.base.retry.time.sleep", return_value=None) - @patch("src.base.kafka_handler.HOSTNAME", "default_tid") - @patch("src.base.kafka_handler.uuid") - @patch("src.base.kafka_handler.logger") + @patch("src.base.kafka.config.HOSTNAME", "default_tid") + @patch("src.base.kafka.producer.uuid") + @patch("src.base.kafka.producer.logger") @patch( - "src.base.kafka_handler.KAFKA_BROKERS", + "src.base.kafka.config.KAFKA_BROKERS", [ { "hostname": "127.0.0.1", @@ -67,7 +67,7 @@ def test_init(self, mock_producer, mock_uuid): }, ], ) - @patch("src.base.kafka_handler.Producer") + @patch("src.base.kafka.producer.Producer") def test_init_retries_until_transactions_initialize( self, mock_producer, mock_logger, mock_uuid, mock_sleep ): @@ -97,7 +97,7 @@ def test_init_retries_until_transactions_initialize( class TestSend(unittest.TestCase): @patch( - "src.base.kafka_handler.KAFKA_BROKERS", + "src.base.kafka.config.KAFKA_BROKERS", [ { "hostname": "127.0.0.1", @@ -113,11 +113,11 @@ class TestSend(unittest.TestCase): }, ], ) - @patch("src.base.kafka_handler.Producer") + @patch("src.base.kafka.producer.Producer") @patch( - "src.base.kafka_handler.ExactlyOnceKafkaProduceHandler.commit_transaction_with_retry" + "src.base.kafka.producer.ExactlyOnceKafkaProduceHandler.commit_transaction_with_retry" ) - @patch("src.base.kafka_handler.kafka_delivery_report") + @patch("src.base.kafka.producer.kafka_delivery_report") def test_send_with_data( self, mock_kafka_delivery_report, @@ -140,7 +140,7 @@ def test_send_with_data( mock_producer_instance.begin_transaction.assert_called_once() @patch( - "src.base.kafka_handler.KAFKA_BROKERS", + "src.base.kafka.config.KAFKA_BROKERS", [ { "hostname": "127.0.0.1", @@ -156,7 +156,7 @@ def test_send_with_data( }, ], ) - @patch("src.base.kafka_handler.Producer") + @patch("src.base.kafka.producer.Producer") def test_send_with_empty_data_string(self, mock_producer): sut = ExactlyOnceKafkaProduceHandler() sut.produce("test_topic", "", None) @@ -165,9 +165,9 @@ def test_send_with_empty_data_string(self, mock_producer): mock_producer.produce.assert_not_called() mock_producer.commit_transaction_with_retry.assert_not_called() - @patch("src.base.kafka_handler.logger") + @patch("src.base.kafka.producer.logger") @patch( - "src.base.kafka_handler.KAFKA_BROKERS", + "src.base.kafka.config.KAFKA_BROKERS", [ { "hostname": "127.0.0.1", @@ -183,10 +183,10 @@ def test_send_with_empty_data_string(self, mock_producer): }, ], ) - @patch("src.base.kafka_handler.Producer") - @patch("src.base.kafka_handler.kafka_delivery_report") + @patch("src.base.kafka.producer.Producer") + @patch("src.base.kafka.producer.kafka_delivery_report") @patch( - "src.base.kafka_handler.ExactlyOnceKafkaProduceHandler.commit_transaction_with_retry" + "src.base.kafka.producer.ExactlyOnceKafkaProduceHandler.commit_transaction_with_retry" ) def test_send_fail( self, @@ -218,7 +218,7 @@ def test_send_fail( class TestCommitTransactionWithRetry(unittest.TestCase): # def test_commit_transaction_with_retry(self): @patch( - "src.base.kafka_handler.KAFKA_BROKERS", + "src.base.kafka.config.KAFKA_BROKERS", [ { "hostname": "127.0.0.1", @@ -234,7 +234,7 @@ class TestCommitTransactionWithRetry(unittest.TestCase): }, ], ) - @patch("src.base.kafka_handler.Producer") + @patch("src.base.kafka.producer.Producer") @patch("time.sleep", return_value=None) def test_commit_successful(self, mock_sleep, mock_producer): mock_producer_instance = MagicMock() @@ -248,7 +248,7 @@ def test_commit_successful(self, mock_sleep, mock_producer): mock_sleep.assert_not_called() @patch( - "src.base.kafka_handler.KAFKA_BROKERS", + "src.base.kafka.config.KAFKA_BROKERS", [ { "hostname": "127.0.0.1", @@ -264,7 +264,7 @@ def test_commit_successful(self, mock_sleep, mock_producer): }, ], ) - @patch("src.base.kafka_handler.Producer") + @patch("src.base.kafka.producer.Producer") @patch("time.sleep", return_value=None) def test_commit_retries_then_successful(self, mock_sleep, mock_producer): mock_producer_instance = MagicMock() @@ -282,9 +282,9 @@ def test_commit_retries_then_successful(self, mock_sleep, mock_producer): self.assertEqual(mock_producer_instance.commit_transaction.call_count, 2) mock_sleep.assert_called_once_with(1.0) - @patch("src.base.kafka_handler.logger") + @patch("src.base.kafka.producer.logger") @patch( - "src.base.kafka_handler.KAFKA_BROKERS", + "src.base.kafka.config.KAFKA_BROKERS", [ { "hostname": "127.0.0.1", @@ -300,7 +300,7 @@ def test_commit_retries_then_successful(self, mock_sleep, mock_producer): }, ], ) - @patch("src.base.kafka_handler.Producer") + @patch("src.base.kafka.producer.Producer") @patch("time.sleep", return_value=None) def test_commit_retries_and_fails(self, mock_sleep, mock_producer, mock_logger): mock_producer_instance = MagicMock() @@ -320,7 +320,7 @@ def test_commit_retries_and_fails(self, mock_sleep, mock_producer, mock_logger): self.assertEqual(mock_sleep.call_count, 3) @patch( - "src.base.kafka_handler.KAFKA_BROKERS", + "src.base.kafka.config.KAFKA_BROKERS", [ { "hostname": "127.0.0.1", @@ -336,7 +336,7 @@ def test_commit_retries_and_fails(self, mock_sleep, mock_producer, mock_logger): }, ], ) - @patch("src.base.kafka_handler.Producer") + @patch("src.base.kafka.producer.Producer") @patch("time.sleep", return_value=None) def test_commit_fails_with_other_exception(self, mock_sleep, mock_producer): mock_producer_instance = MagicMock() @@ -356,7 +356,7 @@ def test_commit_fails_with_other_exception(self, mock_sleep, mock_producer): class TestDel(unittest.TestCase): @patch( - "src.base.kafka_handler.KAFKA_BROKERS", + "src.base.kafka.config.KAFKA_BROKERS", [ { "hostname": "127.0.0.1", @@ -372,7 +372,7 @@ class TestDel(unittest.TestCase): }, ], ) - @patch("src.base.kafka_handler.Producer") + @patch("src.base.kafka.producer.Producer") def test_del(self, mock_producer): mock_producer_instance = MagicMock() mock_producer.return_value = mock_producer_instance diff --git a/tests/kafka/test_kafka_consume_handler.py b/tests/kafka/test_kafka_consume_handler.py index 039e4306..d904b061 100644 --- a/tests/kafka/test_kafka_consume_handler.py +++ b/tests/kafka/test_kafka_consume_handler.py @@ -2,14 +2,17 @@ import unittest from unittest.mock import patch, MagicMock -from src.base.kafka_handler import ( +from src.base.kafka.config import KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS +from src.base.kafka import ( build_consumer_group_id, ensure_topics, KafkaConsumeHandler, KafkaMessageFetchException, +) +from src.base.kafka.topics import ( _desired_topic_partitions, - _topic_replication_factor, _topic_config, + _topic_replication_factor, ) @@ -26,14 +29,14 @@ def _metadata(partitions_by_topic: dict[str, int]): class TestConsumerGroupId(unittest.TestCase): - @patch("src.base.kafka_handler.CONSUMER_GROUP_ID", "test_group_id") + @patch("src.base.kafka.config.CONSUMER_GROUP_ID", "test_group_id") def test_build_consumer_group_id_for_single_topic(self): self.assertEqual( "test_group_id.test_topic", build_consumer_group_id("test_topic"), ) - @patch("src.base.kafka_handler.CONSUMER_GROUP_ID", "test_group_id") + @patch("src.base.kafka.config.CONSUMER_GROUP_ID", "test_group_id") def test_build_consumer_group_id_for_multiple_topics(self): self.assertEqual( "test_group_id.test_topic_1__test_topic_2", @@ -42,7 +45,7 @@ def test_build_consumer_group_id_for_multiple_topics(self): class TestTopicReconciliation(unittest.TestCase): - @patch("src.base.kafka_handler.NewTopic") + @patch("src.base.kafka.topics.NewTopic") def test_missing_topic_is_created_with_target_partitions(self, mock_new_topic): admin_client = MagicMock() admin_client.list_topics.side_effect = [ @@ -67,7 +70,7 @@ def test_missing_topic_is_created_with_target_partitions(self, mock_new_topic): admin_client.create_topics.assert_called_once_with([("test_topic", 4, 2)]) admin_client.create_partitions.assert_not_called() - @patch("src.base.kafka_handler.NewPartitions") + @patch("src.base.kafka.topics.NewPartitions") def test_existing_topic_with_too_few_partitions_is_expanded( self, mock_new_partitions ): @@ -144,25 +147,25 @@ def test_auto_expand_can_be_disabled(self): admin_client.create_topics.assert_not_called() admin_client.create_partitions.assert_not_called() - @patch("src.base.kafka_handler.NUMBER_OF_INSTANCES", 6) - @patch("src.base.kafka_handler.KAFKA_TOPIC_DEFAULT_PARTITIONS", 3) + @patch("src.base.kafka.config.NUMBER_OF_INSTANCES", 6) + @patch("src.base.kafka.config.KAFKA_TOPIC_DEFAULT_PARTITIONS", 3) def test_desired_partitions_uses_highest_scale_value(self): self.assertEqual(6, _desired_topic_partitions()) - @patch("src.base.kafka_handler.KAFKA_TOPIC_REPLICATION_FACTOR", 3) + @patch("src.base.kafka.config.KAFKA_TOPIC_REPLICATION_FACTOR", 3) @patch( - "src.base.kafka_handler.KAFKA_BROKERS", + "src.base.kafka.config.KAFKA_BROKERS", [{"hostname": "127.0.0.1", "internal_port": 9999}], ) def test_replication_factor_is_capped_to_configured_brokers(self): self.assertEqual(1, _topic_replication_factor()) @patch( - "src.base.kafka_handler.KAFKA_PIPELINE_TOPIC_PREFIXES", + "src.base.kafka.config.KAFKA_PIPELINE_TOPIC_PREFIXES", {"inspector_to_detector": "pipeline-inspector_to_detector"}, ) @patch( - "src.base.kafka_handler.KAFKA_TOPIC_STAGE_CONFIG", + "src.base.kafka.config.KAFKA_TOPIC_STAGE_CONFIG", {"inspector_to_detector": {"partitions": 7, "replication_factor": 2}}, ) def test_stage_topic_config_is_resolved_from_prefix(self): @@ -174,7 +177,7 @@ def test_stage_topic_config_is_resolved_from_prefix(self): self.assertEqual(2, _topic_replication_factor(topic)) @patch( - "src.base.kafka_handler.KAFKA_TOPIC_EXACT_CONFIG", + "src.base.kafka.config.KAFKA_TOPIC_EXACT_CONFIG", {"hamstring_alerts": {"partitions": 5, "replication_factor": 2}}, ) def test_exact_topic_config_wins(self): @@ -187,9 +190,9 @@ def test_exact_topic_config_wins(self): class TestInit(unittest.TestCase): - @patch("src.base.kafka_handler.CONSUMER_GROUP_ID", "test_group_id") + @patch("src.base.kafka.config.CONSUMER_GROUP_ID", "test_group_id") @patch( - "src.base.kafka_handler.KAFKA_BROKERS", + "src.base.kafka.config.KAFKA_BROKERS", [ { "hostname": "127.0.0.1", @@ -206,11 +209,11 @@ class TestInit(unittest.TestCase): ], ) @patch( - "src.base.kafka_handler.KafkaConsumeHandler._all_topics_created", + "src.base.kafka.consumer.KafkaConsumeHandler._all_topics_created", return_value=True, ) - @patch("src.base.kafka_handler.AdminClient") - @patch("src.base.kafka_handler.Consumer") + @patch("src.base.kafka.consumer.AdminClient") + @patch("src.base.kafka.consumer.Consumer") def test_init_successful( self, mock_consumer, mock_admin_client, mock_all_topics_created ): @@ -224,7 +227,7 @@ def test_init_successful( "enable.auto.commit": False, "auto.offset.reset": "earliest", "enable.partition.eof": True, - "max.poll.interval.ms": 1800000, + "max.poll.interval.ms": KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS, } # Act @@ -236,9 +239,9 @@ def test_init_successful( mock_consumer.assert_called_once_with(expected_conf) mock_consumer_instance.subscribe.assert_called_once() - @patch("src.base.kafka_handler.CONSUMER_GROUP_ID", "test_group_id") + @patch("src.base.kafka.config.CONSUMER_GROUP_ID", "test_group_id") @patch( - "src.base.kafka_handler.KAFKA_BROKERS", + "src.base.kafka.config.KAFKA_BROKERS", [ { "hostname": "127.0.0.1", @@ -255,12 +258,12 @@ def test_init_successful( ], ) @patch( - "src.base.kafka_handler.KafkaConsumeHandler._all_topics_created", + "src.base.kafka.consumer.KafkaConsumeHandler._all_topics_created", side_effect=[False, True], ) @patch("src.base.retry.time.sleep", return_value=None) - @patch("src.base.kafka_handler.AdminClient") - @patch("src.base.kafka_handler.Consumer") + @patch("src.base.kafka.consumer.AdminClient") + @patch("src.base.kafka.consumer.Consumer") def test_init_retries_until_topics_are_visible( self, mock_consumer, mock_admin_client, mock_sleep, mock_all_topics_created ): @@ -274,7 +277,7 @@ def test_init_retries_until_topics_are_visible( "enable.auto.commit": False, "auto.offset.reset": "earliest", "enable.partition.eof": True, - "max.poll.interval.ms": 1800000, + "max.poll.interval.ms": KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS, } # Act @@ -288,9 +291,9 @@ def test_init_retries_until_topics_are_visible( mock_consumer_instance.subscribe.assert_called_once() mock_sleep.assert_called() - @patch("src.base.kafka_handler.CONSUMER_GROUP_ID", "test_group_id") + @patch("src.base.kafka.config.CONSUMER_GROUP_ID", "test_group_id") @patch( - "src.base.kafka_handler.KAFKA_BROKERS", + "src.base.kafka.config.KAFKA_BROKERS", [ { "hostname": "127.0.0.1", @@ -307,11 +310,11 @@ def test_init_retries_until_topics_are_visible( ], ) @patch( - "src.base.kafka_handler.KafkaConsumeHandler._all_topics_created", + "src.base.kafka.consumer.KafkaConsumeHandler._all_topics_created", return_value=True, ) - @patch("src.base.kafka_handler.AdminClient") - @patch("src.base.kafka_handler.Consumer") + @patch("src.base.kafka.consumer.AdminClient") + @patch("src.base.kafka.consumer.Consumer") def test_init_successful_with_list( self, mock_consumer, mock_admin_client, mock_all_topics_created ): @@ -325,7 +328,7 @@ def test_init_successful_with_list( "enable.auto.commit": False, "auto.offset.reset": "earliest", "enable.partition.eof": True, - "max.poll.interval.ms": 1800000, + "max.poll.interval.ms": KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS, } # Act @@ -339,9 +342,9 @@ def test_init_successful_with_list( class TestConsume(unittest.TestCase): - @patch("src.base.kafka_handler.CONSUMER_GROUP_ID", "test_group_id") + @patch("src.base.kafka.config.CONSUMER_GROUP_ID", "test_group_id") @patch( - "src.base.kafka_handler.KAFKA_BROKERS", + "src.base.kafka.config.KAFKA_BROKERS", [ { "hostname": "127.0.0.1", @@ -358,11 +361,11 @@ class TestConsume(unittest.TestCase): ], ) @patch( - "src.base.kafka_handler.KafkaConsumeHandler._all_topics_created", + "src.base.kafka.consumer.KafkaConsumeHandler._all_topics_created", return_value=True, ) - @patch("src.base.kafka_handler.AdminClient") - @patch("src.base.kafka_handler.Consumer") + @patch("src.base.kafka.consumer.AdminClient") + @patch("src.base.kafka.consumer.Consumer") def test_not_implemented( self, mock_consumer, mock_admin_client, mock_all_topics_created ): @@ -375,9 +378,9 @@ def test_not_implemented( class TestConsumeAsJSON(unittest.TestCase): - @patch("src.base.kafka_handler.CONSUMER_GROUP_ID", "test_group_id") + @patch("src.base.kafka.config.CONSUMER_GROUP_ID", "test_group_id") @patch( - "src.base.kafka_handler.KAFKA_BROKERS", + "src.base.kafka.config.KAFKA_BROKERS", [ { "hostname": "127.0.0.1", @@ -394,17 +397,17 @@ class TestConsumeAsJSON(unittest.TestCase): ], ) @patch( - "src.base.kafka_handler.KafkaConsumeHandler._all_topics_created", + "src.base.kafka.consumer.KafkaConsumeHandler._all_topics_created", return_value=True, ) - @patch("src.base.kafka_handler.AdminClient") - @patch("src.base.kafka_handler.Consumer") + @patch("src.base.kafka.consumer.AdminClient") + @patch("src.base.kafka.consumer.Consumer") def setUp(self, mock_consumer, mock_admin_client, mock_all_topics_created): self.sut = KafkaConsumeHandler(topics="test_topic") def test_successful(self): with patch( - "src.base.kafka_handler.KafkaConsumeHandler.consume" + "src.base.kafka.consumer.KafkaConsumeHandler.consume" ) as mock_consume: # Arrange mock_consume.return_value = [ @@ -421,7 +424,7 @@ def test_successful(self): def test_wrong_data_format(self): with patch( - "src.base.kafka_handler.KafkaConsumeHandler.consume" + "src.base.kafka.consumer.KafkaConsumeHandler.consume" ) as mock_consume: # Arrange mock_consume.return_value = ["test_key", "wrong_format", "test_topic"] @@ -432,7 +435,7 @@ def test_wrong_data_format(self): def test_wrong_data_format_list(self): with patch( - "src.base.kafka_handler.KafkaConsumeHandler.consume" + "src.base.kafka.consumer.KafkaConsumeHandler.consume" ) as mock_consume: # Arrange mock_consume.return_value = [ @@ -447,7 +450,7 @@ def test_wrong_data_format_list(self): def test_kafka_message_fetch_exception(self): with patch( - "src.base.kafka_handler.KafkaConsumeHandler.consume", + "src.base.kafka.consumer.KafkaConsumeHandler.consume", side_effect=KafkaMessageFetchException, ): # Act and Assert @@ -456,7 +459,7 @@ def test_kafka_message_fetch_exception(self): def test_keyboard_interrupt(self): with patch( - "src.base.kafka_handler.KafkaConsumeHandler.consume", + "src.base.kafka.consumer.KafkaConsumeHandler.consume", side_effect=KeyboardInterrupt, ): # Act and Assert @@ -465,7 +468,7 @@ def test_keyboard_interrupt(self): def test_kafka_message_else(self): with patch( - "src.base.kafka_handler.KafkaConsumeHandler.consume" + "src.base.kafka.consumer.KafkaConsumeHandler.consume" ) as mock_consume: # Arrange mock_consume.return_value = [None, None, "test_topic"] @@ -478,9 +481,9 @@ def test_kafka_message_else(self): class TestAllTopicsCreated(unittest.TestCase): - @patch("src.base.kafka_handler.CONSUMER_GROUP_ID", "test_group_id") + @patch("src.base.kafka.config.CONSUMER_GROUP_ID", "test_group_id") @patch( - "src.base.kafka_handler.KAFKA_BROKERS", + "src.base.kafka.config.KAFKA_BROKERS", [ { "hostname": "127.0.0.1", @@ -497,15 +500,15 @@ class TestAllTopicsCreated(unittest.TestCase): ], ) @patch( - "src.base.kafka_handler.KafkaConsumeHandler._all_topics_created", + "src.base.kafka.consumer.KafkaConsumeHandler._all_topics_created", return_value=True, ) - @patch("src.base.kafka_handler.AdminClient") - @patch("src.base.kafka_handler.Consumer") + @patch("src.base.kafka.consumer.AdminClient") + @patch("src.base.kafka.consumer.Consumer") def setUp(self, mock_consumer, mock_admin_client, mock_all_topics_created): self.sut = KafkaConsumeHandler(topics=["test_topic", "another_topic"]) - @patch("src.base.kafka_handler.Consumer") + @patch("src.base.kafka.consumer.Consumer") def test_with_all_created(self, mock_consumer): # Arrange self.sut.consumer = MagicMock() @@ -520,8 +523,8 @@ def test_with_all_created(self, mock_consumer): ) ) - @patch("src.base.kafka_handler.time.sleep") - @patch("src.base.kafka_handler.Consumer") + @patch("src.base.kafka.consumer.time.sleep") + @patch("src.base.kafka.consumer.Consumer") def test_with_none_created(self, mock_consumer, mock_sleep): # Arrange mock_topics = MagicMock() @@ -535,8 +538,8 @@ def test_with_none_created(self, mock_consumer, mock_sleep): self.sut._all_topics_created(topics=["test_topic", "another_topic"]) ) - @patch("src.base.kafka_handler.time.sleep") - @patch("src.base.kafka_handler.Consumer") + @patch("src.base.kafka.consumer.time.sleep") + @patch("src.base.kafka.consumer.Consumer") def test_with_too_few_partitions(self, mock_consumer, mock_sleep): # Arrange self.sut.consumer = MagicMock() diff --git a/tests/kafka/test_kafka_eos_batch.py b/tests/kafka/test_kafka_eos_batch.py new file mode 100644 index 00000000..49c54a26 --- /dev/null +++ b/tests/kafka/test_kafka_eos_batch.py @@ -0,0 +1,155 @@ +import unittest +from unittest.mock import MagicMock, call, patch + +from src.base.kafka import ( + ConsumedKafkaMessage, + ExactlyOnceKafkaConsumeHandler, + ExactlyOnceKafkaProduceHandler, +) + + +class TestTransactionalBatch(unittest.TestCase): + @patch("src.base.kafka.producer.Producer") + def test_outputs_and_source_offsets_are_committed_in_one_transaction( + self, mock_producer + ): + producer = mock_producer.return_value + consumer = MagicMock() + offsets = [MagicMock()] + group_metadata = MagicMock() + consumer.offsets_for.return_value = offsets + consumer.group_metadata.return_value = group_metadata + source_messages = [ + ConsumedKafkaMessage("source-key", "source-value", "source", 2, 7) + ] + handler = ExactlyOnceKafkaProduceHandler() + + with handler.transaction_batch(consumer, source_messages): + handler.produce("output", "first", key="routing-key") + handler.produce("output", "second", key="routing-key") + + producer.begin_transaction.assert_called_once_with() + self.assertEqual( + [ + call( + topic="output", + key="routing-key", + value="first", + callback=unittest.mock.ANY, + ), + call( + topic="output", + key="routing-key", + value="second", + callback=unittest.mock.ANY, + ), + ], + producer.produce.call_args_list, + ) + producer.send_offsets_to_transaction.assert_called_once_with( + offsets, group_metadata + ) + producer.commit_transaction.assert_called_once_with() + + @patch("src.base.kafka.producer.Producer") + def test_source_offsets_are_committed_when_processing_has_no_output( + self, mock_producer + ): + producer = mock_producer.return_value + consumer = MagicMock() + source_messages = [ + ConsumedKafkaMessage("source-key", "source-value", "source", 0, 4) + ] + handler = ExactlyOnceKafkaProduceHandler() + + with handler.transaction_batch(consumer, source_messages): + pass + + producer.begin_transaction.assert_called_once_with() + producer.produce.assert_not_called() + producer.send_offsets_to_transaction.assert_called_once_with( + consumer.offsets_for.return_value, + consumer.group_metadata.return_value, + ) + producer.commit_transaction.assert_called_once_with() + + +class TestBatchConsumption(unittest.TestCase): + def setUp(self): + self.handler = object.__new__(ExactlyOnceKafkaConsumeHandler) + self.handler.consumer = MagicMock() + self.handler._last_consumed_message = None + + @staticmethod + def _message(topic, partition, offset, key, value): + message = MagicMock() + message.error.return_value = None + message.topic.return_value = topic + message.partition.return_value = partition + message.offset.return_value = offset + message.key.return_value = key.encode() + message.value.return_value = value.encode() + return message + + def test_consume_batch_fetches_multiple_records_with_one_consumer_call(self): + first = self._message("source", 0, 3, "key-1", "value-1") + second = self._message("source", 1, 8, "key-2", "value-2") + self.handler.consumer.consume.return_value = [first, second] + + records = self.handler.consume_batch(max_messages=2, timeout_ms=10) + + self.handler.consumer.consume.assert_called_once() + self.assertEqual( + [ + ("key-1", "value-1", "source", 0, 3), + ("key-2", "value-2", "source", 1, 8), + ], + [ + ( + record.key, + record.value, + record.topic, + record.partition, + record.offset, + ) + for record in records + ], + ) + + def test_offsets_use_next_offset_and_highest_record_per_partition(self): + records = [ + ConsumedKafkaMessage(None, "one", "source", 0, 3), + ConsumedKafkaMessage(None, "two", "source", 1, 8), + ConsumedKafkaMessage(None, "three", "source", 0, 7), + ] + + offsets = self.handler.offsets_for(records) + + self.assertEqual( + {("source", 0, 8), ("source", 1, 9)}, + {(item.topic, item.partition, item.offset) for item in offsets}, + ) + + def test_commit_batch_commits_all_partition_offsets_synchronously(self): + records = [ + ConsumedKafkaMessage(None, "one", "source", 0, 3), + ConsumedKafkaMessage(None, "two", "source", 1, 8), + ConsumedKafkaMessage(None, "three", "source", 0, 7), + ] + + self.handler.commit(records) + + self.handler.consumer.commit.assert_called_once() + commit_kwargs = self.handler.consumer.commit.call_args.kwargs + self.assertFalse(commit_kwargs["asynchronous"]) + self.assertEqual( + {("source", 0, 8), ("source", 1, 9)}, + { + (item.topic, item.partition, item.offset) + for item in commit_kwargs["offsets"] + }, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/kafka/test_kafka_handler.py b/tests/kafka/test_kafka_handler.py index db696381..bb8a3b11 100644 --- a/tests/kafka/test_kafka_handler.py +++ b/tests/kafka/test_kafka_handler.py @@ -1,6 +1,6 @@ import unittest -from src.base.kafka_handler import KafkaHandler +from src.base.kafka import KafkaHandler class TestInit(unittest.TestCase): diff --git a/tests/kafka/test_kafka_produce_handler.py b/tests/kafka/test_kafka_produce_handler.py index 859bfabf..a9e4d10b 100644 --- a/tests/kafka/test_kafka_produce_handler.py +++ b/tests/kafka/test_kafka_produce_handler.py @@ -1,7 +1,7 @@ import unittest from unittest.mock import patch, Mock -from src.base.kafka_handler import KafkaProduceHandler +from src.base.kafka import KafkaProduceHandler class TestInit(unittest.TestCase): @@ -10,7 +10,7 @@ def test_successful(self): conf = "test_conf" # Act - with patch("src.base.kafka_handler.Producer") as mock_producer: + with patch("src.base.kafka.producer.Producer") as mock_producer: mock_producer_instance = Mock() mock_producer.return_value = mock_producer_instance @@ -23,7 +23,7 @@ def test_successful(self): class TestProduce(unittest.TestCase): - @patch("src.base.kafka_handler.Producer") + @patch("src.base.kafka.producer.Producer") def test_not_implemented(self, mock_producer): # Arrange sut = KafkaProduceHandler("test_conf") @@ -34,7 +34,7 @@ def test_not_implemented(self, mock_producer): class TestDel(unittest.TestCase): - @patch("src.base.kafka_handler.Producer") + @patch("src.base.kafka.producer.Producer") def test_not_implemented(self, mock_producer): # Arrange mock_producer_instance = Mock() diff --git a/tests/kafka/test_simple_kafka_consume_handler.py b/tests/kafka/test_simple_kafka_consume_handler.py index 0b060085..311a64de 100644 --- a/tests/kafka/test_simple_kafka_consume_handler.py +++ b/tests/kafka/test_simple_kafka_consume_handler.py @@ -1,13 +1,14 @@ import unittest from unittest.mock import patch, Mock from confluent_kafka import KafkaError -from src.base.kafka_handler import SimpleKafkaConsumeHandler +from src.base.kafka.config import KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS +from src.base.kafka import SimpleKafkaConsumeHandler class TestInit(unittest.TestCase): - @patch("src.base.kafka_handler.CONSUMER_GROUP_ID", "test_group_id") + @patch("src.base.kafka.config.CONSUMER_GROUP_ID", "test_group_id") @patch( - "src.base.kafka_handler.KAFKA_BROKERS", + "src.base.kafka.config.KAFKA_BROKERS", [ { "hostname": "127.0.0.1", @@ -24,11 +25,11 @@ class TestInit(unittest.TestCase): ], ) @patch( - "src.base.kafka_handler.KafkaConsumeHandler._all_topics_created", + "src.base.kafka.consumer.KafkaConsumeHandler._all_topics_created", return_value=True, ) - @patch("src.base.kafka_handler.AdminClient") - @patch("src.base.kafka_handler.Consumer") + @patch("src.base.kafka.consumer.AdminClient") + @patch("src.base.kafka.consumer.Consumer") def test_init_successful( self, mock_consumer, mock_admin_client, mock_all_topics_created ): @@ -42,7 +43,7 @@ def test_init_successful( "enable.auto.commit": False, "auto.offset.reset": "earliest", "enable.partition.eof": True, - "max.poll.interval.ms": 1800000, + "max.poll.interval.ms": KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS, } # Act @@ -56,9 +57,9 @@ def test_init_successful( class TestConsume(unittest.TestCase): - @patch("src.base.kafka_handler.CONSUMER_GROUP_ID", "test_group_id") + @patch("src.base.kafka.config.CONSUMER_GROUP_ID", "test_group_id") @patch( - "src.base.kafka_handler.KAFKA_BROKERS", + "src.base.kafka.config.KAFKA_BROKERS", [ { "hostname": "127.0.0.1", @@ -75,11 +76,11 @@ class TestConsume(unittest.TestCase): ], ) @patch( - "src.base.kafka_handler.KafkaConsumeHandler._all_topics_created", + "src.base.kafka.consumer.KafkaConsumeHandler._all_topics_created", return_value=True, ) - @patch("src.base.kafka_handler.AdminClient") - @patch("src.base.kafka_handler.Consumer") + @patch("src.base.kafka.consumer.AdminClient") + @patch("src.base.kafka.consumer.Consumer") def setUp(self, mock_consumer, mock_admin_client, mock_all_topics_created): self.mock_consumer = mock_consumer self.topics = ["test_topic_1", "test_topic_2"] diff --git a/tests/kafka/test_simple_kafka_produce_handler.py b/tests/kafka/test_simple_kafka_produce_handler.py index 7919d722..afe086b0 100644 --- a/tests/kafka/test_simple_kafka_produce_handler.py +++ b/tests/kafka/test_simple_kafka_produce_handler.py @@ -1,14 +1,17 @@ import unittest -from unittest.mock import ANY, patch, Mock +from unittest.mock import ANY, Mock, call, patch from confluent_kafka import KafkaError -from src.base.kafka_handler import SimpleKafkaProduceHandler +from src.base.kafka import ( + BufferedKafkaProduceHandler, + SimpleKafkaProduceHandler, +) class TestInit(unittest.TestCase): @patch( - "src.base.kafka_handler.KAFKA_BROKERS", + "src.base.kafka.config.KAFKA_BROKERS", [ { "hostname": "127.0.0.1", @@ -34,7 +37,7 @@ def test_successful(self): } # Act - with patch("src.base.kafka_handler.Producer") as mock_producer: + with patch("src.base.kafka.producer.Producer") as mock_producer: mock_producer_instance = Mock() mock_producer.return_value = mock_producer_instance @@ -48,7 +51,7 @@ def test_successful(self): class TestProduce(unittest.TestCase): def test_with_data(self): - with patch("src.base.kafka_handler.Producer") as mock_producer: + with patch("src.base.kafka.producer.Producer") as mock_producer: # Arrange mock_producer_instance = Mock() mock_producer.return_value = mock_producer_instance @@ -69,7 +72,7 @@ def test_with_data(self): @patch("src.base.retry.time.sleep", return_value=None) def test_with_data_recreates_producer_after_transient_failure(self, mock_sleep): - with patch("src.base.kafka_handler.Producer") as mock_producer: + with patch("src.base.kafka.producer.Producer") as mock_producer: first_producer = Mock() second_producer = Mock() first_producer.flush.side_effect = BufferError("queue full") @@ -90,7 +93,7 @@ def test_with_data_recreates_producer_after_transient_failure(self, mock_sleep): @patch("src.base.retry.time.sleep", return_value=None) def test_with_data_retries_delivery_callback_error(self, mock_sleep): - with patch("src.base.kafka_handler.Producer") as mock_producer: + with patch("src.base.kafka.producer.Producer") as mock_producer: first_producer = Mock() second_producer = Mock() delivery_error = KafkaError(KafkaError._ALL_BROKERS_DOWN) @@ -115,7 +118,7 @@ def fail_delivery(**kwargs): mock_sleep.assert_called() def test_without_data(self): - with patch("src.base.kafka_handler.Producer") as mock_producer: + with patch("src.base.kafka.producer.Producer") as mock_producer: # Arrange mock_producer_instance = Mock() mock_producer.return_value = mock_producer_instance @@ -130,5 +133,37 @@ def test_without_data(self): mock_producer_instance.produce.assert_not_called() +class TestBufferedProduce(unittest.TestCase): + def test_queues_data_without_flushing(self): + with patch("src.base.kafka.producer.Producer") as mock_producer: + producer = Mock() + mock_producer.return_value = producer + sut = BufferedKafkaProduceHandler() + + sut.produce("test_topic", "test_data") + + producer.flush.assert_not_called() + producer.poll.assert_called_once_with(0) + producer.produce.assert_called_once_with( + topic="test_topic", + key=None, + value="test_data", + callback=ANY, + ) + + def test_waits_for_queue_space_without_recreating_producer(self): + with patch("src.base.kafka.producer.Producer") as mock_producer: + producer = Mock() + producer.produce.side_effect = [BufferError("queue full"), None] + mock_producer.return_value = producer + sut = BufferedKafkaProduceHandler() + + sut.produce("test_topic", "test_data") + + mock_producer.assert_called_once() + self.assertEqual(2, producer.produce.call_count) + producer.poll.assert_has_calls([call(0), call(0.1)]) + + if __name__ == "__main__": unittest.main() diff --git a/tests/logcollector/test_collector.py b/tests/logcollector/test_collector.py index afba936e..3742c649 100644 --- a/tests/logcollector/test_collector.py +++ b/tests/logcollector/test_collector.py @@ -5,6 +5,7 @@ import uuid from unittest.mock import MagicMock, patch, AsyncMock, Mock +from src.base.kafka import ConsumedKafkaMessage from src.logcollector.collector import LogCollector, main from src.base.utils import setup_config @@ -110,8 +111,9 @@ def test_handle_kafka_inputs( mock_logline_handler, ): mock_consume_handler = MagicMock() - mock_consume_handler.consume.side_effect = [ - ("key1", "value1", "topic1"), + source_message = ConsumedKafkaMessage("key1", "value1", "topic1", 0, 1) + mock_consume_handler.consume_batch.side_effect = [ + [source_message], _StopFetching(), ] mock_kafka_consume.return_value = mock_consume_handler @@ -136,7 +138,11 @@ def fetch_wrapper(*args, **kwargs): self.sut.fetch() mock_send.assert_called_once() - mock_consume_handler.commit.assert_called_once() + self.sut.batch_handler.flush.assert_called_once_with() + self.sut.batch_handler.kafka_produce_handler.transaction_batch.assert_called_once_with( + mock_consume_handler, [source_message] + ) + mock_consume_handler.commit.assert_not_called() class TestSend(unittest.TestCase): diff --git a/tests/logserver/test_server.py b/tests/logserver/test_server.py index 05d8d4e3..dc631b26 100644 --- a/tests/logserver/test_server.py +++ b/tests/logserver/test_server.py @@ -8,6 +8,7 @@ import aiofiles +from src.base.kafka import ConsumedKafkaMessage from src.logserver.server import LogServer, main LOG_SERVER_IP_ADDR = "192.168.0.1" @@ -133,8 +134,9 @@ async def test_fetch_from_kafka( mock_uuid.return_value = mock_uuid_instance mock_uuid.uuid4.return_value = UUID("bd72ccb4-0ef2-4100-aa22-e787122d6875") mock_consume_handler = MagicMock() - mock_consume_handler.consume.side_effect = [ - ("key1", "value1", "test-topic"), + source_message = ConsumedKafkaMessage("key1", "value1", "test-topic", 0, 1) + mock_consume_handler.consume_batch.side_effect = [ + [source_message], _StopFetching(), ] mock_kafka_consume.return_value = mock_consume_handler @@ -157,7 +159,10 @@ def fetch_wrapper(*args, **kwargs): mock_send.assert_called_once_with( UUID("bd72ccb4-0ef2-4100-aa22-e787122d6875"), "value1" ) - mock_consume_handler.commit.assert_called_once() + self.sut.kafka_produce_handler.transaction_batch.assert_called_once_with( + mock_consume_handler, [source_message] + ) + mock_consume_handler.commit.assert_not_called() # class TestFetchFromFile(unittest.IsolatedAsyncioTestCase): diff --git a/tests/miscellaneous/test_monitoring_agent.py b/tests/miscellaneous/test_monitoring_agent.py index c989b7c6..2765a2d6 100644 --- a/tests/miscellaneous/test_monitoring_agent.py +++ b/tests/miscellaneous/test_monitoring_agent.py @@ -2,12 +2,20 @@ import os import unittest import uuid -from unittest.mock import patch, AsyncMock, Mock, MagicMock, call, mock_open +from unittest.mock import patch, Mock, call, mock_open import marshmallow_dataclass from src.base.data_classes.clickhouse_connectors import ServerLogs -from src.monitoring.monitoring_agent import CREATE_TABLES_DIRECTORY, main +from src.base.kafka import ConsumedKafkaMessage +from src.monitoring.monitoring_agent import ( + CREATE_TABLES_DIRECTORY, + MONITORING_CONSUMER_BATCH_SIZE, + MONITORING_CONSUMER_TIMEOUT_MS, + build_monitoring_worker, + main, + start_monitoring_workers, +) from src.monitoring.monitoring_agent import MonitoringAgent, prepare_all_tables @@ -103,11 +111,12 @@ def test_successful(self): self.assertTrue(all(e.startswith("clickhouse_") for e in sut.topics)) self.assertIsNotNone(sut.kafka_consumer) self.assertIsNotNone(sut.batch_sender) + self.assertEqual(set(sut.table_names), set(sut.data_schemas)) mock_simple_kafka_consume_handler.assert_called_once_with(sut.topics) - mock_clickhouse_batch_sender.assert_called_once() + mock_clickhouse_batch_sender.assert_called_once_with(use_timer=False) -class TestStart(unittest.IsolatedAsyncioTestCase): +class TestRun(unittest.TestCase): def setUp(self): with ( patch("src.monitoring.monitoring_agent.SimpleKafkaConsumeHandler"), @@ -115,7 +124,7 @@ def setUp(self): ): self.sut = MonitoringAgent() - async def test_successful(self): + def test_successful(self): # Arrange data_schema = marshmallow_dataclass.class_schema(ServerLogs)() fixed_id = uuid.UUID("35871c8c-ff72-44ad-a9b7-4f02cf92d484") @@ -127,38 +136,33 @@ async def test_successful(self): "message_text": "test_text", } ) - self.sut.kafka_consumer.consume.return_value = ( - "test_key", - value, - "clickhouse_server_logs", + source_record = ConsumedKafkaMessage( + key="test_key", + value=value, + topic="clickhouse_server_logs", + partition=2, + offset=7, ) + source_records = [source_record] with ( patch( - "src.monitoring.monitoring_agent.asyncio.get_running_loop" - ) as mock_get_running_loop, - patch( - "src.monitoring.monitoring_agent.create_pipeline_executor" - ) as mock_create_pipeline_executor, + "src.monitoring.monitoring_agent.marshmallow_dataclass.class_schema" + ) as mock_class_schema, ): - mock_executor = MagicMock() - mock_create_pipeline_executor.return_value = mock_executor - mock_loop = AsyncMock() - mock_get_running_loop.return_value = mock_loop - - mock_loop.run_in_executor.side_effect = [ - ("test_key", value, "clickhouse_server_logs"), + self.sut.kafka_consumer.consume_batch.side_effect = [ + source_records, KeyboardInterrupt(), ] # Act - await self.sut.start() + self.sut.run() # Assert - mock_loop.run_in_executor.assert_any_await( - mock_executor, self.sut.kafka_consumer.consume + self.sut.kafka_consumer.consume_batch.assert_called_with( + MONITORING_CONSUMER_BATCH_SIZE, + MONITORING_CONSUMER_TIMEOUT_MS, ) - mock_executor.shutdown.assert_called_once_with(wait=False, cancel_futures=True) self.sut.batch_sender.add.assert_called_once_with( "server_logs", { @@ -167,22 +171,47 @@ async def test_successful(self): "message_text": "test_text", }, ) + self.assertEqual(2, self.sut.batch_sender.insert_all.call_count) + self.sut.kafka_consumer.commit.assert_called_once_with(source_records) + mock_class_schema.assert_not_called() + + +class TestScaling(unittest.IsolatedAsyncioTestCase): + @patch("src.monitoring.monitoring_agent.start_pipeline_worker_replicas") + async def test_start_uses_configured_worker_replicas(self, mock_start_replicas): + await start_monitoring_workers() + + mock_start_replicas.assert_awaited_once() + call_kwargs = mock_start_replicas.call_args.kwargs + self.assertEqual("monitoring.agent", call_kwargs["module_name"]) + self.assertEqual("run", call_kwargs["target_name"]) + self.assertIs(build_monitoring_worker, call_kwargs["worker_factory"]) + + def test_worker_factory_assigns_worker_id(self): + with ( + patch("src.monitoring.monitoring_agent.SimpleKafkaConsumeHandler"), + patch("src.monitoring.monitoring_agent.ClickHouseBatchSender"), + ): + worker = build_monitoring_worker("p0-t3") + + self.assertEqual("p0-t3", worker.worker_id) class TestMain(unittest.TestCase): - @patch("src.monitoring.monitoring_agent.MonitoringAgent") + @patch( + "src.monitoring.monitoring_agent.start_monitoring_workers", + new_callable=Mock, + ) @patch("asyncio.run") - def test_main(self, mock_asyncio_run, mock_monitoring_agent): - # Arrange - mock_agent_instance = Mock() - mock_monitoring_agent.return_value = mock_agent_instance - + def test_main(self, mock_asyncio_run, mock_start_monitoring_workers): # Act main() # Assert - mock_monitoring_agent.assert_called_once() - mock_asyncio_run.assert_called_once_with(mock_agent_instance.start()) + mock_start_monitoring_workers.assert_called_once_with() + mock_asyncio_run.assert_called_once_with( + mock_start_monitoring_workers.return_value + ) if __name__ == "__main__": diff --git a/tests/miscellaneous/test_retry.py b/tests/miscellaneous/test_retry.py new file mode 100644 index 00000000..3c53eba7 --- /dev/null +++ b/tests/miscellaneous/test_retry.py @@ -0,0 +1,60 @@ +import unittest +from unittest.mock import MagicMock, patch + +from src.base.retry import RetrySettings, load_retry_settings, retry_forever + + +class TestRetrySettings(unittest.TestCase): + def test_settings_are_loaded_from_existing_application_config(self): + config = { + "pipeline": { + "resilience": { + "retry": { + "initial_delay_seconds": 2, + "max_delay_seconds": 12, + "backoff_multiplier": 3, + "jitter_seconds": 0.5, + "log_every_attempts": 7, + } + } + } + } + + settings = load_retry_settings(config) + + self.assertEqual( + RetrySettings( + initial_delay_seconds=2, + max_delay_seconds=12, + backoff_multiplier=3, + jitter_seconds=0.5, + log_every_attempts=7, + ), + settings, + ) + + @patch("src.base.retry.time.sleep") + def test_retry_reuses_preloaded_settings(self, mock_sleep): + operation = MagicMock(side_effect=[RuntimeError("temporary"), "done"]) + settings = RetrySettings( + initial_delay_seconds=0.25, + max_delay_seconds=1, + backoff_multiplier=2, + jitter_seconds=0, + log_every_attempts=1, + ) + + result = retry_forever( + operation, + "test operation", + settings, + retryable=(RuntimeError,), + ) + + self.assertEqual("done", result) + self.assertEqual(2, operation.call_count) + mock_sleep.assert_called_once_with(0.25) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/prefilter/test_prefilter.py b/tests/prefilter/test_prefilter.py index 71206544..dc80b5a2 100644 --- a/tests/prefilter/test_prefilter.py +++ b/tests/prefilter/test_prefilter.py @@ -4,7 +4,7 @@ from unittest.mock import MagicMock, patch, AsyncMock, call from src.base.data_classes.batch import Batch -from src.base.kafka_handler import KafkaMessageFetchException +from src.base.kafka import KafkaMessageFetchException from src.prefilter.prefilter import Prefilter, main @@ -31,6 +31,7 @@ def test_bootstrap_prefiltering_process_one_iteration( mock_produce_handler.return_value = mock_produce_handler_instance mock_consume_handler_instance = MagicMock() mock_consume_handler.return_value = mock_consume_handler_instance + mock_consume_handler_instance.consume_batch.return_value = [MagicMock()] mock_consume_handler_instance.consume_as_object.return_value = ( "127.0.0.0_24", Batch( @@ -120,6 +121,7 @@ def test_bootstrap_prefiltering_process_with_filtering( mock_produce_handler.return_value = mock_produce_handler_instance mock_consume_handler_instance = MagicMock() mock_consume_handler.return_value = mock_consume_handler_instance + mock_consume_handler_instance.consume_batch.return_value = [MagicMock()] mock_consume_handler_instance.consume_as_object.return_value = ( "127.0.0.0_24", Batch( @@ -162,9 +164,10 @@ def mock_send_filtered_data(): sut.bootstrap_prefiltering_process() spy.assert_called_once() - # Verify data was correctly processed - self.assertEqual(sut.unfiltered_data, [test_entry]) - self.assertEqual(sut.filtered_data, [test_entry]) + # The record was processed and per-record state was cleared afterwards. + sut.logline_handler.check_relevance.assert_called_once() + self.assertEqual(sut.unfiltered_data, []) + self.assertEqual(sut.filtered_data, []) self.assertEqual(sut.subnet_id, "127.0.0.0_24") @patch("src.prefilter.prefilter.logger") @@ -189,6 +192,7 @@ def test_bootstrap_prefiltering_process_with_empty_data( mock_produce_handler.return_value = mock_produce_handler_instance mock_consume_handler_instance = MagicMock() mock_consume_handler.return_value = mock_consume_handler_instance + mock_consume_handler_instance.consume_batch.return_value = [MagicMock()] mock_consume_handler_instance.consume_as_object.return_value = ( "127.0.0.0_24", Batch(