1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
use std::{
    io::Write,
    sync::{Arc, Mutex},
};

use bytes::Bytes;
use snafu::ResultExt;
use tokio::sync::oneshot::{Receiver, Sender};
use vector_lib::{finalization::EventFinalizers, request_metadata::RequestMetadata};

use super::{
    aggregation::Aggregator, build_request, DDTracesMetadata, DatadogTracesEndpoint,
    DatadogTracesEndpointConfiguration, RequestBuilderError, StatsPayload,
    BUCKET_DURATION_NANOSECONDS,
};
use crate::{
    http::{BuildRequestSnafu, HttpClient},
    internal_events::DatadogTracesAPMStatsError,
    sinks::util::{Compression, Compressor},
};

/// Flushes cached APM stats buckets to Datadog on a 10 second interval.
/// When the sink signals this thread that it is shutting down, all remaining
/// buckets are flush before the thread exits.
///
/// # arguments
///
/// * `tripwire`                 - Receiver that the sink signals when shutting down.
/// * `client`                   - HttpClient to use in sending the stats payloads.
/// * `compression`              - Compression to use when creating the HTTP requests.
/// * `endpoint_configuration`   - Endpoint configuration to use when creating the HTTP requests.
/// * `aggregator`               - The Aggregator object containing cached stats buckets.
pub async fn flush_apm_stats_thread(
    mut tripwire: Receiver<Sender<()>>,
    client: HttpClient,
    compression: Compression,
    endpoint_configuration: DatadogTracesEndpointConfiguration,
    aggregator: Arc<Mutex<Aggregator>>,
) {
    let sender = ApmStatsSender {
        client,
        compression,
        endpoint_configuration,
        aggregator,
    };

    // flush on the same interval as the stats buckets
    let mut interval =
        tokio::time::interval(std::time::Duration::from_nanos(BUCKET_DURATION_NANOSECONDS));

    debug!("Starting APM stats flushing thread.");

    loop {
        tokio::select! {

        _ = interval.tick() => {
            // flush the oldest bucket from the cache to Datadog
            sender.flush_apm_stats(false).await;
        },
        signal = &mut tripwire =>  match signal {
            // sink has signaled us that the process is shutting down
            Ok(sink_shutdown_ack_sender) => {

                debug!("APM stats flushing thread received exit condition. Flushing remaining stats before exiting.");
                sender.flush_apm_stats(true).await;

                // signal the sink (who tripped the tripwire), that we are done flushing
                _ = sink_shutdown_ack_sender.send(());
                break;
            }
            Err(_) => {
                error!(
                    internal_log_rate_limit = true,
                    message = "Tokio Sender unexpectedly dropped."
                );
                break;
            },
        }
        }
    }
}

struct ApmStatsSender {
    client: HttpClient,
    compression: Compression,
    endpoint_configuration: DatadogTracesEndpointConfiguration,
    aggregator: Arc<Mutex<Aggregator>>,
}

impl ApmStatsSender {
    async fn flush_apm_stats(&self, force: bool) {
        // explicit scope to minimize duration that the Aggregator is locked.
        if let Some((payload, api_key)) = {
            let mut aggregator = self.aggregator.lock().unwrap();
            let client_stats_payloads = aggregator.flush(force);

            if client_stats_payloads.is_empty() {
                // no sense proceeding if no payloads to flush
                None
            } else {
                let payload = StatsPayload {
                    agent_hostname: aggregator.get_agent_hostname(),
                    agent_env: aggregator.get_agent_env(),
                    stats: client_stats_payloads,
                    agent_version: aggregator.get_agent_version(),
                    client_computed: false,
                };

                Some((payload, aggregator.get_api_key()))
            }
        } {
            if let Err(error) = self.compress_and_send(payload, api_key).await {
                emit!(DatadogTracesAPMStatsError { error });
            }
        }
    }

    async fn compress_and_send(
        &self,
        payload: StatsPayload,
        api_key: Arc<str>,
    ) -> Result<(), Box<dyn std::error::Error>> {
        let (metadata, compressed_payload) = self.build_apm_stats_request_data(api_key, payload)?;

        let request_metadata = RequestMetadata::default();
        let trace_api_request = build_request(
            (metadata, request_metadata),
            compressed_payload,
            self.compression,
            &self.endpoint_configuration,
        );

        let http_request = trace_api_request
            .into_http_request()
            .context(BuildRequestSnafu)?;

        self.client.send(http_request).await?;

        Ok(())
    }

    fn build_apm_stats_request_data(
        &self,
        api_key: Arc<str>,
        payload: StatsPayload,
    ) -> Result<(DDTracesMetadata, Bytes), RequestBuilderError> {
        let encoded_payload =
            rmp_serde::to_vec_named(&payload).map_err(|e| RequestBuilderError::FailedToBuild {
                message: "Encoding failed.",
                reason: e.to_string(),
                dropped_events: 0,
            })?;
        let uncompressed_size = encoded_payload.len();
        let metadata = DDTracesMetadata {
            api_key,
            endpoint: DatadogTracesEndpoint::APMStats,
            finalizers: EventFinalizers::default(),
            uncompressed_size,
            content_type: "application/msgpack".to_string(),
        };

        let mut compressor = Compressor::from(self.compression);
        match compressor.write_all(&encoded_payload) {
            Ok(()) => {
                let bytes = compressor.into_inner().freeze();

                Ok((metadata, bytes))
            }
            Err(e) => Err(RequestBuilderError::FailedToBuild {
                message: "Compression failed.",
                reason: e.to_string(),
                dropped_events: 0,
            }),
        }
    }
}