|
@@ -4,6 +4,7 @@ use serde::Serialize;
|
|
|
use serde_json::{Value, json, from_str};
|
|
use serde_json::{Value, json, from_str};
|
|
|
use futures_util::{StreamExt, stream};
|
|
use futures_util::{StreamExt, stream};
|
|
|
use tokio::sync::oneshot;
|
|
use tokio::sync::oneshot;
|
|
|
|
|
+use chrono::Utc;
|
|
|
use std::sync::{Arc, Mutex};
|
|
use std::sync::{Arc, Mutex};
|
|
|
use crate::{
|
|
use crate::{
|
|
|
http_error::HttpError,
|
|
http_error::HttpError,
|
|
@@ -118,7 +119,8 @@ async fn send_message(prompt: Vec<PromptMessage>) -> Result<PromptResponse, Http
|
|
|
"reasoning_effort": "none",
|
|
"reasoning_effort": "none",
|
|
|
"temperature": 0.7,
|
|
"temperature": 0.7,
|
|
|
"stream": true,
|
|
"stream": true,
|
|
|
- "stream_options": {"include_usage": true}
|
|
|
|
|
|
|
+ "stream_options": {"include_usage": true},
|
|
|
|
|
+ "max_tokens": 4096
|
|
|
});
|
|
});
|
|
|
|
|
|
|
|
let response = client
|
|
let response = client
|
|
@@ -166,8 +168,16 @@ async fn send_message(prompt: Vec<PromptMessage>) -> Result<PromptResponse, Http
|
|
|
))
|
|
))
|
|
|
.unwrap_or((0, 0));
|
|
.unwrap_or((0, 0));
|
|
|
|
|
|
|
|
|
|
+ let meta = json!({
|
|
|
|
|
+ "tokens": input_tokens + (output_tokens * 2),
|
|
|
|
|
+ "created_at": Utc::now(),
|
|
|
|
|
+ "something": "else"
|
|
|
|
|
+ });
|
|
|
|
|
+
|
|
|
|
|
+ let sse_chunk = format!("event: meta\ndata: {}\n\n", meta.to_string());
|
|
|
|
|
+
|
|
|
let _ = tx.send((content, input_tokens, output_tokens));
|
|
let _ = tx.send((content, input_tokens, output_tokens));
|
|
|
- Ok::<Bytes, HttpError>(Bytes::new())
|
|
|
|
|
|
|
+ Ok::<Bytes, HttpError>(Bytes::from(sse_chunk))
|
|
|
}));
|
|
}));
|
|
|
|
|
|
|
|
Ok(PromptResponse {
|
|
Ok(PromptResponse {
|