Add 4th tab in voice panel with quantitive and qualitive metrics
This commit is contained in:
dimitrievgs
parent
2bffd28c5d
commit
526f4e7b94
|
|
@ -385,6 +385,7 @@ def get_voice_status():
|
|||
"transcription": voice_inst.get_context_by_limit(current_obsidian_settings.get('voiceDisplayLimit', 0), order='reversed'),
|
||||
"response1": list(voice_inst.responses["regular"]),
|
||||
"response2": list(voice_inst.responses["commands"]),
|
||||
"metrics": voice_inst.metrics_data,
|
||||
"is_running": voice_inst.is_running,
|
||||
"obsidian_settings_fetched": obsidian_settings_fetched
|
||||
})
|
||||
|
|
|
|||
|
|
@ -154,6 +154,7 @@ class VoiceHeartbeatMonitor(threading.Thread):
|
|||
self.generator = generator
|
||||
self.get_obsidian_settings = get_obsidian_settings_fn # Функция, возвращающая текущий конфиг
|
||||
self.last_regular_run = time.time()
|
||||
self.last_metrics_run = time.time()
|
||||
with self.vs.history_lock:
|
||||
self.last_gm_index = len(self.vs.history) - 1
|
||||
self.gm_buffer = ""
|
||||
|
|
@ -175,6 +176,15 @@ class VoiceHeartbeatMonitor(threading.Thread):
|
|||
'voiceInterval'] * 60:
|
||||
self._run_regular_analysis(obsidian_settings)
|
||||
self.last_regular_run = time.time()
|
||||
|
||||
# 3. НОВОЕ: Интервал метрик (Графики + Вкладка 4 LLM)
|
||||
if time.time() - self.last_metrics_run > obsidian_settings.get('metricsInterval', 15) * 60:
|
||||
# Запускаем питоновские графики
|
||||
threading.Thread(target=self.vs.generate_metrics_graphs, daemon=True).start()
|
||||
# Запускаем LLM анализ качественных метрик
|
||||
self._run_metrics_analysis(obsidian_settings)
|
||||
self.last_metrics_run = time.time()
|
||||
|
||||
time.sleep(3) # каждые 3 секунды проверяем
|
||||
except Exception as e:
|
||||
logger.critical(f"❌🎤 Voice Heartbeat: Неожиданная ошибка: {e}")
|
||||
|
|
@ -237,3 +247,13 @@ class VoiceHeartbeatMonitor(threading.Thread):
|
|||
print(f"🎤 Запуск регулярного анализа контекста ({len(context_text)} симв.)")
|
||||
prompt = f"{(obsidian_settings.get('systemPrompt') or '')}\n\n{(obsidian_settings.get('promptRegular') or '')}"
|
||||
self.generator.add_voice_regular_task(context_text, prompt)
|
||||
|
||||
# Метод для вызова LLM для аналитики
|
||||
def _run_metrics_analysis(self, obsidian_settings):
|
||||
context_text = self.vs.get_context_by_limit(obsidian_settings.get('voiceContextLimit', 30000))
|
||||
if not context_text.strip(): return
|
||||
|
||||
prompt = obsidian_settings.get('promptMetrics')
|
||||
if prompt:
|
||||
print("🎤 Запуск LLM анализа качественных метрик сессии...")
|
||||
self.generator.add_voice_metrics_task(context_text, prompt)
|
||||
|
|
@ -114,7 +114,20 @@ MODELS: Dict[str, Dict[str, Any]] = {
|
|||
"include_reasoning": True
|
||||
}
|
||||
},
|
||||
"gemini-3.1-pro-preview-bh": {
|
||||
"gemini-3.1-pro-openrouter": {
|
||||
"name": "google/gemini-3.1-pro-preview",
|
||||
"provider": "openai",
|
||||
"model_name": "google/gemini-3.1-pro-preview",
|
||||
"apiBase": "https://openrouter.ai/api/v1", # Добавлено /api/v1
|
||||
"apiKey":
|
||||
"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",
|
||||
"stream": True,
|
||||
"capabilities": ["vision", "reasoning"],
|
||||
"model_kwargs": {
|
||||
"include_reasoning": True
|
||||
}
|
||||
},
|
||||
"gemini-3.1-pro-bh": {
|
||||
"name": "gemini-3.1-pro-preview",
|
||||
"provider": "openai", # Изменено на "openai"
|
||||
"model_name":
|
||||
|
|
|
|||
|
|
@ -140,6 +140,8 @@ class TitleGenerator:
|
|||
self._process_voice_command(item["node_id"], item["graph_id"])
|
||||
elif item["item_type"] == "voice_regular":
|
||||
self._process_voice_regular(item["node_id"], item["graph_id"])
|
||||
elif item["item_type"] == "voice_metrics":
|
||||
self._process_voice_metrics(item["node_id"], item["graph_id"])
|
||||
else:
|
||||
# Если очередь пуста, ждем немного
|
||||
time.sleep(QUEUE_POLLING_INTERVAL)
|
||||
|
|
@ -418,3 +420,29 @@ class TitleGenerator:
|
|||
print(f"✅🎤 Голосовая команда обработана.")
|
||||
except Exception as e:
|
||||
print(f"❌🎤 Ошибка обработки команды: {e}")
|
||||
|
||||
# metrics
|
||||
|
||||
def add_voice_metrics_task(self, transcription_text, prompt, priority=10):
|
||||
with self._get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"INSERT INTO title_generation_queue (item_type, graph_id, node_id, priority) VALUES (?, ?, ?, ?)",
|
||||
("voice_metrics", prompt, transcription_text, priority)) # graph_id = prompt, node_id = context
|
||||
conn.commit()
|
||||
|
||||
def _process_voice_metrics(self, text, prompt):
|
||||
try:
|
||||
print(f"🎤 Запрос качественного анализа к LLM...")
|
||||
llm_messages = [
|
||||
SystemMessage(content=prompt),
|
||||
HumanMessage(content=f"Транскрипция для анализа: {text}")
|
||||
]
|
||||
response = self.voice_llm.invoke(llm_messages)
|
||||
|
||||
import app.api as api
|
||||
# Передаем текст в инстанс
|
||||
api.voice_inst.metrics_data["text"] = response
|
||||
print(f"✅🎤 Качественный анализ сессии завершен.")
|
||||
except Exception as e:
|
||||
print(f"❌🎤 Ошибка качественного анализа: {e}")
|
||||
|
|
@ -8,6 +8,14 @@ from vosk import Model, KaldiRecognizer
|
|||
import sounddevice as sd
|
||||
import uuid
|
||||
from collections import deque
|
||||
import io
|
||||
import base64
|
||||
|
||||
# Для генерации графиков на сервере без UI!
|
||||
import matplotlib
|
||||
matplotlib.use('Agg') # Для работы без GUI
|
||||
import matplotlib.pyplot as plt
|
||||
import pandas as pd
|
||||
|
||||
VOICE_COMMANDS_RESPONSE_TO_STORE = 3
|
||||
|
||||
|
|
@ -25,6 +33,10 @@ class VoiceService:
|
|||
"regular": deque(maxlen=VOICE_COMMANDS_RESPONSE_TO_STORE),
|
||||
"commands": deque(maxlen=VOICE_COMMANDS_RESPONSE_TO_STORE)
|
||||
}
|
||||
self.metrics_data = {
|
||||
"graphs": [], # Массив Base64 строчек с HTML-декором
|
||||
"text": "Анализ пока не выполнялся или недостаточно данных..."
|
||||
}
|
||||
self.lock = threading.Lock()
|
||||
self._stop_event = threading.Event()
|
||||
self.abs_logs_path = None
|
||||
|
|
@ -39,8 +51,8 @@ class VoiceService:
|
|||
return
|
||||
|
||||
import re
|
||||
# Регулярка для парсинга строки типа: [10:20:30.123] GM: текст реплики
|
||||
pattern = re.compile(r"\[(\d{2}:\d{2}:\d{2}(?:\.\d+)?)\]\s+(GM|PCs):\s+(.*)")
|
||||
# Паттерн для: [10:20:30.123 -> 10:20:35.456] GM: текст
|
||||
pattern = re.compile(r"\[(\d{2}:\d{2}:\d{2}(?:\.\d+)?)\s*(?:->|-->)\s*(\d{2}:\d{2}:\d{2}(?:\.\d+)?)\]\s+(GM|PCs):\s+(.*)")
|
||||
|
||||
new_history = []
|
||||
try:
|
||||
|
|
@ -48,13 +60,13 @@ class VoiceService:
|
|||
for line in f:
|
||||
match = pattern.match(line.strip())
|
||||
if match:
|
||||
timestamp, role, text = match.groups()
|
||||
start_time, end_time, role, text = match.groups()
|
||||
new_history.append({
|
||||
'role': role,
|
||||
'text': text,
|
||||
'time': timestamp
|
||||
'startTime': start_time,
|
||||
'endTime': end_time
|
||||
})
|
||||
|
||||
with self.history_lock:
|
||||
self.history = new_history
|
||||
print(f"Loaded {len(self.history)} events from existing log.")
|
||||
|
|
@ -64,6 +76,12 @@ class VoiceService:
|
|||
def start_session(self, continue_last=False):
|
||||
if self.is_running: return
|
||||
|
||||
# Очистка вкладок при новом старте
|
||||
if not continue_last:
|
||||
self.responses["regular"].clear()
|
||||
self.responses["commands"].clear()
|
||||
self.metrics_data = {"graphs": [], "text": "Ожидание достаточного количества данных..."}
|
||||
|
||||
log_dir = self.abs_logs_path if self.abs_logs_path else "audio-logs"
|
||||
if not os.path.exists(log_dir): os.makedirs(log_dir)
|
||||
|
||||
|
|
@ -118,6 +136,11 @@ class VoiceService:
|
|||
rec_gm = KaldiRecognizer(self.model, self.samplerate)
|
||||
rec_pcs = KaldiRecognizer(self.model, self.samplerate)
|
||||
|
||||
rec_states = {
|
||||
"GM": {"start": datetime.datetime.now()},
|
||||
"PCs": {"start": datetime.datetime.now()}
|
||||
}
|
||||
|
||||
# 3. Фабрика колбэков
|
||||
def create_callback(rec, role):
|
||||
def callback_func(indata, frames, time_info, status):
|
||||
|
|
@ -130,10 +153,18 @@ class VoiceService:
|
|||
res = json.loads(rec.Result())
|
||||
text = res.get('text', '')
|
||||
if text:
|
||||
now = datetime.datetime.now().strftime("%H:%M:%S")
|
||||
end_dt = datetime.datetime.now()
|
||||
start_dt = rec_states[role]["start"]
|
||||
|
||||
# 1. Запись в файл (как и было)
|
||||
entry = f"[{now}] {role}: {text}\n"
|
||||
# Форматируем с миллисекундами
|
||||
start_str = start_dt.strftime("%H:%M:%S.%f")[:-3]
|
||||
end_str = end_dt.strftime("%H:%M:%S.%f")[:-3]
|
||||
|
||||
# Обновляем старт для следующей фразы
|
||||
rec_states[role]["start"] = end_dt
|
||||
|
||||
# 1. Запись в файл с двумя метками
|
||||
entry = f"[{start_str} --> {end_str}] {role}: {text}\n"
|
||||
try:
|
||||
with self.lock:
|
||||
with open(self.current_file, "a", encoding="utf-8") as f:
|
||||
|
|
@ -141,12 +172,13 @@ class VoiceService:
|
|||
except Exception as e:
|
||||
print(f"File write error: {e}")
|
||||
|
||||
# 2. ЗАПИСЬ В ПАМЯТЬ
|
||||
# 2. Запись в память
|
||||
with self.history_lock:
|
||||
self.history.append({
|
||||
'role': role,
|
||||
'text': text,
|
||||
'time': now
|
||||
'startTime': start_str,
|
||||
'endTime': end_str
|
||||
})
|
||||
return callback_func
|
||||
|
||||
|
|
@ -220,3 +252,92 @@ class VoiceService:
|
|||
]
|
||||
|
||||
return " ".join(new_phrases), current_max_index
|
||||
|
||||
def generate_metrics_graphs(self):
|
||||
"""Анализирует history и генерирует графики (Matplotlib -> Base64)"""
|
||||
with self.history_lock:
|
||||
if not self.history: return
|
||||
history_copy = list(self.history)
|
||||
|
||||
if len(history_copy) < 3:
|
||||
return
|
||||
|
||||
try:
|
||||
df = pd.DataFrame(history_copy)
|
||||
df['tokens'] = df['text'].apply(lambda x: len(str(x).split()))
|
||||
df['chars'] = df['text'].apply(lambda x: len(str(x)))
|
||||
|
||||
stats = df.groupby('role').agg(
|
||||
chars_total=('chars', 'sum'),
|
||||
tokens_total=('tokens', 'sum'),
|
||||
replica_count=('text', 'count')
|
||||
)
|
||||
|
||||
total_tokens = stats['tokens_total'].sum()
|
||||
stats['percent_tokens'] = (stats['tokens_total'] / total_tokens * 100).fillna(0)
|
||||
stats['avg_len'] = (stats['tokens_total'] / stats['replica_count']).fillna(0)
|
||||
|
||||
# Расчет WPM (Скорость речи)
|
||||
wpm_data = {}
|
||||
for role in stats.index:
|
||||
role_df = df[df['role'] == role]
|
||||
total_duration_sec = 0.0
|
||||
for _, row in role_df.iterrows():
|
||||
try:
|
||||
t_format = "%H:%M:%S.%f" if "." in row['startTime'] else "%H:%M:%S"
|
||||
e_format = "%H:%M:%S.%f" if "." in row['endTime'] else "%H:%M:%S"
|
||||
t1 = datetime.datetime.strptime(row['startTime'], t_format)
|
||||
t2 = datetime.datetime.strptime(row['endTime'], e_format)
|
||||
total_duration_sec += (t2 - t1).total_seconds()
|
||||
except: pass
|
||||
duration_min = total_duration_sec / 60.0
|
||||
wpm_data[role] = stats.loc[role, 'tokens_total'] / max(duration_min, 0.05)
|
||||
|
||||
generated_images = []
|
||||
|
||||
def fig_to_base64(fig):
|
||||
buf = io.BytesIO()
|
||||
fig.savefig(buf, format='png', bbox_inches='tight', dpi=100)
|
||||
plt.close(fig)
|
||||
buf.seek(0)
|
||||
return base64.b64encode(buf.read()).decode('utf-8')
|
||||
|
||||
# --- ПЕРВАЯ КАРТИНКА: Распределение (слева) и Средняя длина (справа) ---
|
||||
fig1, (ax1_1, ax1_2) = plt.subplots(1, 2, figsize=(12, 5))
|
||||
|
||||
# Слева: Распределение реплик (%)
|
||||
ax1_1.barh(stats.index, stats["percent_tokens"], color="skyblue", alpha=0.7)
|
||||
ax1_1.set_title("Распределение реплик (по словам, %)")
|
||||
ax1_1.set_xlim(0, 100)
|
||||
ax1_1.grid(axis="x", linestyle="--", alpha=0.6)
|
||||
|
||||
# Справа: Средняя длина
|
||||
ax1_2.bar(stats.index, stats["avg_len"], color="teal", alpha=0.6)
|
||||
ax1_2.set_title("Средняя длина (слов на реплику)")
|
||||
|
||||
fig1.tight_layout()
|
||||
generated_images.append(f'<img src="data:image/png;base64,{fig_to_base64(fig1)}" style="max-width:100%; border-radius: 8px; margin-bottom: 12px;"/>')
|
||||
|
||||
# --- ВТОРАЯ КАРТИНКА: Общее кол-во (слева) и Скорость речи (справа) ---
|
||||
fig2, (ax2_1, ax2_2) = plt.subplots(1, 2, figsize=(12, 5))
|
||||
|
||||
# Слева: Общее количество реплик
|
||||
ax2_1.bar(stats.index, stats["replica_count"], color="coral", alpha=0.6)
|
||||
ax2_1.set_title("Общее кол-во реплик")
|
||||
|
||||
# Справа: Скорость речи (WPM)
|
||||
roles_list = stats.index.tolist()
|
||||
wpm_values = [wpm_data.get(r, 0) for r in roles_list]
|
||||
ax2_2.bar(roles_list, wpm_values, color='#9b59b6', alpha=0.7)
|
||||
ax2_2.set_title("Скорость речи (WPM)")
|
||||
ax2_2.set_ylabel("Слов в минуту")
|
||||
ax2_2.grid(axis='y', linestyle='--', alpha=0.6)
|
||||
|
||||
fig2.tight_layout()
|
||||
generated_images.append(f'<img src="data:image/png;base64,{fig_to_base64(fig2)}" style="max-width:100%; border-radius: 8px; margin-bottom: 12px;"/>')
|
||||
|
||||
self.metrics_data["graphs"] = generated_images
|
||||
print("✅ Аналитика (сдвоенные графики) обновлена.")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Ошибка генерации графиков: {e}")
|
||||
|
|
@ -17,3 +17,6 @@ selenium
|
|||
vosk==0.3.45
|
||||
sounddevice==0.5.1
|
||||
numpy==1.26.4
|
||||
|
||||
matplotlib==3.8.4
|
||||
pandas==2.2.2
|
||||
Loading…
Reference in New Issue
Block a user