audio.py 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839
  1. import hashlib
  2. import json
  3. import logging
  4. import os
  5. import uuid
  6. from functools import lru_cache
  7. from pathlib import Path
  8. from pydub import AudioSegment
  9. from pydub.silence import split_on_silence
  10. import aiohttp
  11. import aiofiles
  12. import requests
  13. import mimetypes
  14. from fastapi import (
  15. Depends,
  16. FastAPI,
  17. File,
  18. HTTPException,
  19. Request,
  20. UploadFile,
  21. status,
  22. APIRouter,
  23. )
  24. from fastapi.middleware.cors import CORSMiddleware
  25. from fastapi.responses import FileResponse
  26. from pydantic import BaseModel
  27. from open_webui.utils.auth import get_admin_user, get_verified_user
  28. from open_webui.config import (
  29. WHISPER_MODEL_AUTO_UPDATE,
  30. WHISPER_MODEL_DIR,
  31. CACHE_DIR,
  32. )
  33. from open_webui.constants import ERROR_MESSAGES
  34. from open_webui.env import (
  35. AIOHTTP_CLIENT_TIMEOUT,
  36. ENV,
  37. SRC_LOG_LEVELS,
  38. DEVICE_TYPE,
  39. ENABLE_FORWARD_USER_INFO_HEADERS,
  40. )
  41. router = APIRouter()
  42. # Constants
  43. MAX_FILE_SIZE_MB = 25
  44. MAX_FILE_SIZE = MAX_FILE_SIZE_MB * 1024 * 1024 # Convert MB to bytes
  45. log = logging.getLogger(__name__)
  46. log.setLevel(SRC_LOG_LEVELS["AUDIO"])
  47. SPEECH_CACHE_DIR = CACHE_DIR / "audio" / "speech"
  48. SPEECH_CACHE_DIR.mkdir(parents=True, exist_ok=True)
  49. ##########################################
  50. #
  51. # Utility functions
  52. #
  53. ##########################################
  54. from pydub import AudioSegment
  55. from pydub.utils import mediainfo
  56. def audio_needs_conversion(file_path):
  57. """Check if the given file needs to be converted to a different format."""
  58. if not os.path.isfile(file_path):
  59. log.error(f"File not found: {file_path}")
  60. return False
  61. info = mediainfo(file_path)
  62. if (
  63. info.get("codec_name") == "aac"
  64. and info.get("codec_type") == "audio"
  65. and info.get("codec_tag_string") == "mp4a"
  66. ):
  67. return "mp4"
  68. elif info.get("format_name") == "ogg":
  69. return "ogg"
  70. elif info.get("format_name") == "matroska,webm":
  71. return "webm"
  72. return None
  73. def convert_audio_to_wav(file_path, output_path, conversion_type):
  74. """Convert MP4/OGG audio file to WAV format."""
  75. audio = AudioSegment.from_file(file_path, format=conversion_type)
  76. audio.export(output_path, format="wav")
  77. log.info(f"Converted {file_path} to {output_path}")
  78. def set_faster_whisper_model(model: str, auto_update: bool = False):
  79. whisper_model = None
  80. if model:
  81. from faster_whisper import WhisperModel
  82. faster_whisper_kwargs = {
  83. "model_size_or_path": model,
  84. "device": DEVICE_TYPE if DEVICE_TYPE and DEVICE_TYPE == "cuda" else "cpu",
  85. "compute_type": "int8",
  86. "download_root": WHISPER_MODEL_DIR,
  87. "local_files_only": not auto_update,
  88. }
  89. try:
  90. whisper_model = WhisperModel(**faster_whisper_kwargs)
  91. except Exception:
  92. log.warning(
  93. "WhisperModel initialization failed, attempting download with local_files_only=False"
  94. )
  95. faster_whisper_kwargs["local_files_only"] = False
  96. whisper_model = WhisperModel(**faster_whisper_kwargs)
  97. return whisper_model
  98. ##########################################
  99. #
  100. # Audio API
  101. #
  102. ##########################################
  103. class TTSConfigForm(BaseModel):
  104. OPENAI_API_BASE_URL: str
  105. OPENAI_API_KEY: str
  106. API_KEY: str
  107. ENGINE: str
  108. MODEL: str
  109. VOICE: str
  110. SPLIT_ON: str
  111. AZURE_SPEECH_REGION: str
  112. AZURE_SPEECH_OUTPUT_FORMAT: str
  113. class STTConfigForm(BaseModel):
  114. OPENAI_API_BASE_URL: str
  115. OPENAI_API_KEY: str
  116. ENGINE: str
  117. MODEL: str
  118. WHISPER_MODEL: str
  119. DEEPGRAM_API_KEY: str
  120. class AudioConfigUpdateForm(BaseModel):
  121. tts: TTSConfigForm
  122. stt: STTConfigForm
  123. @router.get("/config")
  124. async def get_audio_config(request: Request, user=Depends(get_admin_user)):
  125. return {
  126. "tts": {
  127. "OPENAI_API_BASE_URL": request.app.state.config.TTS_OPENAI_API_BASE_URL,
  128. "OPENAI_API_KEY": request.app.state.config.TTS_OPENAI_API_KEY,
  129. "API_KEY": request.app.state.config.TTS_API_KEY,
  130. "ENGINE": request.app.state.config.TTS_ENGINE,
  131. "MODEL": request.app.state.config.TTS_MODEL,
  132. "VOICE": request.app.state.config.TTS_VOICE,
  133. "SPLIT_ON": request.app.state.config.TTS_SPLIT_ON,
  134. "AZURE_SPEECH_REGION": request.app.state.config.TTS_AZURE_SPEECH_REGION,
  135. "AZURE_SPEECH_OUTPUT_FORMAT": request.app.state.config.TTS_AZURE_SPEECH_OUTPUT_FORMAT,
  136. },
  137. "stt": {
  138. "OPENAI_API_BASE_URL": request.app.state.config.STT_OPENAI_API_BASE_URL,
  139. "OPENAI_API_KEY": request.app.state.config.STT_OPENAI_API_KEY,
  140. "ENGINE": request.app.state.config.STT_ENGINE,
  141. "MODEL": request.app.state.config.STT_MODEL,
  142. "WHISPER_MODEL": request.app.state.config.WHISPER_MODEL,
  143. "DEEPGRAM_API_KEY": request.app.state.config.DEEPGRAM_API_KEY,
  144. },
  145. }
  146. @router.post("/config/update")
  147. async def update_audio_config(
  148. request: Request, form_data: AudioConfigUpdateForm, user=Depends(get_admin_user)
  149. ):
  150. request.app.state.config.TTS_OPENAI_API_BASE_URL = form_data.tts.OPENAI_API_BASE_URL
  151. request.app.state.config.TTS_OPENAI_API_KEY = form_data.tts.OPENAI_API_KEY
  152. request.app.state.config.TTS_API_KEY = form_data.tts.API_KEY
  153. request.app.state.config.TTS_ENGINE = form_data.tts.ENGINE
  154. request.app.state.config.TTS_MODEL = form_data.tts.MODEL
  155. request.app.state.config.TTS_VOICE = form_data.tts.VOICE
  156. request.app.state.config.TTS_SPLIT_ON = form_data.tts.SPLIT_ON
  157. request.app.state.config.TTS_AZURE_SPEECH_REGION = form_data.tts.AZURE_SPEECH_REGION
  158. request.app.state.config.TTS_AZURE_SPEECH_OUTPUT_FORMAT = (
  159. form_data.tts.AZURE_SPEECH_OUTPUT_FORMAT
  160. )
  161. request.app.state.config.STT_OPENAI_API_BASE_URL = form_data.stt.OPENAI_API_BASE_URL
  162. request.app.state.config.STT_OPENAI_API_KEY = form_data.stt.OPENAI_API_KEY
  163. request.app.state.config.STT_ENGINE = form_data.stt.ENGINE
  164. request.app.state.config.STT_MODEL = form_data.stt.MODEL
  165. request.app.state.config.WHISPER_MODEL = form_data.stt.WHISPER_MODEL
  166. request.app.state.config.DEEPGRAM_API_KEY = form_data.stt.DEEPGRAM_API_KEY
  167. if request.app.state.config.STT_ENGINE == "":
  168. request.app.state.faster_whisper_model = set_faster_whisper_model(
  169. form_data.stt.WHISPER_MODEL, WHISPER_MODEL_AUTO_UPDATE
  170. )
  171. return {
  172. "tts": {
  173. "OPENAI_API_BASE_URL": request.app.state.config.TTS_OPENAI_API_BASE_URL,
  174. "OPENAI_API_KEY": request.app.state.config.TTS_OPENAI_API_KEY,
  175. "API_KEY": request.app.state.config.TTS_API_KEY,
  176. "ENGINE": request.app.state.config.TTS_ENGINE,
  177. "MODEL": request.app.state.config.TTS_MODEL,
  178. "VOICE": request.app.state.config.TTS_VOICE,
  179. "SPLIT_ON": request.app.state.config.TTS_SPLIT_ON,
  180. "AZURE_SPEECH_REGION": request.app.state.config.TTS_AZURE_SPEECH_REGION,
  181. "AZURE_SPEECH_OUTPUT_FORMAT": request.app.state.config.TTS_AZURE_SPEECH_OUTPUT_FORMAT,
  182. },
  183. "stt": {
  184. "OPENAI_API_BASE_URL": request.app.state.config.STT_OPENAI_API_BASE_URL,
  185. "OPENAI_API_KEY": request.app.state.config.STT_OPENAI_API_KEY,
  186. "ENGINE": request.app.state.config.STT_ENGINE,
  187. "MODEL": request.app.state.config.STT_MODEL,
  188. "WHISPER_MODEL": request.app.state.config.WHISPER_MODEL,
  189. "DEEPGRAM_API_KEY": request.app.state.config.DEEPGRAM_API_KEY,
  190. },
  191. }
  192. def load_speech_pipeline(request):
  193. from transformers import pipeline
  194. from datasets import load_dataset
  195. if request.app.state.speech_synthesiser is None:
  196. request.app.state.speech_synthesiser = pipeline(
  197. "text-to-speech", "microsoft/speecht5_tts"
  198. )
  199. if request.app.state.speech_speaker_embeddings_dataset is None:
  200. request.app.state.speech_speaker_embeddings_dataset = load_dataset(
  201. "Matthijs/cmu-arctic-xvectors", split="validation"
  202. )
  203. @router.post("/speech")
  204. async def speech(request: Request, user=Depends(get_verified_user)):
  205. body = await request.body()
  206. name = hashlib.sha256(
  207. body
  208. + str(request.app.state.config.TTS_ENGINE).encode("utf-8")
  209. + str(request.app.state.config.TTS_MODEL).encode("utf-8")
  210. ).hexdigest()
  211. file_path = SPEECH_CACHE_DIR.joinpath(f"{name}.mp3")
  212. file_body_path = SPEECH_CACHE_DIR.joinpath(f"{name}.json")
  213. # Check if the file already exists in the cache
  214. if file_path.is_file():
  215. return FileResponse(file_path)
  216. payload = None
  217. try:
  218. payload = json.loads(body.decode("utf-8"))
  219. except Exception as e:
  220. log.exception(e)
  221. raise HTTPException(status_code=400, detail="Invalid JSON payload")
  222. if request.app.state.config.TTS_ENGINE == "openai":
  223. payload["model"] = request.app.state.config.TTS_MODEL
  224. try:
  225. timeout = aiohttp.ClientTimeout(total=AIOHTTP_CLIENT_TIMEOUT)
  226. async with aiohttp.ClientSession(
  227. timeout=timeout, trust_env=True
  228. ) as session:
  229. async with session.post(
  230. url=f"{request.app.state.config.TTS_OPENAI_API_BASE_URL}/audio/speech",
  231. json=payload,
  232. headers={
  233. "Content-Type": "application/json",
  234. "Authorization": f"Bearer {request.app.state.config.TTS_OPENAI_API_KEY}",
  235. **(
  236. {
  237. "X-OpenWebUI-User-Name": user.name,
  238. "X-OpenWebUI-User-Id": user.id,
  239. "X-OpenWebUI-User-Email": user.email,
  240. "X-OpenWebUI-User-Role": user.role,
  241. }
  242. if ENABLE_FORWARD_USER_INFO_HEADERS
  243. else {}
  244. ),
  245. },
  246. ) as r:
  247. r.raise_for_status()
  248. async with aiofiles.open(file_path, "wb") as f:
  249. await f.write(await r.read())
  250. async with aiofiles.open(file_body_path, "w") as f:
  251. await f.write(json.dumps(payload))
  252. return FileResponse(file_path)
  253. except Exception as e:
  254. log.exception(e)
  255. detail = None
  256. try:
  257. if r.status != 200:
  258. res = await r.json()
  259. if "error" in res:
  260. detail = f"External: {res['error'].get('message', '')}"
  261. except Exception:
  262. detail = f"External: {e}"
  263. raise HTTPException(
  264. status_code=getattr(r, "status", 500),
  265. detail=detail if detail else "Open WebUI: Server Connection Error",
  266. )
  267. elif request.app.state.config.TTS_ENGINE == "elevenlabs":
  268. voice_id = payload.get("voice", "")
  269. if voice_id not in get_available_voices(request):
  270. raise HTTPException(
  271. status_code=400,
  272. detail="Invalid voice id",
  273. )
  274. try:
  275. timeout = aiohttp.ClientTimeout(total=AIOHTTP_CLIENT_TIMEOUT)
  276. async with aiohttp.ClientSession(
  277. timeout=timeout, trust_env=True
  278. ) as session:
  279. async with session.post(
  280. f"https://api.elevenlabs.io/v1/text-to-speech/{voice_id}",
  281. json={
  282. "text": payload["input"],
  283. "model_id": request.app.state.config.TTS_MODEL,
  284. "voice_settings": {"stability": 0.5, "similarity_boost": 0.5},
  285. },
  286. headers={
  287. "Accept": "audio/mpeg",
  288. "Content-Type": "application/json",
  289. "xi-api-key": request.app.state.config.TTS_API_KEY,
  290. },
  291. ) as r:
  292. r.raise_for_status()
  293. async with aiofiles.open(file_path, "wb") as f:
  294. await f.write(await r.read())
  295. async with aiofiles.open(file_body_path, "w") as f:
  296. await f.write(json.dumps(payload))
  297. return FileResponse(file_path)
  298. except Exception as e:
  299. log.exception(e)
  300. detail = None
  301. try:
  302. if r.status != 200:
  303. res = await r.json()
  304. if "error" in res:
  305. detail = f"External: {res['error'].get('message', '')}"
  306. except Exception:
  307. detail = f"External: {e}"
  308. raise HTTPException(
  309. status_code=getattr(r, "status", 500),
  310. detail=detail if detail else "Open WebUI: Server Connection Error",
  311. )
  312. elif request.app.state.config.TTS_ENGINE == "azure":
  313. try:
  314. payload = json.loads(body.decode("utf-8"))
  315. except Exception as e:
  316. log.exception(e)
  317. raise HTTPException(status_code=400, detail="Invalid JSON payload")
  318. region = request.app.state.config.TTS_AZURE_SPEECH_REGION
  319. language = request.app.state.config.TTS_VOICE
  320. locale = "-".join(request.app.state.config.TTS_VOICE.split("-")[:1])
  321. output_format = request.app.state.config.TTS_AZURE_SPEECH_OUTPUT_FORMAT
  322. try:
  323. data = f"""<speak version="1.0" xmlns="http://www.w3.org/2001/10/synthesis" xml:lang="{locale}">
  324. <voice name="{language}">{payload["input"]}</voice>
  325. </speak>"""
  326. timeout = aiohttp.ClientTimeout(total=AIOHTTP_CLIENT_TIMEOUT)
  327. async with aiohttp.ClientSession(
  328. timeout=timeout, trust_env=True
  329. ) as session:
  330. async with session.post(
  331. f"https://{region}.tts.speech.microsoft.com/cognitiveservices/v1",
  332. headers={
  333. "Ocp-Apim-Subscription-Key": request.app.state.config.TTS_API_KEY,
  334. "Content-Type": "application/ssml+xml",
  335. "X-Microsoft-OutputFormat": output_format,
  336. },
  337. data=data,
  338. ) as r:
  339. r.raise_for_status()
  340. async with aiofiles.open(file_path, "wb") as f:
  341. await f.write(await r.read())
  342. async with aiofiles.open(file_body_path, "w") as f:
  343. await f.write(json.dumps(payload))
  344. return FileResponse(file_path)
  345. except Exception as e:
  346. log.exception(e)
  347. detail = None
  348. try:
  349. if r.status != 200:
  350. res = await r.json()
  351. if "error" in res:
  352. detail = f"External: {res['error'].get('message', '')}"
  353. except Exception:
  354. detail = f"External: {e}"
  355. raise HTTPException(
  356. status_code=getattr(r, "status", 500),
  357. detail=detail if detail else "Open WebUI: Server Connection Error",
  358. )
  359. elif request.app.state.config.TTS_ENGINE == "transformers":
  360. payload = None
  361. try:
  362. payload = json.loads(body.decode("utf-8"))
  363. except Exception as e:
  364. log.exception(e)
  365. raise HTTPException(status_code=400, detail="Invalid JSON payload")
  366. import torch
  367. import soundfile as sf
  368. load_speech_pipeline(request)
  369. embeddings_dataset = request.app.state.speech_speaker_embeddings_dataset
  370. speaker_index = 6799
  371. try:
  372. speaker_index = embeddings_dataset["filename"].index(
  373. request.app.state.config.TTS_MODEL
  374. )
  375. except Exception:
  376. pass
  377. speaker_embedding = torch.tensor(
  378. embeddings_dataset[speaker_index]["xvector"]
  379. ).unsqueeze(0)
  380. speech = request.app.state.speech_synthesiser(
  381. payload["input"],
  382. forward_params={"speaker_embeddings": speaker_embedding},
  383. )
  384. sf.write(file_path, speech["audio"], samplerate=speech["sampling_rate"])
  385. async with aiofiles.open(file_body_path, "w") as f:
  386. await f.write(json.dumps(payload))
  387. return FileResponse(file_path)
  388. def transcribe(request: Request, file_path):
  389. log.info(f"transcribe: {file_path}")
  390. filename = os.path.basename(file_path)
  391. file_dir = os.path.dirname(file_path)
  392. id = filename.split(".")[0]
  393. if request.app.state.config.STT_ENGINE == "":
  394. if request.app.state.faster_whisper_model is None:
  395. request.app.state.faster_whisper_model = set_faster_whisper_model(
  396. request.app.state.config.WHISPER_MODEL
  397. )
  398. model = request.app.state.faster_whisper_model
  399. segments, info = model.transcribe(file_path, beam_size=5)
  400. log.info(
  401. "Detected language '%s' with probability %f"
  402. % (info.language, info.language_probability)
  403. )
  404. transcript = "".join([segment.text for segment in list(segments)])
  405. data = {"text": transcript.strip()}
  406. # save the transcript to a json file
  407. transcript_file = f"{file_dir}/{id}.json"
  408. with open(transcript_file, "w") as f:
  409. json.dump(data, f)
  410. log.debug(data)
  411. return data
  412. elif request.app.state.config.STT_ENGINE == "openai":
  413. conversion_type = audio_needs_conversion(file_path)
  414. if conversion_type:
  415. os.rename(file_path, file_path.replace(".wav", f".{conversion_type}"))
  416. # Convert unsupported audio file to WAV format
  417. convert_audio_to_wav(
  418. file_path.replace(".wav", f".{conversion_type}"),
  419. file_path,
  420. conversion_type,
  421. )
  422. r = None
  423. try:
  424. r = requests.post(
  425. url=f"{request.app.state.config.STT_OPENAI_API_BASE_URL}/audio/transcriptions",
  426. headers={
  427. "Authorization": f"Bearer {request.app.state.config.STT_OPENAI_API_KEY}"
  428. },
  429. files={"file": (filename, open(file_path, "rb"))},
  430. data={"model": request.app.state.config.STT_MODEL},
  431. )
  432. r.raise_for_status()
  433. data = r.json()
  434. # save the transcript to a json file
  435. transcript_file = f"{file_dir}/{id}.json"
  436. with open(transcript_file, "w") as f:
  437. json.dump(data, f)
  438. return data
  439. except Exception as e:
  440. log.exception(e)
  441. detail = None
  442. if r is not None:
  443. try:
  444. res = r.json()
  445. if "error" in res:
  446. detail = f"External: {res['error'].get('message', '')}"
  447. except Exception:
  448. detail = f"External: {e}"
  449. raise Exception(detail if detail else "Open WebUI: Server Connection Error")
  450. elif request.app.state.config.STT_ENGINE == "deepgram":
  451. try:
  452. # Determine the MIME type of the file
  453. mime, _ = mimetypes.guess_type(file_path)
  454. if not mime:
  455. mime = "audio/wav" # fallback to wav if undetectable
  456. # Read the audio file
  457. with open(file_path, "rb") as f:
  458. file_data = f.read()
  459. # Build headers and parameters
  460. headers = {
  461. "Authorization": f"Token {request.app.state.config.DEEPGRAM_API_KEY}",
  462. "Content-Type": mime,
  463. }
  464. # Add model if specified
  465. params = {}
  466. if request.app.state.config.STT_MODEL:
  467. params["model"] = request.app.state.config.STT_MODEL
  468. # Make request to Deepgram API
  469. r = requests.post(
  470. "https://api.deepgram.com/v1/listen",
  471. headers=headers,
  472. params=params,
  473. data=file_data,
  474. )
  475. r.raise_for_status()
  476. response_data = r.json()
  477. # Extract transcript from Deepgram response
  478. try:
  479. transcript = response_data["results"]["channels"][0]["alternatives"][
  480. 0
  481. ].get("transcript", "")
  482. except (KeyError, IndexError) as e:
  483. log.error(f"Malformed response from Deepgram: {str(e)}")
  484. raise Exception(
  485. "Failed to parse Deepgram response - unexpected response format"
  486. )
  487. data = {"text": transcript.strip()}
  488. # Save transcript
  489. transcript_file = f"{file_dir}/{id}.json"
  490. with open(transcript_file, "w") as f:
  491. json.dump(data, f)
  492. return data
  493. except Exception as e:
  494. log.exception(e)
  495. detail = None
  496. if r is not None:
  497. try:
  498. res = r.json()
  499. if "error" in res:
  500. detail = f"External: {res['error'].get('message', '')}"
  501. except Exception:
  502. detail = f"External: {e}"
  503. raise Exception(detail if detail else "Open WebUI: Server Connection Error")
  504. def compress_audio(file_path):
  505. if os.path.getsize(file_path) > MAX_FILE_SIZE:
  506. file_dir = os.path.dirname(file_path)
  507. audio = AudioSegment.from_file(file_path)
  508. audio = audio.set_frame_rate(16000).set_channels(1) # Compress audio
  509. compressed_path = f"{file_dir}/{id}_compressed.opus"
  510. audio.export(compressed_path, format="opus", bitrate="32k")
  511. log.debug(f"Compressed audio to {compressed_path}")
  512. if (
  513. os.path.getsize(compressed_path) > MAX_FILE_SIZE
  514. ): # Still larger than MAX_FILE_SIZE after compression
  515. raise Exception(ERROR_MESSAGES.FILE_TOO_LARGE(size=f"{MAX_FILE_SIZE_MB}MB"))
  516. return compressed_path
  517. else:
  518. return file_path
  519. @router.post("/transcriptions")
  520. def transcription(
  521. request: Request,
  522. file: UploadFile = File(...),
  523. user=Depends(get_verified_user),
  524. ):
  525. log.info(f"file.content_type: {file.content_type}")
  526. supported_filetypes = ("audio/mpeg", "audio/wav", "audio/ogg", "audio/x-m4a")
  527. if not file.content_type.startswith(supported_filetypes):
  528. raise HTTPException(
  529. status_code=status.HTTP_400_BAD_REQUEST,
  530. detail=ERROR_MESSAGES.FILE_NOT_SUPPORTED,
  531. )
  532. try:
  533. ext = file.filename.split(".")[-1]
  534. id = uuid.uuid4()
  535. filename = f"{id}.{ext}"
  536. contents = file.file.read()
  537. file_dir = f"{CACHE_DIR}/audio/transcriptions"
  538. os.makedirs(file_dir, exist_ok=True)
  539. file_path = f"{file_dir}/{filename}"
  540. with open(file_path, "wb") as f:
  541. f.write(contents)
  542. try:
  543. try:
  544. file_path = compress_audio(file_path)
  545. except Exception as e:
  546. log.exception(e)
  547. raise HTTPException(
  548. status_code=status.HTTP_400_BAD_REQUEST,
  549. detail=ERROR_MESSAGES.DEFAULT(e),
  550. )
  551. data = transcribe(request, file_path)
  552. file_path = file_path.split("/")[-1]
  553. return {**data, "filename": file_path}
  554. except Exception as e:
  555. log.exception(e)
  556. raise HTTPException(
  557. status_code=status.HTTP_400_BAD_REQUEST,
  558. detail=ERROR_MESSAGES.DEFAULT(e),
  559. )
  560. except Exception as e:
  561. log.exception(e)
  562. raise HTTPException(
  563. status_code=status.HTTP_400_BAD_REQUEST,
  564. detail=ERROR_MESSAGES.DEFAULT(e),
  565. )
  566. def get_available_models(request: Request) -> list[dict]:
  567. available_models = []
  568. if request.app.state.config.TTS_ENGINE == "openai":
  569. # Use custom endpoint if not using the official OpenAI API URL
  570. if not request.app.state.config.TTS_OPENAI_API_BASE_URL.startswith(
  571. "https://api.openai.com"
  572. ):
  573. try:
  574. response = requests.get(
  575. f"{request.app.state.config.TTS_OPENAI_API_BASE_URL}/audio/models"
  576. )
  577. response.raise_for_status()
  578. data = response.json()
  579. available_models = data.get("models", [])
  580. except Exception as e:
  581. log.error(f"Error fetching models from custom endpoint: {str(e)}")
  582. available_models = [{"id": "tts-1"}, {"id": "tts-1-hd"}]
  583. else:
  584. available_models = [{"id": "tts-1"}, {"id": "tts-1-hd"}]
  585. elif request.app.state.config.TTS_ENGINE == "elevenlabs":
  586. try:
  587. response = requests.get(
  588. "https://api.elevenlabs.io/v1/models",
  589. headers={
  590. "xi-api-key": request.app.state.config.TTS_API_KEY,
  591. "Content-Type": "application/json",
  592. },
  593. timeout=5,
  594. )
  595. response.raise_for_status()
  596. models = response.json()
  597. available_models = [
  598. {"name": model["name"], "id": model["model_id"]} for model in models
  599. ]
  600. except requests.RequestException as e:
  601. log.error(f"Error fetching voices: {str(e)}")
  602. return available_models
  603. @router.get("/models")
  604. async def get_models(request: Request, user=Depends(get_verified_user)):
  605. return {"models": get_available_models(request)}
  606. def get_available_voices(request) -> dict:
  607. """Returns {voice_id: voice_name} dict"""
  608. available_voices = {}
  609. if request.app.state.config.TTS_ENGINE == "openai":
  610. # Use custom endpoint if not using the official OpenAI API URL
  611. if not request.app.state.config.TTS_OPENAI_API_BASE_URL.startswith(
  612. "https://api.openai.com"
  613. ):
  614. try:
  615. response = requests.get(
  616. f"{request.app.state.config.TTS_OPENAI_API_BASE_URL}/audio/voices"
  617. )
  618. response.raise_for_status()
  619. data = response.json()
  620. voices_list = data.get("voices", [])
  621. available_voices = {voice["id"]: voice["name"] for voice in voices_list}
  622. except Exception as e:
  623. log.error(f"Error fetching voices from custom endpoint: {str(e)}")
  624. available_voices = {
  625. "alloy": "alloy",
  626. "echo": "echo",
  627. "fable": "fable",
  628. "onyx": "onyx",
  629. "nova": "nova",
  630. "shimmer": "shimmer",
  631. }
  632. else:
  633. available_voices = {
  634. "alloy": "alloy",
  635. "echo": "echo",
  636. "fable": "fable",
  637. "onyx": "onyx",
  638. "nova": "nova",
  639. "shimmer": "shimmer",
  640. }
  641. elif request.app.state.config.TTS_ENGINE == "elevenlabs":
  642. try:
  643. available_voices = get_elevenlabs_voices(
  644. api_key=request.app.state.config.TTS_API_KEY
  645. )
  646. except Exception:
  647. # Avoided @lru_cache with exception
  648. pass
  649. elif request.app.state.config.TTS_ENGINE == "azure":
  650. try:
  651. region = request.app.state.config.TTS_AZURE_SPEECH_REGION
  652. url = f"https://{region}.tts.speech.microsoft.com/cognitiveservices/voices/list"
  653. headers = {
  654. "Ocp-Apim-Subscription-Key": request.app.state.config.TTS_API_KEY
  655. }
  656. response = requests.get(url, headers=headers)
  657. response.raise_for_status()
  658. voices = response.json()
  659. for voice in voices:
  660. available_voices[voice["ShortName"]] = (
  661. f"{voice['DisplayName']} ({voice['ShortName']})"
  662. )
  663. except requests.RequestException as e:
  664. log.error(f"Error fetching voices: {str(e)}")
  665. return available_voices
  666. @lru_cache
  667. def get_elevenlabs_voices(api_key: str) -> dict:
  668. """
  669. Note, set the following in your .env file to use Elevenlabs:
  670. AUDIO_TTS_ENGINE=elevenlabs
  671. AUDIO_TTS_API_KEY=sk_... # Your Elevenlabs API key
  672. AUDIO_TTS_VOICE=EXAVITQu4vr4xnSDxMaL # From https://api.elevenlabs.io/v1/voices
  673. AUDIO_TTS_MODEL=eleven_multilingual_v2
  674. """
  675. try:
  676. # TODO: Add retries
  677. response = requests.get(
  678. "https://api.elevenlabs.io/v1/voices",
  679. headers={
  680. "xi-api-key": api_key,
  681. "Content-Type": "application/json",
  682. },
  683. )
  684. response.raise_for_status()
  685. voices_data = response.json()
  686. voices = {}
  687. for voice in voices_data.get("voices", []):
  688. voices[voice["voice_id"]] = voice["name"]
  689. except requests.RequestException as e:
  690. # Avoid @lru_cache with exception
  691. log.error(f"Error fetching voices: {str(e)}")
  692. raise RuntimeError(f"Error fetching voices: {str(e)}")
  693. return voices
  694. @router.get("/voices")
  695. async def get_voices(request: Request, user=Depends(get_verified_user)):
  696. return {
  697. "voices": [
  698. {"id": k, "name": v} for k, v in get_available_voices(request).items()
  699. ]
  700. }