Commit Β·
8b80b7f
0
Parent(s):
Audio Transcriber - ESP32 real-time speech-to-text with product images
Browse files- .gitattributes +1 -0
- .gitignore +7 -0
- Procfile +1 -0
- README.md +256 -0
- esp32_firmware/esp32_firmware.ino +146 -0
- mise.toml +2 -0
- product-media/product-image-1.jpeg +0 -0
- product-media/product-image-2.jpeg +0 -0
- requirements.txt +4 -0
- runtime.txt +1 -0
- server.py +1439 -0
.gitattributes
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
*.mp4 filter=lfs diff=lfs merge=lfs -text
|
.gitignore
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__pycache__/
|
| 2 |
+
*.pyc
|
| 3 |
+
Lib/
|
| 4 |
+
Scripts/
|
| 5 |
+
Include/
|
| 6 |
+
pyvenv.cfg
|
| 7 |
+
.env
|
Procfile
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
web: uvicorn server:app --host 0.0.0.0 --port $PORT
|
README.md
ADDED
|
@@ -0,0 +1,256 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ποΈ ESP32 Real-Time Audio Transcriber
|
| 2 |
+
|
| 3 |
+
A real-time speech-to-text system that captures audio from an **ESP32 + INMP441** microphone, streams it to a cloud-hosted **FastAPI** server, transcribes it via **Deepgram Nova-3**, and generates conversational AI responses using **Google Gemini 2.5 Flash** β all viewable in a sleek live web dashboard.
|
| 4 |
+
|
| 5 |
+
---
|
| 6 |
+
|
| 7 |
+
## πΈ Product Images
|
| 8 |
+
|
| 9 |
+
<p align="center">
|
| 10 |
+
<img src="product-media/product-image-1.jpeg" alt="Product Image 1" width="400"/>
|
| 11 |
+
|
| 12 |
+
<img src="product-media/product-image-2.jpeg" alt="Product Image 2" width="400"/>
|
| 13 |
+
</p>
|
| 14 |
+
|
| 15 |
+
<p align="center"><em>ESP32 + INMP441 assembly housed in a custom 3D printed case</em></p>
|
| 16 |
+
|
| 17 |
+
### π₯ Demo Video
|
| 18 |
+
|
| 19 |
+
[βΆ Watch the product demo video](product-media/product-demo.mp4)
|
| 20 |
+
|
| 21 |
+
---
|
| 22 |
+
|
| 23 |
+
## β¨ Features
|
| 24 |
+
|
| 25 |
+
- **Real-time transcription** β Live interim + final transcripts powered by Deepgram Nova-3
|
| 26 |
+
- **AI conversation** β Press **Q** or click *Ask Gemini* to get a conversational response from Gemini 2.5 Flash with multi-turn context
|
| 27 |
+
- **WiFi captive portal** β No hardcoded credentials; configure WiFi via phone/laptop on first boot
|
| 28 |
+
- **One-click deploy** β Server deploys to [Railway](https://railway.app) with zero configuration
|
| 29 |
+
- **Live status dashboard** β Monitor WebSocket, Deepgram, ESP32, and Gemini connection states in real time
|
| 30 |
+
- **Debug panel** β Built-in debug log overlay for troubleshooting
|
| 31 |
+
|
| 32 |
+
---
|
| 33 |
+
|
| 34 |
+
## ποΈ Architecture
|
| 35 |
+
|
| 36 |
+
```
|
| 37 |
+
βββββββββββββββ WSS (binary audio) ββββββββββββββββββββ WSS βββββββββββββ
|
| 38 |
+
β ESP32 + β βββββββββββββββββββββββββββββββΆ β FastAPI Server β βββββββββββββββΆ β Deepgram β
|
| 39 |
+
β INMP441 β β (Railway) β βββββββββββββββ β Nova-3 β
|
| 40 |
+
βββββββββββββββ β β transcripts βββββββββββββ
|
| 41 |
+
β β
|
| 42 |
+
β β REST βββββββββββββ
|
| 43 |
+
βββββββββββββββ WSS (text events) β β βββββββββββββββΆ β Gemini β
|
| 44 |
+
β Browser β βββββββββββββββββββββββββββββββ β β βββββββββββββββ β 2.5 Flashβ
|
| 45 |
+
β Dashboard β βββββββββββββββββββββββββββββββΆ ββββββββββββββββββββ AI response βββββββββββββ
|
| 46 |
+
βββββββββββββββ "PROCESS" / "CLEAR"
|
| 47 |
+
```
|
| 48 |
+
|
| 49 |
+
---
|
| 50 |
+
|
| 51 |
+
## π Project Structure
|
| 52 |
+
|
| 53 |
+
```
|
| 54 |
+
audio_transcriber/
|
| 55 |
+
βββ server.py # FastAPI server (transcription + LLM + web UI)
|
| 56 |
+
βββ esp32_firmware/
|
| 57 |
+
β βββ esp32_firmware.ino # Arduino sketch for ESP32 + INMP441
|
| 58 |
+
βββ requirements.txt # Python dependencies
|
| 59 |
+
βββ Procfile # Railway process definition
|
| 60 |
+
βββ runtime.txt # Python version for Railway
|
| 61 |
+
βββ .gitignore # Ignored files (venv, cache, etc.)
|
| 62 |
+
βββ README.md # This file
|
| 63 |
+
```
|
| 64 |
+
|
| 65 |
+
---
|
| 66 |
+
|
| 67 |
+
## π§ Hardware Requirements
|
| 68 |
+
|
| 69 |
+
| Component | Description |
|
| 70 |
+
|---|---|
|
| 71 |
+
| **ESP32 Dev Board** | Any ESP32-WROOM-32 based board |
|
| 72 |
+
| **INMP441** | I2S MEMS microphone module |
|
| 73 |
+
| **Jumper wires** | 5 connections (see wiring below) |
|
| 74 |
+
| **USB cable** | For flashing and serial monitor |
|
| 75 |
+
|
| 76 |
+
### Wiring Diagram
|
| 77 |
+
|
| 78 |
+
| ESP32 Pin | INMP441 Pin | Function |
|
| 79 |
+
|---|---|---|
|
| 80 |
+
| `GPIO 26` | `SCK` | Bit Clock |
|
| 81 |
+
| `GPIO 32` | `WS` | Word Select (LRCLK) |
|
| 82 |
+
| `GPIO 33` | `SD` | Serial Data Out |
|
| 83 |
+
| `3.3V` | `VDD` | Power |
|
| 84 |
+
| `GND` | `GND` + `L/R` | Ground (L/R β GND for left channel) |
|
| 85 |
+
|
| 86 |
+
---
|
| 87 |
+
|
| 88 |
+
## π Getting Started
|
| 89 |
+
|
| 90 |
+
### 1. Deploy the Server to Railway
|
| 91 |
+
|
| 92 |
+
1. Push this repository to GitHub
|
| 93 |
+
2. Go to [railway.app](https://railway.app) β **New Project** β **Deploy from GitHub Repo**
|
| 94 |
+
3. Railway auto-detects the `Procfile` and `runtime.txt` β no extra config needed
|
| 95 |
+
4. Note your deployed URL (e.g. `audio-transcriber.up.railway.app`)
|
| 96 |
+
|
| 97 |
+
### 2. Flash the ESP32 Firmware
|
| 98 |
+
|
| 99 |
+
1. Open `esp32_firmware/esp32_firmware.ino` in **Arduino IDE**
|
| 100 |
+
2. Install the required libraries via **Library Manager**:
|
| 101 |
+
- `WiFiManager` by tzapu (β₯ 2.0)
|
| 102 |
+
- `WebSockets` by Markus Sattler (β₯ 2.4)
|
| 103 |
+
3. Update `SERVER_HOST` in the sketch if your Railway URL differs:
|
| 104 |
+
```cpp
|
| 105 |
+
const char* SERVER_HOST = "your-app-name.up.railway.app";
|
| 106 |
+
```
|
| 107 |
+
4. Select board: **ESP32 Dev Module**
|
| 108 |
+
5. Flash and open the Serial Monitor at **115200 baud**
|
| 109 |
+
|
| 110 |
+
### 3. Connect to WiFi
|
| 111 |
+
|
| 112 |
+
1. On first boot, the ESP32 creates a WiFi AP named **`Transcriber-Setup`**
|
| 113 |
+
2. Connect to it from your phone or laptop
|
| 114 |
+
3. Select your home WiFi network and enter the password
|
| 115 |
+
4. The ESP32 saves the credentials and auto-connects on subsequent boots
|
| 116 |
+
|
| 117 |
+
> π‘ **Reset WiFi:** Hold the **BOOT** button (GPIO 0) while powering on to clear saved credentials.
|
| 118 |
+
|
| 119 |
+
### 4. View the Dashboard
|
| 120 |
+
|
| 121 |
+
Open your Railway URL in a browser:
|
| 122 |
+
```
|
| 123 |
+
https://audio-transcriber.up.railway.app
|
| 124 |
+
```
|
| 125 |
+
|
| 126 |
+
You'll see live transcription appear as the ESP32 streams audio.
|
| 127 |
+
|
| 128 |
+
---
|
| 129 |
+
|
| 130 |
+
## π§ Using Gemini AI
|
| 131 |
+
|
| 132 |
+
Once transcription lines appear:
|
| 133 |
+
|
| 134 |
+
1. Press **Q** on your keyboard or click the **⬑ Ask Gemini** button
|
| 135 |
+
2. Gemini reads the buffered transcript and responds in a casual, conversational tone
|
| 136 |
+
3. The conversation supports **multi-turn context** β keep talking and asking
|
| 137 |
+
4. Click **Reset Chat** to clear the conversation history
|
| 138 |
+
|
| 139 |
+
---
|
| 140 |
+
|
| 141 |
+
## βοΈ Configuration
|
| 142 |
+
|
| 143 |
+
### Server Environment
|
| 144 |
+
|
| 145 |
+
| Variable | Default | Description |
|
| 146 |
+
|---|---|---|
|
| 147 |
+
| `PORT` | `8000` | HTTP port (auto-set by Railway) |
|
| 148 |
+
|
| 149 |
+
### API Keys
|
| 150 |
+
|
| 151 |
+
API keys are currently embedded in `server.py`. For production, move them to environment variables:
|
| 152 |
+
|
| 153 |
+
```python
|
| 154 |
+
DEEPGRAM_API_KEY = os.environ.get("DEEPGRAM_API_KEY")
|
| 155 |
+
GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY")
|
| 156 |
+
```
|
| 157 |
+
|
| 158 |
+
### ESP32 Firmware
|
| 159 |
+
|
| 160 |
+
| Constant | Default | Description |
|
| 161 |
+
|---|---|---|
|
| 162 |
+
| `SERVER_HOST` | `audio-transcriber.up.railway.app` | Railway domain |
|
| 163 |
+
| `SERVER_PORT` | `443` | HTTPS/WSS port |
|
| 164 |
+
| `WS_PATH` | `/ws/audio` | WebSocket endpoint |
|
| 165 |
+
| `SAMPLE_RATE` | `16000` | Audio sample rate (Hz) |
|
| 166 |
+
| `I2S_READ_LEN` | `256` | Samples per I2S read cycle |
|
| 167 |
+
|
| 168 |
+
---
|
| 169 |
+
|
| 170 |
+
## π οΈ Local Development
|
| 171 |
+
|
| 172 |
+
```bash
|
| 173 |
+
# Clone the repo
|
| 174 |
+
git clone https://github.com/<your-username>/audio_transcriber.git
|
| 175 |
+
cd audio_transcriber
|
| 176 |
+
|
| 177 |
+
# Create a virtual environment
|
| 178 |
+
python -m venv venv
|
| 179 |
+
source venv/bin/activate # Linux/Mac
|
| 180 |
+
# venv\Scripts\activate # Windows
|
| 181 |
+
|
| 182 |
+
# Install dependencies
|
| 183 |
+
pip install -r requirements.txt
|
| 184 |
+
|
| 185 |
+
# Run the server locally
|
| 186 |
+
python server.py
|
| 187 |
+
# β Server starts at http://localhost:8000
|
| 188 |
+
```
|
| 189 |
+
|
| 190 |
+
> Update `SERVER_HOST` in the ESP32 sketch to your local IP for local testing.
|
| 191 |
+
|
| 192 |
+
---
|
| 193 |
+
|
| 194 |
+
## π¦ Dependencies
|
| 195 |
+
|
| 196 |
+
### Python (Server)
|
| 197 |
+
|
| 198 |
+
| Package | Purpose |
|
| 199 |
+
|---|---|
|
| 200 |
+
| `fastapi` | Async web framework & WebSocket handling |
|
| 201 |
+
| `uvicorn` | ASGI server |
|
| 202 |
+
| `websockets` | Async WebSocket client for Deepgram |
|
| 203 |
+
| `langchain-google-genai` | LangChain integration for Gemini |
|
| 204 |
+
|
| 205 |
+
### Arduino (ESP32)
|
| 206 |
+
|
| 207 |
+
| Library | Purpose |
|
| 208 |
+
|---|---|
|
| 209 |
+
| `WiFiManager` | Captive portal for WiFi provisioning |
|
| 210 |
+
| `WebSockets` | WebSocket client with SSL support |
|
| 211 |
+
|
| 212 |
+
---
|
| 213 |
+
|
| 214 |
+
## π API Endpoints
|
| 215 |
+
|
| 216 |
+
| Endpoint | Type | Description |
|
| 217 |
+
|---|---|---|
|
| 218 |
+
| `GET /` | HTTP | Serves the live transcription dashboard |
|
| 219 |
+
| `WS /ws` | WebSocket | Browser β Server (transcripts + LLM events) |
|
| 220 |
+
| `WS /ws/audio` | WebSocket | ESP32 β Server (raw 16-bit PCM audio) |
|
| 221 |
+
|
| 222 |
+
### Browser WebSocket Messages
|
| 223 |
+
|
| 224 |
+
| Direction | Message | Description |
|
| 225 |
+
|---|---|---|
|
| 226 |
+
| Browser β Server | `PROCESS` | Trigger Gemini on buffered transcript |
|
| 227 |
+
| Browser β Server | `CLEAR_HISTORY` | Reset conversation history |
|
| 228 |
+
| Server β Browser | `__FINAL__:<text>` | Final transcript line |
|
| 229 |
+
| Server β Browser | `__INTERIM__:<text>` | Interim (partial) transcript |
|
| 230 |
+
| Server β Browser | `__LLM_START__` | Gemini processing started |
|
| 231 |
+
| Server β Browser | `__LLM_TOKEN__:<text>` | Gemini response text |
|
| 232 |
+
| Server β Browser | `__LLM_DONE__` | Gemini processing complete |
|
| 233 |
+
| Server β Browser | `__LLM_ERROR__:<msg>` | Gemini error message |
|
| 234 |
+
| Server β Browser | `__STATUS__:<state>` | Connection state change |
|
| 235 |
+
|
| 236 |
+
---
|
| 237 |
+
|
| 238 |
+
## π€ Contributing
|
| 239 |
+
|
| 240 |
+
1. Fork the repository
|
| 241 |
+
2. Create your feature branch (`git checkout -b feature/amazing-feature`)
|
| 242 |
+
3. Commit your changes (`git commit -m 'Add amazing feature'`)
|
| 243 |
+
4. Push to the branch (`git push origin feature/amazing-feature`)
|
| 244 |
+
5. Open a Pull Request
|
| 245 |
+
|
| 246 |
+
---
|
| 247 |
+
|
| 248 |
+
## π License
|
| 249 |
+
|
| 250 |
+
This project is open source and available under the [MIT License](LICENSE).
|
| 251 |
+
|
| 252 |
+
---
|
| 253 |
+
|
| 254 |
+
<p align="center">
|
| 255 |
+
Built with β€οΈ using ESP32 Β· FastAPI Β· Deepgram Β· Gemini
|
| 256 |
+
</p>
|
esp32_firmware/esp32_firmware.ino
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/*
|
| 2 |
+
* ESP32 Real-Time Audio Transcriber Firmware
|
| 3 |
+
*
|
| 4 |
+
* Hardware: ESP32 + INMP441 MEMS Microphone
|
| 5 |
+
*
|
| 6 |
+
* Captures 16-bit 16 kHz mono audio from an INMP441 microphone via I2S
|
| 7 |
+
* and streams it over a secure WebSocket connection to a FastAPI server
|
| 8 |
+
* hosted on Railway for real-time transcription via Deepgram.
|
| 9 |
+
*
|
| 10 |
+
* WiFi credentials are managed via WiFiManager (captive portal).
|
| 11 |
+
* Hold GPIO 0 (BOOT) low during startup to reset saved WiFi settings.
|
| 12 |
+
*
|
| 13 |
+
* Wiring (ESP32 β INMP441):
|
| 14 |
+
* GPIO 26 β SCK (Bit Clock)
|
| 15 |
+
* GPIO 32 β WS (Word Select / LRCLK)
|
| 16 |
+
* GPIO 33 β SD (Serial Data)
|
| 17 |
+
* 3.3 V β VDD
|
| 18 |
+
* GND β GND & L/R (left channel = GND)
|
| 19 |
+
*
|
| 20 |
+
* Dependencies (install via Arduino Library Manager):
|
| 21 |
+
* - WiFiManager by tzapu (>= 2.0)
|
| 22 |
+
* - WebSockets by Markus Sattler (>= 2.4)
|
| 23 |
+
*
|
| 24 |
+
* Board: ESP32 Dev Module | Partition: Default 4 MB
|
| 25 |
+
*/
|
| 26 |
+
|
| 27 |
+
#include <WiFiManager.h>
|
| 28 |
+
#include <WebSocketsClient.h>
|
| 29 |
+
#include "driver/i2s.h"
|
| 30 |
+
|
| 31 |
+
// βββ Server Configuration βββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 32 |
+
const char* SERVER_HOST = "audio-transcriber.up.railway.app";
|
| 33 |
+
const uint16_t SERVER_PORT = 443;
|
| 34 |
+
const char* WS_PATH = "/ws/audio";
|
| 35 |
+
|
| 36 |
+
// βββ Pin Definitions ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 37 |
+
#define WIFI_RESET_PIN 0 // BOOT button β hold LOW on startup to reset WiFi
|
| 38 |
+
#define STATUS_LED 2 // On-board LED β HIGH = WebSocket connected
|
| 39 |
+
|
| 40 |
+
// βββ Audio Configuration ββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 41 |
+
#define SAMPLE_RATE 16000
|
| 42 |
+
#define I2S_READ_LEN 256 // Samples per I2S read cycle
|
| 43 |
+
|
| 44 |
+
// βββ Globals ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 45 |
+
WebSocketsClient webSocket;
|
| 46 |
+
bool wsConnected = false;
|
| 47 |
+
|
| 48 |
+
// βββ I2S Configuration (INMP441 in master-receive mode) βββββββββββββββββββββ
|
| 49 |
+
i2s_config_t i2s_config = {
|
| 50 |
+
.mode = (i2s_mode_t)(I2S_MODE_MASTER | I2S_MODE_RX),
|
| 51 |
+
.sample_rate = SAMPLE_RATE,
|
| 52 |
+
.bits_per_sample = I2S_BITS_PER_SAMPLE_32BIT,
|
| 53 |
+
.channel_format = I2S_CHANNEL_FMT_ONLY_LEFT,
|
| 54 |
+
.communication_format = I2S_COMM_FORMAT_STAND_I2S,
|
| 55 |
+
.intr_alloc_flags = ESP_INTR_FLAG_LEVEL1,
|
| 56 |
+
.dma_buf_count = 8, .dma_buf_len = 256,
|
| 57 |
+
.use_apll = false, .tx_desc_auto_clear = false, .fixed_mclk = 0
|
| 58 |
+
};
|
| 59 |
+
|
| 60 |
+
i2s_pin_config_t pin_config = {
|
| 61 |
+
.bck_io_num = 26, .ws_io_num = 32,
|
| 62 |
+
.data_out_num = I2S_PIN_NO_CHANGE, .data_in_num = 33
|
| 63 |
+
};
|
| 64 |
+
|
| 65 |
+
// βββ WebSocket Event Handler ββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 66 |
+
void webSocketEvent(WStype_t type, uint8_t* payload, size_t length) {
|
| 67 |
+
switch (type) {
|
| 68 |
+
case WStype_CONNECTED:
|
| 69 |
+
wsConnected = true;
|
| 70 |
+
digitalWrite(STATUS_LED, HIGH);
|
| 71 |
+
Serial.println("[WS] Connected!");
|
| 72 |
+
break;
|
| 73 |
+
case WStype_DISCONNECTED:
|
| 74 |
+
wsConnected = false;
|
| 75 |
+
digitalWrite(STATUS_LED, LOW);
|
| 76 |
+
Serial.println("[WS] Disconnected");
|
| 77 |
+
break;
|
| 78 |
+
case WStype_TEXT:
|
| 79 |
+
Serial.printf("[WS] Server: %s\n", payload);
|
| 80 |
+
break;
|
| 81 |
+
default:
|
| 82 |
+
break;
|
| 83 |
+
}
|
| 84 |
+
}
|
| 85 |
+
|
| 86 |
+
// βββ Setup ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 87 |
+
void setup() {
|
| 88 |
+
Serial.begin(115200);
|
| 89 |
+
pinMode(WIFI_RESET_PIN, INPUT_PULLUP);
|
| 90 |
+
pinMode(STATUS_LED, OUTPUT);
|
| 91 |
+
|
| 92 |
+
Serial.println("\n=== ESP32 Transcriber v2.0 ===");
|
| 93 |
+
|
| 94 |
+
// ββ WiFi (captive portal) ββ
|
| 95 |
+
// Always clear saved credentials so the portal opens on every boot.
|
| 96 |
+
// No need to press BOOT button β just power cycle to pick a new network.
|
| 97 |
+
WiFiManager wm;
|
| 98 |
+
wm.resetSettings();
|
| 99 |
+
Serial.println("[WiFi] Credentials cleared β opening captive portal...");
|
| 100 |
+
wm.setConfigPortalTimeout(180);
|
| 101 |
+
if (!wm.autoConnect("Transcriber-Setup")) { ESP.restart(); }
|
| 102 |
+
|
| 103 |
+
IPAddress dns(8, 8, 8, 8);
|
| 104 |
+
WiFi.config(WiFi.localIP(), WiFi.gatewayIP(), WiFi.subnetMask(), dns);
|
| 105 |
+
Serial.println("[WiFi] Connected: " + WiFi.localIP().toString());
|
| 106 |
+
|
| 107 |
+
// ββ I2S (INMP441 microphone) ββ
|
| 108 |
+
i2s_driver_install(I2S_NUM_0, &i2s_config, 0, NULL);
|
| 109 |
+
i2s_set_pin(I2S_NUM_0, &pin_config);
|
| 110 |
+
i2s_zero_dma_buffer(I2S_NUM_0);
|
| 111 |
+
|
| 112 |
+
// Flush initial garbage samples
|
| 113 |
+
int32_t dummy[64];
|
| 114 |
+
size_t br;
|
| 115 |
+
for (int i = 0; i < 5; i++) {
|
| 116 |
+
i2s_read(I2S_NUM_0, dummy, sizeof(dummy), &br, pdMS_TO_TICKS(10));
|
| 117 |
+
}
|
| 118 |
+
Serial.println("[Mic] INMP441 ready");
|
| 119 |
+
|
| 120 |
+
// ββ WebSocket (SSL) ββ
|
| 121 |
+
webSocket.beginSSL(SERVER_HOST, SERVER_PORT, WS_PATH);
|
| 122 |
+
webSocket.onEvent(webSocketEvent);
|
| 123 |
+
webSocket.setReconnectInterval(5000);
|
| 124 |
+
webSocket.enableHeartbeat(15000, 10000, 2);
|
| 125 |
+
Serial.println("[WS] Connecting...");
|
| 126 |
+
}
|
| 127 |
+
|
| 128 |
+
// βββ Main Loop ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 129 |
+
void loop() {
|
| 130 |
+
webSocket.loop();
|
| 131 |
+
if (!wsConnected) return;
|
| 132 |
+
|
| 133 |
+
static int32_t raw32[I2S_READ_LEN];
|
| 134 |
+
static int16_t buffer16[I2S_READ_LEN];
|
| 135 |
+
size_t bytesRead;
|
| 136 |
+
|
| 137 |
+
i2s_read(I2S_NUM_0, raw32, sizeof(raw32), &bytesRead, pdMS_TO_TICKS(100));
|
| 138 |
+
|
| 139 |
+
if (bytesRead > 0) {
|
| 140 |
+
int samples = bytesRead / 4;
|
| 141 |
+
for (int i = 0; i < samples; i++) {
|
| 142 |
+
buffer16[i] = (int16_t)(raw32[i] >> 16);
|
| 143 |
+
}
|
| 144 |
+
webSocket.sendBIN((uint8_t*)buffer16, samples * 2);
|
| 145 |
+
}
|
| 146 |
+
}
|
mise.toml
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[settings]
|
| 2 |
+
python.github_attestations = false
|
product-media/product-image-1.jpeg
ADDED
|
product-media/product-image-2.jpeg
ADDED
|
requirements.txt
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
fastapi==0.136.1
|
| 2 |
+
uvicorn==0.46.0
|
| 3 |
+
websockets==16.0
|
| 4 |
+
langchain-google-genai==4.2.2
|
runtime.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
python-3.12.0
|
server.py
ADDED
|
@@ -0,0 +1,1439 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
FastAPI server that:
|
| 3 |
+
1. Accepts raw audio from ESP32 over WebSocket (same port 8000)
|
| 4 |
+
2. Streams audio to Deepgram via raw WebSocket (no SDK)
|
| 5 |
+
3. Pushes transcripts to a browser via WebSocket
|
| 6 |
+
4. When user presses Q/Stop, sends full transcript to Gemini via LangChain
|
| 7 |
+
and streams the conversational response + follow-up question back to browser
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
import asyncio
|
| 11 |
+
import json
|
| 12 |
+
import os
|
| 13 |
+
from contextlib import asynccontextmanager
|
| 14 |
+
|
| 15 |
+
import websockets
|
| 16 |
+
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
|
| 17 |
+
from fastapi.responses import HTMLResponse
|
| 18 |
+
|
| 19 |
+
from langchain_google_genai import ChatGoogleGenerativeAI
|
| 20 |
+
|
| 21 |
+
# βββ Configuration βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 22 |
+
|
| 23 |
+
DEEPGRAM_API_KEY = "Enter DeepGram API Key"
|
| 24 |
+
GOOGLE_API_KEY = "Enter Google API Key"
|
| 25 |
+
GOOGLE_API_KEY_CHAT = "Enter Google API Key"
|
| 26 |
+
|
| 27 |
+
SAMPLE_RATE = 16000
|
| 28 |
+
FASTAPI_HOST = "0.0.0.0"
|
| 29 |
+
FASTAPI_PORT = int(os.environ.get("PORT", 8000))
|
| 30 |
+
|
| 31 |
+
DEEPGRAM_URL = (
|
| 32 |
+
f"wss://api.deepgram.com/v1/listen"
|
| 33 |
+
f"?model=nova-3&language=en&encoding=linear16"
|
| 34 |
+
f"&sample_rate={SAMPLE_RATE}&channels=1"
|
| 35 |
+
f"&interim_results=true&punctuate=true"
|
| 36 |
+
f"&smart_format=true&endpointing=500"
|
| 37 |
+
f"&utterance_end_ms=1500"
|
| 38 |
+
)
|
| 39 |
+
|
| 40 |
+
# βββ Shared state ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 41 |
+
|
| 42 |
+
browser_clients: list[WebSocket] = []
|
| 43 |
+
|
| 44 |
+
# Accumulates FINAL transcript lines during an active ESP32 session
|
| 45 |
+
transcript_buffer: list[str] = []
|
| 46 |
+
|
| 47 |
+
# Conversation history for multi-turn LLM summarization
|
| 48 |
+
conversation_history: list[tuple] = []
|
| 49 |
+
|
| 50 |
+
# Separate chat history for user β Sync AI conversations
|
| 51 |
+
chat_history: list[tuple] = []
|
| 52 |
+
|
| 53 |
+
# Stores ALL generated summaries so chat can reference them
|
| 54 |
+
summary_history: list[str] = []
|
| 55 |
+
|
| 56 |
+
# βββ LLM setup βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 57 |
+
|
| 58 |
+
# LLM for transcript summarization (Sync AI button)
|
| 59 |
+
llm = ChatGoogleGenerativeAI(
|
| 60 |
+
model="gemini-2.5-flash",
|
| 61 |
+
google_api_key=GOOGLE_API_KEY,
|
| 62 |
+
temperature=1.0,
|
| 63 |
+
max_tokens=None,
|
| 64 |
+
timeout=None,
|
| 65 |
+
max_retries=2,
|
| 66 |
+
)
|
| 67 |
+
|
| 68 |
+
# Separate LLM for chat conversations (summary-aware Q&A)
|
| 69 |
+
chat_llm = ChatGoogleGenerativeAI(
|
| 70 |
+
model="gemini-2.5-flash",
|
| 71 |
+
google_api_key=GOOGLE_API_KEY_CHAT,
|
| 72 |
+
temperature=0.7,
|
| 73 |
+
max_tokens=None,
|
| 74 |
+
timeout=None,
|
| 75 |
+
max_retries=2,
|
| 76 |
+
)
|
| 77 |
+
|
| 78 |
+
LLM_SYSTEM_PROMPT = (
|
| 79 |
+
"You are Sync AI, a smart and concise assistant. "
|
| 80 |
+
"When the user provides a transcript or message, summarise the key points "
|
| 81 |
+
"from the conversation so far in 2-3 sentences. "
|
| 82 |
+
"Then ask exactly 2 relevant follow-up questions to the user "
|
| 83 |
+
"to help deepen the discussion or clarify important details."
|
| 84 |
+
)
|
| 85 |
+
|
| 86 |
+
CHAT_SYSTEM_PROMPT = (
|
| 87 |
+
"You are Sync AI, a knowledgeable and friendly conversational assistant. "
|
| 88 |
+
"You have access to ALL the AI-generated summaries from the user's audio "
|
| 89 |
+
"transcription session. Use these summaries as your primary knowledge base "
|
| 90 |
+
"to answer the user's questions accurately. "
|
| 91 |
+
"If the user asks about something covered in the summaries, reference the "
|
| 92 |
+
"relevant summary details in your answer. "
|
| 93 |
+
"Keep answers clear and concise (2-4 sentences) unless the user asks for detail. "
|
| 94 |
+
"Be natural and conversational."
|
| 95 |
+
)
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
async def run_llm_on_transcript(full_transcript: str):
|
| 99 |
+
"""
|
| 100 |
+
Send the buffered transcript to Gemini via LangChain.
|
| 101 |
+
Uses tuple-based messages and run_in_executor (same as new_with_llmv1.py).
|
| 102 |
+
Streams the response back to browser clients.
|
| 103 |
+
Each call is independent β previous summaries are NOT carried forward.
|
| 104 |
+
"""
|
| 105 |
+
if not full_transcript.strip():
|
| 106 |
+
await broadcast("__LLM_ERROR__:No transcript to process.")
|
| 107 |
+
return
|
| 108 |
+
|
| 109 |
+
await broadcast("__LLM_START__")
|
| 110 |
+
print(f"[LLM] Processing transcript ({len(full_transcript)} chars)...")
|
| 111 |
+
|
| 112 |
+
# Clear conversation history so each sync is independent (no appending)
|
| 113 |
+
conversation_history.clear()
|
| 114 |
+
|
| 115 |
+
# Build message list using tuples: (role, content)
|
| 116 |
+
messages = [("system", LLM_SYSTEM_PROMPT)]
|
| 117 |
+
|
| 118 |
+
# Add current transcript as the only user turn (no previous turns)
|
| 119 |
+
messages.append(("human", full_transcript))
|
| 120 |
+
|
| 121 |
+
try:
|
| 122 |
+
# Call Gemini in a thread (same pattern as new_with_llmv1.py)
|
| 123 |
+
loop = asyncio.get_event_loop()
|
| 124 |
+
ai_msg = await loop.run_in_executor(None, llm.invoke, messages)
|
| 125 |
+
full_response = ai_msg.content
|
| 126 |
+
|
| 127 |
+
# Send the full response to browser
|
| 128 |
+
await broadcast(f"__LLM_TOKEN__:{full_response}")
|
| 129 |
+
|
| 130 |
+
# Save this summary so chat can reference it later
|
| 131 |
+
summary_history.append(full_response)
|
| 132 |
+
|
| 133 |
+
# Clear the transcript buffer so next sync only gets NEW lines
|
| 134 |
+
transcript_buffer.clear()
|
| 135 |
+
|
| 136 |
+
await broadcast("__LLM_DONE__")
|
| 137 |
+
print(f"[LLM] Response: {full_response[:100]}...")
|
| 138 |
+
|
| 139 |
+
except Exception as e:
|
| 140 |
+
err = str(e)
|
| 141 |
+
print(f"[LLM] Error: {err}")
|
| 142 |
+
await broadcast(f"__LLM_ERROR__:{err}")
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
async def run_chat_response(user_message: str):
|
| 146 |
+
"""
|
| 147 |
+
Handle a chat message from the user.
|
| 148 |
+
Uses chat_llm (separate Gemini instance) with ALL generated summaries
|
| 149 |
+
as context so the user can ask questions about any summary.
|
| 150 |
+
Runs in parallel with STT β does not block audio processing.
|
| 151 |
+
"""
|
| 152 |
+
if not user_message.strip():
|
| 153 |
+
await broadcast("__CHAT_ERROR__:Empty message.")
|
| 154 |
+
return
|
| 155 |
+
|
| 156 |
+
await broadcast("__CHAT_START__")
|
| 157 |
+
print(f"[Chat] User: {user_message[:80]}...")
|
| 158 |
+
|
| 159 |
+
# Build message list with chat system prompt
|
| 160 |
+
messages = [("system", CHAT_SYSTEM_PROMPT)]
|
| 161 |
+
|
| 162 |
+
# Inject ALL generated summaries as context
|
| 163 |
+
if summary_history:
|
| 164 |
+
summaries_context = "\n\n".join(
|
| 165 |
+
f"--- Summary {i+1} ---\n{s}" for i, s in enumerate(summary_history)
|
| 166 |
+
)
|
| 167 |
+
messages.append((
|
| 168 |
+
"system",
|
| 169 |
+
f"Here are all the AI-generated summaries from this session "
|
| 170 |
+
f"({len(summary_history)} total). Use these to answer the user's "
|
| 171 |
+
f"questions:\n\n{summaries_context}"
|
| 172 |
+
))
|
| 173 |
+
|
| 174 |
+
# Include current (unsummarized) transcript if available
|
| 175 |
+
if transcript_buffer:
|
| 176 |
+
transcript_context = " ".join(transcript_buffer)
|
| 177 |
+
messages.append((
|
| 178 |
+
"system",
|
| 179 |
+
f"Current live transcript (not yet summarized): {transcript_context}"
|
| 180 |
+
))
|
| 181 |
+
|
| 182 |
+
# Replay previous chat turns
|
| 183 |
+
for role, content in chat_history:
|
| 184 |
+
messages.append((role, content))
|
| 185 |
+
|
| 186 |
+
# Add current user message
|
| 187 |
+
messages.append(("human", user_message))
|
| 188 |
+
chat_history.append(("human", user_message))
|
| 189 |
+
|
| 190 |
+
try:
|
| 191 |
+
loop = asyncio.get_event_loop()
|
| 192 |
+
ai_msg = await loop.run_in_executor(None, chat_llm.invoke, messages)
|
| 193 |
+
|
| 194 |
+
full_response = ai_msg.content
|
| 195 |
+
|
| 196 |
+
await broadcast(f"__CHAT_TOKEN__:{full_response}")
|
| 197 |
+
chat_history.append(("ai", full_response))
|
| 198 |
+
await broadcast("__CHAT_DONE__")
|
| 199 |
+
print(f"[Chat] Sync AI: {full_response[:100]}...")
|
| 200 |
+
|
| 201 |
+
except Exception as e:
|
| 202 |
+
err = str(e)
|
| 203 |
+
print(f"[Chat] Error: {err}")
|
| 204 |
+
await broadcast(f"__CHAT_ERROR__:{err}")
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
# βββ Helpers βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 208 |
+
|
| 209 |
+
async def broadcast(message: str):
|
| 210 |
+
"""Send a message to every connected browser client."""
|
| 211 |
+
disconnected = []
|
| 212 |
+
for ws in browser_clients:
|
| 213 |
+
try:
|
| 214 |
+
await ws.send_text(message)
|
| 215 |
+
except Exception:
|
| 216 |
+
disconnected.append(ws)
|
| 217 |
+
for ws in disconnected:
|
| 218 |
+
browser_clients.remove(ws)
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
# βββ FastAPI lifecycle βββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 222 |
+
|
| 223 |
+
@asynccontextmanager
|
| 224 |
+
async def lifespan(app: FastAPI):
|
| 225 |
+
print("[FastAPI] Server ready")
|
| 226 |
+
yield
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
app = FastAPI(lifespan=lifespan)
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
# βββ WebSocket: browser clients βββββββββββββββββββββββββββββββββββββββββββββ
|
| 233 |
+
|
| 234 |
+
@app.websocket("/ws")
|
| 235 |
+
async def websocket_browser(ws: WebSocket):
|
| 236 |
+
await ws.accept()
|
| 237 |
+
browser_clients.append(ws)
|
| 238 |
+
try:
|
| 239 |
+
while True:
|
| 240 |
+
msg = await ws.receive_text()
|
| 241 |
+
# Browser sends "PROCESS" when user clicks Stop/Q
|
| 242 |
+
if msg == "PROCESS":
|
| 243 |
+
full_text = " ".join(transcript_buffer)
|
| 244 |
+
asyncio.create_task(run_llm_on_transcript(full_text))
|
| 245 |
+
# Browser sends chat message as "CHAT:message"
|
| 246 |
+
elif msg.startswith("CHAT:"):
|
| 247 |
+
user_msg = msg[5:]
|
| 248 |
+
asyncio.create_task(run_chat_response(user_msg))
|
| 249 |
+
# Browser sends "CLEAR_HISTORY" to reset conversation
|
| 250 |
+
elif msg == "CLEAR_HISTORY":
|
| 251 |
+
conversation_history.clear()
|
| 252 |
+
chat_history.clear()
|
| 253 |
+
transcript_buffer.clear()
|
| 254 |
+
summary_history.clear()
|
| 255 |
+
await broadcast("__STATUS__:history_cleared")
|
| 256 |
+
except WebSocketDisconnect:
|
| 257 |
+
if ws in browser_clients:
|
| 258 |
+
browser_clients.remove(ws)
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
# βββ WebSocket: ESP32 audio β Deepgram (fully async, no threads) ββββββββββββ
|
| 262 |
+
|
| 263 |
+
@app.websocket("/ws/audio")
|
| 264 |
+
async def websocket_audio(esp_ws: WebSocket):
|
| 265 |
+
await esp_ws.accept()
|
| 266 |
+
await broadcast("__STATUS__:esp32_connected")
|
| 267 |
+
print("[ESP32] Connected via WebSocket")
|
| 268 |
+
|
| 269 |
+
# Clear buffer for new session
|
| 270 |
+
transcript_buffer.clear()
|
| 271 |
+
|
| 272 |
+
dg_ws = None
|
| 273 |
+
recv_task = None
|
| 274 |
+
chunks = 0
|
| 275 |
+
try:
|
| 276 |
+
headers = {"Authorization": f"Token {DEEPGRAM_API_KEY}"}
|
| 277 |
+
print(f"[Deepgram] Connecting to: {DEEPGRAM_URL[:80]}...")
|
| 278 |
+
print(f"[Deepgram] API key: {DEEPGRAM_API_KEY[:8]}...{DEEPGRAM_API_KEY[-4:]}")
|
| 279 |
+
dg_ws = await websockets.connect(
|
| 280 |
+
DEEPGRAM_URL,
|
| 281 |
+
additional_headers=headers,
|
| 282 |
+
ping_interval=20,
|
| 283 |
+
ping_timeout=20,
|
| 284 |
+
close_timeout=5,
|
| 285 |
+
)
|
| 286 |
+
await broadcast("__STATUS__:deepgram_connected")
|
| 287 |
+
print(f"[Deepgram] Connected (state: open={dg_ws.protocol.state if hasattr(dg_ws, 'protocol') else 'n/a'})")
|
| 288 |
+
|
| 289 |
+
async def receive_transcripts():
|
| 290 |
+
"""Receive and process Deepgram transcript messages."""
|
| 291 |
+
nonlocal dg_ws
|
| 292 |
+
print("[Deepgram] Receive task started β waiting for messages...")
|
| 293 |
+
msg_count = 0
|
| 294 |
+
try:
|
| 295 |
+
while True:
|
| 296 |
+
try:
|
| 297 |
+
msg = await dg_ws.recv()
|
| 298 |
+
except websockets.ConnectionClosed as e:
|
| 299 |
+
print(f"[Deepgram] Connection closed during recv: code={e.code} reason={e.reason}")
|
| 300 |
+
await broadcast(f"__DEBUG__:[DG] Closed: {e.code} {e.reason}")
|
| 301 |
+
break
|
| 302 |
+
|
| 303 |
+
msg_count += 1
|
| 304 |
+
# Log first few raw messages for debugging
|
| 305 |
+
if msg_count <= 3:
|
| 306 |
+
raw_preview = str(msg)[:300] if isinstance(msg, str) else f"<bytes len={len(msg)}>"
|
| 307 |
+
print(f"[Deepgram] Raw msg #{msg_count}: {raw_preview}")
|
| 308 |
+
|
| 309 |
+
try:
|
| 310 |
+
data = json.loads(msg)
|
| 311 |
+
except (json.JSONDecodeError, TypeError) as je:
|
| 312 |
+
print(f"[Deepgram] JSON parse error: {je} β raw: {str(msg)[:200]}")
|
| 313 |
+
continue
|
| 314 |
+
|
| 315 |
+
msg_type = data.get("type", "")
|
| 316 |
+
|
| 317 |
+
if msg_type != "Results":
|
| 318 |
+
print(f"[Deepgram] Event: {msg_type} β {str(data)[:200]}")
|
| 319 |
+
await broadcast(f"__DEBUG__:[DG] {msg_type}: {str(data)[:150]}")
|
| 320 |
+
continue
|
| 321 |
+
|
| 322 |
+
channel = data.get("channel")
|
| 323 |
+
if not isinstance(channel, dict):
|
| 324 |
+
continue
|
| 325 |
+
|
| 326 |
+
alternatives = channel.get("alternatives", [])
|
| 327 |
+
if not alternatives:
|
| 328 |
+
continue
|
| 329 |
+
|
| 330 |
+
transcript = alternatives[0].get("transcript", "")
|
| 331 |
+
if transcript:
|
| 332 |
+
is_final = data.get("is_final", False)
|
| 333 |
+
prefix = "FINAL" if is_final else "INTERIM"
|
| 334 |
+
await broadcast(f"__{prefix}__:{transcript}")
|
| 335 |
+
print(f"[{prefix}] {transcript}")
|
| 336 |
+
|
| 337 |
+
# Buffer only final lines for LLM
|
| 338 |
+
if is_final:
|
| 339 |
+
transcript_buffer.append(transcript)
|
| 340 |
+
|
| 341 |
+
except asyncio.CancelledError:
|
| 342 |
+
print(f"[Deepgram] Receive task cancelled after {msg_count} messages")
|
| 343 |
+
except Exception as e:
|
| 344 |
+
import traceback
|
| 345 |
+
print(f"[Deepgram] Receive error: {type(e).__name__}: {e}")
|
| 346 |
+
traceback.print_exc()
|
| 347 |
+
await broadcast(f"__DEBUG__:[DG] Recv error: {e}")
|
| 348 |
+
|
| 349 |
+
def task_exception_callback(task: asyncio.Task):
|
| 350 |
+
"""Catch any unhandled exception from the receive task."""
|
| 351 |
+
if task.cancelled():
|
| 352 |
+
return
|
| 353 |
+
exc = task.exception()
|
| 354 |
+
if exc:
|
| 355 |
+
print(f"[Deepgram] TASK EXCEPTION: {type(exc).__name__}: {exc}")
|
| 356 |
+
|
| 357 |
+
recv_task = asyncio.create_task(receive_transcripts())
|
| 358 |
+
recv_task.add_done_callback(task_exception_callback)
|
| 359 |
+
|
| 360 |
+
while True:
|
| 361 |
+
audio = await esp_ws.receive_bytes()
|
| 362 |
+
try:
|
| 363 |
+
await dg_ws.send(audio)
|
| 364 |
+
except Exception as send_err:
|
| 365 |
+
print(f"[Deepgram] Send failed: {type(send_err).__name__}: {send_err}")
|
| 366 |
+
print("[Deepgram] Connection lost, reconnecting...")
|
| 367 |
+
try:
|
| 368 |
+
dg_ws = await websockets.connect(
|
| 369 |
+
DEEPGRAM_URL,
|
| 370 |
+
additional_headers=headers,
|
| 371 |
+
ping_interval=20,
|
| 372 |
+
ping_timeout=20,
|
| 373 |
+
close_timeout=5,
|
| 374 |
+
)
|
| 375 |
+
recv_task.cancel()
|
| 376 |
+
recv_task = asyncio.create_task(receive_transcripts())
|
| 377 |
+
recv_task.add_done_callback(task_exception_callback)
|
| 378 |
+
await dg_ws.send(audio)
|
| 379 |
+
print("[Deepgram] Reconnected!")
|
| 380 |
+
except Exception as re_err:
|
| 381 |
+
print(f"[Deepgram] Reconnect failed: {re_err}")
|
| 382 |
+
break
|
| 383 |
+
|
| 384 |
+
chunks += 1
|
| 385 |
+
if chunks == 1:
|
| 386 |
+
print(f"[Audio] First chunk: {len(audio)} bytes")
|
| 387 |
+
if chunks % 500 == 0:
|
| 388 |
+
print(f"[Audio] Forwarded {chunks} chunks")
|
| 389 |
+
# Check if receive task is still alive
|
| 390 |
+
if recv_task.done():
|
| 391 |
+
print("[Deepgram] WARNING: Receive task died! Restarting...")
|
| 392 |
+
recv_task = asyncio.create_task(receive_transcripts())
|
| 393 |
+
recv_task.add_done_callback(task_exception_callback)
|
| 394 |
+
|
| 395 |
+
except WebSocketDisconnect:
|
| 396 |
+
print(f"[ESP32] Disconnected after {chunks} chunks")
|
| 397 |
+
except Exception as e:
|
| 398 |
+
import traceback
|
| 399 |
+
print(f"[Error] {type(e).__name__}: {e}")
|
| 400 |
+
traceback.print_exc()
|
| 401 |
+
finally:
|
| 402 |
+
if recv_task and not recv_task.done():
|
| 403 |
+
recv_task.cancel()
|
| 404 |
+
try:
|
| 405 |
+
if dg_ws:
|
| 406 |
+
await dg_ws.close()
|
| 407 |
+
except Exception:
|
| 408 |
+
pass
|
| 409 |
+
await broadcast("__STATUS__:esp32_disconnected")
|
| 410 |
+
await broadcast("__STATUS__:deepgram_disconnected")
|
| 411 |
+
print("[Cleanup] Session ended")
|
| 412 |
+
|
| 413 |
+
|
| 414 |
+
# βββ Serve the single-page UI βββββββββββββββββββββββββββββββββββββββββββββββ
|
| 415 |
+
|
| 416 |
+
PAGE_HTML = """\
|
| 417 |
+
<!DOCTYPE html>
|
| 418 |
+
<html lang="en">
|
| 419 |
+
<head>
|
| 420 |
+
<meta charset="utf-8" />
|
| 421 |
+
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
| 422 |
+
<title>SyncScribe</title>
|
| 423 |
+
<link href="https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:wght@400;500;600&family=Source+Serif+4:ital,opsz,wght@0,8..60,400;0,8..60,600;1,8..60,400&display=swap" rel="stylesheet">
|
| 424 |
+
<style>
|
| 425 |
+
:root {
|
| 426 |
+
--bg: #0c0e12;
|
| 427 |
+
--surface: #14171d;
|
| 428 |
+
--border: #23272f;
|
| 429 |
+
--text: #d4d8e0;
|
| 430 |
+
--muted: #6b7280;
|
| 431 |
+
--accent: #34d399;
|
| 432 |
+
--accent-dim:#1a7a52;
|
| 433 |
+
--danger: #f87171;
|
| 434 |
+
--warn: #fbbf24;
|
| 435 |
+
--llm: #818cf8;
|
| 436 |
+
--llm-dim: #312e81;
|
| 437 |
+
}
|
| 438 |
+
|
| 439 |
+
* { margin: 0; padding: 0; box-sizing: border-box; }
|
| 440 |
+
|
| 441 |
+
body {
|
| 442 |
+
background: var(--bg);
|
| 443 |
+
color: var(--text);
|
| 444 |
+
font-family: 'Source Serif 4', Georgia, serif;
|
| 445 |
+
min-height: 100vh;
|
| 446 |
+
display: flex;
|
| 447 |
+
flex-direction: column;
|
| 448 |
+
}
|
| 449 |
+
|
| 450 |
+
/* ββ Split layout wrapper βββββββββββββββββββββ */
|
| 451 |
+
.app-body {
|
| 452 |
+
display: flex;
|
| 453 |
+
flex: 1;
|
| 454 |
+
overflow: hidden;
|
| 455 |
+
height: calc(100vh - 80px);
|
| 456 |
+
}
|
| 457 |
+
|
| 458 |
+
/* ββ Header (glassmorphism) ββββββββββββββββββββ */
|
| 459 |
+
header {
|
| 460 |
+
padding: 28px 32px 20px;
|
| 461 |
+
display: flex;
|
| 462 |
+
align-items: center;
|
| 463 |
+
justify-content: space-between;
|
| 464 |
+
flex-wrap: wrap;
|
| 465 |
+
gap: 12px;
|
| 466 |
+
position: sticky;
|
| 467 |
+
top: 0;
|
| 468 |
+
z-index: 100;
|
| 469 |
+
/* Glass background */
|
| 470 |
+
background: rgba(20, 23, 29, 0.55);
|
| 471 |
+
backdrop-filter: blur(14px);
|
| 472 |
+
-webkit-backdrop-filter: blur(14px);
|
| 473 |
+
/* Glass border & glow */
|
| 474 |
+
border: 1px solid rgba(255, 255, 255, 0.08);
|
| 475 |
+
border-top: none;
|
| 476 |
+
box-shadow:
|
| 477 |
+
0 8px 32px rgba(0, 0, 0, 0.25),
|
| 478 |
+
inset 0 1px 0 rgba(255, 255, 255, 0.12),
|
| 479 |
+
inset 0 -1px 0 rgba(255, 255, 255, 0.04),
|
| 480 |
+
inset 0 0 12px 2px rgba(255, 255, 255, 0.03);
|
| 481 |
+
}
|
| 482 |
+
|
| 483 |
+
/* Top edge light streak */
|
| 484 |
+
header::before {
|
| 485 |
+
content: '';
|
| 486 |
+
position: absolute;
|
| 487 |
+
top: 0;
|
| 488 |
+
left: 0;
|
| 489 |
+
right: 0;
|
| 490 |
+
height: 1px;
|
| 491 |
+
background: linear-gradient(
|
| 492 |
+
90deg,
|
| 493 |
+
transparent 5%,
|
| 494 |
+
rgba(255, 255, 255, 0.35),
|
| 495 |
+
rgba(52, 211, 153, 0.25),
|
| 496 |
+
rgba(255, 255, 255, 0.35),
|
| 497 |
+
transparent 95%
|
| 498 |
+
);
|
| 499 |
+
}
|
| 500 |
+
|
| 501 |
+
/* Left edge light streak */
|
| 502 |
+
header::after {
|
| 503 |
+
content: '';
|
| 504 |
+
position: absolute;
|
| 505 |
+
top: 0;
|
| 506 |
+
left: 0;
|
| 507 |
+
width: 1px;
|
| 508 |
+
height: 100%;
|
| 509 |
+
background: linear-gradient(
|
| 510 |
+
180deg,
|
| 511 |
+
rgba(255, 255, 255, 0.3),
|
| 512 |
+
transparent 50%,
|
| 513 |
+
rgba(255, 255, 255, 0.08)
|
| 514 |
+
);
|
| 515 |
+
}
|
| 516 |
+
|
| 517 |
+
header h1 {
|
| 518 |
+
font-family: 'IBM Plex Mono', monospace;
|
| 519 |
+
font-weight: 600;
|
| 520 |
+
font-size: 18px;
|
| 521 |
+
letter-spacing: -0.02em;
|
| 522 |
+
color: var(--text);
|
| 523 |
+
}
|
| 524 |
+
|
| 525 |
+
.status-row {
|
| 526 |
+
display: flex;
|
| 527 |
+
gap: 20px;
|
| 528 |
+
font-family: 'IBM Plex Mono', monospace;
|
| 529 |
+
font-size: 12px;
|
| 530 |
+
color: var(--muted);
|
| 531 |
+
}
|
| 532 |
+
|
| 533 |
+
.status-item {
|
| 534 |
+
display: flex;
|
| 535 |
+
align-items: center;
|
| 536 |
+
gap: 6px;
|
| 537 |
+
}
|
| 538 |
+
|
| 539 |
+
.dot {
|
| 540 |
+
width: 8px; height: 8px;
|
| 541 |
+
border-radius: 50%;
|
| 542 |
+
background: var(--muted);
|
| 543 |
+
flex-shrink: 0;
|
| 544 |
+
transition: background 0.3s;
|
| 545 |
+
}
|
| 546 |
+
.dot.ok { background: var(--accent); box-shadow: 0 0 6px var(--accent-dim); }
|
| 547 |
+
.dot.warn { background: var(--warn); }
|
| 548 |
+
.dot.err { background: var(--danger); }
|
| 549 |
+
.dot.llm { background: var(--llm); box-shadow: 0 0 6px var(--llm-dim); }
|
| 550 |
+
|
| 551 |
+
/* ββ Transcript area (left panel) ββββββββββββββ */
|
| 552 |
+
main {
|
| 553 |
+
flex: 1;
|
| 554 |
+
padding: 24px 32px 80px;
|
| 555 |
+
overflow-y: auto;
|
| 556 |
+
min-width: 0;
|
| 557 |
+
}
|
| 558 |
+
|
| 559 |
+
#transcript-container {
|
| 560 |
+
max-width: 720px;
|
| 561 |
+
margin: 0 auto;
|
| 562 |
+
display: flex;
|
| 563 |
+
flex-direction: column;
|
| 564 |
+
gap: 6px;
|
| 565 |
+
}
|
| 566 |
+
|
| 567 |
+
.line {
|
| 568 |
+
padding: 8px 14px;
|
| 569 |
+
border-radius: 6px;
|
| 570 |
+
font-size: 17px;
|
| 571 |
+
line-height: 1.65;
|
| 572 |
+
animation: fadein 0.25s ease;
|
| 573 |
+
}
|
| 574 |
+
|
| 575 |
+
.line.final {
|
| 576 |
+
background: var(--surface);
|
| 577 |
+
border-left: 3px solid var(--accent-dim);
|
| 578 |
+
color: var(--text);
|
| 579 |
+
}
|
| 580 |
+
|
| 581 |
+
.line.interim {
|
| 582 |
+
background: transparent;
|
| 583 |
+
border-left: 3px solid var(--border);
|
| 584 |
+
color: var(--muted);
|
| 585 |
+
font-style: italic;
|
| 586 |
+
}
|
| 587 |
+
|
| 588 |
+
/* ββ LLM response block βββββββββββββββββββββββ */
|
| 589 |
+
.llm-block {
|
| 590 |
+
margin-top: 16px;
|
| 591 |
+
background: #1a1b2e;
|
| 592 |
+
border: 1px solid var(--llm-dim);
|
| 593 |
+
border-left: 3px solid var(--llm);
|
| 594 |
+
border-radius: 8px;
|
| 595 |
+
padding: 14px 18px;
|
| 596 |
+
font-size: 17px;
|
| 597 |
+
line-height: 1.75;
|
| 598 |
+
color: #c7d2fe;
|
| 599 |
+
animation: fadein 0.3s ease;
|
| 600 |
+
white-space: pre-wrap;
|
| 601 |
+
word-break: break-word;
|
| 602 |
+
}
|
| 603 |
+
|
| 604 |
+
.llm-block .llm-label {
|
| 605 |
+
font-family: 'IBM Plex Mono', monospace;
|
| 606 |
+
font-size: 10px;
|
| 607 |
+
color: var(--llm);
|
| 608 |
+
text-transform: uppercase;
|
| 609 |
+
letter-spacing: 0.08em;
|
| 610 |
+
margin-bottom: 8px;
|
| 611 |
+
display: flex;
|
| 612 |
+
align-items: center;
|
| 613 |
+
gap: 6px;
|
| 614 |
+
}
|
| 615 |
+
|
| 616 |
+
.llm-block .llm-label .pulse {
|
| 617 |
+
display: inline-block;
|
| 618 |
+
width: 6px; height: 6px;
|
| 619 |
+
border-radius: 50%;
|
| 620 |
+
background: var(--llm);
|
| 621 |
+
animation: blink 1s infinite;
|
| 622 |
+
}
|
| 623 |
+
.llm-block .llm-label .pulse.done {
|
| 624 |
+
animation: none;
|
| 625 |
+
background: var(--accent);
|
| 626 |
+
}
|
| 627 |
+
|
| 628 |
+
.divider {
|
| 629 |
+
display: flex;
|
| 630 |
+
align-items: center;
|
| 631 |
+
gap: 12px;
|
| 632 |
+
margin: 16px 0 10px;
|
| 633 |
+
color: var(--muted);
|
| 634 |
+
font-family: 'IBM Plex Mono', monospace;
|
| 635 |
+
font-size: 11px;
|
| 636 |
+
}
|
| 637 |
+
.divider::before, .divider::after {
|
| 638 |
+
content: '';
|
| 639 |
+
flex: 1;
|
| 640 |
+
height: 1px;
|
| 641 |
+
background: var(--border);
|
| 642 |
+
}
|
| 643 |
+
|
| 644 |
+
.line .ts {
|
| 645 |
+
font-family: 'IBM Plex Mono', monospace;
|
| 646 |
+
font-size: 11px;
|
| 647 |
+
color: var(--muted);
|
| 648 |
+
margin-right: 10px;
|
| 649 |
+
user-select: none;
|
| 650 |
+
}
|
| 651 |
+
|
| 652 |
+
.empty-state {
|
| 653 |
+
text-align: center;
|
| 654 |
+
margin-top: 20vh;
|
| 655 |
+
color: var(--muted);
|
| 656 |
+
font-size: 15px;
|
| 657 |
+
line-height: 2;
|
| 658 |
+
font-family: 'IBM Plex Mono', monospace;
|
| 659 |
+
}
|
| 660 |
+
|
| 661 |
+
/* ββ Footer bar (inside left panel) ββββββββββββ */
|
| 662 |
+
footer {
|
| 663 |
+
position: sticky;
|
| 664 |
+
bottom: 0;
|
| 665 |
+
padding: 12px 24px;
|
| 666 |
+
background: var(--surface);
|
| 667 |
+
border-top: 1px solid var(--border);
|
| 668 |
+
display: flex;
|
| 669 |
+
align-items: center;
|
| 670 |
+
justify-content: space-between;
|
| 671 |
+
font-family: 'IBM Plex Mono', monospace;
|
| 672 |
+
font-size: 12px;
|
| 673 |
+
color: var(--muted);
|
| 674 |
+
gap: 12px;
|
| 675 |
+
flex-wrap: wrap;
|
| 676 |
+
z-index: 10;
|
| 677 |
+
}
|
| 678 |
+
|
| 679 |
+
.footer-btns { display: flex; gap: 8px; }
|
| 680 |
+
|
| 681 |
+
footer button {
|
| 682 |
+
background: none;
|
| 683 |
+
border: 1px solid var(--border);
|
| 684 |
+
color: var(--muted);
|
| 685 |
+
font-family: inherit;
|
| 686 |
+
font-size: 12px;
|
| 687 |
+
padding: 6px 14px;
|
| 688 |
+
border-radius: 4px;
|
| 689 |
+
cursor: pointer;
|
| 690 |
+
transition: all 0.15s;
|
| 691 |
+
}
|
| 692 |
+
footer button:hover {
|
| 693 |
+
border-color: var(--accent-dim);
|
| 694 |
+
color: var(--text);
|
| 695 |
+
}
|
| 696 |
+
|
| 697 |
+
footer button#btn-stop {
|
| 698 |
+
border-color: var(--llm-dim);
|
| 699 |
+
color: var(--llm);
|
| 700 |
+
}
|
| 701 |
+
footer button#btn-stop:hover {
|
| 702 |
+
border-color: var(--llm);
|
| 703 |
+
background: #1e1b4b44;
|
| 704 |
+
}
|
| 705 |
+
footer button#btn-stop:disabled {
|
| 706 |
+
opacity: 0.35;
|
| 707 |
+
cursor: default;
|
| 708 |
+
pointer-events: none;
|
| 709 |
+
}
|
| 710 |
+
|
| 711 |
+
footer button#btn-clear-history {
|
| 712 |
+
border-color: var(--border);
|
| 713 |
+
color: var(--muted);
|
| 714 |
+
}
|
| 715 |
+
|
| 716 |
+
@keyframes fadein {
|
| 717 |
+
from { opacity: 0; transform: translateY(4px); }
|
| 718 |
+
to { opacity: 1; transform: translateY(0); }
|
| 719 |
+
}
|
| 720 |
+
|
| 721 |
+
@keyframes blink {
|
| 722 |
+
0%, 100% { opacity: 1; }
|
| 723 |
+
50% { opacity: 0.2; }
|
| 724 |
+
}
|
| 725 |
+
|
| 726 |
+
/* ββ Right chat panel ββββββββββββββββββββββββββ */
|
| 727 |
+
.chat-panel {
|
| 728 |
+
width: 380px;
|
| 729 |
+
min-width: 320px;
|
| 730 |
+
border-left: 1px solid var(--border);
|
| 731 |
+
display: flex;
|
| 732 |
+
flex-direction: column;
|
| 733 |
+
background: var(--surface);
|
| 734 |
+
}
|
| 735 |
+
/* On desktop, .hidden class is ignored β both panels always show */
|
| 736 |
+
@media (min-width: 769px) {
|
| 737 |
+
.chat-panel.hidden { display: flex !important; }
|
| 738 |
+
.left-panel.hidden { display: flex !important; }
|
| 739 |
+
}
|
| 740 |
+
|
| 741 |
+
.chat-panel-header {
|
| 742 |
+
padding: 14px 18px;
|
| 743 |
+
border-bottom: 1px solid var(--border);
|
| 744 |
+
font-family: 'IBM Plex Mono', monospace;
|
| 745 |
+
font-size: 13px;
|
| 746 |
+
font-weight: 600;
|
| 747 |
+
color: var(--llm);
|
| 748 |
+
display: flex;
|
| 749 |
+
align-items: center;
|
| 750 |
+
gap: 8px;
|
| 751 |
+
background: rgba(20, 23, 29, 0.55);
|
| 752 |
+
backdrop-filter: blur(14px);
|
| 753 |
+
-webkit-backdrop-filter: blur(14px);
|
| 754 |
+
}
|
| 755 |
+
.chat-panel-header .panel-dot {
|
| 756 |
+
width: 8px; height: 8px;
|
| 757 |
+
border-radius: 50%;
|
| 758 |
+
background: var(--llm);
|
| 759 |
+
box-shadow: 0 0 6px var(--llm-dim);
|
| 760 |
+
animation: blink 2s infinite;
|
| 761 |
+
}
|
| 762 |
+
|
| 763 |
+
.chat-messages {
|
| 764 |
+
flex: 1;
|
| 765 |
+
overflow-y: auto;
|
| 766 |
+
padding: 16px 14px;
|
| 767 |
+
display: flex;
|
| 768 |
+
flex-direction: column;
|
| 769 |
+
gap: 8px;
|
| 770 |
+
}
|
| 771 |
+
|
| 772 |
+
.chat-empty {
|
| 773 |
+
text-align: center;
|
| 774 |
+
margin-top: 30%;
|
| 775 |
+
color: var(--muted);
|
| 776 |
+
font-size: 13px;
|
| 777 |
+
line-height: 1.8;
|
| 778 |
+
font-family: 'IBM Plex Mono', monospace;
|
| 779 |
+
}
|
| 780 |
+
|
| 781 |
+
.chat-input-bar {
|
| 782 |
+
padding: 12px 14px;
|
| 783 |
+
border-top: 1px solid var(--border);
|
| 784 |
+
display: flex;
|
| 785 |
+
align-items: center;
|
| 786 |
+
gap: 8px;
|
| 787 |
+
background: var(--surface);
|
| 788 |
+
}
|
| 789 |
+
|
| 790 |
+
.chat-input-bar input {
|
| 791 |
+
flex: 1;
|
| 792 |
+
background: var(--bg);
|
| 793 |
+
border: 1px solid var(--border);
|
| 794 |
+
color: var(--text);
|
| 795 |
+
font-family: 'Source Serif 4', Georgia, serif;
|
| 796 |
+
font-size: 14px;
|
| 797 |
+
padding: 10px 12px;
|
| 798 |
+
border-radius: 8px;
|
| 799 |
+
outline: none;
|
| 800 |
+
transition: border-color 0.2s;
|
| 801 |
+
}
|
| 802 |
+
.chat-input-bar input:focus {
|
| 803 |
+
border-color: var(--llm);
|
| 804 |
+
}
|
| 805 |
+
.chat-input-bar input::placeholder {
|
| 806 |
+
color: var(--muted);
|
| 807 |
+
}
|
| 808 |
+
|
| 809 |
+
.chat-input-bar button {
|
| 810 |
+
background: var(--llm-dim);
|
| 811 |
+
border: 1px solid var(--llm);
|
| 812 |
+
color: var(--llm);
|
| 813 |
+
font-family: 'IBM Plex Mono', monospace;
|
| 814 |
+
font-size: 12px;
|
| 815 |
+
padding: 10px 16px;
|
| 816 |
+
border-radius: 8px;
|
| 817 |
+
cursor: pointer;
|
| 818 |
+
transition: all 0.15s;
|
| 819 |
+
white-space: nowrap;
|
| 820 |
+
}
|
| 821 |
+
.chat-input-bar button:hover {
|
| 822 |
+
background: #312e81;
|
| 823 |
+
color: #c7d2fe;
|
| 824 |
+
}
|
| 825 |
+
.chat-input-bar button:disabled {
|
| 826 |
+
opacity: 0.35;
|
| 827 |
+
cursor: default;
|
| 828 |
+
pointer-events: none;
|
| 829 |
+
}
|
| 830 |
+
|
| 831 |
+
/* ββ Chat bubbles βββββββββββββββββββββββββββββ */
|
| 832 |
+
.chat-user {
|
| 833 |
+
align-self: flex-end;
|
| 834 |
+
background: #1a3a2a;
|
| 835 |
+
border: 1px solid var(--accent-dim);
|
| 836 |
+
border-radius: 12px 12px 2px 12px;
|
| 837 |
+
padding: 10px 14px;
|
| 838 |
+
font-size: 14px;
|
| 839 |
+
line-height: 1.6;
|
| 840 |
+
color: var(--accent);
|
| 841 |
+
max-width: 90%;
|
| 842 |
+
animation: fadein 0.2s ease;
|
| 843 |
+
}
|
| 844 |
+
.chat-user .chat-label {
|
| 845 |
+
font-family: 'IBM Plex Mono', monospace;
|
| 846 |
+
font-size: 10px;
|
| 847 |
+
color: var(--accent-dim);
|
| 848 |
+
text-transform: uppercase;
|
| 849 |
+
letter-spacing: 0.08em;
|
| 850 |
+
margin-bottom: 4px;
|
| 851 |
+
}
|
| 852 |
+
|
| 853 |
+
.chat-ai {
|
| 854 |
+
align-self: flex-start;
|
| 855 |
+
background: #1a1b2e;
|
| 856 |
+
border: 1px solid var(--llm-dim);
|
| 857 |
+
border-radius: 12px 12px 12px 2px;
|
| 858 |
+
padding: 10px 14px;
|
| 859 |
+
font-size: 14px;
|
| 860 |
+
line-height: 1.6;
|
| 861 |
+
color: #c7d2fe;
|
| 862 |
+
max-width: 90%;
|
| 863 |
+
animation: fadein 0.25s ease;
|
| 864 |
+
white-space: pre-wrap;
|
| 865 |
+
word-break: break-word;
|
| 866 |
+
}
|
| 867 |
+
.chat-ai .chat-label {
|
| 868 |
+
font-family: 'IBM Plex Mono', monospace;
|
| 869 |
+
font-size: 10px;
|
| 870 |
+
color: var(--llm);
|
| 871 |
+
text-transform: uppercase;
|
| 872 |
+
letter-spacing: 0.08em;
|
| 873 |
+
margin-bottom: 4px;
|
| 874 |
+
display: flex;
|
| 875 |
+
align-items: center;
|
| 876 |
+
gap: 6px;
|
| 877 |
+
}
|
| 878 |
+
|
| 879 |
+
/* ββ Mobile tab bar ββββββββββββββββββββββββββββ */
|
| 880 |
+
.mobile-tabs {
|
| 881 |
+
display: none;
|
| 882 |
+
}
|
| 883 |
+
|
| 884 |
+
/* ββ Responsive: mobile layout βββββββββββββββ */
|
| 885 |
+
@media (max-width: 768px) {
|
| 886 |
+
header {
|
| 887 |
+
padding: 14px 16px 12px;
|
| 888 |
+
}
|
| 889 |
+
header h1 {
|
| 890 |
+
font-size: 15px;
|
| 891 |
+
}
|
| 892 |
+
.status-row {
|
| 893 |
+
gap: 10px;
|
| 894 |
+
font-size: 10px;
|
| 895 |
+
}
|
| 896 |
+
|
| 897 |
+
.mobile-tabs {
|
| 898 |
+
display: flex;
|
| 899 |
+
background: var(--surface);
|
| 900 |
+
border-bottom: 1px solid var(--border);
|
| 901 |
+
padding: 0;
|
| 902 |
+
z-index: 50;
|
| 903 |
+
}
|
| 904 |
+
.mobile-tabs button {
|
| 905 |
+
flex: 1;
|
| 906 |
+
padding: 10px 0;
|
| 907 |
+
background: none;
|
| 908 |
+
border: none;
|
| 909 |
+
border-bottom: 2px solid transparent;
|
| 910 |
+
color: var(--muted);
|
| 911 |
+
font-family: 'IBM Plex Mono', monospace;
|
| 912 |
+
font-size: 12px;
|
| 913 |
+
font-weight: 600;
|
| 914 |
+
cursor: pointer;
|
| 915 |
+
transition: all 0.2s;
|
| 916 |
+
}
|
| 917 |
+
.mobile-tabs button.active {
|
| 918 |
+
color: var(--llm);
|
| 919 |
+
border-bottom-color: var(--llm);
|
| 920 |
+
}
|
| 921 |
+
|
| 922 |
+
.app-body {
|
| 923 |
+
flex-direction: column;
|
| 924 |
+
height: calc(100vh - 100px);
|
| 925 |
+
}
|
| 926 |
+
|
| 927 |
+
.left-panel {
|
| 928 |
+
display: flex !important;
|
| 929 |
+
flex: 1;
|
| 930 |
+
min-height: 0;
|
| 931 |
+
}
|
| 932 |
+
.left-panel.hidden {
|
| 933 |
+
display: none !important;
|
| 934 |
+
}
|
| 935 |
+
|
| 936 |
+
.chat-panel {
|
| 937 |
+
display: flex !important;
|
| 938 |
+
width: 100% !important;
|
| 939 |
+
min-width: 0 !important;
|
| 940 |
+
flex: 1;
|
| 941 |
+
border-left: none !important;
|
| 942 |
+
border-top: 1px solid var(--border);
|
| 943 |
+
}
|
| 944 |
+
.chat-panel.hidden {
|
| 945 |
+
display: none !important;
|
| 946 |
+
}
|
| 947 |
+
|
| 948 |
+
main {
|
| 949 |
+
padding: 16px 16px 60px;
|
| 950 |
+
}
|
| 951 |
+
|
| 952 |
+
footer {
|
| 953 |
+
padding: 8px 12px;
|
| 954 |
+
}
|
| 955 |
+
.footer-btns {
|
| 956 |
+
gap: 4px;
|
| 957 |
+
}
|
| 958 |
+
footer button {
|
| 959 |
+
padding: 5px 10px;
|
| 960 |
+
font-size: 11px;
|
| 961 |
+
}
|
| 962 |
+
|
| 963 |
+
.chat-input-bar {
|
| 964 |
+
padding: 8px 10px;
|
| 965 |
+
}
|
| 966 |
+
|
| 967 |
+
#debug-panel {
|
| 968 |
+
display: none !important;
|
| 969 |
+
}
|
| 970 |
+
}
|
| 971 |
+
|
| 972 |
+
</style>
|
| 973 |
+
</head>
|
| 974 |
+
<body>
|
| 975 |
+
|
| 976 |
+
<header>
|
| 977 |
+
<h1>SyncScribe</h1>
|
| 978 |
+
<div class="status-row">
|
| 979 |
+
<div class="status-item">
|
| 980 |
+
<span class="dot" id="dot-ws"></span>
|
| 981 |
+
<span id="lbl-ws">WebSocket</span>
|
| 982 |
+
</div>
|
| 983 |
+
<div class="status-item">
|
| 984 |
+
<span class="dot" id="dot-dg"></span>
|
| 985 |
+
<span id="lbl-dg">Deepgram</span>
|
| 986 |
+
</div>
|
| 987 |
+
<div class="status-item">
|
| 988 |
+
<span class="dot" id="dot-esp"></span>
|
| 989 |
+
<span id="lbl-esp">ESP32</span>
|
| 990 |
+
</div>
|
| 991 |
+
<div class="status-item">
|
| 992 |
+
<span class="dot" id="dot-llm"></span>
|
| 993 |
+
<span id="lbl-llm">Sync AI</span>
|
| 994 |
+
</div>
|
| 995 |
+
</div>
|
| 996 |
+
</header>
|
| 997 |
+
|
| 998 |
+
<!-- Mobile tab bar -->
|
| 999 |
+
<div class="mobile-tabs" id="mobile-tabs">
|
| 1000 |
+
<button class="active" id="tab-transcript" onclick="switchTab('transcript')">π Transcript</button>
|
| 1001 |
+
<button id="tab-chat" onclick="switchTab('chat')">π¬ Chat</button>
|
| 1002 |
+
</div>
|
| 1003 |
+
|
| 1004 |
+
<div class="app-body">
|
| 1005 |
+
|
| 1006 |
+
<!-- Left panel: transcript + summaries -->
|
| 1007 |
+
<div class="left-panel" id="left-panel" style="flex:1;display:flex;flex-direction:column;overflow:hidden;">
|
| 1008 |
+
<main>
|
| 1009 |
+
<div id="transcript-container">
|
| 1010 |
+
<div class="empty-state" id="empty">
|
| 1011 |
+
Waiting for audio stream from ESP32 ...<br>
|
| 1012 |
+
ESP32 connects via WebSocket on /ws/audio
|
| 1013 |
+
</div>
|
| 1014 |
+
</div>
|
| 1015 |
+
</main>
|
| 1016 |
+
|
| 1017 |
+
<footer>
|
| 1018 |
+
<span id="line-count">0 lines</span>
|
| 1019 |
+
<div class="footer-btns">
|
| 1020 |
+
<button id="btn-stop" onclick="triggerLLM()" disabled
|
| 1021 |
+
title="Process transcript with Sync AI (or press Q)">
|
| 1022 |
+
⬑ Ask Sync AI
|
| 1023 |
+
</button>
|
| 1024 |
+
<button onclick="clearTranscript()">Clear</button>
|
| 1025 |
+
<button id="btn-clear-history" onclick="clearHistory()">Reset Chat</button>
|
| 1026 |
+
</div>
|
| 1027 |
+
</footer>
|
| 1028 |
+
</div>
|
| 1029 |
+
|
| 1030 |
+
<!-- Right panel: chat with Sync AI -->
|
| 1031 |
+
<aside class="chat-panel hidden" id="chat-panel">
|
| 1032 |
+
<div class="chat-panel-header">
|
| 1033 |
+
<span class="panel-dot"></span>
|
| 1034 |
+
Chat with Sync AI
|
| 1035 |
+
</div>
|
| 1036 |
+
<div class="chat-messages" id="chat-messages">
|
| 1037 |
+
<div class="chat-empty" id="chat-empty">
|
| 1038 |
+
Ask questions about your<br>AI-generated summaries...
|
| 1039 |
+
</div>
|
| 1040 |
+
</div>
|
| 1041 |
+
<div class="chat-input-bar">
|
| 1042 |
+
<input type="text" id="chat-input" placeholder="Ask about summaries..." autocomplete="off" />
|
| 1043 |
+
<button id="btn-chat-send" onclick="sendChat()">Send</button>
|
| 1044 |
+
</div>
|
| 1045 |
+
</aside>
|
| 1046 |
+
|
| 1047 |
+
</div>
|
| 1048 |
+
|
| 1049 |
+
<div id="debug-panel" style="position:fixed;bottom:50px;left:0;width:380px;max-height:30vh;overflow-y:auto;background:#0d0f14;border:1px solid #23272f;border-radius:0 8px 0 0;padding:10px;font-family:'IBM Plex Mono',monospace;font-size:11px;color:#6b7280;z-index:999;">
|
| 1050 |
+
<div style="display:flex;justify-content:space-between;margin-bottom:6px;">
|
| 1051 |
+
<span style="color:#818cf8;">⬑ Debug Log</span>
|
| 1052 |
+
<button onclick="document.getElementById('debug-panel').style.display='none'" style="background:none;border:none;color:#6b7280;cursor:pointer;font-size:11px;">β</button>
|
| 1053 |
+
</div>
|
| 1054 |
+
<div id="debug-log"></div>
|
| 1055 |
+
</div>
|
| 1056 |
+
|
| 1057 |
+
<script>
|
| 1058 |
+
const container = document.getElementById('transcript-container');
|
| 1059 |
+
const emptyEl = document.getElementById('empty');
|
| 1060 |
+
const countEl = document.getElementById('line-count');
|
| 1061 |
+
const dotWs = document.getElementById('dot-ws');
|
| 1062 |
+
const dotDg = document.getElementById('dot-dg');
|
| 1063 |
+
const dotEsp = document.getElementById('dot-esp');
|
| 1064 |
+
const dotLlm = document.getElementById('dot-llm');
|
| 1065 |
+
const btnStop = document.getElementById('btn-stop');
|
| 1066 |
+
const chatInput = document.getElementById('chat-input');
|
| 1067 |
+
const btnChatSend = document.getElementById('btn-chat-send');
|
| 1068 |
+
const chatContainer = document.getElementById('chat-messages');
|
| 1069 |
+
const chatEmptyEl = document.getElementById('chat-empty');
|
| 1070 |
+
const leftPanel = document.getElementById('left-panel');
|
| 1071 |
+
const chatPanel = document.getElementById('chat-panel');
|
| 1072 |
+
const tabTranscript = document.getElementById('tab-transcript');
|
| 1073 |
+
const tabChat = document.getElementById('tab-chat');
|
| 1074 |
+
|
| 1075 |
+
// ββ Mobile tab switching ββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 1076 |
+
function switchTab(tab) {
|
| 1077 |
+
if (tab === 'transcript') {
|
| 1078 |
+
leftPanel.classList.remove('hidden');
|
| 1079 |
+
chatPanel.classList.add('hidden');
|
| 1080 |
+
tabTranscript.classList.add('active');
|
| 1081 |
+
tabChat.classList.remove('active');
|
| 1082 |
+
} else {
|
| 1083 |
+
leftPanel.classList.add('hidden');
|
| 1084 |
+
chatPanel.classList.remove('hidden');
|
| 1085 |
+
tabTranscript.classList.remove('active');
|
| 1086 |
+
tabChat.classList.add('active');
|
| 1087 |
+
scrollChatBottom();
|
| 1088 |
+
}
|
| 1089 |
+
}
|
| 1090 |
+
|
| 1091 |
+
let interimEl = null;
|
| 1092 |
+
let lineCount = 0;
|
| 1093 |
+
let hasTranscript = false; // true once at least one FINAL line exists
|
| 1094 |
+
let llmStreaming = false;
|
| 1095 |
+
let chatStreaming = false;
|
| 1096 |
+
let currentLlmEl = null; // the active streaming .llm-block
|
| 1097 |
+
let currentChatAiEl = null; // the active streaming chat-ai bubble
|
| 1098 |
+
let ws;
|
| 1099 |
+
|
| 1100 |
+
// ββ Keyboard shortcut Q (skip when typing in chat input) ββββββββββββββββββ
|
| 1101 |
+
document.addEventListener('keydown', (e) => {
|
| 1102 |
+
if (document.activeElement === chatInput) return;
|
| 1103 |
+
if ((e.key === 'q' || e.key === 'Q') && !e.ctrlKey && !e.metaKey) {
|
| 1104 |
+
triggerLLM();
|
| 1105 |
+
}
|
| 1106 |
+
});
|
| 1107 |
+
|
| 1108 |
+
// ββ Enter key in chat input βββββββββββββββββββββββββββββββββββββββββββββββ
|
| 1109 |
+
chatInput.addEventListener('keydown', (e) => {
|
| 1110 |
+
if (e.key === 'Enter' && !e.shiftKey) {
|
| 1111 |
+
e.preventDefault();
|
| 1112 |
+
sendChat();
|
| 1113 |
+
}
|
| 1114 |
+
});
|
| 1115 |
+
|
| 1116 |
+
// ββ Helpers βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 1117 |
+
function now() {
|
| 1118 |
+
return new Date().toLocaleTimeString('en-GB', { hour12: false });
|
| 1119 |
+
}
|
| 1120 |
+
|
| 1121 |
+
function scrollBottom() {
|
| 1122 |
+
const mainEl = document.querySelector('main');
|
| 1123 |
+
if (mainEl) mainEl.scrollTo({ top: mainEl.scrollHeight, behavior: 'smooth' });
|
| 1124 |
+
}
|
| 1125 |
+
|
| 1126 |
+
function scrollChatBottom() {
|
| 1127 |
+
chatContainer.scrollTo({ top: chatContainer.scrollHeight, behavior: 'smooth' });
|
| 1128 |
+
}
|
| 1129 |
+
|
| 1130 |
+
function escHtml(s) {
|
| 1131 |
+
const d = document.createElement('div');
|
| 1132 |
+
d.textContent = s;
|
| 1133 |
+
return d.innerHTML;
|
| 1134 |
+
}
|
| 1135 |
+
|
| 1136 |
+
// ββ Transcript lines ββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 1137 |
+
function addFinalLine(text) {
|
| 1138 |
+
if (interimEl) { interimEl.remove(); interimEl = null; }
|
| 1139 |
+
emptyEl.style.display = 'none';
|
| 1140 |
+
const div = document.createElement('div');
|
| 1141 |
+
div.className = 'line final';
|
| 1142 |
+
div.innerHTML = '<span class="ts">' + now() + '</span>' + escHtml(text);
|
| 1143 |
+
container.appendChild(div);
|
| 1144 |
+
lineCount++;
|
| 1145 |
+
countEl.textContent = lineCount + ' line' + (lineCount === 1 ? '' : 's');
|
| 1146 |
+
hasTranscript = true;
|
| 1147 |
+
btnStop.disabled = false;
|
| 1148 |
+
scrollBottom();
|
| 1149 |
+
}
|
| 1150 |
+
|
| 1151 |
+
function showInterim(text) {
|
| 1152 |
+
if (!interimEl) {
|
| 1153 |
+
interimEl = document.createElement('div');
|
| 1154 |
+
interimEl.className = 'line interim';
|
| 1155 |
+
container.appendChild(interimEl);
|
| 1156 |
+
}
|
| 1157 |
+
interimEl.innerHTML = '<span class="ts">' + now() + '</span>' + escHtml(text);
|
| 1158 |
+
scrollBottom();
|
| 1159 |
+
}
|
| 1160 |
+
|
| 1161 |
+
// ββ LLM UI ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 1162 |
+
function triggerLLM() {
|
| 1163 |
+
if (!hasTranscript || llmStreaming) return;
|
| 1164 |
+
if (!ws || ws.readyState !== WebSocket.OPEN) return;
|
| 1165 |
+
ws.send('PROCESS');
|
| 1166 |
+
}
|
| 1167 |
+
|
| 1168 |
+
function clearHistory() {
|
| 1169 |
+
if (!ws || ws.readyState !== WebSocket.OPEN) return;
|
| 1170 |
+
ws.send('CLEAR_HISTORY');
|
| 1171 |
+
}
|
| 1172 |
+
|
| 1173 |
+
// ββ Chat functions ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 1174 |
+
function sendChat() {
|
| 1175 |
+
const msg = chatInput.value.trim();
|
| 1176 |
+
if (!msg || chatStreaming) return;
|
| 1177 |
+
if (!ws || ws.readyState !== WebSocket.OPEN) return;
|
| 1178 |
+
|
| 1179 |
+
// Hide empty state
|
| 1180 |
+
if (chatEmptyEl) chatEmptyEl.style.display = 'none';
|
| 1181 |
+
|
| 1182 |
+
// Show user bubble in chat panel
|
| 1183 |
+
const userBubble = document.createElement('div');
|
| 1184 |
+
userBubble.className = 'chat-user';
|
| 1185 |
+
userBubble.innerHTML = '<div class="chat-label">You</div>' + escHtml(msg);
|
| 1186 |
+
chatContainer.appendChild(userBubble);
|
| 1187 |
+
scrollChatBottom();
|
| 1188 |
+
|
| 1189 |
+
// Send to server
|
| 1190 |
+
ws.send('CHAT:' + msg);
|
| 1191 |
+
chatInput.value = '';
|
| 1192 |
+
chatInput.focus();
|
| 1193 |
+
}
|
| 1194 |
+
|
| 1195 |
+
function startChatBlock() {
|
| 1196 |
+
chatStreaming = true;
|
| 1197 |
+
dotLlm.className = 'dot llm';
|
| 1198 |
+
btnChatSend.disabled = true;
|
| 1199 |
+
|
| 1200 |
+
currentChatAiEl = document.createElement('div');
|
| 1201 |
+
currentChatAiEl.className = 'chat-ai';
|
| 1202 |
+
currentChatAiEl.innerHTML =
|
| 1203 |
+
'<div class="chat-label"><span class="pulse chat-pulse"></span> Sync AI</div>' +
|
| 1204 |
+
'<span class="chat-ai-text"></span>';
|
| 1205 |
+
chatContainer.appendChild(currentChatAiEl);
|
| 1206 |
+
scrollChatBottom();
|
| 1207 |
+
}
|
| 1208 |
+
|
| 1209 |
+
function appendChatToken(token) {
|
| 1210 |
+
if (!currentChatAiEl) return;
|
| 1211 |
+
const textEl = currentChatAiEl.querySelector('.chat-ai-text');
|
| 1212 |
+
if (textEl) {
|
| 1213 |
+
textEl.textContent += token;
|
| 1214 |
+
scrollChatBottom();
|
| 1215 |
+
}
|
| 1216 |
+
}
|
| 1217 |
+
|
| 1218 |
+
function finishChatBlock() {
|
| 1219 |
+
chatStreaming = false;
|
| 1220 |
+
dotLlm.className = 'dot ok';
|
| 1221 |
+
btnChatSend.disabled = false;
|
| 1222 |
+
|
| 1223 |
+
if (currentChatAiEl) {
|
| 1224 |
+
const pulse = currentChatAiEl.querySelector('.chat-pulse');
|
| 1225 |
+
if (pulse) pulse.className = 'pulse done chat-pulse';
|
| 1226 |
+
}
|
| 1227 |
+
|
| 1228 |
+
scrollChatBottom();
|
| 1229 |
+
chatInput.focus();
|
| 1230 |
+
}
|
| 1231 |
+
|
| 1232 |
+
function showChatError(msg) {
|
| 1233 |
+
chatStreaming = false;
|
| 1234 |
+
dotLlm.className = 'dot err';
|
| 1235 |
+
btnChatSend.disabled = false;
|
| 1236 |
+
if (currentChatAiEl) {
|
| 1237 |
+
const textEl = currentChatAiEl.querySelector('.chat-ai-text');
|
| 1238 |
+
if (textEl) textEl.textContent = 'β ' + msg;
|
| 1239 |
+
}
|
| 1240 |
+
}
|
| 1241 |
+
|
| 1242 |
+
|
| 1243 |
+
|
| 1244 |
+
function startLlmBlock() {
|
| 1245 |
+
llmStreaming = true;
|
| 1246 |
+
dotLlm.className = 'dot llm';
|
| 1247 |
+
btnStop.disabled = true;
|
| 1248 |
+
|
| 1249 |
+
// Add a divider
|
| 1250 |
+
const div = document.createElement('div');
|
| 1251 |
+
div.className = 'divider';
|
| 1252 |
+
div.textContent = 'Sync AI';
|
| 1253 |
+
container.appendChild(div);
|
| 1254 |
+
|
| 1255 |
+
// Create the LLM response block
|
| 1256 |
+
currentLlmEl = document.createElement('div');
|
| 1257 |
+
currentLlmEl.className = 'llm-block';
|
| 1258 |
+
currentLlmEl.innerHTML =
|
| 1259 |
+
'<div class="llm-label"><span class="pulse llm-pulse"></span> Sync AI is thinking...</div>' +
|
| 1260 |
+
'<span class="llm-text"></span>';
|
| 1261 |
+
container.appendChild(currentLlmEl);
|
| 1262 |
+
scrollBottom();
|
| 1263 |
+
}
|
| 1264 |
+
|
| 1265 |
+
function appendLlmToken(token) {
|
| 1266 |
+
if (!currentLlmEl) return;
|
| 1267 |
+
const textEl = currentLlmEl.querySelector('.llm-text');
|
| 1268 |
+
if (textEl) {
|
| 1269 |
+
textEl.textContent += token;
|
| 1270 |
+
scrollBottom();
|
| 1271 |
+
}
|
| 1272 |
+
}
|
| 1273 |
+
|
| 1274 |
+
function finishLlmBlock() {
|
| 1275 |
+
llmStreaming = false;
|
| 1276 |
+
dotLlm.className = 'dot ok';
|
| 1277 |
+
|
| 1278 |
+
const pulse = currentLlmEl && currentLlmEl.querySelector('.llm-pulse');
|
| 1279 |
+
if (pulse) pulse.className = 'pulse done llm-pulse';
|
| 1280 |
+
|
| 1281 |
+
const label = currentLlmEl && currentLlmEl.querySelector('.llm-label');
|
| 1282 |
+
if (label) label.innerHTML = '<span class="pulse done"></span> Sync AI';
|
| 1283 |
+
|
| 1284 |
+
btnStop.disabled = !hasTranscript;
|
| 1285 |
+
scrollBottom();
|
| 1286 |
+
|
| 1287 |
+
// Re-enable the button after a brief pause so user can ask follow-up
|
| 1288 |
+
setTimeout(() => {
|
| 1289 |
+
if (hasTranscript) btnStop.disabled = false;
|
| 1290 |
+
}, 500);
|
| 1291 |
+
}
|
| 1292 |
+
|
| 1293 |
+
function showLlmError(msg) {
|
| 1294 |
+
llmStreaming = false;
|
| 1295 |
+
dotLlm.className = 'dot err';
|
| 1296 |
+
if (currentLlmEl) {
|
| 1297 |
+
const textEl = currentLlmEl.querySelector('.llm-text');
|
| 1298 |
+
if (textEl) textEl.textContent = 'β ' + msg;
|
| 1299 |
+
const label = currentLlmEl.querySelector('.llm-label');
|
| 1300 |
+
if (label) label.innerHTML = '<span class="pulse done" style="background:var(--danger)"></span> Error';
|
| 1301 |
+
}
|
| 1302 |
+
btnStop.disabled = false;
|
| 1303 |
+
}
|
| 1304 |
+
|
| 1305 |
+
// ββ Clear functions βββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 1306 |
+
function clearTranscript() {
|
| 1307 |
+
container.querySelectorAll('.line, .llm-block, .divider').forEach(el => el.remove());
|
| 1308 |
+
interimEl = null;
|
| 1309 |
+
currentLlmEl = null;
|
| 1310 |
+
lineCount = 0;
|
| 1311 |
+
hasTranscript = false;
|
| 1312 |
+
countEl.textContent = '0 lines';
|
| 1313 |
+
emptyEl.style.display = '';
|
| 1314 |
+
btnStop.disabled = true;
|
| 1315 |
+
}
|
| 1316 |
+
|
| 1317 |
+
function clearChatPanel() {
|
| 1318 |
+
chatContainer.querySelectorAll('.chat-user, .chat-ai').forEach(el => el.remove());
|
| 1319 |
+
currentChatAiEl = null;
|
| 1320 |
+
if (chatEmptyEl) chatEmptyEl.style.display = '';
|
| 1321 |
+
}
|
| 1322 |
+
|
| 1323 |
+
// ββ WebSocket βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 1324 |
+
function connect() {
|
| 1325 |
+
const proto = location.protocol === 'https:' ? 'wss' : 'ws';
|
| 1326 |
+
ws = new WebSocket(proto + '://' + location.host + '/ws');
|
| 1327 |
+
|
| 1328 |
+
ws.onopen = () => {
|
| 1329 |
+
dotWs.className = 'dot ok';
|
| 1330 |
+
};
|
| 1331 |
+
|
| 1332 |
+
ws.onclose = () => {
|
| 1333 |
+
dotWs.className = 'dot err';
|
| 1334 |
+
setTimeout(connect, 2000);
|
| 1335 |
+
};
|
| 1336 |
+
|
| 1337 |
+
ws.onerror = () => {
|
| 1338 |
+
dotWs.className = 'dot err';
|
| 1339 |
+
};
|
| 1340 |
+
|
| 1341 |
+
ws.onmessage = (e) => {
|
| 1342 |
+
const msg = e.data;
|
| 1343 |
+
|
| 1344 |
+
// ββ Status updates βββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 1345 |
+
if (msg.startsWith('__STATUS__:')) {
|
| 1346 |
+
const status = msg.split(':')[1];
|
| 1347 |
+
if (status === 'deepgram_connected') dotDg.className = 'dot ok';
|
| 1348 |
+
if (status === 'deepgram_disconnected') dotDg.className = 'dot err';
|
| 1349 |
+
if (status === 'esp32_connected') dotEsp.className = 'dot ok';
|
| 1350 |
+
if (status === 'esp32_disconnected') dotEsp.className = 'dot err';
|
| 1351 |
+
if (status === 'history_cleared') {
|
| 1352 |
+
dotLlm.className = 'dot';
|
| 1353 |
+
clearChatPanel();
|
| 1354 |
+
console.log('[Chat] History cleared');
|
| 1355 |
+
}
|
| 1356 |
+
return;
|
| 1357 |
+
}
|
| 1358 |
+
|
| 1359 |
+
// ββ Transcript lines βββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 1360 |
+
if (msg.startsWith('__FINAL__:')) {
|
| 1361 |
+
addFinalLine(msg.substring('__FINAL__:'.length));
|
| 1362 |
+
return;
|
| 1363 |
+
}
|
| 1364 |
+
if (msg.startsWith('__INTERIM__:')) {
|
| 1365 |
+
showInterim(msg.substring('__INTERIM__:'.length));
|
| 1366 |
+
return;
|
| 1367 |
+
}
|
| 1368 |
+
|
| 1369 |
+
// ββ LLM events βββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 1370 |
+
if (msg === '__LLM_START__') {
|
| 1371 |
+
startLlmBlock();
|
| 1372 |
+
return;
|
| 1373 |
+
}
|
| 1374 |
+
if (msg.startsWith('__LLM_TOKEN__:')) {
|
| 1375 |
+
appendLlmToken(msg.substring('__LLM_TOKEN__:'.length));
|
| 1376 |
+
return;
|
| 1377 |
+
}
|
| 1378 |
+
if (msg === '__LLM_DONE__') {
|
| 1379 |
+
finishLlmBlock();
|
| 1380 |
+
return;
|
| 1381 |
+
}
|
| 1382 |
+
if (msg.startsWith('__LLM_ERROR__:')) {
|
| 1383 |
+
showLlmError(msg.substring('__LLM_ERROR__:'.length));
|
| 1384 |
+
return;
|
| 1385 |
+
}
|
| 1386 |
+
|
| 1387 |
+
// ββ Chat events βββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 1388 |
+
if (msg === '__CHAT_START__') {
|
| 1389 |
+
startChatBlock();
|
| 1390 |
+
return;
|
| 1391 |
+
}
|
| 1392 |
+
if (msg.startsWith('__CHAT_TOKEN__:')) {
|
| 1393 |
+
appendChatToken(msg.substring('__CHAT_TOKEN__:'.length));
|
| 1394 |
+
return;
|
| 1395 |
+
}
|
| 1396 |
+
if (msg === '__CHAT_DONE__') {
|
| 1397 |
+
finishChatBlock();
|
| 1398 |
+
return;
|
| 1399 |
+
}
|
| 1400 |
+
if (msg.startsWith('__CHAT_ERROR__:')) {
|
| 1401 |
+
showChatError(msg.substring('__CHAT_ERROR__:'.length));
|
| 1402 |
+
return;
|
| 1403 |
+
}
|
| 1404 |
+
|
| 1405 |
+
|
| 1406 |
+
// ββ Debug log βββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 1407 |
+
if (msg.startsWith('__DEBUG__:')) {
|
| 1408 |
+
const logEl = document.getElementById('debug-log');
|
| 1409 |
+
if (logEl) {
|
| 1410 |
+
const d = document.createElement('div');
|
| 1411 |
+
d.style.borderBottom = '1px solid #1a1d24';
|
| 1412 |
+
d.style.padding = '3px 0';
|
| 1413 |
+
d.style.wordBreak = 'break-all';
|
| 1414 |
+
d.textContent = new Date().toLocaleTimeString() + ' ' + msg.substring('__DEBUG__:'.length);
|
| 1415 |
+
logEl.appendChild(d);
|
| 1416 |
+
logEl.scrollTop = logEl.scrollHeight;
|
| 1417 |
+
}
|
| 1418 |
+
return;
|
| 1419 |
+
}
|
| 1420 |
+
};
|
| 1421 |
+
}
|
| 1422 |
+
|
| 1423 |
+
connect();
|
| 1424 |
+
</script>
|
| 1425 |
+
</body>
|
| 1426 |
+
</html>
|
| 1427 |
+
"""
|
| 1428 |
+
|
| 1429 |
+
|
| 1430 |
+
@app.get("/", response_class=HTMLResponse)
|
| 1431 |
+
async def index():
|
| 1432 |
+
return PAGE_HTML
|
| 1433 |
+
|
| 1434 |
+
|
| 1435 |
+
# βββ Entrypoint ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 1436 |
+
|
| 1437 |
+
if __name__ == "__main__":
|
| 1438 |
+
import uvicorn
|
| 1439 |
+
uvicorn.run(app, host=FASTAPI_HOST, port=FASTAPI_PORT)
|