package com.jovan.assistant.service import android.Manifest import android.annotation.SuppressLint import android.app.Notification import android.app.NotificationChannel import android.app.NotificationManager import android.app.PendingIntent import android.app.Service import android.content.Context import android.content.Intent import android.content.pm.PackageManager import android.content.pm.ServiceInfo import android.os.Build import android.os.Handler import android.os.IBinder import android.os.Looper import android.os.PowerManager import android.speech.RecognitionListener import android.speech.RecognizerIntent import android.speech.SpeechRecognizer import android.speech.tts.TextToSpeech import android.speech.tts.UtteranceProgressListener import androidx.core.app.NotificationCompat import androidx.core.content.ContextCompat import com.jovan.assistant.MainActivity import com.jovan.assistant.R import com.jovan.assistant.data.AssistantState import com.jovan.assistant.data.FirebaseRepo import com.jovan.assistant.data.Prefs import com.jovan.assistant.data.WakeStatus import com.jovan.assistant.security.SecurityGuard import java.util.Locale /** * Foreground service (type "microphone") that loops: listen for wake word -> say "Yes, tell me" -> * listen for command -> execute -> back to wake-word listening. * SpeechRecognizer must be used from the main thread, so everything here runs on the main Handler. */ class VoiceService : Service() { private enum class Mode { WAKE, COMMAND } private val main = Handler(Looper.getMainLooper()) private var recognizer: SpeechRecognizer? = null private var tts: TextToSpeech? = null private var ttsReady = false private var mode = Mode.WAKE private var running = false private var errorStreak = 0 private var wakeLock: PowerManager.WakeLock? = null private var removeWakeWordListener: (() -> Unit)? = null private var pendingAfterSpeech: (() -> Unit)? = null private val speechFallback = Runnable { finishSpeech() } override fun onBind(intent: Intent?): IBinder? = null @SuppressLint("WakelockTimeout") override fun onCreate() { super.onCreate() val micOk = ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED if (!micOk || FirebaseRepo.uid == null || !SpeechRecognizer.isRecognitionAvailable(this)) { startInForeground() // must still call startForeground after startForegroundService() stopSelf() return } startInForeground() running = true wakeLock = (getSystemService(Context.POWER_SERVICE) as PowerManager) .newWakeLock(PowerManager.PARTIAL_WAKE_LOCK, "jovan:voice").apply { acquire() } removeWakeWordListener = FirebaseRepo.observeWakeWord() FirebaseRepo.setDeviceStatus("online") FirebaseRepo.log("SERVICE", "Voice service started (wake word: \"${Prefs.wakeWord}\")") tts = TextToSpeech(this) { status -> ttsReady = status == TextToSpeech.SUCCESS if (ttsReady) { tts?.language = Locale.getDefault() tts?.setOnUtteranceProgressListener(object : UtteranceProgressListener() { override fun onStart(id: String?) {} override fun onDone(id: String?) { main.post { finishSpeech() } } @Deprecated("Deprecated in Java") override fun onError(id: String?) { main.post { finishSpeech() } } }) } } buildRecognizer() startListening(Mode.WAKE) } override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { if (intent?.action == ACTION_STOP) { stopSelf() return START_NOT_STICKY } return START_STICKY } // ---------------------------------------------------------------- listening loop private fun buildRecognizer() { recognizer?.destroy() recognizer = SpeechRecognizer.createSpeechRecognizer(this).apply { setRecognitionListener(listener) } } private fun startListening(m: Mode) { if (!running) return mode = m if (SecurityGuard.isBlocked()) { AssistantState.wakeStatus.value = WakeStatus.BLOCKED main.postDelayed({ startListening(Mode.WAKE) }, 5_000) return } AssistantState.wakeStatus.value = if (m == Mode.WAKE) WakeStatus.LISTENING_WAKE else WakeStatus.AWAITING_COMMAND val intent = Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply { putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM) putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 3) putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, false) putExtra(RecognizerIntent.EXTRA_PREFER_OFFLINE, true) putExtra(RecognizerIntent.EXTRA_CALLING_PACKAGE, packageName) } try { recognizer?.startListening(intent) } catch (e: Exception) { buildRecognizer() restartLater(1_000) } } private fun restartLater(delayMs: Long) { main.postDelayed({ startListening(Mode.WAKE) }, delayMs) } private val listener = object : RecognitionListener { override fun onReadyForSpeech(params: android.os.Bundle?) {} override fun onBeginningOfSpeech() {} override fun onRmsChanged(rmsdB: Float) {} override fun onBufferReceived(buffer: ByteArray?) {} override fun onEndOfSpeech() {} override fun onPartialResults(partialResults: android.os.Bundle?) {} override fun onEvent(eventType: Int, params: android.os.Bundle?) {} override fun onResults(results: android.os.Bundle?) { errorStreak = 0 val list = results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION).orEmpty() handleResults(list) } override fun onError(error: Int) { when (error) { SpeechRecognizer.ERROR_NO_MATCH, SpeechRecognizer.ERROR_SPEECH_TIMEOUT -> { errorStreak = 0 restartLater(200) } SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS -> { FirebaseRepo.log("ERROR", "Microphone permission lost", "error") stopSelf() } SpeechRecognizer.ERROR_RECOGNIZER_BUSY, SpeechRecognizer.ERROR_CLIENT -> { buildRecognizer() restartLater(1_000) } else -> { // network / server / audio errors: back off errorStreak++ if (errorStreak >= 10) { AssistantState.wakeStatus.value = WakeStatus.ERROR FirebaseRepo.log("ERROR", "Speech recognizer failing (code $error)", "error") errorStreak = 0 buildRecognizer() restartLater(30_000) } else { restartLater(1_000L * errorStreak) } } } } } private fun handleResults(candidates: List) { if (mode == Mode.WAKE) { val wake = Prefs.wakeWord for (text in candidates) { val rest = remainderAfterWake(text, wake) ?: continue FirebaseRepo.log("WAKE", "Wake word detected") if (rest.isBlank()) { speak("Yes, tell me") { startListening(Mode.COMMAND) } } else { runCommand(rest) } return } main.postDelayed({ startListening(Mode.WAKE) }, 150) } else { val text = candidates.firstOrNull().orEmpty() if (text.isBlank()) restartLater(150) else runCommand(text) } } private fun runCommand(text: String) { AssistantState.wakeStatus.value = WakeStatus.PROCESSING AssistantState.lastHeard.value = text if (FirebaseRepo.uid == null) { SecurityGuard.report("Command received while signed out: \"$text\"") speak("Please sign in to the app first.") { startListening(Mode.WAKE) } return } if (SecurityGuard.isBlocked()) { startListening(Mode.WAKE) return } val result = CommandProcessor.handle(this, text) FirebaseRepo.log("COMMAND", "\"$text\" -> ${result.reply}") if (SecurityGuard.isBlocked()) { // guard tripped during this command; stay quiet, screen is locked startListening(Mode.WAKE) return } speak(result.reply) { if (result.stopListening) stopSelf() else startListening(Mode.WAKE) } } // ---------------------------------------------------------------- wake-word matching /** Returns the text spoken after the wake word, or null if the wake word wasn't heard. */ private fun remainderAfterWake(text: String, wake: String): String? { val tokens = normalize(text).split(" ").filter { it.isNotEmpty() } val wakeTokens = normalize(wake).split(" ").filter { it.isNotEmpty() } if (wakeTokens.isEmpty()) return null for (i in 0..tokens.size - wakeTokens.size) { val match = wakeTokens.indices.all { j -> close(tokens[i + j], wakeTokens[j]) } if (match) return tokens.drop(i + wakeTokens.size).joinToString(" ") } return null } private fun normalize(s: String) = s.lowercase().replace(Regex("[^\\p{L}\\p{N} ]"), " ").replace(Regex("\\s+"), " ").trim() /** Tolerates one character of recognizer error for words of 5+ letters. */ private fun close(a: String, b: String): Boolean = a == b || (b.length >= 5 && levenshtein(a, b) <= 1) private fun levenshtein(a: String, b: String): Int { var prev = IntArray(b.length + 1) { it } for (i in 1..a.length) { val cur = IntArray(b.length + 1).also { it[0] = i } for (j in 1..b.length) { cur[j] = minOf(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + if (a[i - 1] == b[j - 1]) 0 else 1) } prev = cur } return prev[b.length] } // ---------------------------------------------------------------- speech output private fun speak(text: String, then: () -> Unit) { if (!ttsReady) { then(); return } pendingAfterSpeech = then main.removeCallbacks(speechFallback) main.postDelayed(speechFallback, 10_000) // in case the TTS engine never reports completion tts?.speak(text, TextToSpeech.QUEUE_FLUSH, null, "utt-${System.nanoTime()}") } private fun finishSpeech() { main.removeCallbacks(speechFallback) val cb = pendingAfterSpeech pendingAfterSpeech = null cb?.invoke() } // ---------------------------------------------------------------- notification & teardown private fun startInForeground() { val nm = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager nm.createNotificationChannel( NotificationChannel(CHANNEL_ID, "Voice assistant", NotificationManager.IMPORTANCE_LOW) ) val open = PendingIntent.getActivity( this, 0, Intent(this, MainActivity::class.java), PendingIntent.FLAG_IMMUTABLE ) val stop = PendingIntent.getService( this, 1, Intent(this, VoiceService::class.java).setAction(ACTION_STOP), PendingIntent.FLAG_IMMUTABLE ) val n: Notification = NotificationCompat.Builder(this, CHANNEL_ID) .setSmallIcon(R.drawable.ic_notification) .setContentTitle("AI Voice Assistant is listening") .setContentText("Say \"${Prefs.wakeWord}\" followed by a command") .setOngoing(true) .setContentIntent(open) .addAction(0, "Stop", stop) .build() if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { startForeground(NOTIF_ID, n, ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE) } else { startForeground(NOTIF_ID, n) } } override fun onDestroy() { running = false main.removeCallbacksAndMessages(null) recognizer?.destroy(); recognizer = null tts?.stop(); tts?.shutdown(); tts = null removeWakeWordListener?.invoke() if (wakeLock?.isHeld == true) wakeLock?.release() AssistantState.wakeStatus.value = WakeStatus.OFF FirebaseRepo.log("SERVICE", "Voice service stopped") FirebaseRepo.setDeviceStatus("offline") super.onDestroy() } companion object { const val ACTION_STOP = "com.jovan.assistant.STOP" private const val CHANNEL_ID = "voice_service" private const val NOTIF_ID = 42 fun start(ctx: Context) = ContextCompat.startForegroundService(ctx, Intent(ctx, VoiceService::class.java)) fun stop(ctx: Context) { ctx.stopService(Intent(ctx, VoiceService::class.java)) } } }