diff --git a/.github/workflows/android-build.yml b/.github/workflows/android-build.yml
new file mode 100644
index 00000000000..5724405aa09
--- /dev/null
+++ b/.github/workflows/android-build.yml
@@ -0,0 +1,41 @@
+
+name: Build Android Object Detection Demo
+
+on:
+ workflow_dispatch:
+ push:
+ paths:
+ - "lite/examples/object_detection/android/**"
+ - ".github/workflows/android-build.yml"
+
+permissions:
+ contents: read
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ - name: Set up Java
+ uses: actions/setup-java@v4
+ with:
+ distribution: temurin
+ java-version: "11"
+
+ - name: Build debug APK
+ working-directory: lite/examples/object_detection/android
+ run: |
+ chmod +x gradlew
+ ./gradlew assembleDebug --stacktrace
+
+ - name: Upload APK
+ if: success()
+ uses: actions/upload-artifact@v4
+ with:
+ name: blind-assist-debug-apk
+ path: lite/examples/object_detection/android/app/build/outputs/apk/debug/*.apk
+ if-no-files-found: error
+ retention-days: 7
diff --git a/lite/examples/object_detection/android/app/src/main/AndroidManifest.xml b/lite/examples/object_detection/android/app/src/main/AndroidManifest.xml
index a2e00b2b63b..51fbbe13108 100644
--- a/lite/examples/object_detection/android/app/src/main/AndroidManifest.xml
+++ b/lite/examples/object_detection/android/app/src/main/AndroidManifest.xml
@@ -1,18 +1,8 @@
-
+
-
-
-
+
-
+
+
+
+
+
+
+
+
+
-
-
-
+
-
-
diff --git a/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/MainActivity.kt b/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/MainActivity.kt
index f65924e40e1..54fd042cf59 100644
--- a/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/MainActivity.kt
+++ b/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/MainActivity.kt
@@ -1,44 +1,58 @@
+
/*
* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Licensed under the Apache License, Version 2.0.
*/
package org.tensorflow.lite.examples.objectdetection
import android.os.Build
import android.os.Bundle
+import android.speech.tts.TextToSpeech
import androidx.appcompat.app.AppCompatActivity
import org.tensorflow.lite.examples.objectdetection.databinding.ActivityMainBinding
+import java.util.Locale
-/**
- * Main entry point into our app. This app follows the single-activity pattern, and all
- * functionality is implemented in the form of fragments.
- */
class MainActivity : AppCompatActivity() {
private lateinit var activityMainBinding: ActivityMainBinding
+ private var textToSpeech: TextToSpeech? = null
+ private var ttsReady = false
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
activityMainBinding = ActivityMainBinding.inflate(layoutInflater)
setContentView(activityMainBinding.root)
+
+ textToSpeech = TextToSpeech(this) { status ->
+ if (status == TextToSpeech.SUCCESS) {
+ val result = textToSpeech?.setLanguage(Locale.US)
+ ttsReady = result != TextToSpeech.LANG_MISSING_DATA &&
+ result != TextToSpeech.LANG_NOT_SUPPORTED
+ }
+ }
+ }
+
+ private fun speakTestWarning() {
+ if (ttsReady) {
+ textToSpeech?.speak(
+ "Person ahead",
+ TextToSpeech.QUEUE_FLUSH,
+ null,
+ "test_warning"
+ )
+ }
+ }
+
+ override fun onDestroy() {
+ textToSpeech?.stop()
+ textToSpeech?.shutdown()
+ textToSpeech = null
+ super.onDestroy()
}
override fun onBackPressed() {
if (Build.VERSION.SDK_INT == Build.VERSION_CODES.Q) {
- // Workaround for Android Q memory leak issue in IRequestFinishCallback$Stub.
- // (https://issuetracker.google.com/issues/139738913)
finishAfterTransition()
} else {
super.onBackPressed()
diff --git a/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/ObjectDetectorHelper.kt b/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/ObjectDetectorHelper.kt
index 7fb4f0ef282..563c008fa4a 100644
--- a/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/ObjectDetectorHelper.kt
+++ b/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/ObjectDetectorHelper.kt
@@ -1,3 +1,4 @@
+
/*
* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
*
@@ -13,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.tensorflow.lite.examples.objectdetection
import android.content.Context
@@ -28,126 +30,197 @@ import org.tensorflow.lite.task.vision.detector.Detection
import org.tensorflow.lite.task.vision.detector.ObjectDetector
class ObjectDetectorHelper(
- var threshold: Float = 0.5f,
- var numThreads: Int = 2,
- var maxResults: Int = 3,
- var currentDelegate: Int = 0,
- var currentModel: Int = 0,
- val context: Context,
- val objectDetectorListener: DetectorListener?
+ var threshold: Float = DEFAULT_THRESHOLD,
+ var numThreads: Int = DEFAULT_NUM_THREADS,
+ var maxResults: Int = DEFAULT_MAX_RESULTS,
+ var currentDelegate: Int = DELEGATE_CPU,
+ var currentModel: Int = MODEL_EFFICIENTDETV0,
+ val context: Context,
+ val objectDetectorListener: DetectorListener?
) {
- // For this example this needs to be a var so it can be reset on changes. If the ObjectDetector
- // will not change, a lazy val would be preferable.
+ companion object {
+ private const val TAG = "ObjectDetectorHelper"
+
+ // Detection configuration
+ const val DEFAULT_THRESHOLD = 0.45f
+ const val DEFAULT_NUM_THREADS = 4
+ const val DEFAULT_MAX_RESULTS = 5
+
+ // Hardware delegates
+ const val DELEGATE_CPU = 0
+ const val DELEGATE_GPU = 1
+ const val DELEGATE_NNAPI = 2
+
+ // Model identifiers
+ const val MODEL_MOBILENETV1 = 0
+ const val MODEL_EFFICIENTDETV0 = 1
+ const val MODEL_EFFICIENTDETV1 = 2
+ const val MODEL_EFFICIENTDETV2 = 3
+ }
+
private var objectDetector: ObjectDetector? = null
init {
setupObjectDetector()
}
+ /**
+ * Releases the current detector reference.
+ * Call setupObjectDetector() afterwards to initialize it again.
+ */
fun clearObjectDetector() {
objectDetector = null
}
- // Initialize the object detector using current settings on the
- // thread that is using it. CPU and NNAPI delegates can be used with detectors
- // that are created on the main thread and used on a background thread, but
- // the GPU delegate needs to be used on the thread that initialized the detector
+ /**
+ * Creates the detector using the selected model, confidence threshold,
+ * result limit, thread count and hardware delegate.
+ */
+ @Synchronized
fun setupObjectDetector() {
- // Create the base options for the detector using specifies max results and score threshold
- val optionsBuilder =
- ObjectDetector.ObjectDetectorOptions.builder()
- .setScoreThreshold(threshold)
- .setMaxResults(maxResults)
-
- // Set general detection options, including number of used threads
- val baseOptionsBuilder = BaseOptions.builder().setNumThreads(numThreads)
-
- // Use the specified hardware for running the model. Default to CPU
- when (currentDelegate) {
- DELEGATE_CPU -> {
- // Default
- }
- DELEGATE_GPU -> {
- if (CompatibilityList().isDelegateSupportedOnThisDevice) {
- baseOptionsBuilder.useGpu()
- } else {
- objectDetectorListener?.onError("GPU is not supported on this device")
+ // Release the previous detector reference before changing configuration.
+ objectDetector = null
+
+ try {
+ val optionsBuilder =
+ ObjectDetector.ObjectDetectorOptions.builder()
+ .setScoreThreshold(threshold.coerceIn(0.0f, 1.0f))
+ .setMaxResults(maxResults.coerceAtLeast(1))
+
+ val baseOptionsBuilder =
+ BaseOptions.builder()
+ .setNumThreads(numThreads.coerceAtLeast(1))
+
+ when (currentDelegate) {
+ DELEGATE_CPU -> {
+ // CPU is the default and most compatible option.
+ }
+
+ DELEGATE_GPU -> {
+ val compatibilityList = CompatibilityList()
+
+ if (compatibilityList.isDelegateSupportedOnThisDevice) {
+ baseOptionsBuilder.useGpu()
+ } else {
+ objectDetectorListener?.onError(
+ "GPU is not supported on this device. Using CPU instead."
+ )
+ Log.w(TAG, "GPU delegate unsupported; using CPU.")
+ }
+ }
+
+ DELEGATE_NNAPI -> {
+ baseOptionsBuilder.useNnapi()
+ }
+
+ else -> {
+ Log.w(TAG, "Unknown delegate $currentDelegate; using CPU.")
}
}
- DELEGATE_NNAPI -> {
- baseOptionsBuilder.useNnapi()
- }
- }
- optionsBuilder.setBaseOptions(baseOptionsBuilder.build())
+ optionsBuilder.setBaseOptions(baseOptionsBuilder.build())
- val modelName =
- when (currentModel) {
+ val modelName = when (currentModel) {
MODEL_MOBILENETV1 -> "mobilenetv1.tflite"
MODEL_EFFICIENTDETV0 -> "efficientdet-lite0.tflite"
MODEL_EFFICIENTDETV1 -> "efficientdet-lite1.tflite"
MODEL_EFFICIENTDETV2 -> "efficientdet-lite2.tflite"
- else -> "mobilenetv1.tflite"
+ else -> {
+ Log.w(TAG, "Unknown model $currentModel; using MobileNet V1.")
+ "mobilenetv1.tflite"
+ }
}
- try {
objectDetector =
- ObjectDetector.createFromFileAndOptions(context, modelName, optionsBuilder.build())
- } catch (e: IllegalStateException) {
+ ObjectDetector.createFromFileAndOptions(
+ context,
+ modelName,
+ optionsBuilder.build()
+ )
+
+ Log.i(
+ TAG,
+ "Detector initialized: model=$modelName, " +
+ "threshold=$threshold, threads=$numThreads, " +
+ "maxResults=$maxResults, delegate=$currentDelegate"
+ )
+ } catch (e: Exception) {
+ objectDetector = null
+
+ Log.e(
+ TAG,
+ "Failed to initialize object detector.",
+ e
+ )
+
objectDetectorListener?.onError(
- "Object detector failed to initialize. See error logs for details"
+ "Could not initialize the object detector. " +
+ "Check the model file and device configuration."
)
- Log.e("Test", "TFLite failed to load model with error: " + e.message)
}
}
+ /**
+ * Runs object detection on a bitmap.
+ *
+ * imageRotation should be the camera rotation in degrees, normally
+ * 0, 90, 180 or 270, as supplied by the existing camera pipeline.
+ */
fun detect(image: Bitmap, imageRotation: Int) {
- if (objectDetector == null) {
+ val detector = objectDetector ?: run {
setupObjectDetector()
+ objectDetector
}
- // Inference time is the difference between the system time at the start and finish of the
- // process
- var inferenceTime = SystemClock.uptimeMillis()
-
- // Create preprocessor for the image.
- // See https://www.tensorflow.org/lite/inference_with_metadata/
- // lite_support#imageprocessor_architecture
- val imageProcessor =
- ImageProcessor.Builder()
- .add(Rot90Op(-imageRotation / 90))
- .build()
-
- // Preprocess the image and convert it into a TensorImage for detection.
- val tensorImage = imageProcessor.process(TensorImage.fromBitmap(image))
-
- val results = objectDetector?.detect(tensorImage)
- inferenceTime = SystemClock.uptimeMillis() - inferenceTime
- objectDetectorListener?.onResults(
- results,
- inferenceTime,
- tensorImage.height,
- tensorImage.width)
+ if (detector == null) {
+ Log.e(TAG, "Detection skipped because the detector is unavailable.")
+ return
+ }
+
+ val startTime = SystemClock.elapsedRealtime()
+
+ try {
+ val rotation = ((imageRotation % 360) + 360) % 360
+
+ val imageProcessor =
+ ImageProcessor.Builder()
+ .add(Rot90Op(-rotation / 90))
+ .build()
+
+ val tensorImage =
+ imageProcessor.process(
+ TensorImage.fromBitmap(image)
+ )
+
+ val results = detector.detect(tensorImage)
+
+ val inferenceTime =
+ SystemClock.elapsedRealtime() - startTime
+
+ objectDetectorListener?.onResults(
+ results,
+ inferenceTime,
+ tensorImage.height,
+ tensorImage.width
+ )
+ } catch (e: Exception) {
+ Log.e(TAG, "Object detection failed.", e)
+
+ objectDetectorListener?.onError(
+ "Object detection failed. Please try again."
+ )
+ }
}
interface DetectorListener {
fun onError(error: String)
+
fun onResults(
- results: MutableList?,
- inferenceTime: Long,
- imageHeight: Int,
- imageWidth: Int
+ results: MutableList?,
+ inferenceTime: Long,
+ imageHeight: Int,
+ imageWidth: Int
)
}
-
- companion object {
- const val DELEGATE_CPU = 0
- const val DELEGATE_GPU = 1
- const val DELEGATE_NNAPI = 2
- const val MODEL_MOBILENETV1 = 0
- const val MODEL_EFFICIENTDETV0 = 1
- const val MODEL_EFFICIENTDETV1 = 2
- const val MODEL_EFFICIENTDETV2 = 3
- }
}
diff --git a/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/OverlayView.kt b/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/OverlayView.kt
index 605afff3c60..b660fe84150 100644
--- a/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/OverlayView.kt
+++ b/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/OverlayView.kt
@@ -1,3 +1,4 @@
+
/*
* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
*
@@ -26,96 +27,196 @@ import android.util.AttributeSet
import android.view.View
import androidx.core.content.ContextCompat
import java.util.LinkedList
+import java.util.Locale
import kotlin.math.max
+import kotlin.math.min
import org.tensorflow.lite.task.vision.detector.Detection
-class OverlayView(context: Context?, attrs: AttributeSet?) : View(context, attrs) {
+class OverlayView(
+ context: Context?,
+ attrs: AttributeSet?
+) : View(context, attrs) {
+
+ private var results: List = LinkedList()
- private var results: List = LinkedList()
- private var boxPaint = Paint()
- private var textBackgroundPaint = Paint()
- private var textPaint = Paint()
+ private val boxPaint = Paint(Paint.ANTI_ALIAS_FLAG)
+ private val textBackgroundPaint = Paint(Paint.ANTI_ALIAS_FLAG)
+ private val textPaint = Paint(Paint.ANTI_ALIAS_FLAG)
private var scaleFactor: Float = 1f
+ private val bounds = Rect()
- private var bounds = Rect()
+ // Layout settings
+ private val boxStrokeWidth = 5f
+ private val textSizeSp = 14f
+ private val textPadding = 8f
+ private val cornerRadius = 8f
init {
initPaints()
}
+ private fun initPaints() {
+ val density = resources.displayMetrics.density
+ val scaledTextSize = textSizeSp * density
+
+ boxPaint.apply {
+ color = ContextCompat.getColor(
+ context ?: return@apply,
+ R.color.bounding_box_color
+ )
+ style = Paint.Style.STROKE
+ strokeWidth = boxStrokeWidth * density
+ }
+
+ textBackgroundPaint.apply {
+ color = Color.BLACK
+ alpha = 220
+ style = Paint.Style.FILL
+ }
+
+ textPaint.apply {
+ color = Color.WHITE
+ style = Paint.Style.FILL
+ textSize = scaledTextSize
+ typeface = android.graphics.Typeface.create(
+ android.graphics.Typeface.DEFAULT,
+ android.graphics.Typeface.BOLD
+ )
+ }
+ }
+
+ /**
+ * Clears all displayed detections.
+ */
fun clear() {
- textPaint.reset()
- textBackgroundPaint.reset()
- boxPaint.reset()
+ results = emptyList()
invalidate()
- initPaints()
}
- private fun initPaints() {
- textBackgroundPaint.color = Color.BLACK
- textBackgroundPaint.style = Paint.Style.FILL
- textBackgroundPaint.textSize = 50f
+ override fun onDraw(canvas: Canvas) {
+ super.onDraw(canvas)
- textPaint.color = Color.WHITE
- textPaint.style = Paint.Style.FILL
- textPaint.textSize = 50f
+ if (results.isEmpty() || width <= 0 || height <= 0) {
+ return
+ }
- boxPaint.color = ContextCompat.getColor(context!!, R.color.bounding_box_color)
- boxPaint.strokeWidth = 8F
- boxPaint.style = Paint.Style.STROKE
- }
+ for (result in results) {
+ val category = result.categories.maxByOrNull { it.score }
+ ?: continue
- override fun draw(canvas: Canvas) {
- super.draw(canvas)
+ val label = category.label.ifBlank { "Unknown object" }
+ val confidence = (category.score * 100f)
+ .coerceIn(0f, 100f)
- for (result in results) {
val boundingBox = result.boundingBox
- val top = boundingBox.top * scaleFactor
- val bottom = boundingBox.bottom * scaleFactor
val left = boundingBox.left * scaleFactor
+ val top = boundingBox.top * scaleFactor
val right = boundingBox.right * scaleFactor
+ val bottom = boundingBox.bottom * scaleFactor
- // Draw bounding box around detected objects
- val drawableRect = RectF(left, top, right, bottom)
- canvas.drawRect(drawableRect, boxPaint)
-
- // Create text to display alongside detected objects
- val drawableText =
- result.categories[0].label + " " +
- String.format("%.2f", result.categories[0].score)
-
- // Draw rect behind display text
- textBackgroundPaint.getTextBounds(drawableText, 0, drawableText.length, bounds)
- val textWidth = bounds.width()
- val textHeight = bounds.height()
- canvas.drawRect(
+ val drawableRect = RectF(
left,
top,
- left + textWidth + Companion.BOUNDING_RECT_TEXT_PADDING,
- top + textHeight + Companion.BOUNDING_RECT_TEXT_PADDING,
+ right,
+ bottom
+ )
+
+ // Draw the bounding box.
+ canvas.drawRoundRect(
+ drawableRect,
+ cornerRadius,
+ cornerRadius,
+ boxPaint
+ )
+
+ // Display label and confidence as a percentage.
+ val drawableText = String.format(
+ Locale.US,
+ "%s %.0f%%",
+ label,
+ confidence
+ )
+
+ textPaint.getTextBounds(
+ drawableText,
+ 0,
+ drawableText.length,
+ bounds
+ )
+
+ val textWidth = textPaint.measureText(drawableText)
+ val textHeight = bounds.height().toFloat()
+
+ val backgroundWidth = textWidth + textPadding * 2
+ val backgroundHeight = textHeight + textPadding * 2
+
+ // Prefer placing the label above the bounding box.
+ // If there is insufficient room, place it inside the box.
+ val backgroundTop = if (top >= backgroundHeight) {
+ top - backgroundHeight
+ } else {
+ top
+ }
+
+ val backgroundLeft = left.coerceAtLeast(0f)
+
+ val backgroundRect = RectF(
+ backgroundLeft,
+ backgroundTop,
+ min(backgroundLeft + backgroundWidth, width.toFloat()),
+ min(backgroundTop + backgroundHeight, height.toFloat())
+ )
+
+ canvas.drawRoundRect(
+ backgroundRect,
+ cornerRadius,
+ cornerRadius,
textBackgroundPaint
)
- // Draw text for detected object
- canvas.drawText(drawableText, left, top + bounds.height(), textPaint)
+ // Keep the text inside the visible canvas.
+ val textX = backgroundRect.left + textPadding
+ val textBaseline = min(
+ backgroundRect.bottom - textPadding,
+ backgroundRect.top + textPadding + textHeight
+ )
+
+ canvas.drawText(
+ drawableText,
+ textX,
+ textBaseline,
+ textPaint
+ )
}
}
+ /**
+ * Updates detections and scales bounding boxes to match the preview.
+ */
fun setResults(
- detectionResults: MutableList,
- imageHeight: Int,
- imageWidth: Int,
+ detectionResults: MutableList,
+ imageHeight: Int,
+ imageWidth: Int
) {
- results = detectionResults
-
- // PreviewView is in FILL_START mode. So we need to scale up the bounding box to match with
- // the size that the captured images will be displayed.
- scaleFactor = max(width * 1f / imageWidth, height * 1f / imageHeight)
- }
+ results = detectionResults.toList()
+
+ if (
+ imageWidth <= 0 ||
+ imageHeight <= 0 ||
+ width <= 0 ||
+ height <= 0
+ ) {
+ scaleFactor = 1f
+ } else {
+ // Match the PreviewView's FILL_START scaling behavior.
+ scaleFactor = max(
+ width.toFloat() / imageWidth,
+ height.toFloat() / imageHeight
+ )
+ }
- companion object {
- private const val BOUNDING_RECT_TEXT_PADDING = 8
+ invalidate()
}
}
diff --git a/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/fragments/CameraFragment.kt b/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/fragments/CameraFragment.kt
index afc13d8f3fc..2ea0b972860 100644
--- a/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/fragments/CameraFragment.kt
+++ b/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/fragments/CameraFragment.kt
@@ -1,24 +1,20 @@
+
/*
* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Licensed under the Apache License, Version 2.0.
+ * See https://www.apache.org/licenses/LICENSE-2.0
*/
+
package org.tensorflow.lite.examples.objectdetection.fragments
import android.annotation.SuppressLint
import android.content.res.Configuration
import android.graphics.Bitmap
import android.os.Bundle
+import android.os.SystemClock
+import android.speech.tts.TextToSpeech
+import android.speech.tts.UtteranceProgressListener
import android.util.Log
import android.view.LayoutInflater
import android.view.View
@@ -36,17 +32,41 @@ import androidx.camera.lifecycle.ProcessCameraProvider
import androidx.core.content.ContextCompat
import androidx.fragment.app.Fragment
import androidx.navigation.Navigation
-import java.util.LinkedList
-import java.util.concurrent.ExecutorService
-import java.util.concurrent.Executors
import org.tensorflow.lite.examples.objectdetection.ObjectDetectorHelper
import org.tensorflow.lite.examples.objectdetection.R
import org.tensorflow.lite.examples.objectdetection.databinding.FragmentCameraBinding
import org.tensorflow.lite.task.vision.detector.Detection
+import java.io.DataOutputStream
+import java.io.File
+import java.io.FileInputStream
+import java.net.InetSocketAddress
+import java.net.Socket
+import java.util.LinkedList
+import java.util.Locale
+import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Executors
-class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener {
-
- private val TAG = "ObjectDetection"
+class CameraFragment :
+ Fragment(),
+ ObjectDetectorHelper.DetectorListener {
+
+ companion object {
+ private const val TAG = "ObjectDetection"
+
+ // Raspberry Pi connection settings
+ private const val PI_HOST = "10.242.215.42"
+ private const val PI_PORT = 5001
+ private const val CONNECT_TIMEOUT_MS = 3000
+ private const val SOCKET_TIMEOUT_MS = 5000
+ private const val MAX_AUDIO_SIZE = 10 * 1024 * 1024
+
+ // Detection speech settings
+ private const val REQUIRED_STABLE_FRAMES = 3
+ private const val MIN_ANNOUNCEMENT_GAP_MS = 2000L
+ private const val SAME_LABEL_REPEAT_DELAY_MS = 6000L
+ private const val SPEECH_CONFIDENCE_THRESHOLD = 0.60f
+ }
private var _fragmentCameraBinding: FragmentCameraBinding? = null
@@ -55,272 +75,656 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener {
private lateinit var objectDetectorHelper: ObjectDetectorHelper
private lateinit var bitmapBuffer: Bitmap
+ private lateinit var cameraExecutor: ExecutorService
+ private lateinit var audioExecutor: ExecutorService
+
private var preview: Preview? = null
private var imageAnalyzer: ImageAnalysis? = null
private var camera: Camera? = null
private var cameraProvider: ProcessCameraProvider? = null
- /** Blocking camera operations are performed using this executor */
- private lateinit var cameraExecutor: ExecutorService
+ // Text-to-Speech
+ private var textToSpeech: TextToSpeech? = null
+ private var ttsReady = false
+ private var speechPending = false
+
+ // Synthesized audio files, indexed by utterance ID
+ private val audioFiles = ConcurrentHashMap()
+
+ // Stable detection tracking
+ private var candidateLabel = ""
+ private var candidateCount = 0
+
+ // Announcement tracking
+ private var lastSpokenLabel = ""
+ private var lastAnnouncementTime = 0L
+ private var utteranceCounter = 0
override fun onResume() {
super.onResume()
- // Make sure that all permissions are still present, since the
- // user could have removed them while the app was in paused state.
+
if (!PermissionsFragment.hasPermissions(requireContext())) {
- Navigation.findNavController(requireActivity(), R.id.fragment_container)
- .navigate(CameraFragmentDirections.actionCameraToPermissions())
+ Navigation.findNavController(
+ requireActivity(),
+ R.id.fragment_container
+ ).navigate(
+ CameraFragmentDirections.actionCameraToPermissions()
+ )
}
}
override fun onDestroyView() {
+ // Stop camera analysis before releasing the view.
+ imageAnalyzer?.clearAnalyzer()
+ cameraProvider?.unbindAll()
+
+ imageAnalyzer = null
+ preview = null
+ camera = null
+ cameraProvider = null
+
+ // Stop TTS and release the engine.
+ textToSpeech?.stop()
+ textToSpeech?.shutdown()
+ textToSpeech = null
+ ttsReady = false
+ speechPending = false
+
+ // Release generated files.
+ audioFiles.values.forEach { file ->
+ runCatching { file.delete() }
+ }
+ audioFiles.clear()
+
+ if (::cameraExecutor.isInitialized) {
+ cameraExecutor.shutdown()
+ }
+
+ if (::audioExecutor.isInitialized) {
+ audioExecutor.shutdownNow()
+ }
+
_fragmentCameraBinding = null
super.onDestroyView()
-
- // Shut down our background executor
- cameraExecutor.shutdown()
}
override fun onCreateView(
- inflater: LayoutInflater,
- container: ViewGroup?,
- savedInstanceState: Bundle?
+ inflater: LayoutInflater,
+ container: ViewGroup?,
+ savedInstanceState: Bundle?
): View {
- _fragmentCameraBinding = FragmentCameraBinding.inflate(inflater, container, false)
+ _fragmentCameraBinding =
+ FragmentCameraBinding.inflate(inflater, container, false)
return fragmentCameraBinding.root
}
@SuppressLint("MissingPermission")
- override fun onViewCreated(view: View, savedInstanceState: Bundle?) {
+ override fun onViewCreated(
+ view: View,
+ savedInstanceState: Bundle?
+ ) {
super.onViewCreated(view, savedInstanceState)
+ cameraExecutor = Executors.newSingleThreadExecutor()
+ audioExecutor = Executors.newSingleThreadExecutor()
+
+ // Hide bounding boxes and object labels for the demonstration.
+ // Detection still runs; only the visual overlay is hidden.
+ fragmentCameraBinding.overlay.visibility = View.GONE
+
+ initializeTextToSpeech()
+
objectDetectorHelper = ObjectDetectorHelper(
context = requireContext(),
- objectDetectorListener = this)
-
- // Initialize our background executor
- cameraExecutor = Executors.newSingleThreadExecutor()
+ objectDetectorListener = this
+ )
- // Wait for the views to be properly laid out
fragmentCameraBinding.viewFinder.post {
- // Set up the camera and its use cases
- setUpCamera()
+ if (_fragmentCameraBinding != null) {
+ setUpCamera()
+ }
}
- // Attach listeners to UI control widgets
initBottomSheetControls()
+ updateControlsUi()
}
- private fun initBottomSheetControls() {
- // When clicked, lower detection score threshold floor
- fragmentCameraBinding.bottomSheetLayout.thresholdMinus.setOnClickListener {
- if (objectDetectorHelper.threshold >= 0.1) {
- objectDetectorHelper.threshold -= 0.1f
- updateControlsUi()
+ private fun initializeTextToSpeech() {
+ ttsReady = false
+
+ textToSpeech = TextToSpeech(requireContext()) { status ->
+ if (status != TextToSpeech.SUCCESS) {
+ Log.e(TAG, "TTS initialization failed")
+ showToast("Text-to-Speech initialization failed.")
+ return@TextToSpeech
}
- }
- // When clicked, raise detection score threshold floor
- fragmentCameraBinding.bottomSheetLayout.thresholdPlus.setOnClickListener {
- if (objectDetectorHelper.threshold <= 0.8) {
- objectDetectorHelper.threshold += 0.1f
- updateControlsUi()
+ val engine = textToSpeech ?: return@TextToSpeech
+ val languageResult = engine.setLanguage(Locale.US)
+
+ ttsReady =
+ languageResult != TextToSpeech.LANG_MISSING_DATA &&
+ languageResult != TextToSpeech.LANG_NOT_SUPPORTED
+
+ if (!ttsReady) {
+ Log.e(TAG, "TTS language is unavailable")
+ showToast("English speech is unavailable.")
+ return@TextToSpeech
}
+
+ engine.setOnUtteranceProgressListener(
+ object : UtteranceProgressListener() {
+
+ override fun onStart(utteranceId: String?) {
+ Log.d(TAG, "Speech synthesis started: $utteranceId")
+ }
+
+ override fun onDone(utteranceId: String?) {
+ if (utteranceId == null) return
+
+ val file = audioFiles[utteranceId] ?: return
+
+ if (!::audioExecutor.isInitialized ||
+ audioExecutor.isShutdown
+ ) {
+ audioFiles.remove(utteranceId)
+ file.delete()
+ return
+ }
+
+ try {
+ audioExecutor.execute {
+ try {
+ sendAudioToPi(file)
+ } catch (e: Exception) {
+ Log.e(
+ TAG,
+ "Failed to transmit synthesized audio",
+ e
+ )
+ showToast(
+ "Could not send audio to Raspberry Pi."
+ )
+ } finally {
+ audioFiles.remove(utteranceId)
+ file.delete()
+
+ activity?.runOnUiThread {
+ speechPending = false
+ }
+ }
+ }
+ } catch (e: Exception) {
+ audioFiles.remove(utteranceId)
+ file.delete()
+ Log.e(TAG, "Audio executor unavailable", e)
+ activity?.runOnUiThread {
+ speechPending = false
+ }
+ }
+ }
+
+ @Deprecated("Deprecated in Android API")
+ override fun onError(utteranceId: String?) {
+ handleSynthesisError(utteranceId)
+ }
+
+ override fun onError(
+ utteranceId: String?,
+ errorCode: Int
+ ) {
+ Log.e(
+ TAG,
+ "Speech synthesis error: $errorCode"
+ )
+ handleSynthesisError(utteranceId)
+ }
+ }
+ )
+
+ Log.d(TAG, "TTS ready: $ttsReady")
}
+ }
- // When clicked, reduce the number of objects that can be detected at a time
- fragmentCameraBinding.bottomSheetLayout.maxResultsMinus.setOnClickListener {
- if (objectDetectorHelper.maxResults > 1) {
- objectDetectorHelper.maxResults--
- updateControlsUi()
- }
+ private fun handleSynthesisError(utteranceId: String?) {
+ if (utteranceId != null) {
+ audioFiles.remove(utteranceId)?.delete()
}
- // When clicked, increase the number of objects that can be detected at a time
- fragmentCameraBinding.bottomSheetLayout.maxResultsPlus.setOnClickListener {
- if (objectDetectorHelper.maxResults < 5) {
- objectDetectorHelper.maxResults++
- updateControlsUi()
- }
+ activity?.runOnUiThread {
+ speechPending = false
+ }
+ }
+
+ /**
+ * Generate a WAV file locally, then send it to the Raspberry Pi.
+ * This function should be called on the main thread.
+ */
+ private fun synthesizeAndSend(text: String) {
+ val engine = textToSpeech
+
+ if (!ttsReady || engine == null || speechPending) {
+ return
}
- // When clicked, decrease the number of threads used for detection
- fragmentCameraBinding.bottomSheetLayout.threadsMinus.setOnClickListener {
- if (objectDetectorHelper.numThreads > 1) {
- objectDetectorHelper.numThreads--
- updateControlsUi()
+ if (!::audioExecutor.isInitialized || audioExecutor.isShutdown) {
+ return
+ }
+
+ val utteranceId = "blind_assist_${++utteranceCounter}"
+ val outputFile = File(
+ requireContext().cacheDir,
+ "$utteranceId.wav"
+ )
+
+ speechPending = true
+ audioFiles[utteranceId] = outputFile
+
+ val result = engine.synthesizeToFile(
+ text,
+ Bundle(),
+ outputFile,
+ utteranceId
+ )
+
+ if (result != TextToSpeech.SUCCESS) {
+ audioFiles.remove(utteranceId)
+ outputFile.delete()
+ speechPending = false
+ Log.e(TAG, "Could not start WAV synthesis")
+ }
+ }
+
+ /**
+ * Protocol expected by audio_receiver.py:
+ * 4-byte big-endian WAV length, followed by WAV bytes.
+ *
+ * Network work runs on audioExecutor, never on the UI thread.
+ */
+ private fun sendAudioToPi(file: File) {
+ if (!file.exists()) {
+ throw IllegalStateException("Synthesized WAV file is missing")
+ }
+
+ val audioLength = file.length()
+
+ if (audioLength <= 0 || audioLength > MAX_AUDIO_SIZE) {
+ throw IllegalStateException(
+ "Invalid audio file size: $audioLength"
+ )
+ }
+
+ Socket().use { socket ->
+ socket.connect(
+ InetSocketAddress(PI_HOST, PI_PORT),
+ CONNECT_TIMEOUT_MS
+ )
+ socket.soTimeout = SOCKET_TIMEOUT_MS
+ socket.tcpNoDelay = true
+
+ DataOutputStream(socket.getOutputStream()).use { output ->
+ // DataOutputStream writes integers in big-endian order.
+ output.writeInt(audioLength.toInt())
+
+ FileInputStream(file).use { input ->
+ val buffer = ByteArray(8192)
+ var totalSent = 0L
+
+ while (true) {
+ val count = input.read(buffer)
+ if (count == -1) break
+
+ output.write(buffer, 0, count)
+ totalSent += count
+ }
+
+ output.flush()
+
+ if (totalSent != audioLength) {
+ throw IllegalStateException(
+ "Audio transmission was incomplete"
+ )
+ }
+ }
}
}
- // When clicked, increase the number of threads used for detection
- fragmentCameraBinding.bottomSheetLayout.threadsPlus.setOnClickListener {
- if (objectDetectorHelper.numThreads < 4) {
- objectDetectorHelper.numThreads++
- updateControlsUi()
+ Log.i(TAG, "Audio sent to Raspberry Pi ($audioLength bytes)")
+ }
+
+ /**
+ * Select the highest-confidence label, require stable frames,
+ * then synthesize speech and transmit it to the Pi.
+ */
+ private fun announceDetection(results: MutableList?) {
+ if (!ttsReady || speechPending) return
+
+ val bestCategory = results
+ ?.asSequence()
+ ?.mapNotNull { detection ->
+ detection.categories.maxByOrNull { it.score }
+ }
+ ?.filter {
+ it.score >= SPEECH_CONFIDENCE_THRESHOLD &&
+ it.label.isNotBlank()
}
+ ?.maxByOrNull { it.score }
+
+ if (bestCategory == null) {
+ candidateLabel = ""
+ candidateCount = 0
+ return
}
- // When clicked, change the underlying hardware used for inference. Current options are CPU
- // GPU, and NNAPI
- fragmentCameraBinding.bottomSheetLayout.spinnerDelegate.setSelection(0, false)
- fragmentCameraBinding.bottomSheetLayout.spinnerDelegate.onItemSelectedListener =
- object : AdapterView.OnItemSelectedListener {
- override fun onItemSelected(p0: AdapterView<*>?, p1: View?, p2: Int, p3: Long) {
- objectDetectorHelper.currentDelegate = p2
+ val label = bestCategory.label
+
+ if (label == candidateLabel) {
+ candidateCount++
+ } else {
+ candidateLabel = label
+ candidateCount = 1
+ }
+
+ if (candidateCount < REQUIRED_STABLE_FRAMES) return
+
+ val now = SystemClock.elapsedRealtime()
+
+ if (now - lastAnnouncementTime < MIN_ANNOUNCEMENT_GAP_MS) {
+ return
+ }
+
+ if (
+ label == lastSpokenLabel &&
+ now - lastAnnouncementTime < SAME_LABEL_REPEAT_DELAY_MS
+ ) {
+ return
+ }
+
+ // Reserve the speech slot before starting synthesis.
+ lastSpokenLabel = label
+ lastAnnouncementTime = now
+ candidateLabel = ""
+ candidateCount = 0
+
+ val spokenLabel = label.replaceFirstChar { it.titlecase(Locale.US) }
+ val message = "$spokenLabel is detected."
+
+ Log.i(TAG, "Announcing detected object: $message")
+ synthesizeAndSend(message)
+ }
+
+ private fun initBottomSheetControls() {
+ fragmentCameraBinding.bottomSheetLayout.thresholdMinus
+ .setOnClickListener {
+ if (objectDetectorHelper.threshold > 0.1f) {
+ objectDetectorHelper.threshold =
+ (objectDetectorHelper.threshold - 0.1f)
+ .coerceAtLeast(0.1f)
updateControlsUi()
}
+ }
- override fun onNothingSelected(p0: AdapterView<*>?) {
- /* no op */
+ fragmentCameraBinding.bottomSheetLayout.thresholdPlus
+ .setOnClickListener {
+ if (objectDetectorHelper.threshold < 0.9f) {
+ objectDetectorHelper.threshold =
+ (objectDetectorHelper.threshold + 0.1f)
+ .coerceAtMost(0.9f)
+ updateControlsUi()
}
}
- // When clicked, change the underlying model used for object detection
- fragmentCameraBinding.bottomSheetLayout.spinnerModel.setSelection(0, false)
- fragmentCameraBinding.bottomSheetLayout.spinnerModel.onItemSelectedListener =
- object : AdapterView.OnItemSelectedListener {
- override fun onItemSelected(p0: AdapterView<*>?, p1: View?, p2: Int, p3: Long) {
- objectDetectorHelper.currentModel = p2
+ fragmentCameraBinding.bottomSheetLayout.maxResultsMinus
+ .setOnClickListener {
+ if (objectDetectorHelper.maxResults > 1) {
+ objectDetectorHelper.maxResults--
updateControlsUi()
}
+ }
- override fun onNothingSelected(p0: AdapterView<*>?) {
- /* no op */
+ fragmentCameraBinding.bottomSheetLayout.maxResultsPlus
+ .setOnClickListener {
+ if (objectDetectorHelper.maxResults < 5) {
+ objectDetectorHelper.maxResults++
+ updateControlsUi()
}
}
+
+ fragmentCameraBinding.bottomSheetLayout.threadsMinus
+ .setOnClickListener {
+ if (objectDetectorHelper.numThreads > 1) {
+ objectDetectorHelper.numThreads--
+ updateControlsUi()
+ }
+ }
+
+ fragmentCameraBinding.bottomSheetLayout.threadsPlus
+ .setOnClickListener {
+ if (objectDetectorHelper.numThreads < 4) {
+ objectDetectorHelper.numThreads++
+ updateControlsUi()
+ }
+ }
+
+ fragmentCameraBinding.bottomSheetLayout.spinnerDelegate
+ .setSelection(0, false)
+
+ fragmentCameraBinding.bottomSheetLayout.spinnerDelegate
+ .onItemSelectedListener =
+ object : AdapterView.OnItemSelectedListener {
+
+ override fun onItemSelected(
+ parent: AdapterView<*>?,
+ view: View?,
+ position: Int,
+ id: Long
+ ) {
+ if (objectDetectorHelper.currentDelegate != position) {
+ objectDetectorHelper.currentDelegate = position
+ updateControlsUi()
+ }
+ }
+
+ override fun onNothingSelected(parent: AdapterView<*>?) = Unit
+ }
+
+ fragmentCameraBinding.bottomSheetLayout.spinnerModel
+ .setSelection(0, false)
+
+ fragmentCameraBinding.bottomSheetLayout.spinnerModel
+ .onItemSelectedListener =
+ object : AdapterView.OnItemSelectedListener {
+
+ override fun onItemSelected(
+ parent: AdapterView<*>?,
+ view: View?,
+ position: Int,
+ id: Long
+ ) {
+ if (objectDetectorHelper.currentModel != position) {
+ objectDetectorHelper.currentModel = position
+ updateControlsUi()
+ }
+ }
+
+ override fun onNothingSelected(parent: AdapterView<*>?) = Unit
+ }
}
- // Update the values displayed in the bottom sheet. Reset detector.
private fun updateControlsUi() {
- fragmentCameraBinding.bottomSheetLayout.maxResultsValue.text =
+ val binding = _fragmentCameraBinding ?: return
+
+ binding.bottomSheetLayout.maxResultsValue.text =
objectDetectorHelper.maxResults.toString()
- fragmentCameraBinding.bottomSheetLayout.thresholdValue.text =
- String.format("%.2f", objectDetectorHelper.threshold)
- fragmentCameraBinding.bottomSheetLayout.threadsValue.text =
+
+ binding.bottomSheetLayout.thresholdValue.text =
+ String.format(
+ Locale.US,
+ "%.2f",
+ objectDetectorHelper.threshold
+ )
+
+ binding.bottomSheetLayout.threadsValue.text =
objectDetectorHelper.numThreads.toString()
- // Needs to be cleared instead of reinitialized because the GPU
- // delegate needs to be initialized on the thread using it when applicable
+ candidateLabel = ""
+ candidateCount = 0
+
objectDetectorHelper.clearObjectDetector()
- fragmentCameraBinding.overlay.clear()
+ binding.overlay.clear()
}
- // Initialize CameraX, and prepare to bind the camera use cases
private fun setUpCamera() {
- val cameraProviderFuture = ProcessCameraProvider.getInstance(requireContext())
+ val safeContext = context ?: return
+
+ val cameraProviderFuture =
+ ProcessCameraProvider.getInstance(safeContext)
+
cameraProviderFuture.addListener(
{
- // CameraProvider
- cameraProvider = cameraProviderFuture.get()
-
- // Build and bind the camera use cases
- bindCameraUseCases()
+ if (_fragmentCameraBinding == null) return@addListener
+
+ try {
+ cameraProvider = cameraProviderFuture.get()
+ bindCameraUseCases()
+ } catch (e: Exception) {
+ Log.e(TAG, "Camera initialization failed", e)
+ showToast("Unable to initialize the camera.")
+ }
},
- ContextCompat.getMainExecutor(requireContext())
+ ContextCompat.getMainExecutor(safeContext)
)
}
- // Declare and bind preview, capture and analysis use cases
@SuppressLint("UnsafeOptInUsageError")
private fun bindCameraUseCases() {
+ val provider = cameraProvider ?: run {
+ showToast("Camera provider is unavailable.")
+ return
+ }
+
+ val binding = _fragmentCameraBinding ?: return
+
+ val cameraSelector = CameraSelector.Builder()
+ .requireLensFacing(CameraSelector.LENS_FACING_BACK)
+ .build()
- // CameraProvider
- val cameraProvider =
- cameraProvider ?: throw IllegalStateException("Camera initialization failed.")
-
- // CameraSelector - makes assumption that we're only using the back camera
- val cameraSelector =
- CameraSelector.Builder().requireLensFacing(CameraSelector.LENS_FACING_BACK).build()
-
- // Preview. Only using the 4:3 ratio because this is the closest to our models
- preview =
- Preview.Builder()
- .setTargetAspectRatio(AspectRatio.RATIO_4_3)
- .setTargetRotation(fragmentCameraBinding.viewFinder.display.rotation)
- .build()
-
- // ImageAnalysis. Using RGBA 8888 to match how our models work
- imageAnalyzer =
- ImageAnalysis.Builder()
- .setTargetAspectRatio(AspectRatio.RATIO_4_3)
- .setTargetRotation(fragmentCameraBinding.viewFinder.display.rotation)
- .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
- .setOutputImageFormat(OUTPUT_IMAGE_FORMAT_RGBA_8888)
- .build()
- // The analyzer can then be assigned to the instance
- .also {
- it.setAnalyzer(cameraExecutor) { image ->
- if (!::bitmapBuffer.isInitialized) {
- // The image rotation and RGB image buffer are initialized only once
- // the analyzer has started running
+ preview = Preview.Builder()
+ .setTargetAspectRatio(AspectRatio.RATIO_4_3)
+ .setTargetRotation(binding.viewFinder.display.rotation)
+ .build()
+
+ imageAnalyzer = ImageAnalysis.Builder()
+ .setTargetAspectRatio(AspectRatio.RATIO_4_3)
+ .setTargetRotation(binding.viewFinder.display.rotation)
+ .setBackpressureStrategy(
+ ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST
+ )
+ .setOutputImageFormat(OUTPUT_IMAGE_FORMAT_RGBA_8888)
+ .build()
+ .also { analyzer ->
+ analyzer.setAnalyzer(cameraExecutor) { image ->
+ try {
+ if (_fragmentCameraBinding == null) {
+ return@setAnalyzer
+ }
+
+ if (
+ !::bitmapBuffer.isInitialized ||
+ bitmapBuffer.width != image.width ||
+ bitmapBuffer.height != image.height
+ ) {
bitmapBuffer = Bitmap.createBitmap(
- image.width,
- image.height,
- Bitmap.Config.ARGB_8888
+ image.width,
+ image.height,
+ Bitmap.Config.ARGB_8888
)
}
detectObjects(image)
+ } catch (e: Exception) {
+ Log.e(TAG, "Object detection failed", e)
+ } finally {
+ image.close()
}
}
+ }
- // Must unbind the use-cases before rebinding them
- cameraProvider.unbindAll()
+ provider.unbindAll()
try {
- // A variable number of use-cases can be passed here -
- // camera provides access to CameraControl & CameraInfo
- camera = cameraProvider.bindToLifecycle(this, cameraSelector, preview, imageAnalyzer)
-
- // Attach the viewfinder's surface provider to preview use case
- preview?.setSurfaceProvider(fragmentCameraBinding.viewFinder.surfaceProvider)
- } catch (exc: Exception) {
- Log.e(TAG, "Use case binding failed", exc)
+ camera = provider.bindToLifecycle(
+ this,
+ cameraSelector,
+ preview,
+ imageAnalyzer
+ )
+
+ preview?.setSurfaceProvider(binding.viewFinder.surfaceProvider)
+ } catch (e: Exception) {
+ Log.e(TAG, "Use case binding failed", e)
+ showToast("Unable to start the camera preview.")
}
}
private fun detectObjects(image: ImageProxy) {
- // Copy out RGB bits to the shared bitmap buffer
- image.use { bitmapBuffer.copyPixelsFromBuffer(image.planes[0].buffer) }
+ if (!::bitmapBuffer.isInitialized) return
+
+ bitmapBuffer.copyPixelsFromBuffer(image.planes[0].buffer)
- val imageRotation = image.imageInfo.rotationDegrees
- // Pass Bitmap and rotation to the object detector helper for processing and detection
- objectDetectorHelper.detect(bitmapBuffer, imageRotation)
+ objectDetectorHelper.detect(
+ bitmapBuffer,
+ image.imageInfo.rotationDegrees
+ )
}
override fun onConfigurationChanged(newConfig: Configuration) {
super.onConfigurationChanged(newConfig)
- imageAnalyzer?.targetRotation = fragmentCameraBinding.viewFinder.display.rotation
+
+ _fragmentCameraBinding?.let { binding ->
+ imageAnalyzer?.targetRotation =
+ binding.viewFinder.display.rotation
+ }
}
- // Update UI after objects have been detected. Extracts original image height/width
- // to scale and place bounding boxes properly through OverlayView
override fun onResults(
- results: MutableList?,
- inferenceTime: Long,
- imageHeight: Int,
- imageWidth: Int
+ results: MutableList?,
+ inferenceTime: Long,
+ imageHeight: Int,
+ imageWidth: Int
) {
activity?.runOnUiThread {
- fragmentCameraBinding.bottomSheetLayout.inferenceTimeVal.text =
- String.format("%d ms", inferenceTime)
+ val binding = _fragmentCameraBinding
+ ?: return@runOnUiThread
+
+ binding.bottomSheetLayout.inferenceTimeVal.text =
+ String.format(Locale.US, "%d ms", inferenceTime)
- // Pass necessary information to OverlayView for drawing on the canvas
- fragmentCameraBinding.overlay.setResults(
- results ?: LinkedList(),
+ binding.overlay.setResults(
+ results ?: LinkedList(),
imageHeight,
imageWidth
)
- // Force a redraw
- fragmentCameraBinding.overlay.invalidate()
+ binding.overlay.invalidate()
+ announceDetection(results)
}
}
override fun onError(error: String) {
+ showToast(error)
+ }
+
+ private fun showToast(message: String) {
activity?.runOnUiThread {
- Toast.makeText(requireContext(), error, Toast.LENGTH_SHORT).show()
+ context?.let { safeContext ->
+ Toast.makeText(
+ safeContext,
+ message,
+ Toast.LENGTH_SHORT
+ ).show()
+ }
}
}
}
diff --git a/lite/examples/object_detection/android/gradle/wrapper/gradle-wrapper.properties b/lite/examples/object_detection/android/gradle/wrapper/gradle-wrapper.properties
index 9b7cb4af38a..ef426240827 100644
--- a/lite/examples/object_detection/android/gradle/wrapper/gradle-wrapper.properties
+++ b/lite/examples/object_detection/android/gradle/wrapper/gradle-wrapper.properties
@@ -1,6 +1,6 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-9.8.0-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip
networkTimeout=10000
retries=0
retryBackOffMs=500