From 4083d51326d801e4e185cddedda1eb6f38ded7de Mon Sep 17 00:00:00 2001 From: vona-ben <108658664+vona-ben@users.noreply.github.com> Date: Fri, 13 Sep 2024 04:07:51 +0800 Subject: [PATCH] VIDCS-2430: Fix crash when publishing/unpublishing quickly (#518) --- .../MirrorVideoCapturer.java | 299 +++++++++++------- 1 file changed, 179 insertions(+), 120 deletions(-) diff --git a/Basic-Video-Capturer-Camera-2-Java/app/src/main/java/com/tokbox/sample/basicvideocapturercamera2/MirrorVideoCapturer.java b/Basic-Video-Capturer-Camera-2-Java/app/src/main/java/com/tokbox/sample/basicvideocapturercamera2/MirrorVideoCapturer.java index ee270f0d..40ddf3a3 100644 --- a/Basic-Video-Capturer-Camera-2-Java/app/src/main/java/com/tokbox/sample/basicvideocapturercamera2/MirrorVideoCapturer.java +++ b/Basic-Video-Capturer-Camera-2-Java/app/src/main/java/com/tokbox/sample/basicvideocapturercamera2/MirrorVideoCapturer.java @@ -24,12 +24,15 @@ import android.view.Display; import android.view.Surface; import android.view.WindowManager; + +import androidx.annotation.NonNull; import androidx.annotation.RequiresApi; import com.opentok.android.BaseVideoCapturer; import com.opentok.android.Publisher; +import java.io.PrintWriter; +import java.io.StringWriter; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.List; @@ -67,12 +70,12 @@ private enum CameraState { private final Size frameDimensions; private final int desiredFps; private Range camFps; - private final List runtimeExceptionList; private Runnable executeAfterClosed; private Runnable executeAfterCameraOpened; + private Runnable executeAfterCameraSessionConfigured; - private static final SparseIntArray rotationTable = new SparseIntArray() { + static final SparseIntArray rotationTable = new SparseIntArray() { { append(Surface.ROTATION_0, 0); append(Surface.ROTATION_90, 90); @@ -100,7 +103,7 @@ private enum CameraState { /* Observers/Notification callback objects */ private final CameraDevice.StateCallback cameraObserver = new CameraDevice.StateCallback() { @Override - public void onOpened(CameraDevice camera) { + public void onOpened(@NonNull CameraDevice camera) { Log.d(TAG,"CameraDevice.StateCallback onOpened() enter"); cameraState = CameraState.OPEN; MirrorVideoCapturer.this.camera = camera; @@ -112,30 +115,29 @@ public void onOpened(CameraDevice camera) { } @Override - public void onDisconnected(CameraDevice camera) { + public void onDisconnected(@NonNull CameraDevice camera) { Log.d(TAG,"CameraDevice.StateCallback onDisconnected() enter"); try { executeAfterClosed = null; MirrorVideoCapturer.this.camera.close(); - } catch (NullPointerException e) { - // does nothing + } catch (Exception exception) { + handleException(exception); } Log.d(TAG,"CameraDevice.StateCallback onDisconnected() exit"); } + @Override - public void onError(CameraDevice camera, int error) { + public void onError(@NonNull CameraDevice camera, int error) { Log.d(TAG,"CameraDevice.StateCallback onError() enter"); try { MirrorVideoCapturer.this.camera.close(); // wait for condition variable - } catch (NullPointerException e) { - // does nothing + } catch (Exception exception) { + handleException(exception); } - postAsyncException(new Camera2Exception("Camera Open Error: " + error)); Log.d(TAG,"CameraDevice.StateCallback onError() exit"); } - @Override public void onClosed(CameraDevice camera) { Log.d(TAG,"CameraDevice.StateCallback onClosed() enter."); @@ -159,12 +161,10 @@ public void onImageAvailable(ImageReader reader) { if (frame == null || (frame.getPlanes().length > 0 && frame.getPlanes()[0].getBuffer() == null) || (frame.getPlanes().length > 1 && frame.getPlanes()[1].getBuffer() == null) - || (frame.getPlanes().length > 2 && frame.getPlanes()[2].getBuffer() == null)) - { + || (frame.getPlanes().length > 2 && frame.getPlanes()[2].getBuffer() == null)) { Log.d(TAG,"onImageAvailable frame provided has no image data"); return; } - if (CameraState.CAPTURE == cameraState) { provideBufferFramePlanar( frame.getPlanes()[0].getBuffer(), @@ -184,7 +184,7 @@ public void onImageAvailable(ImageReader reader) { } frame.close(); } catch (IllegalStateException e) { - Log.e(TAG,"ImageReader.acquireNextImage() throws error !"); + Log.d(TAG,"ImageReader.acquireNextImage() throws error !"); throw (new Camera2Exception(e.getMessage())); } } @@ -193,29 +193,41 @@ public void onImageAvailable(ImageReader reader) { private final CameraCaptureSession.StateCallback captureSessionObserver = new CameraCaptureSession.StateCallback() { @Override - public void onConfigured(CameraCaptureSession session) { + public void onConfigured(@NonNull CameraCaptureSession session) { Log.d(TAG,"CameraCaptureSession.StateCallback onConfigured() enter."); try { cameraState = CameraState.CAPTURE; captureSession = session; CaptureRequest captureRequest = captureRequestBuilder.build(); captureSession.setRepeatingRequest(captureRequest, captureNotification, null); - } catch (CameraAccessException e) { - e.printStackTrace(); + } catch (Exception exception) { + handleException(exception); + } + + if (executeAfterCameraSessionConfigured != null) { + executeAfterCameraSessionConfigured.run(); + executeAfterCameraSessionConfigured = null; + } + synchronized (lock) { + if (cycleCameraInProgress) { + cycleCameraInProgress = false; + onCameraChanged(getCameraIndex()); + } } Log.d(TAG,"CameraCaptureSession.StateCallback onConfigured() exit."); + } @Override - public void onConfigureFailed(CameraCaptureSession session) { + public void onConfigureFailed(@NonNull CameraCaptureSession session) { Log.d(TAG,"CameraCaptureSession.StateCallback onFailed() enter."); cameraState = CameraState.ERROR; - postAsyncException(new Camera2Exception("Camera session configuration failed")); Log.d(TAG,"CameraCaptureSession.StateCallback onFailed() exit."); + } @Override - public void onClosed(CameraCaptureSession session) { + public void onClosed(@NonNull CameraCaptureSession session) { Log.d(TAG,"CameraCaptureSession.StateCallback onClosed() enter."); if (camera != null) { camera.close(); @@ -227,21 +239,19 @@ public void onClosed(CameraCaptureSession session) { private final CameraCaptureSession.CaptureCallback captureNotification = new CameraCaptureSession.CaptureCallback() { @Override - public void onCaptureStarted(CameraCaptureSession session, CaptureRequest request, + public void onCaptureStarted(@NonNull CameraCaptureSession session, + @NonNull CaptureRequest request, long timestamp, long frameNumber) { super.onCaptureStarted(session, request, timestamp, frameNumber); } }; - /* caching of camera characteristics & display orientation for performance */ private static class CameraInfoCache { - private final CameraCharacteristics info; - private boolean frontFacing = false; - private int sensorOrientation = 0; + private final boolean frontFacing; + private final int sensorOrientation; public CameraInfoCache(CameraCharacteristics info) { - this.info = info; /* its actually faster to cache these results then to always look them up, and since they are queried every frame... */ @@ -249,11 +259,6 @@ public CameraInfoCache(CameraCharacteristics info) { == CameraCharacteristics.LENS_FACING_FRONT; sensorOrientation = info.get(CameraCharacteristics.SENSOR_ORIENTATION).intValue(); } - - public T get(CameraCharacteristics.Key key) { - return info.get(key); - } - public boolean isFrontFacing() { return frontFacing; } @@ -264,16 +269,16 @@ public int sensorOrientation() { } private static class DisplayOrientationCache implements Runnable { - private static final int POLL_DELAY_MS = 750; /* 750 ms */ + private static final int POLL_DELAY_MS = 750; /* 750 ms */ private int displayRotation; private final Display display; private final Handler handler; - public DisplayOrientationCache(Display dsp, Handler hndlr) { + public DisplayOrientationCache(Display dsp, Handler handler) { display = dsp; - handler = hndlr; + this.handler = handler; displayRotation = rotationTable.get(display.getRotation()); - handler.postDelayed(this, POLL_DELAY_MS); + this.handler.postDelayed(this, POLL_DELAY_MS); } public int getOrientation() { @@ -296,26 +301,30 @@ public Camera2Exception(String message) { /* Constructors etc... */ public MirrorVideoCapturer(Context ctx, - Publisher.CameraCaptureResolution resolution, - Publisher.CameraCaptureFrameRate fps) { + Publisher.CameraCaptureResolution resolution, + Publisher.CameraCaptureFrameRate fps) { cameraManager = (CameraManager) ctx.getSystemService(Context.CAMERA_SERVICE); display = ((WindowManager) ctx.getSystemService(Context.WINDOW_SERVICE)).getDefaultDisplay(); camera = null; cameraState = CameraState.CLOSED; frameDimensions = resolutionTable.get(resolution.ordinal()); desiredFps = frameRateTable.get(fps.ordinal()); - runtimeExceptionList = new ArrayList(); try { String camId = selectCamera(PREFERRED_FACING_CAMERA); /* if default camera facing direction is not found, use first camera */ if (null == camId && (0 < cameraManager.getCameraIdList().length)) { camId = cameraManager.getCameraIdList()[0]; } - cameraIndex = findCameraIndex(camId); - initCameraFrame(); - } catch (CameraAccessException e) { - throw new Camera2Exception(e.getMessage()); + setCameraIndex(findCameraIndex(camId)); + if (getCameraIndex() == -1) { + Log.d(TAG,"Exception!. Camera Index cannot be -1."); + } else { + initCameraFrame(); + } + } catch (Exception exception) { + handleException(exception); } + } private void doInit() { @@ -337,17 +346,13 @@ private void doInit() { public synchronized void init() { Log.d(TAG,"init() enter"); - if (cameraState == CameraState.CLOSING) { - executeAfterClosed = () -> doInit(); - } else { - doInit(); - } + doInit(); cameraState = CameraState.SETUP; Log.d(TAG,"init() exit"); } - private int doStartCapture() { + private void doStartCapture() { Log.d(TAG,"doStartCapture() enter"); cameraState = CameraState.CREATESESSION; try { @@ -368,11 +373,6 @@ private int doStartCapture() { CaptureRequest.CONTROL_SCENE_MODE, CaptureRequest.CONTROL_SCENE_MODE_FACE_PRIORITY ); - camera.createCaptureSession( - Arrays.asList(cameraFrame.getSurface()), - captureSessionObserver, - null - ); } else { captureRequestBuilder = camera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD); captureRequestBuilder.addTarget(cameraFrame.getSurface()); @@ -381,19 +381,17 @@ private int doStartCapture() { CaptureRequest.CONTROL_AF_MODE, CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_PICTURE ); - camera.createCaptureSession( - Arrays.asList(cameraFrame.getSurface()), - captureSessionObserver, - null - ); } - } catch (CameraAccessException e) { - throw new Camera2Exception(e.getMessage()); + camera.createCaptureSession( + Collections.singletonList(cameraFrame.getSurface()), + captureSessionObserver, + null + ); + } catch (CameraAccessException exception) { + handleException(exception); } Log.d(TAG,"doStartCapture() exit"); - return 0; } - /** * Starts capturing video. */ @@ -418,21 +416,20 @@ public synchronized int startCapture() { /** * Starts capturing video. */ - public synchronized int scheduleStartCapture() { + private synchronized void scheduleStartCapture() { Log.d(TAG,"scheduleStartCapture() enter (cameraState: " + cameraState + ")"); if (null != camera && CameraState.OPEN == cameraState) { - return doStartCapture(); + doStartCapture(); + return; } else if (CameraState.SETUP == cameraState) { Log.d(TAG,"camera not yet ready, queuing the start until camera is opened."); executeAfterCameraOpened = this::doStartCapture; } else if (CameraState.CREATESESSION == cameraState) { Log.d(TAG,"Camera session creation already requested"); - } - else { + } else { Log.d(TAG,"Start Capture called before init successfully completed."); } Log.d(TAG,"scheduleStartCapture() exit"); - return 0; } /** @@ -440,21 +437,32 @@ public synchronized int scheduleStartCapture() { */ @Override public synchronized int stopCapture() { - Log.d(TAG,"stopCapture enter"); - if (null != camera && null != captureSession && CameraState.CLOSED != cameraState) { - cameraState = CameraState.CLOSING; + Log.d(TAG,"stopCapture() enter (cameraState: " + cameraState + ")"); + if (null != camera && null != captureSession && CameraState.CAPTURE == cameraState) { try { captureSession.stopRepeating(); - } catch (CameraAccessException e) { - e.printStackTrace(); + } catch (CameraAccessException exception) { + handleException(exception); } captureSession.close(); cameraInfoCache = null; + cameraState = CameraState.CLOSING; } else if (null != camera && CameraState.OPEN == cameraState) { cameraState = CameraState.CLOSING; camera.close(); } else if (CameraState.SETUP == cameraState) { - executeAfterCameraOpened = null; + executeAfterCameraOpened = () -> { + cameraState = CameraState.CLOSING; + if (camera != null) { + camera.close(); + } + }; + } else if (CameraState.CREATESESSION == cameraState) { + executeAfterCameraSessionConfigured = () -> { + captureSession.close(); + cameraState = CameraState.CLOSING; + executeAfterCameraSessionConfigured = null; + }; } Log.d(TAG,"stopCapture exit"); return 0; @@ -491,13 +499,15 @@ public boolean isCaptureStarted() { */ @Override public synchronized CaptureSettings getCaptureSettings() { - CaptureSettings captureSettings = new CaptureSettings(); - captureSettings.fps = desiredFps; - captureSettings.width = (null != cameraFrame) ? cameraFrame.getWidth() : 0; - captureSettings.height = (null != cameraFrame) ? cameraFrame.getHeight() : 0; - captureSettings.format = BaseVideoCapturer.NV21; - captureSettings.expectedDelay = 0; - return captureSettings; + CaptureSettings retObj = new CaptureSettings(); + retObj.fps = desiredFps; + retObj.width = (null != cameraFrame) ? cameraFrame.getWidth() : -1; + retObj.height = (null != cameraFrame) ? cameraFrame.getHeight() : -1; + retObj.format = BaseVideoCapturer.NV21; + retObj.expectedDelay = 0; + retObj.mirrorInLocalRender = frameMirrorX; + + return retObj; } /** @@ -559,7 +569,7 @@ private Size[] getCameraOutputSizes(String cameraId) throws CameraAccessExceptio CameraCharacteristics characteristics = cameraManager.getCameraCharacteristics(cameraId); StreamConfigurationMap dimMap = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); - return dimMap.getOutputSizes(PIXEL_FORMAT); + return dimMap != null ? dimMap.getOutputSizes(PIXEL_FORMAT) : new Size[0]; } private int getNextSupportedCameraIndex() throws CameraAccessException { @@ -572,10 +582,11 @@ private int getNextSupportedCameraIndex() throws CameraAccessException { // We use +1 so that the algorithm will rollover and check the // current camera too. At minimum, the current camera *should* have // supported outputs. - int nextCameraIndex = (cameraIndex + i + 1) % numCameraIds; + int nextCameraIndex = (getCameraIndex() + i + 1) % numCameraIds; Size[] outputSizes = getCameraOutputSizes(cameraIds[nextCameraIndex]); boolean hasSupportedOutputs = outputSizes != null && outputSizes.length > 0; + // OPENTOK-48451. Best guess is that the crash is happening when sdk is // trying to open depth sensor cameras while doing cycleCamera() function. boolean isDepthOutputCamera = isDepthOutputCamera(cameraIds[nextCameraIndex]); @@ -592,32 +603,47 @@ private int getNextSupportedCameraIndex() throws CameraAccessException { @Override public synchronized void cycleCamera() { + synchronized (lock) { + if (cycleCameraInProgress) { + Log.d(TAG, "cycleCamera is still in progress."); + return; + } + cycleCameraInProgress = true; + } + Log.d(TAG,"cycleCamera() enter"); try { int nextCameraIndex = getNextSupportedCameraIndex(); - boolean canSwapCamera = nextCameraIndex != -1; + setCameraIndex(nextCameraIndex); + boolean canSwapCamera = getCameraIndex() != -1; // I think all devices *should* have at least one camera with // supported outputs, but adding this just in case. if (!canSwapCamera) { - throw new CameraAccessException(CameraAccessException.CAMERA_ERROR, "No cameras with supported outputs found"); + handleException(new Camera2Exception("No cameras with supported outputs found")); + } else { + swapCamera(getCameraIndex()); } - - cameraIndex = nextCameraIndex; - swapCamera(cameraIndex); - } catch (CameraAccessException e) { - e.printStackTrace(); - throw new Camera2Exception(e.getMessage()); + } catch (Exception exception) { + handleException(exception); } + Log.d(TAG,"cycleCamera() exit"); } + private boolean cycleCameraInProgress = false; + private final Object lock = new Object(); + @Override public int getCameraIndex() { return cameraIndex; } + private void setCameraIndex(int index) { + cameraIndex = index; + } + @Override public synchronized void swapCamera(int cameraId) { - Log.d(TAG,"swapCamera() enter"); + Log.d(TAG,"swapCamera() enter. cameraState = " + cameraState); CameraState oldState = cameraState; /* shutdown old camera but not the camera-callback thread */ @@ -625,12 +651,18 @@ public synchronized void swapCamera(int cameraId) { case CAPTURE: stopCapture(); break; + case ERROR: //Previous camera open attempt failed. + case CLOSED: + initCameraFrame(); + initCamera(); + startCapture(); + break; case SETUP: default: break; } /* set camera ID */ - cameraIndex = cameraId; + setCameraIndex(cameraId); executeAfterClosed = () -> { switch (oldState) { case CAPTURE: @@ -645,7 +677,6 @@ public synchronized void swapCamera(int cameraId) { }; Log.d(TAG,"swapCamera() exit"); } - private boolean isFrontCamera() { return (cameraInfoCache != null) && cameraInfoCache.isFrontFacing(); } @@ -663,10 +694,8 @@ private void stopCamThread() { try { cameraThread.quitSafely(); cameraThread.join(); - } catch (InterruptedException e) { - e.printStackTrace(); - } catch (NullPointerException e) { - // does nothing + } catch (Exception exception) { + handleException(exception); } finally { cameraThread = null; cameraThreadHandler = null; @@ -674,12 +703,13 @@ private void stopCamThread() { Log.d(TAG,"stopCamThread() exit"); } - private String selectCamera(int lenseDirection) throws CameraAccessException { + private String selectCamera(int lensDirection) throws CameraAccessException { for (String id : cameraManager.getCameraIdList()) { CameraCharacteristics info = cameraManager.getCameraCharacteristics(id); /* discard cameras that don't face the right direction */ - if (lenseDirection == info.get(CameraCharacteristics.LENS_FACING)) { - Log.d(TAG,"selectCamera() Direction the camera faces relative to device screen: " + info.get(CameraCharacteristics.LENS_FACING)); + if (lensDirection == info.get(CameraCharacteristics.LENS_FACING)) { + Log.d(TAG,"selectCamera() Direction the camera faces relative to device screen: " + + info.get(CameraCharacteristics.LENS_FACING)); return id; } } @@ -693,25 +723,26 @@ private Range selectCameraFpsRange(String camId, final int fps) throws List> fpsLst = new ArrayList<>(); Collections.addAll(fpsLst, info.get(CameraCharacteristics.CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES)); - /* sort list by error from desired fps * - * Android seems to do a better job at color correction/avoid 'dark frames' issue by - * selecting camera settings with the smallest lower bound on allowed frame rate - * range. */ - return Collections.min(fpsLst, new Comparator>() { + + Log.d(TAG,"Supported fps ranges = " + fpsLst); + Range selectedRange = Collections.min(fpsLst, new Comparator>() { @Override public int compare(Range lhs, Range rhs) { return calcError(lhs) - calcError(rhs); } private int calcError(Range val) { - return val.getLower() + Math.abs(val.getUpper() - fps); + return Math.abs(val.getLower() - fps) + Math.abs(val.getUpper() - fps); } }); + Log.d(TAG,"Desired fps = " + fps + " || Selected frame rate range = " + selectedRange); + return selectedRange; } } return null; } + private int findCameraIndex(String camId) throws CameraAccessException { String[] idList = cameraManager.getCameraIdList(); for (int ndx = 0; ndx < idList.length; ++ndx) { @@ -758,10 +789,14 @@ private int calculateCamRotation() { } private void initCameraFrame() { + if (getCameraIndex() == -1) { + Log.d(TAG," Camera Index cannot be -1. initCameraFrame() unsuccessful."); + return; + } Log.d(TAG,"initCameraFrame() enter."); try { String[] cameraIdList = cameraManager.getCameraIdList(); - String camId = cameraIdList[cameraIndex]; + String camId = cameraIdList[getCameraIndex()]; Size preferredSize = selectPreferredSize( camId, frameDimensions.getWidth(), @@ -775,39 +810,63 @@ private void initCameraFrame() { PIXEL_FORMAT, 3 ); - } catch (CameraAccessException exp) { - throw new Camera2Exception(exp.getMessage()); + } catch (Exception exception) { + handleException(exception); } + Log.d(TAG,"initCameraFrame() exit."); } - @SuppressLint("all") + @SuppressLint("MissingPermission") private void initCamera() { + if (getCameraIndex() == -1) { + Log.d(TAG," Camera Index cannot be -1. initCamera() unsuccessful."); + return; + } Log.d(TAG,"initCamera() enter."); try { cameraState = CameraState.SETUP; - // find desired camera & camera ouput size + // find desired camera & camera output size String[] cameraIdList = cameraManager.getCameraIdList(); - String camId = cameraIdList[cameraIndex]; + String camId = cameraIdList[getCameraIndex()]; camFps = selectCameraFpsRange(camId, desiredFps); cameraFrame.setOnImageAvailableListener(frameObserver, cameraThreadHandler); cameraInfoCache = new CameraInfoCache(cameraManager.getCameraCharacteristics(camId)); cameraManager.openCamera(camId, cameraObserver, null); - } catch (CameraAccessException exp) { - throw new Camera2Exception(exp.getMessage()); + } catch (Exception exception) { + Log.d(TAG,"Camera cannot be opened. Check the error message below."); + handleException(exception); } Log.d(TAG,"initCamera() exit."); } - private void postAsyncException(RuntimeException exp) { - runtimeExceptionList.add(exp); + private void handleException(Exception exception) { + cameraState = CameraState.ERROR; + synchronized (lock) { + cycleCameraInProgress = false; + } + //Log exception as an error + StringWriter writer = new StringWriter(); + PrintWriter printWriter = new PrintWriter(writer); + exception.printStackTrace(printWriter); + printWriter.flush(); + String stackTrace = writer.toString(); + Log.d(TAG,stackTrace); + + //Send the exception to client + onCaptureError(exception); } + private void startDisplayOrientationCache() { displayOrientationCache = new DisplayOrientationCache(display, cameraThreadHandler); } private void stopDisplayOrientationCache() { - cameraThreadHandler.removeCallbacks(displayOrientationCache); + if (cameraThreadHandler != null) { + if (displayOrientationCache != null) { + cameraThreadHandler.removeCallbacks(displayOrientationCache); + } + } } -} \ No newline at end of file +}