Displaying Camera Preview on Android with Unity

PreviewTestNew.cs

using UnityEngine;
using System.Collections;
using System.Runtime.InteropServices;
using System;
using UnityEngine.Assertions;

public class PreviewTestNew : MonoBehaviour {

    AndroidJavaClass androidNativeCam;
    AndroidJavaObject androidNativeCamActivity;

    Texture2D camTexture;

    private int texWidth;
    private int texHeight;

    // Use this for initialization
    void Start() {
#if UNITY_ANDROID
        //find the plugin
        AndroidJNI.AttachCurrentThread();
        androidNativeCam = new AndroidJavaClass("com.paloghas.cameracomponent.AndroidNativeCam");
        Assert.IsNotNull(androidNativeCam);
        androidNativeCamActivity = androidNativeCam.GetStatic<AndroidJavaObject>("mContext");

        //start cam (this will generate a texture on the java side)
        int nativeTextureID = androidNativeCamActivity.Call<int>("startCamera");
        texWidth = androidNativeCamActivity.Call<int>("getPreviewSizeWidth");
        texHeight = androidNativeCamActivity.Call<int>("getPreviewSizeHeight");
        
        Assert.IsTrue(nativeTextureID > 0, "nativeTextureID=" + nativeTextureID);
        Assert.IsTrue(nativeTextureID > 0, "width=" + texWidth);
        Assert.IsTrue(nativeTextureID > 0, "height=" + texHeight);
        
        camTexture = Texture2D.CreateExternalTexture(texWidth, texHeight, TextureFormat.YUY2, false, true, new IntPtr(nativeTextureID));
        this.GetComponent<Renderer>().material.mainTexture = camTexture; // TODO this line causes the error
#endif
    }

    // Update is called once per frame
    void Update() {
        //if (texWidth != camTexture.width || texHeight != camTexture.height)
        //{
        //    Debug.LogWarning("texWidth != camTexture.width || texHeight != camTexture.height");
        //    camTexture.Resize(texWidth, texHeight, TextureFormat.YUY2, false);
        //    camTexture.Apply();
        //}

        androidNativeCamActivity.Call("updateTexture");
        transform.Rotate(Time.deltaTime * 10, Time.deltaTime * 30, 0);
    }

    void OnGUI() {
        GUI.Label(new Rect(10, 10, Screen.width - 20, Screen.height - 20), msg);
    }
}

AndroidManifest.xml

<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android" package="com.paloghas.cameracomponent" android:theme="@android:style/Theme.NoTitleBar" android:versionName="1.0" android:versionCode="1" android:installLocation="preferExternal">
  <uses-sdk android:minSdkVersion="11" android:targetSdkVersion="22" />
  <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
  <uses-permission android:name="android.permission.INTERNET" />
  <!-- Microphone permissions -->
  <uses-permission android:name="android.permission.RECORD_AUDIO" />
  <!-- Camera -->
  <uses-permission android:name="android.permission.CAMERA" />
  <uses-feature android:name="android.hardware.camera" />
  <uses-feature android:name="android.hardware.camera.autofocus" />
  <!-- Require OpenGL ES >= 2.0. -->
  <uses-feature android:glEsVersion="0x00020000" android:required="true" />
  <application android:icon="@drawable/app_icon" android:label="@string/app_name" android:debuggable="true" android:isGame="true" android:banner="@drawable/app_banner">
    <!-- <activity android:name="com.unity3d.player.UnityPlayerNativeActivity" -->
    <activity android:name=".AndroidNativeCam" android:label="@string/app_name" android:screenOrientation="fullSensor" android:launchMode="singleTask" android:configChanges="mcc|mnc|locale|touchscreen|keyboard|keyboardHidden|navigation|orientation|screenLayout|uiMode|screenSize|smallestScreenSize|fontScale">
      <intent-filter>
        <action android:name="android.intent.action.MAIN" />
        <category android:name="android.intent.category.LAUNCHER" />
        <category android:name="android.intent.category.LEANBACK_LAUNCHER" />
      </intent-filter>
      <meta-data android:name="unityplayer.UnityActivity" android:value="true" />
      <meta-data android:name="unityplayer.ForwardNativeEventsToDalvik" android:value="false" />
    </activity>
  </application>
  <uses-permission android:name="android.permission.READ_PHONE_STATE" />
  <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
  <uses-feature android:name="android.hardware.touchscreen" android:required="false" />
  <uses-feature android:name="android.hardware.touchscreen.multitouch" android:required="false" />
  <uses-feature android:name="android.hardware.touchscreen.multitouch.distinct" android:required="false" />
</manifest>

AndroidNativeCam.java

package com.paloghas.cameracomponent;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;

import com.unity3d.player.UnityPlayerActivity;

import android.annotation.SuppressLint;
import android.content.Context;
import android.graphics.SurfaceTexture;
import android.hardware.Camera;
import android.opengl.GLES11Ext;
import android.opengl.GLES20;
import android.os.Bundle; 
import android.util.Log;

public class AndroidNativeCam extends UnityPlayerActivity implements
		SurfaceTexture.OnFrameAvailableListener {

	private static final String LOG_TAG = AndroidNativeCam.class
			.getSimpleName();

	public static Context mContext;

	private Camera mCamera;
	private SurfaceTexture texture;

	// unity texture
	private int nativeTexturePointer = -1;

	private int prevHeight;
	private int prevWidth;

//  private ByteBuffer mPixelBuf;

	@Override
	public void onCreate(Bundle savedInstanceState) {
		mContext = this;
		Log.d(LOG_TAG, "now mContext=" + mContext);
		super.onCreate(savedInstanceState);
	}

	@Override
	protected void onDestroy() {
		mCamera.stopPreview();
		mCamera.release();
	}

	/*
	 * JAVA texture creation
	 */
	public int startCamera() {
		// create the texture
		nativeTexturePointer = createExternalTexture();
		texture = new SurfaceTexture(nativeTexturePointer);
		texture.setOnFrameAvailableListener(this);

    // open the camera
		mCamera = Camera.open(); 
		setupCamera();

		Log.d(LOG_TAG, "camera opened: " + (mCamera != null));

		try {
			mCamera.setPreviewTexture(texture); 
			mCamera.startPreview();

		} catch (IOException ioe) {
			Log.w("MainActivity", "CAM LAUNCH FAILED");
		}
		
		Log.d(LOG_TAG, "nativeTexturePointer="+nativeTexturePointer);
		return nativeTexturePointer;  
	}

	@SuppressLint("NewApi")
	private void setupCamera() {
		Camera.Parameters parms = mCamera.getParameters();

		// Give the camera a hint that we're recording video. This can have a
		// big impact on frame rate.
		parms.setRecordingHint(true);
		parms.setPreviewFormat(20);

		// leave the frame rate set to default
		mCamera.setParameters(parms);

		Camera.Size mCameraPreviewSize = parms.getPreviewSize();
		prevWidth = parms.getPreviewSize().width;
		prevHeight = parms.getPreviewSize().height;
		
//		mPixelBuf = ByteBuffer.allocateDirect(prevWidth * prevHeight * 4);
//		mPixelBuf.order(ByteOrder.LITTLE_ENDIAN);

		// only for debugging output
		int[] fpsRange = new int[2];
		parms.getPreviewFpsRange(fpsRange); 
		String previewFacts = mCameraPreviewSize.width + "x"
				+ mCameraPreviewSize.height;
		if (fpsRange[0] == fpsRange[1]) {
			previewFacts += " @" + (fpsRange[0] / 1000.0) + "fps";
		} else {
			previewFacts += " @[" + (fpsRange[0] / 1000.0) + " - "
					+ (fpsRange[1] / 1000.0) + "] fps";
		}

//		previewFacts += ", supported Preview Formats: ";
//		List<Integer> formats = parms.getSupportedPreviewFormats();
//		for (int i = 0; i < formats.size(); i++) {
//			previewFacts += formats.get(i).toString() + " ";
//		}
//		Integer format = parms.getPreviewFormat();
//		previewFacts += ", Preview Format: ";
//		previewFacts += format.toString();

		Log.i(LOG_TAG, "previewFacts=" + previewFacts);
		
		checkGlError("endSetupCamera");
	}

	public void updateTexture() {
	  // check for errors at the beginning
		checkGlError("begin_updateTexture()");

		Log.d(LOG_TAG, "GLES20.glActiveTexture.."); 
		GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
		checkGlError("glActiveTexture");
		Log.d(LOG_TAG, "GLES20.glBindTexture..");
		GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,
				nativeTexturePointer);
		checkGlError("glBindTexture");

		Log.d(LOG_TAG,"ThreadID="+Thread.currentThread().getId());
		Log.d(LOG_TAG, "texture.updateTexImage..");

		texture.updateTexImage();
		checkGlError("updateTexImage");

//		mPixelBuf.rewind();
//		Log.d(LOG_TAG, "GLES20.glReadPixels..");
//		GLES20.glReadPixels(0, 0, prevWidth, prevHeight, GLES20.GL_RGBA,
//				GLES20.GL_UNSIGNED_SHORT_4_4_4_4, mPixelBuf);
//		checkGlError("glReadPixels");

//		Log.d(LOG_TAG, "mPixelBuf.get(0)=" + mPixelBuf.get(0));
	}

	public int getPreviewSizeWidth() {
		return prevWidth;
	}

	public int getPreviewSizeHeight() {

		return prevHeight;
	}

	@Override
	public void onFrameAvailable(SurfaceTexture arg0) {

		Log.d(LOG_TAG, "onFrameAvailable");
	}

	// create texture here instead by Unity
	private int createExternalTexture() {
		int[] textureIdContainer = new int[1];
		GLES20.glGenTextures(1, textureIdContainer, 0);
		GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,
				textureIdContainer[0]);

		GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,
				GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST);
		GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,
				GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
		GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,
				GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
		GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,
				GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);

		return textureIdContainer[0];
	}

  // check for OpenGL errors
	private void checkGlError(String op) {
	    int error;
	    while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) {
	        Log.e(LOG_TAG, op + ": glError 0x" + Integer.toHexString(error));
	    }
	}

}

Hey, this code seems pretty interesting!

I tried implementing it but I seem to get something wrong. My Unity-Script dies at the following line:

androidNativeCamActivity = androidNativeCam.GetStatic<AndroidJavaObject>("mContext");

The error basically says mContext is null. It seems it hasn't been initialized because onCreate of androidNativeCam hasn't been called.

The line before you initialize androidNativeCam like this:
androidNativeCam = new AndroidJavaClass("com.paloghas.cameracomponent.AndroidNativeCam");

But by just creating the Object, the onCreate-Method isn't implicitly called, right? How is mContext supposed to be initialized then?

If you could help me with this issue I would appreciate very much :)

Hello
I have been trying to make this work.
Something is wrong with texture id I guess...
camTexture = Texture2D.CreateExternalTexture(texWidth, texHeight, TextureFormat.YUY2, false, true, new IntPtr(nativeTextureID));
Did you found the solution?

"call to OpenGL ES API with no current context (logged once per thread)"
when call createExternalTexture();

please help

It do not work!
Most Android Device Camera don't support YUY2,
So “parms.setPreviewFormat(20)” will throw out an error or just come with discard texture.

Unity seems not support anymore TextureFormat.YUY2 on Android platform since 2017.x! It used to work quite good!

Any updates on that issue ?