RICOH THETA Plug-in sample code by KA-2 for TensorFlow Lite plug-in

MainActivity.java

/**
 * Copyright 2018 Ricoh Company, Ltd.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.theta360.extendedpreview;

import android.content.Context;
import android.content.pm.ActivityInfo;
import android.os.AsyncTask;
import android.util.Log;
import android.os.Bundle;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.List;

import android.view.KeyEvent;

import com.theta360.pluginlibrary.activity.PluginActivity;
import com.theta360.pluginlibrary.callback.KeyCallback;
import com.theta360.pluginlibrary.receiver.KeyReceiver;
import com.theta360.pluginapplication.task.TakePictureTask;
import com.theta360.pluginapplication.task.TakePictureTask.Callback;
import com.theta360.pluginapplication.task.GetLiveViewTask;
import com.theta360.pluginapplication.task.MjisTimeOutTask;
import com.theta360.pluginapplication.view.MJpegInputStream;
import com.theta360.pluginapplication.oled.Oled;
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;

import android.graphics.Canvas;
import android.graphics.Paint;
import android.graphics.Color;
import android.graphics.RectF;


public class MainActivity extends PluginActivity {
    private static final String TAG = "ExtendedPreview";

    //Button Resorce
    private boolean onKeyDownModeButton = false;
    private boolean onKeyLongPressWlan = false;
    private boolean onKeyLongPressFn = false;

    //Preview Resorce
    private int previewFormatNo;
    GetLiveViewTask mGetLiveViewTask;
    private byte[]		latestLvFrame;
    private byte[]		latestFrame_Result;

    //Preview Timeout Resorce
    private static final long FRAME_READ_TIMEOUT_MSEC  = 1000;
    MjisTimeOutTask mTimeOutTask;
    MJpegInputStream mjis;

    //WebServer Resorce
    private Context context;
    private WebServer webServer;

    //OLED Dislay Resorce
    Oled oledDisplay = null;
    private boolean mFinished;



    private TakePictureTask.Callback mTakePictureTaskCallback = new Callback() {
        @Override
        public void onTakePicture(String fileUrl) {
            startPreview(mGetLiveViewTaskCallback, previewFormatNo);
        }
    };

    @Override
    protected void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_main);

        // Set enable to close by pluginlibrary, If you set false, please call close() after finishing your end processing.
        setAutoClose(true);

        setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_PORTRAIT);

        //init OLED
        oledDisplay = new Oled(getApplicationContext());
        oledDisplay.brightness(100);
        oledDisplay.clear(oledDisplay.black);
        oledDisplay.draw();

        // Set a callback when a button operation event is acquired.
        setKeyCallback(new KeyCallback() {
            @Override
            public void onKeyDown(int keyCode, KeyEvent event) {
                switch (keyCode) {
                    case KeyReceiver.KEYCODE_CAMERA :
                        stopPreview();
                        new TakePictureTask(mTakePictureTaskCallback).execute();
                        break;
                    case KeyReceiver.KEYCODE_MEDIA_RECORD :
                        // Disable onKeyUp of startup operation.
                        onKeyDownModeButton = true;
                        break;
                    default:
                        break;
                }
            }

            @Override
            public void onKeyUp(int keyCode, KeyEvent event) {

                switch (keyCode) {
                    case KeyReceiver.KEYCODE_WLAN_ON_OFF :
                        if (onKeyLongPressWlan) {
                            onKeyLongPressWlan=false;
                        } else {

                            //reset Object detection dir
                            lastDetectYaw = equiW/2; // Front
                            lastDetectPitch = equiH/2;

                        }

                        break;
                    case KeyReceiver.KEYCODE_MEDIA_RECORD :
                        if (onKeyDownModeButton) {
                            if (mGetLiveViewTask!=null) {
                                stopPreview();
                            } else {
                                startPreview(mGetLiveViewTaskCallback, previewFormatNo);
                            }
                            onKeyDownModeButton = false;
                        }
                        break;
                    case KeyEvent.KEYCODE_FUNCTION :
                        if (onKeyLongPressFn) {
                            onKeyLongPressFn=false;
                        } else {

                            //reset Object detection dir
                            lastDetectYaw = 0/*(equiW/2)*/; //Back
                            lastDetectPitch = equiH/2;

                        }

                        break;
                    default:
                        break;
                }

            }

            @Override
            public void onKeyLongPress(int keyCode, KeyEvent event) {
                switch (keyCode) {
                    case KeyReceiver.KEYCODE_WLAN_ON_OFF:
                        onKeyLongPressWlan=true;

                        //NOP : KEYCODE_WLAN_ON_OFF

                        break;
                    case KeyEvent.KEYCODE_FUNCTION :
                        onKeyLongPressFn=true;

                        //NOP : KEYCODE_FUNCTION

                        break;
                    default:
                        break;
                }

            }
        });

        this.context = getApplicationContext();
        this.webServer = new WebServer(this.context, mWebServerCallback);
        try {
            this.webServer.start();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    @Override
    protected void onResume() {
        super.onResume();

        if (isApConnected()) {

        }

        //Start LivePreview
        previewFormatNo = GetLiveViewTask.FORMAT_NO_1024_8FPS;
        startPreview(mGetLiveViewTaskCallback, previewFormatNo);

        //Start OLED thread
        mFinished = false;
        imageProcessingThread();
    }

    @Override
    protected void onPause() {
        // Do end processing
        //close();

        //Stop Web server
        this.webServer.stop();

        //Stop LivePreview
        stopPreview();

        //Stop OLED thread
        mFinished = true;

        super.onPause();
    }

    @Override
    protected void onDestroy() {
        super.onDestroy();
        if (this.webServer != null) {
            this.webServer.stop();
        }
    }

    private void startPreview(GetLiveViewTask.Callback callback, int formatNo){
        if (mGetLiveViewTask!=null) {
            stopPreview();

            try {
                Thread.sleep(400);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }

        mGetLiveViewTask = new GetLiveViewTask(callback, formatNo);
        mGetLiveViewTask.executeOnExecutor(AsyncTask.THREAD_POOL_EXECUTOR);
    }

    private void stopPreview(){
        //At the intended stop, timeout monitoring also stops.
        if (mTimeOutTask!=null) {
            mTimeOutTask.cancel(false);
            mTimeOutTask=null;
        }

        if (mGetLiveViewTask!=null) {
            mGetLiveViewTask.cancel(false);
            mGetLiveViewTask = null;
        }
    }


    /**
     * GetLiveViewTask Callback.
     */
    private GetLiveViewTask.Callback mGetLiveViewTaskCallback = new GetLiveViewTask.Callback() {

        @Override
        public void onGetResorce(MJpegInputStream inMjis) {
            mjis = inMjis;
        }

        @Override
        public void onLivePreviewFrame(byte[] previewByteArray) {
            latestLvFrame = previewByteArray;

            //Update timeout monitor
            if (mTimeOutTask!=null) {
                mTimeOutTask.cancel(false);
                mTimeOutTask=null;
            }
            mTimeOutTask = new MjisTimeOutTask(mMjisTimeOutTaskCallback, FRAME_READ_TIMEOUT_MSEC);
            mTimeOutTask.execute();
        }

        @Override
        public void onCancelled(Boolean inTimeoutOccurred) {
            mGetLiveViewTask = null;
            latestLvFrame = null;

            if (inTimeoutOccurred) {
                startPreview(mGetLiveViewTaskCallback, previewFormatNo);
            }
        }

    };


    /**
     * MjisTimeOutTask Callback.
     */
    private MjisTimeOutTask.Callback mMjisTimeOutTaskCallback = new MjisTimeOutTask.Callback() {
        @Override
        public void onTimeoutExec(){
            if (mjis!=null) {
                try {
                    // Force an IOException to `mjis.readMJpegFrame()' in GetLiveViewTask()
                    mjis.close();
                } catch (IOException e) {
                    Log.d(TAG, "[timeout] mjis.close() IOException");
                    e.printStackTrace();
                }
                mjis=null;
            }
        }
    };

    /**
     * WebServer Callback.
     */
    private WebServer.Callback mWebServerCallback = new WebServer.Callback() {
        @Override
        public void execStartPreview(int format) {
            previewFormatNo = format;
            startPreview(mGetLiveViewTaskCallback, format);
        }

        @Override
        public void execStopPreview() {
            stopPreview();
        }

        @Override
        public boolean execGetPreviewStat() {
            if (mGetLiveViewTask==null) {
                return false;
            } else {
                return true;
            }
        }

        @Override
        public byte[] getLatestFrame() {
            //return latestLvFrame;
            return latestFrame_Result;
        }
    };

    //==============================================================
    // Image processing Thread
    //==============================================================
    private static final String TF_OD_API_MODEL_FILE = "detect.tflite";
    private static final String TF_OD_API_LABELS_FILE = "file:///android_asset/labelmap.txt";
    private static final int TF_OD_API_INPUT_SIZE = 300;
    private static final boolean TF_OD_API_IS_QUANTIZED = true;

    //Object Detection dir
    private int equiW = 0;
    private int equiH = 0;

    boolean detectFlag = false;
    private int lastDetectYaw=512;
    private int lastDetectPitch=256;

    public void imageProcessingThread() {
        new Thread(new Runnable() {
            @Override
            public void run() {
                int outFps=0;
                long startTime = System.currentTimeMillis();
                android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_BACKGROUND);

                ///////////////////////////////////////////////////////////////////////
                // TFLite Initial detector
                ///////////////////////////////////////////////////////////////////////
                Classifier detector=null;
                try {
                    Log.d(TAG, "### TFLite Initial detector ###");
                    detector = TFLiteObjectDetectionAPIModel.create(
                            getAssets(),
                            TF_OD_API_MODEL_FILE,
                            TF_OD_API_LABELS_FILE,
                            TF_OD_API_INPUT_SIZE,
                            TF_OD_API_IS_QUANTIZED);
                } catch (final IOException e) {
                    e.printStackTrace();
                    Log.d(TAG, "IOException:" + e);
                    mFinished = true;
                }

                //set detection area offset
                int offsetX=0;
                int offsetY=0;


                while (mFinished == false) {
                    detectFlag = false;

                    //set detection area offset
                    if ( (previewFormatNo==GetLiveViewTask.FORMAT_NO_640_8FPS) ||
                            (previewFormatNo==GetLiveViewTask.FORMAT_NO_640_30FPS) ) {
                        offsetX = 170;
                        offsetY = 10;
                        equiW = 640;
                    } else if ( (previewFormatNo==GetLiveViewTask.FORMAT_NO_1024_8FPS) ||
                            (previewFormatNo==GetLiveViewTask.FORMAT_NO_1024_30FPS) ) {
                        offsetX = 362;
                        offsetY = 106;
                        equiW = 1024;
                    } else if ( (previewFormatNo==GetLiveViewTask.FORMAT_NO_1920_8FPS) ) {
                        offsetX = 810;
                        offsetY = 330;
                        equiW = 1920;
                    } else {
                        offsetX = 170;
                        offsetY = 10;
                        equiW = 640;
                    }
                    equiH = equiW/2;


                    byte[] jpegFrame = latestLvFrame;
                    if ( jpegFrame != null ) {

                        //JPEG -> Bitmap
                        BitmapFactory.Options options = new  BitmapFactory.Options();
                        options.inMutable = true;
                        Bitmap bitmap = BitmapFactory.decodeByteArray(jpegFrame, 0, jpegFrame.length, options);

                        //rotation yaw
                        bitmap = rotationYaw(lastDetectYaw, equiW, bitmap);

                        //crop detect area
                        Bitmap cropBitmap = Bitmap.createBitmap(bitmap, offsetX, offsetY, TF_OD_API_INPUT_SIZE, TF_OD_API_INPUT_SIZE, null, true);

                        //make result canvas
                        Canvas resultCanvas = new Canvas(bitmap);
                        Paint mPaint = new Paint();
                        mPaint.setStyle(Paint.Style.STROKE);
                        mPaint.setColor( Color.GREEN );
                        resultCanvas.drawRect(offsetX, offsetY, offsetX+TF_OD_API_INPUT_SIZE, offsetY+TF_OD_API_INPUT_SIZE, mPaint);

                        ///////////////////////////////////////////////////////////////////////
                        // TFLite Object detection
                        ///////////////////////////////////////////////////////////////////////
                        final List<Classifier.Recognition> results = detector.recognizeImage(cropBitmap);
                        Log.d(TAG, "### TFLite Object detection [result] ###");
                        for (final Classifier.Recognition result : results) {
                            drawDetectResult(result, resultCanvas, mPaint, offsetX, offsetY);
                        }

                        //set result image
                        ByteArrayOutputStream baos = new ByteArrayOutputStream();
                        bitmap.compress(Bitmap.CompressFormat.JPEG, 100, baos);
                        latestFrame_Result = baos.toByteArray();

                        outFps++;
                    } else {
                        try {
                            Thread.sleep(33);
                        } catch (InterruptedException e) {
                            e.printStackTrace();
                        }
                    }

                    //Dislpay Detect Dir to OLED
                    double lastDetectYawDig = (lastDetectYaw-equiW/2)*360.0/equiW;
                    double lastDetectPitchDig = (equiH/2-lastDetectPitch)*180/equiH;
                    displayResult(lastDetectYawDig, lastDetectPitchDig, detectFlag);

                    long curTime = System.currentTimeMillis();
                    long diffTime = curTime - startTime;
                    if (diffTime >= 1000 ) {
                        Log.d(TAG, "[OLED]" + String.valueOf(outFps) + "[fps]" );
                        startTime = curTime;
                        outFps =0;
                    }
                }
            }
        }).start();
    }


    private Bitmap rotationYaw(int inLastDetectYaw, int equiW, Bitmap inBitmap) {
        //Yaw axis rotation [Moving the detection frame]
        Log.d(TAG, "### Yaw axis rotation START [result] ###");

        //Yaw axis rotation [Image rotation]
        Bitmap rotationBmp = Bitmap.createBitmap(equiW, (equiW/2), Bitmap.Config.ARGB_8888);
        Canvas rotationCanvas = new Canvas(rotationBmp);
        if ( (equiW/2) < inLastDetectYaw ) {
            Log.d(TAG, "Case 1 [result]");

            int leftWidth = (equiW/2) + ( equiW - inLastDetectYaw ) ;

            Bitmap leftBmp = Bitmap.createBitmap(inBitmap, (inLastDetectYaw-(equiW/2)), 0, leftWidth, (equiW/2), null, true);
            Bitmap rightBmp = Bitmap.createBitmap(inBitmap, 0, 0, (inLastDetectYaw-(equiW/2)), (equiW/2), null, true);
            Paint mPaint = new Paint();
            rotationCanvas.drawBitmap(leftBmp, 0, 0, mPaint);
            rotationCanvas.drawBitmap(rightBmp, leftWidth, 0, mPaint);

        } else if ( inLastDetectYaw<(equiW/2) ) {
            Log.d(TAG, "Case 2 [result]");

            Bitmap leftBmp = Bitmap.createBitmap(inBitmap, (inLastDetectYaw+(equiW/2)), 0, ((equiW/2)-inLastDetectYaw), (equiW/2), null, true);
            Bitmap rightBmp = Bitmap.createBitmap(inBitmap, 0, 0, (inLastDetectYaw+(equiW/2)), (equiW/2), null, true);
            Paint mPaint = new Paint();
            rotationCanvas.drawBitmap(leftBmp, 0, 0, mPaint);
            rotationCanvas.drawBitmap(rightBmp, ((equiW/2)-inLastDetectYaw), 0, mPaint);

        } else {
            Log.d(TAG, "Case 3 [result]");

            Paint mPaint = new Paint();
            rotationCanvas.drawBitmap(inBitmap, 0, 0, mPaint);
        }
        Log.d(TAG, "### Yaw axis rotation END [result] ###");

        return rotationBmp;
    }


    private void drawDetectResult(Classifier.Recognition inResult, Canvas inResultCanvas, Paint inPaint, int inOffsetX, int inOffsetY){
        double confidence = Double.valueOf(inResult.getConfidence());
        if ( confidence >= 0.54 ) {
            Log.d(TAG, "[result] Title:" + inResult.getTitle());
            Log.d(TAG, "[result] Confidence:" + inResult.getConfidence());
            Log.d(TAG, "[result] Location:" + inResult.getLocation());

            // draw result
            if (confidence >= 0.56) {
                String title = inResult.getTitle();
                if ( title.equals("apple")) {
                    inPaint.setColor( Color.RED );
                } else if ( title.equals("banana") ) {
                    inPaint.setColor( Color.YELLOW );

                    detectFlag = true;
                    updateDetectInfo(inResult, inOffsetX, inOffsetY);

                } else if ( title.equals("orange") ) {
                    inPaint.setColor(Color.CYAN );
                } else {
                    inPaint.setColor( Color.BLUE );
                }
            } else {
                inPaint.setColor( Color.DKGRAY );
            }
            RectF offsetRectF = new RectF(inResult.getLocation().left, inResult.getLocation().top, inResult.getLocation().right, inResult.getLocation().bottom);
            offsetRectF.offset( (float) inOffsetX, (float) inOffsetY );
            inResultCanvas.drawRect( offsetRectF, inPaint );
            inResultCanvas.drawText(inResult.getTitle() + " : " + inResult.getConfidence(), offsetRectF.left, offsetRectF.top, inPaint);
        }
    }

    private void updateDetectInfo(Classifier.Recognition inResult, int inOffsetX, int inOffsetY){
        int tmp = lastDetectYaw;
        int curDetectYaw = (int)( inOffsetX + inResult.getLocation().left + ((inResult.getLocation().right-inResult.getLocation().left)/2) );
        if ( curDetectYaw <= (equiW/2) ) {
            lastDetectYaw -= ((equiW/2)-curDetectYaw);
        } else {
            lastDetectYaw += (curDetectYaw-(equiW/2));
        }
        if ( equiW < lastDetectYaw ) {
            lastDetectYaw -= equiW ;
        } else if (lastDetectYaw<0) {
            lastDetectYaw = equiW + lastDetectYaw;
        }
        Log.d(TAG, "[result] lastDetectYaw=" + String.valueOf(lastDetectYaw) + ", befor=" +String.valueOf(tmp) );

        int curDetectPitch = (int)( inOffsetY + inResult.getLocation().top + ((inResult.getLocation().bottom-inResult.getLocation().top)/2) );
        lastDetectPitch = curDetectPitch;
    }

    private void displayResult(double detectYawDig, double detectPitchDig, boolean inDetectFlag) {

        double lineLength = 10.0;
        double lineEndDig = detectYawDig-90.0;
        double lineEndX = lineLength * Math.cos( Math.toRadians( lineEndDig ) );
        double lineEndY = lineLength * Math.sin( Math.toRadians( lineEndDig ) );

        double arrowLength = 6.0;
        double arrowEndX1 = arrowLength * Math.cos( Math.toRadians( lineEndDig+210.0 ) );
        double arrowEndY1 = arrowLength * Math.sin( Math.toRadians( lineEndDig+210.0 ) );
        double arrowEndX2 = arrowLength * Math.cos( Math.toRadians( lineEndDig-210.0 ) );
        double arrowEndY2 = arrowLength * Math.sin( Math.toRadians( lineEndDig-210.0 ) );

        int centerX = 15;
        int centerY = 12;

        oledDisplay.clear();

        oledDisplay.circle(centerX, centerY, 11);
        oledDisplay.line(centerX, centerY, (int)(centerX+lineEndX+0.5), (int)(centerY+lineEndY+0.5));
        oledDisplay.line((int)(centerX+lineEndX+0.5), (int)(centerY+lineEndY+0.5), (int)(centerX+lineEndX+arrowEndX1+0.5), (int)(centerY+lineEndY+arrowEndY1+0.5) );
        oledDisplay.line((int)(centerX+lineEndX+0.5), (int)(centerY+lineEndY+0.5), (int)(centerX+lineEndX+arrowEndX2+0.5), (int)(centerY+lineEndY+arrowEndY2+0.5) );

        String line1Str = "";
        if (mGetLiveViewTask!=null) {
            if (inDetectFlag) {
                line1Str = "** Lock-On! **";
            } else {
                line1Str = "- can't find -";
            }
        } else {
            line1Str = "STOP Detection";
        }
        String line2Str = "Yaw   : " + String.valueOf( (int)detectYawDig );
        String line3Str = "Pitch : " + String.valueOf( (int)detectPitchDig );

        int textLine1 = 0;
        int textLine2 = 8;
        int textLine3 = 16;
        oledDisplay.setString(35, textLine1,line1Str);
        oledDisplay.setString(35, textLine2,line2Str);
        oledDisplay.setString(35, textLine3,line3Str);

        oledDisplay.draw();
    }

}