main.dart for MLKit version

main.dart

import 'package:flutter/material.dart';
import 'package:flutter/services.dart';
import 'package:camera/camera.dart';
import 'package:soundpool/soundpool.dart';
import 'dart:ui' as ui;
import 'dart:typed_data';
import 'package:image/image.dart' as imglib;
import 'package:flutter/foundation.dart';
import 'breedinfo.dart';
import 'package:image_gallery_saver/image_gallery_saver.dart';
import 'package:permission_handler/permission_handler.dart';
import 'package:intent/intent.dart' as android_intent;
import 'package:intent/action.dart' as android_action;
import 'package:intent/category.dart' as android_category;
import 'package:mini_ml/mini_ml.dart';
import 'package:csv/csv.dart';

void main() => runApp(MyApp());

class MyApp extends StatelessWidget {
  @override
  Widget build(BuildContext context) {
    return MaterialApp(
      home: MyHomePage(),
    );
  }
}

class MyHomePage extends StatefulWidget {
  MyHomePage({Key key}) : super(key: key);
  @override
  _MyHomePageState createState() => _MyHomePageState();
}

class _MyHomePageState extends State<MyHomePage> {
  CameraController _controller;
  bool _cameraInitialized = false;
  bool _isDetecting = false;
  String _topText = '';
  Soundpool _pool;
  int _soundId;
  CameraImage _savedImage;
  Rect _savedRect;
  Uint8List _snapShot;
  ui.Image _buttonImage;
  bool _showingWiki = false;
  bool _showSnapshot = false;
  List breedTable;

  FirebaseVisionObjectDetector objectDetector =
      FirebaseVisionObjectDetector.instance;
  FirebaseVisionLabelDetector labelDetector =
      FirebaseVisionLabelDetector.instance;

  @override
  void initState() {
    super.initState();
    SystemChrome.setEnabledSystemUIOverlays([SystemUiOverlay.bottom]);
    SystemChrome.setSystemUIOverlayStyle(SystemUiOverlayStyle.dark);
    _initializeApp();
  }

  void _initializeApp() async {
    await PermissionHandler().requestPermissions(<PermissionGroup>[
      PermissionGroup.camera,
      PermissionGroup.storage,
    ]);

    _pool = Soundpool(streamType: StreamType.notification);
    _soundId = await rootBundle
        .load("assets/178186__snapper4298__camera-click-nikon.wav")
        .then((ByteData soundData) {
      return _pool.load(soundData);
    });

    List<CameraDescription> cameras = await availableCameras();
    _controller = CameraController(cameras[0], ResolutionPreset.medium);
    _controller.initialize().then((_) async {
      _cameraInitialized = true;
      await _controller
          .startImageStream((CameraImage image) => _processCameraImage(image));
      setState(() {});
    });

    String csvTable = await rootBundle.loadString("assets/Breeds.csv");
    breedTable = CsvToListConverter(fieldDelimiter: ',').convert(csvTable);
  }

  void _processCameraImage(CameraImage image) async {
    if (_showingWiki) return;
    if (_isDetecting) return;
    _isDetecting = true;
    Future findDogFuture = _findDog(image);
    List results = await Future.wait(
        [findDogFuture, Future.delayed(Duration(milliseconds: 500))]);
    setState(() {
      _savedImage = image;
      _savedRect = results[0];
    });
    _isDetecting = false;
  }

  @override
  void dispose() {
    _controller?.dispose();
    super.dispose();
  }

  @override
  Widget build(BuildContext context) {
    return Scaffold(
        body: GestureDetector(
      onTapDown: (TapDownDetails details) async {
        double mediaHeight = MediaQuery.of(context).size.height;
        double mediaWidth = MediaQuery.of(context).size.width;
        int marginToScreen =
            ((mediaHeight * _controller.value.aspectRatio - mediaWidth) / 2)
                .round();

        if (details.localPosition.dy < mediaHeight * 0.8)
          return; // Tapped above button area
        // double mediaWidth = MediaQuery.of(context).size.width;
        double xTap = details.localPosition.dx;

        if (xTap < mediaWidth * 0.35) {
          // Left button tapped: show Gallery.
          var intent = android_intent.Intent();
          intent.setAction(android_action.Action.ACTION_MAIN);
          intent.addCategory(android_category.Category.CATEGORY_APP_GALLERY);
          intent.startActivity().catchError((e) => print(e));
        } else if (xTap < mediaWidth * 0.65) {
          //
          // Middle button tapped: process dog inside [_savedRect] in [_savedImage]
          if (_showSnapshot) {
            // Stop showing snapshot if tapped while doing so
            _showSnapshot = false;
            return;
          } else {
            _pool.play(_soundId);
            imglib.Image img = _convertCameraImage(_savedImage);
            imglib.Image convertedImage = imglib.copyRotate(img, 90);
            imglib.Image fullImage =
                imglib.copyResize(convertedImage, height: mediaHeight.round());
            imglib.Image croppedImage = fullImage;
            if (_savedRect != null) {
              _topText = await _classifyDog(img);
              imglib.drawString(
                  fullImage, imglib.arial_24, marginToScreen, 20, _topText);
              croppedImage = imglib.copyCrop(
                  convertedImage,
                  _savedRect.left.round(),
                  _savedRect.top.round(),
                  _savedRect.width.round(),
                  _savedRect.height.round());
            }

            _snapShot = imglib.encodePng(fullImage);

            imglib.Image button = imglib.copyResizeCropSquare(croppedImage, 40);
            Uint8List buttonPng = imglib.encodePng(button);
            ui.Codec codec = await ui.instantiateImageCodec(buttonPng);
            ui.FrameInfo fi = await codec.getNextFrame();
            _buttonImage = fi.image;
            await ImageGallerySaver.saveImage(_snapShot);
            // Show the snapshot with text for four seconds
            setState(() {
              _showSnapshot = true;
            });
            Future.delayed(const Duration(seconds: 4), () {
              setState(() {
                _showSnapshot = false;
              });
            });
          }
        } else {
          //
          // Right button tapped: show Wikipedia info about the dog's breed.
          _showingWiki = true;
          String breed = _topText.split('(')[0];
          await Navigator.push(context,
              MaterialPageRoute(builder: (context) => BreedInfo(breed: breed)));
          _showingWiki = false;
        }
      },
      child: Container(
        child: _cameraInitialized
            ? OverflowBox(
                maxWidth: double.infinity,
                child: AspectRatio(
                    aspectRatio: _controller.value.aspectRatio,
                    child: _showSnapshot
                        ? Stack(fit: StackFit.expand, children: <Widget>[
                            _snapShot != null
                                ? Image.memory(
                                    _snapShot,
                                  )
                                : Text('wait'),
                            CustomPaint(painter: ButtonsPainter(_buttonImage)),
                          ])
                        : Stack(fit: StackFit.expand, children: <Widget>[
                            CameraPreview(_controller),
                            CustomPaint(painter: ButtonsPainter(_buttonImage)),
                            CustomPaint(
                                painter: RectPainter(
                                    _savedRect, _controller.value.previewSize))
                            // RectPainter(_savedRect, Size(MediaQuery.of(context).size.height, MediaQuery.of(context).size.width)))
                          ])))
            : Text(
                ' Waiting for camera initialization',
                style: TextStyle(fontSize: 20),
              ),
      ),
    ));
  }

  Future<Rect> _findDog(CameraImage image) async {
    imglib.Image img = _convertCameraImage(image);
    Uint8List png = imglib.encodePng(img);
    List<VisionLabel> _onDeviceLabels =
        await labelDetector.detectFromBinary(png, false);
    bool foundDog = false;
    for (int i = 0; i < _onDeviceLabels.length; i++) {
      if (_onDeviceLabels[i].label == "Dog") foundDog = true;
    }
    if (foundDog) {
      List<VisionObject> _foundObjects =
          await objectDetector.detectFromBinary(png);
      if (_foundObjects.length > 0)
        return _foundObjects[0].bounds;
      else
        return Rect.fromLTRB(50.0, 25.0, img.height - 50.0, img.width - 25.0);
    } else
      return null;
  }

  static imglib.Image _convertCameraImage(CameraImage image) {
    int width = image.width;
    int height = image.height;
    var img = imglib.Image(width, height); // Create Image buffer
    const int hexFF = 0xFF000000;
    final int uvyButtonStride = image.planes[1].bytesPerRow;
    final int uvPixelStride = image.planes[1].bytesPerPixel;
    for (int x = 0; x < width; x++) {
      for (int y = 0; y < height; y++) {
        final int uvIndex =
            uvPixelStride * (x / 2).floor() + uvyButtonStride * (y / 2).floor();
        final int index = y * width + x;
        final yp = image.planes[0].bytes[index];
        final up = image.planes[1].bytes[uvIndex];
        final vp = image.planes[2].bytes[uvIndex];
        // Calculate pixel color
        int r = (yp + vp * 1436 / 1024 - 179).round().clamp(0, 255);
        int g = (yp - up * 46549 / 131072 + 44 - vp * 93604 / 131072 + 91)
            .round()
            .clamp(0, 255);
        int b = (yp + up * 1814 / 1024 - 227).round().clamp(0, 255);
        // color: 0x FF  FF  FF  FF
        //           A   B   G   R
        img.data[index] = hexFF | (b << 16) | (g << 8) | r;
      }
    }
    return img;
  }

  Future<String> _classifyDog(imglib.Image img) async {
    Uint8List png = imglib.encodePng(img);
    List<VisionLabel> _cloudLabels =
        await labelDetector.detectFromBinary(png, true);
    for (int i = 0; i < _cloudLabels.length; i++) {
      String foundLabel = _cloudLabels[i].label;
      if (isBreed(foundLabel)) {
        String conf = (_cloudLabels[i].confidence * 100).toStringAsFixed(0);
        return (foundLabel + ' (' + conf + '%)');
      }
    }
    return null;
  }

  bool isBreed(String label) {
    List labelWords = label.split(' ');
    for (int i = 0; i < labelWords.length; i++) {
      String word = labelWords[i];
      List<String> ignoreLabels = [
        'Dog',
        'Dog breed',
        'Canidae',
        'Vertebrate',
        'Carnivore',
        'Mammal',
        'Snout',
        'Puppy',
        'Grass',
        'Sky'
      ];
      if (!ignoreLabels.contains(word)) {
        for (int i = 0; i < breedTable.length; i++) {
          String aBreed = breedTable[i][0];
          if (aBreed.contains(word.toUpperCase())) return true;
        }
      }
    }
    return false;
  }
}

class RectPainter extends CustomPainter {
  Rect rect;
  Size imageSize;
  RectPainter(this.rect, this.imageSize);
  @override
  void paint(Canvas canvas, Size size) {
    if (rect != null) {
      final paint = Paint();
      paint.color = Colors.yellow;
      paint.style = PaintingStyle.stroke;
      paint.strokeWidth = 2.0;

      final _heightRatio = imageSize.width / size.height;
      double _widthRatio = imageSize.height / size.width;

      final rect1 = Rect.fromLTRB(
          (rect.left) / _widthRatio,
          rect.top / _heightRatio,
          (rect.right) / _widthRatio,
          rect.bottom / _heightRatio);
      canvas.drawRect(rect1, paint);
    }
  }

  @override
  bool shouldRepaint(RectPainter oldDelegate) => oldDelegate.rect != rect;
}

class ButtonsPainter extends CustomPainter {
  ui.Image buttonImage;
  ButtonsPainter(this.buttonImage);

  @override
  void paint(Canvas canvas, Size size) {
    var paint = Paint();
    // First paint black field around buttons with low opacity
    paint.color = Colors.black.withOpacity(0.1);
    Rect rect =
        Offset(0.0, size.height * 0.8) & Size(size.width, size.height * 0.2);
    canvas.drawRect(rect, paint);

    // Draw buttons at 10% from the bottom
    final double yButton = size.height * 0.9;
    paint.style = PaintingStyle.fill;
    paint.color = Colors.grey;
    final double canvasWidth = size.width;
    double xButton;
    var icon;

    // Paint left button if no  buttonImage supplied
    if (buttonImage == null) {
      xButton = canvasWidth * 0.3;
      icon = Icons.photo_library;
      canvas.drawCircle(Offset(xButton, yButton), 22.0, paint);
      var builder = ui.ParagraphBuilder(ui.ParagraphStyle(
        fontFamily: icon.fontFamily,
        fontSize: 25.0,
      ))
        ..addText(String.fromCharCode(icon.codePoint));
      var para = builder.build();
      para.layout(const ui.ParagraphConstraints(width: 100.0));
      canvas.drawParagraph(para, Offset(xButton - 12.5, yButton - 12.5));
    }

    //Paint middle button.
    xButton = canvasWidth * 0.5;
    canvas.drawCircle(Offset(xButton, yButton), 32.0, paint);
    paint.color = Colors.white;
    canvas.drawCircle(Offset(xButton, yButton), 28.0, paint);

    // Paint right button.
    xButton = canvasWidth * 0.7;
    icon = Icons.info_outline;
    paint.color = Colors.grey;
    canvas.drawCircle(Offset(xButton, yButton), 22.0, paint);
    var builder = ui.ParagraphBuilder(ui.ParagraphStyle(
      fontFamily: icon.fontFamily,
      fontSize: 25.0,
    ))
      ..addText(String.fromCharCode(icon.codePoint));
    var para = builder.build();
    para.layout(const ui.ParagraphConstraints(width: 100.0));
    canvas.drawParagraph(para, Offset(xButton - 12.5, yButton - 12.5));

    // Paint image on left button
    if (buttonImage != null) {
      xButton = canvasWidth * 0.3;
      // First set up a round clipping area.
      double radius = 22.0;
      double l, t, r, b;
      l = xButton - radius;
      r = xButton + radius;
      t = yButton - radius;
      b = yButton + radius;
      ui.Rect clippingRect = Rect.fromLTRB(l, t, r, b);
      RRect clippingArea =
          RRect.fromRectAndRadius(clippingRect, Radius.circular(radius));
      canvas.clipRRect(clippingArea);
      // Then draw the square button image over the round clipping area
      double x, y = 0.0;
      x = xButton - buttonImage.height / 2.0;
      y = yButton - buttonImage.width / 2.0;
      Offset buttonOffset = Offset(x, y);
      canvas.drawImage(buttonImage, buttonOffset, Paint());
    }
  }

  @override
  bool shouldRepaint(ButtonsPainter oldDelegate) =>
      oldDelegate.buttonImage != buttonImage;
}

Hi Bo, I have a pixel 3a i am testing on, and the image from the _convertCameraImage function returns a vertically striped image, see attached. I'd appreciate your thoughts, maybe there are other yuv formats that should be taken into account?

Hi Kevin! Thanks for your report. I know that the _convertCameraImage function does not work on the old Samsung Galaxy S4, but I was hoping all newer phones were OK. I don't have access to a Pixel 3a so it is difficult for me to debug this. Does it work with the front-facing camera?