Clicking Selenium elements with OpenCV library

button.png

example.robot

*** Settings ***

Library  BuiltIn
Library  SeleniumLibrary
Library  VisionLibrary

Test teardown  Close all browsers

*** Test Cases ***

Google search succeeds
    Open browser  https://www.google.fi/
    Wait until page contains  Google-haku
    Page should contain template  button.png
    Input text  name=q  Robot Framework
    Click template  button.png
    Wait until page contains  robotframework.org

Google search fails with missing button
    Open browser  https://www.google.fi/
    Wait until page contains  Google-haku
    Run keyword and expect error  Template not found *
    ...  Page should contain template  missing.png

missing.png

requirements.nix

# Generated by pip2nix 0.7.0.dev1
# See https://github.com/johbo/pip2nix

{ pkgs, fetchurl, fetchgit, fetchhg }:

self: super: {
  "robotframework" = super.buildPythonPackage {
    name = "robotframework-3.0.2";
    doCheck = false;
    src = fetchurl {
      url = "https://pypi.python.org/packages/3e/79/d8b9a7ea833cf4f33d51c0d5f24b825ac72105bf30c147b472da10895143/robotframework-3.0.2.tar.gz";
      sha256 = "1xqzxv00lxf9xi4vdxdsyd1bfmx18gi96vrnijpzj9w2aqrz4610";
    };
  };
  "robotframework-seleniumlibrary" = super.buildPythonPackage {
    name = "robotframework-seleniumlibrary-3.0.1";
    doCheck = false;
    propagatedBuildInputs = [
      self."selenium"
      self."robotframework"
    ];
    src = fetchurl {
      url = "https://pypi.python.org/packages/19/24/8c2259193aa8a6ca919623e2313ac5ae8d85152795c6772c1d303cd13768/robotframework-seleniumlibrary-3.0.1.tar.gz";
      sha256 = "0bkan2j39784cifgfxjl2m1zcqn4x2ygqay5plkk2icpxv0f9p8p";
    };
  };
  "selenium" = super.buildPythonPackage {
    name = "selenium-3.8.1";
    doCheck = false;
    src = fetchurl {
      url = "https://pypi.python.org/packages/59/2e/43cc1233703228e6752dc5ddfa59a254c430312c96ea6817419239fac137/selenium-3.8.1.tar.gz";
      sha256 = "1lqm2md84g11g7lqi94xqb5lydm93vgmlznfhf27g6sy9ayjvgcs";
    };
  };

### Test requirements

  
}

requirements.txt

robotframework
robotframework-seleniumlibrary

setup.cfg

[metadata]
name = robotframework-visionlibrary
version = 0.0.1.dev0

[options]
install_requires =
    robotframework
    robotframework-seleniumlibrary
py_modules =
    VisionLibrary

setup.nix

{ pkgs ? import <nixpkgs> {}
, pythonPackages ? pkgs.python2Packages
, setup ? import (pkgs.fetchFromGitHub {
    owner = "datakurre";
    repo = "setup.nix";
    rev = "b1bfe61cd2f60f5446c8e3e74e99663fdbbaf7f6";
    sha256 = "1iw3ib6zwgyqnyr9gapsrmwprawws8k331cb600724ddv30xrpkg";
  })
}:

setup {
  inherit pkgs pythonPackages;
  src = ./.;
  propagatedBuildInputs = [
    pkgs.geckodriver
    pythonPackages.opencv
  ];
}

setup.py

from setuptools import setup; setup()

VisionLibrary.py

from robot.libraries.BuiltIn import BuiltIn

import cv2
import shutil
import tempfile


class TemporaryDirectory(object):
    def __enter__(self):
        self.name = tempfile.mkdtemp()
        return self.name

    def __exit__(self, exc_type, exc_value, traceback):
        shutil.rmtree(self.name)


class VisionLibrary:

    def page_should_contain_template(self, filename, similarity=0.9):
        similarity = float(similarity)
        sl = BuiltIn().get_library_instance('SeleniumLibrary')
        with TemporaryDirectory() as path:
            sl.driver.save_screenshot(path + '/image.png')
            image = cv2.imread(path + '/image.png')
            template = cv2.imread(filename)
            result = cv2.matchTemplate(image, template, cv2.TM_CCOEFF_NORMED)
            min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(result)
            assert max_val >= similarity, \
                'Template not found ({} < {})'.format(max_val, similarity)
            return (int(max_loc[0] + template.shape[1] / 2),
                    int(max_loc[1] + template.shape[0] / 2))

    def click_template(self, filename, similarity=0.9):
        x, y = self.page_should_contain_template(filename, similarity)
        sl = BuiltIn().get_library_instance('SeleniumLibrary')
        el = sl.driver.execute_script(
            'return document.elementFromPoint({}, {});'.format(x, y))
        el.click()