For QT5 this is an easy way to use a QActionGroup or a QButtonGroup to allow for an exclusive selection/checking of Actions/Buttons or no selection at all.

CustomQActionGroup.cpp

/*
    Keywords: C++, QT, QT5, GUI, QActionGroup, QButtonGroup

    By default a QActionGroup allows for an exclusive selection of actions (only one action can be selected at a time).
    However there is no way to configure it out of the box to allow for exclusive selection or no selection at all.
    Here is an easy way how to use the QActionGroup to enable also no selection, without the need to subclass it.
*/

#include <QActionGroup>
#include <QAction>

QActionGroup * actionGroup = new QActionGroup(this);
// we implement a custom version of exclusive selection, which allows for no selection, so we have to disable the default here
actionGroup->setExclusive(false);

// define actions and add to the group (remember to make actions checkable)
QAction* action1 = new QAction("Action 1");
action1->setCheckable(true);
QAction* action2 = new QAction("Action 2");
action2->setCheckable(true);
QAction* action3 = new QAction("Action 3");
action3->setCheckable(true);
actionGroup->addAction(action1);
actionGroup->addAction(action2);
actionGroup->addAction(action3);

connect(actionGroup, &QActionGroup::triggered, this, [=](QAction* action) {
  // if an action is selected
  if (action->isChecked())
  {
    // deselect all other actions except the selected one
    QAction * action_i;
    foreach(action_i, actionGroup->actions())
      if (action != action_i) action_i->setChecked(false);
  }
  else
  {
    // at this point no action is selected and we can perform any operation that may be needed for this case
  }
});

CustomQButtonGroup.cpp

/*
  The implementation for the button group is similar to the QActionGroup, 
  however a bit uglier in the connection part because of the overloaded buttonClicked signal.
*/

#include <QButtonGroup>
#include <QPushButton>

QButtonGroup * buttonGroup = new QButtonGroup(this);
// we implement a custom version of exclusive selection, which allows for no selection, so we have to disable the default here
buttonGroup->setExclusive(false);

// Create the buttons and make them checkable
QPushButton* button1 = new QPushButton("Button 1");
button1->setCheckable(true);
QPushButton* button2 = new QPushButton("Button 2");
button2->setCheckable(true);
QPushButton* button3 = new QPushButton("Button 3");
button3->setCheckable(true);
buttonGroup->addButton(button1);
buttonGroup->addButton(button2);
buttonGroup->addButton(button3);

connect(buttonGroup, static_cast<void(QButtonGroup::*)(QAbstractButton *)>(&QButtonGroup::buttonClicked), this, [=](QAbstractButton * button) {
  // if a button is selected
  if (button->isChecked())
  {
    // deselect all other buttons except the selected one
    QAbstractButton * button_i;
    foreach(button_i, buttonGroup->buttons())
      if (button_i != button) button_i->setChecked(false);
  }
  else
  {
    // at this point no button is selected and we can perform any operation that may be needed for this case
  }
});