Rework canonicalizations in `SCEV::SimplifyICmpOperands`

canon_icmps_scev.cpp

// If possible, canonicalize GE/LE comparisons to GT/LT comparisons, by
// adding or subtracting 1 from one of the operands.
enum class OperandKind { LHS, RHS };

struct Rule {
  struct OpStep {
    uint64_t Adjustment;
    SCEV::NoWrapFlags Flags;
  };

  ICmpInst::Predicate PredOut;
  OpStep LHS, RHS;
};

using P = ICmpInst::Predicate;
// clang-format off
const SmallDenseMap<P, Rule, 4> CanonRules = {
  // PredIn      PredOut       LHSAdj        LHSFlags          RHSAdj  RHSFlags
  {P::ICMP_SLE, {P::ICMP_SLT, { (uint64_t)-1, SCEV::FlagNSW }, { 1, SCEV::FlagNSW }}},
  {P::ICMP_SGE, {P::ICMP_SGT, { 1, SCEV::FlagNSW }, { (uint64_t)-1, SCEV::FlagNSW }}},
  {P::ICMP_ULE, {P::ICMP_ULT, { (uint64_t)-1, SCEV::FlagAnyWrap }, { 1, SCEV::FlagNUW }}},
  {P::ICMP_UGE, {P::ICMP_UGT, { 1, SCEV::FlagNUW }, { (uint64_t)-1, SCEV::FlagAnyWrap }}},
};
// clang-format on

auto CreateCanonicalizedAddExpr = [&](const Rule &R, OperandKind OpKind) {
  const auto *&Op = OpKind == OperandKind::RHS ? RHS : LHS;
  const auto OpStep = OpKind == OperandKind::RHS ? R.RHS : R.LHS;
  Op = getAddExpr(getConstant(Op->getType(), OpStep.Adjustment, true), Op,
                  OpStep.Flags);
  Pred = R.PredOut;
  Changed = true;
};

if (auto It = CanonRules.find(Pred); It != CanonRules.end()) {
  const Rule &R = It->second;
  bool IsSigned = ICmpInst::isSigned(Pred);
  auto IsAtMax = [&](const SCEV *S) {
    return IsSigned ? getSignedRangeMax(S).isMaxSignedValue()
                    : getUnsignedRangeMax(S).isMaxValue();
  };
  auto IsAtMin = [&](const SCEV *S) {
    return IsSigned ? getSignedRangeMin(S).isMinSignedValue()
                    : getUnsignedRangeMin(S).isMinValue();
  };
  auto MayNeedFlagPreservation = [](const SCEV *S) {
    return isa<SCEVConstant>(S) ||
            (isa<SCEVAddExpr, SCEVAddRecExpr>(S) &&
            isa<SCEVConstant>(cast<SCEVNAryExpr>(S)->getOperand(0)));
  };

  if (ICmpInst::isLE(Pred)) {
    if (!IsAtMax(RHS))
      CreateCanonicalizedAddExpr(R, OperandKind::RHS);
    else if (!IsAtMin(LHS))
      CreateCanonicalizedAddExpr(R, OperandKind::LHS);
  } else {
    // For UGE, if RHS is an op we can fold the -1, try that first.
    if (!IsSigned && MayNeedFlagPreservation(RHS) && !IsAtMin(RHS))
      CreateCanonicalizedAddExpr(R, OperandKind::RHS);
    else if (!IsAtMin(RHS))
      CreateCanonicalizedAddExpr(R, OperandKind::RHS);
    else if (!IsAtMax(LHS))
      CreateCanonicalizedAddExpr(R, OperandKind::LHS);
  }
}