Minimal reproduction of CannotMixIPBasedAddressesAndIPConfigurationsOnLoadBalancerBackendAddressPool error

main.tf

locals {
    location = "Australia East"
    zones    = [1, 2, 3]
}
resource "azurerm_resource_group" "my_rg" {
    location = local.location
    name = "minimal_rg"
}

resource "azurerm_public_ip" "my_lb_public_ip_address" {
  name                = "pip-lb-1"
  resource_group_name = azurerm_resource_group.my_rg.name
  location            = local.location
  sku               = "Standard"
  allocation_method = "Static"
  zones             = local.zones
  domain_name_label = "my-dns-label"
  idle_timeout_in_minutes = 15
}

resource "azurerm_virtual_network" "my_vnet" {
  name                = "vnet-1"
  resource_group_name = azurerm_resource_group.my_rg.name
  location            = local.location
  address_space       = ["10.0.0.0/16"]
}

resource "azurerm_subnet" "my_subnet" {
  name                 = "snet-1"
  resource_group_name  = azurerm_resource_group.my_rg.name
  virtual_network_name = azurerm_virtual_network.my_vnet.name
  address_prefixes     = ["10.0.1.0/24"]
}

resource "azurerm_application_security_group" "my_app_sg" {
  name                = "asg-1"
  resource_group_name = azurerm_resource_group.my_rg.name
  location            = local.location
}

resource "azurerm_network_security_group" "my_net_sg" {
  name                = "nsg-1"
  resource_group_name = azurerm_resource_group.my_rg.name
  location            = local.location
}

resource "azurerm_subnet_network_security_group_association" "graphdb_vmss" {
  network_security_group_id = azurerm_network_security_group.my_net_sg.id
  subnet_id                 = azurerm_subnet.my_subnet.id
}

resource "azurerm_lb" "my_lb" {
  name                = "lb-1"
  resource_group_name = azurerm_resource_group.my_rg.name
  location            = local.location
  sku                 = "Standard"
    
  frontend_ip_configuration {
    name                 = "lb-frontend-ip-configuration-1"
    public_ip_address_id = azurerm_public_ip.my_lb_public_ip_address.id
  }
}

resource "azurerm_lb_backend_address_pool" "my_lb_backend_pool" {
  name                = "lb-backend-pool-1"
  loadbalancer_id     = azurerm_lb.my_lb.id
  virtual_network_id  = azurerm_virtual_network.my_vnet.id
}

resource "azurerm_lb_probe" "my_ssh_probe" {
  name                = "ssh-running-probe"
  loadbalancer_id     = azurerm_lb.my_lb.id
  protocol            = "Tcp"
  port                = 22
  interval_in_seconds = 15
}

resource "azurerm_lb_rule" "my_lb_rule" {
  loadbalancer_id                = azurerm_lb.my_lb.id
  backend_address_pool_ids       = [azurerm_lb_backend_address_pool.my_lb_backend_pool.id]
  name                           = "LBRule"
  protocol                       = "Tcp"
  frontend_port                  = 80
  backend_port                   = 80
  probe_id                       = azurerm_lb_probe.my_ssh_probe.id
  frontend_ip_configuration_name = "lb-frontend-ip-configuration-1"
}

resource "azurerm_network_interface" "my_nic" {
  resource_group_name = azurerm_resource_group.my_rg.name
  location            = local.location
  name                = "nic-1"
  ip_configuration {
    name                          = "nic-ip-config"
    primary                       = true
    subnet_id                     = azurerm_subnet.my_subnet.id
    private_ip_address_allocation = "Dynamic"
  }
}

resource "azurerm_linux_virtual_machine" "graphdb" {
  name                 = "vm-1"
  resource_group_name  = azurerm_resource_group.my_rg.name
  location             = local.location
  zone                 = local.zones[0]
  size                 = "Standard_D2_v4"
  network_interface_ids = [azurerm_network_interface.my_nic.id]


  source_image_reference {
    publisher = "Canonical"
    offer     = "0001-com-ubuntu-server-jammy"
    sku       = "22_04-lts"
    version   = "latest"
  }

  admin_ssh_key {
    username   = "ubuntu"
    public_key = file("~/.ssh/id_rsa.pub")
  }
  
  admin_username                  = "ubuntu"
  disable_password_authentication = true
  encryption_at_host_enabled      = false

  os_disk {
    caching              = "ReadWrite"
    storage_account_type = "Standard_LRS"
  }
}

resource "azurerm_network_interface_backend_address_pool_association" "my_nic_address_pool" {
  network_interface_id    = azurerm_network_interface.my_nic.id
  ip_configuration_name   = "nic-ip-config"
  backend_address_pool_id = azurerm_lb_backend_address_pool.my_lb_backend_pool.id
}

resource "azurerm_network_interface_application_security_group_association" "graphdb" {
  network_interface_id    = azurerm_network_interface.my_nic.id
  application_security_group_id = azurerm_application_security_group.my_app_sg.id
}