Setup EC2 instance with two network interfaces.

multiple_interfaces_on_ec2.txt

>>> See comments

src: https://unix.stackexchange.com/questions/4420/reply-on-same-interface-as-incoming
Detailed infos: http://irp.nain-t.net/doku.php/100iproute:020_iproute2

Deploy semasim :

On load balancer:
/etc/network/interfaces

# interfaces(5) file used by ifup(8) and ifdown(8)
# Include files from /etc/network/interfaces.d:
source-directory /etc/network/interfaces.d
auto eth0
iface eth0 inet dhcp
  #172.31.19.1 - 52.58.64.189 ( semasim.com )

  # Packet that have been marked "0x1" are looked up by table t_marked
  up ip rule add fwmark 1 table t_marked

  # talbe t_marked consist in sending every packet to localhost
  # ( so they can be handled by nginx )
  up ip route add local 0.0.0.0/0 dev lo table t_marked

  # We mark 0x1 every packet that match the folowing:
  # 1) Arive on this interface. ( --in-interface eth0 )
  # 2) Are not adressed to this interface. ( ! -d 172.31.19.1/32 )
  # 3) Come from one of the interfaces affected to runing semasim instances. ( -s 172.31.20.0/24 )
  up iptables -t mangle -A PREROUTING -p tcp --in-interface eth0 ! -d 172.31.19.1/32 -s 172.31.20.0/24 -j MARK --set-xmark 0x1/0xffffffff
  up iptables -t mangle -A PREROUTING -p udp --in-interface eth0 ! -d 172.31.19.1/32 -s 172.31.20.0/24 -j MARK --set-xmark 0x1/0xffffffff

  # When a packet toward a semasim instance has to be sent
  # by nginx we ensure that it is sent via eth0
  up ip rule add from all table t_pri
  up ip route add 172.31.20.0/24 dev eth0  proto kernel  scope link  src 172.31.19.1 table t_pri

iface eth1 inet dhcp
#172.31.19.2 - 52.57.54.73 ( sip.semasim.com )

  # We ensure that packet originated from interface eth1 are
  # correctly routed.
  up ip rule add from 172.31.19.2 table t_eth1
  up ip route add default via 172.31.16.1 dev eth1 table t_eth1
  up ip route add 172.31.16.0/20 dev eth1  proto kernel  scope link  src 172.31.19.2 table t_eth1

iface eth2 inet dhcp
iface eth3 inet dhcp
iface eth4 inet dhcp
iface eth5 inet dhcp
iface eth6 inet dhcp
iface eth7 inet dhcp

/etc/iproute2/rt_tables

#
# reserved values
#
255     local
254     main
253     default
0       unspec
#
# local
#
#1      inr.ruhep
#
200 t_pri
201 t_marked
202 t_eth1

For deploy semasim !

On instances:

/etc/network/interfaces

# interfaces(5) file used by ifup(8) and ifdown(8)
# Include files from /etc/network/interfaces.d:
source-directory /etc/network/interfaces.d
auto eth0
iface eth0 inet dhcp
#172.31.21.X - access only via VPN

  #Add route to the VPN gateway
  up ip route add 10.8.0.0/24 via 172.31.19.1 dev eth0

iface eth1 inet dhcp
  #172.31.20.X - interface dedicated to running semasim instances
  #This interface cannot access internet.

  # We route all trafic originated from this interface to the
  # load balancer except for local trafic that we route normaly.
  # [/etc/iproute2/rt_ables ] 200 t_eth1
  up ip rule add from 172.31.20.0/24 table t_eth1
  up ip route add default via 172.31.19.1 dev eth1 table t_eth1
  up ip route add 172.31.16.0/20 dev eth1  proto kernel  scope link  table t_eth1

iface eth2 inet dhcp
iface eth3 inet dhcp
iface eth4 inet dhcp
iface eth5 inet dhcp
iface eth6 inet dhcp
iface eth7 inet dhcp

/etc/iproute2/rt_tables

#
# reserved values
#
255     local
254     main
253     default
0       unspec
#
# local
#
#1      inr.ruhep
#
200 t_eth1

Pour savoir dans quelle ordre sont évaluer les tables on fait ip rule list le nombre gauche représente la prio, évaluer dans l'ordre croissant.
Pour changer la priorité d'une table quand on fait ip rule add from ... table .... prio XXXX