前分析過linux kernel 2.6.32的bridge轉發邏輯，下面分析一下linux kernel 3.10的bridge轉發邏輯。這樣正是CentOS 5和CentOS 7對應的內核。3.10 kernel中bridge邏輯的最大改變就是增加了vlan處理邏輯以及brdige入口函數的設置。

1. netdev_rx_handler_register

在分析之前首先要介紹一個重要函數：netdev_rx_handler_register，這個函數是2.6內核所沒有的。

netdev_rx_handler_register

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

/*

* dev: 要注冊接收函數的dev

* rx_handler: 要注冊的接收函數

* rx_handler_data: 指向rx_handler_data使用的數據

*/

int netdev_rx_handler_register(struct net_device *dev,

rx_handler_func_t *rx_handler,

void *rx_handler_data)

{

ASSERT_RTNL();

if (dev->rx_handler)

return -EBUSY;

/* Note: rx_handler_data must be set before rx_handler */

rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);

rcu_assign_pointer(dev->rx_handler, rx_handler);

return 0;

}

這個函數可以給設備（net_device）注冊接收函數，然后在__netif_receive_skb函數中根據接收skb的設備接口，再調用這個被注冊的接收函數。比如為網橋下的接口注冊br_handle_frame函數，為bonding接口注冊bond_handle_frame函數。這相對于老式的網橋處理更靈活，有了這個機制也可以在模塊中自行注冊處理函數。比如3.10中的openvswitch（OpenvSwitch在3.10已經合入了內核）創建netdev vport的函數netdev_create。

netdev_create

1

2

3

4

5

6

7

static struct vport *netdev_create(const struct vport_parms *parms)

{

struct vport *vport;

/....../

err = netdev_rx_handler_register(netdev_vport->dev, netdev_frame_hook,vport);

/....../

}

這個函數在創建netdev vport時將設備的接收函數設置為netdev_frame_hook函數，這也是整個openvswitch的入口函數，如果查看OpenvSwitch的源碼可以看到當安裝于2.6內核時這里是替換掉bridge的br_handle_frame_hook函數，從而由bridge邏輯進入OpenvSwitch邏輯。

2. Bridge轉發邏輯分析

還是先從netif_receive_skb函數分析，這個函數算是進入協議棧的入口。

netif_receive_skb

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

int netif_receive_skb(struct sk_buff *skb)

{

int ret;

if (skb_defer_rx_timestamp(skb))

return NET_RX_SUCCESS;

rcu_read_lock();

/*RPS邏輯處理，現在內核中使用了RPS機制, 將報文分散到各個cpu的接收隊列中進行負載均衡處理*/

#ifdef CONFIG_RPS

if (static_key_false(&rps_needed)) {

struct rps_dev_flow voidflow, *rflow = &voidflow;

int cpu = get_rps_cpu(skb->dev, skb, &rflow);

if (cpu >= 0) {

ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);

rcu_read_unlock();

return ret;

}

}

#endif

ret = __netif_receive_skb(skb);

rcu_read_unlock();

return ret;

}

netif_receive_skb只是對數據包進行了RPS的處理，然后調用__netif_receive_skb。

__netif_receive_skb并沒有其他多余的處理邏輯，主要調用 __netif_receive_skb_core，這個函數才真正相當于2.6內核的netif_receive_skb。以下代碼省略了和bridge無關的邏輯。

__netif_receive_skb_core

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)

{

struct packet_type *ptype, *pt_prev;

rx_handler_func_t *rx_handler;

struct net_device *orig_dev;

struct net_device *null_or_dev;

bool deliver_exact = false;

int ret = NET_RX_DROP;

__be16 type;

/*......*/

orig_dev = skb->dev;

skb_reset_network_header(skb);

pt_prev = NULL;

skb->skb_iif = skb->dev->ifindex;

/*ptype_all協議處理，tcpdump抓包就在這里*/

list_for_each_entry_rcu(ptype, &ptype_all, list) {

if (!ptype->dev || ptype->dev == skb->dev) {

if (pt_prev)

ret = deliver_skb(skb, pt_prev, orig_dev);

pt_prev = ptype;

}

}

/*調用接收設備的rx_handler*/

rx_handler = rcu_dereference(skb->dev->rx_handler);

if (rx_handler) {

if (pt_prev) {

ret = deliver_skb(skb, pt_prev, orig_dev);

pt_prev = NULL;

}

switch (rx_handler(&skb)) {

case RX_HANDLER_CONSUMED:

ret = NET_RX_SUCCESS;

goto out;

case RX_HANDLER_ANOTHER:

goto another_round;

case RX_HANDLER_EXACT:

deliver_exact = true;

case RX_HANDLER_PASS:

break;

default:

BUG();

}

}

/*根據 skb->protocol傳遞給上層協議*/

type = skb->protocol;

list_for_each_entry_rcu(ptype,&ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {

if (ptype->type == type && (ptype->dev == null_or_dev || ptype->dev == skb->dev ||ptype->dev == orig_dev)) {

if (pt_prev)

ret = deliver_skb(skb, pt_prev, orig_dev);

pt_prev = ptype;

}

}

if (pt_prev) {

if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))

goto drop;

else

ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);

} else {

drop:

atomic_long_inc(&skb->dev->rx_dropped);

kfree_skb(skb);

ret = NET_RX_DROP;

}

out:

return ret;

}

如果一個dev被添加到一個bridge（做為bridge的一個接口），的這個接口設備的rx_handler被設置為br_handle_frame函數，這是在br_add_if函數中設置的，而br_add_if (net/bridge/br_if.c)是在向網橋設備上添加接口時設置的。進入br_handle_frame也就進入了bridge的邏輯代碼。

br_add_if

1

2

3

4

5

6

int br_add_if(struct net_bridge *br, struct net_device *dev)

{

/*......*/

err = netdev_rx_handler_register(dev, br_handle_frame, p);

/*......*/

}

br_handle_frame

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

rx_handler_result_t br_handle_frame(struct sk_buff **pskb)

{

struct net_bridge_port *p;

struct sk_buff *skb = *pskb;

const unsigned char *dest = eth_hdr(skb)->h_dest;

br_should_route_hook_t *rhook;

if (unlikely(skb->pkt_type == PACKET_LOOPBACK))

return RX_HANDLER_PASS;

if (!is_valid_ether_addr(eth_hdr(skb)->h_source))

goto drop;

skb = skb_share_check(skb, GFP_ATOMIC);

if (!skb)

return RX_HANDLER_CONSUMED;

/*獲取dev對應的bridge port*/

p = br_port_get_rcu(skb->dev);

/*特殊目的mac地址的處理*/

if (unlikely(is_link_local_ether_addr(dest))) {

/*

* See IEEE 802.1D Table 7-10 Reserved addresses

*

* Assignment Value

* Bridge Group Address 01-80-C2-00-00-00

* (MAC Control) 802.3 01-80-C2-00-00-01

* (Link Aggregation) 802.3 01-80-C2-00-00-02

* 802.1X PAE address 01-80-C2-00-00-03

*

* 802.1AB LLDP 01-80-C2-00-00-0E

*

* Others reserved for future standardization

*/

switch (dest[5]) {

case 0x00: /* Bridge Group Address */

/* If STP is turned off,then must forward to keep loop detection */

if (p->br->stp_enabled == BR_NO_STP)

goto forward;

break;

case 0x01: /* IEEE MAC (Pause) */

goto drop;

default:

/* Allow selective forwarding for most other protocols */

if (p->br->group_fwd_mask & (1u << dest[5]))

goto forward;

}

/* LOCAL_IN hook點，注意經過這個hook點并不代表發送到主機協議棧（只有特殊目的mac 01-80-C2才會走到這里）*/

if (NF_HOOK(NFPROTO_BRIDGE, NF_BR_LOCAL_IN, skb, skb->dev,

NULL, br_handle_local_finish)) {

return RX_HANDLER_CONSUMED; /* consumed by filter */

} else {

*pskb = skb;

return RX_HANDLER_PASS; /* continue processing */

}

}

/*轉發邏輯*/

forward:

switch (p->state) {

case BR_STATE_FORWARDING:

rhook = rcu_dereference(br_should_route_hook);

if (rhook) {

if ((*rhook)(skb)) {

*pskb = skb;

return RX_HANDLER_PASS;

}

dest = eth_hdr(skb)->h_dest;

}

/* fall through */

case BR_STATE_LEARNING:

/*skb的目的mac和bridge的mac一樣，則將skb發往本機協議棧*/

if (ether_addr_equal(p->br->dev->dev_addr, dest))

skb->pkt_type = PACKET_HOST;

/*NF_BR_PRE_ROUTING hook點*/

NF_HOOK(NFPROTO_BRIDGE, NF_BR_PRE_ROUTING, skb, skb->dev, NULL,br_handle_frame_finish);

break;

default:

drop:

kfree_skb(skb);

}

return RX_HANDLER_CONSUMED;

}

經過NF_BR_LOCAL_IN hook點會執行br_handle_local_finish函數。

br_handle_local_finish

1

2

3

4

5

6

7

8

9

10

static int br_handle_local_finish(struct sk_buff *skb)

{

struct net_bridge_port *p = br_port_get_rcu(skb->dev);

u16 vid = 0;

/*獲取skb的vlan id(3.10的bridge支持vlan)*/

br_vlan_get_tag(skb, &vid);

/*更新bridge的mac表，注意vlan id也是參數，說明每個vlan有一個獨立的mac表*/

br_fdb_update(p->br, p, eth_hdr(skb)->h_source, vid);

return 0; /* process further */

}

經過NF_BR_PRE_ROUTING hook點會執行br_handle_frame_finish函數。

br_handle_frame_finish

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

int br_handle_frame_finish(struct sk_buff *skb)

{

const unsigned char *dest = eth_hdr(skb)->h_dest;

struct net_bridge_port *p = br_port_get_rcu(skb->dev);

struct net_bridge *br;

struct net_bridge_fdb_entry *dst;

struct net_bridge_mdb_entry *mdst;

struct sk_buff *skb2;

u16 vid = 0;

if (!p || p->state == BR_STATE_DISABLED)

goto drop;

/*這個判斷主要是vlan的相關檢查，如是否和接收接口配置的vlan相同*/

if (!br_allowed_ingress(p->br, nbp_get_vlan_info(p), skb, &vid))

goto out;

/* insert into forwarding database after filtering to avoid spoofing */

br = p->br;

/*更新轉發數據庫*/

br_fdb_update(br, p, eth_hdr(skb)->h_source, vid);

/*多播mac的處理*/

if (!is_broadcast_ether_addr(dest) && is_multicast_ether_addr(dest) &&

br_multicast_rcv(br, p, skb))

goto drop;

if (p->state == BR_STATE_LEARNING)

goto drop;

BR_INPUT_SKB_CB(skb)->brdev = br->dev;

/* The packet skb2 goes to the local host (NULL to skip). */

skb2 = NULL;

/*如果網橋被設置為混雜模式*/

if (br->dev->flags & IFF_PROMISC)

skb2 = skb;

dst = NULL;

/*如果skb的目的mac是廣播*/

if (is_broadcast_ether_addr(dest))

skb2 = skb;

else if (is_multicast_ether_addr(dest)) { /*多播*/

mdst = br_mdb_get(br, skb, vid);

if (mdst || BR_INPUT_SKB_CB_MROUTERS_ONLY(skb)) {

if ((mdst && mdst->mglist) ||

br_multicast_is_router(br))

skb2 = skb;

br_multicast_forward(mdst, skb, skb2);

skb = NULL;

if (!skb2)

goto out;

} else

skb2 = skb;

br->dev->stats.multicast++;

} else if ((dst = __br_fdb_get(br, dest, vid)) && dst->is_local) {/*目的地址是本機mac，則發往本機協議棧*/

skb2 = skb;

/* Do not forward the packet since it's local. */

skb = NULL;

}

if (skb) {

if (dst) {

dst->used = jiffies;

br_forward(dst->dst, skb, skb2); //轉發給目的接口

} else

br_flood_forward(br, skb, skb2); //找不到目的接口則廣播

}

if (skb2)

return br_pass_frame_up(skb2); //發往本機協議棧

out:

return 0;

drop:

kfree_skb(skb);

goto out;

}

{C}

我們先看發往本機協議棧的函數br_pass_frame_up。

br_pass_frame_up

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

static int br_pass_frame_up(struct sk_buff *skb)

{

struct net_device *indev, *brdev = BR_INPUT_SKB_CB(skb)->brdev;

struct net_bridge *br = netdev_priv(brdev);

//更新統計計數(略)

/* Bridge is just like any other port. Make sure the

* packet is allowed except in promisc modue when someone

* may be running packet capture.

*/

if (!(brdev->flags & IFF_PROMISC) && !br_allowed_egress(br, br_get_vlan_info(br), skb)) {

kfree_skb(skb); //如果不是混雜模式且vlan處理不合要求則丟棄

return NET_RX_DROP;

}

//vlan處理邏輯

skb = br_handle_vlan(br, br_get_vlan_info(br), skb);

if (!skb)

return NET_RX_DROP;

indev = skb->dev;

skb->dev = brdev; //重點，這里修改了skb->dev為bridge

//經過NF_BR_LOCAL_IN再次進入協議棧

return NF_HOOK(NFPROTO_BRIDGE, NF_BR_LOCAL_IN, skb, indev, NULL,

netif_receive_skb);

}

{C}

再次進入netif_receive_skb，由于skb-dev被設置成了bridge，而bridge設備的rx_handler函數是沒有被設置的，所以就不會再次進入bridge邏輯，而直接進入了主機上層協議棧。

下面看轉發邏輯，轉發邏輯主要在br_forward函數中，而br_forward主要調用__br_forward函數。

__br_forward

1

2

3

4

5

6

7

8

9

10

11

12

13

14

static void __br_forward(const struct net_bridge_port *to, struct sk_buff *skb)

{

struct net_device *indev;

//vlan處理

skb = br_handle_vlan(to->br, nbp_get_vlan_info(to), skb);

if (!skb)

return;

indev = skb->dev;

skb->dev = to->dev; //skb->dev設置為出口設備dev

skb_forward_csum(skb);

//經過NF_BR_FORWARD hook點，調用br_forward_finish

NF_HOOK(NFPROTO_BRIDGE, NF_BR_FORWARD, skb, indev, skb->dev,

br_forward_finish);

}

{C}

br_forward_finish

1

2

3

4

5

int br_forward_finish(struct sk_buff *skb)

{

//經過NF_BR_POST_ROUTING hook點,調用br_dev_queue_push_xmit

return NF_HOOK(NFPROTO_BRIDGE, NF_BR_POST_ROUTING, skb, NULL, skb->dev, br_dev_queue_push_xmit);

}

{C}

br_dev_queue_push_xmit

1

2

3

4

5

6

7

8

9

10

11

12

int br_dev_queue_push_xmit(struct sk_buff *skb)

{

/* ip_fragment doesn't copy the MAC header */

if (nf_bridge_maybe_copy_header(skb) || (packet_length(skb) > skb->dev->mtu && !skb_is_gso(skb))) {

kfree_skb(skb);

} else {

skb_push(skb, ETH_HLEN);

br_drop_fake_rtable(skb);

dev_queue_xmit(skb); //發送到鏈路層

}

return 0;

}

Skb進入dev_queue_xmit就會調用相應設備驅動的發送函數。也就出了bridge邏輯。所以整個3.10kernel的bridge轉發邏輯如下圖所示：

注意，和2.6kernel一樣，bridge的OUTPUT hook點在bridge dev的發送函數中，這里不再分析列出。

?