1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2 /* Copyright (c) 2018 Mellanox Technologies. */
3
4 #include <net/vxlan.h>
5 #include <net/gre.h>
6 #include <net/geneve.h>
7 #include <net/bareudp.h>
8 #include "en/tc_tun.h"
9 #include "en/tc_priv.h"
10 #include "en_tc.h"
11 #include "rep/tc.h"
12 #include "rep/neigh.h"
13 #include "lag/lag.h"
14 #include "lag/mp.h"
15
16 struct mlx5e_tc_tun_route_attr {
17 struct net_device *out_dev;
18 struct net_device *route_dev;
19 union {
20 struct flowi4 fl4;
21 struct flowi6 fl6;
22 } fl;
23 struct neighbour *n;
24 u8 ttl;
25 };
26
27 #define TC_TUN_ROUTE_ATTR_INIT(name) struct mlx5e_tc_tun_route_attr name = {}
28
mlx5e_tc_tun_route_attr_cleanup(struct mlx5e_tc_tun_route_attr * attr)29 static void mlx5e_tc_tun_route_attr_cleanup(struct mlx5e_tc_tun_route_attr *attr)
30 {
31 if (attr->n)
32 neigh_release(attr->n);
33 if (attr->route_dev)
34 dev_put(attr->route_dev);
35 }
36
mlx5e_get_tc_tun(struct net_device * tunnel_dev)37 struct mlx5e_tc_tunnel *mlx5e_get_tc_tun(struct net_device *tunnel_dev)
38 {
39 if (netif_is_vxlan(tunnel_dev))
40 return &vxlan_tunnel;
41 else if (netif_is_geneve(tunnel_dev))
42 return &geneve_tunnel;
43 else if (netif_is_gretap(tunnel_dev) ||
44 netif_is_ip6gretap(tunnel_dev))
45 return &gre_tunnel;
46 else if (netif_is_bareudp(tunnel_dev))
47 return &mplsoudp_tunnel;
48 else
49 return NULL;
50 }
51
get_route_and_out_devs(struct mlx5e_priv * priv,struct net_device * dev,struct net_device ** route_dev,struct net_device ** out_dev)52 static int get_route_and_out_devs(struct mlx5e_priv *priv,
53 struct net_device *dev,
54 struct net_device **route_dev,
55 struct net_device **out_dev)
56 {
57 struct net_device *uplink_dev, *uplink_upper, *real_dev;
58 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
59 bool dst_is_lag_dev;
60
61 real_dev = is_vlan_dev(dev) ? vlan_dev_real_dev(dev) : dev;
62 uplink_dev = mlx5_eswitch_uplink_get_proto_dev(esw, REP_ETH);
63
64 rcu_read_lock();
65 uplink_upper = netdev_master_upper_dev_get_rcu(uplink_dev);
66 /* mlx5_lag_is_sriov() is a blocking function which can't be called
67 * while holding rcu read lock. Take the net_device for correctness
68 * sake.
69 */
70 if (uplink_upper)
71 dev_hold(uplink_upper);
72 rcu_read_unlock();
73
74 dst_is_lag_dev = (uplink_upper &&
75 netif_is_lag_master(uplink_upper) &&
76 real_dev == uplink_upper &&
77 mlx5_lag_is_sriov(priv->mdev));
78 if (uplink_upper)
79 dev_put(uplink_upper);
80
81 /* if the egress device isn't on the same HW e-switch or
82 * it's a LAG device, use the uplink
83 */
84 *route_dev = dev;
85 if (!netdev_port_same_parent_id(priv->netdev, real_dev) ||
86 dst_is_lag_dev || is_vlan_dev(*route_dev) ||
87 netif_is_ovs_master(*route_dev))
88 *out_dev = uplink_dev;
89 else if (mlx5e_eswitch_rep(dev) &&
90 mlx5e_is_valid_eswitch_fwd_dev(priv, dev))
91 *out_dev = *route_dev;
92 else
93 return -EOPNOTSUPP;
94
95 if (!(mlx5e_eswitch_rep(*out_dev) &&
96 mlx5e_is_uplink_rep(netdev_priv(*out_dev))))
97 return -EOPNOTSUPP;
98
99 if (mlx5e_eswitch_uplink_rep(priv->netdev) && *out_dev != priv->netdev)
100 return -EOPNOTSUPP;
101
102 return 0;
103 }
104
mlx5e_route_lookup_ipv4_get(struct mlx5e_priv * priv,struct net_device * mirred_dev,struct mlx5e_tc_tun_route_attr * attr)105 static int mlx5e_route_lookup_ipv4_get(struct mlx5e_priv *priv,
106 struct net_device *mirred_dev,
107 struct mlx5e_tc_tun_route_attr *attr)
108 {
109 struct net_device *route_dev;
110 struct net_device *out_dev;
111 struct neighbour *n;
112 struct rtable *rt;
113
114 #if IS_ENABLED(CONFIG_INET)
115 struct mlx5_core_dev *mdev = priv->mdev;
116 struct net_device *uplink_dev;
117 int ret;
118
119 if (mlx5_lag_is_multipath(mdev)) {
120 struct mlx5_eswitch *esw = mdev->priv.eswitch;
121
122 uplink_dev = mlx5_eswitch_uplink_get_proto_dev(esw, REP_ETH);
123 attr->fl.fl4.flowi4_oif = uplink_dev->ifindex;
124 } else {
125 struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(mirred_dev);
126
127 if (tunnel && tunnel->get_remote_ifindex)
128 attr->fl.fl4.flowi4_oif = tunnel->get_remote_ifindex(mirred_dev);
129 }
130
131 rt = ip_route_output_key(dev_net(mirred_dev), &attr->fl.fl4);
132 if (IS_ERR(rt))
133 return PTR_ERR(rt);
134
135 if (rt->rt_type != RTN_UNICAST) {
136 ret = -ENETUNREACH;
137 goto err_rt_release;
138 }
139
140 if (mlx5_lag_is_multipath(mdev) && rt->rt_gw_family != AF_INET) {
141 ret = -ENETUNREACH;
142 goto err_rt_release;
143 }
144 #else
145 return -EOPNOTSUPP;
146 #endif
147
148 ret = get_route_and_out_devs(priv, rt->dst.dev, &route_dev, &out_dev);
149 if (ret < 0)
150 goto err_rt_release;
151 dev_hold(route_dev);
152
153 if (!attr->ttl)
154 attr->ttl = ip4_dst_hoplimit(&rt->dst);
155 n = dst_neigh_lookup(&rt->dst, &attr->fl.fl4.daddr);
156 if (!n) {
157 ret = -ENOMEM;
158 goto err_dev_release;
159 }
160
161 ip_rt_put(rt);
162 attr->route_dev = route_dev;
163 attr->out_dev = out_dev;
164 attr->n = n;
165 return 0;
166
167 err_dev_release:
168 dev_put(route_dev);
169 err_rt_release:
170 ip_rt_put(rt);
171 return ret;
172 }
173
mlx5e_route_lookup_ipv4_put(struct mlx5e_tc_tun_route_attr * attr)174 static void mlx5e_route_lookup_ipv4_put(struct mlx5e_tc_tun_route_attr *attr)
175 {
176 mlx5e_tc_tun_route_attr_cleanup(attr);
177 }
178
mlx5e_netdev_kind(struct net_device * dev)179 static const char *mlx5e_netdev_kind(struct net_device *dev)
180 {
181 if (dev->rtnl_link_ops)
182 return dev->rtnl_link_ops->kind;
183 else
184 return "unknown";
185 }
186
mlx5e_gen_ip_tunnel_header(char buf[],__u8 * ip_proto,struct mlx5e_encap_entry * e)187 static int mlx5e_gen_ip_tunnel_header(char buf[], __u8 *ip_proto,
188 struct mlx5e_encap_entry *e)
189 {
190 if (!e->tunnel) {
191 pr_warn("mlx5: Cannot generate tunnel header for this tunnel\n");
192 return -EOPNOTSUPP;
193 }
194
195 return e->tunnel->generate_ip_tun_hdr(buf, ip_proto, e);
196 }
197
gen_eth_tnl_hdr(char * buf,struct net_device * dev,struct mlx5e_encap_entry * e,u16 proto)198 static char *gen_eth_tnl_hdr(char *buf, struct net_device *dev,
199 struct mlx5e_encap_entry *e,
200 u16 proto)
201 {
202 struct ethhdr *eth = (struct ethhdr *)buf;
203 char *ip;
204
205 ether_addr_copy(eth->h_dest, e->h_dest);
206 ether_addr_copy(eth->h_source, dev->dev_addr);
207 if (is_vlan_dev(dev)) {
208 struct vlan_hdr *vlan = (struct vlan_hdr *)
209 ((char *)eth + ETH_HLEN);
210 ip = (char *)vlan + VLAN_HLEN;
211 eth->h_proto = vlan_dev_vlan_proto(dev);
212 vlan->h_vlan_TCI = htons(vlan_dev_vlan_id(dev));
213 vlan->h_vlan_encapsulated_proto = htons(proto);
214 } else {
215 eth->h_proto = htons(proto);
216 ip = (char *)eth + ETH_HLEN;
217 }
218
219 return ip;
220 }
221
mlx5e_tc_tun_create_header_ipv4(struct mlx5e_priv * priv,struct net_device * mirred_dev,struct mlx5e_encap_entry * e)222 int mlx5e_tc_tun_create_header_ipv4(struct mlx5e_priv *priv,
223 struct net_device *mirred_dev,
224 struct mlx5e_encap_entry *e)
225 {
226 int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size);
227 const struct ip_tunnel_key *tun_key = &e->tun_info->key;
228 struct mlx5_pkt_reformat_params reformat_params;
229 struct mlx5e_neigh m_neigh = {};
230 TC_TUN_ROUTE_ATTR_INIT(attr);
231 int ipv4_encap_size;
232 char *encap_header;
233 struct iphdr *ip;
234 u8 nud_state;
235 int err;
236
237 /* add the IP fields */
238 attr.fl.fl4.flowi4_tos = tun_key->tos;
239 attr.fl.fl4.daddr = tun_key->u.ipv4.dst;
240 attr.fl.fl4.saddr = tun_key->u.ipv4.src;
241 attr.ttl = tun_key->ttl;
242
243 err = mlx5e_route_lookup_ipv4_get(priv, mirred_dev, &attr);
244 if (err)
245 return err;
246
247 ipv4_encap_size =
248 (is_vlan_dev(attr.route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) +
249 sizeof(struct iphdr) +
250 e->tunnel->calc_hlen(e);
251
252 if (max_encap_size < ipv4_encap_size) {
253 mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n",
254 ipv4_encap_size, max_encap_size);
255 err = -EOPNOTSUPP;
256 goto release_neigh;
257 }
258
259 encap_header = kzalloc(ipv4_encap_size, GFP_KERNEL);
260 if (!encap_header) {
261 err = -ENOMEM;
262 goto release_neigh;
263 }
264
265 m_neigh.family = attr.n->ops->family;
266 memcpy(&m_neigh.dst_ip, attr.n->primary_key, attr.n->tbl->key_len);
267 e->out_dev = attr.out_dev;
268 e->route_dev_ifindex = attr.route_dev->ifindex;
269
270 /* It's important to add the neigh to the hash table before checking
271 * the neigh validity state. So if we'll get a notification, in case the
272 * neigh changes it's validity state, we would find the relevant neigh
273 * in the hash.
274 */
275 err = mlx5e_rep_encap_entry_attach(netdev_priv(attr.out_dev), e, &m_neigh, attr.n->dev);
276 if (err)
277 goto free_encap;
278
279 read_lock_bh(&attr.n->lock);
280 nud_state = attr.n->nud_state;
281 ether_addr_copy(e->h_dest, attr.n->ha);
282 read_unlock_bh(&attr.n->lock);
283
284 /* add ethernet header */
285 ip = (struct iphdr *)gen_eth_tnl_hdr(encap_header, attr.route_dev, e,
286 ETH_P_IP);
287
288 /* add ip header */
289 ip->tos = tun_key->tos;
290 ip->version = 0x4;
291 ip->ihl = 0x5;
292 ip->ttl = attr.ttl;
293 ip->daddr = attr.fl.fl4.daddr;
294 ip->saddr = attr.fl.fl4.saddr;
295
296 /* add tunneling protocol header */
297 err = mlx5e_gen_ip_tunnel_header((char *)ip + sizeof(struct iphdr),
298 &ip->protocol, e);
299 if (err)
300 goto destroy_neigh_entry;
301
302 e->encap_size = ipv4_encap_size;
303 e->encap_header = encap_header;
304
305 if (!(nud_state & NUD_VALID)) {
306 neigh_event_send(attr.n, NULL);
307 /* the encap entry will be made valid on neigh update event
308 * and not used before that.
309 */
310 goto release_neigh;
311 }
312
313 memset(&reformat_params, 0, sizeof(reformat_params));
314 reformat_params.type = e->reformat_type;
315 reformat_params.size = ipv4_encap_size;
316 reformat_params.data = encap_header;
317 e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, &reformat_params,
318 MLX5_FLOW_NAMESPACE_FDB);
319 if (IS_ERR(e->pkt_reformat)) {
320 err = PTR_ERR(e->pkt_reformat);
321 goto destroy_neigh_entry;
322 }
323
324 e->flags |= MLX5_ENCAP_ENTRY_VALID;
325 mlx5e_rep_queue_neigh_stats_work(netdev_priv(attr.out_dev));
326 mlx5e_route_lookup_ipv4_put(&attr);
327 return err;
328
329 destroy_neigh_entry:
330 mlx5e_rep_encap_entry_detach(netdev_priv(e->out_dev), e);
331 free_encap:
332 kfree(encap_header);
333 release_neigh:
334 mlx5e_route_lookup_ipv4_put(&attr);
335 return err;
336 }
337
mlx5e_tc_tun_update_header_ipv4(struct mlx5e_priv * priv,struct net_device * mirred_dev,struct mlx5e_encap_entry * e)338 int mlx5e_tc_tun_update_header_ipv4(struct mlx5e_priv *priv,
339 struct net_device *mirred_dev,
340 struct mlx5e_encap_entry *e)
341 {
342 int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size);
343 const struct ip_tunnel_key *tun_key = &e->tun_info->key;
344 struct mlx5_pkt_reformat_params reformat_params;
345 TC_TUN_ROUTE_ATTR_INIT(attr);
346 int ipv4_encap_size;
347 char *encap_header;
348 struct iphdr *ip;
349 u8 nud_state;
350 int err;
351
352 /* add the IP fields */
353 attr.fl.fl4.flowi4_tos = tun_key->tos;
354 attr.fl.fl4.daddr = tun_key->u.ipv4.dst;
355 attr.fl.fl4.saddr = tun_key->u.ipv4.src;
356 attr.ttl = tun_key->ttl;
357
358 err = mlx5e_route_lookup_ipv4_get(priv, mirred_dev, &attr);
359 if (err)
360 return err;
361
362 ipv4_encap_size =
363 (is_vlan_dev(attr.route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) +
364 sizeof(struct iphdr) +
365 e->tunnel->calc_hlen(e);
366
367 if (max_encap_size < ipv4_encap_size) {
368 mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n",
369 ipv4_encap_size, max_encap_size);
370 err = -EOPNOTSUPP;
371 goto release_neigh;
372 }
373
374 encap_header = kzalloc(ipv4_encap_size, GFP_KERNEL);
375 if (!encap_header) {
376 err = -ENOMEM;
377 goto release_neigh;
378 }
379
380 e->route_dev_ifindex = attr.route_dev->ifindex;
381
382 read_lock_bh(&attr.n->lock);
383 nud_state = attr.n->nud_state;
384 ether_addr_copy(e->h_dest, attr.n->ha);
385 WRITE_ONCE(e->nhe->neigh_dev, attr.n->dev);
386 read_unlock_bh(&attr.n->lock);
387
388 /* add ethernet header */
389 ip = (struct iphdr *)gen_eth_tnl_hdr(encap_header, attr.route_dev, e,
390 ETH_P_IP);
391
392 /* add ip header */
393 ip->tos = tun_key->tos;
394 ip->version = 0x4;
395 ip->ihl = 0x5;
396 ip->ttl = attr.ttl;
397 ip->daddr = attr.fl.fl4.daddr;
398 ip->saddr = attr.fl.fl4.saddr;
399
400 /* add tunneling protocol header */
401 err = mlx5e_gen_ip_tunnel_header((char *)ip + sizeof(struct iphdr),
402 &ip->protocol, e);
403 if (err)
404 goto free_encap;
405
406 e->encap_size = ipv4_encap_size;
407 kfree(e->encap_header);
408 e->encap_header = encap_header;
409
410 if (!(nud_state & NUD_VALID)) {
411 neigh_event_send(attr.n, NULL);
412 /* the encap entry will be made valid on neigh update event
413 * and not used before that.
414 */
415 goto release_neigh;
416 }
417
418 memset(&reformat_params, 0, sizeof(reformat_params));
419 reformat_params.type = e->reformat_type;
420 reformat_params.size = ipv4_encap_size;
421 reformat_params.data = encap_header;
422 e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, &reformat_params,
423 MLX5_FLOW_NAMESPACE_FDB);
424 if (IS_ERR(e->pkt_reformat)) {
425 err = PTR_ERR(e->pkt_reformat);
426 goto free_encap;
427 }
428
429 e->flags |= MLX5_ENCAP_ENTRY_VALID;
430 mlx5e_rep_queue_neigh_stats_work(netdev_priv(attr.out_dev));
431 mlx5e_route_lookup_ipv4_put(&attr);
432 return err;
433
434 free_encap:
435 kfree(encap_header);
436 release_neigh:
437 mlx5e_route_lookup_ipv4_put(&attr);
438 return err;
439 }
440
441 #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6)
mlx5e_route_lookup_ipv6_get(struct mlx5e_priv * priv,struct net_device * mirred_dev,struct mlx5e_tc_tun_route_attr * attr)442 static int mlx5e_route_lookup_ipv6_get(struct mlx5e_priv *priv,
443 struct net_device *mirred_dev,
444 struct mlx5e_tc_tun_route_attr *attr)
445 {
446 struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(mirred_dev);
447 struct net_device *route_dev;
448 struct net_device *out_dev;
449 struct dst_entry *dst;
450 struct neighbour *n;
451 int ret;
452
453 if (tunnel && tunnel->get_remote_ifindex)
454 attr->fl.fl6.flowi6_oif = tunnel->get_remote_ifindex(mirred_dev);
455 dst = ipv6_stub->ipv6_dst_lookup_flow(dev_net(mirred_dev), NULL, &attr->fl.fl6,
456 NULL);
457 if (IS_ERR(dst))
458 return PTR_ERR(dst);
459
460 if (!attr->ttl)
461 attr->ttl = ip6_dst_hoplimit(dst);
462
463 ret = get_route_and_out_devs(priv, dst->dev, &route_dev, &out_dev);
464 if (ret < 0)
465 goto err_dst_release;
466
467 dev_hold(route_dev);
468 n = dst_neigh_lookup(dst, &attr->fl.fl6.daddr);
469 if (!n) {
470 ret = -ENOMEM;
471 goto err_dev_release;
472 }
473
474 dst_release(dst);
475 attr->out_dev = out_dev;
476 attr->route_dev = route_dev;
477 attr->n = n;
478 return 0;
479
480 err_dev_release:
481 dev_put(route_dev);
482 err_dst_release:
483 dst_release(dst);
484 return ret;
485 }
486
mlx5e_route_lookup_ipv6_put(struct mlx5e_tc_tun_route_attr * attr)487 static void mlx5e_route_lookup_ipv6_put(struct mlx5e_tc_tun_route_attr *attr)
488 {
489 mlx5e_tc_tun_route_attr_cleanup(attr);
490 }
491
mlx5e_tc_tun_create_header_ipv6(struct mlx5e_priv * priv,struct net_device * mirred_dev,struct mlx5e_encap_entry * e)492 int mlx5e_tc_tun_create_header_ipv6(struct mlx5e_priv *priv,
493 struct net_device *mirred_dev,
494 struct mlx5e_encap_entry *e)
495 {
496 int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size);
497 const struct ip_tunnel_key *tun_key = &e->tun_info->key;
498 struct mlx5_pkt_reformat_params reformat_params;
499 struct mlx5e_neigh m_neigh = {};
500 TC_TUN_ROUTE_ATTR_INIT(attr);
501 struct ipv6hdr *ip6h;
502 int ipv6_encap_size;
503 char *encap_header;
504 u8 nud_state;
505 int err;
506
507 attr.ttl = tun_key->ttl;
508 attr.fl.fl6.flowlabel = ip6_make_flowinfo(RT_TOS(tun_key->tos), tun_key->label);
509 attr.fl.fl6.daddr = tun_key->u.ipv6.dst;
510 attr.fl.fl6.saddr = tun_key->u.ipv6.src;
511
512 err = mlx5e_route_lookup_ipv6_get(priv, mirred_dev, &attr);
513 if (err)
514 return err;
515
516 ipv6_encap_size =
517 (is_vlan_dev(attr.route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) +
518 sizeof(struct ipv6hdr) +
519 e->tunnel->calc_hlen(e);
520
521 if (max_encap_size < ipv6_encap_size) {
522 mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n",
523 ipv6_encap_size, max_encap_size);
524 err = -EOPNOTSUPP;
525 goto release_neigh;
526 }
527
528 encap_header = kzalloc(ipv6_encap_size, GFP_KERNEL);
529 if (!encap_header) {
530 err = -ENOMEM;
531 goto release_neigh;
532 }
533
534 m_neigh.family = attr.n->ops->family;
535 memcpy(&m_neigh.dst_ip, attr.n->primary_key, attr.n->tbl->key_len);
536 e->out_dev = attr.out_dev;
537 e->route_dev_ifindex = attr.route_dev->ifindex;
538
539 /* It's important to add the neigh to the hash table before checking
540 * the neigh validity state. So if we'll get a notification, in case the
541 * neigh changes it's validity state, we would find the relevant neigh
542 * in the hash.
543 */
544 err = mlx5e_rep_encap_entry_attach(netdev_priv(attr.out_dev), e, &m_neigh, attr.n->dev);
545 if (err)
546 goto free_encap;
547
548 read_lock_bh(&attr.n->lock);
549 nud_state = attr.n->nud_state;
550 ether_addr_copy(e->h_dest, attr.n->ha);
551 read_unlock_bh(&attr.n->lock);
552
553 /* add ethernet header */
554 ip6h = (struct ipv6hdr *)gen_eth_tnl_hdr(encap_header, attr.route_dev, e,
555 ETH_P_IPV6);
556
557 /* add ip header */
558 ip6_flow_hdr(ip6h, tun_key->tos, 0);
559 /* the HW fills up ipv6 payload len */
560 ip6h->hop_limit = attr.ttl;
561 ip6h->daddr = attr.fl.fl6.daddr;
562 ip6h->saddr = attr.fl.fl6.saddr;
563
564 /* add tunneling protocol header */
565 err = mlx5e_gen_ip_tunnel_header((char *)ip6h + sizeof(struct ipv6hdr),
566 &ip6h->nexthdr, e);
567 if (err)
568 goto destroy_neigh_entry;
569
570 e->encap_size = ipv6_encap_size;
571 e->encap_header = encap_header;
572
573 if (!(nud_state & NUD_VALID)) {
574 neigh_event_send(attr.n, NULL);
575 /* the encap entry will be made valid on neigh update event
576 * and not used before that.
577 */
578 goto release_neigh;
579 }
580
581 memset(&reformat_params, 0, sizeof(reformat_params));
582 reformat_params.type = e->reformat_type;
583 reformat_params.size = ipv6_encap_size;
584 reformat_params.data = encap_header;
585 e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, &reformat_params,
586 MLX5_FLOW_NAMESPACE_FDB);
587 if (IS_ERR(e->pkt_reformat)) {
588 err = PTR_ERR(e->pkt_reformat);
589 goto destroy_neigh_entry;
590 }
591
592 e->flags |= MLX5_ENCAP_ENTRY_VALID;
593 mlx5e_rep_queue_neigh_stats_work(netdev_priv(attr.out_dev));
594 mlx5e_route_lookup_ipv6_put(&attr);
595 return err;
596
597 destroy_neigh_entry:
598 mlx5e_rep_encap_entry_detach(netdev_priv(e->out_dev), e);
599 free_encap:
600 kfree(encap_header);
601 release_neigh:
602 mlx5e_route_lookup_ipv6_put(&attr);
603 return err;
604 }
605
mlx5e_tc_tun_update_header_ipv6(struct mlx5e_priv * priv,struct net_device * mirred_dev,struct mlx5e_encap_entry * e)606 int mlx5e_tc_tun_update_header_ipv6(struct mlx5e_priv *priv,
607 struct net_device *mirred_dev,
608 struct mlx5e_encap_entry *e)
609 {
610 int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size);
611 const struct ip_tunnel_key *tun_key = &e->tun_info->key;
612 struct mlx5_pkt_reformat_params reformat_params;
613 TC_TUN_ROUTE_ATTR_INIT(attr);
614 struct ipv6hdr *ip6h;
615 int ipv6_encap_size;
616 char *encap_header;
617 u8 nud_state;
618 int err;
619
620 attr.ttl = tun_key->ttl;
621
622 attr.fl.fl6.flowlabel = ip6_make_flowinfo(RT_TOS(tun_key->tos), tun_key->label);
623 attr.fl.fl6.daddr = tun_key->u.ipv6.dst;
624 attr.fl.fl6.saddr = tun_key->u.ipv6.src;
625
626 err = mlx5e_route_lookup_ipv6_get(priv, mirred_dev, &attr);
627 if (err)
628 return err;
629
630 ipv6_encap_size =
631 (is_vlan_dev(attr.route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) +
632 sizeof(struct ipv6hdr) +
633 e->tunnel->calc_hlen(e);
634
635 if (max_encap_size < ipv6_encap_size) {
636 mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n",
637 ipv6_encap_size, max_encap_size);
638 err = -EOPNOTSUPP;
639 goto release_neigh;
640 }
641
642 encap_header = kzalloc(ipv6_encap_size, GFP_KERNEL);
643 if (!encap_header) {
644 err = -ENOMEM;
645 goto release_neigh;
646 }
647
648 e->route_dev_ifindex = attr.route_dev->ifindex;
649
650 read_lock_bh(&attr.n->lock);
651 nud_state = attr.n->nud_state;
652 ether_addr_copy(e->h_dest, attr.n->ha);
653 WRITE_ONCE(e->nhe->neigh_dev, attr.n->dev);
654 read_unlock_bh(&attr.n->lock);
655
656 /* add ethernet header */
657 ip6h = (struct ipv6hdr *)gen_eth_tnl_hdr(encap_header, attr.route_dev, e,
658 ETH_P_IPV6);
659
660 /* add ip header */
661 ip6_flow_hdr(ip6h, tun_key->tos, 0);
662 /* the HW fills up ipv6 payload len */
663 ip6h->hop_limit = attr.ttl;
664 ip6h->daddr = attr.fl.fl6.daddr;
665 ip6h->saddr = attr.fl.fl6.saddr;
666
667 /* add tunneling protocol header */
668 err = mlx5e_gen_ip_tunnel_header((char *)ip6h + sizeof(struct ipv6hdr),
669 &ip6h->nexthdr, e);
670 if (err)
671 goto free_encap;
672
673 e->encap_size = ipv6_encap_size;
674 kfree(e->encap_header);
675 e->encap_header = encap_header;
676
677 if (!(nud_state & NUD_VALID)) {
678 neigh_event_send(attr.n, NULL);
679 /* the encap entry will be made valid on neigh update event
680 * and not used before that.
681 */
682 goto release_neigh;
683 }
684
685 memset(&reformat_params, 0, sizeof(reformat_params));
686 reformat_params.type = e->reformat_type;
687 reformat_params.size = ipv6_encap_size;
688 reformat_params.data = encap_header;
689 e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, &reformat_params,
690 MLX5_FLOW_NAMESPACE_FDB);
691 if (IS_ERR(e->pkt_reformat)) {
692 err = PTR_ERR(e->pkt_reformat);
693 goto free_encap;
694 }
695
696 e->flags |= MLX5_ENCAP_ENTRY_VALID;
697 mlx5e_rep_queue_neigh_stats_work(netdev_priv(attr.out_dev));
698 mlx5e_route_lookup_ipv6_put(&attr);
699 return err;
700
701 free_encap:
702 kfree(encap_header);
703 release_neigh:
704 mlx5e_route_lookup_ipv6_put(&attr);
705 return err;
706 }
707 #endif
708
mlx5e_tc_tun_route_lookup(struct mlx5e_priv * priv,struct mlx5_flow_spec * spec,struct mlx5_flow_attr * flow_attr)709 int mlx5e_tc_tun_route_lookup(struct mlx5e_priv *priv,
710 struct mlx5_flow_spec *spec,
711 struct mlx5_flow_attr *flow_attr)
712 {
713 struct mlx5_esw_flow_attr *esw_attr = flow_attr->esw_attr;
714 struct mlx5e_tc_int_port *int_port;
715 TC_TUN_ROUTE_ATTR_INIT(attr);
716 u16 vport_num;
717 int err = 0;
718
719 if (flow_attr->tun_ip_version == 4) {
720 /* Addresses are swapped for decap */
721 attr.fl.fl4.saddr = esw_attr->rx_tun_attr->dst_ip.v4;
722 attr.fl.fl4.daddr = esw_attr->rx_tun_attr->src_ip.v4;
723 err = mlx5e_route_lookup_ipv4_get(priv, priv->netdev, &attr);
724 }
725 #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6)
726 else if (flow_attr->tun_ip_version == 6) {
727 /* Addresses are swapped for decap */
728 attr.fl.fl6.saddr = esw_attr->rx_tun_attr->dst_ip.v6;
729 attr.fl.fl6.daddr = esw_attr->rx_tun_attr->src_ip.v6;
730 err = mlx5e_route_lookup_ipv6_get(priv, priv->netdev, &attr);
731 }
732 #endif
733 else
734 return 0;
735
736 if (err)
737 return err;
738
739 if (attr.route_dev->netdev_ops == &mlx5e_netdev_ops &&
740 mlx5e_tc_is_vf_tunnel(attr.out_dev, attr.route_dev)) {
741 err = mlx5e_tc_query_route_vport(attr.out_dev, attr.route_dev, &vport_num);
742 if (err)
743 goto out;
744
745 esw_attr->rx_tun_attr->vni = MLX5_GET(fte_match_param, spec->match_value,
746 misc_parameters.vxlan_vni);
747 esw_attr->rx_tun_attr->decap_vport = vport_num;
748 } else if (netif_is_ovs_master(attr.route_dev)) {
749 int_port = mlx5e_tc_int_port_get(mlx5e_get_int_port_priv(priv),
750 attr.route_dev->ifindex,
751 MLX5E_TC_INT_PORT_INGRESS);
752 if (IS_ERR(int_port)) {
753 err = PTR_ERR(int_port);
754 goto out;
755 }
756 esw_attr->int_port = int_port;
757 }
758
759 out:
760 if (flow_attr->tun_ip_version == 4)
761 mlx5e_route_lookup_ipv4_put(&attr);
762 #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6)
763 else if (flow_attr->tun_ip_version == 6)
764 mlx5e_route_lookup_ipv6_put(&attr);
765 #endif
766 return err;
767 }
768
mlx5e_tc_tun_device_to_offload(struct mlx5e_priv * priv,struct net_device * netdev)769 bool mlx5e_tc_tun_device_to_offload(struct mlx5e_priv *priv,
770 struct net_device *netdev)
771 {
772 struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(netdev);
773
774 if (tunnel && tunnel->can_offload(priv))
775 return true;
776 else
777 return false;
778 }
779
mlx5e_tc_tun_init_encap_attr(struct net_device * tunnel_dev,struct mlx5e_priv * priv,struct mlx5e_encap_entry * e,struct netlink_ext_ack * extack)780 int mlx5e_tc_tun_init_encap_attr(struct net_device *tunnel_dev,
781 struct mlx5e_priv *priv,
782 struct mlx5e_encap_entry *e,
783 struct netlink_ext_ack *extack)
784 {
785 struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(tunnel_dev);
786
787 if (!tunnel) {
788 e->reformat_type = -1;
789 return -EOPNOTSUPP;
790 }
791
792 return tunnel->init_encap_attr(tunnel_dev, priv, e, extack);
793 }
794
mlx5e_tc_tun_parse(struct net_device * filter_dev,struct mlx5e_priv * priv,struct mlx5_flow_spec * spec,struct flow_cls_offload * f,u8 * match_level)795 int mlx5e_tc_tun_parse(struct net_device *filter_dev,
796 struct mlx5e_priv *priv,
797 struct mlx5_flow_spec *spec,
798 struct flow_cls_offload *f,
799 u8 *match_level)
800 {
801 struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(filter_dev);
802 struct flow_rule *rule = flow_cls_offload_flow_rule(f);
803 void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
804 outer_headers);
805 void *headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
806 outer_headers);
807 struct netlink_ext_ack *extack = f->common.extack;
808 int err = 0;
809
810 if (!tunnel) {
811 netdev_warn(priv->netdev,
812 "decapsulation offload is not supported for %s net device\n",
813 mlx5e_netdev_kind(filter_dev));
814 err = -EOPNOTSUPP;
815 goto out;
816 }
817
818 *match_level = tunnel->match_level;
819
820 if (tunnel->parse_udp_ports) {
821 err = tunnel->parse_udp_ports(priv, spec, f,
822 headers_c, headers_v);
823 if (err)
824 goto out;
825 }
826
827 if (tunnel->parse_tunnel) {
828 err = tunnel->parse_tunnel(priv, spec, f,
829 headers_c, headers_v);
830 if (err)
831 goto out;
832 }
833
834 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_CONTROL)) {
835 struct flow_dissector_key_basic key_basic = {};
836 struct flow_dissector_key_basic mask_basic = {
837 .n_proto = htons(0xFFFF),
838 };
839 struct flow_match_basic match_basic = {
840 .key = &key_basic, .mask = &mask_basic,
841 };
842 struct flow_match_control match;
843 u16 addr_type;
844
845 flow_rule_match_enc_control(rule, &match);
846 addr_type = match.key->addr_type;
847
848 /* For tunnel addr_type used same key id`s as for non-tunnel */
849 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
850 struct flow_match_ipv4_addrs match;
851
852 flow_rule_match_enc_ipv4_addrs(rule, &match);
853 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
854 src_ipv4_src_ipv6.ipv4_layout.ipv4,
855 ntohl(match.mask->src));
856 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
857 src_ipv4_src_ipv6.ipv4_layout.ipv4,
858 ntohl(match.key->src));
859
860 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
861 dst_ipv4_dst_ipv6.ipv4_layout.ipv4,
862 ntohl(match.mask->dst));
863 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
864 dst_ipv4_dst_ipv6.ipv4_layout.ipv4,
865 ntohl(match.key->dst));
866
867 key_basic.n_proto = htons(ETH_P_IP);
868 mlx5e_tc_set_ethertype(priv->mdev, &match_basic, true,
869 headers_c, headers_v);
870 } else if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
871 struct flow_match_ipv6_addrs match;
872
873 flow_rule_match_enc_ipv6_addrs(rule, &match);
874 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
875 src_ipv4_src_ipv6.ipv6_layout.ipv6),
876 &match.mask->src, MLX5_FLD_SZ_BYTES(ipv6_layout,
877 ipv6));
878 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
879 src_ipv4_src_ipv6.ipv6_layout.ipv6),
880 &match.key->src, MLX5_FLD_SZ_BYTES(ipv6_layout,
881 ipv6));
882
883 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
884 dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
885 &match.mask->dst, MLX5_FLD_SZ_BYTES(ipv6_layout,
886 ipv6));
887 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
888 dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
889 &match.key->dst, MLX5_FLD_SZ_BYTES(ipv6_layout,
890 ipv6));
891
892 key_basic.n_proto = htons(ETH_P_IPV6);
893 mlx5e_tc_set_ethertype(priv->mdev, &match_basic, true,
894 headers_c, headers_v);
895 }
896 }
897
898 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_IP)) {
899 struct flow_match_ip match;
900
901 flow_rule_match_enc_ip(rule, &match);
902 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_ecn,
903 match.mask->tos & 0x3);
904 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn,
905 match.key->tos & 0x3);
906
907 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_dscp,
908 match.mask->tos >> 2);
909 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp,
910 match.key->tos >> 2);
911
912 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ttl_hoplimit,
913 match.mask->ttl);
914 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ttl_hoplimit,
915 match.key->ttl);
916
917 if (match.mask->ttl &&
918 !MLX5_CAP_ESW_FLOWTABLE_FDB
919 (priv->mdev,
920 ft_field_support.outer_ipv4_ttl)) {
921 NL_SET_ERR_MSG_MOD(extack,
922 "Matching on TTL is not supported");
923 err = -EOPNOTSUPP;
924 goto out;
925 }
926 }
927
928 /* let software handle IP fragments */
929 MLX5_SET(fte_match_set_lyr_2_4, headers_c, frag, 1);
930 MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0);
931
932 return 0;
933
934 out:
935 return err;
936 }
937
mlx5e_tc_tun_parse_udp_ports(struct mlx5e_priv * priv,struct mlx5_flow_spec * spec,struct flow_cls_offload * f,void * headers_c,void * headers_v)938 int mlx5e_tc_tun_parse_udp_ports(struct mlx5e_priv *priv,
939 struct mlx5_flow_spec *spec,
940 struct flow_cls_offload *f,
941 void *headers_c,
942 void *headers_v)
943 {
944 struct flow_rule *rule = flow_cls_offload_flow_rule(f);
945 struct netlink_ext_ack *extack = f->common.extack;
946 struct flow_match_ports enc_ports;
947
948 /* Full udp dst port must be given */
949
950 if (!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_PORTS)) {
951 NL_SET_ERR_MSG_MOD(extack,
952 "UDP tunnel decap filter must include enc_dst_port condition");
953 netdev_warn(priv->netdev,
954 "UDP tunnel decap filter must include enc_dst_port condition\n");
955 return -EOPNOTSUPP;
956 }
957
958 flow_rule_match_enc_ports(rule, &enc_ports);
959
960 if (memchr_inv(&enc_ports.mask->dst, 0xff,
961 sizeof(enc_ports.mask->dst))) {
962 NL_SET_ERR_MSG_MOD(extack,
963 "UDP tunnel decap filter must match enc_dst_port fully");
964 netdev_warn(priv->netdev,
965 "UDP tunnel decap filter must match enc_dst_port fully\n");
966 return -EOPNOTSUPP;
967 }
968
969 /* match on UDP protocol and dst port number */
970
971 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, ip_protocol);
972 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
973
974 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport,
975 ntohs(enc_ports.mask->dst));
976 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
977 ntohs(enc_ports.key->dst));
978
979 /* UDP src port on outer header is generated by HW,
980 * so it is probably a bad idea to request matching it.
981 * Nonetheless, it is allowed.
982 */
983
984 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport,
985 ntohs(enc_ports.mask->src));
986 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
987 ntohs(enc_ports.key->src));
988
989 return 0;
990 }
991