Update Linux to v5.4.2
Change-Id: Idf6911045d9d382da2cfe01b1edff026404ac8fd
diff --git a/drivers/net/veth.c b/drivers/net/veth.c
index 41a00cd..9f3c839 100644
--- a/drivers/net/veth.c
+++ b/drivers/net/veth.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0-only
/*
* drivers/net/veth.c
*
@@ -24,6 +25,7 @@
#include <linux/filter.h>
#include <linux/ptr_ring.h>
#include <linux/bpf_trace.h>
+#include <linux/net_tstamp.h>
#define DRV_NAME "veth"
#define DRV_VERSION "1.0"
@@ -36,9 +38,12 @@
#define VETH_XDP_TX BIT(0)
#define VETH_XDP_REDIR BIT(1)
-struct pcpu_vstats {
- u64 packets;
- u64 bytes;
+#define VETH_XDP_TX_BULK_SIZE 16
+
+struct veth_rq_stats {
+ u64 xdp_packets;
+ u64 xdp_bytes;
+ u64 xdp_drops;
struct u64_stats_sync syncp;
};
@@ -47,6 +52,7 @@
struct net_device *dev;
struct bpf_prog __rcu *xdp_prog;
struct xdp_mem_info xdp_mem;
+ struct veth_rq_stats stats;
bool rx_notify_masked;
struct ptr_ring xdp_ring;
struct xdp_rxq_info xdp_rxq;
@@ -60,10 +66,30 @@
unsigned int requested_headroom;
};
+struct veth_xdp_tx_bq {
+ struct xdp_frame *q[VETH_XDP_TX_BULK_SIZE];
+ unsigned int count;
+};
+
/*
* ethtool interface
*/
+struct veth_q_stat_desc {
+ char desc[ETH_GSTRING_LEN];
+ size_t offset;
+};
+
+#define VETH_RQ_STAT(m) offsetof(struct veth_rq_stats, m)
+
+static const struct veth_q_stat_desc veth_rq_stats_desc[] = {
+ { "xdp_packets", VETH_RQ_STAT(xdp_packets) },
+ { "xdp_bytes", VETH_RQ_STAT(xdp_bytes) },
+ { "xdp_drops", VETH_RQ_STAT(xdp_drops) },
+};
+
+#define VETH_RQ_STATS_LEN ARRAY_SIZE(veth_rq_stats_desc)
+
static struct {
const char string[ETH_GSTRING_LEN];
} ethtool_stats_keys[] = {
@@ -88,9 +114,21 @@
static void veth_get_strings(struct net_device *dev, u32 stringset, u8 *buf)
{
+ char *p = (char *)buf;
+ int i, j;
+
switch(stringset) {
case ETH_SS_STATS:
- memcpy(buf, ðtool_stats_keys, sizeof(ethtool_stats_keys));
+ memcpy(p, ðtool_stats_keys, sizeof(ethtool_stats_keys));
+ p += sizeof(ethtool_stats_keys);
+ for (i = 0; i < dev->real_num_rx_queues; i++) {
+ for (j = 0; j < VETH_RQ_STATS_LEN; j++) {
+ snprintf(p, ETH_GSTRING_LEN,
+ "rx_queue_%u_%.11s",
+ i, veth_rq_stats_desc[j].desc);
+ p += ETH_GSTRING_LEN;
+ }
+ }
break;
}
}
@@ -99,7 +137,8 @@
{
switch (sset) {
case ETH_SS_STATS:
- return ARRAY_SIZE(ethtool_stats_keys);
+ return ARRAY_SIZE(ethtool_stats_keys) +
+ VETH_RQ_STATS_LEN * dev->real_num_rx_queues;
default:
return -EOPNOTSUPP;
}
@@ -110,8 +149,25 @@
{
struct veth_priv *priv = netdev_priv(dev);
struct net_device *peer = rtnl_dereference(priv->peer);
+ int i, j, idx;
data[0] = peer ? peer->ifindex : 0;
+ idx = 1;
+ for (i = 0; i < dev->real_num_rx_queues; i++) {
+ const struct veth_rq_stats *rq_stats = &priv->rq[i].stats;
+ const void *stats_base = (void *)rq_stats;
+ unsigned int start;
+ size_t offset;
+
+ do {
+ start = u64_stats_fetch_begin_irq(&rq_stats->syncp);
+ for (j = 0; j < VETH_RQ_STATS_LEN; j++) {
+ offset = veth_rq_stats_desc[j].offset;
+ data[idx + j] = *(u64 *)(stats_base + offset);
+ }
+ } while (u64_stats_fetch_retry_irq(&rq_stats->syncp, start));
+ idx += VETH_RQ_STATS_LEN;
+ }
}
static const struct ethtool_ops veth_ethtool_ops = {
@@ -121,6 +177,7 @@
.get_sset_count = veth_get_sset_count,
.get_ethtool_stats = veth_get_ethtool_stats,
.get_link_ksettings = veth_get_link_ksettings,
+ .get_ts_info = ethtool_op_get_ts_info,
};
/* general routines */
@@ -201,13 +258,16 @@
skb_record_rx_queue(skb, rxq);
}
+ skb_tx_timestamp(skb);
if (likely(veth_forward_skb(rcv, skb, rq, rcv_xdp) == NET_RX_SUCCESS)) {
- struct pcpu_vstats *stats = this_cpu_ptr(dev->vstats);
+ if (!rcv_xdp) {
+ struct pcpu_lstats *stats = this_cpu_ptr(dev->lstats);
- u64_stats_update_begin(&stats->syncp);
- stats->bytes += length;
- stats->packets++;
- u64_stats_update_end(&stats->syncp);
+ u64_stats_update_begin(&stats->syncp);
+ stats->bytes += length;
+ stats->packets++;
+ u64_stats_update_end(&stats->syncp);
+ }
} else {
drop:
atomic64_inc(&priv->dropped);
@@ -221,7 +281,7 @@
return NETDEV_TX_OK;
}
-static u64 veth_stats_one(struct pcpu_vstats *result, struct net_device *dev)
+static u64 veth_stats_tx(struct pcpu_lstats *result, struct net_device *dev)
{
struct veth_priv *priv = netdev_priv(dev);
int cpu;
@@ -229,7 +289,7 @@
result->packets = 0;
result->bytes = 0;
for_each_possible_cpu(cpu) {
- struct pcpu_vstats *stats = per_cpu_ptr(dev->vstats, cpu);
+ struct pcpu_lstats *stats = per_cpu_ptr(dev->lstats, cpu);
u64 packets, bytes;
unsigned int start;
@@ -244,23 +304,58 @@
return atomic64_read(&priv->dropped);
}
+static void veth_stats_rx(struct veth_rq_stats *result, struct net_device *dev)
+{
+ struct veth_priv *priv = netdev_priv(dev);
+ int i;
+
+ result->xdp_packets = 0;
+ result->xdp_bytes = 0;
+ result->xdp_drops = 0;
+ for (i = 0; i < dev->num_rx_queues; i++) {
+ struct veth_rq_stats *stats = &priv->rq[i].stats;
+ u64 packets, bytes, drops;
+ unsigned int start;
+
+ do {
+ start = u64_stats_fetch_begin_irq(&stats->syncp);
+ packets = stats->xdp_packets;
+ bytes = stats->xdp_bytes;
+ drops = stats->xdp_drops;
+ } while (u64_stats_fetch_retry_irq(&stats->syncp, start));
+ result->xdp_packets += packets;
+ result->xdp_bytes += bytes;
+ result->xdp_drops += drops;
+ }
+}
+
static void veth_get_stats64(struct net_device *dev,
struct rtnl_link_stats64 *tot)
{
struct veth_priv *priv = netdev_priv(dev);
struct net_device *peer;
- struct pcpu_vstats one;
+ struct veth_rq_stats rx;
+ struct pcpu_lstats tx;
- tot->tx_dropped = veth_stats_one(&one, dev);
- tot->tx_bytes = one.bytes;
- tot->tx_packets = one.packets;
+ tot->tx_dropped = veth_stats_tx(&tx, dev);
+ tot->tx_bytes = tx.bytes;
+ tot->tx_packets = tx.packets;
+
+ veth_stats_rx(&rx, dev);
+ tot->rx_dropped = rx.xdp_drops;
+ tot->rx_bytes = rx.xdp_bytes;
+ tot->rx_packets = rx.xdp_packets;
rcu_read_lock();
peer = rcu_dereference(priv->peer);
if (peer) {
- tot->rx_dropped = veth_stats_one(&one, peer);
- tot->rx_bytes = one.bytes;
- tot->rx_packets = one.packets;
+ tot->rx_dropped += veth_stats_tx(&tx, peer);
+ tot->rx_bytes += tx.bytes;
+ tot->rx_packets += tx.packets;
+
+ veth_stats_rx(&rx, peer);
+ tot->tx_bytes += rx.xdp_bytes;
+ tot->tx_packets += rx.xdp_packets;
}
rcu_read_unlock();
}
@@ -299,16 +394,20 @@
{
struct veth_priv *rcv_priv, *priv = netdev_priv(dev);
struct net_device *rcv;
+ int i, ret, drops = n;
unsigned int max_len;
struct veth_rq *rq;
- int i, drops = 0;
- if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
- return -EINVAL;
+ if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) {
+ ret = -EINVAL;
+ goto drop;
+ }
rcv = rcu_dereference(priv->peer);
- if (unlikely(!rcv))
- return -ENXIO;
+ if (unlikely(!rcv)) {
+ ret = -ENXIO;
+ goto drop;
+ }
rcv_priv = netdev_priv(rcv);
rq = &rcv_priv->rq[veth_select_rxq(rcv)];
@@ -316,9 +415,12 @@
* side. This means an XDP program is loaded on the peer and the peer
* device is up.
*/
- if (!rcu_access_pointer(rq->xdp_prog))
- return -ENXIO;
+ if (!rcu_access_pointer(rq->xdp_prog)) {
+ ret = -ENXIO;
+ goto drop;
+ }
+ drops = 0;
max_len = rcv->mtu + rcv->hard_header_len + VLAN_HLEN;
spin_lock(&rq->xdp_ring.producer_lock);
@@ -337,16 +439,40 @@
if (flags & XDP_XMIT_FLUSH)
__veth_xdp_flush(rq);
- return n - drops;
+ if (likely(!drops))
+ return n;
+
+ ret = n - drops;
+drop:
+ atomic64_add(drops, &priv->dropped);
+
+ return ret;
}
-static void veth_xdp_flush(struct net_device *dev)
+static void veth_xdp_flush_bq(struct net_device *dev, struct veth_xdp_tx_bq *bq)
+{
+ int sent, i, err = 0;
+
+ sent = veth_xdp_xmit(dev, bq->count, bq->q, 0);
+ if (sent < 0) {
+ err = sent;
+ sent = 0;
+ for (i = 0; i < bq->count; i++)
+ xdp_return_frame(bq->q[i]);
+ }
+ trace_xdp_bulk_tx(dev, sent, bq->count - sent, err);
+
+ bq->count = 0;
+}
+
+static void veth_xdp_flush(struct net_device *dev, struct veth_xdp_tx_bq *bq)
{
struct veth_priv *rcv_priv, *priv = netdev_priv(dev);
struct net_device *rcv;
struct veth_rq *rq;
rcu_read_lock();
+ veth_xdp_flush_bq(dev, bq);
rcv = rcu_dereference(priv->peer);
if (unlikely(!rcv))
goto out;
@@ -362,19 +488,26 @@
rcu_read_unlock();
}
-static int veth_xdp_tx(struct net_device *dev, struct xdp_buff *xdp)
+static int veth_xdp_tx(struct net_device *dev, struct xdp_buff *xdp,
+ struct veth_xdp_tx_bq *bq)
{
struct xdp_frame *frame = convert_to_xdp_frame(xdp);
if (unlikely(!frame))
return -EOVERFLOW;
- return veth_xdp_xmit(dev, 1, &frame, 0);
+ if (unlikely(bq->count == VETH_XDP_TX_BULK_SIZE))
+ veth_xdp_flush_bq(dev, bq);
+
+ bq->q[bq->count++] = frame;
+
+ return 0;
}
static struct sk_buff *veth_xdp_rcv_one(struct veth_rq *rq,
struct xdp_frame *frame,
- unsigned int *xdp_xmit)
+ unsigned int *xdp_xmit,
+ struct veth_xdp_tx_bq *bq)
{
void *hard_start = frame->data - frame->headroom;
void *head = hard_start - sizeof(struct xdp_frame);
@@ -407,7 +540,7 @@
orig_frame = *frame;
xdp.data_hard_start = head;
xdp.rxq->mem = frame->mem;
- if (unlikely(veth_xdp_tx(rq->dev, &xdp) < 0)) {
+ if (unlikely(veth_xdp_tx(rq->dev, &xdp, bq) < 0)) {
trace_xdp_exception(rq->dev, xdp_prog, act);
frame = &orig_frame;
goto err_xdp;
@@ -428,8 +561,10 @@
goto xdp_xmit;
default:
bpf_warn_invalid_xdp_action(act);
+ /* fall through */
case XDP_ABORTED:
trace_xdp_exception(rq->dev, xdp_prog, act);
+ /* fall through */
case XDP_DROP:
goto err_xdp;
}
@@ -443,6 +578,7 @@
goto err;
}
+ xdp_release_frame(frame);
xdp_scrub_frame(frame);
skb->protocol = eth_type_trans(skb, rq->dev);
err:
@@ -455,7 +591,8 @@
}
static struct sk_buff *veth_xdp_rcv_skb(struct veth_rq *rq, struct sk_buff *skb,
- unsigned int *xdp_xmit)
+ unsigned int *xdp_xmit,
+ struct veth_xdp_tx_bq *bq)
{
u32 pktlen, headroom, act, metalen;
void *orig_data, *orig_data_end;
@@ -531,7 +668,7 @@
get_page(virt_to_page(xdp.data));
consume_skb(skb);
xdp.rxq->mem = rq->xdp_mem;
- if (unlikely(veth_xdp_tx(rq->dev, &xdp) < 0)) {
+ if (unlikely(veth_xdp_tx(rq->dev, &xdp, bq) < 0)) {
trace_xdp_exception(rq->dev, xdp_prog, act);
goto err_xdp;
}
@@ -549,8 +686,10 @@
goto xdp_xmit;
default:
bpf_warn_invalid_xdp_action(act);
+ /* fall through */
case XDP_ABORTED:
trace_xdp_exception(rq->dev, xdp_prog, act);
+ /* fall through */
case XDP_DROP:
goto drop;
}
@@ -584,30 +723,45 @@
return NULL;
}
-static int veth_xdp_rcv(struct veth_rq *rq, int budget, unsigned int *xdp_xmit)
+static int veth_xdp_rcv(struct veth_rq *rq, int budget, unsigned int *xdp_xmit,
+ struct veth_xdp_tx_bq *bq)
{
- int i, done = 0;
+ int i, done = 0, drops = 0, bytes = 0;
for (i = 0; i < budget; i++) {
void *ptr = __ptr_ring_consume(&rq->xdp_ring);
+ unsigned int xdp_xmit_one = 0;
struct sk_buff *skb;
if (!ptr)
break;
if (veth_is_xdp_frame(ptr)) {
- skb = veth_xdp_rcv_one(rq, veth_ptr_to_xdp(ptr),
- xdp_xmit);
+ struct xdp_frame *frame = veth_ptr_to_xdp(ptr);
+
+ bytes += frame->len;
+ skb = veth_xdp_rcv_one(rq, frame, &xdp_xmit_one, bq);
} else {
- skb = veth_xdp_rcv_skb(rq, ptr, xdp_xmit);
+ skb = ptr;
+ bytes += skb->len;
+ skb = veth_xdp_rcv_skb(rq, skb, &xdp_xmit_one, bq);
}
+ *xdp_xmit |= xdp_xmit_one;
if (skb)
napi_gro_receive(&rq->xdp_napi, skb);
+ else if (!xdp_xmit_one)
+ drops++;
done++;
}
+ u64_stats_update_begin(&rq->stats.syncp);
+ rq->stats.xdp_packets += done;
+ rq->stats.xdp_bytes += bytes;
+ rq->stats.xdp_drops += drops;
+ u64_stats_update_end(&rq->stats.syncp);
+
return done;
}
@@ -616,10 +770,13 @@
struct veth_rq *rq =
container_of(napi, struct veth_rq, xdp_napi);
unsigned int xdp_xmit = 0;
+ struct veth_xdp_tx_bq bq;
int done;
+ bq.count = 0;
+
xdp_set_return_frame_no_direct();
- done = veth_xdp_rcv(rq, budget, &xdp_xmit);
+ done = veth_xdp_rcv(rq, budget, &xdp_xmit, &bq);
if (done < budget && napi_complete_done(napi, done)) {
/* Write rx_notify_masked before reading ptr_ring */
@@ -631,7 +788,7 @@
}
if (xdp_xmit & VETH_XDP_TX)
- veth_xdp_flush(rq->dev);
+ veth_xdp_flush(rq->dev, &bq);
if (xdp_xmit & VETH_XDP_REDIR)
xdp_do_flush_map();
xdp_clear_return_frame_no_direct();
@@ -798,8 +955,10 @@
if (!priv->rq)
return -ENOMEM;
- for (i = 0; i < dev->num_rx_queues; i++)
+ for (i = 0; i < dev->num_rx_queues; i++) {
priv->rq[i].dev = dev;
+ u64_stats_init(&priv->rq[i].stats.syncp);
+ }
return 0;
}
@@ -815,13 +974,13 @@
{
int err;
- dev->vstats = netdev_alloc_pcpu_stats(struct pcpu_vstats);
- if (!dev->vstats)
+ dev->lstats = netdev_alloc_pcpu_stats(struct pcpu_lstats);
+ if (!dev->lstats)
return -ENOMEM;
err = veth_alloc_queues(dev);
if (err) {
- free_percpu(dev->vstats);
+ free_percpu(dev->lstats);
return err;
}
@@ -831,7 +990,7 @@
static void veth_dev_free(struct net_device *dev)
{
veth_free_queues(dev);
- free_percpu(dev->vstats);
+ free_percpu(dev->lstats);
}
#ifdef CONFIG_NET_POLL_CONTROLLER
@@ -1125,7 +1284,7 @@
return PTR_ERR(net);
peer = rtnl_create_link(net, ifname, name_assign_type,
- &veth_link_ops, tbp);
+ &veth_link_ops, tbp, extack);
if (IS_ERR(peer)) {
put_net(net);
return PTR_ERR(peer);