SchOS/ldls/kpi/skb.c

// SPDX-License-Identifier: Apache-2.0
//
// Socket buffers. See ldls/kpi/kpi.h for the provenance rule.
//
// An `sk_buff` is the unit a Linux driver receives into and transmits from, so
// a host that cannot make one cannot carry a packet. It is also the structure
// with the most fields a driver touches directly, which is why every offset
// here comes from BTF rather than from anything written down.
//
// The layout is worth one note because it is not what the names suggest:
// `head` and `data` are pointers, but `tail` and `end` are **32-bit offsets
// from head**, not pointers. Treating them as pointers produces something that
// works for small packets on a lucky allocation and then does not.

#include "types.h"
#include "user.h"
#include "posix.h"
#include "kpi.h"

// Room in front of the data for headers a stack might push. Linux reserves
// this so that adding an Ethernet header does not mean copying the payload;
// nothing here pushes anything, but a driver may rely on `skb_reserve` having
// somewhere to go.
#define KPI_HEADROOM 64

// Room past `end` for `struct skb_shared_info`.
//
// `skb_shinfo(skb)` is `(struct skb_shared_info *)(skb->head + skb->end)` --
// it is not a field, it lives immediately after the data, and every allocator
// in Linux reserves space for it. 8139too never looked, so an skb that ended
// exactly at `end` worked; e1000 reads `nr_frags` on every single transmit,
// and read it from whatever followed the allocation.
//
// The struct is 320 bytes on x86-64 and this is deliberately more. It is not
// read from BTF only because getting it wrong in the small direction is
// silent, and a fixed size that is comfortably too large cannot be.
#define KPI_SHINFO 512

static unsigned int
rd32(void *base, unsigned int off)
{
  unsigned int v;
  unsigned char *p = (unsigned char *)base + off;
  int i;
  for (i = 0; i < 4; i++) ((unsigned char *)&v)[i] = p[i];
  return v;
}

static void
wr32(void *base, unsigned int off, unsigned int v)
{
  unsigned char *p = (unsigned char *)base + off;
  int i;
  for (i = 0; i < 4; i++) p[i] = ((unsigned char *)&v)[i];
}

static void *
rdptr(void *base, unsigned int off)
{
  void *v;
  unsigned char *p = (unsigned char *)base + off;
  int i;
  for (i = 0; i < 8; i++) ((unsigned char *)&v)[i] = p[i];
  return v;
}

static void
wrptr(void *base, unsigned int off, void *v)
{
  unsigned char *p = (unsigned char *)base + off;
  int i;
  for (i = 0; i < 8; i++) p[i] = ((unsigned char *)&v)[i];
}

// One buffer: the control block, then headroom, then the frame.
//
// Allocated in one piece so that freeing is a no-op like every other
// allocation here -- see the note in kpi.c about a hosted driver's lifetime.
void *
kpi_skb_alloc(unsigned int len)
{
  unsigned char *p;
  unsigned int total = kpi_l.skb_size + KPI_HEADROOM + len + 64 + KPI_SHINFO;
  unsigned char *buf;

  if (kpi_l.skb_size == 0)
    return 0;

  // From the DMA arena, not from mmap.
  //
  // A driver that maps an skb for the device -- e1000 does, 8139too did not --
  // needs the bus address of its data, and a process can only know that for
  // memory the kernel handed it through PCIIOC_DMA. Putting the control block
  // in there too wastes a little arena and keeps one buffer one allocation.
  p = kpi_dma_heap(total, 64);
  if (p == 0)
    return 0;
  {
    unsigned int i;
    for (i = 0; i < total; i++)
      p[i] = 0;                  // the arena is recycled; mmap's zeroes are not
  }

  buf = p + kpi_l.skb_size;
  wrptr(p, kpi_l.skb_head, buf);
  wrptr(p, kpi_l.skb_data, buf + KPI_HEADROOM);
  wr32(p, kpi_l.skb_tail, KPI_HEADROOM);
  wr32(p, kpi_l.skb_end,  KPI_HEADROOM + len + 64);
  wr32(p, kpi_l.skb_len,  0);
  return p;
}

// napi_build_skb(data, frag_len) -- an skb wrapped around a buffer that
// already exists.
//
// The other half of the receive path, and the one that carries anything
// larger than `copybreak` (256 bytes by default). e1000 hands the device a
// raw fragment, and when a frame arrives it wraps that same fragment rather
// than copying it -- so the control block is allocated here and the data is
// not. A DHCP offer is 590 bytes, which is why nothing larger than a ping
// reply worked until this existed.
//
// `skb_shared_info` lives at the *end of the buffer* in this arrangement,
// which is what makes `end` smaller than the fragment: the driver picked a
// fragment size that already includes room for it.
void *
kpi_build_skb(void *data, unsigned int frag_len)
{
  unsigned char *p;
  unsigned int i, room;

  if (kpi_l.skb_size == 0 || data == 0 || frag_len <= KPI_SHINFO)
    return 0;

  p = kpi_dma_heap(kpi_l.skb_size, 64);
  if (!p)
    return 0;
  for (i = 0; i < kpi_l.skb_size; i++)
    p[i] = 0;

  room = frag_len - KPI_SHINFO;
  // And zero the shared info, because this buffer is recycled arena and the
  // driver reads nr_frags out of it without writing it first.
  for (i = 0; i < KPI_SHINFO; i++)
    ((unsigned char *)data)[room + i] = 0;

  wrptr(p, kpi_l.skb_head, data);
  wrptr(p, kpi_l.skb_data, data);
  wr32(p, kpi_l.skb_tail, 0);
  wr32(p, kpi_l.skb_end,  room);
  wr32(p, kpi_l.skb_len,  0);
  return p;
}

void *kpi_napi_alloc_skb(void *napi, unsigned int len)
{ (void)napi; return kpi_skb_alloc(len); }

void *kpi_netdev_alloc_skb(void *dev, unsigned int len)
{ (void)dev; return kpi_skb_alloc(len); }

// Claim `n` bytes at the tail and return where they start. The classic, and
// the one every receive path is built on.
void *
kpi_skb_put(void *skb, unsigned int n)
{
  unsigned char *head = (unsigned char *)rdptr(skb, kpi_l.skb_head);
  unsigned int   tail = rd32(skb, kpi_l.skb_tail);

  wr32(skb, kpi_l.skb_tail, tail + n);
  wr32(skb, kpi_l.skb_len, rd32(skb, kpi_l.skb_len) + n);
  return head + tail;
}

void *
kpi_skb_reserve(void *skb, int n)
{
  unsigned char *d = (unsigned char *)rdptr(skb, kpi_l.skb_data);
  wrptr(skb, kpi_l.skb_data, d + n);
  wr32(skb, kpi_l.skb_tail, rd32(skb, kpi_l.skb_tail) + (unsigned int)n);
  return 0;
}

unsigned char *kpi_skb_data(void *skb) { return rdptr(skb, kpi_l.skb_data); }
unsigned int   kpi_skb_len(void *skb)  { return rd32(skb, kpi_l.skb_len); }

// Copy a frame out of an skb, which is what a driver does on transmit when it
// cannot hand the buffer to the hardware directly. The checksum half of the
// name does not apply: nothing here offloads one.
void
kpi_skb_copy_and_csum_dev(void *skb, unsigned char *to)
{
  unsigned char *d = rdptr(skb, kpi_l.skb_data);
  unsigned int   n = rd32(skb, kpi_l.skb_len), i;
  for (i = 0; i < n; i++)
    to[i] = d[i];
}

// The driver has a frame. Hand it to the stack.
//
// "The stack" is the kernel on the other side of /dev/netdev, and this is the
// point at which a packet that arrived on real hardware becomes an ordinary
// frame on an ordinary interface. Everything before it is the driver's; nothing
// after it knows a Linux driver was involved.
int
kpi_netif_receive_skb(void *skb)
{
  unsigned char *d = rdptr(skb, kpi_l.skb_data);
  unsigned int   n = rd32(skb, kpi_l.skb_len);

  if (kpi.netfd >= 0 && n > 0 && n <= 1536)
    write(kpi.netfd, (char *)d, (int)n);
  return 0;
}

// gro_receive_skb(gro, skb) -- the same thing, with the GRO node in front.
//
// `napi_gro_receive` is an inline wrapper in this kernel, so what a module
// actually calls is this, and the skb is the *second* argument. Taking the
// first would hand the stack a `struct gro_node` and call it a frame.
int
kpi_gro_receive_skb(void *gro, void *skb)
{
  (void)gro;
  return kpi_netif_receive_skb(skb);
}

// eth_type_trans(skb, dev) -- consume the Ethernet header and report the
// protocol. The stack below wants the whole frame including that header, so
// unlike Linux's this does not pull it off; what it returns is the type field,
// which the driver passes on and nothing here reads.
unsigned short
kpi_eth_type_trans(void *skb, void *dev)
{
  unsigned char *d = rdptr(skb, kpi_l.skb_data);
  unsigned int   n = rd32(skb, kpi_l.skb_len);

  wrptr(skb, kpi_l.skb_dev, dev);
  if (n < 14)
    return 0;
  return (unsigned short)((d[12] << 8) | d[13]);
}

void kpi_kfree_skb(void *skb) { (void)skb; }
void kpi_kfree_skb_reason(void *skb, int r) { (void)skb; (void)r; }