head 1.22; access; symbols netbsd-10-0-RELEASE:1.21 netbsd-10-0-RC6:1.21 netbsd-10-0-RC5:1.21 netbsd-10-0-RC4:1.21 netbsd-10-0-RC3:1.21 netbsd-10-0-RC2:1.21 thorpej-ifq:1.22.0.4 thorpej-ifq-base:1.22 thorpej-altq-separation:1.22.0.2 thorpej-altq-separation-base:1.22 netbsd-10-0-RC1:1.21 netbsd-10:1.21.0.4 netbsd-10-base:1.21 bouyer-sunxi-drm:1.21.0.2 bouyer-sunxi-drm-base:1.21 netbsd-9-3-RELEASE:1.10 thorpej-i2c-spi-conf2:1.13.0.12 thorpej-i2c-spi-conf2-base:1.13 thorpej-futex2:1.13.0.10 thorpej-futex2-base:1.13 thorpej-cfargs2:1.13.0.8 thorpej-cfargs2-base:1.13 cjep_sun2x-base1:1.13 cjep_sun2x:1.13.0.6 cjep_sun2x-base:1.13 cjep_staticlib_x-base1:1.13 netbsd-9-2-RELEASE:1.10 cjep_staticlib_x:1.13.0.4 cjep_staticlib_x-base:1.13 thorpej-i2c-spi-conf:1.13.0.2 thorpej-i2c-spi-conf-base:1.13 thorpej-cfargs:1.12.0.4 thorpej-cfargs-base:1.13 thorpej-futex:1.12.0.2 thorpej-futex-base:1.13 netbsd-9-1-RELEASE:1.10 bouyer-xenpvh-base2:1.11 phil-wifi-20200421:1.11 bouyer-xenpvh-base1:1.11 phil-wifi-20200411:1.11 bouyer-xenpvh:1.11.0.4 bouyer-xenpvh-base:1.11 is-mlppp:1.11.0.2 is-mlppp-base:1.11 phil-wifi-20200406:1.11 netbsd-8-2-RELEASE:1.9 ad-namecache-base3:1.11 netbsd-9-0-RELEASE:1.10 netbsd-9-0-RC2:1.10 ad-namecache-base2:1.10 ad-namecache-base1:1.10 ad-namecache:1.10.0.6 ad-namecache-base:1.10 netbsd-9-0-RC1:1.10 phil-wifi-20191119:1.10 netbsd-9:1.10.0.4 netbsd-9-base:1.10 phil-wifi-20190609:1.10 netbsd-8-1-RELEASE:1.9 netbsd-8-1-RC1:1.9 isaki-audio2:1.10.0.2 isaki-audio2-base:1.10 pgoyette-compat-merge-20190127:1.9.8.1 pgoyette-compat-20190127:1.10 pgoyette-compat-20190118:1.10 pgoyette-compat-1226:1.10 pgoyette-compat-1126:1.10 pgoyette-compat-1020:1.10 pgoyette-compat-0930:1.10 pgoyette-compat-0906:1.10 netbsd-7-2-RELEASE:1.8 pgoyette-compat-0728:1.9 netbsd-8-0-RELEASE:1.9 phil-wifi:1.9.0.10 phil-wifi-base:1.9 pgoyette-compat-0625:1.9 netbsd-8-0-RC2:1.9 pgoyette-compat-0521:1.9 pgoyette-compat-0502:1.9 pgoyette-compat-0422:1.9 netbsd-8-0-RC1:1.9 pgoyette-compat-0415:1.9 pgoyette-compat-0407:1.9 pgoyette-compat-0330:1.9 pgoyette-compat-0322:1.9 pgoyette-compat-0315:1.9 netbsd-7-1-2-RELEASE:1.8 pgoyette-compat:1.9.0.8 pgoyette-compat-base:1.9 netbsd-7-1-1-RELEASE:1.8 tls-maxphys-base-20171202:1.9 matt-nb8-mediatek:1.9.0.6 matt-nb8-mediatek-base:1.9 nick-nhusb-base-20170825:1.9 perseant-stdc-iso10646:1.9.0.4 perseant-stdc-iso10646-base:1.9 netbsd-8:1.9.0.2 netbsd-8-base:1.9 prg-localcount2-base3:1.8 prg-localcount2-base2:1.8 prg-localcount2-base1:1.8 prg-localcount2:1.8.0.22 prg-localcount2-base:1.8 pgoyette-localcount-20170426:1.8 bouyer-socketcan-base1:1.8 jdolecek-ncq:1.8.0.20 jdolecek-ncq-base:1.8 pgoyette-localcount-20170320:1.8 netbsd-7-1:1.8.0.18 netbsd-7-1-RELEASE:1.8 netbsd-7-1-RC2:1.8 nick-nhusb-base-20170204:1.8 netbsd-7-nhusb-base-20170116:1.8 bouyer-socketcan:1.8.0.16 bouyer-socketcan-base:1.8 pgoyette-localcount-20170107:1.8 netbsd-7-1-RC1:1.8 nick-nhusb-base-20161204:1.8 pgoyette-localcount-20161104:1.8 netbsd-7-0-2-RELEASE:1.8 nick-nhusb-base-20161004:1.8 localcount-20160914:1.8 netbsd-7-nhusb:1.8.0.14 netbsd-7-nhusb-base:1.8 pgoyette-localcount-20160806:1.8 pgoyette-localcount-20160726:1.8 pgoyette-localcount:1.8.0.12 pgoyette-localcount-base:1.8 nick-nhusb-base-20160907:1.8 nick-nhusb-base-20160529:1.8 netbsd-7-0-1-RELEASE:1.8 nick-nhusb-base-20160422:1.8 nick-nhusb-base-20160319:1.8 nick-nhusb-base-20151226:1.8 netbsd-7-0:1.8.0.10 netbsd-7-0-RELEASE:1.8 nick-nhusb-base-20150921:1.8 netbsd-7-0-RC3:1.8 netbsd-7-0-RC2:1.8 netbsd-7-0-RC1:1.8 nick-nhusb-base-20150606:1.8 nick-nhusb-base-20150406:1.8 nick-nhusb:1.8.0.8 nick-nhusb-base:1.8 tls-maxphys-base:1.8 tls-maxphys:1.8.0.6 netbsd-7:1.8.0.4 netbsd-7-base:1.8 tls-earlyentropy:1.8.0.2 tls-earlyentropy-base:1.8; locks; strict; comment @ * @; 1.22 date 2023.05.28.08.09.34; author andvar; state Exp; branches; next 1.21; commitid tVjUQ3b8clHgKHqE; 1.21 date 2022.09.04.17.34.43; author thorpej; state Exp; branches; next 1.20; commitid 27JyFNrtEsz9ozSD; 1.20 date 2022.09.02.05.50.36; author thorpej; state Exp; branches; next 1.19; commitid 2fOHICBxLwKjzfSD; 1.19 date 2022.09.02.03.50.00; author thorpej; state Exp; branches; next 1.18; commitid v7NyDhjKKDi7VeSD; 1.18 date 2022.09.01.05.04.22; author thorpej; state Exp; branches; next 1.17; commitid jJfZ1sEhg2jYm7SD; 1.17 date 2022.09.01.02.35.06; author thorpej; state Exp; branches; next 1.16; commitid dUX7ItdPZNswx6SD; 1.16 date 2021.12.21.04.09.32; author knakahara; state Exp; branches; next 1.15; commitid tgmoDjkHHAS6etlD; 1.15 date 2021.12.15.07.47.22; author knakahara; state Exp; branches; next 1.14; commitid d9KrkWRAdzPZCIkD; 1.14 date 2021.10.11.05.13.11; author knakahara; state Exp; branches; next 1.13; commitid XtwcpcwCe61wRlcD; 1.13 date 2021.03.25.08.18.03; author skrll; state Exp; branches; next 1.12; commitid CcypTN5JKZpzjFMC; 1.12 date 2020.09.11.14.29.00; author riastradh; state Exp; branches 1.12.2.1 1.12.4.1; next 1.11; commitid 5puXzgRZZN3vDDnC; 1.11 date 2020.02.07.12.35.33; author thorpej; state Exp; branches; next 1.10; commitid 94Ksk2mJncCBYJVB; 1.10 date 2018.08.10.07.24.09; author msaitoh; state Exp; branches 1.10.6.1; next 1.9; commitid ZsvezIAuKSumOxNA; 1.9 date 2017.06.01.02.45.14; author chs; state Exp; branches 1.9.8.1 1.9.10.1; next 1.8; commitid QWuKA1CxvBGsfCTz; 1.8 date 2014.07.04.01.50.22; author ozaki-r; state Exp; branches 1.8.2.1 1.8.6.1 1.8.8.1; next 1.7; commitid HMGKYSLZuxM740Hx; 1.7 date 2014.07.02.07.30.37; author ozaki-r; state Exp; branches; next 1.6; commitid YaUyU7ANOLrR2MGx; 1.6 date 2014.06.16.00.40.10; author ozaki-r; state Exp; branches; next 1.5; commitid bWgBkDBe8ozbiGEx; 1.5 date 2014.06.16.00.33.39; author ozaki-r; state Exp; branches; next 1.4; commitid kyBfozL4qc3vfGEx; 1.4 date 2014.06.09.14.44.48; author rmind; state Exp; branches; next 1.3; commitid 0h2shzPUrnhHcRDx; 1.3 date 2014.06.09.13.03.16; author rmind; state Exp; branches; next 1.2; commitid d8RMLuWUgo4FDQDx; 1.2 date 2014.06.09.12.57.04; author rmind; state Exp; branches; next 1.1; commitid EyVxp2I1mbjEBQDx; 1.1 date 2014.06.05.23.48.16; author rmind; state Exp; branches; next ; commitid P7hs4C9IyZuIkoDx; 1.12.2.1 date 2021.04.03.22.29.01; author thorpej; state Exp; branches; next ; commitid 1gqS07EfPjskJTNC; 1.12.4.1 date 2021.04.03.21.45.01; author thorpej; state Exp; branches; next ; commitid KB7Cr288FrZluTNC; 1.10.6.1 date 2020.02.29.20.21.06; author ad; state Exp; branches; next ; commitid OjSb8ro7YQETQBYB; 1.9.8.1 date 2018.09.06.06.56.44; author pgoyette; state Exp; branches; next ; commitid HCi1bXD317XIK0RA; 1.9.10.1 date 2019.06.10.22.09.45; author christos; state Exp; branches; next 1.9.10.2; commitid jtc8rnCzWiEEHGqB; 1.9.10.2 date 2020.04.08.14.08.57; author martin; state Exp; branches; next ; commitid Qli2aW9E74UFuA3C; 1.8.2.1 date 2014.07.04.01.50.22; author tls; state dead; branches; next 1.8.2.2; commitid VywzU7NDeggtCMLx; 1.8.2.2 date 2014.08.10.06.56.15; author tls; state Exp; branches; next ; commitid VywzU7NDeggtCMLx; 1.8.6.1 date 2014.07.04.01.50.22; author tls; state dead; branches; next 1.8.6.2; commitid jTnpym9Qu0o4R1Nx; 1.8.6.2 date 2014.08.20.00.04.34; author tls; state Exp; branches; next 1.8.6.3; commitid jTnpym9Qu0o4R1Nx; 1.8.6.3 date 2017.12.03.11.39.02; author jdolecek; state Exp; branches; next ; commitid XcIYRZTAh1LmerhA; 1.8.8.1 date 2017.08.28.17.53.11; author skrll; state Exp; branches; next ; commitid UQQpnjvcNkUZn05A; desc @@ 1.22 log @s/explcit/explicit/ in comment. @ text @/* $NetBSD: pktqueue.c,v 1.21 2022/09/04 17:34:43 thorpej Exp $ */ /*- * Copyright (c) 2014 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation * by Mindaugas Rasiukevicius. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * The packet queue (pktqueue) interface is a lockless IP input queue * which also abstracts and handles network ISR scheduling. It provides * a mechanism to enable receiver-side packet steering (RPS). */ #include __KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.21 2022/09/04 17:34:43 thorpej Exp $"); #ifdef _KERNEL_OPT #include "opt_net_mpsafe.h" #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include struct pktqueue { /* * The lock used for a barrier mechanism. The barrier counter, * as well as the drop counter, are managed atomically though. * Ensure this group is in a separate cache line. */ union { struct { kmutex_t pq_lock; volatile u_int pq_barrier; }; uint8_t _pad[COHERENCY_UNIT]; }; /* The size of the queue, counters and the interrupt handler. */ u_int pq_maxlen; percpu_t * pq_counters; void * pq_sih; /* The per-CPU queues. */ struct percpu * pq_pcq; /* struct pcq * */ /* The linkage on the list of all pktqueues. */ LIST_ENTRY(pktqueue) pq_list; }; /* The counters of the packet queue. */ #define PQCNT_ENQUEUE 0 #define PQCNT_DEQUEUE 1 #define PQCNT_DROP 2 #define PQCNT_NCOUNTERS 3 typedef struct { uint64_t count[PQCNT_NCOUNTERS]; } pktq_counters_t; /* Special marker value used by pktq_barrier() mechanism. */ #define PKTQ_MARKER ((void *)(~0ULL)) /* * This is a list of all pktqueues. This list is used by * pktq_ifdetach() to issue a barrier on every pktqueue. * * The r/w lock is acquired for writing in pktq_create() and * pktq_destroy(), and for reading in pktq_ifdetach(). * * This list is not performance critical, and will seldom be * accessed. */ static LIST_HEAD(, pktqueue) pktqueue_list __read_mostly; static krwlock_t pktqueue_list_lock __read_mostly; static once_t pktqueue_list_init_once __read_mostly; static int pktqueue_list_init(void) { LIST_INIT(&pktqueue_list); rw_init(&pktqueue_list_lock); return 0; } static void pktq_init_cpu(void *vqp, void *vpq, struct cpu_info *ci) { struct pcq **qp = vqp; struct pktqueue *pq = vpq; *qp = pcq_create(pq->pq_maxlen, KM_SLEEP); } static void pktq_fini_cpu(void *vqp, void *vpq, struct cpu_info *ci) { struct pcq **qp = vqp, *q = *qp; KASSERT(pcq_peek(q) == NULL); pcq_destroy(q); *qp = NULL; /* paranoia */ } static struct pcq * pktq_pcq(struct pktqueue *pq, struct cpu_info *ci) { struct pcq **qp, *q; /* * As long as preemption is disabled, the xcall to swap percpu * buffers can't complete, so it is safe to read the pointer. */ KASSERT(kpreempt_disabled()); qp = percpu_getptr_remote(pq->pq_pcq, ci); q = *qp; return q; } pktqueue_t * pktq_create(size_t maxlen, void (*intrh)(void *), void *sc) { const u_int sflags = SOFTINT_NET | SOFTINT_MPSAFE | SOFTINT_RCPU; pktqueue_t *pq; percpu_t *pc; void *sih; RUN_ONCE(&pktqueue_list_init_once, pktqueue_list_init); pc = percpu_alloc(sizeof(pktq_counters_t)); if ((sih = softint_establish(sflags, intrh, sc)) == NULL) { percpu_free(pc, sizeof(pktq_counters_t)); return NULL; } pq = kmem_zalloc(sizeof(*pq), KM_SLEEP); mutex_init(&pq->pq_lock, MUTEX_DEFAULT, IPL_NONE); pq->pq_maxlen = maxlen; pq->pq_counters = pc; pq->pq_sih = sih; pq->pq_pcq = percpu_create(sizeof(struct pcq *), pktq_init_cpu, pktq_fini_cpu, pq); rw_enter(&pktqueue_list_lock, RW_WRITER); LIST_INSERT_HEAD(&pktqueue_list, pq, pq_list); rw_exit(&pktqueue_list_lock); return pq; } void pktq_destroy(pktqueue_t *pq) { KASSERT(pktqueue_list_init_once.o_status == ONCE_DONE); rw_enter(&pktqueue_list_lock, RW_WRITER); LIST_REMOVE(pq, pq_list); rw_exit(&pktqueue_list_lock); percpu_free(pq->pq_pcq, sizeof(struct pcq *)); percpu_free(pq->pq_counters, sizeof(pktq_counters_t)); softint_disestablish(pq->pq_sih); mutex_destroy(&pq->pq_lock); kmem_free(pq, sizeof(*pq)); } /* * - pktq_inc_counter: increment the counter given an ID. * - pktq_collect_counts: handler to sum up the counts from each CPU. * - pktq_getcount: return the effective count given an ID. */ static inline void pktq_inc_count(pktqueue_t *pq, u_int i) { percpu_t *pc = pq->pq_counters; pktq_counters_t *c; c = percpu_getref(pc); c->count[i]++; percpu_putref(pc); } static void pktq_collect_counts(void *mem, void *arg, struct cpu_info *ci) { const pktq_counters_t *c = mem; pktq_counters_t *sum = arg; int s = splnet(); for (u_int i = 0; i < PQCNT_NCOUNTERS; i++) { sum->count[i] += c->count[i]; } splx(s); } static uint64_t pktq_get_count(pktqueue_t *pq, pktq_count_t c) { pktq_counters_t sum; if (c != PKTQ_MAXLEN) { memset(&sum, 0, sizeof(sum)); percpu_foreach_xcall(pq->pq_counters, XC_HIGHPRI_IPL(IPL_SOFTNET), pktq_collect_counts, &sum); } switch (c) { case PKTQ_NITEMS: return sum.count[PQCNT_ENQUEUE] - sum.count[PQCNT_DEQUEUE]; case PKTQ_DROPS: return sum.count[PQCNT_DROP]; case PKTQ_MAXLEN: return pq->pq_maxlen; } return 0; } uint32_t pktq_rps_hash(const pktq_rps_hash_func_t *funcp, const struct mbuf *m) { pktq_rps_hash_func_t func = atomic_load_relaxed(funcp); KASSERT(func != NULL); return (*func)(m); } static uint32_t pktq_rps_hash_zero(const struct mbuf *m __unused) { return 0; } static uint32_t pktq_rps_hash_curcpu(const struct mbuf *m __unused) { return cpu_index(curcpu()); } static uint32_t pktq_rps_hash_toeplitz(const struct mbuf *m) { struct ip *ip; /* * Disable UDP port - IP fragments aren't currently being handled * and so we end up with a mix of 2-tuple and 4-tuple * traffic. */ const u_int flag = RSS_TOEPLITZ_USE_TCP_PORT; /* glance IP version */ if ((m->m_flags & M_PKTHDR) == 0) return 0; ip = mtod(m, struct ip *); if (ip->ip_v == IPVERSION) { if (__predict_false(m->m_len < sizeof(struct ip))) return 0; return rss_toeplitz_hash_from_mbuf_ipv4(m, flag); } else if (ip->ip_v == 6) { if (__predict_false(m->m_len < sizeof(struct ip6_hdr))) return 0; return rss_toeplitz_hash_from_mbuf_ipv6(m, flag); } return 0; } /* * toeplitz without curcpu. * Generally, this has better performance than toeplitz. */ static uint32_t pktq_rps_hash_toeplitz_othercpus(const struct mbuf *m) { uint32_t hash; if (ncpu == 1) return 0; hash = pktq_rps_hash_toeplitz(m); hash %= ncpu - 1; if (hash >= cpu_index(curcpu())) return hash + 1; else return hash; } static struct pktq_rps_hash_table { const char* prh_type; pktq_rps_hash_func_t prh_func; } const pktq_rps_hash_tab[] = { { "zero", pktq_rps_hash_zero }, { "curcpu", pktq_rps_hash_curcpu }, { "toeplitz", pktq_rps_hash_toeplitz }, { "toeplitz-othercpus", pktq_rps_hash_toeplitz_othercpus }, }; const pktq_rps_hash_func_t pktq_rps_hash_default = #ifdef NET_MPSAFE pktq_rps_hash_curcpu; #else pktq_rps_hash_zero; #endif static const char * pktq_get_rps_hash_type(pktq_rps_hash_func_t func) { for (int i = 0; i < __arraycount(pktq_rps_hash_tab); i++) { if (func == pktq_rps_hash_tab[i].prh_func) { return pktq_rps_hash_tab[i].prh_type; } } return NULL; } static int pktq_set_rps_hash_type(pktq_rps_hash_func_t *func, const char *type) { if (strcmp(type, pktq_get_rps_hash_type(*func)) == 0) return 0; for (int i = 0; i < __arraycount(pktq_rps_hash_tab); i++) { if (strcmp(type, pktq_rps_hash_tab[i].prh_type) == 0) { atomic_store_relaxed(func, pktq_rps_hash_tab[i].prh_func); return 0; } } return ENOENT; } int sysctl_pktq_rps_hash_handler(SYSCTLFN_ARGS) { struct sysctlnode node; pktq_rps_hash_func_t *func; int error; char type[PKTQ_RPS_HASH_NAME_LEN]; node = *rnode; func = node.sysctl_data; strlcpy(type, pktq_get_rps_hash_type(*func), PKTQ_RPS_HASH_NAME_LEN); node.sysctl_data = &type; node.sysctl_size = sizeof(type); error = sysctl_lookup(SYSCTLFN_CALL(&node)); if (error || newp == NULL) return error; error = pktq_set_rps_hash_type(func, type); return error; } /* * pktq_enqueue: inject the packet into the end of the queue. * * => Must be called from the interrupt or with the preemption disabled. * => Consumes the packet and returns true on success. * => Returns false on failure; caller is responsible to free the packet. */ bool pktq_enqueue(pktqueue_t *pq, struct mbuf *m, const u_int hash __unused) { #if defined(_RUMPKERNEL) || defined(_RUMP_NATIVE_ABI) struct cpu_info *ci = curcpu(); #else struct cpu_info *ci = cpu_lookup(hash % ncpu); #endif KASSERT(kpreempt_disabled()); if (__predict_false(!pcq_put(pktq_pcq(pq, ci), m))) { pktq_inc_count(pq, PQCNT_DROP); return false; } softint_schedule_cpu(pq->pq_sih, ci); pktq_inc_count(pq, PQCNT_ENQUEUE); return true; } /* * pktq_dequeue: take a packet from the queue. * * => Must be called with preemption disabled. * => Must ensure there are not concurrent dequeue calls. */ struct mbuf * pktq_dequeue(pktqueue_t *pq) { struct cpu_info *ci = curcpu(); struct mbuf *m; KASSERT(kpreempt_disabled()); m = pcq_get(pktq_pcq(pq, ci)); if (__predict_false(m == PKTQ_MARKER)) { /* Note the marker entry. */ atomic_inc_uint(&pq->pq_barrier); /* Get the next queue entry. */ m = pcq_get(pktq_pcq(pq, ci)); /* * There can only be one barrier operation pending * on a pktqueue at any given time, so we can assert * that the next item is not a marker. */ KASSERT(m != PKTQ_MARKER); } if (__predict_true(m != NULL)) { pktq_inc_count(pq, PQCNT_DEQUEUE); } return m; } /* * pktq_barrier: waits for a grace period when all packets enqueued at * the moment of calling this routine will be processed. This is used * to ensure that e.g. packets referencing some interface were drained. */ void pktq_barrier(pktqueue_t *pq) { CPU_INFO_ITERATOR cii; struct cpu_info *ci; u_int pending = 0; mutex_enter(&pq->pq_lock); KASSERT(pq->pq_barrier == 0); for (CPU_INFO_FOREACH(cii, ci)) { struct pcq *q; kpreempt_disable(); q = pktq_pcq(pq, ci); kpreempt_enable(); /* If the queue is empty - nothing to do. */ if (pcq_peek(q) == NULL) { continue; } /* Otherwise, put the marker and entry. */ while (!pcq_put(q, PKTQ_MARKER)) { kpause("pktqsync", false, 1, NULL); } kpreempt_disable(); softint_schedule_cpu(pq->pq_sih, ci); kpreempt_enable(); pending++; } /* Wait for each queue to process the markers. */ while (pq->pq_barrier != pending) { kpause("pktqsync", false, 1, NULL); } pq->pq_barrier = 0; mutex_exit(&pq->pq_lock); } /* * pktq_ifdetach: issue a barrier on all pktqueues when a network * interface is detached. */ void pktq_ifdetach(void) { pktqueue_t *pq; /* Just in case no pktqueues have been created yet... */ RUN_ONCE(&pktqueue_list_init_once, pktqueue_list_init); rw_enter(&pktqueue_list_lock, RW_READER); LIST_FOREACH(pq, &pktqueue_list, pq_list) { pktq_barrier(pq); } rw_exit(&pktqueue_list_lock); } /* * pktq_flush: free mbufs in all queues. * * => The caller must ensure there are no concurrent writers or flush calls. */ void pktq_flush(pktqueue_t *pq) { CPU_INFO_ITERATOR cii; struct cpu_info *ci; struct mbuf *m, *m0 = NULL; ASSERT_SLEEPABLE(); /* * Run a dummy softint at IPL_SOFTNET on all CPUs to ensure that any * already running handler for this pktqueue is no longer running. */ xc_barrier(XC_HIGHPRI_IPL(IPL_SOFTNET)); /* * Acquire the barrier lock. While the caller ensures that * no explicit pktq_barrier() calls will be issued, this holds * off any implicit pktq_barrier() calls that would happen * as the result of pktq_ifdetach(). */ mutex_enter(&pq->pq_lock); for (CPU_INFO_FOREACH(cii, ci)) { struct pcq *q; kpreempt_disable(); q = pktq_pcq(pq, ci); kpreempt_enable(); /* * Pull the packets off the pcq and chain them into * a list to be freed later. */ while ((m = pcq_get(q)) != NULL) { pktq_inc_count(pq, PQCNT_DEQUEUE); m->m_nextpkt = m0; m0 = m; } } mutex_exit(&pq->pq_lock); /* Free the packets now that the critical section is over. */ while ((m = m0) != NULL) { m0 = m->m_nextpkt; m_freem(m); } } static void pktq_set_maxlen_cpu(void *vpq, void *vqs) { struct pktqueue *pq = vpq; struct pcq **qp, *q, **qs = vqs; unsigned i = cpu_index(curcpu()); int s; s = splnet(); qp = percpu_getref(pq->pq_pcq); q = *qp; *qp = qs[i]; qs[i] = q; percpu_putref(pq->pq_pcq); splx(s); } /* * pktq_set_maxlen: create per-CPU queues using a new size and replace * the existing queues without losing any packets. * * XXX ncpu must remain stable throughout. */ int pktq_set_maxlen(pktqueue_t *pq, size_t maxlen) { const u_int slotbytes = ncpu * sizeof(pcq_t *); pcq_t **qs; if (!maxlen || maxlen > PCQ_MAXLEN) return EINVAL; if (pq->pq_maxlen == maxlen) return 0; /* First, allocate the new queues. */ qs = kmem_zalloc(slotbytes, KM_SLEEP); for (u_int i = 0; i < ncpu; i++) { qs[i] = pcq_create(maxlen, KM_SLEEP); } /* * Issue an xcall to replace the queue pointers on each CPU. * This implies all the necessary memory barriers. */ mutex_enter(&pq->pq_lock); xc_wait(xc_broadcast(XC_HIGHPRI, pktq_set_maxlen_cpu, pq, qs)); pq->pq_maxlen = maxlen; mutex_exit(&pq->pq_lock); /* * At this point, the new packets are flowing into the new * queues. However, the old queues may have some packets * present which are no longer being processed. We are going * to re-enqueue them. This may change the order of packet * arrival, but it is not considered an issue. * * There may be in-flight interrupts calling pktq_dequeue() * which reference the old queues. Issue a barrier to ensure * that we are going to be the only pcq_get() callers on the * old queues. */ pktq_barrier(pq); for (u_int i = 0; i < ncpu; i++) { struct pcq *q; struct mbuf *m; kpreempt_disable(); q = pktq_pcq(pq, cpu_lookup(i)); kpreempt_enable(); while ((m = pcq_get(qs[i])) != NULL) { while (!pcq_put(q, m)) { kpause("pktqrenq", false, 1, NULL); } } pcq_destroy(qs[i]); } /* Well, that was fun. */ kmem_free(qs, slotbytes); return 0; } static int sysctl_pktq_maxlen(SYSCTLFN_ARGS) { struct sysctlnode node = *rnode; pktqueue_t * const pq = node.sysctl_data; u_int nmaxlen = pktq_get_count(pq, PKTQ_MAXLEN); int error; node.sysctl_data = &nmaxlen; error = sysctl_lookup(SYSCTLFN_CALL(&node)); if (error || newp == NULL) return error; return pktq_set_maxlen(pq, nmaxlen); } static int sysctl_pktq_count(SYSCTLFN_ARGS, u_int count_id) { struct sysctlnode node = *rnode; pktqueue_t * const pq = node.sysctl_data; uint64_t count = pktq_get_count(pq, count_id); node.sysctl_data = &count; return sysctl_lookup(SYSCTLFN_CALL(&node)); } static int sysctl_pktq_nitems(SYSCTLFN_ARGS) { return sysctl_pktq_count(SYSCTLFN_CALL(rnode), PKTQ_NITEMS); } static int sysctl_pktq_drops(SYSCTLFN_ARGS) { return sysctl_pktq_count(SYSCTLFN_CALL(rnode), PKTQ_DROPS); } /* * pktqueue_sysctl_setup: set up the sysctl nodes for a pktqueue * using standardized names at the specified parent node and * node ID (or CTL_CREATE). */ void pktq_sysctl_setup(pktqueue_t * const pq, struct sysctllog ** const clog, const struct sysctlnode * const parent_node, const int qid) { const struct sysctlnode *rnode = parent_node, *cnode; KASSERT(pq != NULL); KASSERT(parent_node != NULL); KASSERT(qid == CTL_CREATE || qid >= 0); /* Create the "ifq" node below the parent node. */ sysctl_createv(clog, 0, &rnode, &cnode, CTLFLAG_PERMANENT, CTLTYPE_NODE, "ifq", SYSCTL_DESCR("Protocol input queue controls"), NULL, 0, NULL, 0, qid, CTL_EOL); /* Now create the standard child nodes below "ifq". */ rnode = cnode; sysctl_createv(clog, 0, &rnode, &cnode, CTLFLAG_PERMANENT, CTLTYPE_QUAD, "len", SYSCTL_DESCR("Current input queue length"), sysctl_pktq_nitems, 0, (void *)pq, 0, IFQCTL_LEN, CTL_EOL); sysctl_createv(clog, 0, &rnode, &cnode, CTLFLAG_PERMANENT | CTLFLAG_READWRITE, CTLTYPE_INT, "maxlen", SYSCTL_DESCR("Maximum allowed input queue length"), sysctl_pktq_maxlen, 0, (void *)pq, 0, IFQCTL_MAXLEN, CTL_EOL); sysctl_createv(clog, 0, &rnode, &cnode, CTLFLAG_PERMANENT, CTLTYPE_QUAD, "drops", SYSCTL_DESCR("Packets dropped due to full input queue"), sysctl_pktq_drops, 0, (void *)pq, 0, IFQCTL_DROPS, CTL_EOL); } @ 1.21 log @In pktq_flush(): - Run a dummy softint at IPL_SOFTNET on all CPUs to ensure that the ISR for this pktqueue is not running (addresses a pre-existing XXX). - Hold the barrier lock around the critical section to ensure that implicit pktq_barrier() calls via pktq_ifdetach() are held off during the critical section. - Ensure the critical section completes in minimal time by not freeing memory during the critical section; instead, just build a list of the packets pulled out of the per-CPU queues and free them after the critical section is over. @ text @d1 1 a1 1 /* $NetBSD: pktqueue.c,v 1.20 2022/09/02 05:50:36 thorpej Exp $ */ d39 1 a39 1 __KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.20 2022/09/02 05:50:36 thorpej Exp $"); d555 1 a555 1 * no explcit pktq_barrier() calls will be issued, this holds @ 1.20 log @Re-factor how pktq_barrier() is issued by if_detach(). Rather than excplicitly referencing ip_pktq and ip6_pktq in if_detach(), instead add all pktqueues to a global list. This list is then used in the new pktq_ifdetach() function to issue a barrier on all pktqueues. Note that the performance of this list is not critical; it will seldom be accessed (then pktqueues are created/destroyed and when network interfaces are detached), and so a simple synchronization strategy using a rwlock is sufficient. @ text @d1 1 a1 1 /* $NetBSD: pktqueue.c,v 1.19 2022/09/02 03:50:00 thorpej Exp $ */ d39 1 a39 1 __KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.19 2022/09/02 03:50:00 thorpej Exp $"); d543 17 a559 1 struct mbuf *m; d569 2 a570 2 * XXX This can't be right -- if the softint is running * then pcq_get isn't safe here. d574 2 a575 1 m_freem(m); d578 8 @ 1.19 log @pktqueue: Re-factor sysctl handling. Provide a new pktq_sysctl_setup() function that attaches standard pktq sysctl nodes below a specified parent node, with either a fixed node ID or CTL_CREATE to dynamically assign node IDs. Make all of the sysctl handlers private to pktqueue.c, and remove the INET- and INET6-specific pktqueue sysctl code from net/if.c. @ text @d1 1 a1 1 /* $NetBSD: pktqueue.c,v 1.18 2022/09/01 05:04:22 thorpej Exp $ */ d39 1 a39 1 __KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.18 2022/09/01 05:04:22 thorpej Exp $"); d56 3 d86 1 a86 1 /* Finally, per-CPU queues. */ d88 3 d106 22 d172 2 d188 4 d199 6 d515 19 @ 1.18 log @pktq_rps_hash(): Make the "funcp" argument const. @ text @d1 1 a1 1 /* $NetBSD: pktqueue.c,v 1.17 2022/09/01 02:35:06 thorpej Exp $ */ d39 1 a39 1 __KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.17 2022/09/01 02:35:06 thorpej Exp $"); d204 1 a204 1 uint64_t d588 2 a589 2 int sysctl_pktq_maxlen(SYSCTLFN_ARGS, pktqueue_t *pq) d591 2 a593 1 struct sysctlnode node = *rnode; d603 2 a604 2 int sysctl_pktq_count(SYSCTLFN_ARGS, pktqueue_t *pq, u_int count_id) d606 2 a608 1 struct sysctlnode node = *rnode; d613 58 @ 1.17 log @pktq_dequeue(): Prevent packets from getting stuck beind barrier markers. pktq_barrier() ensures that all packets enqueued before the barrier have been dequeued before the barrier returns. However, previously, pktq_dequeue() would return NULL when a barrier marker was encountered. If there were packets queued up behind the marker and no additional softint were scheduled for the pktqueue, those packets would end up stranded. pktq_dequeue() now continues to the next slot after the marker, ensuring that processing can continue after the barrier has been signaled. @ text @d1 1 a1 1 /* $NetBSD: pktqueue.c,v 1.16 2021/12/21 04:09:32 knakahara Exp $ */ d39 1 a39 1 __KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.16 2021/12/21 04:09:32 knakahara Exp $"); d226 1 a226 1 pktq_rps_hash(pktq_rps_hash_func_t *funcp, const struct mbuf *m) @ 1.16 log @Fix net.*.rps_hash=toeplitz-othercpus on one CPU systems. @ text @d1 1 a1 1 /* $NetBSD: pktqueue.c,v 1.15 2021/12/15 07:47:22 knakahara Exp $ */ d39 1 a39 1 __KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.15 2021/12/15 07:47:22 knakahara Exp $"); d413 10 a422 1 return NULL; @ 1.15 log @Fix typo in comment. @ text @d1 1 a1 1 /* $NetBSD: pktqueue.c,v 1.14 2021/10/11 05:13:11 knakahara Exp $ */ d39 1 a39 1 __KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.14 2021/10/11 05:13:11 knakahara Exp $"); d287 3 @ 1.14 log @Make pktq_rps_hash() pluggable for each interface type. Reviewed by gdt@@n.o, thorpej@@n.o, and riastradh@@n.o, thanks. @ text @d1 1 a1 1 /* $NetBSD: pktqueue.c,v 1.13 2021/03/25 08:18:03 skrll Exp $ */ d39 1 a39 1 __KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.13 2021/03/25 08:18:03 skrll Exp $"); d279 2 a280 2 * topelitz without curcpu. * Generally, this has better performance than topelitz. @ 1.13 log @Remove strange padding #define and replace with anonymous struct/union @ text @d1 1 a1 1 /* $NetBSD: pktqueue.c,v 1.12 2020/09/11 14:29:00 riastradh Exp $ */ d39 5 a43 1 __KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.12 2020/09/11 14:29:00 riastradh Exp $"); d58 5 d226 11 a236 1 pktq_rps_hash(const struct mbuf *m __unused) d238 15 d254 3 a256 2 * XXX: No distribution yet; the softnet_lock contention * XXX: must be eliminated first. d258 17 d279 87 @ 1.12 log @pktqueue(9): Use percpu_create to allow early initialization. Otherwise pktqueues can't be created before all CPUs are detected -- they will have a queue only for the primary CPU, not for others. This will also be necessary if we want to add CPU hotplug (still need some way to block hotplug during pktq_set_maxlen but it's a start). @ text @d1 1 a1 1 /* $NetBSD: pktqueue.c,v 1.11 2020/02/07 12:35:33 thorpej Exp $ */ d39 1 a39 1 __KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.11 2020/02/07 12:35:33 thorpej Exp $"); a54 6 /* * WARNING: update this if struct pktqueue changes. */ #define PKTQ_CLPAD \ MAX(COHERENCY_UNIT, COHERENCY_UNIT - sizeof(kmutex_t) - sizeof(u_int)) d61 7 a67 3 kmutex_t pq_lock; volatile u_int pq_barrier; uint8_t _pad[PKTQ_CLPAD]; @ 1.12.2.1 log @Sync with HEAD. @ text @d1 1 a1 1 /* $NetBSD: pktqueue.c,v 1.13 2021/03/25 08:18:03 skrll Exp $ */ d39 1 a39 1 __KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.13 2021/03/25 08:18:03 skrll Exp $"); d55 6 d67 3 a69 7 union { struct { kmutex_t pq_lock; volatile u_int pq_barrier; }; uint8_t _pad[COHERENCY_UNIT]; }; @ 1.12.4.1 log @Sync with HEAD. @ text @d1 1 a1 1 /* $NetBSD: pktqueue.c,v 1.13 2021/03/25 08:18:03 skrll Exp $ */ d39 1 a39 1 __KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.13 2021/03/25 08:18:03 skrll Exp $"); d55 6 d67 3 a69 7 union { struct { kmutex_t pq_lock; volatile u_int pq_barrier; }; uint8_t _pad[COHERENCY_UNIT]; }; @ 1.11 log @Use percpu_foreach_xcall() to gather volatile per-cpu counters. These must be serialized against the interrupts / soft-interrupts in which they're manipulated, as well as protected from non-atomic 64-bit memory loads on 32-bit platforms. @ text @d1 1 a1 1 /* $NetBSD: pktqueue.c,v 1.10 2018/08/10 07:24:09 msaitoh Exp $ */ d39 1 a39 1 __KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.10 2018/08/10 07:24:09 msaitoh Exp $"); d77 1 a77 1 pcq_t * pq_queue[]; d93 35 a127 5 /* * The total size of pktqueue_t which depends on the number of CPUs. */ #define PKTQUEUE_STRUCT_LEN(ncpu) \ roundup2(offsetof(pktqueue_t, pq_queue[ncpu]), coherency_unit) a132 1 const size_t len = PKTQUEUE_STRUCT_LEN(ncpu); d143 1 a143 4 pq = kmem_zalloc(len, KM_SLEEP); for (u_int i = 0; i < ncpu; i++) { pq->pq_queue[i] = pcq_create(maxlen, KM_SLEEP); } d148 2 a156 1 const size_t len = PKTQUEUE_STRUCT_LEN(ncpu); d158 1 a158 5 for (u_int i = 0; i < ncpu; i++) { pcq_t *q = pq->pq_queue[i]; KASSERT(pcq_peek(q) == NULL); pcq_destroy(q); } d162 1 a162 1 kmem_free(pq, len); d239 1 a239 1 const unsigned cpuid = curcpu()->ci_index; d241 1 a241 1 const unsigned cpuid = hash % ncpu; d246 1 a246 1 if (__predict_false(!pcq_put(pq->pq_queue[cpuid], m))) { d250 1 a250 1 softint_schedule_cpu(pq->pq_sih, cpu_lookup(cpuid)); d264 1 a264 2 const struct cpu_info *ci = curcpu(); const unsigned cpuid = cpu_index(ci); d267 3 a269 1 m = pcq_get(pq->pq_queue[cpuid]); d289 2 d296 6 a301 2 for (u_int i = 0; i < ncpu; i++) { pcq_t *q = pq->pq_queue[i]; d312 1 a312 1 softint_schedule_cpu(pq->pq_sih, cpu_lookup(i)); d333 2 d337 12 a348 2 for (u_int i = 0; i < ncpu; i++) { while ((m = pcq_get(pq->pq_queue[i])) != NULL) { d355 17 d375 2 d389 1 a389 1 /* First, allocate the new queues and replace them. */ d394 5 d400 1 a400 6 for (u_int i = 0; i < ncpu; i++) { /* Swap: store of a word is atomic. */ pcq_t *q = pq->pq_queue[i]; pq->pq_queue[i] = qs[i]; qs[i] = q; } d419 1 d422 4 d427 1 a427 1 while (!pcq_put(pq->pq_queue[i], m)) { @ 1.10 log @- Fix a bug that drop counter shows incorrect vaule like "net.inet.ip.ifq.drops = 72059810241052672" - Change pktq's length sysctl to uint64_t. @ text @d1 1 a1 1 /* $NetBSD: pktqueue.c,v 1.9 2017/06/01 02:45:14 chs Exp $ */ d39 1 a39 1 __KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.9 2017/06/01 02:45:14 chs Exp $"); d51 1 d165 2 d170 2 d181 2 a182 1 percpu_foreach(pq->pq_counters, pktq_collect_counts, &sum); @ 1.10.6.1 log @Sync with head. @ text @d1 1 a1 1 /* $NetBSD: pktqueue.c,v 1.11 2020/02/07 12:35:33 thorpej Exp $ */ d39 1 a39 1 __KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.11 2020/02/07 12:35:33 thorpej Exp $"); a50 1 #include a163 2 int s = splnet(); a166 2 splx(s); d176 1 a176 2 percpu_foreach_xcall(pq->pq_counters, XC_HIGHPRI_IPL(IPL_SOFTNET), pktq_collect_counts, &sum); @ 1.9 log @remove checks for failure after memory allocation calls that cannot fail: kmem_alloc() with KM_SLEEP kmem_zalloc() with KM_SLEEP percpu_alloc() pserialize_create() psref_class_create() all of these paths include an assertion that the allocation has not failed, so callers should not assert that again. @ text @d1 1 a1 1 /* $NetBSD: pktqueue.c,v 1.8 2014/07/04 01:50:22 ozaki-r Exp $ */ d39 1 a39 1 __KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.8 2014/07/04 01:50:22 ozaki-r Exp $"); d384 1 a384 1 int count = pktq_get_count(pq, count_id); d386 1 @ 1.9.10.1 log @Sync with HEAD @ text @d1 1 a1 1 /* $NetBSD: pktqueue.c,v 1.10 2018/08/10 07:24:09 msaitoh Exp $ */ d39 1 a39 1 __KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.10 2018/08/10 07:24:09 msaitoh Exp $"); d384 1 a384 1 uint64_t count = pktq_get_count(pq, count_id); a385 1 @ 1.9.10.2 log @Merge changes from current as of 20200406 @ text @d1 1 a1 1 /* $NetBSD$ */ d39 1 a39 1 __KERNEL_RCSID(0, "$NetBSD$"); a50 1 #include a163 2 int s = splnet(); a166 2 splx(s); d176 1 a176 2 percpu_foreach_xcall(pq->pq_counters, XC_HIGHPRI_IPL(IPL_SOFTNET), pktq_collect_counts, &sum); @ 1.9.8.1 log @Sync with HEAD Resolve a couple of conflicts (result of the uimin/uimax changes) @ text @d1 1 a1 1 /* $NetBSD: pktqueue.c,v 1.10 2018/08/10 07:24:09 msaitoh Exp $ */ d39 1 a39 1 __KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.10 2018/08/10 07:24:09 msaitoh Exp $"); d384 1 a384 1 uint64_t count = pktq_get_count(pq, count_id); a385 1 @ 1.8 log @Fix pktq_enqueue for rump Add _RUMP_NATIVE_ABI to the macro condition for i386 and x86_64 because _RUMPKERNEL is not defined for them. See sys/rump/Makefile.rump. Found by ATF @ text @d1 1 a1 1 /* $NetBSD: pktqueue.c,v 1.7 2014/07/02 07:30:37 ozaki-r Exp $ */ d39 1 a39 1 __KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.7 2014/07/02 07:30:37 ozaki-r Exp $"); d107 1 a107 3 if ((pc = percpu_alloc(sizeof(pktq_counters_t))) == NULL) { return NULL; } @ 1.8.8.1 log @Sync with HEAD @ text @d1 1 a1 1 /* $NetBSD: pktqueue.c,v 1.9 2017/06/01 02:45:14 chs Exp $ */ d39 1 a39 1 __KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.9 2017/06/01 02:45:14 chs Exp $"); d107 3 a109 1 pc = percpu_alloc(sizeof(pktq_counters_t)); @ 1.8.6.1 log @file pktqueue.c was added on branch tls-maxphys on 2014-08-20 00:04:34 +0000 @ text @d1 390 @ 1.8.6.2 log @Rebase to HEAD as of a few days ago. @ text @a0 390 /* $NetBSD$ */ /*- * Copyright (c) 2014 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation * by Mindaugas Rasiukevicius. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * The packet queue (pktqueue) interface is a lockless IP input queue * which also abstracts and handles network ISR scheduling. It provides * a mechanism to enable receiver-side packet steering (RPS). */ #include __KERNEL_RCSID(0, "$NetBSD$"); #include #include #include #include #include #include #include #include #include #include /* * WARNING: update this if struct pktqueue changes. */ #define PKTQ_CLPAD \ MAX(COHERENCY_UNIT, COHERENCY_UNIT - sizeof(kmutex_t) - sizeof(u_int)) struct pktqueue { /* * The lock used for a barrier mechanism. The barrier counter, * as well as the drop counter, are managed atomically though. * Ensure this group is in a separate cache line. */ kmutex_t pq_lock; volatile u_int pq_barrier; uint8_t _pad[PKTQ_CLPAD]; /* The size of the queue, counters and the interrupt handler. */ u_int pq_maxlen; percpu_t * pq_counters; void * pq_sih; /* Finally, per-CPU queues. */ pcq_t * pq_queue[]; }; /* The counters of the packet queue. */ #define PQCNT_ENQUEUE 0 #define PQCNT_DEQUEUE 1 #define PQCNT_DROP 2 #define PQCNT_NCOUNTERS 3 typedef struct { uint64_t count[PQCNT_NCOUNTERS]; } pktq_counters_t; /* Special marker value used by pktq_barrier() mechanism. */ #define PKTQ_MARKER ((void *)(~0ULL)) /* * The total size of pktqueue_t which depends on the number of CPUs. */ #define PKTQUEUE_STRUCT_LEN(ncpu) \ roundup2(offsetof(pktqueue_t, pq_queue[ncpu]), coherency_unit) pktqueue_t * pktq_create(size_t maxlen, void (*intrh)(void *), void *sc) { const u_int sflags = SOFTINT_NET | SOFTINT_MPSAFE | SOFTINT_RCPU; const size_t len = PKTQUEUE_STRUCT_LEN(ncpu); pktqueue_t *pq; percpu_t *pc; void *sih; if ((pc = percpu_alloc(sizeof(pktq_counters_t))) == NULL) { return NULL; } if ((sih = softint_establish(sflags, intrh, sc)) == NULL) { percpu_free(pc, sizeof(pktq_counters_t)); return NULL; } pq = kmem_zalloc(len, KM_SLEEP); for (u_int i = 0; i < ncpu; i++) { pq->pq_queue[i] = pcq_create(maxlen, KM_SLEEP); } mutex_init(&pq->pq_lock, MUTEX_DEFAULT, IPL_NONE); pq->pq_maxlen = maxlen; pq->pq_counters = pc; pq->pq_sih = sih; return pq; } void pktq_destroy(pktqueue_t *pq) { const size_t len = PKTQUEUE_STRUCT_LEN(ncpu); for (u_int i = 0; i < ncpu; i++) { pcq_t *q = pq->pq_queue[i]; KASSERT(pcq_peek(q) == NULL); pcq_destroy(q); } percpu_free(pq->pq_counters, sizeof(pktq_counters_t)); softint_disestablish(pq->pq_sih); mutex_destroy(&pq->pq_lock); kmem_free(pq, len); } /* * - pktq_inc_counter: increment the counter given an ID. * - pktq_collect_counts: handler to sum up the counts from each CPU. * - pktq_getcount: return the effective count given an ID. */ static inline void pktq_inc_count(pktqueue_t *pq, u_int i) { percpu_t *pc = pq->pq_counters; pktq_counters_t *c; c = percpu_getref(pc); c->count[i]++; percpu_putref(pc); } static void pktq_collect_counts(void *mem, void *arg, struct cpu_info *ci) { const pktq_counters_t *c = mem; pktq_counters_t *sum = arg; for (u_int i = 0; i < PQCNT_NCOUNTERS; i++) { sum->count[i] += c->count[i]; } } uint64_t pktq_get_count(pktqueue_t *pq, pktq_count_t c) { pktq_counters_t sum; if (c != PKTQ_MAXLEN) { memset(&sum, 0, sizeof(sum)); percpu_foreach(pq->pq_counters, pktq_collect_counts, &sum); } switch (c) { case PKTQ_NITEMS: return sum.count[PQCNT_ENQUEUE] - sum.count[PQCNT_DEQUEUE]; case PKTQ_DROPS: return sum.count[PQCNT_DROP]; case PKTQ_MAXLEN: return pq->pq_maxlen; } return 0; } uint32_t pktq_rps_hash(const struct mbuf *m __unused) { /* * XXX: No distribution yet; the softnet_lock contention * XXX: must be eliminated first. */ return 0; } /* * pktq_enqueue: inject the packet into the end of the queue. * * => Must be called from the interrupt or with the preemption disabled. * => Consumes the packet and returns true on success. * => Returns false on failure; caller is responsible to free the packet. */ bool pktq_enqueue(pktqueue_t *pq, struct mbuf *m, const u_int hash __unused) { #if defined(_RUMPKERNEL) || defined(_RUMP_NATIVE_ABI) const unsigned cpuid = curcpu()->ci_index; #else const unsigned cpuid = hash % ncpu; #endif KASSERT(kpreempt_disabled()); if (__predict_false(!pcq_put(pq->pq_queue[cpuid], m))) { pktq_inc_count(pq, PQCNT_DROP); return false; } softint_schedule_cpu(pq->pq_sih, cpu_lookup(cpuid)); pktq_inc_count(pq, PQCNT_ENQUEUE); return true; } /* * pktq_dequeue: take a packet from the queue. * * => Must be called with preemption disabled. * => Must ensure there are not concurrent dequeue calls. */ struct mbuf * pktq_dequeue(pktqueue_t *pq) { const struct cpu_info *ci = curcpu(); const unsigned cpuid = cpu_index(ci); struct mbuf *m; m = pcq_get(pq->pq_queue[cpuid]); if (__predict_false(m == PKTQ_MARKER)) { /* Note the marker entry. */ atomic_inc_uint(&pq->pq_barrier); return NULL; } if (__predict_true(m != NULL)) { pktq_inc_count(pq, PQCNT_DEQUEUE); } return m; } /* * pktq_barrier: waits for a grace period when all packets enqueued at * the moment of calling this routine will be processed. This is used * to ensure that e.g. packets referencing some interface were drained. */ void pktq_barrier(pktqueue_t *pq) { u_int pending = 0; mutex_enter(&pq->pq_lock); KASSERT(pq->pq_barrier == 0); for (u_int i = 0; i < ncpu; i++) { pcq_t *q = pq->pq_queue[i]; /* If the queue is empty - nothing to do. */ if (pcq_peek(q) == NULL) { continue; } /* Otherwise, put the marker and entry. */ while (!pcq_put(q, PKTQ_MARKER)) { kpause("pktqsync", false, 1, NULL); } kpreempt_disable(); softint_schedule_cpu(pq->pq_sih, cpu_lookup(i)); kpreempt_enable(); pending++; } /* Wait for each queue to process the markers. */ while (pq->pq_barrier != pending) { kpause("pktqsync", false, 1, NULL); } pq->pq_barrier = 0; mutex_exit(&pq->pq_lock); } /* * pktq_flush: free mbufs in all queues. * * => The caller must ensure there are no concurrent writers or flush calls. */ void pktq_flush(pktqueue_t *pq) { struct mbuf *m; for (u_int i = 0; i < ncpu; i++) { while ((m = pcq_get(pq->pq_queue[i])) != NULL) { pktq_inc_count(pq, PQCNT_DEQUEUE); m_freem(m); } } } /* * pktq_set_maxlen: create per-CPU queues using a new size and replace * the existing queues without losing any packets. */ int pktq_set_maxlen(pktqueue_t *pq, size_t maxlen) { const u_int slotbytes = ncpu * sizeof(pcq_t *); pcq_t **qs; if (!maxlen || maxlen > PCQ_MAXLEN) return EINVAL; if (pq->pq_maxlen == maxlen) return 0; /* First, allocate the new queues and replace them. */ qs = kmem_zalloc(slotbytes, KM_SLEEP); for (u_int i = 0; i < ncpu; i++) { qs[i] = pcq_create(maxlen, KM_SLEEP); } mutex_enter(&pq->pq_lock); for (u_int i = 0; i < ncpu; i++) { /* Swap: store of a word is atomic. */ pcq_t *q = pq->pq_queue[i]; pq->pq_queue[i] = qs[i]; qs[i] = q; } pq->pq_maxlen = maxlen; mutex_exit(&pq->pq_lock); /* * At this point, the new packets are flowing into the new * queues. However, the old queues may have some packets * present which are no longer being processed. We are going * to re-enqueue them. This may change the order of packet * arrival, but it is not considered an issue. * * There may be in-flight interrupts calling pktq_dequeue() * which reference the old queues. Issue a barrier to ensure * that we are going to be the only pcq_get() callers on the * old queues. */ pktq_barrier(pq); for (u_int i = 0; i < ncpu; i++) { struct mbuf *m; while ((m = pcq_get(qs[i])) != NULL) { while (!pcq_put(pq->pq_queue[i], m)) { kpause("pktqrenq", false, 1, NULL); } } pcq_destroy(qs[i]); } /* Well, that was fun. */ kmem_free(qs, slotbytes); return 0; } int sysctl_pktq_maxlen(SYSCTLFN_ARGS, pktqueue_t *pq) { u_int nmaxlen = pktq_get_count(pq, PKTQ_MAXLEN); struct sysctlnode node = *rnode; int error; node.sysctl_data = &nmaxlen; error = sysctl_lookup(SYSCTLFN_CALL(&node)); if (error || newp == NULL) return error; return pktq_set_maxlen(pq, nmaxlen); } int sysctl_pktq_count(SYSCTLFN_ARGS, pktqueue_t *pq, u_int count_id) { int count = pktq_get_count(pq, count_id); struct sysctlnode node = *rnode; node.sysctl_data = &count; return sysctl_lookup(SYSCTLFN_CALL(&node)); } @ 1.8.6.3 log @update from HEAD @ text @d107 3 a109 1 pc = percpu_alloc(sizeof(pktq_counters_t)); @ 1.8.2.1 log @file pktqueue.c was added on branch tls-earlyentropy on 2014-08-10 06:56:15 +0000 @ text @d1 390 @ 1.8.2.2 log @Rebase. @ text @a0 390 /* $NetBSD: pktqueue.c,v 1.8 2014/07/04 01:50:22 ozaki-r Exp $ */ /*- * Copyright (c) 2014 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation * by Mindaugas Rasiukevicius. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * The packet queue (pktqueue) interface is a lockless IP input queue * which also abstracts and handles network ISR scheduling. It provides * a mechanism to enable receiver-side packet steering (RPS). */ #include __KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.8 2014/07/04 01:50:22 ozaki-r Exp $"); #include #include #include #include #include #include #include #include #include #include /* * WARNING: update this if struct pktqueue changes. */ #define PKTQ_CLPAD \ MAX(COHERENCY_UNIT, COHERENCY_UNIT - sizeof(kmutex_t) - sizeof(u_int)) struct pktqueue { /* * The lock used for a barrier mechanism. The barrier counter, * as well as the drop counter, are managed atomically though. * Ensure this group is in a separate cache line. */ kmutex_t pq_lock; volatile u_int pq_barrier; uint8_t _pad[PKTQ_CLPAD]; /* The size of the queue, counters and the interrupt handler. */ u_int pq_maxlen; percpu_t * pq_counters; void * pq_sih; /* Finally, per-CPU queues. */ pcq_t * pq_queue[]; }; /* The counters of the packet queue. */ #define PQCNT_ENQUEUE 0 #define PQCNT_DEQUEUE 1 #define PQCNT_DROP 2 #define PQCNT_NCOUNTERS 3 typedef struct { uint64_t count[PQCNT_NCOUNTERS]; } pktq_counters_t; /* Special marker value used by pktq_barrier() mechanism. */ #define PKTQ_MARKER ((void *)(~0ULL)) /* * The total size of pktqueue_t which depends on the number of CPUs. */ #define PKTQUEUE_STRUCT_LEN(ncpu) \ roundup2(offsetof(pktqueue_t, pq_queue[ncpu]), coherency_unit) pktqueue_t * pktq_create(size_t maxlen, void (*intrh)(void *), void *sc) { const u_int sflags = SOFTINT_NET | SOFTINT_MPSAFE | SOFTINT_RCPU; const size_t len = PKTQUEUE_STRUCT_LEN(ncpu); pktqueue_t *pq; percpu_t *pc; void *sih; if ((pc = percpu_alloc(sizeof(pktq_counters_t))) == NULL) { return NULL; } if ((sih = softint_establish(sflags, intrh, sc)) == NULL) { percpu_free(pc, sizeof(pktq_counters_t)); return NULL; } pq = kmem_zalloc(len, KM_SLEEP); for (u_int i = 0; i < ncpu; i++) { pq->pq_queue[i] = pcq_create(maxlen, KM_SLEEP); } mutex_init(&pq->pq_lock, MUTEX_DEFAULT, IPL_NONE); pq->pq_maxlen = maxlen; pq->pq_counters = pc; pq->pq_sih = sih; return pq; } void pktq_destroy(pktqueue_t *pq) { const size_t len = PKTQUEUE_STRUCT_LEN(ncpu); for (u_int i = 0; i < ncpu; i++) { pcq_t *q = pq->pq_queue[i]; KASSERT(pcq_peek(q) == NULL); pcq_destroy(q); } percpu_free(pq->pq_counters, sizeof(pktq_counters_t)); softint_disestablish(pq->pq_sih); mutex_destroy(&pq->pq_lock); kmem_free(pq, len); } /* * - pktq_inc_counter: increment the counter given an ID. * - pktq_collect_counts: handler to sum up the counts from each CPU. * - pktq_getcount: return the effective count given an ID. */ static inline void pktq_inc_count(pktqueue_t *pq, u_int i) { percpu_t *pc = pq->pq_counters; pktq_counters_t *c; c = percpu_getref(pc); c->count[i]++; percpu_putref(pc); } static void pktq_collect_counts(void *mem, void *arg, struct cpu_info *ci) { const pktq_counters_t *c = mem; pktq_counters_t *sum = arg; for (u_int i = 0; i < PQCNT_NCOUNTERS; i++) { sum->count[i] += c->count[i]; } } uint64_t pktq_get_count(pktqueue_t *pq, pktq_count_t c) { pktq_counters_t sum; if (c != PKTQ_MAXLEN) { memset(&sum, 0, sizeof(sum)); percpu_foreach(pq->pq_counters, pktq_collect_counts, &sum); } switch (c) { case PKTQ_NITEMS: return sum.count[PQCNT_ENQUEUE] - sum.count[PQCNT_DEQUEUE]; case PKTQ_DROPS: return sum.count[PQCNT_DROP]; case PKTQ_MAXLEN: return pq->pq_maxlen; } return 0; } uint32_t pktq_rps_hash(const struct mbuf *m __unused) { /* * XXX: No distribution yet; the softnet_lock contention * XXX: must be eliminated first. */ return 0; } /* * pktq_enqueue: inject the packet into the end of the queue. * * => Must be called from the interrupt or with the preemption disabled. * => Consumes the packet and returns true on success. * => Returns false on failure; caller is responsible to free the packet. */ bool pktq_enqueue(pktqueue_t *pq, struct mbuf *m, const u_int hash __unused) { #if defined(_RUMPKERNEL) || defined(_RUMP_NATIVE_ABI) const unsigned cpuid = curcpu()->ci_index; #else const unsigned cpuid = hash % ncpu; #endif KASSERT(kpreempt_disabled()); if (__predict_false(!pcq_put(pq->pq_queue[cpuid], m))) { pktq_inc_count(pq, PQCNT_DROP); return false; } softint_schedule_cpu(pq->pq_sih, cpu_lookup(cpuid)); pktq_inc_count(pq, PQCNT_ENQUEUE); return true; } /* * pktq_dequeue: take a packet from the queue. * * => Must be called with preemption disabled. * => Must ensure there are not concurrent dequeue calls. */ struct mbuf * pktq_dequeue(pktqueue_t *pq) { const struct cpu_info *ci = curcpu(); const unsigned cpuid = cpu_index(ci); struct mbuf *m; m = pcq_get(pq->pq_queue[cpuid]); if (__predict_false(m == PKTQ_MARKER)) { /* Note the marker entry. */ atomic_inc_uint(&pq->pq_barrier); return NULL; } if (__predict_true(m != NULL)) { pktq_inc_count(pq, PQCNT_DEQUEUE); } return m; } /* * pktq_barrier: waits for a grace period when all packets enqueued at * the moment of calling this routine will be processed. This is used * to ensure that e.g. packets referencing some interface were drained. */ void pktq_barrier(pktqueue_t *pq) { u_int pending = 0; mutex_enter(&pq->pq_lock); KASSERT(pq->pq_barrier == 0); for (u_int i = 0; i < ncpu; i++) { pcq_t *q = pq->pq_queue[i]; /* If the queue is empty - nothing to do. */ if (pcq_peek(q) == NULL) { continue; } /* Otherwise, put the marker and entry. */ while (!pcq_put(q, PKTQ_MARKER)) { kpause("pktqsync", false, 1, NULL); } kpreempt_disable(); softint_schedule_cpu(pq->pq_sih, cpu_lookup(i)); kpreempt_enable(); pending++; } /* Wait for each queue to process the markers. */ while (pq->pq_barrier != pending) { kpause("pktqsync", false, 1, NULL); } pq->pq_barrier = 0; mutex_exit(&pq->pq_lock); } /* * pktq_flush: free mbufs in all queues. * * => The caller must ensure there are no concurrent writers or flush calls. */ void pktq_flush(pktqueue_t *pq) { struct mbuf *m; for (u_int i = 0; i < ncpu; i++) { while ((m = pcq_get(pq->pq_queue[i])) != NULL) { pktq_inc_count(pq, PQCNT_DEQUEUE); m_freem(m); } } } /* * pktq_set_maxlen: create per-CPU queues using a new size and replace * the existing queues without losing any packets. */ int pktq_set_maxlen(pktqueue_t *pq, size_t maxlen) { const u_int slotbytes = ncpu * sizeof(pcq_t *); pcq_t **qs; if (!maxlen || maxlen > PCQ_MAXLEN) return EINVAL; if (pq->pq_maxlen == maxlen) return 0; /* First, allocate the new queues and replace them. */ qs = kmem_zalloc(slotbytes, KM_SLEEP); for (u_int i = 0; i < ncpu; i++) { qs[i] = pcq_create(maxlen, KM_SLEEP); } mutex_enter(&pq->pq_lock); for (u_int i = 0; i < ncpu; i++) { /* Swap: store of a word is atomic. */ pcq_t *q = pq->pq_queue[i]; pq->pq_queue[i] = qs[i]; qs[i] = q; } pq->pq_maxlen = maxlen; mutex_exit(&pq->pq_lock); /* * At this point, the new packets are flowing into the new * queues. However, the old queues may have some packets * present which are no longer being processed. We are going * to re-enqueue them. This may change the order of packet * arrival, but it is not considered an issue. * * There may be in-flight interrupts calling pktq_dequeue() * which reference the old queues. Issue a barrier to ensure * that we are going to be the only pcq_get() callers on the * old queues. */ pktq_barrier(pq); for (u_int i = 0; i < ncpu; i++) { struct mbuf *m; while ((m = pcq_get(qs[i])) != NULL) { while (!pcq_put(pq->pq_queue[i], m)) { kpause("pktqrenq", false, 1, NULL); } } pcq_destroy(qs[i]); } /* Well, that was fun. */ kmem_free(qs, slotbytes); return 0; } int sysctl_pktq_maxlen(SYSCTLFN_ARGS, pktqueue_t *pq) { u_int nmaxlen = pktq_get_count(pq, PKTQ_MAXLEN); struct sysctlnode node = *rnode; int error; node.sysctl_data = &nmaxlen; error = sysctl_lookup(SYSCTLFN_CALL(&node)); if (error || newp == NULL) return error; return pktq_set_maxlen(pq, nmaxlen); } int sysctl_pktq_count(SYSCTLFN_ARGS, pktqueue_t *pq, u_int count_id) { int count = pktq_get_count(pq, count_id); struct sysctlnode node = *rnode; node.sysctl_data = &count; return sysctl_lookup(SYSCTLFN_CALL(&node)); } @ 1.7 log @Restore RPS of pktq_enqueue unless _RUMPKERNEL It's a workaround and would be fixed in rump soon. ok pooka@@ @ text @d1 1 a1 1 /* $NetBSD: pktqueue.c,v 1.6 2014/06/16 00:40:10 ozaki-r Exp $ */ d39 1 a39 1 __KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.6 2014/06/16 00:40:10 ozaki-r Exp $"); d211 1 a211 1 #ifdef _RUMPKERNEL @ 1.6 log @Move sysctl_pktq_{maxlen,count} to pktqueue.c and make them global They will be used by bridge. ok rmind@@ @ text @d1 1 a1 1 /* $NetBSD: pktqueue.c,v 1.5 2014/06/16 00:33:39 ozaki-r Exp $ */ d39 1 a39 1 __KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.5 2014/06/16 00:33:39 ozaki-r Exp $"); d211 5 a215 1 const unsigned cpuid = curcpu()->ci_index /* hash % ncpu */; @ 1.5 log @Add 3rd argument to pktq_create to pass sc It will be used to pass bridge sc for bridge_forward softint. ok rmind@@ @ text @d1 1 a1 1 /* $NetBSD: pktqueue.c,v 1.4 2014/06/09 14:44:48 rmind Exp $ */ d39 1 a39 1 __KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.4 2014/06/09 14:44:48 rmind Exp $"); d364 23 @ 1.4 log @pktqueue: add or fix some comments, remove some header inclusions. @ text @d1 1 a1 1 /* $NetBSD: pktqueue.c,v 1.3 2014/06/09 13:03:16 rmind Exp $ */ d39 1 a39 1 __KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.3 2014/06/09 13:03:16 rmind Exp $"); d99 1 a99 1 pktq_create(size_t maxlen, void (*intrh)(void *)) d110 1 a110 1 if ((sih = softint_establish(sflags, intrh, NULL)) == NULL) { @ 1.3 log @Restore the assert in RUMP's softint_schedule_cpu() and just ensure curcpu() in the caller. @ text @d1 1 a1 1 /* $NetBSD: pktqueue.c,v 1.2 2014/06/09 12:57:04 rmind Exp $ */ d32 6 d39 1 a39 1 __KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.2 2014/06/09 12:57:04 rmind Exp $"); a51 4 #include #include #include d290 1 a290 1 * => The caller must ensure there are no concurrent writers or flush. d337 2 a338 2 * queues. However, the old queues may have same packets * present which are no longer being present. We are going d342 1 a342 1 * There may also in-flight interrupts calling pktq_dequeue() @ 1.2 log @Implement pktq_set_maxlen() and let sysctl net.inet.{ip,ip6}.ifq.maxlen be changed on the fly again. @ text @d1 1 a1 1 /* $NetBSD: pktqueue.c,v 1.1 2014/06/05 23:48:16 rmind Exp $ */ d33 1 a33 1 __KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.1 2014/06/05 23:48:16 rmind Exp $"); d207 1 a207 1 pktq_enqueue(pktqueue_t *pq, struct mbuf *m, const u_int hash) d209 1 a209 1 const unsigned cpuid = hash % ncpu; @ 1.1 log @- Implement pktqueue interface for lockless IP input queue. - Replace ipintrq and ip6intrq with the pktqueue mechanism. - Eliminate kernel-lock from ipintr() and ip6intr(). - Some preparation work to push softnet_lock out of ipintr(). Discussed on tech-net. @ text @d1 1 a1 1 /* $NetBSD$ */ d33 1 a33 1 __KERNEL_RCSID(0, "$NetBSD$"); d302 60 @