diff --git a/README.md b/README.md index 81a0c1a644b0f..a81da6f9e6d59 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,8 @@ +``` +OUT="$PWD/.selftests-bpf-veristat-final" ./script.sh +sudo .selftests-bpf-veristat-final/veristat /Users/javierhonduco/src/bpf/.selftests-bpf-veristat-final/lightswitch_bpf/profiler.bpf.o +``` + # BPF CI GitHub Actions worfklows This repository contains GitHub Actions workflow definitions, scripts and configuration files used by those workflows. diff --git a/include/uapi/linux/netfilter/xt_CONNMARK.h b/include/uapi/linux/netfilter/xt_CONNMARK.h deleted file mode 100644 index 36cc956ead1ae..0000000000000 --- a/include/uapi/linux/netfilter/xt_CONNMARK.h +++ /dev/null @@ -1,7 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -#ifndef _XT_CONNMARK_H_target -#define _XT_CONNMARK_H_target - -#include - -#endif /*_XT_CONNMARK_H_target*/ diff --git a/include/uapi/linux/netfilter/xt_DSCP.h b/include/uapi/linux/netfilter/xt_DSCP.h deleted file mode 100644 index 223d635e8b6f7..0000000000000 --- a/include/uapi/linux/netfilter/xt_DSCP.h +++ /dev/null @@ -1,27 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* x_tables module for setting the IPv4/IPv6 DSCP field - * - * (C) 2002 Harald Welte - * based on ipt_FTOS.c (C) 2000 by Matthew G. Marsh - * This software is distributed under GNU GPL v2, 1991 - * - * See RFC2474 for a description of the DSCP field within the IP Header. - * - * xt_DSCP.h,v 1.7 2002/03/14 12:03:13 laforge Exp -*/ -#ifndef _XT_DSCP_TARGET_H -#define _XT_DSCP_TARGET_H -#include -#include - -/* target info */ -struct xt_DSCP_info { - __u8 dscp; -}; - -struct xt_tos_target_info { - __u8 tos_value; - __u8 tos_mask; -}; - -#endif /* _XT_DSCP_TARGET_H */ diff --git a/include/uapi/linux/netfilter/xt_MARK.h b/include/uapi/linux/netfilter/xt_MARK.h deleted file mode 100644 index f1fe2b4be9332..0000000000000 --- a/include/uapi/linux/netfilter/xt_MARK.h +++ /dev/null @@ -1,7 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -#ifndef _XT_MARK_H_target -#define _XT_MARK_H_target - -#include - -#endif /*_XT_MARK_H_target */ diff --git a/include/uapi/linux/netfilter/xt_RATEEST.h b/include/uapi/linux/netfilter/xt_RATEEST.h deleted file mode 100644 index 2b87a71e6266e..0000000000000 --- a/include/uapi/linux/netfilter/xt_RATEEST.h +++ /dev/null @@ -1,17 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -#ifndef _XT_RATEEST_TARGET_H -#define _XT_RATEEST_TARGET_H - -#include -#include - -struct xt_rateest_target_info { - char name[IFNAMSIZ]; - __s8 interval; - __u8 ewma_log; - - /* Used internally by the kernel */ - struct xt_rateest *est __attribute__((aligned(8))); -}; - -#endif /* _XT_RATEEST_TARGET_H */ diff --git a/include/uapi/linux/netfilter/xt_TCPMSS.h b/include/uapi/linux/netfilter/xt_TCPMSS.h deleted file mode 100644 index 65ea6c9dab4b6..0000000000000 --- a/include/uapi/linux/netfilter/xt_TCPMSS.h +++ /dev/null @@ -1,13 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -#ifndef _XT_TCPMSS_H -#define _XT_TCPMSS_H - -#include - -struct xt_tcpmss_info { - __u16 mss; -}; - -#define XT_TCPMSS_CLAMP_PMTU 0xffff - -#endif /* _XT_TCPMSS_H */ diff --git a/include/uapi/linux/netfilter_ipv4/ipt_ECN.h b/include/uapi/linux/netfilter_ipv4/ipt_ECN.h deleted file mode 100644 index e3630fd045b8d..0000000000000 --- a/include/uapi/linux/netfilter_ipv4/ipt_ECN.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* Header file for iptables ipt_ECN target - * - * (C) 2002 by Harald Welte - * - * This software is distributed under GNU GPL v2, 1991 - * - * ipt_ECN.h,v 1.3 2002/05/29 12:17:40 laforge Exp -*/ -#ifndef _IPT_ECN_TARGET_H -#define _IPT_ECN_TARGET_H - -#include -#include - -#define IPT_ECN_IP_MASK (~XT_DSCP_MASK) - -#define IPT_ECN_OP_SET_IP 0x01 /* set ECN bits of IPv4 header */ -#define IPT_ECN_OP_SET_ECE 0x10 /* set ECE bit of TCP header */ -#define IPT_ECN_OP_SET_CWR 0x20 /* set CWR bit of TCP header */ - -#define IPT_ECN_OP_MASK 0xce - -struct ipt_ECN_info { - __u8 operation; /* bitset of operations */ - __u8 ip_ect; /* ECT codepoint of IPv4 header, pre-shifted */ - union { - struct { - __u8 ece:1, cwr:1; /* TCP ECT bits */ - } tcp; - } proto; -}; - -#endif /* _IPT_ECN_TARGET_H */ diff --git a/include/uapi/linux/netfilter_ipv4/ipt_TTL.h b/include/uapi/linux/netfilter_ipv4/ipt_TTL.h deleted file mode 100644 index 57d2fc67a9437..0000000000000 --- a/include/uapi/linux/netfilter_ipv4/ipt_TTL.h +++ /dev/null @@ -1,24 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* TTL modification module for IP tables - * (C) 2000 by Harald Welte */ - -#ifndef _IPT_TTL_H -#define _IPT_TTL_H - -#include - -enum { - IPT_TTL_SET = 0, - IPT_TTL_INC, - IPT_TTL_DEC -}; - -#define IPT_TTL_MAXMODE IPT_TTL_DEC - -struct ipt_TTL_info { - __u8 mode; - __u8 ttl; -}; - - -#endif diff --git a/include/uapi/linux/netfilter_ipv6/ip6t_HL.h b/include/uapi/linux/netfilter_ipv6/ip6t_HL.h deleted file mode 100644 index eaed56a287b47..0000000000000 --- a/include/uapi/linux/netfilter_ipv6/ip6t_HL.h +++ /dev/null @@ -1,25 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* Hop Limit modification module for ip6tables - * Maciej Soltysiak - * Based on HW's TTL module */ - -#ifndef _IP6T_HL_H -#define _IP6T_HL_H - -#include - -enum { - IP6T_HL_SET = 0, - IP6T_HL_INC, - IP6T_HL_DEC -}; - -#define IP6T_HL_MAXMODE IP6T_HL_DEC - -struct ip6t_HL_info { - __u8 mode; - __u8 hop_limit; -}; - - -#endif diff --git a/net/netfilter/xt_DSCP.c b/net/netfilter/xt_DSCP.c deleted file mode 100644 index cfa44515ab72d..0000000000000 --- a/net/netfilter/xt_DSCP.c +++ /dev/null @@ -1,161 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* x_tables module for setting the IPv4/IPv6 DSCP field, Version 1.8 - * - * (C) 2002 by Harald Welte - * based on ipt_FTOS.c (C) 2000 by Matthew G. Marsh - * - * See RFC2474 for a description of the DSCP field within the IP Header. -*/ -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include -#include -#include -#include -#include - -#include -#include - -MODULE_AUTHOR("Harald Welte "); -MODULE_DESCRIPTION("Xtables: DSCP/TOS field modification"); -MODULE_LICENSE("GPL"); -MODULE_ALIAS("ipt_DSCP"); -MODULE_ALIAS("ip6t_DSCP"); -MODULE_ALIAS("ipt_TOS"); -MODULE_ALIAS("ip6t_TOS"); - -#define XT_DSCP_ECN_MASK 3u - -static unsigned int -dscp_tg(struct sk_buff *skb, const struct xt_action_param *par) -{ - const struct xt_DSCP_info *dinfo = par->targinfo; - u_int8_t dscp = ipv4_get_dsfield(ip_hdr(skb)) >> XT_DSCP_SHIFT; - - if (dscp != dinfo->dscp) { - if (skb_ensure_writable(skb, sizeof(struct iphdr))) - return NF_DROP; - - ipv4_change_dsfield(ip_hdr(skb), XT_DSCP_ECN_MASK, - dinfo->dscp << XT_DSCP_SHIFT); - - } - return XT_CONTINUE; -} - -static unsigned int -dscp_tg6(struct sk_buff *skb, const struct xt_action_param *par) -{ - const struct xt_DSCP_info *dinfo = par->targinfo; - u_int8_t dscp = ipv6_get_dsfield(ipv6_hdr(skb)) >> XT_DSCP_SHIFT; - - if (dscp != dinfo->dscp) { - if (skb_ensure_writable(skb, sizeof(struct ipv6hdr))) - return NF_DROP; - - ipv6_change_dsfield(ipv6_hdr(skb), XT_DSCP_ECN_MASK, - dinfo->dscp << XT_DSCP_SHIFT); - } - return XT_CONTINUE; -} - -static int dscp_tg_check(const struct xt_tgchk_param *par) -{ - const struct xt_DSCP_info *info = par->targinfo; - - if (info->dscp > XT_DSCP_MAX) - return -EDOM; - return 0; -} - -static unsigned int -tos_tg(struct sk_buff *skb, const struct xt_action_param *par) -{ - const struct xt_tos_target_info *info = par->targinfo; - struct iphdr *iph = ip_hdr(skb); - u_int8_t orig, nv; - - orig = ipv4_get_dsfield(iph); - nv = (orig & ~info->tos_mask) ^ info->tos_value; - - if (orig != nv) { - if (skb_ensure_writable(skb, sizeof(struct iphdr))) - return NF_DROP; - iph = ip_hdr(skb); - ipv4_change_dsfield(iph, 0, nv); - } - - return XT_CONTINUE; -} - -static unsigned int -tos_tg6(struct sk_buff *skb, const struct xt_action_param *par) -{ - const struct xt_tos_target_info *info = par->targinfo; - struct ipv6hdr *iph = ipv6_hdr(skb); - u_int8_t orig, nv; - - orig = ipv6_get_dsfield(iph); - nv = (orig & ~info->tos_mask) ^ info->tos_value; - - if (orig != nv) { - if (skb_ensure_writable(skb, sizeof(struct iphdr))) - return NF_DROP; - iph = ipv6_hdr(skb); - ipv6_change_dsfield(iph, 0, nv); - } - - return XT_CONTINUE; -} - -static struct xt_target dscp_tg_reg[] __read_mostly = { - { - .name = "DSCP", - .family = NFPROTO_IPV4, - .checkentry = dscp_tg_check, - .target = dscp_tg, - .targetsize = sizeof(struct xt_DSCP_info), - .table = "mangle", - .me = THIS_MODULE, - }, - { - .name = "DSCP", - .family = NFPROTO_IPV6, - .checkentry = dscp_tg_check, - .target = dscp_tg6, - .targetsize = sizeof(struct xt_DSCP_info), - .table = "mangle", - .me = THIS_MODULE, - }, - { - .name = "TOS", - .revision = 1, - .family = NFPROTO_IPV4, - .table = "mangle", - .target = tos_tg, - .targetsize = sizeof(struct xt_tos_target_info), - .me = THIS_MODULE, - }, - { - .name = "TOS", - .revision = 1, - .family = NFPROTO_IPV6, - .table = "mangle", - .target = tos_tg6, - .targetsize = sizeof(struct xt_tos_target_info), - .me = THIS_MODULE, - }, -}; - -static int __init dscp_tg_init(void) -{ - return xt_register_targets(dscp_tg_reg, ARRAY_SIZE(dscp_tg_reg)); -} - -static void __exit dscp_tg_exit(void) -{ - xt_unregister_targets(dscp_tg_reg, ARRAY_SIZE(dscp_tg_reg)); -} - -module_init(dscp_tg_init); -module_exit(dscp_tg_exit); diff --git a/net/netfilter/xt_HL.c b/net/netfilter/xt_HL.c deleted file mode 100644 index 7873b834c3004..0000000000000 --- a/net/netfilter/xt_HL.c +++ /dev/null @@ -1,159 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * TTL modification target for IP tables - * (C) 2000,2005 by Harald Welte - * - * Hop Limit modification target for ip6tables - * Maciej Soltysiak - */ -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include -#include -#include -#include -#include - -#include -#include -#include - -MODULE_AUTHOR("Harald Welte "); -MODULE_AUTHOR("Maciej Soltysiak "); -MODULE_DESCRIPTION("Xtables: Hoplimit/TTL Limit field modification target"); -MODULE_LICENSE("GPL"); - -static unsigned int -ttl_tg(struct sk_buff *skb, const struct xt_action_param *par) -{ - struct iphdr *iph; - const struct ipt_TTL_info *info = par->targinfo; - int new_ttl; - - if (skb_ensure_writable(skb, sizeof(*iph))) - return NF_DROP; - - iph = ip_hdr(skb); - - switch (info->mode) { - case IPT_TTL_SET: - new_ttl = info->ttl; - break; - case IPT_TTL_INC: - new_ttl = iph->ttl + info->ttl; - if (new_ttl > 255) - new_ttl = 255; - break; - case IPT_TTL_DEC: - new_ttl = iph->ttl - info->ttl; - if (new_ttl < 0) - new_ttl = 0; - break; - default: - new_ttl = iph->ttl; - break; - } - - if (new_ttl != iph->ttl) { - csum_replace2(&iph->check, htons(iph->ttl << 8), - htons(new_ttl << 8)); - iph->ttl = new_ttl; - } - - return XT_CONTINUE; -} - -static unsigned int -hl_tg6(struct sk_buff *skb, const struct xt_action_param *par) -{ - struct ipv6hdr *ip6h; - const struct ip6t_HL_info *info = par->targinfo; - int new_hl; - - if (skb_ensure_writable(skb, sizeof(*ip6h))) - return NF_DROP; - - ip6h = ipv6_hdr(skb); - - switch (info->mode) { - case IP6T_HL_SET: - new_hl = info->hop_limit; - break; - case IP6T_HL_INC: - new_hl = ip6h->hop_limit + info->hop_limit; - if (new_hl > 255) - new_hl = 255; - break; - case IP6T_HL_DEC: - new_hl = ip6h->hop_limit - info->hop_limit; - if (new_hl < 0) - new_hl = 0; - break; - default: - new_hl = ip6h->hop_limit; - break; - } - - ip6h->hop_limit = new_hl; - - return XT_CONTINUE; -} - -static int ttl_tg_check(const struct xt_tgchk_param *par) -{ - const struct ipt_TTL_info *info = par->targinfo; - - if (info->mode > IPT_TTL_MAXMODE) - return -EINVAL; - if (info->mode != IPT_TTL_SET && info->ttl == 0) - return -EINVAL; - return 0; -} - -static int hl_tg6_check(const struct xt_tgchk_param *par) -{ - const struct ip6t_HL_info *info = par->targinfo; - - if (info->mode > IP6T_HL_MAXMODE) - return -EINVAL; - if (info->mode != IP6T_HL_SET && info->hop_limit == 0) - return -EINVAL; - return 0; -} - -static struct xt_target hl_tg_reg[] __read_mostly = { - { - .name = "TTL", - .revision = 0, - .family = NFPROTO_IPV4, - .target = ttl_tg, - .targetsize = sizeof(struct ipt_TTL_info), - .table = "mangle", - .checkentry = ttl_tg_check, - .me = THIS_MODULE, - }, - { - .name = "HL", - .revision = 0, - .family = NFPROTO_IPV6, - .target = hl_tg6, - .targetsize = sizeof(struct ip6t_HL_info), - .table = "mangle", - .checkentry = hl_tg6_check, - .me = THIS_MODULE, - }, -}; - -static int __init hl_tg_init(void) -{ - return xt_register_targets(hl_tg_reg, ARRAY_SIZE(hl_tg_reg)); -} - -static void __exit hl_tg_exit(void) -{ - xt_unregister_targets(hl_tg_reg, ARRAY_SIZE(hl_tg_reg)); -} - -module_init(hl_tg_init); -module_exit(hl_tg_exit); -MODULE_ALIAS("ipt_TTL"); -MODULE_ALIAS("ip6t_HL"); diff --git a/net/netfilter/xt_RATEEST.c b/net/netfilter/xt_RATEEST.c deleted file mode 100644 index 91270d467ffdf..0000000000000 --- a/net/netfilter/xt_RATEEST.c +++ /dev/null @@ -1,248 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * (C) 2007 Patrick McHardy - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#define RATEEST_HSIZE 16 - -struct xt_rateest_net { - struct mutex hash_lock; - struct hlist_head hash[RATEEST_HSIZE]; -}; - -static unsigned int xt_rateest_id; - -static unsigned int jhash_rnd __read_mostly; - -static unsigned int xt_rateest_hash(const char *name) -{ - return jhash(name, sizeof_field(struct xt_rateest, name), jhash_rnd) & - (RATEEST_HSIZE - 1); -} - -static void xt_rateest_hash_insert(struct xt_rateest_net *xn, - struct xt_rateest *est) -{ - unsigned int h; - - h = xt_rateest_hash(est->name); - hlist_add_head(&est->list, &xn->hash[h]); -} - -static struct xt_rateest *__xt_rateest_lookup(struct xt_rateest_net *xn, - const char *name) -{ - struct xt_rateest *est; - unsigned int h; - - h = xt_rateest_hash(name); - hlist_for_each_entry(est, &xn->hash[h], list) { - if (strcmp(est->name, name) == 0) { - est->refcnt++; - return est; - } - } - - return NULL; -} - -struct xt_rateest *xt_rateest_lookup(struct net *net, const char *name) -{ - struct xt_rateest_net *xn = net_generic(net, xt_rateest_id); - struct xt_rateest *est; - - mutex_lock(&xn->hash_lock); - est = __xt_rateest_lookup(xn, name); - mutex_unlock(&xn->hash_lock); - return est; -} -EXPORT_SYMBOL_GPL(xt_rateest_lookup); - -void xt_rateest_put(struct net *net, struct xt_rateest *est) -{ - struct xt_rateest_net *xn = net_generic(net, xt_rateest_id); - - mutex_lock(&xn->hash_lock); - if (--est->refcnt == 0) { - hlist_del(&est->list); - gen_kill_estimator(&est->rate_est); - /* - * gen_estimator est_timer() might access est->lock or bstats, - * wait a RCU grace period before freeing 'est' - */ - kfree_rcu(est, rcu); - } - mutex_unlock(&xn->hash_lock); -} -EXPORT_SYMBOL_GPL(xt_rateest_put); - -static unsigned int -xt_rateest_tg(struct sk_buff *skb, const struct xt_action_param *par) -{ - const struct xt_rateest_target_info *info = par->targinfo; - struct gnet_stats_basic_sync *stats = &info->est->bstats; - - spin_lock_bh(&info->est->lock); - u64_stats_add(&stats->bytes, skb->len); - u64_stats_inc(&stats->packets); - spin_unlock_bh(&info->est->lock); - - return XT_CONTINUE; -} - -static int xt_rateest_tg_checkentry(const struct xt_tgchk_param *par) -{ - struct xt_rateest_net *xn = net_generic(par->net, xt_rateest_id); - struct xt_rateest_target_info *info = par->targinfo; - struct xt_rateest *est; - struct { - struct nlattr opt; - struct gnet_estimator est; - } cfg; - int ret; - - if (strnlen(info->name, sizeof(est->name)) >= sizeof(est->name)) - return -ENAMETOOLONG; - - net_get_random_once(&jhash_rnd, sizeof(jhash_rnd)); - - mutex_lock(&xn->hash_lock); - est = __xt_rateest_lookup(xn, info->name); - if (est) { - mutex_unlock(&xn->hash_lock); - /* - * If estimator parameters are specified, they must match the - * existing estimator. - */ - if ((!info->interval && !info->ewma_log) || - (info->interval != est->params.interval || - info->ewma_log != est->params.ewma_log)) { - xt_rateest_put(par->net, est); - return -EINVAL; - } - info->est = est; - return 0; - } - - ret = -ENOMEM; - est = kzalloc_obj(*est); - if (!est) - goto err1; - - gnet_stats_basic_sync_init(&est->bstats); - strscpy(est->name, info->name, sizeof(est->name)); - spin_lock_init(&est->lock); - est->refcnt = 1; - est->params.interval = info->interval; - est->params.ewma_log = info->ewma_log; - - cfg.opt.nla_len = nla_attr_size(sizeof(cfg.est)); - cfg.opt.nla_type = TCA_STATS_RATE_EST; - cfg.est.interval = info->interval; - cfg.est.ewma_log = info->ewma_log; - - ret = gen_new_estimator(&est->bstats, NULL, &est->rate_est, - &est->lock, NULL, &cfg.opt); - if (ret < 0) - goto err2; - - info->est = est; - xt_rateest_hash_insert(xn, est); - mutex_unlock(&xn->hash_lock); - return 0; - -err2: - kfree(est); -err1: - mutex_unlock(&xn->hash_lock); - return ret; -} - -static void xt_rateest_tg_destroy(const struct xt_tgdtor_param *par) -{ - struct xt_rateest_target_info *info = par->targinfo; - - xt_rateest_put(par->net, info->est); -} - -static struct xt_target xt_rateest_tg_reg[] __read_mostly = { - { - .name = "RATEEST", - .revision = 0, - .family = NFPROTO_IPV4, - .target = xt_rateest_tg, - .checkentry = xt_rateest_tg_checkentry, - .destroy = xt_rateest_tg_destroy, - .targetsize = sizeof(struct xt_rateest_target_info), - .usersize = offsetof(struct xt_rateest_target_info, est), - .me = THIS_MODULE, - }, -#if IS_ENABLED(CONFIG_IP6_NF_IPTABLES) - { - .name = "RATEEST", - .revision = 0, - .family = NFPROTO_IPV6, - .target = xt_rateest_tg, - .checkentry = xt_rateest_tg_checkentry, - .destroy = xt_rateest_tg_destroy, - .targetsize = sizeof(struct xt_rateest_target_info), - .usersize = offsetof(struct xt_rateest_target_info, est), - .me = THIS_MODULE, - }, -#endif -}; - -static __net_init int xt_rateest_net_init(struct net *net) -{ - struct xt_rateest_net *xn = net_generic(net, xt_rateest_id); - int i; - - mutex_init(&xn->hash_lock); - for (i = 0; i < ARRAY_SIZE(xn->hash); i++) - INIT_HLIST_HEAD(&xn->hash[i]); - return 0; -} - -static struct pernet_operations xt_rateest_net_ops = { - .init = xt_rateest_net_init, - .id = &xt_rateest_id, - .size = sizeof(struct xt_rateest_net), -}; - -static int __init xt_rateest_tg_init(void) -{ - int err = register_pernet_subsys(&xt_rateest_net_ops); - - if (err) - return err; - return xt_register_targets(xt_rateest_tg_reg, ARRAY_SIZE(xt_rateest_tg_reg)); -} - -static void __exit xt_rateest_tg_fini(void) -{ - xt_unregister_targets(xt_rateest_tg_reg, ARRAY_SIZE(xt_rateest_tg_reg)); - unregister_pernet_subsys(&xt_rateest_net_ops); -} - - -MODULE_AUTHOR("Patrick McHardy "); -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("Xtables: packet rate estimator"); -MODULE_ALIAS("ipt_RATEEST"); -MODULE_ALIAS("ip6t_RATEEST"); -module_init(xt_rateest_tg_init); -module_exit(xt_rateest_tg_fini); diff --git a/net/netfilter/xt_TCPMSS.c b/net/netfilter/xt_TCPMSS.c deleted file mode 100644 index 80e1634bc51f8..0000000000000 --- a/net/netfilter/xt_TCPMSS.c +++ /dev/null @@ -1,346 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * This is a module which is used for setting the MSS option in TCP packets. - * - * Copyright (C) 2000 Marc Boucher - * Copyright (C) 2007 Patrick McHardy - */ -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Marc Boucher "); -MODULE_DESCRIPTION("Xtables: TCP Maximum Segment Size (MSS) adjustment"); -MODULE_ALIAS("ipt_TCPMSS"); -MODULE_ALIAS("ip6t_TCPMSS"); - -static inline unsigned int -optlen(const u_int8_t *opt, unsigned int offset) -{ - /* Beware zero-length options: make finite progress */ - if (opt[offset] <= TCPOPT_NOP || opt[offset+1] == 0) - return 1; - else - return opt[offset+1]; -} - -static u_int32_t tcpmss_reverse_mtu(struct net *net, - const struct sk_buff *skb, - unsigned int family) -{ - struct flowi fl; - struct rtable *rt = NULL; - u_int32_t mtu = ~0U; - - if (family == PF_INET) { - struct flowi4 *fl4 = &fl.u.ip4; - memset(fl4, 0, sizeof(*fl4)); - fl4->daddr = ip_hdr(skb)->saddr; - } else { - struct flowi6 *fl6 = &fl.u.ip6; - - memset(fl6, 0, sizeof(*fl6)); - fl6->daddr = ipv6_hdr(skb)->saddr; - } - - nf_route(net, (struct dst_entry **)&rt, &fl, false, family); - if (rt != NULL) { - mtu = dst_mtu(&rt->dst); - dst_release(&rt->dst); - } - return mtu; -} - -static int -tcpmss_mangle_packet(struct sk_buff *skb, - const struct xt_action_param *par, - unsigned int family, - unsigned int tcphoff, - unsigned int minlen) -{ - const struct xt_tcpmss_info *info = par->targinfo; - struct tcphdr *tcph; - int len, tcp_hdrlen; - unsigned int i; - __be16 oldval; - u16 newmss; - u8 *opt; - - /* This is a fragment, no TCP header is available */ - if (par->fragoff != 0) - return 0; - - if (skb_ensure_writable(skb, skb->len)) - return -1; - - len = skb->len - tcphoff; - if (len < (int)sizeof(struct tcphdr)) - return -1; - - tcph = (struct tcphdr *)(skb_network_header(skb) + tcphoff); - tcp_hdrlen = tcph->doff * 4; - - if (len < tcp_hdrlen || tcp_hdrlen < sizeof(struct tcphdr)) - return -1; - - if (info->mss == XT_TCPMSS_CLAMP_PMTU) { - struct net *net = xt_net(par); - unsigned int in_mtu = tcpmss_reverse_mtu(net, skb, family); - unsigned int min_mtu = min(dst_mtu(skb_dst(skb)), in_mtu); - - if (min_mtu <= minlen) { - net_err_ratelimited("unknown or invalid path-MTU (%u)\n", - min_mtu); - return -1; - } - newmss = min_mtu - minlen; - } else - newmss = info->mss; - - opt = (u_int8_t *)tcph; - for (i = sizeof(struct tcphdr); i <= tcp_hdrlen - TCPOLEN_MSS; i += optlen(opt, i)) { - if (opt[i] == TCPOPT_MSS && opt[i+1] == TCPOLEN_MSS) { - u_int16_t oldmss; - - oldmss = (opt[i+2] << 8) | opt[i+3]; - - /* Never increase MSS, even when setting it, as - * doing so results in problems for hosts that rely - * on MSS being set correctly. - */ - if (oldmss <= newmss) - return 0; - - opt[i+2] = (newmss & 0xff00) >> 8; - opt[i+3] = newmss & 0x00ff; - - inet_proto_csum_replace2(&tcph->check, skb, - htons(oldmss), htons(newmss), - false); - return 0; - } - } - - /* There is data after the header so the option can't be added - * without moving it, and doing so may make the SYN packet - * itself too large. Accept the packet unmodified instead. - */ - if (len > tcp_hdrlen) - return 0; - - /* tcph->doff has 4 bits, do not wrap it to 0 */ - if (tcp_hdrlen >= 15 * 4) - return 0; - - /* - * MSS Option not found ?! add it.. - */ - if (skb_tailroom(skb) < TCPOLEN_MSS) { - if (pskb_expand_head(skb, 0, - TCPOLEN_MSS - skb_tailroom(skb), - GFP_ATOMIC)) - return -1; - tcph = (struct tcphdr *)(skb_network_header(skb) + tcphoff); - } - - skb_put(skb, TCPOLEN_MSS); - - /* - * IPv4: RFC 1122 states "If an MSS option is not received at - * connection setup, TCP MUST assume a default send MSS of 536". - * IPv6: RFC 2460 states IPv6 has a minimum MTU of 1280 and a minimum - * length IPv6 header of 60, ergo the default MSS value is 1220 - * Since no MSS was provided, we must use the default values - */ - if (xt_family(par) == NFPROTO_IPV4) - newmss = min(newmss, (u16)536); - else - newmss = min(newmss, (u16)1220); - - opt = (u_int8_t *)tcph + sizeof(struct tcphdr); - memmove(opt + TCPOLEN_MSS, opt, len - sizeof(struct tcphdr)); - - inet_proto_csum_replace2(&tcph->check, skb, - htons(len), htons(len + TCPOLEN_MSS), true); - opt[0] = TCPOPT_MSS; - opt[1] = TCPOLEN_MSS; - opt[2] = (newmss & 0xff00) >> 8; - opt[3] = newmss & 0x00ff; - - inet_proto_csum_replace4(&tcph->check, skb, 0, *((__be32 *)opt), false); - - oldval = ((__be16 *)tcph)[6]; - tcph->doff += TCPOLEN_MSS/4; - inet_proto_csum_replace2(&tcph->check, skb, - oldval, ((__be16 *)tcph)[6], false); - return TCPOLEN_MSS; -} - -static unsigned int -tcpmss_tg4(struct sk_buff *skb, const struct xt_action_param *par) -{ - struct iphdr *iph = ip_hdr(skb); - __be16 newlen; - int ret; - - ret = tcpmss_mangle_packet(skb, par, - PF_INET, - iph->ihl * 4, - sizeof(*iph) + sizeof(struct tcphdr)); - if (ret < 0) - return NF_DROP; - if (ret > 0) { - iph = ip_hdr(skb); - newlen = htons(ntohs(iph->tot_len) + ret); - csum_replace2(&iph->check, iph->tot_len, newlen); - iph->tot_len = newlen; - } - return XT_CONTINUE; -} - -#if IS_ENABLED(CONFIG_IP6_NF_IPTABLES) -static unsigned int -tcpmss_tg6(struct sk_buff *skb, const struct xt_action_param *par) -{ - struct ipv6hdr *ipv6h = ipv6_hdr(skb); - u8 nexthdr; - __be16 frag_off, oldlen, newlen; - int tcphoff; - int ret; - - nexthdr = ipv6h->nexthdr; - tcphoff = ipv6_skip_exthdr(skb, sizeof(*ipv6h), &nexthdr, &frag_off); - if (tcphoff < 0) - return NF_DROP; - ret = tcpmss_mangle_packet(skb, par, - PF_INET6, - tcphoff, - sizeof(*ipv6h) + sizeof(struct tcphdr)); - if (ret < 0) - return NF_DROP; - if (ret > 0) { - ipv6h = ipv6_hdr(skb); - oldlen = ipv6h->payload_len; - newlen = htons(ntohs(oldlen) + ret); - if (skb->ip_summed == CHECKSUM_COMPLETE) - skb->csum = csum_add(csum_sub(skb->csum, (__force __wsum)oldlen), - (__force __wsum)newlen); - ipv6h->payload_len = newlen; - } - return XT_CONTINUE; -} -#endif - -static int tcpmss_tg4_check_hooks(const struct xt_tgchk_param *par) -{ - const struct xt_tcpmss_info *info = par->targinfo; - - if (info->mss == XT_TCPMSS_CLAMP_PMTU && - (par->hook_mask & ~((1 << NF_INET_FORWARD) | - (1 << NF_INET_LOCAL_OUT) | - (1 << NF_INET_POST_ROUTING))) != 0) { - pr_info_ratelimited("path-MTU clamping only supported in FORWARD, OUTPUT and POSTROUTING hooks\n"); - return -EINVAL; - } - - return 0; -} - -/* Must specify -p tcp --syn */ -static inline bool find_syn_match(const struct xt_entry_match *m) -{ - const struct xt_tcp *tcpinfo = (const struct xt_tcp *)m->data; - - if (strcmp(m->u.kernel.match->name, "tcp") == 0 && - tcpinfo->flg_cmp & TCPHDR_SYN && - !(tcpinfo->invflags & XT_TCP_INV_FLAGS)) - return true; - - return false; -} - -static int tcpmss_tg4_check(const struct xt_tgchk_param *par) -{ - const struct ipt_entry *e = par->entryinfo; - const struct xt_entry_match *ematch; - - if (par->nft_compat) - return 0; - - xt_ematch_foreach(ematch, e) - if (find_syn_match(ematch)) - return 0; - pr_info_ratelimited("Only works on TCP SYN packets\n"); - return -EINVAL; -} - -#if IS_ENABLED(CONFIG_IP6_NF_IPTABLES) -static int tcpmss_tg6_check(const struct xt_tgchk_param *par) -{ - const struct ip6t_entry *e = par->entryinfo; - const struct xt_entry_match *ematch; - - if (par->nft_compat) - return 0; - - xt_ematch_foreach(ematch, e) - if (find_syn_match(ematch)) - return 0; - pr_info_ratelimited("Only works on TCP SYN packets\n"); - return -EINVAL; -} -#endif - -static struct xt_target tcpmss_tg_reg[] __read_mostly = { - { - .family = NFPROTO_IPV4, - .name = "TCPMSS", - .check_hooks = tcpmss_tg4_check_hooks, - .checkentry = tcpmss_tg4_check, - .target = tcpmss_tg4, - .targetsize = sizeof(struct xt_tcpmss_info), - .proto = IPPROTO_TCP, - .me = THIS_MODULE, - }, -#if IS_ENABLED(CONFIG_IP6_NF_IPTABLES) - { - .family = NFPROTO_IPV6, - .name = "TCPMSS", - .check_hooks = tcpmss_tg4_check_hooks, - .checkentry = tcpmss_tg6_check, - .target = tcpmss_tg6, - .targetsize = sizeof(struct xt_tcpmss_info), - .proto = IPPROTO_TCP, - .me = THIS_MODULE, - }, -#endif -}; - -static int __init tcpmss_tg_init(void) -{ - return xt_register_targets(tcpmss_tg_reg, ARRAY_SIZE(tcpmss_tg_reg)); -} - -static void __exit tcpmss_tg_exit(void) -{ - xt_unregister_targets(tcpmss_tg_reg, ARRAY_SIZE(tcpmss_tg_reg)); -} - -module_init(tcpmss_tg_init); -module_exit(tcpmss_tg_exit); diff --git a/script.sh b/script.sh new file mode 100755 index 0000000000000..8b876642b2342 --- /dev/null +++ b/script.sh @@ -0,0 +1,196 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: GPL-2.0 + +set -euo pipefail + +ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +OUT="${OUT:-$ROOT/.selftests-bpf}" +if [[ -z "${JOBS:-}" ]]; then + if command -v nproc >/dev/null 2>&1; then + JOBS="$(nproc)" + elif command -v sysctl >/dev/null 2>&1; then + JOBS="$(sysctl -n hw.ncpu)" + else + JOBS=1 + fi +fi +LIGHTSWITCH_BPF_SRC="${LIGHTSWITCH_BPF_SRC:-/Users/javierhonduco/src/lightswitch/bpf}" +if [[ ! -d "$LIGHTSWITCH_BPF_SRC" && -d /Users/javierhonduco/src/lightswitch/src/bpf ]]; then + LIGHTSWITCH_BPF_SRC=/Users/javierhonduco/src/lightswitch/src/bpf +fi +LIGHTSWITCH_BPF_DIR="${LIGHTSWITCH_BPF_DIR:-$ROOT/tools/testing/selftests/bpf/progs/lightswitch}" + +usage() { + cat <&2 <&2 + exit 1 +fi + +if [[ ! -d "$LIGHTSWITCH_BPF_SRC" ]]; then + echo "lightswitch BPF source directory not found: $LIGHTSWITCH_BPF_SRC" >&2 + exit 1 +fi + +cd "$ROOT" +mkdir -p "$OUT" + +elfutils_dev="$(nix build --no-link --print-out-paths nixpkgs#elfutils.dev)" +elfutils_out="$(nix build --no-link --print-out-paths nixpkgs#elfutils.out)" +openssl_dev="$(nix build --no-link --print-out-paths nixpkgs#openssl.dev)" +openssl_out="$(nix build --no-link --print-out-paths nixpkgs#openssl.out)" +zlib_dev="$(nix build --no-link --print-out-paths nixpkgs#zlib.dev)" +zlib_out="$(nix build --no-link --print-out-paths nixpkgs#zlib.out)" + +extra_cflags="${EXTRA_CFLAGS:-} -I$elfutils_dev/include -I$openssl_dev/include -I$zlib_dev/include" +extra_ldflags="${EXTRA_LDFLAGS:-} -L$elfutils_out/lib -L$openssl_out/lib -L$zlib_out/lib" +export PKG_CONFIG_PATH="$openssl_dev/lib/pkgconfig:$elfutils_dev/lib/pkgconfig:$zlib_dev/share/pkgconfig:${PKG_CONFIG_PATH:-}" +export CPATH="$elfutils_dev/include:$openssl_dev/include:$zlib_dev/include:${CPATH:-}" +export C_INCLUDE_PATH="$elfutils_dev/include:$openssl_dev/include:$zlib_dev/include:${C_INCLUDE_PATH:-}" +export LIBRARY_PATH="$elfutils_out/lib:$openssl_out/lib:$zlib_out/lib:${LIBRARY_PATH:-}" +export LD_LIBRARY_PATH="$elfutils_out/lib:$openssl_out/lib:$zlib_out/lib:${LD_LIBRARY_PATH:-}" + +make_bpf_selftests() { + make -C tools/testing/selftests/bpf \ + OUTPUT="$OUT" \ + LLVM=1 \ + LIGHTSWITCH_BPF_DIR="$LIGHTSWITCH_BPF_DIR" \ + EXTRA_CFLAGS="$extra_cflags" \ + HOST_EXTRACFLAGS="$extra_cflags" \ + EXTRA_LDFLAGS="$extra_ldflags" \ + -j"$JOBS" \ + "$@" +} + +case "${target_args[0]}" in + -h|--help|help) + usage + ;; + smoke) + make_bpf_selftests \ + "$OUT/veristat" \ + "$OUT/lightswitch_bpf/profiler.bpf.o" \ + "$OUT/lightswitch_bpf/tracers.bpf.o" + ( + cd "$OUT" + "$ROOT/tools/testing/selftests/bpf/test_lightswitch_bpf_build.sh" + ) + ;; + full) + make_bpf_selftests run_tests + ;; + vm) + vm_args=("${target_args[@]:1}") + if [[ ${#vm_args[@]} -eq 0 ]]; then + vm_args=(-- env BPF_STRICT_BUILD=0 SKIP_DOCS=1 make run_tests) + fi + exec env \ + OUTPUT_DIR="$OUT/vm" \ + LIGHTSWITCH_BPF_DIR="$LIGHTSWITCH_BPF_DIR" \ + BPF_STRICT_BUILD="${BPF_STRICT_BUILD:-0}" \ + SKIP_DOCS="${SKIP_DOCS:-1}" \ + EXTRA_CFLAGS="$extra_cflags" \ + HOSTCFLAGS="$extra_cflags" \ + HOST_EXTRACFLAGS="$extra_cflags" \ + EXTRA_LDFLAGS="$extra_ldflags" \ + "$ROOT/tools/testing/selftests/bpf/vmtest.sh" \ + -d "$OUT/vm" \ + -j "$JOBS" \ + "${vm_args[@]}" + ;; + *) + make_bpf_selftests "${target_args[@]}" + ;; +esac diff --git a/tools/memory-model/litmus-tests/Z6.0+pooncelock+poonceLock+pombonce.litmus b/tools/memory-model/litmus-tests/Z6.0+pooncelock+poonceLock+pombonce.litmus deleted file mode 100644 index 415248fb66990..0000000000000 --- a/tools/memory-model/litmus-tests/Z6.0+pooncelock+poonceLock+pombonce.litmus +++ /dev/null @@ -1,42 +0,0 @@ -C Z6.0+pooncelock+poonceLock+pombonce - -(* - * Result: Never - * - * This litmus test demonstrates how smp_mb__after_spinlock() may be - * used to ensure that accesses in different critical sections for a - * given lock running on different CPUs are nevertheless seen in order - * by CPUs not holding that lock. - *) - -{} - -P0(int *x, int *y, spinlock_t *mylock) -{ - spin_lock(mylock); - WRITE_ONCE(*x, 1); - WRITE_ONCE(*y, 1); - spin_unlock(mylock); -} - -P1(int *y, int *z, spinlock_t *mylock) -{ - int r0; - - spin_lock(mylock); - smp_mb__after_spinlock(); - r0 = READ_ONCE(*y); - WRITE_ONCE(*z, 1); - spin_unlock(mylock); -} - -P2(int *x, int *z) -{ - int r1; - - WRITE_ONCE(*z, 2); - smp_mb(); - r1 = READ_ONCE(*x); -} - -exists (1:r0=1 /\ z=2 /\ 2:r1=0) diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index b642ee489ea64..a1121530ae308 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -123,6 +123,7 @@ TEST_FILES = xsk_prereqs.sh $(wildcard progs/btf_dump_test_case_*.c) # Order correspond to 'make run_tests' order TEST_PROGS := test_kmod.sh \ + test_lightswitch_bpf_build.sh \ test_lirc_mode2.sh \ test_bpftool_build.sh \ test_doc_build.sh \ @@ -150,6 +151,11 @@ TEST_GEN_PROGS_EXTENDED = \ TEST_GEN_FILES += $(TEST_KMODS) liburandom_read.so urandom_read sign-file uprobe_multi +LIGHTSWITCH_BPF_DIR ?= $(CURDIR)/progs/lightswitch +LIGHTSWITCH_BPF_SRCS := $(wildcard $(LIGHTSWITCH_BPF_DIR)/*.bpf.c) +LIGHTSWITCH_BPF_OBJS := $(patsubst %.bpf.c,lightswitch_bpf/%.bpf.o,$(notdir $(LIGHTSWITCH_BPF_SRCS))) +TEST_GEN_FILES += $(LIGHTSWITCH_BPF_OBJS) + ifneq ($(V),1) submake_extras := feature_display=0 endif @@ -495,6 +501,18 @@ define CLANG_BPF_BUILD_RULE $(Q)$(CLANG) $3 -O2 $(BPF_TARGET_ENDIAN) -c $1 -mcpu=v3 -o $2 $(if $(PERMISSIVE),|| \ ($(RM) $2; printf ' %-12s %s\n' 'SKIP-BPF' '$(notdir $2)' 1>&2)) endef + +$(OUTPUT)/lightswitch_bpf: + $(call msg,MKDIR,,$@) + $(Q)mkdir -p $@ + +$(OUTPUT)/lightswitch_bpf/%.bpf.o: $(LIGHTSWITCH_BPF_DIR)/%.bpf.c \ + $(LIGHTSWITCH_BPF_DIR)/*.h \ + $(INCLUDE_DIR)/vmlinux.h \ + $(BPFOBJ) | $(OUTPUT)/lightswitch_bpf + $(call msg,CLNG-BPF,lightswitch_bpf,$@) + $(Q)$(CLANG) $(BPF_CFLAGS) $(CLANG_CFLAGS) -Wno-unused-function -I$(LIGHTSWITCH_BPF_DIR) -O2 $(BPF_TARGET_ENDIAN) -c $< -mcpu=v3 -o $@ + # Similar to CLANG_BPF_BUILD_RULE, but with disabled alu32 define CLANG_NOALU32_BPF_BUILD_RULE $(call msg,CLNG-BPF,$4,$2) diff --git a/tools/testing/selftests/bpf/progs/lightswitch/basic_types.h b/tools/testing/selftests/bpf/progs/lightswitch/basic_types.h new file mode 100644 index 0000000000000..65271be666e98 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/lightswitch/basic_types.h @@ -0,0 +1,19 @@ +typedef signed char __s8; +typedef unsigned char __u8; +typedef short int __s16; +typedef short unsigned int __u16; +typedef int __s32; +typedef unsigned int __u32; +typedef long long int __s64; +typedef long long unsigned int __u64; + +typedef __s8 s8; +typedef __u8 u8; +typedef __s16 s16; +typedef __u16 u16; +typedef __s32 s32; +typedef __u32 u32; +typedef __s64 s64; +typedef __u64 u64; + +#include diff --git a/tools/testing/selftests/bpf/progs/lightswitch/constants.h b/tools/testing/selftests/bpf/progs/lightswitch/constants.h new file mode 100644 index 0000000000000..297f0e0a1b20a --- /dev/null +++ b/tools/testing/selftests/bpf/progs/lightswitch/constants.h @@ -0,0 +1,20 @@ +#ifndef __LIGHTSWITCH_LINUX_PAGE_CONSTANTS__ +#define __LIGHTSWITCH_LINUX_PAGE_CONSTANTS__ + +#define PAGE_SIZE (1UL << 12) // 4KB +#define PAGE_MASK ~(PAGE_SIZE - 1) + +// Values for x86_64 as of 6.0.18-200. +#define TOP_OF_KERNEL_STACK_PADDING 0 +#define THREAD_SIZE_ORDER 2 +#define THREAD_SIZE (PAGE_SIZE << THREAD_SIZE_ORDER) + +#endif + +#ifndef __LIGHTSWITCH_ERROR_CONSTANTS__ +#define __LIGHTSWITCH_ERROR_CONSTANTS__ + +#define EFAULT 14 +#define EEXIST 17 + +#endif diff --git a/tools/testing/selftests/bpf/progs/lightswitch/profiler.bpf.c b/tools/testing/selftests/bpf/progs/lightswitch/profiler.bpf.c new file mode 100644 index 0000000000000..2fa2108596dfd --- /dev/null +++ b/tools/testing/selftests/bpf/progs/lightswitch/profiler.bpf.c @@ -0,0 +1,679 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright 2022 The Parca Authors +// Copyright 2024 The Lightswitch Authors + +#include "vmlinux.h" +#include "profiler.h" +#include "shared_maps.h" +#include "shared_helpers.h" + +#include +#include +#include +#include + +struct { + __uint(type, BPF_MAP_TYPE_RINGBUF); + __uint(max_entries, 256 * 1024 /* 256 KB */); +} stacks_rb SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); + __uint(key_size, sizeof(u32)); + __uint(value_size, sizeof(u32)); +} stacks SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, 1); + __type(key, u32); + __type(value, unwind_state_t); +} heap SEC(".maps"); + +struct unwind_table_inner_map { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, UNWIND_INFO_PAGE_SIZE); + __type(key, u32); + __type(value, stack_unwind_row_t); +}; + +// Holds BPF array maps which store unwind information. + +struct { + __uint(type, BPF_MAP_TYPE_HASH_OF_MAPS); + __uint(max_entries, MAX_OUTER_UNWIND_MAP_ENTRIES); + __type(key, u64); + __type(value, u32); + __array(values, struct unwind_table_inner_map); +} outer_map SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 5); + __type(key, u32); + __type(value, u32); +} programs SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); + __uint(key_size, sizeof(u32)); + __uint(value_size, sizeof(u32)); + __uint(max_entries, 0); +} events SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_RINGBUF); + __uint(max_entries, 256 * 1024 /* 256 KB */); +} events_rb SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, MAX_EXECUTABLE_TO_PAGE_ENTRIES); + __type(key, page_key_t); + __type(value, page_value_t); +} executable_to_page SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, MAX_PROCESSES); + __type(key, u64); + __type(value, bool); +} rate_limits SEC(".maps"); + +// Binary search the unwind table to find the row index containing the unwind +// information for a given program counter (pc) relative to the object file. +static __always_inline u64 find_offset_for_pc(void *inner_map, u16 pc_low, u64 left, + u64 right) { + u64 found = BINARY_SEARCH_DEFAULT; + +// On kernels ~6.8 and greater the verifier fails with argument list too long. This did not use to +// happen before and it's probably due to a regression in the way the verifier accounts for the explored +// paths. I have tried many other things, such as two mid variables but that did not do it. +// Perhaps unrolling the loop works is that the verifier doesn't have as many states to explore +// per iteration. +#pragma unroll + for (int i = 0; i < MAX_BINARY_SEARCH_DEPTH; i++) { + // TODO(javierhonduco): ensure that this condition is right as we use + // unsigned values... + if (left >= right) { + LOG("\t.done"); + return found; + } + + u32 mid = (left + right) / 2; + + stack_unwind_row_t *row = bpf_map_lookup_elem(inner_map, &mid); + if (row == NULL) { + return BINARY_SEARCH_DEFAULT; + } + + if (row->pc_low <= pc_low) { + found = mid; + left = mid + 1; + } else { + right = mid; + } + } + return BINARY_SEARCH_EXHAUSTED_ITERATIONS; +} + +// Finds the shard information for a given pid and program counter. Optionally, +// and offset can be passed that will be filled in with the mapping's load +// address. +static __always_inline void * +find_page(mapping_t *mapping, u64 object_relative_pc, u64 *low_index, u64 *high_index) { + page_key_t page_key = { + .executable_id = mapping->executable_id, + .file_offset = object_relative_pc, + }; + + page_value_t *found_page = bpf_map_lookup_elem(&executable_to_page, &page_key); + + if (found_page != NULL) { + void *inner_map = bpf_map_lookup_elem(&outer_map, &mapping->executable_id); + if (inner_map != NULL) { + *low_index = found_page->low_index; + *high_index = found_page->high_index; + return inner_map; + } + } + + LOG("[error] could not find page for executable_id: %llx at file_offset: %llx", mapping->executable_id, object_relative_pc); + bump_unwind_error_page_not_found(); + return NULL; +} + +static __always_inline void send_event(Event *event, struct bpf_perf_event_data *ctx) { + u64 event_id = ((u64)event->pid << 32 | event->type); + bool *is_rate_limited = bpf_map_lookup_elem(&rate_limits, &event_id); + if (is_rate_limited != NULL && *is_rate_limited) { + LOG("[debug] send_event was rate limited for process %d", event->pid); + return; + } + + int ret = 0; + if (lightswitch_config.use_ring_buffers) { + ret = bpf_ringbuf_output(&events_rb, event, sizeof(Event), 0); + } else { + ret = bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, event, sizeof(Event)); + } + if (ret < 0) { + LOG("[error] failed to send event for process %d", event->pid); + + if (event->type == EVENT_NEW_PROCESS) { + bump_unwind_error_sending_new_process_event(); + } else if (event->type == EVENT_NEED_UNWIND_INFO) { + bump_unwind_error_sending_need_unwind_info_event(); + } + } + + LOG("[debug] event type %d sent for process %d", event->type, event->pid); + bool rate_limited = true; + + bpf_map_update_elem(&rate_limits, &event_id, &rate_limited, BPF_ANY); +} + +// The return address points as the the instruction at which execution +// will resume after returning from a function call, we need to get the +// previous instruction's address. +static __always_inline u64 previous_instruction_addr(u64 addr) { +#ifdef __TARGET_ARCH_x86 + // On x86 it's not possible to find the previous instruction address + // without fully disassembling the whole executable from the start. + // By substracting 1 byte, we make sure to fall within the previous + // instruction. + return addr - 1; +#elif __TARGET_ARCH_arm64 + return addr - 4; +#endif +} + +#ifdef __TARGET_ARCH_x86 +static __always_inline u64 remove_pac(u64 addr) { + return addr; +} +#elif __TARGET_ARCH_arm64 +// Arm64 supports pointer authentication, we need to remove the signatured during +// unwinding. +static __always_inline u64 remove_pac(u64 addr) { + // The signature is stored in the top 55 - virtual address size bits [0], which + // is typically 48 bytes, hence we need to clear the top 7 bits. Clearing 8 bits + // as they are all the non-addressable anyways. + // - [0]: https://docs.kernel.org/arch/arm64/pointer-authentication.html#basic-support + addr &= 0x0000FFFFFFFFFFFF; + return addr; +} +#endif + +// Kernel addresses have the top bits set. +static __always_inline bool in_kernel(u64 ip) { return ip & (1UL << 63); } + +// kthreads mm's is not set. +// +// We don't check for the return value of `retrieve_task_registers`, it's +// caller due the verifier not liking that code. +static __always_inline bool is_kthread() { + struct task_struct *task = current_task(); + if (task == NULL) { + return false; + } + + void *mm; + int err = bpf_probe_read_kernel(&mm, 8, &task->mm); + if (err) { + LOG("[warn] bpf_probe_read_kernel failed with %d", err); + return false; + } + + return mm == NULL; +} + +// avoid R0 invalid mem access 'scalar' +// Port of `task_pt_regs` in BPF. +static __always_inline bool retrieve_task_registers(u64 *ip, u64 *sp, u64 *bp, u64 *lr) { + if (ip == NULL || sp == NULL || bp == NULL || lr == NULL) { + return false; + } + + if (is_kthread()) { + return false; + } + + struct task_struct *task = current_task(); + struct pt_regs *regs = pt_regs(task); + if (regs == NULL) { + return false; + } + + *ip = PT_REGS_IP_CORE(regs); + *sp = PT_REGS_SP_CORE(regs); + *bp = PT_REGS_FP_CORE(regs); +#ifdef __TARGET_ARCH_arm64 + *lr = PT_REGS_RET_CORE(regs); +#endif + return true; +} + +static __always_inline void add_stack(struct bpf_perf_event_data *ctx, + unwind_state_t *unwind_state) { + // Unwind and copy kernel stack. + u32 ulen = unwind_state->sample.stack.ulen; + if (ulen < MAX_STACK_DEPTH) { + int ret = bpf_get_stack(ctx, &unwind_state->sample.stack.addresses[ulen], MAX_STACK_DEPTH * sizeof(u64), 0); + if (ret > 0) { + unwind_state->sample.stack.klen = ret / sizeof(u64); + } + } + + struct task_struct *task = current_task(); + unsigned int level = lightswitch_config.userspace_pid_ns_level; + int per_process_id = BPF_CORE_READ(task, group_leader, thread_pid, numbers[level].nr); + int per_thread_id = BPF_CORE_READ(task, thread_pid, numbers[level].nr); + + unwind_state->sample.pid = per_process_id; + unwind_state->sample.tid = per_thread_id; + unwind_state->sample.collected_at = bpf_ktime_get_boot_ns(); + + u32 sample_size = sizeof(sample_t) + // Remove the actual stack buffer which was doubled to appease the verifier. + - 2 * MAX_STACK_DEPTH * sizeof(u64) + // Add the actual stack size in bytes. + + (unwind_state->sample.stack.ulen + unwind_state->sample.stack.klen) * sizeof(u64); + + // Appease the verifier. + if (sample_size > sizeof(sample_t)) { + return; + } + + int ret = 0; + if (lightswitch_config.use_ring_buffers) { + ret = bpf_ringbuf_output(&stacks_rb, &(unwind_state->sample), sample_size, 0); + } else { + ret = bpf_perf_event_output(ctx, &stacks, BPF_F_CURRENT_CPU, &(unwind_state->sample), sample_size); + } + + if (ret < 0) { + bpf_printk( + "add_stack failed ret=%d, use_ring_buffers=%d", + ret, + lightswitch_config.use_ring_buffers); + bump_unwind_error_failure_sending_stack(); + } +} + +// The unwinding machinery lives here. +SEC("perf_event") +int dwarf_unwind(struct bpf_perf_event_data *ctx) { + struct task_struct *task = current_task(); + unsigned int level = lightswitch_config.userspace_pid_ns_level; + int per_process_id = BPF_CORE_READ(task, group_leader, thread_pid, numbers[level].nr); + + bool reached_bottom_of_stack = false; + u64 zero = 0; + + unwind_state_t *unwind_state = bpf_map_lookup_elem(&heap, &zero); + if (unwind_state == NULL) { + LOG("unwind_state is NULL, should not happen"); + return 1; + } + + for (int i = 0; i < MAX_STACK_DEPTH_PER_PROGRAM; i++) { + // LOG("[debug] Within unwinding machinery loop"); + LOG("## frame: %d", unwind_state->sample.stack.ulen); + LOG("\tcurrent pc: %llx", unwind_state->ip); + LOG("\tcurrent sp: %llx", unwind_state->sp); + LOG("\tcurrent bp: %llx", unwind_state->bp); + + mapping_t *mapping = find_mapping(per_process_id, unwind_state->ip); + + if (mapping == NULL) { + LOG("[error] no mapping found for pc %llx", unwind_state->ip); + bump_unwind_error_mapping_not_found(); + return 1; + } + + if (unwind_state->ip < mapping->begin || unwind_state->ip >= mapping->end) { + LOG("[error] pc %llx not contained within begin: %llx end: %llx", unwind_state->ip, mapping->begin, mapping->end); + bump_unwind_error_mapping_does_not_contain_pc(); + return 1; + } + + if (mapping->type == MAPPING_TYPE_ANON) { + LOG("JIT section, stopping"); + bump_unwind_jit_encountered(); + return 1; + } + + if (mapping->type == MAPPING_TYPE_VDSO) { + LOG("vDSO section"); + bump_unwind_vdso_encountered(); + } + + u64 object_relative_pc = unwind_state->ip - mapping->load_address; + u64 object_relative_pc_high = HIGH_PC(object_relative_pc); + u16 object_relative_pc_low = LOW_PC(object_relative_pc); + + u64 low_index = 0; + u64 high_index = 0; + void *inner = find_page(mapping, object_relative_pc_high, &low_index, &high_index); + if (inner == NULL) { + Event event = { + .type = EVENT_NEED_UNWIND_INFO, + .pid = per_process_id, + // Assume 4KB pages, hence the offset within the page does not offer any + // additional information to find the right memory mapping. This way the + // rate limiting logic will work better due to the reduced cardinality of + // the rate limiting key. + .address = unwind_state->ip & PAGE_MASK, + }; + send_event(&event, ctx); + return 1; + } + + u64 table_idx = find_offset_for_pc(inner, object_relative_pc_low, low_index, high_index); + + if (table_idx == BINARY_SEARCH_DEFAULT || + table_idx == BINARY_SEARCH_SHOULD_NEVER_HAPPEN || + table_idx == BINARY_SEARCH_EXHAUSTED_ITERATIONS) { + bool in_previous_page = false; + + if (table_idx == BINARY_SEARCH_DEFAULT) { + low_index -= 1; + stack_unwind_row_t *previous_row = bpf_map_lookup_elem(inner, &low_index); + if (previous_row != NULL && object_relative_pc > PREVIOUS_PAGE(object_relative_pc_high) + previous_row->pc_low) { + table_idx = low_index; + in_previous_page = true; + } + } + + if (!in_previous_page) { + LOG("[error] binary search failed with %llx, pc: %llx", table_idx, unwind_state->ip); + if (table_idx == BINARY_SEARCH_EXHAUSTED_ITERATIONS) { + bump_unwind_error_binary_search_exhausted_iterations(); + } + return 1; + } + } + + LOG("\t=> table_index: %d", table_idx); + LOG("\t=> object relative pc: %llx", object_relative_pc); + + stack_unwind_row_t *row = bpf_map_lookup_elem(inner, &table_idx); + if (row == NULL) { + return 1; + } + + u64 found_pc = object_relative_pc_high + row->pc_low; + u8 found_cfa_type = row->cfa_type; + u8 found_rbp_type = row->rbp_type; + s16 found_cfa_offset = row->cfa_offset; + s16 found_rbp_offset = row->rbp_offset; + LOG("\tcfa type: %d, offset: %d (row pc: %llx)", found_cfa_type, + found_cfa_offset, found_pc); + + if (found_cfa_type == CFA_TYPE_OFFSET_DID_NOT_FIT) { + bump_unwind_error_cfa_offset_did_not_fit(); + return 1; + } + + if (found_cfa_type == CFA_TYPE_END_OF_FDE_MARKER) { + LOG("[info] pc %llx not contained in the unwind info, found marker", + unwind_state->ip); + reached_bottom_of_stack = true; + break; + } + + if (found_rbp_type == RBP_TYPE_OFFSET_DID_NOT_FIT) { + bump_unwind_error_rbp_offset_did_not_fit(); + return 1; + } + + if (found_rbp_type == RBP_TYPE_UNDEFINED_RETURN_ADDRESS) { + LOG("[info] null return address, end of stack", unwind_state->ip); + reached_bottom_of_stack = true; + break; + } + + // Add address to stack. + u32 ulen = unwind_state->sample.stack.ulen; + // Appease the verifier. + if (ulen < MAX_STACK_DEPTH) { + unwind_state->sample.stack.addresses[ulen] = unwind_state->ip; + unwind_state->sample.stack.ulen++; + } + + if (found_rbp_type == RBP_TYPE_REGISTER || + found_rbp_type == RBP_TYPE_EXPRESSION) { + LOG("\t[error] frame pointer is %d (register or exp), bailing out", + found_rbp_type); + bump_unwind_error_unsupported_frame_pointer_action(); + return 1; + } + + u64 previous_rsp = 0; + if (found_cfa_type == CFA_TYPE_RBP) { + previous_rsp = unwind_state->bp + found_cfa_offset; + } else if (found_cfa_type == CFA_TYPE_RSP) { + previous_rsp = unwind_state->sp + found_cfa_offset; + } else if (found_cfa_type == CFA_TYPE_DEREF_AND_ADD) { + u8 offset = found_cfa_offset >> 8; + u8 addition = found_cfa_offset; + LOG("dwarf exp: *($rsp + %d) + %d", offset, addition); + int ret = + bpf_probe_read_user(&previous_rsp, 8, (void *)(unwind_state->sp + offset)); + if (ret < 0) { + LOG("[error] reading previous rsp failed with %d", ret); + bump_unwind_error_previous_rsp_read(); + } + previous_rsp += addition; + } else if (found_cfa_type == CFA_TYPE_CFA_TYPE_UNSUP_EXP) { + bump_unwind_error_unsupported_expression(); + return 1; + } else if (found_cfa_type == CFA_TYPE_PLT1 || found_cfa_type == CFA_TYPE_PLT2) { + LOG("CFA expression found with id %d", found_cfa_offset); + u64 threshold = found_cfa_type == CFA_TYPE_PLT1 ? 11 : 10; + + if (threshold == 0) { + bump_unwind_error_should_never_happen(); + return 1; + } + previous_rsp = unwind_state->sp + 8 + + ((((unwind_state->ip & 15) >= threshold)) << 3); + } else { + LOG("\t[unsup] cfa type %d not valid at ip: %llx", found_cfa_type, object_relative_pc); + bump_unwind_error_unsupported_cfa_register(); + return 1; + } + + // TODO(javierhonduco): A possible check could be to see whether this value + // is within the stack. This check could be quite brittle though, so if we + // add it, it would be best to add it only during development. + if (previous_rsp == 0) { + LOG("[error] previous_rsp should not be zero."); + bump_unwind_error_previous_rsp_zero(); + return 1; + } + + // Set rbp register. + u64 previous_rbp = 0; + u64 previous_rbp_addr = previous_rsp + found_rbp_offset; + + if (found_rbp_type == RBP_TYPE_UNCHANGED) { + previous_rbp = unwind_state->bp; + } else { + LOG("\t(bp_offset: %d, bp value stored at %llx)", found_rbp_offset, + previous_rbp_addr); + int ret = + bpf_probe_read_user(&previous_rbp, 8, (void *)(previous_rbp_addr)); + if (ret < 0) { + LOG("[error] previous_rbp read failed with %d", ret); + bump_unwind_error_previous_rbp_read(); + return 1; + } + } + + u64 previous_rip = 0; + u64 previous_rip_addr = 0; + int err = 0; + +#ifdef __TARGET_ARCH_x86 + // The return address is guaranteed to be 8 bytes ahead of + // the previous stack pointer in x86_64. + previous_rip_addr = previous_rsp - 8; + err = bpf_probe_read_user(&previous_rip, 8, (void *)(previous_rip_addr)); + +#endif + +#ifdef __TARGET_ARCH_arm64 + // Special handling for leaf frame. Binaries with .eh_frame will have + // a explicit usage of the LR register, while syntethised binaries do + // not, such as Golang and Arm64 vDSO. + if (found_rbp_type == RBP_TYPE_ARM64_RETURN_ADDRESS_LR) { + previous_rip = unwind_state->lr; + } else if (found_rbp_type == RBP_TYPE_ARM64_RETURN_ADDRESS_FRAME) { + // The binary was compiled with frame pointers, the location of the return + // address is guaranteed by the Aarch64 ABI. + previous_rip_addr = previous_rbp_addr + 8; + err = bpf_probe_read_user(&previous_rip, 8, (void *)(previous_rip_addr)); + } else { + // If there are not frame pointers, we need to fetch the return address somewhere + // else in the stack. + previous_rip_addr = previous_rsp + found_rbp_offset; + err = bpf_probe_read_user(&previous_rip, 8, (void *)(previous_rip_addr)); + } +#endif + + if (err < 0) { + LOG("[error] previous_rip read failed, ret=%d @ %llx", + err, previous_rip_addr); + bump_unwind_error_previous_rip_read(); + return 1; + } + + if (previous_rip == 0) { + LOG("[error] previous_rip is zero, ret=%d @ %llx", err, previous_rip_addr); + bump_unwind_error_previous_rip_zero(); + return 1; + } + + LOG("\tprevious ip: %llx (@ %llx)", previous_rip, previous_rip_addr); + LOG("\tprevious sp: %llx", previous_rsp); + // Set rsp and rip registers + unwind_state->ip = previous_instruction_addr(remove_pac(previous_rip)); + unwind_state->sp = previous_rsp; + // Set rbp + LOG("\tprevious bp: %llx", previous_rbp); + unwind_state->bp = previous_rbp; + + // Frame finished! :) + } + + if (reached_bottom_of_stack) { +#ifdef __TARGET_ARCH_x86 + // We've reached the bottom of the stack once we don't find an unwind + // entry for the given program counter and the current frame pointer + // is 0. As per the x86_64 ABI: + // + // From 3.4.1 Initial Stack and Register State + // > %rbp The content of this register is unspecified at process + // > initialization time, but the user code should mark the deepest + // > stack frame by setting the frame pointer to zero. + // + // Note: the initial register state only applies to processes not to threads. + // + // https://refspecs.linuxbase.org/elf/x86_64-abi-0.99.pdf + int per_thread_id = BPF_CORE_READ(task, thread_pid, numbers[level].nr); + bool main_thread = per_process_id == per_thread_id; + if (main_thread && unwind_state->bp != 0) { + LOG("[error] Expected rbp to be 0 on main thread but found %llx, pc: %llx", unwind_state->bp, unwind_state->ip); + bump_unwind_bp_non_zero_for_bottom_frame(); + } +#endif + LOG("======= reached bottom frame! ======="); + add_stack(ctx, unwind_state); + bump_unwind_success_dwarf(); + return 0; + + } else if (unwind_state->sample.stack.ulen < MAX_STACK_DEPTH && + unwind_state->tail_calls < MAX_TAIL_CALLS) { + LOG("Continuing walking the stack in a tail call, current tail %d", + unwind_state->tail_calls); + unwind_state->tail_calls++; + bpf_tail_call(ctx, &programs, PROGRAM_NATIVE_UNWINDER); + } + + // We couldn't get the whole stacktrace. + LOG("Truncated stack, won't be sent"); + bump_unwind_error_truncated(); + return 0; +} + +// Set up the initial unwinding state. +static __always_inline bool set_initial_state(unwind_state_t *unwind_state, bpf_user_pt_regs_t *regs) { + unwind_state->sample.stack.ulen = 0; + unwind_state->sample.stack.klen = 0; + unwind_state->tail_calls = 0; + + unwind_state->sample.pid = 0; + unwind_state->sample.tid = 0; + unwind_state->sample.collected_at = 0; + + if (in_kernel(PT_REGS_IP(regs))) { + if (!retrieve_task_registers(&unwind_state->ip, &unwind_state->sp, &unwind_state->bp, &unwind_state->lr)) { + // in kernelspace, but failed, probs a kworker + // todo: bump counter + return false; + } + } else { + // Currently executing userspace code. + unwind_state->ip = PT_REGS_IP(regs); + unwind_state->sp = PT_REGS_SP(regs); + unwind_state->bp = PT_REGS_FP(regs); + unwind_state->lr = remove_pac(PT_REGS_RET(regs)); + } + + return true; +} + +SEC("perf_event") +int on_event(struct bpf_perf_event_data *ctx) { + struct task_struct *task = current_task(); + unsigned int level = lightswitch_config.userspace_pid_ns_level; + int per_process_id = BPF_CORE_READ(task, group_leader, thread_pid, numbers[level].nr); + + // There's no point in checking for the swapper process. + if (per_process_id == 0) { + return 0; + } + + // Discard kworkers. + if (is_kthread()) { + return 0; + } + + if (process_is_known(per_process_id)) { + bump_unwind_total(); + + u32 zero = 0; + unwind_state_t *profiler_state = bpf_map_lookup_elem(&heap, &zero); + if (profiler_state == NULL) { + LOG("[error] profiler state should never be NULL"); + return 0; + } + set_initial_state(profiler_state, &ctx->regs); + + bpf_tail_call(ctx, &programs, PROGRAM_NATIVE_UNWINDER); + return 0; + } + + Event event = { + .type = EVENT_NEW_PROCESS, + .pid = per_process_id, + }; + // Send the main thread's name. + BPF_CORE_READ_STR_INTO(&event.comm, task, group_leader, comm); + send_event(&event, ctx); + return 0; +} + +char LICENSE[] SEC("license") = "Dual MIT/GPL"; diff --git a/tools/testing/selftests/bpf/progs/lightswitch/profiler.h b/tools/testing/selftests/bpf/progs/lightswitch/profiler.h new file mode 100644 index 0000000000000..28d9dcb192bcc --- /dev/null +++ b/tools/testing/selftests/bpf/progs/lightswitch/profiler.h @@ -0,0 +1,201 @@ +#include "basic_types.h" + +// Number of frames to walk per tail call iteration. +#define MAX_STACK_DEPTH_PER_PROGRAM 7 +// Number of BPF tail calls that will be attempted. +#define MAX_TAIL_CALLS 30 +// Maximum number of frames. +#define MAX_STACK_DEPTH 200 +_Static_assert(MAX_TAIL_CALLS *MAX_STACK_DEPTH_PER_PROGRAM >= MAX_STACK_DEPTH, + "enough iterations to traverse the whole stack"); +// Number of unique stacks. +#define MAX_STACK_TRACES_ENTRIES 64000 +// Number of items in the stack counts aggregation map. +#define MAX_STACK_COUNTS_ENTRIES 10240 +// Maximum number of processes we are willing to track. +#define MAX_PROCESSES 5000 +// Maximum number of memory mappings entries we can store in the LPM trie. This is shared across all +// the processes and assumes an average of 200 entries per process. These are LPM entries that in most +// cases will be higher than the number of mappings. +#define MAX_MAPPINGS MAX_PROCESSES * 200 +// Binary search iterations to find unwind information. +#define MAX_BINARY_SEARCH_DEPTH 17 +// Number of entries in the 'outer' unwind map. +#define MAX_OUTER_UNWIND_MAP_ENTRIES 3000 + +#define UNWIND_INFO_PAGE_BIT_LEN 16 +#define UNWIND_INFO_PAGE_SIZE (1 << UNWIND_INFO_PAGE_BIT_LEN) +#define PREVIOUS_PAGE(address) (address - UNWIND_INFO_PAGE_SIZE) +_Static_assert(MAX_BINARY_SEARCH_DEPTH >= UNWIND_INFO_PAGE_BIT_LEN, "unwind table is big enough"); +#define HIGH_PC_MASK 0x0000FFFFFFFF0000LLU +#define LOW_PC_MASK 0x000000000000FFFFLLU +#define HIGH_PC(addr) (addr & HIGH_PC_MASK) +#define LOW_PC(addr) (addr & LOW_PC_MASK) + +#define MAX_EXECUTABLE_TO_PAGE_ENTRIES 500 * 1000 + +#define MAPPING_TYPE_FILE 0 +#define MAPPING_TYPE_ANON 1 +#define MAPPING_TYPE_VDSO 2 + +typedef struct { + u64 executable_id; + u64 file_offset; +} page_key_t; + +typedef struct { + u32 low_index; + u32 high_index; +} page_value_t; + +// Values for the unwind table's CFA type. +#define CFA_TYPE_RBP 1 +#define CFA_TYPE_RSP 2 +#define CFA_TYPE_CFA_TYPE_UNSUP_EXP 3 +#define CFA_TYPE_PLT1 4 +#define CFA_TYPE_PLT2 5 +#define CFA_TYPE_DEREF_AND_ADD 6 +#define CFA_TYPE_END_OF_FDE_MARKER 7 +#define CFA_TYPE_UNSUP_REGISTER_OFFSET 8 // not used in the unwinder yet. +#define CFA_TYPE_OFFSET_DID_NOT_FIT 9 + +// Values for the unwind table's frame pointer type. +#define RBP_TYPE_UNCHANGED 0 +#define RBP_TYPE_OFFSET 1 +#define RBP_TYPE_REGISTER 2 +#define RBP_TYPE_EXPRESSION 3 +#define RBP_TYPE_UNDEFINED_RETURN_ADDRESS 4 +#define RBP_TYPE_OFFSET_DID_NOT_FIT 5 +#define RBP_TYPE_ARM64_RETURN_ADDRESS_LR 6 +#define RBP_TYPE_ARM64_RETURN_ADDRESS_FRAME 7 +#define RBP_TYPE_ARM64_RETURN_ADDRESS_ELSEWHERE 8 + +// Binary search error codes. +#define BINARY_SEARCH_DEFAULT 0xFABADAFABADAULL +#define BINARY_SEARCH_SHOULD_NEVER_HAPPEN 0xDEADBEEFDEADBEEFULL +#define BINARY_SEARCH_EXHAUSTED_ITERATIONS 0xBADFADBADFADBADULL + +struct lightswitch_config_t { + bool verbose_logging; + bool use_ring_buffers; + bool use_task_pt_regs_helper; + bool use_btf_helpers; + unsigned int userspace_pid_ns_level; +}; + +struct unwinder_stats_t { + u64 total; + u64 success_dwarf; + u64 error_truncated; + u64 error_unsupported_expression; + u64 error_unsupported_frame_pointer_action; + u64 error_unsupported_cfa_register; + u64 error_previous_rsp_read; + u64 error_previous_rsp_zero; + u64 error_previous_rip_read; + u64 error_previous_rip_zero; + u64 error_previous_rbp_read; + u64 error_should_never_happen; + u64 error_mapping_not_found; + u64 error_mapping_does_not_contain_pc; + u64 error_page_not_found; + u64 error_binary_search_exhausted_iterations; + u64 error_sending_new_process_event; + u64 error_sending_need_unwind_info_event; + u64 error_cfa_offset_did_not_fit; + u64 error_rbp_offset_did_not_fit; + u64 error_failure_sending_stack; + u64 bp_non_zero_for_bottom_frame; + u64 vdso_encountered; + u64 jit_encountered; +}; + +const volatile struct lightswitch_config_t lightswitch_config = { + .verbose_logging = false, + .use_ring_buffers = false, + .use_task_pt_regs_helper = false, + .use_btf_helpers = false, + .userspace_pid_ns_level = -1, +}; + +#define LOG(fmt, ...) \ + ({ \ + if (lightswitch_config.verbose_logging) { \ + bpf_printk(fmt, ##__VA_ARGS__); \ + } \ + }) + +// Represents an executable mapping. +typedef struct { + u64 executable_id; + u64 load_address; + u64 begin; + u64 end; + u32 type; +} mapping_t; + +// Key for the longest prefix matching. This is defined +// in the kernel in struct bpf_lpm_trie_key. +struct exec_mappings_key { + u32 prefix_len; + // Per /usr/include/bits/typesizes.h, __PID_T_TYPE is S32 (i32 in Rust) + s32 pid; + u64 data; +}; + +// Prefix size in bits, excluding the prefix length. +#define PREFIX_LEN (sizeof(struct exec_mappings_key) - sizeof(u32)) * 8; + +typedef struct __attribute__((packed)) { + u16 pc_low; + u8 cfa_type; + u8 rbp_type; + u16 cfa_offset; + s16 rbp_offset; +} stack_unwind_row_t; + +_Static_assert(sizeof(stack_unwind_row_t) == 8, + "unwind row has the expected size"); + +// The addresses of a native stack trace. +typedef struct { + u32 ulen; + u32 klen; + // Needed as the verifier won't operate with dynamically computed offsets and + // wants to ensure that any write won't be out of bounds. Note that only the + // actual unwound stack will be sent to userspace. + u64 addresses[MAX_STACK_DEPTH * 2]; +} native_stack_t; + +typedef struct { + int pid; + int tid; + u64 collected_at; + native_stack_t stack; +} sample_t; + +typedef struct { + unsigned long long ip; + unsigned long long sp; + unsigned long long bp; + unsigned long long lr; + u64 tail_calls; + sample_t sample; +} unwind_state_t; + +enum event_type { + EVENT_NEW_PROCESS = 1, + EVENT_NEED_UNWIND_INFO = 2, +}; + +typedef struct { + enum event_type type; + int pid; // use right name here (tgid?) + u64 address; + // Not using char as it's signed in arm64. + u8 comm[16]; +} Event; + +enum program { + PROGRAM_NATIVE_UNWINDER = 0, +}; diff --git a/tools/testing/selftests/bpf/progs/lightswitch/shared_helpers.h b/tools/testing/selftests/bpf/progs/lightswitch/shared_helpers.h new file mode 100644 index 0000000000000..a6ef2a451af10 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/lightswitch/shared_helpers.h @@ -0,0 +1,48 @@ +#include "constants.h" + +static __always_inline struct task_struct *current_task() { + if (lightswitch_config.use_task_pt_regs_helper && lightswitch_config.use_btf_helpers) { + return bpf_get_current_task_btf(); + } else { + return (struct task_struct *)bpf_get_current_task(); + } +} + +static __always_inline struct pt_regs *pt_regs(struct task_struct *task) { + if (lightswitch_config.use_task_pt_regs_helper && lightswitch_config.use_btf_helpers) { + if (task == NULL) { + return NULL; + } + return (struct pt_regs *)bpf_task_pt_regs(task); + } else { + if (task == NULL) { + return NULL; + } + void *stack; + int err = bpf_probe_read_kernel(&stack, 8, &task->stack); + if (err) { + LOG("[warn] bpf_probe_read_kernel failed with %d", err); + return NULL; + } + void *ptr = stack + THREAD_SIZE - TOP_OF_KERNEL_STACK_PADDING; + return ((struct pt_regs *)ptr) - 1; + } +} + +static __always_inline mapping_t *find_mapping(int per_process_id, u64 pc) { + struct exec_mappings_key key = {}; + key.prefix_len = PREFIX_LEN; + key.pid = __builtin_bswap32((u32)per_process_id); + key.data = __builtin_bswap64(pc); + + return bpf_map_lookup_elem(&exec_mappings, &key); +} + +static __always_inline bool process_is_known(int per_process_id) { + struct exec_mappings_key key = {}; + key.prefix_len = PREFIX_LEN; + key.pid = __builtin_bswap32((u32)per_process_id); + key.data = 0; + + return bpf_map_lookup_elem(&exec_mappings, &key) != NULL; +} diff --git a/tools/testing/selftests/bpf/progs/lightswitch/shared_maps.h b/tools/testing/selftests/bpf/progs/lightswitch/shared_maps.h new file mode 100644 index 0000000000000..037d6fe2cd45c --- /dev/null +++ b/tools/testing/selftests/bpf/progs/lightswitch/shared_maps.h @@ -0,0 +1,58 @@ +#ifndef __LIGHTSWITCH_SHARED_BPF_MAPS__ +#define __LIGHTSWITCH_SHARED_BPF_MAPS__ + +#include + +struct { + __uint(type, BPF_MAP_TYPE_LPM_TRIE); + __type(key, struct exec_mappings_key); + __type(value, mapping_t); + __uint(map_flags, BPF_F_NO_PREALLOC); + __uint(max_entries, MAX_MAPPINGS); +} exec_mappings SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, 1); + __type(key, u32); + __type(value, struct unwinder_stats_t); +} percpu_stats SEC(".maps"); + +#define DEFINE_COUNTER(__func__name) \ + static void bump_unwind_##__func__name() { \ + u32 zero = 0; \ + struct unwinder_stats_t *unwinder_stats = \ + bpf_map_lookup_elem(&percpu_stats, &zero); \ + if (unwinder_stats != NULL) { \ + unwinder_stats->__func__name++; \ + } \ + } + +DEFINE_COUNTER(total); +DEFINE_COUNTER(success_dwarf); +DEFINE_COUNTER(error_truncated); +DEFINE_COUNTER(error_unsupported_expression); +DEFINE_COUNTER(error_unsupported_frame_pointer_action); +DEFINE_COUNTER(error_unsupported_cfa_register); +DEFINE_COUNTER(error_previous_rsp_read); +DEFINE_COUNTER(error_previous_rsp_zero); +DEFINE_COUNTER(error_previous_rip_read); +DEFINE_COUNTER(error_previous_rip_zero); +DEFINE_COUNTER(error_previous_rbp_read); +DEFINE_COUNTER(error_should_never_happen); +DEFINE_COUNTER(error_mapping_not_found); +DEFINE_COUNTER(error_mapping_does_not_contain_pc); +DEFINE_COUNTER(error_page_not_found); +DEFINE_COUNTER(error_binary_search_exhausted_iterations); +DEFINE_COUNTER(error_sending_new_process_event); +DEFINE_COUNTER(error_sending_need_unwind_info_event); +DEFINE_COUNTER(error_cfa_offset_did_not_fit); +DEFINE_COUNTER(error_rbp_offset_did_not_fit); +DEFINE_COUNTER(error_failure_sending_stack); +#ifdef __TARGET_ARCH_x86 +DEFINE_COUNTER(bp_non_zero_for_bottom_frame); +#endif +DEFINE_COUNTER(vdso_encountered); +DEFINE_COUNTER(jit_encountered); + +#endif diff --git a/tools/testing/selftests/bpf/progs/lightswitch/tracers.bpf.c b/tools/testing/selftests/bpf/progs/lightswitch/tracers.bpf.c new file mode 100644 index 0000000000000..1f259cd2cbd7e --- /dev/null +++ b/tools/testing/selftests/bpf/progs/lightswitch/tracers.bpf.c @@ -0,0 +1,166 @@ +#include "vmlinux.h" +#include "profiler.h" +#include "shared_maps.h" +#include "shared_helpers.h" +#include "tracers.h" + +#include +#include + +typedef struct { + u32 tid; +} mmap_data_key_t; + +typedef struct { + u64 start_address; + u64 end_address; +} mmap_data_value_t; + +struct { + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); + __uint(key_size, sizeof(u32)); + __uint(value_size, sizeof(u32)); + __uint(max_entries, 0); +} tracer_events SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_RINGBUF); + __uint(max_entries, 256 * 1024 /* 256 KB */); +} tracer_events_rb SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 500); + __type(key, mmap_data_key_t); + __type(value, mmap_data_value_t); +} tracked_munmap SEC(".maps"); + +// Arguments from +// /sys/kernel/debug/tracing/events/syscalls/sys_enter_munmap/format +struct munmap_entry_args { + unsigned short common_type; + unsigned char common_flags; + unsigned char common_preempt_count; + int common_pid; + int __syscall_nr; + unsigned long addr; + size_t len; +}; + +SEC("tracepoint/sched/sched_process_exit") +int tracer_process_exit(void *ctx) { + struct task_struct *task = current_task(); + unsigned int level = lightswitch_config.userspace_pid_ns_level; + int per_process_id = BPF_CORE_READ(task, group_leader, thread_pid, numbers[level].nr); + int per_thread_id = BPF_CORE_READ(task, thread_pid, numbers[level].nr); + + if (!process_is_known(per_process_id)) { + return 0; + } + + // Only report main thread terminating. + if (per_process_id != per_thread_id) { + return 0; + } + + tracer_event_t event = { + .type = TRACER_EVENT_TYPE_PROCESS_EXIT, + .pid = per_process_id, + .start_address = 0, + }; + + int ret = 0; + if (lightswitch_config.use_ring_buffers) { + ret = bpf_ringbuf_output(&tracer_events_rb, &event, sizeof(tracer_event_t), 0); + } else { + ret = bpf_perf_event_output(ctx, &tracer_events, BPF_F_CURRENT_CPU, &event, sizeof(tracer_event_t)); + } + if (ret < 0) { + LOG("[error] failed to send process exit tracer event"); + return 0; + } + + LOG("[debug] sent process exit tracer event"); + return 0; +} + +SEC("tracepoint/syscalls/sys_enter_munmap") +int tracer_enter_munmap(struct munmap_entry_args *args) { + u64 start_address = args->addr; + u64 end_address = args->addr + args->len; + + struct task_struct *task = current_task(); + unsigned int level = lightswitch_config.userspace_pid_ns_level; + int per_process_id = BPF_CORE_READ(task, group_leader, thread_pid, numbers[level].nr); + + // We might not know about some mappings, but also we definitely don't want to notify + // of non-executable mappings being unmapped. + mapping_t *mapping = find_mapping(per_process_id, start_address); + if (mapping == NULL) { + return 0; + } + + // Ensure we didn't get a process entry. + if (start_address < mapping->begin || start_address >= mapping->end) { + return 0; + } + + int per_thread_id = BPF_CORE_READ(task, thread_pid, numbers[level].nr); + mmap_data_key_t key = { + .tid = per_thread_id, + }; + mmap_data_value_t value = { + .start_address = start_address, + .end_address = end_address, + }; + bpf_map_update_elem(&tracked_munmap, &key, &value, BPF_ANY); + + return 0; +} + +SEC("tracepoint/syscalls/sys_exit_munmap") +int tracer_exit_munmap(struct syscall_trace_exit *ctx) { + struct task_struct *task = current_task(); + unsigned int level = lightswitch_config.userspace_pid_ns_level; + int per_thread_id = BPF_CORE_READ(task, thread_pid, numbers[level].nr); + + mmap_data_key_t key = { + .tid = per_thread_id, + }; + + mmap_data_value_t *value = bpf_map_lookup_elem(&tracked_munmap, &key); + if (value == NULL) { + return 0; + } + + if (ctx->ret != 0) { + bpf_map_delete_elem(&tracked_munmap, &key); + return 0; + } + + LOG("[debug] sending munmap event"); + + int per_process_id = BPF_CORE_READ(task, group_leader, thread_pid, numbers[level].nr); + tracer_event_t event = { + .type = TRACER_EVENT_TYPE_MUNMAP, + .pid = per_process_id, + .start_address = value->start_address, + .end_address = value->end_address, + }; + + int ret; + + if (lightswitch_config.use_ring_buffers) { + ret = bpf_ringbuf_output(&tracer_events_rb, &event, sizeof(tracer_event_t), 0); + } else { + ret = bpf_perf_event_output(ctx, &tracer_events, BPF_F_CURRENT_CPU, &event, sizeof(tracer_event_t)); + } + if (ret < 0) { + LOG("[error] failed to send munmap tracer event"); + } + + bpf_map_delete_elem(&tracked_munmap, &key); + return 0; +} + +char LICENSE[] SEC("license") = "Dual MIT/GPL"; diff --git a/tools/testing/selftests/bpf/progs/lightswitch/tracers.h b/tools/testing/selftests/bpf/progs/lightswitch/tracers.h new file mode 100644 index 0000000000000..fea7abc891ff5 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/lightswitch/tracers.h @@ -0,0 +1,13 @@ +#include "basic_types.h" + +enum tracer_event_type { + TRACER_EVENT_TYPE_PROCESS_EXIT = 1, + TRACER_EVENT_TYPE_MUNMAP = 2, +}; + +typedef struct { + u32 type; + int pid; + u64 start_address; + u64 end_address; +} tracer_event_t; diff --git a/tools/testing/selftests/bpf/test_lightswitch_bpf_build.sh b/tools/testing/selftests/bpf/test_lightswitch_bpf_build.sh new file mode 100755 index 0000000000000..6acb3e82ad4b6 --- /dev/null +++ b/tools/testing/selftests/bpf/test_lightswitch_bpf_build.sh @@ -0,0 +1,28 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 + +set -eu + +ksft_skip=4 +obj_dir="lightswitch_bpf" + +if [ ! -d "$obj_dir" ]; then + echo "lightswitch BPF build output not found" + exit "$ksft_skip" +fi + +set -- "$obj_dir"/*.bpf.o +if [ ! -e "$1" ]; then + echo "no lightswitch BPF objects were built" + exit 1 +fi + +for obj in "$@"; do + if [ ! -s "$obj" ]; then + echo "empty lightswitch BPF object: $obj" + exit 1 + fi + echo "built $obj" +done + +exit 0