Contributors: 20
Author Tokens Token Proportion Commits Commit Proportion
Yonghong Song 4702 54.70% 9 18.37%
Kui-Feng Lee 1654 19.24% 2 4.08%
Hou Tao 547 6.36% 4 8.16%
Song Liu 466 5.42% 2 4.08%
Florent Revest 352 4.09% 3 6.12%
Dmitrii Dolgov 232 2.70% 3 6.12%
Alan Maguire 168 1.95% 4 8.16%
Yuran Pereira 126 1.47% 2 4.08%
Andrii Nakryiko 114 1.33% 7 14.29%
Dave Marchevsky 94 1.09% 3 6.12%
Kenny Yu 42 0.49% 1 2.04%
Kuniyuki Iwashima 30 0.35% 1 2.04%
Jiri Olsa 21 0.24% 1 2.04%
Wang Nan 18 0.21% 1 2.04%
Chuyi Zhou 9 0.10% 1 2.04%
Jordan Rome 7 0.08% 1 2.04%
Maxim Mikityanskiy 7 0.08% 1 2.04%
Ilya Leoshkevich 4 0.05% 1 2.04%
Toke Höiland-Jörgensen 2 0.02% 1 2.04%
Geliang Tang 1 0.01% 1 2.04%
Total 8596 49 
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <test_progs.h>
#include <unistd.h>
#include <sys/syscall.h>
#include <task_local_storage_helpers.h>
#include "bpf_iter_ipv6_route.skel.h"
#include "bpf_iter_netlink.skel.h"
#include "bpf_iter_bpf_map.skel.h"
#include "bpf_iter_tasks.skel.h"
#include "bpf_iter_task_stack.skel.h"
#include "bpf_iter_task_file.skel.h"
#include "bpf_iter_task_vmas.skel.h"
#include "bpf_iter_task_btf.skel.h"
#include "bpf_iter_tcp4.skel.h"
#include "bpf_iter_tcp6.skel.h"
#include "bpf_iter_udp4.skel.h"
#include "bpf_iter_udp6.skel.h"
#include "bpf_iter_unix.skel.h"
#include "bpf_iter_vma_offset.skel.h"
#include "bpf_iter_test_kern1.skel.h"
#include "bpf_iter_test_kern2.skel.h"
#include "bpf_iter_test_kern3.skel.h"
#include "bpf_iter_test_kern4.skel.h"
#include "bpf_iter_bpf_hash_map.skel.h"
#include "bpf_iter_bpf_percpu_hash_map.skel.h"
#include "bpf_iter_bpf_array_map.skel.h"
#include "bpf_iter_bpf_percpu_array_map.skel.h"
#include "bpf_iter_bpf_sk_storage_helpers.skel.h"
#include "bpf_iter_bpf_sk_storage_map.skel.h"
#include "bpf_iter_test_kern5.skel.h"
#include "bpf_iter_test_kern6.skel.h"
#include "bpf_iter_bpf_link.skel.h"
#include "bpf_iter_ksym.skel.h"
#include "bpf_iter_sockmap.skel.h"

static void test_btf_id_or_null(void)
{
	struct bpf_iter_test_kern3 *skel;

	skel = bpf_iter_test_kern3__open_and_load();
	if (!ASSERT_ERR_PTR(skel, "bpf_iter_test_kern3__open_and_load")) {
		bpf_iter_test_kern3__destroy(skel);
		return;
	}
}

static void do_dummy_read_opts(struct bpf_program *prog, struct bpf_iter_attach_opts *opts)
{
	struct bpf_link *link;
	char buf[16] = {};
	int iter_fd, len;

	link = bpf_program__attach_iter(prog, opts);
	if (!ASSERT_OK_PTR(link, "attach_iter"))
		return;

	iter_fd = bpf_iter_create(bpf_link__fd(link));
	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
		goto free_link;

	/* not check contents, but ensure read() ends without error */
	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
		;
	ASSERT_GE(len, 0, "read");

	close(iter_fd);

free_link:
	bpf_link__destroy(link);
}

static void do_dummy_read(struct bpf_program *prog)
{
	do_dummy_read_opts(prog, NULL);
}

static void do_read_map_iter_fd(struct bpf_object_skeleton **skel, struct bpf_program *prog,
				struct bpf_map *map)
{
	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
	union bpf_iter_link_info linfo;
	struct bpf_link *link;
	char buf[16] = {};
	int iter_fd, len;

	memset(&linfo, 0, sizeof(linfo));
	linfo.map.map_fd = bpf_map__fd(map);
	opts.link_info = &linfo;
	opts.link_info_len = sizeof(linfo);
	link = bpf_program__attach_iter(prog, &opts);
	if (!ASSERT_OK_PTR(link, "attach_map_iter"))
		return;

	iter_fd = bpf_iter_create(bpf_link__fd(link));
	if (!ASSERT_GE(iter_fd, 0, "create_map_iter")) {
		bpf_link__destroy(link);
		return;
	}

	/* Close link and map fd prematurely */
	bpf_link__destroy(link);
	bpf_object__destroy_skeleton(*skel);
	*skel = NULL;

	/* Try to let map free work to run first if map is freed */
	usleep(100);
	/* Memory used by both sock map and sock local storage map are
	 * freed after two synchronize_rcu() calls, so wait for it
	 */
	kern_sync_rcu();
	kern_sync_rcu();

	/* Read after both map fd and link fd are closed */
	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
		;
	ASSERT_GE(len, 0, "read_iterator");

	close(iter_fd);
}

static int read_fd_into_buffer(int fd, char *buf, int size)
{
	int bufleft = size;
	int len;

	do {
		len = read(fd, buf, bufleft);
		if (len > 0) {
			buf += len;
			bufleft -= len;
		}
	} while (len > 0);

	return len < 0 ? len : size - bufleft;
}

static void test_ipv6_route(void)
{
	struct bpf_iter_ipv6_route *skel;

	skel = bpf_iter_ipv6_route__open_and_load();
	if (!ASSERT_OK_PTR(skel, "bpf_iter_ipv6_route__open_and_load"))
		return;

	do_dummy_read(skel->progs.dump_ipv6_route);

	bpf_iter_ipv6_route__destroy(skel);
}

static void test_netlink(void)
{
	struct bpf_iter_netlink *skel;

	skel = bpf_iter_netlink__open_and_load();
	if (!ASSERT_OK_PTR(skel, "bpf_iter_netlink__open_and_load"))
		return;

	do_dummy_read(skel->progs.dump_netlink);

	bpf_iter_netlink__destroy(skel);
}

static void test_bpf_map(void)
{
	struct bpf_iter_bpf_map *skel;

	skel = bpf_iter_bpf_map__open_and_load();
	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_map__open_and_load"))
		return;

	do_dummy_read(skel->progs.dump_bpf_map);

	bpf_iter_bpf_map__destroy(skel);
}

static void check_bpf_link_info(const struct bpf_program *prog)
{
	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
	union bpf_iter_link_info linfo;
	struct bpf_link_info info = {};
	struct bpf_link *link;
	__u32 info_len;
	int err;

	memset(&linfo, 0, sizeof(linfo));
	linfo.task.tid = getpid();
	opts.link_info = &linfo;
	opts.link_info_len = sizeof(linfo);

	link = bpf_program__attach_iter(prog, &opts);
	if (!ASSERT_OK_PTR(link, "attach_iter"))
		return;

	info_len = sizeof(info);
	err = bpf_link_get_info_by_fd(bpf_link__fd(link), &info, &info_len);
	ASSERT_OK(err, "bpf_link_get_info_by_fd");
	ASSERT_EQ(info.iter.task.tid, getpid(), "check_task_tid");

	bpf_link__destroy(link);
}

static pthread_mutex_t do_nothing_mutex;

static void *do_nothing_wait(void *arg)
{
	pthread_mutex_lock(&do_nothing_mutex);
	pthread_mutex_unlock(&do_nothing_mutex);

	pthread_exit(arg);
}

static void test_task_common_nocheck(struct bpf_iter_attach_opts *opts,
				     int *num_unknown, int *num_known)
{
	struct bpf_iter_tasks *skel;
	pthread_t thread_id;
	void *ret;

	skel = bpf_iter_tasks__open_and_load();
	if (!ASSERT_OK_PTR(skel, "bpf_iter_tasks__open_and_load"))
		return;

	ASSERT_OK(pthread_mutex_lock(&do_nothing_mutex), "pthread_mutex_lock");

	ASSERT_OK(pthread_create(&thread_id, NULL, &do_nothing_wait, NULL),
		  "pthread_create");

	skel->bss->tid = getpid();

	do_dummy_read_opts(skel->progs.dump_task, opts);

	*num_unknown = skel->bss->num_unknown_tid;
	*num_known = skel->bss->num_known_tid;

	ASSERT_OK(pthread_mutex_unlock(&do_nothing_mutex), "pthread_mutex_unlock");
	ASSERT_FALSE(pthread_join(thread_id, &ret) || ret != NULL,
		     "pthread_join");

	bpf_iter_tasks__destroy(skel);
}

static void test_task_common(struct bpf_iter_attach_opts *opts, int num_unknown, int num_known)
{
	int num_unknown_tid, num_known_tid;

	test_task_common_nocheck(opts, &num_unknown_tid, &num_known_tid);
	ASSERT_EQ(num_unknown_tid, num_unknown, "check_num_unknown_tid");
	ASSERT_EQ(num_known_tid, num_known, "check_num_known_tid");
}

static void test_task_tid(void)
{
	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
	union bpf_iter_link_info linfo;
	int num_unknown_tid, num_known_tid;

	memset(&linfo, 0, sizeof(linfo));
	linfo.task.tid = getpid();
	opts.link_info = &linfo;
	opts.link_info_len = sizeof(linfo);
	test_task_common(&opts, 0, 1);

	linfo.task.tid = 0;
	linfo.task.pid = getpid();
	test_task_common(&opts, 1, 1);

	test_task_common_nocheck(NULL, &num_unknown_tid, &num_known_tid);
	ASSERT_GT(num_unknown_tid, 1, "check_num_unknown_tid");
	ASSERT_EQ(num_known_tid, 1, "check_num_known_tid");
}

static void test_task_pid(void)
{
	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
	union bpf_iter_link_info linfo;

	memset(&linfo, 0, sizeof(linfo));
	linfo.task.pid = getpid();
	opts.link_info = &linfo;
	opts.link_info_len = sizeof(linfo);

	test_task_common(&opts, 1, 1);
}

static void test_task_pidfd(void)
{
	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
	union bpf_iter_link_info linfo;
	int pidfd;

	pidfd = sys_pidfd_open(getpid(), 0);
	if (!ASSERT_GT(pidfd, 0, "sys_pidfd_open"))
		return;

	memset(&linfo, 0, sizeof(linfo));
	linfo.task.pid_fd = pidfd;
	opts.link_info = &linfo;
	opts.link_info_len = sizeof(linfo);

	test_task_common(&opts, 1, 1);

	close(pidfd);
}

static void test_task_sleepable(void)
{
	struct bpf_iter_tasks *skel;

	skel = bpf_iter_tasks__open_and_load();
	if (!ASSERT_OK_PTR(skel, "bpf_iter_tasks__open_and_load"))
		return;

	do_dummy_read(skel->progs.dump_task_sleepable);

	ASSERT_GT(skel->bss->num_expected_failure_copy_from_user_task, 0,
		  "num_expected_failure_copy_from_user_task");
	ASSERT_GT(skel->bss->num_success_copy_from_user_task, 0,
		  "num_success_copy_from_user_task");

	bpf_iter_tasks__destroy(skel);
}

static void test_task_stack(void)
{
	struct bpf_iter_task_stack *skel;

	skel = bpf_iter_task_stack__open_and_load();
	if (!ASSERT_OK_PTR(skel, "bpf_iter_task_stack__open_and_load"))
		return;

	do_dummy_read(skel->progs.dump_task_stack);
	do_dummy_read(skel->progs.get_task_user_stacks);

	ASSERT_EQ(skel->bss->num_user_stacks, 1, "num_user_stacks");

	bpf_iter_task_stack__destroy(skel);
}

static void test_task_file(void)
{
	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
	struct bpf_iter_task_file *skel;
	union bpf_iter_link_info linfo;
	pthread_t thread_id;
	void *ret;

	skel = bpf_iter_task_file__open_and_load();
	if (!ASSERT_OK_PTR(skel, "bpf_iter_task_file__open_and_load"))
		return;

	skel->bss->tgid = getpid();

	ASSERT_OK(pthread_mutex_lock(&do_nothing_mutex), "pthread_mutex_lock");

	ASSERT_OK(pthread_create(&thread_id, NULL, &do_nothing_wait, NULL),
		  "pthread_create");

	memset(&linfo, 0, sizeof(linfo));
	linfo.task.tid = getpid();
	opts.link_info = &linfo;
	opts.link_info_len = sizeof(linfo);

	do_dummy_read_opts(skel->progs.dump_task_file, &opts);

	ASSERT_EQ(skel->bss->count, 0, "check_count");
	ASSERT_EQ(skel->bss->unique_tgid_count, 1, "check_unique_tgid_count");

	skel->bss->last_tgid = 0;
	skel->bss->count = 0;
	skel->bss->unique_tgid_count = 0;

	do_dummy_read(skel->progs.dump_task_file);

	ASSERT_EQ(skel->bss->count, 0, "check_count");
	ASSERT_GT(skel->bss->unique_tgid_count, 1, "check_unique_tgid_count");

	check_bpf_link_info(skel->progs.dump_task_file);

	ASSERT_OK(pthread_mutex_unlock(&do_nothing_mutex), "pthread_mutex_unlock");
	ASSERT_OK(pthread_join(thread_id, &ret), "pthread_join");
	ASSERT_NULL(ret, "pthread_join");

	bpf_iter_task_file__destroy(skel);
}

#define TASKBUFSZ		32768

static char taskbuf[TASKBUFSZ];

static int do_btf_read(struct bpf_iter_task_btf *skel)
{
	struct bpf_program *prog = skel->progs.dump_task_struct;
	struct bpf_iter_task_btf__bss *bss = skel->bss;
	int iter_fd = -1, err;
	struct bpf_link *link;
	char *buf = taskbuf;
	int ret = 0;

	link = bpf_program__attach_iter(prog, NULL);
	if (!ASSERT_OK_PTR(link, "attach_iter"))
		return ret;

	iter_fd = bpf_iter_create(bpf_link__fd(link));
	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
		goto free_link;

	err = read_fd_into_buffer(iter_fd, buf, TASKBUFSZ);
	if (bss->skip) {
		printf("%s:SKIP:no __builtin_btf_type_id\n", __func__);
		ret = 1;
		test__skip();
		goto free_link;
	}

	if (!ASSERT_GE(err, 0, "read"))
		goto free_link;

	ASSERT_HAS_SUBSTR(taskbuf, "(struct task_struct)",
	      "check for btf representation of task_struct in iter data");
free_link:
	if (iter_fd > 0)
		close(iter_fd);
	bpf_link__destroy(link);
	return ret;
}

static void test_task_btf(void)
{
	struct bpf_iter_task_btf__bss *bss;
	struct bpf_iter_task_btf *skel;
	int ret;

	skel = bpf_iter_task_btf__open_and_load();
	if (!ASSERT_OK_PTR(skel, "bpf_iter_task_btf__open_and_load"))
		return;

	bss = skel->bss;

	ret = do_btf_read(skel);
	if (ret)
		goto cleanup;

	if (!ASSERT_NEQ(bss->tasks, 0, "no task iteration, did BPF program run?"))
		goto cleanup;

	ASSERT_EQ(bss->seq_err, 0, "check for unexpected err");

cleanup:
	bpf_iter_task_btf__destroy(skel);
}

static void test_tcp4(void)
{
	struct bpf_iter_tcp4 *skel;

	skel = bpf_iter_tcp4__open_and_load();
	if (!ASSERT_OK_PTR(skel, "bpf_iter_tcp4__open_and_load"))
		return;

	do_dummy_read(skel->progs.dump_tcp4);

	bpf_iter_tcp4__destroy(skel);
}

static void test_tcp6(void)
{
	struct bpf_iter_tcp6 *skel;

	skel = bpf_iter_tcp6__open_and_load();
	if (!ASSERT_OK_PTR(skel, "bpf_iter_tcp6__open_and_load"))
		return;

	do_dummy_read(skel->progs.dump_tcp6);

	bpf_iter_tcp6__destroy(skel);
}

static void test_udp4(void)
{
	struct bpf_iter_udp4 *skel;

	skel = bpf_iter_udp4__open_and_load();
	if (!ASSERT_OK_PTR(skel, "bpf_iter_udp4__open_and_load"))
		return;

	do_dummy_read(skel->progs.dump_udp4);

	bpf_iter_udp4__destroy(skel);
}

static void test_udp6(void)
{
	struct bpf_iter_udp6 *skel;

	skel = bpf_iter_udp6__open_and_load();
	if (!ASSERT_OK_PTR(skel, "bpf_iter_udp6__open_and_load"))
		return;

	do_dummy_read(skel->progs.dump_udp6);

	bpf_iter_udp6__destroy(skel);
}

static void test_unix(void)
{
	struct bpf_iter_unix *skel;

	skel = bpf_iter_unix__open_and_load();
	if (!ASSERT_OK_PTR(skel, "bpf_iter_unix__open_and_load"))
		return;

	do_dummy_read(skel->progs.dump_unix);

	bpf_iter_unix__destroy(skel);
}

/* The expected string is less than 16 bytes */
static int do_read_with_fd(int iter_fd, const char *expected,
			   bool read_one_char)
{
	int len, read_buf_len, start;
	char buf[16] = {};

	read_buf_len = read_one_char ? 1 : 16;
	start = 0;
	while ((len = read(iter_fd, buf + start, read_buf_len)) > 0) {
		start += len;
		if (!ASSERT_LT(start, 16, "read"))
			return -1;
		read_buf_len = read_one_char ? 1 : 16 - start;
	}
	if (!ASSERT_GE(len, 0, "read"))
		return -1;

	if (!ASSERT_STREQ(buf, expected, "read"))
		return -1;

	return 0;
}

static void test_anon_iter(bool read_one_char)
{
	struct bpf_iter_test_kern1 *skel;
	struct bpf_link *link;
	int iter_fd, err;

	skel = bpf_iter_test_kern1__open_and_load();
	if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern1__open_and_load"))
		return;

	err = bpf_iter_test_kern1__attach(skel);
	if (!ASSERT_OK(err, "bpf_iter_test_kern1__attach")) {
		goto out;
	}

	link = skel->links.dump_task;
	iter_fd = bpf_iter_create(bpf_link__fd(link));
	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
		goto out;

	do_read_with_fd(iter_fd, "abcd", read_one_char);
	close(iter_fd);

out:
	bpf_iter_test_kern1__destroy(skel);
}

static int do_read(const char *path, const char *expected)
{
	int err, iter_fd;

	iter_fd = open(path, O_RDONLY);
	if (!ASSERT_GE(iter_fd, 0, "open"))
		return -1;

	err = do_read_with_fd(iter_fd, expected, false);
	close(iter_fd);
	return err;
}

static void test_file_iter(void)
{
	const char *path = "/sys/fs/bpf/bpf_iter_test1";
	struct bpf_iter_test_kern1 *skel1;
	struct bpf_iter_test_kern2 *skel2;
	struct bpf_link *link;
	int err;

	skel1 = bpf_iter_test_kern1__open_and_load();
	if (!ASSERT_OK_PTR(skel1, "bpf_iter_test_kern1__open_and_load"))
		return;

	link = bpf_program__attach_iter(skel1->progs.dump_task, NULL);
	if (!ASSERT_OK_PTR(link, "attach_iter"))
		goto out;

	/* unlink this path if it exists. */
	unlink(path);

	err = bpf_link__pin(link, path);
	if (!ASSERT_OK(err, "pin_iter"))
		goto free_link;

	err = do_read(path, "abcd");
	if (err)
		goto unlink_path;

	/* file based iterator seems working fine. Let us a link update
	 * of the underlying link and `cat` the iterator again, its content
	 * should change.
	 */
	skel2 = bpf_iter_test_kern2__open_and_load();
	if (!ASSERT_OK_PTR(skel2, "bpf_iter_test_kern2__open_and_load"))
		goto unlink_path;

	err = bpf_link__update_program(link, skel2->progs.dump_task);
	if (!ASSERT_OK(err, "update_prog"))
		goto destroy_skel2;

	do_read(path, "ABCD");

destroy_skel2:
	bpf_iter_test_kern2__destroy(skel2);
unlink_path:
	unlink(path);
free_link:
	bpf_link__destroy(link);
out:
	bpf_iter_test_kern1__destroy(skel1);
}

static void test_overflow(bool test_e2big_overflow, bool ret1)
{
	__u32 map_info_len, total_read_len, expected_read_len;
	int err, iter_fd, map1_fd, map2_fd, len;
	struct bpf_map_info map_info = {};
	struct bpf_iter_test_kern4 *skel;
	struct bpf_link *link;
	__u32 iter_size;
	char *buf;

	skel = bpf_iter_test_kern4__open();
	if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern4__open"))
		return;

	/* create two maps: bpf program will only do bpf_seq_write
	 * for these two maps. The goal is one map output almost
	 * fills seq_file buffer and then the other will trigger
	 * overflow and needs restart.
	 */
	map1_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, 4, 8, 1, NULL);
	if (!ASSERT_GE(map1_fd, 0, "bpf_map_create"))
		goto out;
	map2_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, 4, 8, 1, NULL);
	if (!ASSERT_GE(map2_fd, 0, "bpf_map_create"))
		goto free_map1;

	/* bpf_seq_printf kernel buffer is 8 pages, so one map
	 * bpf_seq_write will mostly fill it, and the other map
	 * will partially fill and then trigger overflow and need
	 * bpf_seq_read restart.
	 */
	iter_size = sysconf(_SC_PAGE_SIZE) << 3;

	if (test_e2big_overflow) {
		skel->rodata->print_len = (iter_size + 8) / 8;
		expected_read_len = 2 * (iter_size + 8);
	} else if (!ret1) {
		skel->rodata->print_len = (iter_size - 8) / 8;
		expected_read_len = 2 * (iter_size - 8);
	} else {
		skel->rodata->print_len = 1;
		expected_read_len = 2 * 8;
	}
	skel->rodata->ret1 = ret1;

	if (!ASSERT_OK(bpf_iter_test_kern4__load(skel),
		  "bpf_iter_test_kern4__load"))
		goto free_map2;

	/* setup filtering map_id in bpf program */
	map_info_len = sizeof(map_info);
	err = bpf_map_get_info_by_fd(map1_fd, &map_info, &map_info_len);
	if (!ASSERT_OK(err, "get_map_info"))
		goto free_map2;
	skel->bss->map1_id = map_info.id;

	err = bpf_map_get_info_by_fd(map2_fd, &map_info, &map_info_len);
	if (!ASSERT_OK(err, "get_map_info"))
		goto free_map2;
	skel->bss->map2_id = map_info.id;

	link = bpf_program__attach_iter(skel->progs.dump_bpf_map, NULL);
	if (!ASSERT_OK_PTR(link, "attach_iter"))
		goto free_map2;

	iter_fd = bpf_iter_create(bpf_link__fd(link));
	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
		goto free_link;

	buf = malloc(expected_read_len);
	if (!ASSERT_OK_PTR(buf, "malloc"))
		goto close_iter;

	/* do read */
	total_read_len = 0;
	if (test_e2big_overflow) {
		while ((len = read(iter_fd, buf, expected_read_len)) > 0)
			total_read_len += len;

		ASSERT_EQ(len, -1, "read");
		ASSERT_EQ(errno, E2BIG, "read");
		goto free_buf;
	} else if (!ret1) {
		while ((len = read(iter_fd, buf, expected_read_len)) > 0)
			total_read_len += len;

		if (!ASSERT_GE(len, 0, "read"))
			goto free_buf;
	} else {
		do {
			len = read(iter_fd, buf, expected_read_len);
			if (len > 0)
				total_read_len += len;
		} while (len > 0 || len == -EAGAIN);

		if (!ASSERT_GE(len, 0, "read"))
			goto free_buf;
	}

	if (!ASSERT_EQ(total_read_len, expected_read_len, "read"))
		goto free_buf;

	if (!ASSERT_EQ(skel->bss->map1_accessed, 1, "map1_accessed"))
		goto free_buf;

	if (!ASSERT_EQ(skel->bss->map2_accessed, 2, "map2_accessed"))
		goto free_buf;

	ASSERT_EQ(skel->bss->map2_seqnum1, skel->bss->map2_seqnum2, "map2_seqnum");

free_buf:
	free(buf);
close_iter:
	close(iter_fd);
free_link:
	bpf_link__destroy(link);
free_map2:
	close(map2_fd);
free_map1:
	close(map1_fd);
out:
	bpf_iter_test_kern4__destroy(skel);
}

static void test_bpf_hash_map(void)
{
	__u32 expected_key_a = 0, expected_key_b = 0;
	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
	struct bpf_iter_bpf_hash_map *skel;
	int err, i, len, map_fd, iter_fd;
	union bpf_iter_link_info linfo;
	__u64 val, expected_val = 0;
	struct bpf_link *link;
	struct key_t {
		int a;
		int b;
		int c;
	} key;
	char buf[64];

	skel = bpf_iter_bpf_hash_map__open();
	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_hash_map__open"))
		return;

	skel->bss->in_test_mode = true;

	err = bpf_iter_bpf_hash_map__load(skel);
	if (!ASSERT_OK(err, "bpf_iter_bpf_hash_map__load"))
		goto out;

	/* iterator with hashmap2 and hashmap3 should fail */
	memset(&linfo, 0, sizeof(linfo));
	linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap2);
	opts.link_info = &linfo;
	opts.link_info_len = sizeof(linfo);
	link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);
	if (!ASSERT_ERR_PTR(link, "attach_iter"))
		goto out;

	linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap3);
	link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);
	if (!ASSERT_ERR_PTR(link, "attach_iter"))
		goto out;

	/* hashmap1 should be good, update map values here */
	map_fd = bpf_map__fd(skel->maps.hashmap1);
	for (i = 0; i < bpf_map__max_entries(skel->maps.hashmap1); i++) {
		key.a = i + 1;
		key.b = i + 2;
		key.c = i + 3;
		val = i + 4;
		expected_key_a += key.a;
		expected_key_b += key.b;
		expected_val += val;

		err = bpf_map_update_elem(map_fd, &key, &val, BPF_ANY);
		if (!ASSERT_OK(err, "map_update"))
			goto out;
	}

	/* Sleepable program is prohibited for hash map iterator */
	linfo.map.map_fd = map_fd;
	link = bpf_program__attach_iter(skel->progs.sleepable_dummy_dump, &opts);
	if (!ASSERT_ERR_PTR(link, "attach_sleepable_prog_to_iter"))
		goto out;

	linfo.map.map_fd = map_fd;
	link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);
	if (!ASSERT_OK_PTR(link, "attach_iter"))
		goto out;

	iter_fd = bpf_iter_create(bpf_link__fd(link));
	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
		goto free_link;

	/* do some tests */
	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
		;
	if (!ASSERT_GE(len, 0, "read"))
		goto close_iter;

	/* test results */
	if (!ASSERT_EQ(skel->bss->key_sum_a, expected_key_a, "key_sum_a"))
		goto close_iter;
	if (!ASSERT_EQ(skel->bss->key_sum_b, expected_key_b, "key_sum_b"))
		goto close_iter;
	if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
		goto close_iter;

close_iter:
	close(iter_fd);
free_link:
	bpf_link__destroy(link);
out:
	bpf_iter_bpf_hash_map__destroy(skel);
}

static void test_bpf_percpu_hash_map(void)
{
	__u32 expected_key_a = 0, expected_key_b = 0;
	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
	struct bpf_iter_bpf_percpu_hash_map *skel;
	int err, i, j, len, map_fd, iter_fd;
	union bpf_iter_link_info linfo;
	__u32 expected_val = 0;
	struct bpf_link *link;
	struct key_t {
		int a;
		int b;
		int c;
	} key;
	char buf[64];
	void *val;

	skel = bpf_iter_bpf_percpu_hash_map__open();
	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_hash_map__open"))
		return;

	skel->rodata->num_cpus = bpf_num_possible_cpus();
	val = malloc(8 * bpf_num_possible_cpus());
	if (!ASSERT_OK_PTR(val, "malloc"))
		goto out;

	err = bpf_iter_bpf_percpu_hash_map__load(skel);
	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_hash_map__load"))
		goto out;

	/* update map values here */
	map_fd = bpf_map__fd(skel->maps.hashmap1);
	for (i = 0; i < bpf_map__max_entries(skel->maps.hashmap1); i++) {
		key.a = i + 1;
		key.b = i + 2;
		key.c = i + 3;
		expected_key_a += key.a;
		expected_key_b += key.b;

		for (j = 0; j < bpf_num_possible_cpus(); j++) {
			*(__u32 *)(val + j * 8) = i + j;
			expected_val += i + j;
		}

		err = bpf_map_update_elem(map_fd, &key, val, BPF_ANY);
		if (!ASSERT_OK(err, "map_update"))
			goto out;
	}

	memset(&linfo, 0, sizeof(linfo));
	linfo.map.map_fd = map_fd;
	opts.link_info = &linfo;
	opts.link_info_len = sizeof(linfo);
	link = bpf_program__attach_iter(skel->progs.dump_bpf_percpu_hash_map, &opts);
	if (!ASSERT_OK_PTR(link, "attach_iter"))
		goto out;

	iter_fd = bpf_iter_create(bpf_link__fd(link));
	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
		goto free_link;

	/* do some tests */
	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
		;
	if (!ASSERT_GE(len, 0, "read"))
		goto close_iter;

	/* test results */
	if (!ASSERT_EQ(skel->bss->key_sum_a, expected_key_a, "key_sum_a"))
		goto close_iter;
	if (!ASSERT_EQ(skel->bss->key_sum_b, expected_key_b, "key_sum_b"))
		goto close_iter;
	if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
		goto close_iter;

close_iter:
	close(iter_fd);
free_link:
	bpf_link__destroy(link);
out:
	bpf_iter_bpf_percpu_hash_map__destroy(skel);
	free(val);
}

static void test_bpf_array_map(void)
{
	__u64 val, expected_val = 0, res_first_val, first_val = 0;
	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
	__u32 key, expected_key = 0, res_first_key;
	int err, i, map_fd, hash_fd, iter_fd;
	struct bpf_iter_bpf_array_map *skel;
	union bpf_iter_link_info linfo;
	struct bpf_link *link;
	char buf[64] = {};
	int len, start;

	skel = bpf_iter_bpf_array_map__open_and_load();
	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_array_map__open_and_load"))
		return;

	map_fd = bpf_map__fd(skel->maps.arraymap1);
	for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) {
		val = i + 4;
		expected_key += i;
		expected_val += val;

		if (i == 0)
			first_val = val;

		err = bpf_map_update_elem(map_fd, &i, &val, BPF_ANY);
		if (!ASSERT_OK(err, "map_update"))
			goto out;
	}

	memset(&linfo, 0, sizeof(linfo));
	linfo.map.map_fd = map_fd;
	opts.link_info = &linfo;
	opts.link_info_len = sizeof(linfo);
	link = bpf_program__attach_iter(skel->progs.dump_bpf_array_map, &opts);
	if (!ASSERT_OK_PTR(link, "attach_iter"))
		goto out;

	iter_fd = bpf_iter_create(bpf_link__fd(link));
	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
		goto free_link;

	/* do some tests */
	start = 0;
	while ((len = read(iter_fd, buf + start, sizeof(buf) - start)) > 0)
		start += len;
	if (!ASSERT_GE(len, 0, "read"))
		goto close_iter;

	/* test results */
	res_first_key = *(__u32 *)buf;
	res_first_val = *(__u64 *)(buf + sizeof(__u32));
	if (!ASSERT_EQ(res_first_key, 0, "bpf_seq_write") ||
			!ASSERT_EQ(res_first_val, first_val, "bpf_seq_write"))
		goto close_iter;

	if (!ASSERT_EQ(skel->bss->key_sum, expected_key, "key_sum"))
		goto close_iter;
	if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
		goto close_iter;

	hash_fd = bpf_map__fd(skel->maps.hashmap1);
	for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) {
		err = bpf_map_lookup_elem(map_fd, &i, &val);
		if (!ASSERT_OK(err, "map_lookup arraymap1"))
			goto close_iter;
		if (!ASSERT_EQ(i, val, "invalid_val arraymap1"))
			goto close_iter;

		val = i + 4;
		err = bpf_map_lookup_elem(hash_fd, &val, &key);
		if (!ASSERT_OK(err, "map_lookup hashmap1"))
			goto close_iter;
		if (!ASSERT_EQ(key, val - 4, "invalid_val hashmap1"))
			goto close_iter;
	}

close_iter:
	close(iter_fd);
free_link:
	bpf_link__destroy(link);
out:
	bpf_iter_bpf_array_map__destroy(skel);
}

static void test_bpf_array_map_iter_fd(void)
{
	struct bpf_iter_bpf_array_map *skel;

	skel = bpf_iter_bpf_array_map__open_and_load();
	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_array_map__open_and_load"))
		return;

	do_read_map_iter_fd(&skel->skeleton, skel->progs.dump_bpf_array_map,
			    skel->maps.arraymap1);

	bpf_iter_bpf_array_map__destroy(skel);
}

static void test_bpf_percpu_array_map(void)
{
	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
	struct bpf_iter_bpf_percpu_array_map *skel;
	__u32 expected_key = 0, expected_val = 0;
	union bpf_iter_link_info linfo;
	int err, i, j, map_fd, iter_fd;
	struct bpf_link *link;
	char buf[64];
	void *val;
	int len;

	skel = bpf_iter_bpf_percpu_array_map__open();
	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_array_map__open"))
		return;

	skel->rodata->num_cpus = bpf_num_possible_cpus();
	val = malloc(8 * bpf_num_possible_cpus());
	if (!ASSERT_OK_PTR(val, "malloc"))
		goto out;

	err = bpf_iter_bpf_percpu_array_map__load(skel);
	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_array_map__load"))
		goto out;

	/* update map values here */
	map_fd = bpf_map__fd(skel->maps.arraymap1);
	for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) {
		expected_key += i;

		for (j = 0; j < bpf_num_possible_cpus(); j++) {
			*(__u32 *)(val + j * 8) = i + j;
			expected_val += i + j;
		}

		err = bpf_map_update_elem(map_fd, &i, val, BPF_ANY);
		if (!ASSERT_OK(err, "map_update"))
			goto out;
	}

	memset(&linfo, 0, sizeof(linfo));
	linfo.map.map_fd = map_fd;
	opts.link_info = &linfo;
	opts.link_info_len = sizeof(linfo);
	link = bpf_program__attach_iter(skel->progs.dump_bpf_percpu_array_map, &opts);
	if (!ASSERT_OK_PTR(link, "attach_iter"))
		goto out;

	iter_fd = bpf_iter_create(bpf_link__fd(link));
	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
		goto free_link;

	/* do some tests */
	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
		;
	if (!ASSERT_GE(len, 0, "read"))
		goto close_iter;

	/* test results */
	if (!ASSERT_EQ(skel->bss->key_sum, expected_key, "key_sum"))
		goto close_iter;
	if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
		goto close_iter;

close_iter:
	close(iter_fd);
free_link:
	bpf_link__destroy(link);
out:
	bpf_iter_bpf_percpu_array_map__destroy(skel);
	free(val);
}

/* An iterator program deletes all local storage in a map. */
static void test_bpf_sk_storage_delete(void)
{
	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
	struct bpf_iter_bpf_sk_storage_helpers *skel;
	union bpf_iter_link_info linfo;
	int err, len, map_fd, iter_fd;
	struct bpf_link *link;
	int sock_fd = -1;
	__u32 val = 42;
	char buf[64];

	skel = bpf_iter_bpf_sk_storage_helpers__open_and_load();
	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_helpers__open_and_load"))
		return;

	map_fd = bpf_map__fd(skel->maps.sk_stg_map);

	sock_fd = socket(AF_INET6, SOCK_STREAM, 0);
	if (!ASSERT_GE(sock_fd, 0, "socket"))
		goto out;

	err = bpf_map_update_elem(map_fd, &sock_fd, &val, BPF_NOEXIST);
	if (!ASSERT_OK(err, "map_update"))
		goto out;

	memset(&linfo, 0, sizeof(linfo));
	linfo.map.map_fd = map_fd;
	opts.link_info = &linfo;
	opts.link_info_len = sizeof(linfo);
	link = bpf_program__attach_iter(skel->progs.delete_bpf_sk_storage_map,
					&opts);
	if (!ASSERT_OK_PTR(link, "attach_iter"))
		goto out;

	iter_fd = bpf_iter_create(bpf_link__fd(link));
	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
		goto free_link;

	/* do some tests */
	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
		;
	if (!ASSERT_GE(len, 0, "read"))
		goto close_iter;

	/* test results */
	err = bpf_map_lookup_elem(map_fd, &sock_fd, &val);

	 /* Note: The following assertions serve to ensure
	  * the value was deleted. It does so by asserting
	  * that bpf_map_lookup_elem has failed. This might
	  * seem counterintuitive at first.
	  */
	ASSERT_ERR(err, "bpf_map_lookup_elem");
	ASSERT_EQ(errno, ENOENT, "bpf_map_lookup_elem");

close_iter:
	close(iter_fd);
free_link:
	bpf_link__destroy(link);
out:
	if (sock_fd >= 0)
		close(sock_fd);
	bpf_iter_bpf_sk_storage_helpers__destroy(skel);
}

/* This creates a socket and its local storage. It then runs a task_iter BPF
 * program that replaces the existing socket local storage with the tgid of the
 * only task owning a file descriptor to this socket, this process, prog_tests.
 * It then runs a tcp socket iterator that negates the value in the existing
 * socket local storage, the test verifies that the resulting value is -pid.
 */
static void test_bpf_sk_storage_get(void)
{
	struct bpf_iter_bpf_sk_storage_helpers *skel;
	int err, map_fd, val = -1;
	int sock_fd = -1;

	skel = bpf_iter_bpf_sk_storage_helpers__open_and_load();
	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_helpers__open_and_load"))
		return;

	sock_fd = socket(AF_INET6, SOCK_STREAM, 0);
	if (!ASSERT_GE(sock_fd, 0, "socket"))
		goto out;

	err = listen(sock_fd, 1);
	if (!ASSERT_OK(err, "listen"))
		goto close_socket;

	map_fd = bpf_map__fd(skel->maps.sk_stg_map);

	err = bpf_map_update_elem(map_fd, &sock_fd, &val, BPF_NOEXIST);
	if (!ASSERT_OK(err, "bpf_map_update_elem"))
		goto close_socket;

	do_dummy_read(skel->progs.fill_socket_owner);

	err = bpf_map_lookup_elem(map_fd, &sock_fd, &val);
	if (!ASSERT_OK(err, "bpf_map_lookup_elem") ||
			!ASSERT_EQ(val, getpid(), "bpf_map_lookup_elem"))
		goto close_socket;

	do_dummy_read(skel->progs.negate_socket_local_storage);

	err = bpf_map_lookup_elem(map_fd, &sock_fd, &val);
	ASSERT_OK(err, "bpf_map_lookup_elem");
	ASSERT_EQ(val, -getpid(), "bpf_map_lookup_elem");

close_socket:
	close(sock_fd);
out:
	bpf_iter_bpf_sk_storage_helpers__destroy(skel);
}

static void test_bpf_sk_stoarge_map_iter_fd(void)
{
	struct bpf_iter_bpf_sk_storage_map *skel;

	skel = bpf_iter_bpf_sk_storage_map__open_and_load();
	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_map__open_and_load"))
		return;

	do_read_map_iter_fd(&skel->skeleton, skel->progs.rw_bpf_sk_storage_map,
			    skel->maps.sk_stg_map);

	bpf_iter_bpf_sk_storage_map__destroy(skel);
}

static void test_bpf_sk_storage_map(void)
{
	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
	int err, i, len, map_fd, iter_fd, num_sockets;
	struct bpf_iter_bpf_sk_storage_map *skel;
	union bpf_iter_link_info linfo;
	int sock_fd[3] = {-1, -1, -1};
	__u32 val, expected_val = 0;
	struct bpf_link *link;
	char buf[64];

	skel = bpf_iter_bpf_sk_storage_map__open_and_load();
	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_map__open_and_load"))
		return;

	map_fd = bpf_map__fd(skel->maps.sk_stg_map);
	num_sockets = ARRAY_SIZE(sock_fd);
	for (i = 0; i < num_sockets; i++) {
		sock_fd[i] = socket(AF_INET6, SOCK_STREAM, 0);
		if (!ASSERT_GE(sock_fd[i], 0, "socket"))
			goto out;

		val = i + 1;
		expected_val += val;

		err = bpf_map_update_elem(map_fd, &sock_fd[i], &val,
					  BPF_NOEXIST);
		if (!ASSERT_OK(err, "map_update"))
			goto out;
	}

	memset(&linfo, 0, sizeof(linfo));
	linfo.map.map_fd = map_fd;
	opts.link_info = &linfo;
	opts.link_info_len = sizeof(linfo);
	link = bpf_program__attach_iter(skel->progs.oob_write_bpf_sk_storage_map, &opts);
	err = libbpf_get_error(link);
	if (!ASSERT_EQ(err, -EACCES, "attach_oob_write_iter")) {
		if (!err)
			bpf_link__destroy(link);
		goto out;
	}

	link = bpf_program__attach_iter(skel->progs.rw_bpf_sk_storage_map, &opts);
	if (!ASSERT_OK_PTR(link, "attach_iter"))
		goto out;

	iter_fd = bpf_iter_create(bpf_link__fd(link));
	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
		goto free_link;

	skel->bss->to_add_val = time(NULL);
	/* do some tests */
	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
		;
	if (!ASSERT_GE(len, 0, "read"))
		goto close_iter;

	/* test results */
	if (!ASSERT_EQ(skel->bss->ipv6_sk_count, num_sockets, "ipv6_sk_count"))
		goto close_iter;

	if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
		goto close_iter;

	for (i = 0; i < num_sockets; i++) {
		err = bpf_map_lookup_elem(map_fd, &sock_fd[i], &val);
		if (!ASSERT_OK(err, "map_lookup") ||
		    !ASSERT_EQ(val, i + 1 + skel->bss->to_add_val, "check_map_value"))
			break;
	}

close_iter:
	close(iter_fd);
free_link:
	bpf_link__destroy(link);
out:
	for (i = 0; i < num_sockets; i++) {
		if (sock_fd[i] >= 0)
			close(sock_fd[i]);
	}
	bpf_iter_bpf_sk_storage_map__destroy(skel);
}

static void test_rdonly_buf_out_of_bound(void)
{
	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
	struct bpf_iter_test_kern5 *skel;
	union bpf_iter_link_info linfo;
	struct bpf_link *link;

	skel = bpf_iter_test_kern5__open_and_load();
	if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern5__open_and_load"))
		return;

	memset(&linfo, 0, sizeof(linfo));
	linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap1);
	opts.link_info = &linfo;
	opts.link_info_len = sizeof(linfo);
	link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);
	if (!ASSERT_ERR_PTR(link, "attach_iter"))
		bpf_link__destroy(link);

	bpf_iter_test_kern5__destroy(skel);
}

static void test_buf_neg_offset(void)
{
	struct bpf_iter_test_kern6 *skel;

	skel = bpf_iter_test_kern6__open_and_load();
	if (!ASSERT_ERR_PTR(skel, "bpf_iter_test_kern6__open_and_load"))
		bpf_iter_test_kern6__destroy(skel);
}

static void test_link_iter(void)
{
	struct bpf_iter_bpf_link *skel;

	skel = bpf_iter_bpf_link__open_and_load();
	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_link__open_and_load"))
		return;

	do_dummy_read(skel->progs.dump_bpf_link);

	bpf_iter_bpf_link__destroy(skel);
}

static void test_ksym_iter(void)
{
	struct bpf_iter_ksym *skel;

	skel = bpf_iter_ksym__open_and_load();
	if (!ASSERT_OK_PTR(skel, "bpf_iter_ksym__open_and_load"))
		return;

	do_dummy_read(skel->progs.dump_ksym);

	bpf_iter_ksym__destroy(skel);
}

#define CMP_BUFFER_SIZE 1024
static char task_vma_output[CMP_BUFFER_SIZE];
static char proc_maps_output[CMP_BUFFER_SIZE];

/* remove \0 and \t from str, and only keep the first line */
static void str_strip_first_line(char *str)
{
	char *dst = str, *src = str;

	do {
		if (*src == ' ' || *src == '\t')
			src++;
		else
			*(dst++) = *(src++);

	} while (*src != '\0' && *src != '\n');

	*dst = '\0';
}

static void test_task_vma_common(struct bpf_iter_attach_opts *opts)
{
	int err, iter_fd = -1, proc_maps_fd = -1;
	struct bpf_iter_task_vmas *skel;
	int len, read_size = 4;
	char maps_path[64];

	skel = bpf_iter_task_vmas__open();
	if (!ASSERT_OK_PTR(skel, "bpf_iter_task_vmas__open"))
		return;

	skel->bss->pid = getpid();
	skel->bss->one_task = opts ? 1 : 0;

	err = bpf_iter_task_vmas__load(skel);
	if (!ASSERT_OK(err, "bpf_iter_task_vmas__load"))
		goto out;

	skel->links.proc_maps = bpf_program__attach_iter(
		skel->progs.proc_maps, opts);

	if (!ASSERT_OK_PTR(skel->links.proc_maps, "bpf_program__attach_iter")) {
		skel->links.proc_maps = NULL;
		goto out;
	}

	iter_fd = bpf_iter_create(bpf_link__fd(skel->links.proc_maps));
	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
		goto out;

	/* Read CMP_BUFFER_SIZE (1kB) from bpf_iter. Read in small chunks
	 * to trigger seq_file corner cases.
	 */
	len = 0;
	while (len < CMP_BUFFER_SIZE) {
		err = read_fd_into_buffer(iter_fd, task_vma_output + len,
					  MIN(read_size, CMP_BUFFER_SIZE - len));
		if (!err)
			break;
		if (!ASSERT_GE(err, 0, "read_iter_fd"))
			goto out;
		len += err;
	}
	if (opts)
		ASSERT_EQ(skel->bss->one_task_error, 0, "unexpected task");

	/* read CMP_BUFFER_SIZE (1kB) from /proc/pid/maps */
	snprintf(maps_path, 64, "/proc/%u/maps", skel->bss->pid);
	proc_maps_fd = open(maps_path, O_RDONLY);
	if (!ASSERT_GE(proc_maps_fd, 0, "open_proc_maps"))
		goto out;
	err = read_fd_into_buffer(proc_maps_fd, proc_maps_output, CMP_BUFFER_SIZE);
	if (!ASSERT_GE(err, 0, "read_prog_maps_fd"))
		goto out;

	/* strip and compare the first line of the two files */
	str_strip_first_line(task_vma_output);
	str_strip_first_line(proc_maps_output);

	ASSERT_STREQ(task_vma_output, proc_maps_output, "compare_output");

	check_bpf_link_info(skel->progs.proc_maps);

out:
	close(proc_maps_fd);
	close(iter_fd);
	bpf_iter_task_vmas__destroy(skel);
}

static void test_task_vma_dead_task(void)
{
	struct bpf_iter_task_vmas *skel;
	int wstatus, child_pid = -1;
	time_t start_tm, cur_tm;
	int err, iter_fd = -1;
	int wait_sec = 3;

	skel = bpf_iter_task_vmas__open();
	if (!ASSERT_OK_PTR(skel, "bpf_iter_task_vmas__open"))
		return;

	skel->bss->pid = getpid();

	err = bpf_iter_task_vmas__load(skel);
	if (!ASSERT_OK(err, "bpf_iter_task_vmas__load"))
		goto out;

	skel->links.proc_maps = bpf_program__attach_iter(
		skel->progs.proc_maps, NULL);

	if (!ASSERT_OK_PTR(skel->links.proc_maps, "bpf_program__attach_iter")) {
		skel->links.proc_maps = NULL;
		goto out;
	}

	start_tm = time(NULL);
	cur_tm = start_tm;

	child_pid = fork();
	if (child_pid == 0) {
		/* Fork short-lived processes in the background. */
		while (cur_tm < start_tm + wait_sec) {
			system("echo > /dev/null");
			cur_tm = time(NULL);
		}
		exit(0);
	}

	if (!ASSERT_GE(child_pid, 0, "fork_child"))
		goto out;

	while (cur_tm < start_tm + wait_sec) {
		iter_fd = bpf_iter_create(bpf_link__fd(skel->links.proc_maps));
		if (!ASSERT_GE(iter_fd, 0, "create_iter"))
			goto out;

		/* Drain all data from iter_fd. */
		while (cur_tm < start_tm + wait_sec) {
			err = read_fd_into_buffer(iter_fd, task_vma_output, CMP_BUFFER_SIZE);
			if (!ASSERT_GE(err, 0, "read_iter_fd"))
				goto out;

			cur_tm = time(NULL);

			if (err == 0)
				break;
		}

		close(iter_fd);
		iter_fd = -1;
	}

	check_bpf_link_info(skel->progs.proc_maps);

out:
	waitpid(child_pid, &wstatus, 0);
	close(iter_fd);
	bpf_iter_task_vmas__destroy(skel);
}

void test_bpf_sockmap_map_iter_fd(void)
{
	struct bpf_iter_sockmap *skel;

	skel = bpf_iter_sockmap__open_and_load();
	if (!ASSERT_OK_PTR(skel, "bpf_iter_sockmap__open_and_load"))
		return;

	do_read_map_iter_fd(&skel->skeleton, skel->progs.copy, skel->maps.sockmap);

	bpf_iter_sockmap__destroy(skel);
}

static void test_task_vma(void)
{
	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
	union bpf_iter_link_info linfo;

	memset(&linfo, 0, sizeof(linfo));
	linfo.task.tid = getpid();
	opts.link_info = &linfo;
	opts.link_info_len = sizeof(linfo);

	test_task_vma_common(&opts);
	test_task_vma_common(NULL);
}

/* uprobe attach point */
static noinline int trigger_func(int arg)
{
	asm volatile ("");
	return arg + 1;
}

static void test_task_vma_offset_common(struct bpf_iter_attach_opts *opts, bool one_proc)
{
	struct bpf_iter_vma_offset *skel;
	char buf[16] = {};
	int iter_fd, len;
	int pgsz, shift;

	skel = bpf_iter_vma_offset__open_and_load();
	if (!ASSERT_OK_PTR(skel, "bpf_iter_vma_offset__open_and_load"))
		return;

	skel->bss->pid = getpid();
	skel->bss->address = (uintptr_t)trigger_func;
	for (pgsz = getpagesize(), shift = 0; pgsz > 1; pgsz >>= 1, shift++)
		;
	skel->bss->page_shift = shift;

	skel->links.get_vma_offset = bpf_program__attach_iter(skel->progs.get_vma_offset, opts);
	if (!ASSERT_OK_PTR(skel->links.get_vma_offset, "attach_iter"))
		goto exit;

	iter_fd = bpf_iter_create(bpf_link__fd(skel->links.get_vma_offset));
	if (!ASSERT_GT(iter_fd, 0, "create_iter"))
		goto exit;

	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
		;
	buf[15] = 0;
	ASSERT_EQ(strcmp(buf, "OK\n"), 0, "strcmp");

	ASSERT_EQ(skel->bss->offset, get_uprobe_offset(trigger_func), "offset");
	if (one_proc)
		ASSERT_EQ(skel->bss->unique_tgid_cnt, 1, "unique_tgid_count");
	else
		ASSERT_GT(skel->bss->unique_tgid_cnt, 1, "unique_tgid_count");

	close(iter_fd);

exit:
	bpf_iter_vma_offset__destroy(skel);
}

static void test_task_vma_offset(void)
{
	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
	union bpf_iter_link_info linfo;

	memset(&linfo, 0, sizeof(linfo));
	linfo.task.pid = getpid();
	opts.link_info = &linfo;
	opts.link_info_len = sizeof(linfo);

	test_task_vma_offset_common(&opts, true);

	linfo.task.pid = 0;
	linfo.task.tid = getpid();
	test_task_vma_offset_common(&opts, true);

	test_task_vma_offset_common(NULL, false);
}

void test_bpf_iter(void)
{
	ASSERT_OK(pthread_mutex_init(&do_nothing_mutex, NULL), "pthread_mutex_init");

	if (test__start_subtest("btf_id_or_null"))
		test_btf_id_or_null();
	if (test__start_subtest("ipv6_route"))
		test_ipv6_route();
	if (test__start_subtest("netlink"))
		test_netlink();
	if (test__start_subtest("bpf_map"))
		test_bpf_map();
	if (test__start_subtest("task_tid"))
		test_task_tid();
	if (test__start_subtest("task_pid"))
		test_task_pid();
	if (test__start_subtest("task_pidfd"))
		test_task_pidfd();
	if (test__start_subtest("task_sleepable"))
		test_task_sleepable();
	if (test__start_subtest("task_stack"))
		test_task_stack();
	if (test__start_subtest("task_file"))
		test_task_file();
	if (test__start_subtest("task_vma"))
		test_task_vma();
	if (test__start_subtest("task_vma_dead_task"))
		test_task_vma_dead_task();
	if (test__start_subtest("task_btf"))
		test_task_btf();
	if (test__start_subtest("tcp4"))
		test_tcp4();
	if (test__start_subtest("tcp6"))
		test_tcp6();
	if (test__start_subtest("udp4"))
		test_udp4();
	if (test__start_subtest("udp6"))
		test_udp6();
	if (test__start_subtest("unix"))
		test_unix();
	if (test__start_subtest("anon"))
		test_anon_iter(false);
	if (test__start_subtest("anon-read-one-char"))
		test_anon_iter(true);
	if (test__start_subtest("file"))
		test_file_iter();
	if (test__start_subtest("overflow"))
		test_overflow(false, false);
	if (test__start_subtest("overflow-e2big"))
		test_overflow(true, false);
	if (test__start_subtest("prog-ret-1"))
		test_overflow(false, true);
	if (test__start_subtest("bpf_hash_map"))
		test_bpf_hash_map();
	if (test__start_subtest("bpf_percpu_hash_map"))
		test_bpf_percpu_hash_map();
	if (test__start_subtest("bpf_array_map"))
		test_bpf_array_map();
	if (test__start_subtest("bpf_array_map_iter_fd"))
		test_bpf_array_map_iter_fd();
	if (test__start_subtest("bpf_percpu_array_map"))
		test_bpf_percpu_array_map();
	if (test__start_subtest("bpf_sk_storage_map"))
		test_bpf_sk_storage_map();
	if (test__start_subtest("bpf_sk_storage_map_iter_fd"))
		test_bpf_sk_stoarge_map_iter_fd();
	if (test__start_subtest("bpf_sk_storage_delete"))
		test_bpf_sk_storage_delete();
	if (test__start_subtest("bpf_sk_storage_get"))
		test_bpf_sk_storage_get();
	if (test__start_subtest("rdonly-buf-out-of-bound"))
		test_rdonly_buf_out_of_bound();
	if (test__start_subtest("buf-neg-offset"))
		test_buf_neg_offset();
	if (test__start_subtest("link-iter"))
		test_link_iter();
	if (test__start_subtest("ksym"))
		test_ksym_iter();
	if (test__start_subtest("bpf_sockmap_map_iter_fd"))
		test_bpf_sockmap_map_iter_fd();
	if (test__start_subtest("vma_offset"))
		test_task_vma_offset();
}