cregit-Linux how code gets into the kernel

Release 4.15 kernel/livepatch/core.c

Directory: kernel/livepatch
/*
 * core.c - Kernel Live Patching Core
 *
 * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
 * Copyright (C) 2014 SUSE
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, see <http://www.gnu.org/licenses/>.
 */


#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/mutex.h>
#include <linux/slab.h>
#include <linux/list.h>
#include <linux/kallsyms.h>
#include <linux/livepatch.h>
#include <linux/elf.h>
#include <linux/moduleloader.h>
#include <linux/completion.h>
#include <asm/cacheflush.h>
#include "core.h"
#include "patch.h"
#include "transition.h"

/*
 * klp_mutex is a coarse lock which serializes access to klp data.  All
 * accesses to klp-related variables and structures must have mutex protection,
 * except within the following functions which carefully avoid the need for it:
 *
 * - klp_ftrace_handler()
 * - klp_update_patch_state()
 */

DEFINE_MUTEX(klp_mutex);

static LIST_HEAD(klp_patches);


static struct kobject *klp_root_kobj;


static bool klp_is_module(struct klp_object *obj) { return obj->name; }

Contributors

PersonTokensPropCommitsCommitProp
Seth Jennings16100.00%1100.00%
Total16100.00%1100.00%

/* sets obj->mod if object is not vmlinux and module is found */
static void klp_find_object_module(struct klp_object *obj) { struct module *mod; if (!klp_is_module(obj)) return; mutex_lock(&module_mutex); /* * We do not want to block removal of patched modules and therefore * we do not take a reference here. The patches are removed by * klp_module_going() instead. */ mod = find_module(obj->name); /* * Do not mess work of klp_module_coming() and klp_module_going(). * Note that the patch might still be needed before klp_module_going() * is called. Module functions can be called even in the GOING state * until mod->exit() finishes. This is especially important for * patches that modify semantic of the functions. */ if (mod && mod->klp_alive) obj->mod = mod; mutex_unlock(&module_mutex); }

Contributors

PersonTokensPropCommitsCommitProp
Seth Jennings4166.13%133.33%
Petr Mladek1930.65%133.33%
Jessica Yu23.23%133.33%
Total62100.00%3100.00%


static bool klp_is_patch_registered(struct klp_patch *patch) { struct klp_patch *mypatch; list_for_each_entry(mypatch, &klp_patches, list) if (mypatch == patch) return true; return false; }

Contributors

PersonTokensPropCommitsCommitProp
Seth Jennings36100.00%1100.00%
Total36100.00%1100.00%


static bool klp_initialized(void) { return !!klp_root_kobj; }

Contributors

PersonTokensPropCommitsCommitProp
Seth Jennings1184.62%150.00%
Nicholas Mc Guire215.38%150.00%
Total13100.00%2100.00%

struct klp_find_arg { const char *objname; const char *name; unsigned long addr; unsigned long count; unsigned long pos; };
static int klp_find_callback(void *data, const char *name, struct module *mod, unsigned long addr) { struct klp_find_arg *args = data; if ((mod && !args->objname) || (!mod && args->objname)) return 0; if (strcmp(args->name, name)) return 0; if (args->objname && strcmp(args->objname, mod->name)) return 0; args->addr = addr; args->count++; /* * Finish the search when the symbol is found for the desired position * or the position is not defined for a non-unique symbol. */ if ((args->pos && (args->count == args->pos)) || (!args->pos && (args->count > 1))) return 1; return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Seth Jennings10273.38%150.00%
Chris J Arges3726.62%150.00%
Total139100.00%2100.00%


static int klp_find_object_symbol(const char *objname, const char *name, unsigned long sympos, unsigned long *addr) { struct klp_find_arg args = { .objname = objname, .name = name, .addr = 0, .count = 0, .pos = sympos, }; mutex_lock(&module_mutex); if (objname) module_kallsyms_on_each_symbol(klp_find_callback, &args); else kallsyms_on_each_symbol(klp_find_callback, &args); mutex_unlock(&module_mutex); /* * Ensure an address was found. If sympos is 0, ensure symbol is unique; * otherwise ensure the symbol position count matches sympos. */ if (args.addr == 0) pr_err("symbol '%s' not found in symbol table\n", name); else if (args.count > 1 && sympos == 0) { pr_err("unresolvable ambiguity for symbol '%s' in object '%s'\n", name, objname); } else if (sympos != args.count && sympos > 0) { pr_err("symbol position %lu for symbol '%s' in object '%s' not found\n", sympos, name, objname ? objname : "vmlinux"); } else { *addr = args.addr; return 0; } *addr = 0; return -EINVAL; }

Contributors

PersonTokensPropCommitsCommitProp
Seth Jennings10759.12%120.00%
Chris J Arges4826.52%120.00%
Zhou Chengming137.18%120.00%
Miroslav Benes126.63%120.00%
Petr Mladek10.55%120.00%
Total181100.00%5100.00%


static int klp_resolve_symbols(Elf_Shdr *relasec, struct module *pmod) { int i, cnt, vmlinux, ret; char objname[MODULE_NAME_LEN]; char symname[KSYM_NAME_LEN]; char *strtab = pmod->core_kallsyms.strtab; Elf_Rela *relas; Elf_Sym *sym; unsigned long sympos, addr; /* * Since the field widths for objname and symname in the sscanf() * call are hard-coded and correspond to MODULE_NAME_LEN and * KSYM_NAME_LEN respectively, we must make sure that MODULE_NAME_LEN * and KSYM_NAME_LEN have the values we expect them to have. * * Because the value of MODULE_NAME_LEN can differ among architectures, * we use the smallest/strictest upper bound possible (56, based on * the current definition of MODULE_NAME_LEN) to prevent overflows. */ BUILD_BUG_ON(MODULE_NAME_LEN < 56 || KSYM_NAME_LEN != 128); relas = (Elf_Rela *) relasec->sh_addr; /* For each rela in this klp relocation section */ for (i = 0; i < relasec->sh_size / sizeof(Elf_Rela); i++) { sym = pmod->core_kallsyms.symtab + ELF_R_SYM(relas[i].r_info); if (sym->st_shndx != SHN_LIVEPATCH) { pr_err("symbol %s is not marked as a livepatch symbol\n", strtab + sym->st_name); return -EINVAL; } /* Format: .klp.sym.objname.symname,sympos */ cnt = sscanf(strtab + sym->st_name, ".klp.sym.%55[^.].%127[^,],%lu", objname, symname, &sympos); if (cnt != 3) { pr_err("symbol %s has an incorrectly formatted name\n", strtab + sym->st_name); return -EINVAL; } /* klp_find_object_symbol() treats a NULL objname as vmlinux */ vmlinux = !strcmp(objname, "vmlinux"); ret = klp_find_object_symbol(vmlinux ? NULL : objname, symname, sympos, &addr); if (ret) return ret; sym->st_value = addr; } return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Jessica Yu19080.17%120.00%
Seth Jennings4217.72%120.00%
Peter Zijlstra20.84%120.00%
Josh Poimboeuf20.84%120.00%
Chris J Arges10.42%120.00%
Total237100.00%5100.00%


static int klp_write_object_relocations(struct module *pmod, struct klp_object *obj) { int i, cnt, ret = 0; const char *objname, *secname; char sec_objname[MODULE_NAME_LEN]; Elf_Shdr *sec; if (WARN_ON(!klp_is_object_loaded(obj))) return -EINVAL; objname = klp_is_module(obj) ? obj->name : "vmlinux"; /* For each klp relocation section */ for (i = 1; i < pmod->klp_info->hdr.e_shnum; i++) { sec = pmod->klp_info->sechdrs + i; secname = pmod->klp_info->secstrings + sec->sh_name; if (!(sec->sh_flags & SHF_RELA_LIVEPATCH)) continue; /* * Format: .klp.rela.sec_objname.section_name * See comment in klp_resolve_symbols() for an explanation * of the selected field width value. */ cnt = sscanf(secname, ".klp.rela.%55[^.]", sec_objname); if (cnt != 1) { pr_err("section %s has an incorrectly formatted name\n", secname); ret = -EINVAL; break; } if (strcmp(objname, sec_objname)) continue; ret = klp_resolve_symbols(sec, pmod); if (ret) break; ret = apply_relocate_add(pmod->klp_info->sechdrs, pmod->core_kallsyms.strtab, pmod->klp_info->symndx, i, pmod); if (ret) break; } return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Jessica Yu11652.97%116.67%
Seth Jennings8739.73%116.67%
Chris J Arges73.20%116.67%
Josh Poimboeuf73.20%233.33%
Zhou Chengming20.91%116.67%
Total219100.00%6100.00%


static int __klp_disable_patch(struct klp_patch *patch) { struct klp_object *obj; if (WARN_ON(!patch->enabled)) return -EINVAL; if (klp_transition_patch) return -EBUSY; /* enforce stacking: only the last enabled patch can be disabled */ if (!list_is_last(&patch->list, &klp_patches) && list_next_entry(patch, list)->enabled) return -EBUSY; klp_init_transition(patch, KLP_UNPATCHED); klp_for_each_object(patch, obj) if (obj->patched) klp_pre_unpatch_callback(obj); /* * Enforce the order of the func->transition writes in * klp_init_transition() and the TIF_PATCH_PENDING writes in * klp_start_transition(). In the rare case where klp_ftrace_handler() * is called shortly after klp_update_patch_state() switches the task, * this ensures the handler sees that func->transition is set. */ smp_wmb(); klp_start_transition(); klp_try_complete_transition(); patch->enabled = false; return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Josh Poimboeuf4944.95%440.00%
Joe Lawrence2220.18%110.00%
Seth Jennings1816.51%110.00%
Petr Mladek1412.84%220.00%
Michael Ellerman32.75%110.00%
Li Bin32.75%110.00%
Total109100.00%10100.00%

/** * klp_disable_patch() - disables a registered patch * @patch: The registered, enabled patch to be disabled * * Unregisters the patched functions from ftrace. * * Return: 0 on success, otherwise error */
int klp_disable_patch(struct klp_patch *patch) { int ret; mutex_lock(&klp_mutex); if (!klp_is_patch_registered(patch)) { ret = -EINVAL; goto err; } if (!patch->enabled) { ret = -EINVAL; goto err; } ret = __klp_disable_patch(patch); err: mutex_unlock(&klp_mutex); return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Seth Jennings6590.28%125.00%
Minfei Huang56.94%125.00%
Josh Poimboeuf22.78%250.00%
Total72100.00%4100.00%

EXPORT_SYMBOL_GPL(klp_disable_patch);
static int __klp_enable_patch(struct klp_patch *patch) { struct klp_object *obj; int ret; if (klp_transition_patch) return -EBUSY; if (WARN_ON(patch->enabled)) return -EINVAL; /* enforce stacking: only the first disabled patch can be enabled */ if (patch->list.prev != &klp_patches && !list_prev_entry(patch, list)->enabled) return -EBUSY; /* * A reference is taken on the patch module to prevent it from being * unloaded. * * Note: For immediate (no consistency model) patches we don't allow * patch modules to unload since there is no safe/sane method to * determine if a thread is still running in the patched code contained * in the patch module once the ftrace registration is successful. */ if (!try_module_get(patch->mod)) return -ENODEV; pr_notice("enabling patch '%s'\n", patch->mod->name); klp_init_transition(patch, KLP_PATCHED); /* * Enforce the order of the func->transition writes in * klp_init_transition() and the ops->func_stack writes in * klp_patch_object(), so that klp_ftrace_handler() will see the * func->transition updates before the handler is registered and the * new funcs become visible to the handler. */ smp_wmb(); klp_for_each_object(patch, obj) { if (!klp_is_object_loaded(obj)) continue; ret = klp_pre_patch_callback(obj); if (ret) { pr_warn("pre-patch callback failed for object '%s'\n", klp_is_module(obj) ? obj->name : "vmlinux"); goto err; } ret = klp_patch_object(obj); if (ret) { pr_warn("failed to patch object '%s'\n", klp_is_module(obj) ? obj->name : "vmlinux"); goto err; } } klp_start_transition(); klp_try_complete_transition(); patch->enabled = true; return 0; err: pr_warn("failed to enable patch '%s'\n", patch->mod->name); klp_cancel_transition(); return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Josh Poimboeuf7936.24%457.14%
Seth Jennings7132.57%114.29%
Joe Lawrence6228.44%114.29%
Jiri Slaby62.75%114.29%
Total218100.00%7100.00%

/** * klp_enable_patch() - enables a registered patch * @patch: The registered, disabled patch to be enabled * * Performs the needed symbol lookups and code relocations, * then registers the patched functions with ftrace. * * Return: 0 on success, otherwise error */
int klp_enable_patch(struct klp_patch *patch) { int ret; mutex_lock(&klp_mutex); if (!klp_is_patch_registered(patch)) { ret = -EINVAL; goto err; } ret = __klp_enable_patch(patch); err: mutex_unlock(&klp_mutex); return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Seth Jennings55100.00%1100.00%
Total55100.00%1100.00%

EXPORT_SYMBOL_GPL(klp_enable_patch); /* * Sysfs Interface * * /sys/kernel/livepatch * /sys/kernel/livepatch/<patch> * /sys/kernel/livepatch/<patch>/enabled * /sys/kernel/livepatch/<patch>/transition * /sys/kernel/livepatch/<patch>/<object> * /sys/kernel/livepatch/<patch>/<object>/<function,sympos> */
static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { struct klp_patch *patch; int ret; bool enabled; ret = kstrtobool(buf, &enabled); if (ret) return ret; patch = container_of(kobj, struct klp_patch, kobj); mutex_lock(&klp_mutex); if (!klp_is_patch_registered(patch)) { /* * Module with the patch could either disappear meanwhile or is * not properly initialized yet. */ ret = -EINVAL; goto err; } if (patch->enabled == enabled) { /* already in requested state */ ret = -EINVAL; goto err; } if (patch == klp_transition_patch) { klp_reverse_transition(); } else if (enabled) { ret = __klp_enable_patch(patch); if (ret) goto err; } else { ret = __klp_disable_patch(patch); if (ret) goto err; } mutex_unlock(&klp_mutex); return count; err: mutex_unlock(&klp_mutex); return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Seth Jennings13877.97%120.00%
Josh Poimboeuf3922.03%480.00%
Total177100.00%5100.00%


static ssize_t enabled_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { struct klp_patch *patch; patch = container_of(kobj, struct klp_patch, kobj); return snprintf(buf, PAGE_SIZE-1, "%d\n", patch->enabled); }

Contributors

PersonTokensPropCommitsCommitProp
Seth Jennings5298.11%150.00%
Josh Poimboeuf11.89%150.00%
Total53100.00%2100.00%


static ssize_t transition_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { struct klp_patch *patch; patch = container_of(kobj, struct klp_patch, kobj); return snprintf(buf, PAGE_SIZE-1, "%d\n", patch == klp_transition_patch); }

Contributors

PersonTokensPropCommitsCommitProp
Seth Jennings5094.34%150.00%
Josh Poimboeuf35.66%150.00%
Total53100.00%2100.00%

static struct kobj_attribute enabled_kobj_attr = __ATTR_RW(enabled); static struct kobj_attribute transition_kobj_attr = __ATTR_RO(transition); static struct attribute *klp_patch_attrs[] = { &enabled_kobj_attr.attr, &transition_kobj_attr.attr, NULL };
static void klp_kobj_release_patch(struct kobject *kobj) { struct klp_patch *patch; patch = container_of(kobj, struct klp_patch, kobj); complete(&patch->finish); }

Contributors

PersonTokensPropCommitsCommitProp
Josh Poimboeuf2569.44%150.00%
Seth Jennings1130.56%150.00%
Total36100.00%2100.00%

static struct kobj_type klp_ktype_patch = { .release = klp_kobj_release_patch, .sysfs_ops = &kobj_sysfs_ops, .default_attrs = klp_patch_attrs, };
static void klp_kobj_release_object(struct kobject *kobj) { }

Contributors

PersonTokensPropCommitsCommitProp
Miroslav Benes10100.00%1100.00%
Total10100.00%1100.00%

static struct kobj_type klp_ktype_object = { .release = klp_kobj_release_object, .sysfs_ops = &kobj_sysfs_ops, };
static void klp_kobj_release_func(struct kobject *kobj) { }

Contributors

PersonTokensPropCommitsCommitProp
Seth Jennings990.00%150.00%
Josh Poimboeuf110.00%150.00%
Total10100.00%2100.00%

static struct kobj_type klp_ktype_func = { .release = klp_kobj_release_func, .sysfs_ops = &kobj_sysfs_ops, }; /* * Free all functions' kobjects in the array up to some limit. When limit is * NULL, all kobjects are freed. */
static void klp_free_funcs_limited(struct klp_object *obj, struct klp_func *limit) { struct klp_func *func; for (func = obj->funcs; func->old_name && func != limit; func++) kobject_put(&func->kobj); }

Contributors

PersonTokensPropCommitsCommitProp
Seth Jennings48100.00%1100.00%
Total48100.00%1100.00%

/* Clean up when a patched object is unloaded */
static void klp_free_object_loaded(struct klp_object *obj) { struct klp_func *func; obj->mod = NULL; klp_for_each_func(obj, func) func->old_addr = 0; }

Contributors

PersonTokensPropCommitsCommitProp
Seth Jennings2882.35%150.00%
Jiri Slaby617.65%150.00%
Total34100.00%2100.00%

/* * Free all objects' kobjects in the array up to some limit. When limit is * NULL, all kobjects are freed. */
static void klp_free_objects_limited(struct klp_patch *patch, struct klp_object *limit) { struct klp_object *obj; for (obj = patch->objs; obj->funcs && obj != limit; obj++) { klp_free_funcs_limited(obj, NULL); kobject_put(&obj->kobj); } }

Contributors

PersonTokensPropCommitsCommitProp
Seth Jennings5698.25%150.00%
Miroslav Benes11.75%150.00%
Total57100.00%2100.00%


static void klp_free_patch(struct klp_patch *patch) { klp_free_objects_limited(patch, NULL); if (!list_empty(&patch->list)) list_del(&patch->list); }

Contributors

PersonTokensPropCommitsCommitProp
Seth Jennings37100.00%1100.00%
Total37100.00%1100.00%


static int klp_init_func(struct klp_object *obj, struct klp_func *func) { if (!func->old_name || !func->new_func) return -EINVAL; INIT_LIST_HEAD(&func->stack_node); func->patched = false; func->transition = false; /* The format for the sysfs directory is <function,sympos> where sympos * is the nth occurrence of this symbol in kallsyms for the patched * object. If the user selects 0 for old_sympos, then 1 will be used * since a unique symbol will be the first occurrence. */ return kobject_init_and_add(&func->kobj, &klp_ktype_func, &obj->kobj, "%s,%lu", func->old_name, func->old_sympos ? func->old_sympos : 1); }

Contributors

PersonTokensPropCommitsCommitProp
Seth Jennings4552.33%112.50%
Miroslav Benes1719.77%225.00%
Chris J Arges1213.95%112.50%
Josh Poimboeuf1112.79%337.50%
Jiri Kosina11.16%112.50%
Total86100.00%8100.00%

/* Arches may override this to finish any remaining arch-specific tasks */
void __weak arch_klp_init_object_loaded(struct klp_patch *patch, struct klp_object *obj) { }

Contributors

PersonTokensPropCommitsCommitProp
Jessica Yu15100.00%1100.00%
Total15100.00%1100.00%

/* parts of the initialization that is done only when the object is loaded */
static int klp_init_object_loaded(struct klp_patch *patch, struct klp_object *obj) { struct klp_func *func; int ret; module_disable_ro(patch->mod); ret = klp_write_object_relocations(patch->mod, obj); if (ret) { module_enable_ro(patch->mod, true); return ret; } arch_klp_init_object_loaded(patch, obj); module_enable_ro(patch->mod, true); klp_for_each_func(obj, func) { ret = klp_find_object_symbol(obj->name, func->old_name, func->old_sympos, &func->old_addr); if (ret) return ret; ret = kallsyms_lookup_size_offset(func->old_addr, &func->old_size, NULL); if (!ret) { pr_err("kallsyms size lookup failed for '%s'\n", func->old_name); return -ENOENT; } ret = kallsyms_lookup_size_offset((unsigned long)func->new_func, &func->new_size, NULL); if (!ret) { pr_err("kallsyms size lookup failed for '%s' replacement\n", func->old_name); return -ENOENT; } } return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Josh Poimboeuf7639.58%120.00%
Seth Jennings6232.29%120.00%
Jessica Yu3417.71%120.00%
Chris J Arges147.29%120.00%
Jiri Slaby63.12%120.00%
Total192100.00%5100.00%


static int klp_init_object(struct klp_patch *patch, struct klp_object *obj) { struct klp_func *func; int ret; const char *name; if (!obj->funcs) return -EINVAL; obj->patched = false; obj->mod = NULL; klp_find_object_module(obj); name = klp_is_module(obj) ? obj->name : "vmlinux"; ret = kobject_init_and_add(&obj->kobj, &klp_ktype_object, &patch->kobj, "%s", name); if (ret) return ret; klp_for_each_func(obj, func) { ret = klp_init_func(obj, func); if (ret) goto free; } if (klp_is_object_loaded(obj)) { ret = klp_init_object_loaded(patch, obj); if (ret) goto free; } return 0; free: klp_free_funcs_limited(obj, func); kobject_put(&obj->kobj); return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Seth Jennings14283.04%120.00%
Miroslav Benes158.77%120.00%
Petr Mladek63.51%120.00%
Jiri Slaby63.51%120.00%
Josh Poimboeuf21.17%120.00%
Total171100.00%5100.00%


static int klp_init_patch(struct klp_patch *patch) { struct klp_object *obj; int ret; if (!patch->objs) return -EINVAL; mutex_lock(&klp_mutex); patch->enabled = false; init_completion(&patch->finish); ret = kobject_init_and_add(&patch->kobj, &klp_ktype_patch, klp_root_kobj, "%s", patch->mod->name); if (ret) { mutex_unlock(&klp_mutex); return ret; } klp_for_each_object(patch, obj) { ret = klp_init_object(patch, obj); if (ret) goto free; } list_add_tail(&patch->list, &klp_patches); mutex_unlock(&klp_mutex); return 0; free: klp_free_objects_limited(patch, obj); mutex_unlock(&klp_mutex); kobject_put(&patch->kobj); wait_for_completion(&patch->finish); return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Seth Jennings12776.51%116.67%
Josh Poimboeuf3118.67%350.00%
Jiri Slaby63.61%116.67%
Jiri Kosina21.20%116.67%
Total166100.00%6100.00%

/** * klp_unregister_patch() - unregisters a patch * @patch: Disabled patch to be unregistered * * Frees the data structures and removes the sysfs interface. * * Return: 0 on success, otherwise error */
int klp_unregister_patch(struct klp_patch *patch) { int ret; mutex_lock(&klp_mutex); if (!klp_is_patch_registered(patch)) { ret