cregit-Linux how code gets into the kernel

Release 4.15 kernel/sched/membarrier.c

Directory: kernel/sched
 * Copyright (C) 2010-2017 Mathieu Desnoyers <>
 * membarrier system call
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * GNU General Public License for more details.

#include <linux/syscalls.h>
#include <linux/membarrier.h>
#include <linux/tick.h>
#include <linux/cpumask.h>
#include <linux/atomic.h>

#include "sched.h"	/* for cpu_rq(). */

 * Bitmask made from a "or" of all commands within enum membarrier_cmd,


static void ipi_mb(void *info) { smp_mb(); /* IPIs should be serializing but paranoid. */ }


Mathieu Desnoyers14100.00%1100.00%

static int membarrier_private_expedited(void) { int cpu; bool fallback = false; cpumask_var_t tmpmask; if (!(atomic_read(&current->mm->membarrier_state) & MEMBARRIER_STATE_PRIVATE_EXPEDITED_READY)) return -EPERM; if (num_online_cpus() == 1) return 0; /* * Matches memory barriers around rq->curr modification in * scheduler. */ smp_mb(); /* system call entry is not a mb. */ /* * Expedited membarrier commands guarantee that they won't * block, hence the GFP_NOWAIT allocation flag and fallback * implementation. */ if (!zalloc_cpumask_var(&tmpmask, GFP_NOWAIT)) { /* Fallback for OOM. */ fallback = true; } cpus_read_lock(); for_each_online_cpu(cpu) { struct task_struct *p; /* * Skipping the current CPU is OK even through we can be * migrated at any point. The current CPU, at the point * where we read raw_smp_processor_id(), is ensured to * be in program order with respect to the caller * thread. Therefore, we can skip this CPU from the * iteration. */ if (cpu == raw_smp_processor_id()) continue; rcu_read_lock(); p = task_rcu_dereference(&cpu_rq(cpu)->curr); if (p && p->mm == current->mm) { if (!fallback) __cpumask_set_cpu(cpu, tmpmask); else smp_call_function_single(cpu, ipi_mb, NULL, 1); } rcu_read_unlock(); } if (!fallback) { preempt_disable(); smp_call_function_many(tmpmask, ipi_mb, NULL, 1); preempt_enable(); free_cpumask_var(tmpmask); } cpus_read_unlock(); /* * Memory barrier on the caller thread _after_ we finished * waiting for the last IPI. Matches memory barriers around * rq->curr modification in scheduler. */ smp_mb(); /* exit from system call is not a mb */ return 0; }


Mathieu Desnoyers194100.00%3100.00%

static void membarrier_register_private_expedited(void) { struct task_struct *p = current; struct mm_struct *mm = p->mm; /* * We need to consider threads belonging to different thread * groups, which use the same mm. (CLONE_VM but not * CLONE_THREAD). */ if (atomic_read(&mm->membarrier_state) & MEMBARRIER_STATE_PRIVATE_EXPEDITED_READY) return; atomic_or(MEMBARRIER_STATE_PRIVATE_EXPEDITED_READY, &mm->membarrier_state); }


Mathieu Desnoyers48100.00%2100.00%

/** * sys_membarrier - issue memory barriers on a set of threads * @cmd: Takes command values defined in enum membarrier_cmd. * @flags: Currently needs to be 0. For future extensions. * * If this system call is not implemented, -ENOSYS is returned. If the * command specified does not exist, not available on the running * kernel, or if the command argument is invalid, this system call * returns -EINVAL. For a given command, with flags argument set to 0, * this system call is guaranteed to always return the same value until * reboot. * * All memory accesses performed in program order from each targeted thread * is guaranteed to be ordered with respect to sys_membarrier(). If we use * the semantic "barrier()" to represent a compiler barrier forcing memory * accesses to be performed in program order across the barrier, and * smp_mb() to represent explicit memory barriers forcing full memory * ordering across the barrier, we have the following ordering table for * each pair of barrier(), sys_membarrier() and smp_mb(): * * The pair ordering is detailed as (O: ordered, X: not ordered): * * barrier() smp_mb() sys_membarrier() * barrier() X X O * smp_mb() X O O * sys_membarrier() O O O */ SYSCALL_DEFINE2(membarrier, int, cmd, int, flags) { if (unlikely(flags)) return -EINVAL; switch (cmd) { case MEMBARRIER_CMD_QUERY: { int cmd_mask = MEMBARRIER_CMD_BITMASK; if (tick_nohz_full_enabled()) cmd_mask &= ~MEMBARRIER_CMD_SHARED; return cmd_mask; } case MEMBARRIER_CMD_SHARED: /* MEMBARRIER_CMD_SHARED is not compatible with nohz_full. */ if (tick_nohz_full_enabled()) return -EINVAL; if (num_online_cpus() > 1) synchronize_sched(); return 0; case MEMBARRIER_CMD_PRIVATE_EXPEDITED: return membarrier_private_expedited(); case MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED: membarrier_register_private_expedited(); return 0; default: return -EINVAL; } }

Overall Contributors

Mathieu Desnoyers384100.00%3100.00%
Directory: kernel/sched
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with cregit.