*.bc
*.bin
*.bz2
+*.c.[012]*.*
*.cis
*.cpio
*.csp
+*.dbg
*.dsp
*.dvi
*.elf
*.gcov
*.gen.S
*.gif
+*.gmo
*.grep
*.grp
*.gz
*.tab.h
*.tex
*.ver
+*.vim
*.xml
*.xz
*_MODULES
+*_reg_safe.h
*_vga16.c
*~
\#*#
*.9
-.*
+.[^g]*
+.gen*
.*.d
.mm
53c700_d.h
Module.markers
Module.symvers
PENDING
+PERF*
SCCS
System.map*
TAGS
+TRACEEVENT-CFLAGS
aconf
af_names.h
aic7*reg.h*
aicasm
aicdb.h*
altivec*.c
+ashldi3.S
asm-offsets.h
asm_offsets.h
autoconf.h*
bsetup
btfixupprep
build
+builtin-policy.h
bvmlinux
bzImage*
capability_names.h
capflags.c
classlist.h*
+clut_vga16.c
+common-cmds.h
comp*.log
compile.h*
conf
config
config-*
config_data.h*
+config.c
config.mak
config.mak.autogen
+config.tmp
conmakehash
consolemap_deftbl.c*
cpustr.h
crc32table.h*
cscope.*
defkeymap.c
+devicetable-offsets.h
devlist.h*
dnotify_test
docproc
dslm
+dtc-lexer.lex.c
elf2ecoff
elfconfig.h*
evergreen_reg_safe.h
+exception_policy.conf
fixdep
flask.h
fore200e_mkfirm
gconf
gconf.glade.h
gen-devlist
+gen-kdb_cmds.c
gen_crc32table
gen_init_cpio
generated
genheaders
genksyms
*_gray256.c
+hash
+hid-example
hpet_example
hugepage-mmap
hugepage-shm
int4.c
int8.c
kallsyms
-kconfig
+kern_constants.h
keywords.c
ksym.c*
ksym.h*
kxgettext
lex.c
lex.*.c
-linux
+lib1funcs.S
logo_*.c
logo_*_clut224.c
logo_*_mono.c
machtypes.h
map
map_hugetlb
-media
mconf
+mdp
miboot*
mk_elfconfig
mkboot
mkbugboot
mkcpustr
mkdep
+mkpiggy
mkprep
mkregtable
mktables
page-types
parse.c
parse.h
+parse-events*
+pasyms.h
patches*
pca200e.bin
pca200e_ecd.bin2
piggyback
piggy.gzip
piggy.S
+pmu-*
pnmtologo
ppc_defs.h*
pss_boot.h
r300_reg_safe.h
r420_reg_safe.h
r600_reg_safe.h
+randomize_layout_hash.h
+randomize_layout_seed.h
+realmode.lds
+realmode.relocs
recordmcount
+regdb.c
relocs
rlim_names.h
rn50_reg_safe.h
setup
setup.bin
setup.elf
+signing_key*
+size_overflow_hash.h
sImage
+slabinfo
sm_tbl*
+sortextable
split-include
syscalltab.h
tables.c
timeconst.h
times.h*
trix_boot.h
+user_constants.h
utsrelease.h*
vdso-syms.lds
vdso.lds
vdso32.so.dbg
vdso64.lds
vdso64.so.dbg
+vdsox32.lds
+vdsox32-syms.lds
version.h*
vmImage
vmlinux
vmlinux-*
vmlinux.aout
vmlinux.bin.all
+vmlinux.bin.bz2
vmlinux.lds
+vmlinux.relocs
vmlinuz
voffset.h
vsyscall.lds
wanxlfw.inc
uImage
unifdef
+utsrelease.h
wakeup.bin
wakeup.elf
wakeup.lds
+x509*
zImage*
zconf.hash.c
+zconf.lex.c
zoffset.h
=== 4 Host Program support
--- 4.1 Simple Host Program
--- 4.2 Composite Host Programs
- --- 4.3 Using C++ for host programs
- --- 4.4 Controlling compiler options for host programs
- --- 4.5 When host programs are actually built
- --- 4.6 Using hostprogs-$(CONFIG_FOO)
+ --- 4.3 Defining shared libraries
+ --- 4.4 Using C++ for host programs
+ --- 4.5 Controlling compiler options for host programs
+ --- 4.6 When host programs are actually built
+ --- 4.7 Using hostprogs-$(CONFIG_FOO)
=== 5 Kbuild clean infrastructure
Finally, the two .o files are linked to the executable, lxdialog.
Note: The syntax <executable>-y is not permitted for host-programs.
---- 4.3 Using C++ for host programs
+--- 4.3 Defining shared libraries
+
+ Objects with extension .so are considered shared libraries, and
+ will be compiled as position independent objects.
+ Kbuild provides support for shared libraries, but the usage
+ shall be restricted.
+ In the following example the libkconfig.so shared library is used
+ to link the executable conf.
+
+ Example:
+ #scripts/kconfig/Makefile
+ hostprogs-y := conf
+ conf-objs := conf.o libkconfig.so
+ libkconfig-objs := expr.o type.o
+
+ Shared libraries always require a corresponding -objs line, and
+ in the example above the shared library libkconfig is composed by
+ the two objects expr.o and type.o.
+ expr.o and type.o will be built as position independent code and
+ linked as a shared library libkconfig.so. C++ is not supported for
+ shared libraries.
+
+--- 4.4 Using C++ for host programs
kbuild offers support for host programs written in C++. This was
introduced solely to support kconfig, and is not recommended
qconf-cxxobjs := qconf.o
qconf-objs := check.o
---- 4.4 Controlling compiler options for host programs
+--- 4.5 Controlling compiler options for host programs
When compiling host programs, it is possible to set specific flags.
The programs will always be compiled utilising $(HOSTCC) passed
When linking qconf, it will be passed the extra option
"-L$(QTDIR)/lib".
---- 4.5 When host programs are actually built
+--- 4.6 When host programs are actually built
Kbuild will only build host-programs when they are referenced
as a prerequisite.
This will tell kbuild to build lxdialog even if not referenced in
any rule.
---- 4.6 Using hostprogs-$(CONFIG_FOO)
+--- 4.7 Using hostprogs-$(CONFIG_FOO)
A typical pattern in a Kbuild file looks like this:
Format: <unsigned int> such that (rxsize & ~0x1fffc0) == 0.
Default: 1024
+ grsec_proc_gid= [GRKERNSEC_PROC_USERGROUP] Chooses GID to
+ ignore grsecurity's /proc restrictions
+
+
hashdist= [KNL,NUMA] Large hashes allocated during boot
are distributed across NUMA nodes. Defaults on
for 64-bit NUMA, off otherwise.
noexec=on: enable non-executable mappings (default)
noexec=off: disable non-executable mappings
+ nopcid [X86-64]
+ Disable PCID (Process-Context IDentifier) even if it
+ is supported by the processor.
+
nosmap [X86]
Disable SMAP (Supervisor Mode Access Prevention)
even if it is supported by processor.
the specified number of seconds. This is to be used if
your oopses keep scrolling off the screen.
+ pax_nouderef [X86] disables UDEREF. Most likely needed under certain
+ virtualization environments that don't cope well with the
+ expand down segment used by UDEREF on X86-32 or the frequent
+ page table updates on X86-64.
+
+ pax_sanitize_slab=
+ Format: { 0 | 1 | off | fast | full }
+ Options '0' and '1' are only provided for backward
+ compatibility, 'off' or 'fast' should be used instead.
+ 0|off : disable slab object sanitization
+ 1|fast: enable slab object sanitization excluding
+ whitelisted slabs (default)
+ full : sanitize all slabs, even the whitelisted ones
+
+ pax_softmode= 0/1 to disable/enable PaX softmode on boot already.
+
+ pax_extra_latent_entropy
+ Enable a very simple form of latent entropy extraction
+ from the first 4GB of memory as the bootmem allocator
+ passes the memory pages to the buddy allocator.
+
+ pax_weakuderef [X86-64] enables the weaker but faster form of UDEREF
+ when the processor supports PCID.
+
pcbit= [HW,ISDN]
pcd. [PARIDE]
HOSTCC = gcc
HOSTCXX = g++
HOSTCFLAGS = -Wall -Wmissing-prototypes -Wstrict-prototypes -O2 -fomit-frame-pointer -std=gnu89
-HOSTCXXFLAGS = -O2
+HOSTCFLAGS = -Wall -W -Wmissing-prototypes -Wstrict-prototypes -Wno-unused-parameter -Wno-missing-field-initializers -O2 -fomit-frame-pointer -std=gnu89 -fno-delete-null-pointer-checks
+HOSTCFLAGS += $(call cc-option, -Wno-empty-body)
+HOSTCXXFLAGS = -O2 -Wall -W -Wno-array-bounds
ifeq ($(shell $(HOSTCC) -v 2>&1 | grep -c "clang version"), 1)
HOSTCFLAGS += -Wno-unused-value -Wno-unused-parameter \
# Rules shared between *config targets and build targets
# Basic helpers built in scripts/
-PHONY += scripts_basic
-scripts_basic:
+PHONY += scripts_basic gcc-plugins
+scripts_basic: gcc-plugins
$(Q)$(MAKE) $(build)=scripts/basic
$(Q)rm -f .tmp_quiet_recordmcount
# Tell gcc to never replace conditional load with a non-conditional one
KBUILD_CFLAGS += $(call cc-option,--param=allow-store-data-races=0)
+ifndef DISABLE_PAX_PLUGINS
+ifeq ($(call cc-ifversion, -ge, 0408, y), y)
+PLUGINCC := $(shell $(CONFIG_SHELL) $(srctree)/scripts/gcc-plugin.sh "$(HOSTCXX)" "$(HOSTCXX)" "$(CC)")
+else
+PLUGINCC := $(shell $(CONFIG_SHELL) $(srctree)/scripts/gcc-plugin.sh "$(HOSTCC)" "$(HOSTCXX)" "$(CC)")
+endif
+ifneq ($(PLUGINCC),)
+ifdef CONFIG_PAX_CONSTIFY_PLUGIN
+CONSTIFY_PLUGIN_CFLAGS := -fplugin=$(objtree)/tools/gcc/constify_plugin.so -DCONSTIFY_PLUGIN
+endif
+ifdef CONFIG_PAX_MEMORY_STACKLEAK
+STACKLEAK_PLUGIN_CFLAGS := -fplugin=$(objtree)/tools/gcc/stackleak_plugin.so -DSTACKLEAK_PLUGIN
+STACKLEAK_PLUGIN_CFLAGS += -fplugin-arg-stackleak_plugin-track-lowest-sp=100
+endif
+ifdef CONFIG_KALLOCSTAT_PLUGIN
+KALLOCSTAT_PLUGIN_CFLAGS := -fplugin=$(objtree)/tools/gcc/kallocstat_plugin.so
+endif
+ifdef CONFIG_PAX_KERNEXEC_PLUGIN
+KERNEXEC_PLUGIN_CFLAGS := -fplugin=$(objtree)/tools/gcc/kernexec_plugin.so
+KERNEXEC_PLUGIN_CFLAGS += -fplugin-arg-kernexec_plugin-method=$(CONFIG_PAX_KERNEXEC_PLUGIN_METHOD) -DKERNEXEC_PLUGIN
+KERNEXEC_PLUGIN_AFLAGS := -DKERNEXEC_PLUGIN
+endif
+ifdef CONFIG_GRKERNSEC_RANDSTRUCT
+RANDSTRUCT_PLUGIN_CFLAGS := -fplugin=$(objtree)/tools/gcc/randomize_layout_plugin.so -DRANDSTRUCT_PLUGIN
+ifdef CONFIG_GRKERNSEC_RANDSTRUCT_PERFORMANCE
+RANDSTRUCT_PLUGIN_CFLAGS += -fplugin-arg-randomize_layout_plugin-performance-mode
+endif
+endif
+ifdef CONFIG_CHECKER_PLUGIN
+ifeq ($(call cc-ifversion, -ge, 0406, y), y)
+CHECKER_PLUGIN_CFLAGS := -fplugin=$(objtree)/tools/gcc/checker_plugin.so -DCHECKER_PLUGIN
+endif
+endif
+COLORIZE_PLUGIN_CFLAGS := -fplugin=$(objtree)/tools/gcc/colorize_plugin.so
+ifdef CONFIG_PAX_SIZE_OVERFLOW
+SIZE_OVERFLOW_PLUGIN_CFLAGS := -fplugin=$(objtree)/tools/gcc/size_overflow_plugin/size_overflow_plugin.so -DSIZE_OVERFLOW_PLUGIN
+endif
+ifdef CONFIG_PAX_LATENT_ENTROPY
+LATENT_ENTROPY_PLUGIN_CFLAGS := -fplugin=$(objtree)/tools/gcc/latent_entropy_plugin.so -DLATENT_ENTROPY_PLUGIN
+endif
+ifdef CONFIG_PAX_MEMORY_STRUCTLEAK
+STRUCTLEAK_PLUGIN_CFLAGS := -fplugin=$(objtree)/tools/gcc/structleak_plugin.so -DSTRUCTLEAK_PLUGIN
+endif
+GCC_PLUGINS_CFLAGS := $(CONSTIFY_PLUGIN_CFLAGS) $(STACKLEAK_PLUGIN_CFLAGS) $(KALLOCSTAT_PLUGIN_CFLAGS)
+GCC_PLUGINS_CFLAGS += $(KERNEXEC_PLUGIN_CFLAGS) $(CHECKER_PLUGIN_CFLAGS) $(COLORIZE_PLUGIN_CFLAGS)
+GCC_PLUGINS_CFLAGS += $(SIZE_OVERFLOW_PLUGIN_CFLAGS) $(LATENT_ENTROPY_PLUGIN_CFLAGS) $(STRUCTLEAK_PLUGIN_CFLAGS)
+GCC_PLUGINS_CFLAGS += $(RANDSTRUCT_PLUGIN_CFLAGS)
+GCC_PLUGINS_AFLAGS := $(KERNEXEC_PLUGIN_AFLAGS)
+export PLUGINCC GCC_PLUGINS_CFLAGS GCC_PLUGINS_AFLAGS CONSTIFY_PLUGIN LATENT_ENTROPY_PLUGIN_CFLAGS
+ifeq ($(KBUILD_EXTMOD),)
+gcc-plugins:
+ $(Q)$(MAKE) $(build)=tools/gcc
+else
+gcc-plugins: ;
+endif
+else
+gcc-plugins:
+ifeq ($(call cc-ifversion, -ge, 0405, y), y)
+ $(error Your gcc installation does not support plugins. If the necessary headers for plugin support are missing, they should be installed. On Debian, apt-get install gcc-<ver>-plugin-dev. If you choose to ignore this error and lessen the improvements provided by this patch, re-run make with the DISABLE_PAX_PLUGINS=y argument.))
+else
+ $(Q)echo "warning, your gcc version does not support plugins, you should upgrade it to gcc 4.5 at least"
+endif
+ $(Q)echo "PAX_MEMORY_STACKLEAK, constification, PAX_LATENT_ENTROPY and other features will be less secure. PAX_SIZE_OVERFLOW will not be active."
+endif
+endif
+
ifdef CONFIG_READABLE_ASM
# Disable optimizations that make assembler listings hard to read.
# reorder blocks reorders the control in the function
else
KBUILD_CFLAGS += -g
endif
-KBUILD_AFLAGS += -Wa,-gdwarf-2
+KBUILD_AFLAGS += -Wa,--gdwarf-2
endif
ifdef CONFIG_DEBUG_INFO_DWARF4
KBUILD_CFLAGS += $(call cc-option, -gdwarf-4,)
ifeq ($(KBUILD_EXTMOD),)
-core-y += kernel/ mm/ fs/ ipc/ security/ crypto/ block/
+core-y += kernel/ mm/ fs/ ipc/ security/ crypto/ block/ grsecurity/
vmlinux-dirs := $(patsubst %/,%,$(filter %/, $(init-y) $(init-m) \
$(core-y) $(core-m) $(drivers-y) $(drivers-m) \
# The actual objects are generated when descending,
# make sure no implicit rule kicks in
+$(filter-out $(init-y),$(vmlinux-deps)): KBUILD_CFLAGS += $(GCC_PLUGINS_CFLAGS)
+$(filter-out $(init-y),$(vmlinux-deps)): KBUILD_AFLAGS += $(GCC_PLUGINS_AFLAGS)
$(sort $(vmlinux-deps)): $(vmlinux-dirs) ;
# Handle descending into subdirectories listed in $(vmlinux-dirs)
# Error messages still appears in the original language
PHONY += $(vmlinux-dirs)
-$(vmlinux-dirs): prepare scripts
+$(vmlinux-dirs): gcc-plugins prepare scripts
$(Q)$(MAKE) $(build)=$@
define filechk_kernel.release
archprepare: archheaders archscripts prepare1 scripts_basic
+prepare0: KBUILD_CFLAGS += $(GCC_PLUGINS_CFLAGS)
+prepare0: KBUILD_AFLAGS += $(GCC_PLUGINS_AFLAGS)
prepare0: archprepare FORCE
$(Q)$(MAKE) $(build)=.
# All the preparing..
+prepare: KBUILD_CFLAGS := $(filter-out $(GCC_PLUGINS_CFLAGS),$(KBUILD_CFLAGS))
prepare: prepare0
# Generate some files
# using awk while concatenating to the final file.
PHONY += modules
+modules: KBUILD_CFLAGS += $(GCC_PLUGINS_CFLAGS)
+modules: KBUILD_AFLAGS += $(GCC_PLUGINS_AFLAGS)
modules: $(vmlinux-dirs) $(if $(KBUILD_BUILTIN),vmlinux) modules.builtin
$(Q)$(AWK) '!x[$$0]++' $(vmlinux-dirs:%=$(objtree)/%/modules.order) > $(objtree)/modules.order
@$(kecho) ' Building modules, stage 2.';
# Target to prepare building external modules
PHONY += modules_prepare
-modules_prepare: prepare scripts
+modules_prepare: gcc-plugins prepare scripts
# Target to install modules
PHONY += modules_install
Module.symvers tags TAGS cscope* GPATH GTAGS GRTAGS GSYMS \
signing_key.priv signing_key.x509 x509.genkey \
extra_certificates signing_key.x509.keyid \
- signing_key.x509.signer
+ signing_key.x509.signer \
+ tools/gcc/size_overflow_plugin/size_overflow_hash_aux.h \
+ tools/gcc/size_overflow_plugin/size_overflow_hash.h \
+ tools/gcc/randomize_layout_seed.h
# clean - Delete most, but leave enough to build external modules
#
@find $(srctree) $(RCS_FIND_IGNORE) \
\( -name '*.orig' -o -name '*.rej' -o -name '*~' \
-o -name '*.bak' -o -name '#*#' -o -name '.*.orig' \
- -o -name '.*.rej' -o -name '*%' -o -name 'core' \) \
+ -o -name '.*.rej' -o -name '*.so' -o -name '*%' -o -name 'core' \) \
-type f -print | xargs rm -f
$(module-dirs): crmodverdir $(objtree)/Module.symvers
$(Q)$(MAKE) $(build)=$(patsubst _module_%,%,$@)
+modules: KBUILD_CFLAGS += $(GCC_PLUGINS_CFLAGS)
+modules: KBUILD_AFLAGS += $(GCC_PLUGINS_AFLAGS)
modules: $(module-dirs)
@$(kecho) ' Building modules, stage 2.';
$(Q)$(MAKE) -f $(srctree)/scripts/Makefile.modpost
target-dir = $(if $(KBUILD_EXTMOD),$(dir $<),$(dir $@))
endif
-%.s: %.c prepare scripts FORCE
+%.s: KBUILD_CFLAGS += $(GCC_PLUGINS_CFLAGS)
+%.s: KBUILD_AFLAGS += $(GCC_PLUGINS_AFLAGS)
+%.s: %.c gcc-plugins prepare scripts FORCE
$(Q)$(MAKE) $(build)=$(build-dir) $(target-dir)$(notdir $@)
%.i: %.c prepare scripts FORCE
$(Q)$(MAKE) $(build)=$(build-dir) $(target-dir)$(notdir $@)
-%.o: %.c prepare scripts FORCE
+%.o: KBUILD_CFLAGS += $(GCC_PLUGINS_CFLAGS)
+%.o: KBUILD_AFLAGS += $(GCC_PLUGINS_AFLAGS)
+%.o: %.c gcc-plugins prepare scripts FORCE
$(Q)$(MAKE) $(build)=$(build-dir) $(target-dir)$(notdir $@)
%.lst: %.c prepare scripts FORCE
$(Q)$(MAKE) $(build)=$(build-dir) $(target-dir)$(notdir $@)
-%.s: %.S prepare scripts FORCE
+%.s: %.S gcc-plugins prepare scripts FORCE
$(Q)$(MAKE) $(build)=$(build-dir) $(target-dir)$(notdir $@)
-%.o: %.S prepare scripts FORCE
+%.o: %.S gcc-plugins prepare scripts FORCE
$(Q)$(MAKE) $(build)=$(build-dir) $(target-dir)$(notdir $@)
%.symtypes: %.c prepare scripts FORCE
$(Q)$(MAKE) $(build)=$(build-dir) $(target-dir)$(notdir $@)
$(build)=$(build-dir)
# Make sure the latest headers are built for Documentation
Documentation/: headers_install
-%/: prepare scripts FORCE
+%/: KBUILD_CFLAGS += $(GCC_PLUGINS_CFLAGS)
+%/: KBUILD_AFLAGS += $(GCC_PLUGINS_AFLAGS)
+%/: gcc-plugins prepare scripts FORCE
$(cmd_crmodverdir)
$(Q)$(MAKE) KBUILD_MODULES=$(if $(CONFIG_MODULES),1) \
$(build)=$(build-dir)
-%.ko: prepare scripts FORCE
+%.ko: KBUILD_CFLAGS += $(GCC_PLUGINS_CFLAGS)
+%.ko: KBUILD_AFLAGS += $(GCC_PLUGINS_AFLAGS)
+%.ko: gcc-plugins prepare scripts FORCE
$(cmd_crmodverdir)
$(Q)$(MAKE) KBUILD_MODULES=$(if $(CONFIG_MODULES),1) \
$(build)=$(build-dir) $(@:.ko=.o)
#define atomic_dec(v) atomic_sub(1,(v))
#define atomic64_dec(v) atomic64_sub(1,(v))
+#define atomic64_read_unchecked(v) atomic64_read(v)
+#define atomic64_set_unchecked(v, i) atomic64_set((v), (i))
+#define atomic64_add_unchecked(a, v) atomic64_add((a), (v))
+#define atomic64_add_return_unchecked(a, v) atomic64_add_return((a), (v))
+#define atomic64_sub_unchecked(a, v) atomic64_sub((a), (v))
+#define atomic64_inc_unchecked(v) atomic64_inc(v)
+#define atomic64_inc_return_unchecked(v) atomic64_inc_return(v)
+#define atomic64_dec_unchecked(v) atomic64_dec(v)
+#define atomic64_cmpxchg_unchecked(v, o, n) atomic64_cmpxchg((v), (o), (n))
+
#endif /* _ALPHA_ATOMIC_H */
#ifndef __ARCH_ALPHA_CACHE_H
#define __ARCH_ALPHA_CACHE_H
+#include <linux/const.h>
/* Bytes per L1 (data) cache line. */
#if defined(CONFIG_ALPHA_GENERIC) || defined(CONFIG_ALPHA_EV6)
-# define L1_CACHE_BYTES 64
# define L1_CACHE_SHIFT 6
#else
/* Both EV4 and EV5 are write-through, read-allocate,
direct-mapped, physical.
*/
-# define L1_CACHE_BYTES 32
# define L1_CACHE_SHIFT 5
#endif
+#define L1_CACHE_BYTES (_AC(1,UL) << L1_CACHE_SHIFT)
#define SMP_CACHE_BYTES L1_CACHE_BYTES
#endif
#define ELF_ET_DYN_BASE (TASK_UNMAPPED_BASE + 0x1000000)
+#ifdef CONFIG_PAX_ASLR
+#define PAX_ELF_ET_DYN_BASE (current->personality & ADDR_LIMIT_32BIT ? 0x10000 : 0x120000000UL)
+
+#define PAX_DELTA_MMAP_LEN (current->personality & ADDR_LIMIT_32BIT ? 14 : 28)
+#define PAX_DELTA_STACK_LEN (current->personality & ADDR_LIMIT_32BIT ? 14 : 19)
+#endif
+
/* $0 is set by ld.so to a pointer to a function which might be
registered using atexit. This provides a mean for the dynamic
linker to call DT_FINI functions for shared libraries that have
pgd_set(pgd, pmd);
}
+static inline void
+pgd_populate_kernel(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmd)
+{
+ pgd_populate(mm, pgd, pmd);
+}
+
extern pgd_t *pgd_alloc(struct mm_struct *mm);
static inline void
#define PAGE_SHARED __pgprot(_PAGE_VALID | __ACCESS_BITS)
#define PAGE_COPY __pgprot(_PAGE_VALID | __ACCESS_BITS | _PAGE_FOW)
#define PAGE_READONLY __pgprot(_PAGE_VALID | __ACCESS_BITS | _PAGE_FOW)
+
+#ifdef CONFIG_PAX_PAGEEXEC
+# define PAGE_SHARED_NOEXEC __pgprot(_PAGE_VALID | __ACCESS_BITS | _PAGE_FOE)
+# define PAGE_COPY_NOEXEC __pgprot(_PAGE_VALID | __ACCESS_BITS | _PAGE_FOW | _PAGE_FOE)
+# define PAGE_READONLY_NOEXEC __pgprot(_PAGE_VALID | __ACCESS_BITS | _PAGE_FOW | _PAGE_FOE)
+#else
+# define PAGE_SHARED_NOEXEC PAGE_SHARED
+# define PAGE_COPY_NOEXEC PAGE_COPY
+# define PAGE_READONLY_NOEXEC PAGE_READONLY
+#endif
+
#define PAGE_KERNEL __pgprot(_PAGE_VALID | _PAGE_ASM | _PAGE_KRE | _PAGE_KWE)
#define _PAGE_NORMAL(x) __pgprot(_PAGE_VALID | __ACCESS_BITS | (x))
/* The small sections were sorted to the end of the segment.
The following should definitely cover them. */
- gp = (u64)me->module_core + me->core_size - 0x8000;
+ gp = (u64)me->module_core_rw + me->core_size_rw - 0x8000;
got = sechdrs[me->arch.gotsecindex].sh_addr;
for (i = 0; i < n; i++) {
generic version except that we know how to honor ADDR_LIMIT_32BIT. */
static unsigned long
-arch_get_unmapped_area_1(unsigned long addr, unsigned long len,
- unsigned long limit)
+arch_get_unmapped_area_1(struct file *filp, unsigned long addr, unsigned long len,
+ unsigned long limit, unsigned long flags)
{
struct vm_unmapped_area_info info;
+ unsigned long offset = gr_rand_threadstack_offset(current->mm, filp, flags);
info.flags = 0;
info.length = len;
info.high_limit = limit;
info.align_mask = 0;
info.align_offset = 0;
+ info.threadstack_offset = offset;
return vm_unmapped_area(&info);
}
merely specific addresses, but regions of memory -- perhaps
this feature should be incorporated into all ports? */
+#ifdef CONFIG_PAX_RANDMMAP
+ if (!(current->mm->pax_flags & MF_PAX_RANDMMAP))
+#endif
+
if (addr) {
- addr = arch_get_unmapped_area_1 (PAGE_ALIGN(addr), len, limit);
+ addr = arch_get_unmapped_area_1 (filp, PAGE_ALIGN(addr), len, limit, flags);
if (addr != (unsigned long) -ENOMEM)
return addr;
}
/* Next, try allocating at TASK_UNMAPPED_BASE. */
- addr = arch_get_unmapped_area_1 (PAGE_ALIGN(TASK_UNMAPPED_BASE),
- len, limit);
+ addr = arch_get_unmapped_area_1 (filp, PAGE_ALIGN(current->mm->mmap_base), len, limit, flags);
+
if (addr != (unsigned long) -ENOMEM)
return addr;
/* Finally, try allocating in low memory. */
- addr = arch_get_unmapped_area_1 (PAGE_SIZE, len, limit);
+ addr = arch_get_unmapped_area_1 (filp, PAGE_SIZE, len, limit, flags);
return addr;
}
__reload_thread(pcb);
}
+#ifdef CONFIG_PAX_PAGEEXEC
+/*
+ * PaX: decide what to do with offenders (regs->pc = fault address)
+ *
+ * returns 1 when task should be killed
+ * 2 when patched PLT trampoline was detected
+ * 3 when unpatched PLT trampoline was detected
+ */
+static int pax_handle_fetch_fault(struct pt_regs *regs)
+{
+
+#ifdef CONFIG_PAX_EMUPLT
+ int err;
+
+ do { /* PaX: patched PLT emulation #1 */
+ unsigned int ldah, ldq, jmp;
+
+ err = get_user(ldah, (unsigned int *)regs->pc);
+ err |= get_user(ldq, (unsigned int *)(regs->pc+4));
+ err |= get_user(jmp, (unsigned int *)(regs->pc+8));
+
+ if (err)
+ break;
+
+ if ((ldah & 0xFFFF0000U) == 0x277B0000U &&
+ (ldq & 0xFFFF0000U) == 0xA77B0000U &&
+ jmp == 0x6BFB0000U)
+ {
+ unsigned long r27, addr;
+ unsigned long addrh = (ldah | 0xFFFFFFFFFFFF0000UL) << 16;
+ unsigned long addrl = ldq | 0xFFFFFFFFFFFF0000UL;
+
+ addr = regs->r27 + ((addrh ^ 0x80000000UL) + 0x80000000UL) + ((addrl ^ 0x8000UL) + 0x8000UL);
+ err = get_user(r27, (unsigned long *)addr);
+ if (err)
+ break;
+
+ regs->r27 = r27;
+ regs->pc = r27;
+ return 2;
+ }
+ } while (0);
+
+ do { /* PaX: patched PLT emulation #2 */
+ unsigned int ldah, lda, br;
+
+ err = get_user(ldah, (unsigned int *)regs->pc);
+ err |= get_user(lda, (unsigned int *)(regs->pc+4));
+ err |= get_user(br, (unsigned int *)(regs->pc+8));
+
+ if (err)
+ break;
+
+ if ((ldah & 0xFFFF0000U) == 0x277B0000U &&
+ (lda & 0xFFFF0000U) == 0xA77B0000U &&
+ (br & 0xFFE00000U) == 0xC3E00000U)
+ {
+ unsigned long addr = br | 0xFFFFFFFFFFE00000UL;
+ unsigned long addrh = (ldah | 0xFFFFFFFFFFFF0000UL) << 16;
+ unsigned long addrl = lda | 0xFFFFFFFFFFFF0000UL;
+
+ regs->r27 += ((addrh ^ 0x80000000UL) + 0x80000000UL) + ((addrl ^ 0x8000UL) + 0x8000UL);
+ regs->pc += 12 + (((addr ^ 0x00100000UL) + 0x00100000UL) << 2);
+ return 2;
+ }
+ } while (0);
+
+ do { /* PaX: unpatched PLT emulation */
+ unsigned int br;
+
+ err = get_user(br, (unsigned int *)regs->pc);
+
+ if (!err && (br & 0xFFE00000U) == 0xC3800000U) {
+ unsigned int br2, ldq, nop, jmp;
+ unsigned long addr = br | 0xFFFFFFFFFFE00000UL, resolver;
+
+ addr = regs->pc + 4 + (((addr ^ 0x00100000UL) + 0x00100000UL) << 2);
+ err = get_user(br2, (unsigned int *)addr);
+ err |= get_user(ldq, (unsigned int *)(addr+4));
+ err |= get_user(nop, (unsigned int *)(addr+8));
+ err |= get_user(jmp, (unsigned int *)(addr+12));
+ err |= get_user(resolver, (unsigned long *)(addr+16));
+
+ if (err)
+ break;
+
+ if (br2 == 0xC3600000U &&
+ ldq == 0xA77B000CU &&
+ nop == 0x47FF041FU &&
+ jmp == 0x6B7B0000U)
+ {
+ regs->r28 = regs->pc+4;
+ regs->r27 = addr+16;
+ regs->pc = resolver;
+ return 3;
+ }
+ }
+ } while (0);
+#endif
+
+ return 1;
+}
+
+void pax_report_insns(struct pt_regs *regs, void *pc, void *sp)
+{
+ unsigned long i;
+
+ printk(KERN_ERR "PAX: bytes at PC: ");
+ for (i = 0; i < 5; i++) {
+ unsigned int c;
+ if (get_user(c, (unsigned int *)pc+i))
+ printk(KERN_CONT "???????? ");
+ else
+ printk(KERN_CONT "%08x ", c);
+ }
+ printk("\n");
+}
+#endif
/*
* This routine handles page faults. It determines the address,
good_area:
si_code = SEGV_ACCERR;
if (cause < 0) {
- if (!(vma->vm_flags & VM_EXEC))
+ if (!(vma->vm_flags & VM_EXEC)) {
+
+#ifdef CONFIG_PAX_PAGEEXEC
+ if (!(mm->pax_flags & MF_PAX_PAGEEXEC) || address != regs->pc)
+ goto bad_area;
+
+ up_read(&mm->mmap_sem);
+ switch (pax_handle_fetch_fault(regs)) {
+
+#ifdef CONFIG_PAX_EMUPLT
+ case 2:
+ case 3:
+ return;
+#endif
+
+ }
+ pax_report_fault(regs, (void *)regs->pc, (void *)rdusp());
+ do_group_exit(SIGKILL);
+#else
goto bad_area;
+#endif
+
+ }
} else if (!cause) {
/* Allow reads even for write-only mappings */
if (!(vma->vm_flags & (VM_READ | VM_WRITE)))
config UACCESS_WITH_MEMCPY
bool "Use kernel mem{cpy,set}() for {copy_to,clear}_user()"
- depends on MMU
+ depends on MMU && !PAX_MEMORY_UDEREF
default y if CPU_FEROCEON
help
Implement faster copy_to_user and clear_user methods for CPU
config KEXEC
bool "Kexec system call (EXPERIMENTAL)"
depends on (!SMP || PM_SLEEP_SMP)
+ depends on !GRKERNSEC_KMEM
help
kexec is a system call that implements the ability to shutdown your
current kernel, and to start another kernel. It is like a reboot
#include <asm/barrier.h>
#include <asm/cmpxchg.h>
+#ifdef CONFIG_GENERIC_ATOMIC64
+#include <asm-generic/atomic64.h>
+#endif
+
#define ATOMIC_INIT(i) { (i) }
#ifdef __KERNEL__
+#ifdef CONFIG_THUMB2_KERNEL
+#define REFCOUNT_TRAP_INSN "bkpt 0xf1"
+#else
+#define REFCOUNT_TRAP_INSN "bkpt 0xf103"
+#endif
+
+#define _ASM_EXTABLE(from, to) \
+" .pushsection __ex_table,\"a\"\n"\
+" .align 3\n" \
+" .long " #from ", " #to"\n" \
+" .popsection"
+
/*
* On ARM, ordinary assignment (str instruction) doesn't clear the local
* strex/ldrex monitor on some implementations. The reason we can use it for
* atomic_set() is the clrex or dummy strex done on every exception return.
*/
#define atomic_read(v) ACCESS_ONCE((v)->counter)
+static inline int atomic_read_unchecked(const atomic_unchecked_t *v)
+{
+ return ACCESS_ONCE(v->counter);
+}
#define atomic_set(v,i) (((v)->counter) = (i))
+static inline void atomic_set_unchecked(atomic_unchecked_t *v, int i)
+{
+ v->counter = i;
+}
#if __LINUX_ARM_ARCH__ >= 6
* to ensure that the update happens.
*/
-#define ATOMIC_OP(op, c_op, asm_op) \
-static inline void atomic_##op(int i, atomic_t *v) \
+#ifdef CONFIG_PAX_REFCOUNT
+#define __OVERFLOW_POST \
+ " bvc 3f\n" \
+ "2: " REFCOUNT_TRAP_INSN "\n"\
+ "3:\n"
+#define __OVERFLOW_POST_RETURN \
+ " bvc 3f\n" \
+" mov %0, %1\n" \
+ "2: " REFCOUNT_TRAP_INSN "\n"\
+ "3:\n"
+#define __OVERFLOW_EXTABLE \
+ "4:\n" \
+ _ASM_EXTABLE(2b, 4b)
+#else
+#define __OVERFLOW_POST
+#define __OVERFLOW_POST_RETURN
+#define __OVERFLOW_EXTABLE
+#endif
+
+#define __ATOMIC_OP(op, suffix, c_op, asm_op, post_op, extable) \
+static inline void atomic_##op##suffix(int i, atomic##suffix##_t *v) \
{ \
unsigned long tmp; \
int result; \
\
prefetchw(&v->counter); \
- __asm__ __volatile__("@ atomic_" #op "\n" \
+ __asm__ __volatile__("@ atomic_" #op #suffix "\n" \
"1: ldrex %0, [%3]\n" \
" " #asm_op " %0, %0, %4\n" \
+ post_op \
" strex %1, %0, [%3]\n" \
" teq %1, #0\n" \
-" bne 1b" \
+" bne 1b\n" \
+ extable \
: "=&r" (result), "=&r" (tmp), "+Qo" (v->counter) \
: "r" (&v->counter), "Ir" (i) \
: "cc"); \
} \
-#define ATOMIC_OP_RETURN(op, c_op, asm_op) \
-static inline int atomic_##op##_return(int i, atomic_t *v) \
+#define ATOMIC_OP(op, c_op, asm_op) __ATOMIC_OP(op, , c_op, asm_op, , )\
+ __ATOMIC_OP(op, _unchecked, c_op, asm_op##s, __OVERFLOW_POST, __OVERFLOW_EXTABLE)
+
+#define __ATOMIC_OP_RETURN(op, suffix, c_op, asm_op, post_op, extable) \
+static inline int atomic_##op##_return##suffix(int i, atomic##suffix##_t *v)\
{ \
unsigned long tmp; \
int result; \
smp_mb(); \
prefetchw(&v->counter); \
\
- __asm__ __volatile__("@ atomic_" #op "_return\n" \
+ __asm__ __volatile__("@ atomic_" #op "_return" #suffix "\n" \
"1: ldrex %0, [%3]\n" \
" " #asm_op " %0, %0, %4\n" \
+ post_op \
" strex %1, %0, [%3]\n" \
" teq %1, #0\n" \
-" bne 1b" \
+" bne 1b\n" \
+ extable \
: "=&r" (result), "=&r" (tmp), "+Qo" (v->counter) \
: "r" (&v->counter), "Ir" (i) \
: "cc"); \
return result; \
}
+#define ATOMIC_OP_RETURN(op, c_op, asm_op) __ATOMIC_OP_RETURN(op, , c_op, asm_op, , )\
+ __ATOMIC_OP_RETURN(op, _unchecked, c_op, asm_op##s, __OVERFLOW_POST_RETURN, __OVERFLOW_EXTABLE)
+
static inline int atomic_cmpxchg(atomic_t *ptr, int old, int new)
{
int oldval;
__asm__ __volatile__ ("@ atomic_add_unless\n"
"1: ldrex %0, [%4]\n"
" teq %0, %5\n"
-" beq 2f\n"
-" add %1, %0, %6\n"
+" beq 4f\n"
+" adds %1, %0, %6\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+" bvc 3f\n"
+"2: " REFCOUNT_TRAP_INSN "\n"
+"3:\n"
+#endif
+
" strex %2, %1, [%4]\n"
" teq %2, #0\n"
" bne 1b\n"
-"2:"
+"4:"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ _ASM_EXTABLE(2b, 4b)
+#endif
+
: "=&r" (oldval), "=&r" (newval), "=&r" (tmp), "+Qo" (v->counter)
: "r" (&v->counter), "r" (u), "r" (a)
: "cc");
return oldval;
}
+static inline int atomic_cmpxchg_unchecked(atomic_unchecked_t *ptr, int old, int new)
+{
+ unsigned long oldval, res;
+
+ smp_mb();
+
+ do {
+ __asm__ __volatile__("@ atomic_cmpxchg_unchecked\n"
+ "ldrex %1, [%3]\n"
+ "mov %0, #0\n"
+ "teq %1, %4\n"
+ "strexeq %0, %5, [%3]\n"
+ : "=&r" (res), "=&r" (oldval), "+Qo" (ptr->counter)
+ : "r" (&ptr->counter), "Ir" (old), "r" (new)
+ : "cc");
+ } while (res);
+
+ smp_mb();
+
+ return oldval;
+}
+
#else /* ARM_ARCH_6 */
#ifdef CONFIG_SMP
#error SMP not supported on pre-ARMv6 CPUs
#endif
-#define ATOMIC_OP(op, c_op, asm_op) \
-static inline void atomic_##op(int i, atomic_t *v) \
+#define __ATOMIC_OP(op, suffix, c_op, asm_op) \
+static inline void atomic_##op##suffix(int i, atomic##suffix##_t *v) \
{ \
unsigned long flags; \
\
raw_local_irq_restore(flags); \
} \
-#define ATOMIC_OP_RETURN(op, c_op, asm_op) \
-static inline int atomic_##op##_return(int i, atomic_t *v) \
+#define ATOMIC_OP(op, c_op, asm_op) __ATOMIC_OP(op, , c_op, asm_op) \
+ __ATOMIC_OP(op, _unchecked, c_op, asm_op)
+
+#define __ATOMIC_OP_RETURN(op, suffix, c_op, asm_op) \
+static inline int atomic_##op##_return##suffix(int i, atomic##suffix##_t *v)\
{ \
unsigned long flags; \
int val; \
return val; \
}
+#define ATOMIC_OP_RETURN(op, c_op, asm_op) __ATOMIC_OP_RETURN(op, , c_op, asm_op)\
+ __ATOMIC_OP_RETURN(op, _unchecked, c_op, asm_op)
+
static inline int atomic_cmpxchg(atomic_t *v, int old, int new)
{
int ret;
return ret;
}
+static inline int atomic_cmpxchg_unchecked(atomic_unchecked_t *v, int old, int new)
+{
+ return atomic_cmpxchg((atomic_t *)v, old, new);
+}
+
static inline int __atomic_add_unless(atomic_t *v, int a, int u)
{
int c, old;
#undef ATOMIC_OPS
#undef ATOMIC_OP_RETURN
+#undef __ATOMIC_OP_RETURN
#undef ATOMIC_OP
+#undef __ATOMIC_OP
#define atomic_xchg(v, new) (xchg(&((v)->counter), new))
+static inline int atomic_xchg_unchecked(atomic_unchecked_t *v, int new)
+{
+ return xchg(&v->counter, new);
+}
#define atomic_inc(v) atomic_add(1, v)
+static inline void atomic_inc_unchecked(atomic_unchecked_t *v)
+{
+ atomic_add_unchecked(1, v);
+}
#define atomic_dec(v) atomic_sub(1, v)
+static inline void atomic_dec_unchecked(atomic_unchecked_t *v)
+{
+ atomic_sub_unchecked(1, v);
+}
#define atomic_inc_and_test(v) (atomic_add_return(1, v) == 0)
+static inline int atomic_inc_and_test_unchecked(atomic_unchecked_t *v)
+{
+ return atomic_add_return_unchecked(1, v) == 0;
+}
#define atomic_dec_and_test(v) (atomic_sub_return(1, v) == 0)
#define atomic_inc_return(v) (atomic_add_return(1, v))
+static inline int atomic_inc_return_unchecked(atomic_unchecked_t *v)
+{
+ return atomic_add_return_unchecked(1, v);
+}
#define atomic_dec_return(v) (atomic_sub_return(1, v))
#define atomic_sub_and_test(i, v) (atomic_sub_return(i, v) == 0)
long long counter;
} atomic64_t;
+#ifdef CONFIG_PAX_REFCOUNT
+typedef struct {
+ long long counter;
+} atomic64_unchecked_t;
+#else
+typedef atomic64_t atomic64_unchecked_t;
+#endif
+
#define ATOMIC64_INIT(i) { (i) }
#ifdef CONFIG_ARM_LPAE
return result;
}
+static inline long long atomic64_read_unchecked(const atomic64_unchecked_t *v)
+{
+ long long result;
+
+ __asm__ __volatile__("@ atomic64_read_unchecked\n"
+" ldrd %0, %H0, [%1]"
+ : "=&r" (result)
+ : "r" (&v->counter), "Qo" (v->counter)
+ );
+
+ return result;
+}
+
static inline void atomic64_set(atomic64_t *v, long long i)
{
__asm__ __volatile__("@ atomic64_set\n"
: "r" (&v->counter), "r" (i)
);
}
+
+static inline void atomic64_set_unchecked(atomic64_unchecked_t *v, long long i)
+{
+ __asm__ __volatile__("@ atomic64_set_unchecked\n"
+" strd %2, %H2, [%1]"
+ : "=Qo" (v->counter)
+ : "r" (&v->counter), "r" (i)
+ );
+}
#else
static inline long long atomic64_read(const atomic64_t *v)
{
return result;
}
+static inline long long atomic64_read_unchecked(const atomic64_unchecked_t *v)
+{
+ long long result;
+
+ __asm__ __volatile__("@ atomic64_read_unchecked\n"
+" ldrexd %0, %H0, [%1]"
+ : "=&r" (result)
+ : "r" (&v->counter), "Qo" (v->counter)
+ );
+
+ return result;
+}
+
static inline void atomic64_set(atomic64_t *v, long long i)
{
long long tmp;
: "r" (&v->counter), "r" (i)
: "cc");
}
+
+static inline void atomic64_set_unchecked(atomic64_unchecked_t *v, long long i)
+{
+ long long tmp;
+
+ prefetchw(&v->counter);
+ __asm__ __volatile__("@ atomic64_set_unchecked\n"
+"1: ldrexd %0, %H0, [%2]\n"
+" strexd %0, %3, %H3, [%2]\n"
+" teq %0, #0\n"
+" bne 1b"
+ : "=&r" (tmp), "=Qo" (v->counter)
+ : "r" (&v->counter), "r" (i)
+ : "cc");
+}
#endif
-#define ATOMIC64_OP(op, op1, op2) \
-static inline void atomic64_##op(long long i, atomic64_t *v) \
+#undef __OVERFLOW_POST_RETURN
+#define __OVERFLOW_POST_RETURN \
+ " bvc 3f\n" \
+" mov %0, %1\n" \
+" mov %H0, %H1\n" \
+ "2: " REFCOUNT_TRAP_INSN "\n"\
+ "3:\n"
+
+#define __ATOMIC64_OP(op, suffix, op1, op2, post_op, extable) \
+static inline void atomic64_##op##suffix(long long i, atomic64##suffix##_t *v)\
{ \
long long result; \
unsigned long tmp; \
\
prefetchw(&v->counter); \
- __asm__ __volatile__("@ atomic64_" #op "\n" \
+ __asm__ __volatile__("@ atomic64_" #op #suffix "\n" \
"1: ldrexd %0, %H0, [%3]\n" \
" " #op1 " %Q0, %Q0, %Q4\n" \
" " #op2 " %R0, %R0, %R4\n" \
+ post_op \
" strexd %1, %0, %H0, [%3]\n" \
" teq %1, #0\n" \
-" bne 1b" \
+" bne 1b\n" \
+ extable \
: "=&r" (result), "=&r" (tmp), "+Qo" (v->counter) \
: "r" (&v->counter), "r" (i) \
: "cc"); \
} \
-#define ATOMIC64_OP_RETURN(op, op1, op2) \
-static inline long long atomic64_##op##_return(long long i, atomic64_t *v) \
+#define ATOMIC64_OP(op, op1, op2) __ATOMIC64_OP(op, , op1, op2, , ) \
+ __ATOMIC64_OP(op, _unchecked, op1, op2##s, __OVERFLOW_POST, __OVERFLOW_EXTABLE)
+
+#define __ATOMIC64_OP_RETURN(op, suffix, op1, op2, post_op, extable) \
+static inline long long atomic64_##op##_return##suffix(long long i, atomic64##suffix##_t *v) \
{ \
long long result; \
unsigned long tmp; \
smp_mb(); \
prefetchw(&v->counter); \
\
- __asm__ __volatile__("@ atomic64_" #op "_return\n" \
+ __asm__ __volatile__("@ atomic64_" #op "_return" #suffix "\n" \
"1: ldrexd %0, %H0, [%3]\n" \
" " #op1 " %Q0, %Q0, %Q4\n" \
" " #op2 " %R0, %R0, %R4\n" \
+ post_op \
" strexd %1, %0, %H0, [%3]\n" \
" teq %1, #0\n" \
-" bne 1b" \
+" bne 1b\n" \
+ extable \
: "=&r" (result), "=&r" (tmp), "+Qo" (v->counter) \
: "r" (&v->counter), "r" (i) \
: "cc"); \
return result; \
}
+#define ATOMIC64_OP_RETURN(op, op1, op2) __ATOMIC64_OP_RETURN(op, , op1, op2, , ) \
+ __ATOMIC64_OP_RETURN(op, _unchecked, op1, op2##s, __OVERFLOW_POST_RETURN, __OVERFLOW_EXTABLE)
+
#define ATOMIC64_OPS(op, op1, op2) \
ATOMIC64_OP(op, op1, op2) \
ATOMIC64_OP_RETURN(op, op1, op2)
#undef ATOMIC64_OPS
#undef ATOMIC64_OP_RETURN
+#undef __ATOMIC64_OP_RETURN
#undef ATOMIC64_OP
+#undef __ATOMIC64_OP
+#undef __OVERFLOW_EXTABLE
+#undef __OVERFLOW_POST_RETURN
+#undef __OVERFLOW_POST
static inline long long atomic64_cmpxchg(atomic64_t *ptr, long long old,
long long new)
return oldval;
}
+static inline long long atomic64_cmpxchg_unchecked(atomic64_unchecked_t *ptr, long long old,
+ long long new)
+{
+ long long oldval;
+ unsigned long res;
+
+ smp_mb();
+
+ do {
+ __asm__ __volatile__("@ atomic64_cmpxchg_unchecked\n"
+ "ldrexd %1, %H1, [%3]\n"
+ "mov %0, #0\n"
+ "teq %1, %4\n"
+ "teqeq %H1, %H4\n"
+ "strexdeq %0, %5, %H5, [%3]"
+ : "=&r" (res), "=&r" (oldval), "+Qo" (ptr->counter)
+ : "r" (&ptr->counter), "r" (old), "r" (new)
+ : "cc");
+ } while (res);
+
+ smp_mb();
+
+ return oldval;
+}
+
static inline long long atomic64_xchg(atomic64_t *ptr, long long new)
{
long long result;
static inline long long atomic64_dec_if_positive(atomic64_t *v)
{
long long result;
- unsigned long tmp;
+ u64 tmp;
smp_mb();
prefetchw(&v->counter);
__asm__ __volatile__("@ atomic64_dec_if_positive\n"
-"1: ldrexd %0, %H0, [%3]\n"
-" subs %Q0, %Q0, #1\n"
-" sbc %R0, %R0, #0\n"
+"1: ldrexd %1, %H1, [%3]\n"
+" subs %Q0, %Q1, #1\n"
+" sbcs %R0, %R1, #0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+" bvc 3f\n"
+" mov %Q0, %Q1\n"
+" mov %R0, %R1\n"
+"2: " REFCOUNT_TRAP_INSN "\n"
+"3:\n"
+#endif
+
" teq %R0, #0\n"
-" bmi 2f\n"
+" bmi 4f\n"
" strexd %1, %0, %H0, [%3]\n"
" teq %1, #0\n"
" bne 1b\n"
-"2:"
+"4:\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ _ASM_EXTABLE(2b, 4b)
+#endif
+
: "=&r" (result), "=&r" (tmp), "+Qo" (v->counter)
: "r" (&v->counter)
: "cc");
" teq %0, %5\n"
" teqeq %H0, %H5\n"
" moveq %1, #0\n"
-" beq 2f\n"
+" beq 4f\n"
" adds %Q0, %Q0, %Q6\n"
-" adc %R0, %R0, %R6\n"
+" adcs %R0, %R0, %R6\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+" bvc 3f\n"
+"2: " REFCOUNT_TRAP_INSN "\n"
+"3:\n"
+#endif
+
" strexd %2, %0, %H0, [%4]\n"
" teq %2, #0\n"
" bne 1b\n"
-"2:"
+"4:\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ _ASM_EXTABLE(2b, 4b)
+#endif
+
: "=&r" (val), "+r" (ret), "=&r" (tmp), "+Qo" (v->counter)
: "r" (&v->counter), "r" (u), "r" (a)
: "cc");
#define atomic64_add_negative(a, v) (atomic64_add_return((a), (v)) < 0)
#define atomic64_inc(v) atomic64_add(1LL, (v))
+#define atomic64_inc_unchecked(v) atomic64_add_unchecked(1LL, (v))
#define atomic64_inc_return(v) atomic64_add_return(1LL, (v))
+#define atomic64_inc_return_unchecked(v) atomic64_add_return_unchecked(1LL, (v))
#define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0)
#define atomic64_sub_and_test(a, v) (atomic64_sub_return((a), (v)) == 0)
#define atomic64_dec(v) atomic64_sub(1LL, (v))
+#define atomic64_dec_unchecked(v) atomic64_sub_unchecked(1LL, (v))
#define atomic64_dec_return(v) atomic64_sub_return(1LL, (v))
#define atomic64_dec_and_test(v) (atomic64_dec_return((v)) == 0)
#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1LL, 0LL)
do { \
compiletime_assert_atomic_type(*p); \
smp_mb(); \
- ACCESS_ONCE(*p) = (v); \
+ ACCESS_ONCE_RW(*p) = (v); \
} while (0)
#define smp_load_acquire(p) \
#ifndef __ASMARM_CACHE_H
#define __ASMARM_CACHE_H
+#include <linux/const.h>
+
#define L1_CACHE_SHIFT CONFIG_ARM_L1_CACHE_SHIFT
-#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT)
+#define L1_CACHE_BYTES (_AC(1,UL) << L1_CACHE_SHIFT)
/*
* Memory returned by kmalloc() may be used for DMA, so we must make
#endif
#define __read_mostly __attribute__((__section__(".data..read_mostly")))
+#define __read_only __attribute__ ((__section__(".data..read_only")))
#endif
void (*dma_unmap_area)(const void *, size_t, int);
void (*dma_flush_range)(const void *, const void *);
-};
+} __no_const;
/*
* Select the calling method
csum_partial_copy_nocheck(const void *src, void *dst, int len, __wsum sum);
__wsum
-csum_partial_copy_from_user(const void __user *src, void *dst, int len, __wsum sum, int *err_ptr);
+__csum_partial_copy_from_user(const void __user *src, void *dst, int len, __wsum sum, int *err_ptr);
+
+static inline __wsum
+csum_partial_copy_from_user(const void __user *src, void *dst, int len, __wsum sum, int *err_ptr)
+{
+ __wsum ret;
+ pax_open_userland();
+ ret = __csum_partial_copy_from_user(src, dst, len, sum, err_ptr);
+ pax_close_userland();
+ return ret;
+}
+
+
/*
* Fold a partial checksum without adding pseudo headers
#define xchg(ptr,x) \
((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr))))
+#define xchg_unchecked(ptr,x) \
+ ((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr))))
#include <asm-generic/cmpxchg-local.h>
* Domain types
*/
#define DOMAIN_NOACCESS 0
-#define DOMAIN_CLIENT 1
#ifdef CONFIG_CPU_USE_DOMAINS
+#define DOMAIN_USERCLIENT 1
+#define DOMAIN_KERNELCLIENT 1
#define DOMAIN_MANAGER 3
+#define DOMAIN_VECTORS DOMAIN_USER
+#else
+
+#ifdef CONFIG_PAX_KERNEXEC
+#define DOMAIN_MANAGER 1
+#define DOMAIN_KERNEXEC 3
#else
#define DOMAIN_MANAGER 1
#endif
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+#define DOMAIN_USERCLIENT 0
+#define DOMAIN_UDEREF 1
+#define DOMAIN_VECTORS DOMAIN_KERNEL
+#else
+#define DOMAIN_USERCLIENT 1
+#define DOMAIN_VECTORS DOMAIN_USER
+#endif
+#define DOMAIN_KERNELCLIENT 1
+
+#endif
+
#define domain_val(dom,type) ((type) << (2*(dom)))
#ifndef __ASSEMBLY__
-#ifdef CONFIG_CPU_USE_DOMAINS
+#if defined(CONFIG_CPU_USE_DOMAINS) || defined(CONFIG_PAX_KERNEXEC) || defined(CONFIG_PAX_MEMORY_UDEREF)
static inline void set_domain(unsigned val)
{
asm volatile(
isb();
}
-#define modify_domain(dom,type) \
- do { \
- struct thread_info *thread = current_thread_info(); \
- unsigned int domain = thread->cpu_domain; \
- domain &= ~domain_val(dom, DOMAIN_MANAGER); \
- thread->cpu_domain = domain | domain_val(dom, type); \
- set_domain(thread->cpu_domain); \
- } while (0)
-
+extern void modify_domain(unsigned int dom, unsigned int type);
#else
static inline void set_domain(unsigned val) { }
static inline void modify_domain(unsigned dom, unsigned type) { }
the loader. We need to make sure that it is out of the way of the program
that it will "exec", and that there is sufficient room for the brk. */
-#define ELF_ET_DYN_BASE (2 * TASK_SIZE / 3)
+#define ELF_ET_DYN_BASE (TASK_SIZE / 3 * 2)
+
+#ifdef CONFIG_PAX_ASLR
+#define PAX_ELF_ET_DYN_BASE 0x00008000UL
+
+#define PAX_DELTA_MMAP_LEN ((current->personality == PER_LINUX_32BIT) ? 16 : 10)
+#define PAX_DELTA_STACK_LEN ((current->personality == PER_LINUX_32BIT) ? 16 : 10)
+#endif
/* When the program starts, a1 contains a pointer to a function to be
registered with atexit, as per the SVR4 ABI. A value of 0 means we
extern void elf_set_personality(const struct elf32_hdr *);
#define SET_PERSONALITY(ex) elf_set_personality(&(ex))
-struct mm_struct;
-extern unsigned long arch_randomize_brk(struct mm_struct *mm);
-#define arch_randomize_brk arch_randomize_brk
-
#ifdef CONFIG_MMU
#define ARCH_HAS_SETUP_ADDITIONAL_PAGES 1
struct linux_binprm;
BUG_ON((uintptr_t)(dest_buf) & (FNCPY_ALIGN - 1) || \
(__funcp_address & ~(uintptr_t)1 & (FNCPY_ALIGN - 1))); \
\
+ pax_open_kernel(); \
memcpy(dest_buf, (void const *)(__funcp_address & ~1), size); \
+ pax_close_kernel(); \
flush_icache_range((unsigned long)(dest_buf), \
(unsigned long)(dest_buf) + (size)); \
\
if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32)))
return -EFAULT;
+ pax_open_userland();
+
smp_mb();
/* Prefetching cannot fault */
prefetchw(uaddr);
: "cc", "memory");
smp_mb();
+ pax_close_userland();
+
*uval = val;
return ret;
}
if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32)))
return -EFAULT;
+ pax_open_userland();
+
__asm__ __volatile__("@futex_atomic_cmpxchg_inatomic\n"
"1: " TUSER(ldr) " %1, [%4]\n"
" teq %1, %2\n"
: "r" (oldval), "r" (newval), "r" (uaddr), "Ir" (-EFAULT)
: "cc", "memory");
+ pax_close_userland();
+
*uval = val;
return ret;
}
return -EFAULT;
pagefault_disable(); /* implies preempt_disable() */
+ pax_open_userland();
switch (op) {
case FUTEX_OP_SET:
ret = -ENOSYS;
}
+ pax_close_userland();
pagefault_enable(); /* subsumes preempt_enable() */
if (!ret) {
/*
* This is the "bare minimum". AIO seems to require this.
*/
-#define KM_TYPE_NR 16
+#define KM_TYPE_NR 17
#endif
int (*residue)(unsigned int, dma_t *); /* optional */
int (*setspeed)(unsigned int, dma_t *, int); /* optional */
const char *type;
-};
+} __do_const;
struct dma_struct {
void *addr; /* single DMA address */
/* types 0-3 are defined in asm/io.h */
enum {
- MT_UNCACHED = 4,
- MT_CACHECLEAN,
- MT_MINICLEAN,
+ MT_UNCACHED_RW = 4,
+ MT_CACHECLEAN_RO,
+ MT_MINICLEAN_RO,
MT_LOW_VECTORS,
MT_HIGH_VECTORS,
- MT_MEMORY_RWX,
+ __MT_MEMORY_RWX,
MT_MEMORY_RW,
- MT_ROM,
- MT_MEMORY_RWX_NONCACHED,
+ MT_MEMORY_RX,
+ MT_ROM_RX,
+ MT_MEMORY_RW_NONCACHED,
+ MT_MEMORY_RX_NONCACHED,
MT_MEMORY_RW_DTCM,
- MT_MEMORY_RWX_ITCM,
+ MT_MEMORY_RX_ITCM,
MT_MEMORY_RW_SO,
MT_MEMORY_DMA_READY,
};
/* This is an ARM L2C thing */
void (*write_sec)(unsigned long, unsigned);
-};
+} __no_const;
extern struct outer_cache_fns outer_cache;
#else
+#include <linux/compiler.h>
#include <asm/glue.h>
/*
void (*cpu_clear_user_highpage)(struct page *page, unsigned long vaddr);
void (*cpu_copy_user_highpage)(struct page *to, struct page *from,
unsigned long vaddr, struct vm_area_struct *vma);
-};
+} __no_const;
#ifdef MULTI_USER
extern struct cpu_user_fns cpu_user;
#include <asm/processor.h>
#include <asm/cacheflush.h>
#include <asm/tlbflush.h>
+#include <asm/system_info.h>
#define check_pgt_cache() do { } while (0)
set_pud(pud, __pud(__pa(pmd) | PMD_TYPE_TABLE));
}
+static inline void pud_populate_kernel(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
+{
+ pud_populate(mm, pud, pmd);
+}
+
#else /* !CONFIG_ARM_LPAE */
/*
#define pmd_alloc_one(mm,addr) ({ BUG(); ((pmd_t *)2); })
#define pmd_free(mm, pmd) do { } while (0)
#define pud_populate(mm,pmd,pte) BUG()
+#define pud_populate_kernel(mm,pmd,pte) BUG()
#endif /* CONFIG_ARM_LPAE */
__free_page(pte);
}
+static inline void __section_update(pmd_t *pmdp, unsigned long addr, pmdval_t prot)
+{
+#ifdef CONFIG_ARM_LPAE
+ pmdp[0] = __pmd(pmd_val(pmdp[0]) | prot);
+#else
+ if (addr & SECTION_SIZE)
+ pmdp[1] = __pmd(pmd_val(pmdp[1]) | prot);
+ else
+ pmdp[0] = __pmd(pmd_val(pmdp[0]) | prot);
+#endif
+ flush_pmd_entry(pmdp);
+}
+
static inline void __pmd_populate(pmd_t *pmdp, phys_addr_t pte,
pmdval_t prot)
{
/*
* - section
*/
-#define PMD_SECT_PXN (_AT(pmdval_t, 1) << 0) /* v7 */
+#define PMD_SECT_PXN (_AT(pmdval_t, 1) << 0) /* v7 */
#define PMD_SECT_BUFFERABLE (_AT(pmdval_t, 1) << 2)
#define PMD_SECT_CACHEABLE (_AT(pmdval_t, 1) << 3)
#define PMD_SECT_XN (_AT(pmdval_t, 1) << 4) /* v6 */
#define PMD_SECT_nG (_AT(pmdval_t, 1) << 17) /* v6 */
#define PMD_SECT_SUPER (_AT(pmdval_t, 1) << 18) /* v6 */
#define PMD_SECT_AF (_AT(pmdval_t, 0))
+#define PMD_SECT_RDONLY (_AT(pmdval_t, 0))
#define PMD_SECT_UNCACHED (_AT(pmdval_t, 0))
#define PMD_SECT_BUFFERED (PMD_SECT_BUFFERABLE)
* - extended small page/tiny page
*/
#define PTE_EXT_XN (_AT(pteval_t, 1) << 0) /* v6 */
+#define PTE_EXT_PXN (_AT(pteval_t, 1) << 2) /* v7 */
#define PTE_EXT_AP_MASK (_AT(pteval_t, 3) << 4)
#define PTE_EXT_AP0 (_AT(pteval_t, 1) << 4)
#define PTE_EXT_AP1 (_AT(pteval_t, 2) << 4)
#define L_PTE_SHARED (_AT(pteval_t, 1) << 10) /* shared(v6), coherent(xsc3) */
#define L_PTE_NONE (_AT(pteval_t, 1) << 11)
+/* Two-level page tables only have PXN in the PGD, not in the PTE. */
+#define L_PTE_PXN (_AT(pteval_t, 0))
+
/*
* These are the memory types, defined to be compatible with
* pre-ARMv6 CPUs cacheable and bufferable bits: XXCB
#define L_PTE_USER (_AT(pteval_t, 1) << 6) /* AP[1] */
#define L_PTE_SHARED (_AT(pteval_t, 3) << 8) /* SH[1:0], inner shareable */
#define L_PTE_YOUNG (_AT(pteval_t, 1) << 10) /* AF */
+#define L_PTE_PXN (_AT(pteval_t, 1) << 53) /* PXN */
#define L_PTE_XN (_AT(pteval_t, 1) << 54) /* XN */
#define L_PTE_DIRTY (_AT(pteval_t, 1) << 55)
#define L_PTE_SPECIAL (_AT(pteval_t, 1) << 56)
#define L_PMD_SECT_SPLITTING (_AT(pmdval_t, 1) << 56)
#define L_PMD_SECT_NONE (_AT(pmdval_t, 1) << 57)
#define L_PMD_SECT_RDONLY (_AT(pteval_t, 1) << 58)
+#define PMD_SECT_RDONLY PMD_SECT_AP2
/*
* To be used in assembly code with the upper page attributes.
*/
+#define L_PTE_PXN_HIGH (1 << (53 - 32))
#define L_PTE_XN_HIGH (1 << (54 - 32))
#define L_PTE_DIRTY_HIGH (1 << (55 - 32))
#include <asm/pgtable-2level.h>
#endif
+#define ktla_ktva(addr) (addr)
+#define ktva_ktla(addr) (addr)
+
/*
* Just any arbitrary offset to the start of the vmalloc VM area: the
* current 8MB value just means that there will be a 8MB "hole" after the
#define LIBRARY_TEXT_START 0x0c000000
#ifndef __ASSEMBLY__
+extern pteval_t __supported_pte_mask;
+extern pmdval_t __supported_pmd_mask;
+
extern void __pte_error(const char *file, int line, pte_t);
extern void __pmd_error(const char *file, int line, pmd_t);
extern void __pgd_error(const char *file, int line, pgd_t);
#define pmd_ERROR(pmd) __pmd_error(__FILE__, __LINE__, pmd)
#define pgd_ERROR(pgd) __pgd_error(__FILE__, __LINE__, pgd)
+#define __HAVE_ARCH_PAX_OPEN_KERNEL
+#define __HAVE_ARCH_PAX_CLOSE_KERNEL
+
+#if defined(CONFIG_PAX_KERNEXEC) || defined(CONFIG_PAX_MEMORY_UDEREF)
+#include <asm/domain.h>
+#include <linux/thread_info.h>
+#include <linux/preempt.h>
+
+static inline int test_domain(int domain, int domaintype)
+{
+ return ((current_thread_info()->cpu_domain) & domain_val(domain, 3)) == domain_val(domain, domaintype);
+}
+#endif
+
+#ifdef CONFIG_PAX_KERNEXEC
+static inline unsigned long pax_open_kernel(void) {
+#ifdef CONFIG_ARM_LPAE
+ /* TODO */
+#else
+ preempt_disable();
+ BUG_ON(test_domain(DOMAIN_KERNEL, DOMAIN_KERNEXEC));
+ modify_domain(DOMAIN_KERNEL, DOMAIN_KERNEXEC);
+#endif
+ return 0;
+}
+
+static inline unsigned long pax_close_kernel(void) {
+#ifdef CONFIG_ARM_LPAE
+ /* TODO */
+#else
+ BUG_ON(test_domain(DOMAIN_KERNEL, DOMAIN_MANAGER));
+ /* DOMAIN_MANAGER = "client" under KERNEXEC */
+ modify_domain(DOMAIN_KERNEL, DOMAIN_MANAGER);
+ preempt_enable_no_resched();
+#endif
+ return 0;
+}
+#else
+static inline unsigned long pax_open_kernel(void) { return 0; }
+static inline unsigned long pax_close_kernel(void) { return 0; }
+#endif
+
/*
* This is the lowest virtual address we can permit any user space
* mapping to be mapped at. This is particularly important for
/*
* The pgprot_* and protection_map entries will be fixed up in runtime
* to include the cachable and bufferable bits based on memory policy,
- * as well as any architecture dependent bits like global/ASID and SMP
- * shared mapping bits.
+ * as well as any architecture dependent bits like global/ASID, PXN,
+ * and SMP shared mapping bits.
*/
#define _L_PTE_DEFAULT L_PTE_PRESENT | L_PTE_YOUNG
static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
{
const pteval_t mask = L_PTE_XN | L_PTE_RDONLY | L_PTE_USER |
- L_PTE_NONE | L_PTE_VALID;
+ L_PTE_NONE | L_PTE_VALID | __supported_pte_mask;
pte_val(pte) = (pte_val(pte) & ~mask) | (pgprot_val(newprot) & mask);
return pte;
}
int (*affinity_info)(unsigned long target_affinity,
unsigned long lowest_affinity_level);
int (*migrate_info_type)(void);
-};
+} __no_const;
extern struct psci_operations psci_ops;
extern struct smp_operations psci_smp_ops;
int (*cpu_disable)(unsigned int cpu);
#endif
#endif
-};
+} __no_const;
struct of_cpu_method {
const char *method;
.flags = 0, \
.preempt_count = INIT_PREEMPT_COUNT, \
.addr_limit = KERNEL_DS, \
- .cpu_domain = domain_val(DOMAIN_USER, DOMAIN_MANAGER) | \
- domain_val(DOMAIN_KERNEL, DOMAIN_MANAGER) | \
- domain_val(DOMAIN_IO, DOMAIN_CLIENT), \
+ .cpu_domain = domain_val(DOMAIN_USER, DOMAIN_USERCLIENT) | \
+ domain_val(DOMAIN_KERNEL, DOMAIN_KERNELCLIENT) | \
+ domain_val(DOMAIN_IO, DOMAIN_KERNELCLIENT), \
.restart_block = { \
.fn = do_no_restart_syscall, \
}, \
#define TIF_SYSCALL_AUDIT 9
#define TIF_SYSCALL_TRACEPOINT 10
#define TIF_SECCOMP 11 /* seccomp syscall filtering active */
-#define TIF_NOHZ 12 /* in adaptive nohz mode */
+/* within 8 bits of TIF_SYSCALL_TRACE
+ * to meet flexible second operand requirements
+ */
+#define TIF_GRSEC_SETXID 12
+#define TIF_NOHZ 13 /* in adaptive nohz mode */
#define TIF_USING_IWMMXT 17
#define TIF_MEMDIE 18 /* is terminating due to OOM killer */
#define TIF_RESTORE_SIGMASK 20
#define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT)
#define _TIF_SECCOMP (1 << TIF_SECCOMP)
#define _TIF_USING_IWMMXT (1 << TIF_USING_IWMMXT)
+#define _TIF_GRSEC_SETXID (1 << TIF_GRSEC_SETXID)
/* Checks for any syscall work in entry-common.S */
#define _TIF_SYSCALL_WORK (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \
- _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP)
+ _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP | _TIF_GRSEC_SETXID)
/*
* Change these and you break ASM code in entry-common.S
#include <linux/compiler.h>
#include <asm/thread_info.h>
+#include <asm/pgtable.h>
#ifdef __ASSEMBLY__
#include <asm/asm-offsets.h>
* at 0xffff0fe0 must be used instead. (see
* entry-armv.S for details)
*/
+ pax_open_kernel();
*((unsigned int *)0xffff0ff0) = val;
+ pax_close_kernel();
#endif
}
#include <asm/domain.h>
#include <asm/unified.h>
#include <asm/compiler.h>
+#include <asm/pgtable.h>
#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
#include <asm-generic/uaccess-unaligned.h>
static inline void set_fs(mm_segment_t fs)
{
current_thread_info()->addr_limit = fs;
- modify_domain(DOMAIN_KERNEL, fs ? DOMAIN_CLIENT : DOMAIN_MANAGER);
+ modify_domain(DOMAIN_KERNEL, fs ? DOMAIN_KERNELCLIENT : DOMAIN_MANAGER);
}
#define segment_eq(a,b) ((a) == (b))
+#define __HAVE_ARCH_PAX_OPEN_USERLAND
+#define __HAVE_ARCH_PAX_CLOSE_USERLAND
+
+static inline void pax_open_userland(void)
+{
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ if (segment_eq(get_fs(), USER_DS)) {
+ BUG_ON(test_domain(DOMAIN_USER, DOMAIN_UDEREF));
+ modify_domain(DOMAIN_USER, DOMAIN_UDEREF);
+ }
+#endif
+
+}
+
+static inline void pax_close_userland(void)
+{
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ if (segment_eq(get_fs(), USER_DS)) {
+ BUG_ON(test_domain(DOMAIN_USER, DOMAIN_NOACCESS));
+ modify_domain(DOMAIN_USER, DOMAIN_NOACCESS);
+ }
+#endif
+
+}
+
#define __addr_ok(addr) ({ \
unsigned long flag; \
__asm__("cmp %2, %0; movlo %0, #0" \
#define get_user(x,p) \
({ \
+ int __e; \
might_fault(); \
- __get_user_check(x,p); \
+ pax_open_userland(); \
+ __e = __get_user_check(x,p); \
+ pax_close_userland(); \
+ __e; \
})
extern int __put_user_1(void *, unsigned int);
#define put_user(x,p) \
({ \
+ int __e; \
might_fault(); \
- __put_user_check(x,p); \
+ pax_open_userland(); \
+ __e = __put_user_check(x,p); \
+ pax_close_userland(); \
+ __e; \
})
#else /* CONFIG_MMU */
#endif /* CONFIG_MMU */
+#define access_ok_noprefault(type,addr,size) access_ok((type),(addr),(size))
#define access_ok(type,addr,size) (__range_ok(addr,size) == 0)
#define user_addr_max() \
#define __get_user(x,ptr) \
({ \
long __gu_err = 0; \
+ pax_open_userland(); \
__get_user_err((x),(ptr),__gu_err); \
+ pax_close_userland(); \
__gu_err; \
})
#define __get_user_error(x,ptr,err) \
({ \
+ pax_open_userland(); \
__get_user_err((x),(ptr),err); \
+ pax_close_userland(); \
(void) 0; \
})
#define __put_user(x,ptr) \
({ \
long __pu_err = 0; \
+ pax_open_userland(); \
__put_user_err((x),(ptr),__pu_err); \
+ pax_close_userland(); \
__pu_err; \
})
#define __put_user_error(x,ptr,err) \
({ \
+ pax_open_userland(); \
__put_user_err((x),(ptr),err); \
+ pax_close_userland(); \
(void) 0; \
})
#ifdef CONFIG_MMU
-extern unsigned long __must_check __copy_from_user(void *to, const void __user *from, unsigned long n);
-extern unsigned long __must_check __copy_to_user(void __user *to, const void *from, unsigned long n);
+extern unsigned long __must_check ___copy_from_user(void *to, const void __user *from, unsigned long n);
+extern unsigned long __must_check ___copy_to_user(void __user *to, const void *from, unsigned long n);
+
+static inline unsigned long __must_check __copy_from_user(void *to, const void __user *from, unsigned long n)
+{
+ unsigned long ret;
+
+ check_object_size(to, n, false);
+ pax_open_userland();
+ ret = ___copy_from_user(to, from, n);
+ pax_close_userland();
+ return ret;
+}
+
+static inline unsigned long __must_check __copy_to_user(void __user *to, const void *from, unsigned long n)
+{
+ unsigned long ret;
+
+ check_object_size(from, n, true);
+ pax_open_userland();
+ ret = ___copy_to_user(to, from, n);
+ pax_close_userland();
+ return ret;
+}
+
extern unsigned long __must_check __copy_to_user_std(void __user *to, const void *from, unsigned long n);
-extern unsigned long __must_check __clear_user(void __user *addr, unsigned long n);
+extern unsigned long __must_check ___clear_user(void __user *addr, unsigned long n);
extern unsigned long __must_check __clear_user_std(void __user *addr, unsigned long n);
+
+static inline unsigned long __must_check __clear_user(void __user *addr, unsigned long n)
+{
+ unsigned long ret;
+ pax_open_userland();
+ ret = ___clear_user(addr, n);
+ pax_close_userland();
+ return ret;
+}
+
#else
#define __copy_from_user(to,from,n) (memcpy(to, (void __force *)from, n), 0)
#define __copy_to_user(to,from,n) (memcpy((void __force *)to, from, n), 0)
static inline unsigned long __must_check copy_from_user(void *to, const void __user *from, unsigned long n)
{
+ if ((long)n < 0)
+ return n;
+
if (access_ok(VERIFY_READ, from, n))
n = __copy_from_user(to, from, n);
else /* security hole - plug it */
static inline unsigned long __must_check copy_to_user(void __user *to, const void *from, unsigned long n)
{
+ if ((long)n < 0)
+ return n;
+
if (access_ok(VERIFY_WRITE, to, n))
n = __copy_to_user(to, from, n);
return n;
* ARMv7 groups of PSR bits
*/
#define APSR_MASK 0xf80f0000 /* N, Z, C, V, Q and GE flags */
-#define PSR_ISET_MASK 0x01000010 /* ISA state (J, T) mask */
+#define PSR_ISET_MASK 0x01000020 /* ISA state (J, T) mask */
#define PSR_IT_MASK 0x0600fc00 /* If-Then execution state mask */
#define PSR_ENDIAN_MASK 0x00000200 /* Endianness state mask */
/* networking */
EXPORT_SYMBOL(csum_partial);
-EXPORT_SYMBOL(csum_partial_copy_from_user);
+EXPORT_SYMBOL(__csum_partial_copy_from_user);
EXPORT_SYMBOL(csum_partial_copy_nocheck);
EXPORT_SYMBOL(__csum_ipv6_magic);
#ifdef CONFIG_MMU
EXPORT_SYMBOL(copy_page);
-EXPORT_SYMBOL(__copy_from_user);
-EXPORT_SYMBOL(__copy_to_user);
-EXPORT_SYMBOL(__clear_user);
+EXPORT_SYMBOL(___copy_from_user);
+EXPORT_SYMBOL(___copy_to_user);
+EXPORT_SYMBOL(___clear_user);
EXPORT_SYMBOL(__get_user_1);
EXPORT_SYMBOL(__get_user_2);
9997:
.endm
+ .macro pax_enter_kernel
+#if defined(CONFIG_PAX_KERNEXEC) || defined(CONFIG_PAX_MEMORY_UDEREF)
+ @ make aligned space for saved DACR
+ sub sp, sp, #8
+ @ save regs
+ stmdb sp!, {r1, r2}
+ @ read DACR from cpu_domain into r1
+ mov r2, sp
+ @ assume 8K pages, since we have to split the immediate in two
+ bic r2, r2, #(0x1fc0)
+ bic r2, r2, #(0x3f)
+ ldr r1, [r2, #TI_CPU_DOMAIN]
+ @ store old DACR on stack
+ str r1, [sp, #8]
+#ifdef CONFIG_PAX_KERNEXEC
+ @ set type of DOMAIN_KERNEL to DOMAIN_KERNELCLIENT
+ bic r1, r1, #(domain_val(DOMAIN_KERNEL, 3))
+ orr r1, r1, #(domain_val(DOMAIN_KERNEL, DOMAIN_KERNELCLIENT))
+#endif
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ @ set current DOMAIN_USER to DOMAIN_NOACCESS
+ bic r1, r1, #(domain_val(DOMAIN_USER, 3))
+#endif
+ @ write r1 to current_thread_info()->cpu_domain
+ str r1, [r2, #TI_CPU_DOMAIN]
+ @ write r1 to DACR
+ mcr p15, 0, r1, c3, c0, 0
+ @ instruction sync
+ instr_sync
+ @ restore regs
+ ldmia sp!, {r1, r2}
+#endif
+ .endm
+
+ .macro pax_open_userland
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ @ save regs
+ stmdb sp!, {r0, r1}
+ @ read DACR from cpu_domain into r1
+ mov r0, sp
+ @ assume 8K pages, since we have to split the immediate in two
+ bic r0, r0, #(0x1fc0)
+ bic r0, r0, #(0x3f)
+ ldr r1, [r0, #TI_CPU_DOMAIN]
+ @ set current DOMAIN_USER to DOMAIN_CLIENT
+ bic r1, r1, #(domain_val(DOMAIN_USER, 3))
+ orr r1, r1, #(domain_val(DOMAIN_USER, DOMAIN_UDEREF))
+ @ write r1 to current_thread_info()->cpu_domain
+ str r1, [r0, #TI_CPU_DOMAIN]
+ @ write r1 to DACR
+ mcr p15, 0, r1, c3, c0, 0
+ @ instruction sync
+ instr_sync
+ @ restore regs
+ ldmia sp!, {r0, r1}
+#endif
+ .endm
+
+ .macro pax_close_userland
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ @ save regs
+ stmdb sp!, {r0, r1}
+ @ read DACR from cpu_domain into r1
+ mov r0, sp
+ @ assume 8K pages, since we have to split the immediate in two
+ bic r0, r0, #(0x1fc0)
+ bic r0, r0, #(0x3f)
+ ldr r1, [r0, #TI_CPU_DOMAIN]
+ @ set current DOMAIN_USER to DOMAIN_NOACCESS
+ bic r1, r1, #(domain_val(DOMAIN_USER, 3))
+ @ write r1 to current_thread_info()->cpu_domain
+ str r1, [r0, #TI_CPU_DOMAIN]
+ @ write r1 to DACR
+ mcr p15, 0, r1, c3, c0, 0
+ @ instruction sync
+ instr_sync
+ @ restore regs
+ ldmia sp!, {r0, r1}
+#endif
+ .endm
+
.macro pabt_helper
@ PABORT handler takes pt_regs in r2, fault address in r4 and psr in r5
#ifdef MULTI_PABORT
* Invalid mode handlers
*/
.macro inv_entry, reason
+
+ pax_enter_kernel
+
sub sp, sp, #S_FRAME_SIZE
ARM( stmib sp, {r1 - lr} )
THUMB( stmia sp, {r0 - r12} )
THUMB( str sp, [sp, #S_SP] )
THUMB( str lr, [sp, #S_LR] )
+
mov r1, #\reason
.endm
.macro svc_entry, stack_hole=0, trace=1
UNWIND(.fnstart )
UNWIND(.save {r0 - pc} )
+
+ pax_enter_kernel
+
sub sp, sp, #(S_FRAME_SIZE + \stack_hole - 4)
+
#ifdef CONFIG_THUMB2_KERNEL
SPFIX( str r0, [sp] ) @ temporarily saved
SPFIX( mov r0, sp )
ldmia r0, {r3 - r5}
add r7, sp, #S_SP - 4 @ here for interlock avoidance
mov r6, #-1 @ "" "" "" ""
+#if defined(CONFIG_PAX_KERNEXEC) || defined(CONFIG_PAX_MEMORY_UDEREF)
+ @ offset sp by 8 as done in pax_enter_kernel
+ add r2, sp, #(S_FRAME_SIZE + \stack_hole + 4)
+#else
add r2, sp, #(S_FRAME_SIZE + \stack_hole - 4)
+#endif
SPFIX( addeq r2, r2, #4 )
str r3, [sp, #-4]! @ save the "real" r0 copied
@ from the exception stack
.macro usr_entry, trace=1
UNWIND(.fnstart )
UNWIND(.cantunwind ) @ don't unwind the user space
+
+ pax_enter_kernel_user
+
sub sp, sp, #S_FRAME_SIZE
ARM( stmib sp, {r1 - r12} )
THUMB( stmia sp, {r0 - r12} )
tst r3, #PSR_T_BIT @ Thumb mode?
bne __und_usr_thumb
sub r4, r2, #4 @ ARM instr at LR - 4
+ pax_open_userland
1: ldrt r0, [r4]
+ pax_close_userland
ARM_BE8(rev r0, r0) @ little endian instruction
@ r0 = 32-bit ARM instruction which caused the exception
*/
.arch armv6t2
#endif
+ pax_open_userland
2: ldrht r5, [r4]
+ pax_close_userland
ARM_BE8(rev16 r5, r5) @ little endian instruction
cmp r5, #0xe800 @ 32bit instruction if xx != 0
blo __und_usr_fault_16 @ 16bit undefined instruction
+ pax_open_userland
3: ldrht r0, [r2]
+ pax_close_userland
ARM_BE8(rev16 r0, r0) @ little endian instruction
add r2, r2, #2 @ r2 is PC + 2, make it PC + 4
str r2, [sp, #S_PC] @ it's a 2x16bit instr, update
*/
.pushsection .fixup, "ax"
.align 2
-4: str r4, [sp, #S_PC] @ retry current instruction
+4: pax_close_userland
+ str r4, [sp, #S_PC] @ retry current instruction
ret r9
.popsection
.pushsection __ex_table,"a"
THUMB( str lr, [ip], #4 )
ldr r4, [r2, #TI_TP_VALUE]
ldr r5, [r2, #TI_TP_VALUE + 4]
-#ifdef CONFIG_CPU_USE_DOMAINS
+#if defined(CONFIG_CPU_USE_DOMAINS) || defined(CONFIG_PAX_KERNEXEC) || defined(CONFIG_PAX_MEMORY_UDEREF)
ldr r6, [r2, #TI_CPU_DOMAIN]
#endif
switch_tls r1, r4, r5, r3, r7
ldr r8, =__stack_chk_guard
ldr r7, [r7, #TSK_STACK_CANARY]
#endif
-#ifdef CONFIG_CPU_USE_DOMAINS
+#if defined(CONFIG_CPU_USE_DOMAINS) || defined(CONFIG_PAX_KERNEXEC) || defined(CONFIG_PAX_MEMORY_UDEREF)
mcr p15, 0, r6, c3, c0, 0 @ Set domain register
#endif
mov r5, r0
#include <asm/assembler.h>
#include <asm/unistd.h>
#include <asm/ftrace.h>
+#include <asm/domain.h>
#include <asm/unwind.h>
+#include "entry-header.S"
+
#ifdef CONFIG_NEED_RET_TO_USER
#include <mach/entry-macro.S>
#else
.macro arch_ret_to_user, tmp1, tmp2
+#if defined(CONFIG_PAX_KERNEXEC) || defined(CONFIG_PAX_MEMORY_UDEREF)
+ @ save regs
+ stmdb sp!, {r1, r2}
+ @ read DACR from cpu_domain into r1
+ mov r2, sp
+ @ assume 8K pages, since we have to split the immediate in two
+ bic r2, r2, #(0x1fc0)
+ bic r2, r2, #(0x3f)
+ ldr r1, [r2, #TI_CPU_DOMAIN]
+#ifdef CONFIG_PAX_KERNEXEC
+ @ set type of DOMAIN_KERNEL to DOMAIN_KERNELCLIENT
+ bic r1, r1, #(domain_val(DOMAIN_KERNEL, 3))
+ orr r1, r1, #(domain_val(DOMAIN_KERNEL, DOMAIN_KERNELCLIENT))
+#endif
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ @ set current DOMAIN_USER to DOMAIN_UDEREF
+ bic r1, r1, #(domain_val(DOMAIN_USER, 3))
+ orr r1, r1, #(domain_val(DOMAIN_USER, DOMAIN_UDEREF))
+#endif
+ @ write r1 to current_thread_info()->cpu_domain
+ str r1, [r2, #TI_CPU_DOMAIN]
+ @ write r1 to DACR
+ mcr p15, 0, r1, c3, c0, 0
+ @ instruction sync
+ instr_sync
+ @ restore regs
+ ldmia sp!, {r1, r2}
+#endif
.endm
#endif
-#include "entry-header.S"
-
-
.align 5
/*
* This is the fast syscall return path. We do as little as
USER( ldr scno, [lr, #-4] ) @ get SWI instruction
#endif
+ /*
+ * do this here to avoid a performance hit of wrapping the code above
+ * that directly dereferences userland to parse the SWI instruction
+ */
+ pax_enter_kernel_user
+
adr tbl, sys_call_table @ load syscall table pointer
#if defined(CONFIG_OABI_COMPAT)
msr cpsr_c, \rtemp @ switch back to the SVC mode
.endm
+ .macro pax_enter_kernel_user
+#if defined(CONFIG_PAX_KERNEXEC) || defined(CONFIG_PAX_MEMORY_UDEREF)
+ @ save regs
+ stmdb sp!, {r0, r1}
+ @ read DACR from cpu_domain into r1
+ mov r0, sp
+ @ assume 8K pages, since we have to split the immediate in two
+ bic r0, r0, #(0x1fc0)
+ bic r0, r0, #(0x3f)
+ ldr r1, [r0, #TI_CPU_DOMAIN]
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ @ set current DOMAIN_USER to DOMAIN_NOACCESS
+ bic r1, r1, #(domain_val(DOMAIN_USER, 3))
+#endif
+#ifdef CONFIG_PAX_KERNEXEC
+ @ set current DOMAIN_KERNEL to DOMAIN_KERNELCLIENT
+ bic r1, r1, #(domain_val(DOMAIN_KERNEL, 3))
+ orr r1, r1, #(domain_val(DOMAIN_KERNEL, DOMAIN_KERNELCLIENT))
+#endif
+ @ write r1 to current_thread_info()->cpu_domain
+ str r1, [r0, #TI_CPU_DOMAIN]
+ @ write r1 to DACR
+ mcr p15, 0, r1, c3, c0, 0
+ @ instruction sync
+ instr_sync
+ @ restore regs
+ ldmia sp!, {r0, r1}
+#endif
+ .endm
+
+ .macro pax_exit_kernel
+#if defined(CONFIG_PAX_KERNEXEC) || defined(CONFIG_PAX_MEMORY_UDEREF)
+ @ save regs
+ stmdb sp!, {r0, r1}
+ @ read old DACR from stack into r1
+ ldr r1, [sp, #(8 + S_SP)]
+ sub r1, r1, #8
+ ldr r1, [r1]
+
+ @ write r1 to current_thread_info()->cpu_domain
+ mov r0, sp
+ @ assume 8K pages, since we have to split the immediate in two
+ bic r0, r0, #(0x1fc0)
+ bic r0, r0, #(0x3f)
+ str r1, [r0, #TI_CPU_DOMAIN]
+ @ write r1 to DACR
+ mcr p15, 0, r1, c3, c0, 0
+ @ instruction sync
+ instr_sync
+ @ restore regs
+ ldmia sp!, {r0, r1}
+#endif
+ .endm
+
#ifndef CONFIG_THUMB2_KERNEL
.macro svc_exit, rpsr, irq = 0
.if \irq != 0
blne trace_hardirqs_off
#endif
.endif
+
+ pax_exit_kernel
+
msr spsr_cxsf, \rpsr
#if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_32v6K)
@ We must avoid clrex due to Cortex-A15 erratum #830321
blne trace_hardirqs_off
#endif
.endif
+
+ pax_exit_kernel
+
ldr lr, [sp, #S_SP] @ top of the stack
ldrd r0, r1, [sp, #S_LR] @ calling lr and pc
void *base = vectors_page;
unsigned offset = FIQ_OFFSET;
+ pax_open_kernel();
memcpy(base + offset, start, length);
+ pax_close_kernel();
+
if (!cache_is_vipt_nonaliasing())
flush_icache_range((unsigned long)base + offset, offset +
length);
mov r5, #(domain_val(DOMAIN_USER, DOMAIN_MANAGER) | \
domain_val(DOMAIN_KERNEL, DOMAIN_MANAGER) | \
domain_val(DOMAIN_TABLE, DOMAIN_MANAGER) | \
- domain_val(DOMAIN_IO, DOMAIN_CLIENT))
+ domain_val(DOMAIN_IO, DOMAIN_KERNELCLIENT))
mcr p15, 0, r5, c3, c0, 0 @ load domain access register
mcr p15, 0, r4, c2, c0, 0 @ load page table pointer
#endif
#endif
#ifdef CONFIG_MMU
-void *module_alloc(unsigned long size)
+static inline void *__module_alloc(unsigned long size, pgprot_t prot)
{
+ if (!size || PAGE_ALIGN(size) > MODULES_END - MODULES_VADDR)
+ return NULL;
return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END,
- GFP_KERNEL, PAGE_KERNEL_EXEC, NUMA_NO_NODE,
+ GFP_KERNEL, prot, NUMA_NO_NODE,
__builtin_return_address(0));
}
+
+void *module_alloc(unsigned long size)
+{
+
+#ifdef CONFIG_PAX_KERNEXEC
+ return __module_alloc(size, PAGE_KERNEL);
+#else
+ return __module_alloc(size, PAGE_KERNEL_EXEC);
+#endif
+
+}
+
+#ifdef CONFIG_PAX_KERNEXEC
+void module_memfree_exec(void *module_region)
+{
+ module_memfree(module_region);
+}
+EXPORT_SYMBOL(module_memfree_exec);
+
+void *module_alloc_exec(unsigned long size)
+{
+ return __module_alloc(size, PAGE_KERNEL_EXEC);
+}
+EXPORT_SYMBOL(module_alloc_exec);
+#endif
#endif
int
else
__acquire(&patch_lock);
+ pax_open_kernel();
if (thumb2 && __opcode_is_thumb16(insn)) {
*(u16 *)waddr = __opcode_to_mem_thumb16(insn);
size = sizeof(u16);
*(u32 *)waddr = insn;
size = sizeof(u32);
}
+ pax_close_kernel();
if (waddr != addr) {
flush_kernel_vmap_range(waddr, twopage ? size / 2 : size);
if (pm_power_off)
pm_power_off();
+ BUG();
}
/*
* executing pre-reset code, and using RAM that the primary CPU's code wishes
* to use. Implementing such co-ordination would be essentially impossible.
*/
-void machine_restart(char *cmd)
+__noreturn void machine_restart(char *cmd)
{
local_irq_disable();
smp_send_stop();
show_regs_print_info(KERN_DEFAULT);
- print_symbol("PC is at %s\n", instruction_pointer(regs));
- print_symbol("LR is at %s\n", regs->ARM_lr);
+ printk("PC is at %pA\n", (void *)instruction_pointer(regs));
+ printk("LR is at %pA\n", (void *)regs->ARM_lr);
printk("pc : [<%08lx>] lr : [<%08lx>] psr: %08lx\n"
"sp : %08lx ip : %08lx fp : %08lx\n",
regs->ARM_pc, regs->ARM_lr, regs->ARM_cpsr,
return 0;
}
-unsigned long arch_randomize_brk(struct mm_struct *mm)
-{
- unsigned long range_end = mm->brk + 0x02000000;
- return randomize_range(mm->brk, range_end, 0) ? : mm->brk;
-}
-
#ifdef CONFIG_MMU
#ifdef CONFIG_KUSER_HELPERS
/*
static int __init gate_vma_init(void)
{
- gate_vma.vm_page_prot = PAGE_READONLY_EXEC;
+ gate_vma.vm_page_prot = vm_get_page_prot(gate_vma.vm_flags);
return 0;
}
arch_initcall(gate_vma_init);
return is_gate_vma(vma) ? "[vectors]" : NULL;
}
-/* If possible, provide a placement hint at a random offset from the
- * stack for the signal page.
- */
-static unsigned long sigpage_addr(const struct mm_struct *mm,
- unsigned int npages)
-{
- unsigned long offset;
- unsigned long first;
- unsigned long last;
- unsigned long addr;
- unsigned int slots;
-
- first = PAGE_ALIGN(mm->start_stack);
-
- last = TASK_SIZE - (npages << PAGE_SHIFT);
-
- /* No room after stack? */
- if (first > last)
- return 0;
-
- /* Just enough room? */
- if (first == last)
- return first;
-
- slots = ((last - first) >> PAGE_SHIFT) + 1;
-
- offset = get_random_int() % slots;
-
- addr = first + (offset << PAGE_SHIFT);
-
- return addr;
-}
-
-static struct page *signal_page;
-extern struct page *get_signal_page(void);
-
-static const struct vm_special_mapping sigpage_mapping = {
- .name = "[sigpage]",
- .pages = &signal_page,
-};
-
int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
{
struct mm_struct *mm = current->mm;
- struct vm_area_struct *vma;
- unsigned long addr;
- unsigned long hint;
- int ret = 0;
-
- if (!signal_page)
- signal_page = get_signal_page();
- if (!signal_page)
- return -ENOMEM;
down_write(&mm->mmap_sem);
- hint = sigpage_addr(mm, 1);
- addr = get_unmapped_area(NULL, hint, PAGE_SIZE, 0, 0);
- if (IS_ERR_VALUE(addr)) {
- ret = addr;
- goto up_fail;
- }
-
- vma = _install_special_mapping(mm, addr, PAGE_SIZE,
- VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC,
- &sigpage_mapping);
-
- if (IS_ERR(vma)) {
- ret = PTR_ERR(vma);
- goto up_fail;
- }
-
- mm->context.sigpage = addr;
-
- up_fail:
+ mm->context.sigpage = (PAGE_OFFSET + (get_random_int() % 0x3FFEFFE0)) & 0xFFFFFFFC;
up_write(&mm->mmap_sem);
- return ret;
+ return 0;
}
#endif
#include <asm/psci.h>
#include <asm/system_misc.h>
-struct psci_operations psci_ops;
+struct psci_operations psci_ops __read_only;
static int (*invoke_psci_fn)(u32, u32, u32, u32);
typedef int (*psci_initcall_t)(const struct device_node *);
regs->ARM_ip = ip;
}
+#ifdef CONFIG_GRKERNSEC_SETXID
+extern void gr_delayed_cred_worker(void);
+#endif
+
asmlinkage int syscall_trace_enter(struct pt_regs *regs, int scno)
{
current_thread_info()->syscall = scno;
+#ifdef CONFIG_GRKERNSEC_SETXID
+ if (unlikely(test_and_clear_thread_flag(TIF_GRSEC_SETXID)))
+ gr_delayed_cred_worker();
+#endif
+
/* Do the secure computing check first; failures should be fast. */
#ifdef CONFIG_HAVE_ARCH_SECCOMP_FILTER
if (secure_computing() == -1)
unsigned int elf_hwcap2 __read_mostly;
EXPORT_SYMBOL(elf_hwcap2);
+pteval_t __supported_pte_mask __read_only;
+pmdval_t __supported_pmd_mask __read_only;
#ifdef MULTI_CPU
-struct processor processor __read_mostly;
+struct processor processor __read_only;
#endif
#ifdef MULTI_TLB
-struct cpu_tlb_fns cpu_tlb __read_mostly;
+struct cpu_tlb_fns cpu_tlb __read_only;
#endif
#ifdef MULTI_USER
-struct cpu_user_fns cpu_user __read_mostly;
+struct cpu_user_fns cpu_user __read_only;
#endif
#ifdef MULTI_CACHE
-struct cpu_cache_fns cpu_cache __read_mostly;
+struct cpu_cache_fns cpu_cache __read_only;
#endif
#ifdef CONFIG_OUTER_CACHE
-struct outer_cache_fns outer_cache __read_mostly;
+struct outer_cache_fns outer_cache __read_only;
EXPORT_SYMBOL(outer_cache);
#endif
asm("mrc p15, 0, %0, c0, c1, 4"
: "=r" (mmfr0));
if ((mmfr0 & 0x0000000f) >= 0x00000003 ||
- (mmfr0 & 0x000000f0) >= 0x00000030)
+ (mmfr0 & 0x000000f0) >= 0x00000030) {
cpu_arch = CPU_ARCH_ARMv7;
- else if ((mmfr0 & 0x0000000f) == 0x00000002 ||
+ if ((mmfr0 & 0x0000000f) == 0x00000005 || (mmfr0 & 0x0000000f) == 0x00000004) {
+ __supported_pte_mask |= L_PTE_PXN;
+ __supported_pmd_mask |= PMD_PXNTABLE;
+ }
+ } else if ((mmfr0 & 0x0000000f) == 0x00000002 ||
(mmfr0 & 0x000000f0) == 0x00000020)
cpu_arch = CPU_ARCH_ARMv6;
else
extern const unsigned long sigreturn_codes[7];
-static unsigned long signal_return_offset;
-
#ifdef CONFIG_CRUNCH
static int preserve_crunch_context(struct crunch_sigframe __user *frame)
{
* except when the MPU has protected the vectors
* page from PL0
*/
- retcode = mm->context.sigpage + signal_return_offset +
- (idx << 2) + thumb;
+ retcode = mm->context.sigpage + (idx << 2) + thumb;
} else
#endif
{
} while (thread_flags & _TIF_WORK_MASK);
return 0;
}
-
-struct page *get_signal_page(void)
-{
- unsigned long ptr;
- unsigned offset;
- struct page *page;
- void *addr;
-
- page = alloc_pages(GFP_KERNEL, 0);
-
- if (!page)
- return NULL;
-
- addr = page_address(page);
-
- /* Give the signal return code some randomness */
- offset = 0x200 + (get_random_int() & 0x7fc);
- signal_return_offset = offset;
-
- /*
- * Copy signal return handlers into the vector page, and
- * set sigreturn to be a pointer to these.
- */
- memcpy(addr + offset, sigreturn_codes, sizeof(sigreturn_codes));
-
- ptr = (unsigned long)addr + offset;
- flush_icache_range(ptr, ptr + sizeof(sigreturn_codes));
-
- return page;
-}
static DECLARE_COMPLETION(cpu_running);
-static struct smp_operations smp_ops;
+static struct smp_operations smp_ops __read_only;
void __init smp_set_ops(struct smp_operations *ops)
{
.virtual = ITCM_OFFSET,
.pfn = __phys_to_pfn(ITCM_OFFSET),
.length = 0,
- .type = MT_MEMORY_RWX_ITCM,
+ .type = MT_MEMORY_RX_ITCM,
}
};
start = &__sitcm_text;
end = &__eitcm_text;
ram = &__itcm_start;
+ pax_open_kernel();
memcpy(start, ram, itcm_code_sz);
+ pax_close_kernel();
pr_debug("CPU ITCM: copied code from %p - %p\n",
start, end);
itcm_present = true;
void dump_backtrace_entry(unsigned long where, unsigned long from, unsigned long frame)
{
#ifdef CONFIG_KALLSYMS
- printk("[<%08lx>] (%ps) from [<%08lx>] (%pS)\n", where, (void *)where, from, (void *)from);
+ printk("[<%08lx>] (%pA) from [<%08lx>] (%pA)\n", where, (void *)where, from, (void *)from);
#else
printk("Function entered at [<%08lx>] from [<%08lx>]\n", where, from);
#endif
static int die_owner = -1;
static unsigned int die_nest_count;
+extern void gr_handle_kernel_exploit(void);
+
static unsigned long oops_begin(void)
{
int cpu;
panic("Fatal exception in interrupt");
if (panic_on_oops)
panic("Fatal exception");
+
+ gr_handle_kernel_exploit();
+
if (signr)
do_exit(signr);
}
kuser_init(vectors_base);
flush_icache_range(vectors, vectors + PAGE_SIZE * 2);
- modify_domain(DOMAIN_USER, DOMAIN_CLIENT);
+
+#ifndef CONFIG_PAX_MEMORY_UDEREF
+ modify_domain(DOMAIN_USER, DOMAIN_USERCLIENT);
+#endif
+
#else /* ifndef CONFIG_CPU_V7M */
/*
* on V7-M there is no need to copy the vector table to a dedicated
#endif
#if (defined(CONFIG_SMP_ON_UP) && !defined(CONFIG_DEBUG_SPINLOCK)) || \
- defined(CONFIG_GENERIC_BUG)
+ defined(CONFIG_GENERIC_BUG) || defined(CONFIG_PAX_REFCOUNT)
#define ARM_EXIT_KEEP(x) x
#define ARM_EXIT_DISCARD(x)
#else
#ifdef CONFIG_DEBUG_RODATA
. = ALIGN(1<<SECTION_SHIFT);
#endif
+ _etext = .; /* End of text section */
+
RO_DATA(PAGE_SIZE)
. = ALIGN(4);
NOTES
- _etext = .; /* End of text and rodata section */
-
#ifndef CONFIG_XIP_KERNEL
# ifdef CONFIG_ARM_KERNMEM_PERMS
. = ALIGN(1<<SECTION_SHIFT);
static DEFINE_PER_CPU(struct kvm_vcpu *, kvm_arm_running_vcpu);
/* The VMID used in the VTTBR */
-static atomic64_t kvm_vmid_gen = ATOMIC64_INIT(1);
+static atomic64_unchecked_t kvm_vmid_gen = ATOMIC64_INIT(1);
static u8 kvm_next_vmid;
static DEFINE_SPINLOCK(kvm_vmid_lock);
*/
static bool need_new_vmid_gen(struct kvm *kvm)
{
- return unlikely(kvm->arch.vmid_gen != atomic64_read(&kvm_vmid_gen));
+ return unlikely(kvm->arch.vmid_gen != atomic64_read_unchecked(&kvm_vmid_gen));
}
/**
/* First user of a new VMID generation? */
if (unlikely(kvm_next_vmid == 0)) {
- atomic64_inc(&kvm_vmid_gen);
+ atomic64_inc_unchecked(&kvm_vmid_gen);
kvm_next_vmid = 1;
/*
kvm_call_hyp(__kvm_flush_vm_context);
}
- kvm->arch.vmid_gen = atomic64_read(&kvm_vmid_gen);
+ kvm->arch.vmid_gen = atomic64_read_unchecked(&kvm_vmid_gen);
kvm->arch.vmid = kvm_next_vmid;
kvm_next_vmid++;
/**
* Initialize Hyp-mode and memory mappings on all CPUs.
*/
-int kvm_arch_init(void *opaque)
+int kvm_arch_init(const void *opaque)
{
int err;
int ret, cpu;
.text
-/* Prototype: int __clear_user(void *addr, size_t sz)
+/* Prototype: int ___clear_user(void *addr, size_t sz)
* Purpose : clear some user memory
* Params : addr - user memory address to clear
* : sz - number of bytes to clear
* Returns : number of bytes NOT cleared
*/
ENTRY(__clear_user_std)
-WEAK(__clear_user)
+WEAK(___clear_user)
stmfd sp!, {r1, lr}
mov r2, #0
cmp r1, #4
USER( strnebt r2, [r0])
mov r0, #0
ldmfd sp!, {r1, pc}
-ENDPROC(__clear_user)
+ENDPROC(___clear_user)
ENDPROC(__clear_user_std)
.pushsection .fixup,"ax"
/*
* Prototype:
*
- * size_t __copy_from_user(void *to, const void *from, size_t n)
+ * size_t ___copy_from_user(void *to, const void *from, size_t n)
*
* Purpose:
*
.text
-ENTRY(__copy_from_user)
+ENTRY(___copy_from_user)
#include "copy_template.S"
-ENDPROC(__copy_from_user)
+ENDPROC(___copy_from_user)
.pushsection .fixup,"ax"
.align 0
* ASM optimised string functions
*/
#include <linux/linkage.h>
+#include <linux/const.h>
#include <asm/assembler.h>
#include <asm/asm-offsets.h>
#include <asm/cache.h>
/*
* Prototype:
*
- * size_t __copy_to_user(void *to, const void *from, size_t n)
+ * size_t ___copy_to_user(void *to, const void *from, size_t n)
*
* Purpose:
*
.text
ENTRY(__copy_to_user_std)
-WEAK(__copy_to_user)
+WEAK(___copy_to_user)
#include "copy_template.S"
-ENDPROC(__copy_to_user)
+ENDPROC(___copy_to_user)
ENDPROC(__copy_to_user_std)
.pushsection .fixup,"ax"
* Returns : r0 = checksum, [[sp, #0], #0] = 0 or -EFAULT
*/
-#define FN_ENTRY ENTRY(csum_partial_copy_from_user)
-#define FN_EXIT ENDPROC(csum_partial_copy_from_user)
+#define FN_ENTRY ENTRY(__csum_partial_copy_from_user)
+#define FN_EXIT ENDPROC(__csum_partial_copy_from_user)
#include "csumpartialcopygeneric.S"
/*
* Default to the loop-based delay implementation.
*/
-struct arm_delay_ops arm_delay_ops = {
+struct arm_delay_ops arm_delay_ops __read_only = {
.delay = __loop_delay,
.const_udelay = __loop_const_udelay,
.udelay = __loop_udelay,
}
unsigned long
-__copy_to_user(void __user *to, const void *from, unsigned long n)
+___copy_to_user(void __user *to, const void *from, unsigned long n)
{
/*
* This test is stubbed out of the main function above to keep
return n;
}
-unsigned long __clear_user(void __user *addr, unsigned long n)
+unsigned long ___clear_user(void __user *addr, unsigned long n)
{
/* See rational for this in __copy_to_user() above. */
if (n < 64)
desc->pfn = __phys_to_pfn(base);
desc->length = length;
- desc->type = MT_MEMORY_RWX_NONCACHED;
+ desc->type = MT_MEMORY_RW_NONCACHED;
pr_info("sram at 0x%lx of 0x%x mapped at 0x%lx\n",
base, length, desc->virtual);
#include <linux/syscore_ops.h>
#include <linux/cpu_pm.h>
#include <linux/io.h>
+#include <linux/irq.h>
#include <linux/irqchip/arm-gic.h>
#include <linux/err.h>
#include <linux/regulator/machine.h>
tmp |= pm_data->wake_disable_mask;
pmu_raw_writel(tmp, S5P_WAKEUP_MASK);
- exynos_pm_syscore_ops.suspend = pm_data->pm_suspend;
- exynos_pm_syscore_ops.resume = pm_data->pm_resume;
+ pax_open_kernel();
+ *(void **)&exynos_pm_syscore_ops.suspend = pm_data->pm_suspend;
+ *(void **)&exynos_pm_syscore_ops.resume = pm_data->pm_resume;
+ pax_close_kernel();
register_syscore_ops(&exynos_pm_syscore_ops);
suspend_set_ops(&exynos_suspend_ops);
#include "keystone.h"
-static struct notifier_block platform_nb;
+static notifier_block_no_const platform_nb;
static unsigned long keystone_dma_pfn_offset __read_mostly;
static int keystone_platform_notifier(struct notifier_block *nb,
/*
* This ioremap hook is used on Armada 375/38x to ensure that PCIe
- * memory areas are mapped as MT_UNCACHED instead of MT_DEVICE. This
+ * memory areas are mapped as MT_UNCACHED_RW instead of MT_DEVICE. This
* is needed as a workaround for a deadlock issue between the PCIe
* interface and the cache controller.
*/
mvebu_mbus_get_pcie_mem_aperture(&pcie_mem);
if (pcie_mem.start <= phys_addr && (phys_addr + size) <= pcie_mem.end)
- mtype = MT_UNCACHED;
+ mtype = MT_UNCACHED_RW;
return __arm_ioremap_caller(phys_addr, size, mtype, caller);
}
}
#endif
-struct menelaus_platform_data n8x0_menelaus_platform_data __initdata = {
+struct menelaus_platform_data n8x0_menelaus_platform_data __initconst = {
.late_init = n8x0_menelaus_late_init,
};
void (*resume)(void);
void (*scu_prepare)(unsigned int cpu_id, unsigned int cpu_state);
void (*hotplug_restart)(void);
-};
+} __no_const;
static DEFINE_PER_CPU(struct omap4_cpu_pm_info, omap4_pm_info);
static struct powerdomain *mpuss_pd;
static void dummy_scu_prepare(unsigned int cpu_id, unsigned int cpu_state)
{}
-struct cpu_pm_ops omap_pm_ops = {
+static struct cpu_pm_ops omap_pm_ops __read_only = {
.finish_suspend = default_finish_suspend,
.resume = dummy_cpu_resume,
.scu_prepare = dummy_scu_prepare,
#include <linux/device.h>
#include <linux/smp.h>
#include <linux/io.h>
+#include <linux/irq.h>
#include <linux/irqchip/arm-gic.h>
#include <asm/smp_scu.h>
return NOTIFY_OK;
}
-static struct notifier_block __refdata irq_hotplug_notifier = {
+static struct notifier_block irq_hotplug_notifier = {
.notifier_call = irq_cpu_hotplug_notify,
};
struct platform_device __init *omap_device_build(const char *pdev_name,
int pdev_id,
struct omap_hwmod *oh,
- void *pdata, int pdata_len)
+ const void *pdata, int pdata_len)
{
struct omap_hwmod *ohs[] = { oh };
struct platform_device __init *omap_device_build_ss(const char *pdev_name,
int pdev_id,
struct omap_hwmod **ohs,
- int oh_cnt, void *pdata,
+ int oh_cnt, const void *pdata,
int pdata_len)
{
int ret = -ENOMEM;
/* Core code interface */
struct platform_device *omap_device_build(const char *pdev_name, int pdev_id,
- struct omap_hwmod *oh, void *pdata,
+ struct omap_hwmod *oh, const void *pdata,
int pdata_len);
struct platform_device *omap_device_build_ss(const char *pdev_name, int pdev_id,
struct omap_hwmod **oh, int oh_cnt,
- void *pdata, int pdata_len);
+ const void *pdata, int pdata_len);
struct omap_device *omap_device_alloc(struct platform_device *pdev,
struct omap_hwmod **ohs, int oh_cnt);
int (*init_clkdm)(struct omap_hwmod *oh);
void (*update_context_lost)(struct omap_hwmod *oh);
int (*get_context_lost)(struct omap_hwmod *oh);
-};
+} __no_const;
/* soc_ops: adapts the omap_hwmod code to the currently-booted SoC */
-static struct omap_hwmod_soc_ops soc_ops;
+static struct omap_hwmod_soc_ops soc_ops __read_only;
/* omap_hwmod_list contains all registered struct omap_hwmods */
static LIST_HEAD(omap_hwmod_list);
#include <linux/kernel.h>
#include <linux/init.h>
+#include <asm/pgtable.h>
#include "powerdomain.h"
void __init am43xx_powerdomains_init(void)
{
- omap4_pwrdm_operations.pwrdm_has_voltdm = am43xx_check_vcvp;
+ pax_open_kernel();
+ *(void **)&omap4_pwrdm_operations.pwrdm_has_voltdm = am43xx_check_vcvp;
+ pax_close_kernel();
pwrdm_register_platform_funcs(&omap4_pwrdm_operations);
pwrdm_register_pwrdms(powerdomains_am43xx);
pwrdm_complete_init();
struct omap_hwmod *oh;
char *oh_name = "wd_timer2";
char *dev_name = "omap_wdt";
- struct omap_wd_timer_platform_data pdata;
+ static struct omap_wd_timer_platform_data pdata = {
+ .read_reset_sources = prm_read_reset_sources
+ };
if (!cpu_class_is_omap2() || of_have_populated_dt())
return 0;
return -EINVAL;
}
- pdata.read_reset_sources = prm_read_reset_sources;
-
pdev = omap_device_build(dev_name, id, oh, &pdata,
sizeof(struct omap_wd_timer_platform_data));
WARN(IS_ERR(pdev), "Can't build omap_device for %s:%s.\n",
bool entered_lp2 = false;
if (tegra_pending_sgi())
- ACCESS_ONCE(abort_flag) = true;
+ ACCESS_ONCE_RW(abort_flag) = true;
cpuidle_coupled_parallel_barrier(dev, &abort_barrier);
#include <linux/cpu_pm.h>
#include <linux/interrupt.h>
#include <linux/io.h>
+#include <linux/irq.h>
#include <linux/irqchip/arm-gic.h>
#include <linux/irq.h>
#include <linux/kernel.h>
*/
#include <linux/kernel.h>
+#include <linux/irq.h>
#include <linux/irqchip/arm-gic.h>
#include <linux/delay.h>
#include <linux/io.h>
.type = MT_DEVICE, \
}
-#define __MEM_DEV_DESC(x, sz) { \
- .virtual = IO_ADDRESS(x), \
- .pfn = __phys_to_pfn(x), \
- .length = sz, \
- .type = MT_MEMORY_RWX, \
-}
-
extern struct smp_operations ux500_smp_ops;
extern void ux500_cpu_die(unsigned int cpu);
#include <linux/io.h>
#include <asm/cacheflush.h>
#include <asm/smp_scu.h>
+#include <linux/irq.h>
#include <linux/irqchip/arm-gic.h>
#include "common.h"
config CPU_32v6
bool
+ select CPU_USE_DOMAINS if CPU_V6 && MMU && !PAX_KERNEXEC && !PAX_MEMORY_UDEREF
select TLS_REG_EMUL if !CPU_32v6K && !MMU
config CPU_32v6K
config CPU_USE_DOMAINS
bool
+ depends on !ARM_LPAE && !PAX_KERNEXEC && !PAX_MEMORY_UDEREF
help
This option enables or disables the use of domain switching
via the set_fs() function.
config KUSER_HELPERS
bool "Enable kuser helpers in vector page" if !NEED_KUSER_HELPERS
- depends on MMU
+ depends on MMU && (!(CPU_V6 || CPU_V6K || CPU_V7) || GRKERNSEC_OLD_ARM_USERLAND)
default y
help
Warning: disabling this option may break user programs.
See Documentation/arm/kernel_user_helpers.txt for details.
However, the fixed address nature of these helpers can be used
- by ROP (return orientated programming) authors when creating
+ by ROP (Return Oriented Programming) authors when creating
exploits.
If all of the binaries and libraries which run on your platform
#define __get16_unaligned_check(ins,val,addr) \
do { \
unsigned int err = 0, v, a = addr; \
+ pax_open_userland(); \
__get8_unaligned_check(ins,v,a,err); \
val = v << ((BE) ? 8 : 0); \
__get8_unaligned_check(ins,v,a,err); \
val |= v << ((BE) ? 0 : 8); \
+ pax_close_userland(); \
if (err) \
goto fault; \
} while (0)
#define __get32_unaligned_check(ins,val,addr) \
do { \
unsigned int err = 0, v, a = addr; \
+ pax_open_userland(); \
__get8_unaligned_check(ins,v,a,err); \
val = v << ((BE) ? 24 : 0); \
__get8_unaligned_check(ins,v,a,err); \
val |= v << ((BE) ? 8 : 16); \
__get8_unaligned_check(ins,v,a,err); \
val |= v << ((BE) ? 0 : 24); \
+ pax_close_userland(); \
if (err) \
goto fault; \
} while (0)
#define __put16_unaligned_check(ins,val,addr) \
do { \
unsigned int err = 0, v = val, a = addr; \
+ pax_open_userland(); \
__asm__( FIRST_BYTE_16 \
ARM( "1: "ins" %1, [%2], #1\n" ) \
THUMB( "1: "ins" %1, [%2]\n" ) \
" .popsection\n" \
: "=r" (err), "=&r" (v), "=&r" (a) \
: "0" (err), "1" (v), "2" (a)); \
+ pax_close_userland(); \
if (err) \
goto fault; \
} while (0)
#define __put32_unaligned_check(ins,val,addr) \
do { \
unsigned int err = 0, v = val, a = addr; \
+ pax_open_userland(); \
__asm__( FIRST_BYTE_32 \
ARM( "1: "ins" %1, [%2], #1\n" ) \
THUMB( "1: "ins" %1, [%2]\n" ) \
" .popsection\n" \
: "=r" (err), "=&r" (v), "=&r" (a) \
: "0" (err), "1" (v), "2" (a)); \
+ pax_close_userland(); \
if (err) \
goto fault; \
} while (0)
void (*fixup)(void __iomem *, u32, struct outer_cache_fns *);
void (*save)(void __iomem *);
struct outer_cache_fns outer_cache;
-};
+} __do_const;
#define CACHE_LINE_SIZE 32
#define NUM_USER_ASIDS ASID_FIRST_VERSION
static DEFINE_RAW_SPINLOCK(cpu_asid_lock);
-static atomic64_t asid_generation = ATOMIC64_INIT(ASID_FIRST_VERSION);
+static atomic64_unchecked_t asid_generation = ATOMIC64_INIT(ASID_FIRST_VERSION);
static DECLARE_BITMAP(asid_map, NUM_USER_ASIDS);
static DEFINE_PER_CPU(atomic64_t, active_asids);
{
static u32 cur_idx = 1;
u64 asid = atomic64_read(&mm->context.id);
- u64 generation = atomic64_read(&asid_generation);
+ u64 generation = atomic64_read_unchecked(&asid_generation);
if (asid != 0) {
/*
*/
asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, cur_idx);
if (asid == NUM_USER_ASIDS) {
- generation = atomic64_add_return(ASID_FIRST_VERSION,
+ generation = atomic64_add_return_unchecked(ASID_FIRST_VERSION,
&asid_generation);
flush_context(cpu);
asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, 1);
cpu_set_reserved_ttbr0();
asid = atomic64_read(&mm->context.id);
- if (!((asid ^ atomic64_read(&asid_generation)) >> ASID_BITS)
+ if (!((asid ^ atomic64_read_unchecked(&asid_generation)) >> ASID_BITS)
&& atomic64_xchg(&per_cpu(active_asids, cpu), asid))
goto switch_mm_fastpath;
raw_spin_lock_irqsave(&cpu_asid_lock, flags);
/* Check that our ASID belongs to the current generation. */
asid = atomic64_read(&mm->context.id);
- if ((asid ^ atomic64_read(&asid_generation)) >> ASID_BITS) {
+ if ((asid ^ atomic64_read_unchecked(&asid_generation)) >> ASID_BITS) {
asid = new_context(mm, cpu);
atomic64_set(&mm->context.id, asid);
}
#include <asm/system_misc.h>
#include <asm/system_info.h>
#include <asm/tlbflush.h>
+#include <asm/sections.h>
#include "fault.h"
if (fixup_exception(regs))
return;
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ if (addr < TASK_SIZE) {
+ if (current->signal->curr_ip)
+ printk(KERN_ERR "PAX: From %pI4: %s:%d, uid/euid: %u/%u, attempted to access userland memory at %08lx\n", ¤t->signal->curr_ip, current->comm, task_pid_nr(current),
+ from_kuid_munged(&init_user_ns, current_uid()), from_kuid_munged(&init_user_ns, current_euid()), addr);
+ else
+ printk(KERN_ERR "PAX: %s:%d, uid/euid: %u/%u, attempted to access userland memory at %08lx\n", current->comm, task_pid_nr(current),
+ from_kuid_munged(&init_user_ns, current_uid()), from_kuid_munged(&init_user_ns, current_euid()), addr);
+ }
+#endif
+
+#ifdef CONFIG_PAX_KERNEXEC
+ if ((fsr & FSR_WRITE) &&
+ (((unsigned long)_stext <= addr && addr < init_mm.end_code) ||
+ (MODULES_VADDR <= addr && addr < MODULES_END)))
+ {
+ if (current->signal->curr_ip)
+ printk(KERN_ERR "PAX: From %pI4: %s:%d, uid/euid: %u/%u, attempted to modify kernel code\n", ¤t->signal->curr_ip, current->comm, task_pid_nr(current),
+ from_kuid_munged(&init_user_ns, current_uid()), from_kuid_munged(&init_user_ns, current_euid()));
+ else
+ printk(KERN_ERR "PAX: %s:%d, uid/euid: %u/%u, attempted to modify kernel code\n", current->comm, task_pid_nr(current),
+ from_kuid_munged(&init_user_ns, current_uid()), from_kuid_munged(&init_user_ns, current_euid()));
+ }
+#endif
+
/*
* No handler, we'll have to terminate things with extreme prejudice.
*/
}
#endif
+#ifdef CONFIG_PAX_PAGEEXEC
+ if (fsr & FSR_LNX_PF) {
+ pax_report_fault(regs, (void *)regs->ARM_pc, (void *)regs->ARM_sp);
+ do_group_exit(SIGKILL);
+ }
+#endif
+
tsk->thread.address = addr;
tsk->thread.error_code = fsr;
tsk->thread.trap_no = 14;
}
#endif /* CONFIG_MMU */
+#ifdef CONFIG_PAX_PAGEEXEC
+void pax_report_insns(struct pt_regs *regs, void *pc, void *sp)
+{
+ long i;
+
+ printk(KERN_ERR "PAX: bytes at PC: ");
+ for (i = 0; i < 20; i++) {
+ unsigned char c;
+ if (get_user(c, (__force unsigned char __user *)pc+i))
+ printk(KERN_CONT "?? ");
+ else
+ printk(KERN_CONT "%02x ", c);
+ }
+ printk("\n");
+
+ printk(KERN_ERR "PAX: bytes at SP-4: ");
+ for (i = -1; i < 20; i++) {
+ unsigned long c;
+ if (get_user(c, (__force unsigned long __user *)sp+i))
+ printk(KERN_CONT "???????? ");
+ else
+ printk(KERN_CONT "%08lx ", c);
+ }
+ printk("\n");
+}
+#endif
+
/*
* First Level Translation Fault Handler
*
const struct fsr_info *inf = fsr_info + fsr_fs(fsr);
struct siginfo info;
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ if (addr < TASK_SIZE && is_domain_fault(fsr)) {
+ if (current->signal->curr_ip)
+ printk(KERN_ERR "PAX: From %pI4: %s:%d, uid/euid: %u/%u, attempted to access userland memory at %08lx\n", ¤t->signal->curr_ip, current->comm, task_pid_nr(current),
+ from_kuid_munged(&init_user_ns, current_uid()), from_kuid_munged(&init_user_ns, current_euid()), addr);
+ else
+ printk(KERN_ERR "PAX: %s:%d, uid/euid: %u/%u, attempted to access userland memory at %08lx\n", current->comm, task_pid_nr(current),
+ from_kuid_munged(&init_user_ns, current_uid()), from_kuid_munged(&init_user_ns, current_euid()), addr);
+ goto die;
+ }
+#endif
+
if (!inf->fn(addr, fsr & ~FSR_LNX_PF, regs))
return;
+die:
pr_alert("Unhandled fault: %s (0x%03x) at 0x%08lx\n",
inf->name, fsr, addr);
ifsr_info[nr].name = name;
}
+asmlinkage int sys_sigreturn(struct pt_regs *regs);
+asmlinkage int sys_rt_sigreturn(struct pt_regs *regs);
+
asmlinkage void __exception
do_PrefetchAbort(unsigned long addr, unsigned int ifsr, struct pt_regs *regs)
{
const struct fsr_info *inf = ifsr_info + fsr_fs(ifsr);
struct siginfo info;
+ unsigned long pc = instruction_pointer(regs);
+
+ if (user_mode(regs)) {
+ unsigned long sigpage = current->mm->context.sigpage;
+
+ if (sigpage <= pc && pc < sigpage + 7*4) {
+ if (pc < sigpage + 3*4)
+ sys_sigreturn(regs);
+ else
+ sys_rt_sigreturn(regs);
+ return;
+ }
+ if (pc == 0xffff0f60UL) {
+ /*
+ * PaX: __kuser_cmpxchg64 emulation
+ */
+ // TODO
+ //regs->ARM_pc = regs->ARM_lr;
+ //return;
+ }
+ if (pc == 0xffff0fa0UL) {
+ /*
+ * PaX: __kuser_memory_barrier emulation
+ */
+ // dmb(); implied by the exception
+ regs->ARM_pc = regs->ARM_lr;
+ return;
+ }
+ if (pc == 0xffff0fc0UL) {
+ /*
+ * PaX: __kuser_cmpxchg emulation
+ */
+ // TODO
+ //long new;
+ //int op;
+
+ //op = FUTEX_OP_SET << 28;
+ //new = futex_atomic_op_inuser(op, regs->ARM_r2);
+ //regs->ARM_r0 = old != new;
+ //regs->ARM_pc = regs->ARM_lr;
+ //return;
+ }
+ if (pc == 0xffff0fe0UL) {
+ /*
+ * PaX: __kuser_get_tls emulation
+ */
+ regs->ARM_r0 = current_thread_info()->tp_value[0];
+ regs->ARM_pc = regs->ARM_lr;
+ return;
+ }
+ }
+
+#if defined(CONFIG_PAX_KERNEXEC) || defined(CONFIG_PAX_MEMORY_UDEREF)
+ else if (is_domain_fault(ifsr) || is_xn_fault(ifsr)) {
+ if (current->signal->curr_ip)
+ printk(KERN_ERR "PAX: From %pI4: %s:%d, uid/euid: %u/%u, attempted to execute %s memory at %08lx\n", ¤t->signal->curr_ip, current->comm, task_pid_nr(current),
+ from_kuid_munged(&init_user_ns, current_uid()), from_kuid_munged(&init_user_ns, current_euid()),
+ pc >= TASK_SIZE ? "non-executable kernel" : "userland", pc);
+ else
+ printk(KERN_ERR "PAX: %s:%d, uid/euid: %u/%u, attempted to execute %s memory at %08lx\n", current->comm, task_pid_nr(current),
+ from_kuid_munged(&init_user_ns, current_uid()), from_kuid_munged(&init_user_ns, current_euid()),
+ pc >= TASK_SIZE ? "non-executable kernel" : "userland", pc);
+ goto die;
+ }
+#endif
+
+#ifdef CONFIG_PAX_REFCOUNT
+ if (fsr_fs(ifsr) == FAULT_CODE_DEBUG) {
+#ifdef CONFIG_THUMB2_KERNEL
+ unsigned short bkpt;
+
+ if (!probe_kernel_address(pc, bkpt) && cpu_to_le16(bkpt) == 0xbef1) {
+#else
+ unsigned int bkpt;
+
+ if (!probe_kernel_address(pc, bkpt) && cpu_to_le32(bkpt) == 0xe12f1073) {
+#endif
+ current->thread.error_code = ifsr;
+ current->thread.trap_no = 0;
+ pax_report_refcount_overflow(regs);
+ fixup_exception(regs);
+ return;
+ }
+ }
+#endif
if (!inf->fn(addr, ifsr | FSR_LNX_PF, regs))
return;
+die:
pr_alert("Unhandled prefetch abort: %s (0x%03x) at 0x%08lx\n",
inf->name, ifsr, addr);
/*
* Fault status register encodings. We steal bit 31 for our own purposes.
+ * Set when the FSR value is from an instruction fault.
*/
#define FSR_LNX_PF (1 << 31)
#define FSR_WRITE (1 << 11)
}
#endif
+/* valid for LPAE and !LPAE */
+static inline int is_xn_fault(unsigned int fsr)
+{
+ return ((fsr_fs(fsr) & 0x3c) == 0xc);
+}
+
+static inline int is_domain_fault(unsigned int fsr)
+{
+ return ((fsr_fs(fsr) & 0xD) == 0x9);
+}
+
void do_bad_area(unsigned long addr, unsigned int fsr, struct pt_regs *regs);
unsigned long search_exception_table(unsigned long addr);
{
#ifdef CONFIG_HAVE_TCM
extern char __tcm_start, __tcm_end;
+#endif
+
+#ifdef CONFIG_PAX_KERNEXEC
+ unsigned long addr;
+ pgd_t *pgd;
+ pud_t *pud;
+ pmd_t *pmd;
+ int cpu_arch = cpu_architecture();
+ unsigned int cr = get_cr();
+
+ if (cpu_arch >= CPU_ARCH_ARMv6 && (cr & CR_XP)) {
+ /* make pages tables, etc before .text NX */
+ for (addr = PAGE_OFFSET; addr < (unsigned long)_stext; addr += SECTION_SIZE) {
+ pgd = pgd_offset_k(addr);
+ pud = pud_offset(pgd, addr);
+ pmd = pmd_offset(pud, addr);
+ __section_update(pmd, addr, PMD_SECT_XN);
+ }
+ /* make init NX */
+ for (addr = (unsigned long)__init_begin; addr < (unsigned long)_sdata; addr += SECTION_SIZE) {
+ pgd = pgd_offset_k(addr);
+ pud = pud_offset(pgd, addr);
+ pmd = pmd_offset(pud, addr);
+ __section_update(pmd, addr, PMD_SECT_XN);
+ }
+ /* make kernel code/rodata RX */
+ for (addr = (unsigned long)_stext; addr < (unsigned long)__init_begin; addr += SECTION_SIZE) {
+ pgd = pgd_offset_k(addr);
+ pud = pud_offset(pgd, addr);
+ pmd = pmd_offset(pud, addr);
+#ifdef CONFIG_ARM_LPAE
+ __section_update(pmd, addr, PMD_SECT_RDONLY);
+#else
+ __section_update(pmd, addr, PMD_SECT_APX|PMD_SECT_AP_WRITE);
+#endif
+ }
+ }
+#endif
+#ifdef CONFIG_HAVE_TCM
poison_init_mem(&__tcm_start, &__tcm_end - &__tcm_start);
free_reserved_area(&__tcm_start, &__tcm_end, -1, "TCM link");
#endif
unsigned int mtype;
if (cached)
- mtype = MT_MEMORY_RWX;
+ mtype = MT_MEMORY_RX;
else
- mtype = MT_MEMORY_RWX_NONCACHED;
+ mtype = MT_MEMORY_RX_NONCACHED;
return __arm_ioremap_caller(phys_addr, size, mtype,
__builtin_return_address(0));
struct vm_area_struct *vma;
int do_align = 0;
int aliasing = cache_is_vipt_aliasing();
+ unsigned long offset = gr_rand_threadstack_offset(mm, filp, flags);
struct vm_unmapped_area_info info;
/*
if (len > TASK_SIZE)
return -ENOMEM;
+#ifdef CONFIG_PAX_RANDMMAP
+ if (!(mm->pax_flags & MF_PAX_RANDMMAP))
+#endif
+
if (addr) {
if (do_align)
addr = COLOUR_ALIGN(addr, pgoff);
addr = PAGE_ALIGN(addr);
vma = find_vma(mm, addr);
- if (TASK_SIZE - len >= addr &&
- (!vma || addr + len <= vma->vm_start))
+ if (TASK_SIZE - len >= addr && check_heap_stack_gap(vma, addr, len, offset))
return addr;
}
info.high_limit = TASK_SIZE;
info.align_mask = do_align ? (PAGE_MASK & (SHMLBA - 1)) : 0;
info.align_offset = pgoff << PAGE_SHIFT;
+ info.threadstack_offset = offset;
return vm_unmapped_area(&info);
}
unsigned long addr = addr0;
int do_align = 0;
int aliasing = cache_is_vipt_aliasing();
+ unsigned long offset = gr_rand_threadstack_offset(mm, filp, flags);
struct vm_unmapped_area_info info;
/*
return addr;
}
+#ifdef CONFIG_PAX_RANDMMAP
+ if (!(mm->pax_flags & MF_PAX_RANDMMAP))
+#endif
+
/* requesting a specific address */
if (addr) {
if (do_align)
else
addr = PAGE_ALIGN(addr);
vma = find_vma(mm, addr);
- if (TASK_SIZE - len >= addr &&
- (!vma || addr + len <= vma->vm_start))
+ if (TASK_SIZE - len >= addr && check_heap_stack_gap(vma, addr, len, offset))
return addr;
}
info.high_limit = mm->mmap_base;
info.align_mask = do_align ? (PAGE_MASK & (SHMLBA - 1)) : 0;
info.align_offset = pgoff << PAGE_SHIFT;
+ info.threadstack_offset = offset;
addr = vm_unmapped_area(&info);
/*
{
unsigned long random_factor = 0UL;
+#ifdef CONFIG_PAX_RANDMMAP
+ if (!(mm->pax_flags & MF_PAX_RANDMMAP))
+#endif
+
/* 8 bits of randomness in 20 address space bits */
if ((current->flags & PF_RANDOMIZE) &&
!(current->personality & ADDR_NO_RANDOMIZE))
if (mmap_is_legacy()) {
mm->mmap_base = TASK_UNMAPPED_BASE + random_factor;
+
+#ifdef CONFIG_PAX_RANDMMAP
+ if (mm->pax_flags & MF_PAX_RANDMMAP)
+ mm->mmap_base += mm->delta_mmap;
+#endif
+
mm->get_unmapped_area = arch_get_unmapped_area;
} else {
mm->mmap_base = mmap_base(random_factor);
+
+#ifdef CONFIG_PAX_RANDMMAP
+ if (mm->pax_flags & MF_PAX_RANDMMAP)
+ mm->mmap_base -= mm->delta_mmap + mm->delta_stack;
+#endif
+
mm->get_unmapped_area = arch_get_unmapped_area_topdown;
}
}
#include "mm.h"
#include "tcm.h"
+#if defined(CONFIG_CPU_USE_DOMAINS) || defined(CONFIG_PAX_KERNEXEC) || defined(CONFIG_PAX_MEMORY_UDEREF)
+void modify_domain(unsigned int dom, unsigned int type)
+{
+ struct thread_info *thread = current_thread_info();
+ unsigned int domain = thread->cpu_domain;
+ /*
+ * DOMAIN_MANAGER might be defined to some other value,
+ * use the arch-defined constant
+ */
+ domain &= ~domain_val(dom, 3);
+ thread->cpu_domain = domain | domain_val(dom, type);
+ set_domain(thread->cpu_domain);
+}
+EXPORT_SYMBOL(modify_domain);
+#endif
+
/*
* empty_zero_page is a special page that is used for
* zero-initialized data and COW.
#define PROT_PTE_S2_DEVICE PROT_PTE_DEVICE
#define PROT_SECT_DEVICE PMD_TYPE_SECT|PMD_SECT_AP_WRITE
-static struct mem_type mem_types[] = {
+#ifdef CONFIG_PAX_KERNEXEC
+#define L_PTE_KERNEXEC L_PTE_RDONLY
+#define PMD_SECT_KERNEXEC PMD_SECT_RDONLY
+#else
+#define L_PTE_KERNEXEC L_PTE_DIRTY
+#define PMD_SECT_KERNEXEC PMD_SECT_AP_WRITE
+#endif
+
+static struct mem_type mem_types[] __read_only = {
[MT_DEVICE] = { /* Strongly ordered / ARMv6 shared device */
.prot_pte = PROT_PTE_DEVICE | L_PTE_MT_DEV_SHARED |
L_PTE_SHARED,
.prot_sect = PROT_SECT_DEVICE,
.domain = DOMAIN_IO,
},
- [MT_UNCACHED] = {
+ [MT_UNCACHED_RW] = {
.prot_pte = PROT_PTE_DEVICE,
.prot_l1 = PMD_TYPE_TABLE,
.prot_sect = PMD_TYPE_SECT | PMD_SECT_XN,
.domain = DOMAIN_IO,
},
- [MT_CACHECLEAN] = {
- .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN,
+ [MT_CACHECLEAN_RO] = {
+ .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN | PMD_SECT_RDONLY,
.domain = DOMAIN_KERNEL,
},
#ifndef CONFIG_ARM_LPAE
- [MT_MINICLEAN] = {
- .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN | PMD_SECT_MINICACHE,
+ [MT_MINICLEAN_RO] = {
+ .prot_sect = PMD_TYPE_SECT | PMD_SECT_MINICACHE | PMD_SECT_XN | PMD_SECT_RDONLY,
.domain = DOMAIN_KERNEL,
},
#endif
.prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
L_PTE_RDONLY,
.prot_l1 = PMD_TYPE_TABLE,
- .domain = DOMAIN_USER,
+ .domain = DOMAIN_VECTORS,
},
[MT_HIGH_VECTORS] = {
.prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
L_PTE_USER | L_PTE_RDONLY,
.prot_l1 = PMD_TYPE_TABLE,
- .domain = DOMAIN_USER,
+ .domain = DOMAIN_VECTORS,
},
- [MT_MEMORY_RWX] = {
+ [__MT_MEMORY_RWX] = {
.prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY,
.prot_l1 = PMD_TYPE_TABLE,
.prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE,
.prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE,
.domain = DOMAIN_KERNEL,
},
- [MT_ROM] = {
- .prot_sect = PMD_TYPE_SECT,
+ [MT_MEMORY_RX] = {
+ .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_KERNEXEC,
+ .prot_l1 = PMD_TYPE_TABLE,
+ .prot_sect = PMD_TYPE_SECT | PMD_SECT_KERNEXEC,
+ .domain = DOMAIN_KERNEL,
+ },
+ [MT_ROM_RX] = {
+ .prot_sect = PMD_TYPE_SECT | PMD_SECT_RDONLY,
.domain = DOMAIN_KERNEL,
},
- [MT_MEMORY_RWX_NONCACHED] = {
+ [MT_MEMORY_RW_NONCACHED] = {
.prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
L_PTE_MT_BUFFERABLE,
.prot_l1 = PMD_TYPE_TABLE,
.prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE,
.domain = DOMAIN_KERNEL,
},
+ [MT_MEMORY_RX_NONCACHED] = {
+ .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_KERNEXEC |
+ L_PTE_MT_BUFFERABLE,
+ .prot_l1 = PMD_TYPE_TABLE,
+ .prot_sect = PMD_TYPE_SECT | PMD_SECT_KERNEXEC,
+ .domain = DOMAIN_KERNEL,
+ },
[MT_MEMORY_RW_DTCM] = {
.prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
L_PTE_XN,
.prot_sect = PMD_TYPE_SECT | PMD_SECT_XN,
.domain = DOMAIN_KERNEL,
},
- [MT_MEMORY_RWX_ITCM] = {
- .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY,
+ [MT_MEMORY_RX_ITCM] = {
+ .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_KERNEXEC,
.prot_l1 = PMD_TYPE_TABLE,
+ .prot_sect = PMD_TYPE_SECT | PMD_SECT_KERNEXEC,
.domain = DOMAIN_KERNEL,
},
[MT_MEMORY_RW_SO] = {
* Mark cache clean areas and XIP ROM read only
* from SVC mode and no access from userspace.
*/
- mem_types[MT_ROM].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
- mem_types[MT_MINICLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
- mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
+ mem_types[MT_ROM_RX].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
+#ifdef CONFIG_PAX_KERNEXEC
+ mem_types[MT_MEMORY_RX].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
+ mem_types[MT_MEMORY_RX_NONCACHED].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
+ mem_types[MT_MEMORY_RX_ITCM].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
+#endif
+ mem_types[MT_MINICLEAN_RO].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
+ mem_types[MT_CACHECLEAN_RO].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
#endif
/*
mem_types[MT_DEVICE_WC].prot_pte |= L_PTE_SHARED;
mem_types[MT_DEVICE_CACHED].prot_sect |= PMD_SECT_S;
mem_types[MT_DEVICE_CACHED].prot_pte |= L_PTE_SHARED;
- mem_types[MT_MEMORY_RWX].prot_sect |= PMD_SECT_S;
- mem_types[MT_MEMORY_RWX].prot_pte |= L_PTE_SHARED;
+ mem_types[__MT_MEMORY_RWX].prot_sect |= PMD_SECT_S;
+ mem_types[__MT_MEMORY_RWX].prot_pte |= L_PTE_SHARED;
mem_types[MT_MEMORY_RW].prot_sect |= PMD_SECT_S;
mem_types[MT_MEMORY_RW].prot_pte |= L_PTE_SHARED;
+ mem_types[MT_MEMORY_RX].prot_sect |= PMD_SECT_S;
+ mem_types[MT_MEMORY_RX].prot_pte |= L_PTE_SHARED;
mem_types[MT_MEMORY_DMA_READY].prot_pte |= L_PTE_SHARED;
- mem_types[MT_MEMORY_RWX_NONCACHED].prot_sect |= PMD_SECT_S;
- mem_types[MT_MEMORY_RWX_NONCACHED].prot_pte |= L_PTE_SHARED;
+ mem_types[MT_MEMORY_RW_NONCACHED].prot_sect |= PMD_SECT_S;
+ mem_types[MT_MEMORY_RW_NONCACHED].prot_pte |= L_PTE_SHARED;
+ mem_types[MT_MEMORY_RX_NONCACHED].prot_sect |= PMD_SECT_S;
+ mem_types[MT_MEMORY_RX_NONCACHED].prot_pte |= L_PTE_SHARED;
}
}
if (cpu_arch >= CPU_ARCH_ARMv6) {
if (cpu_arch >= CPU_ARCH_ARMv7 && (cr & CR_TRE)) {
/* Non-cacheable Normal is XCB = 001 */
- mem_types[MT_MEMORY_RWX_NONCACHED].prot_sect |=
+ mem_types[MT_MEMORY_RW_NONCACHED].prot_sect |=
+ PMD_SECT_BUFFERED;
+ mem_types[MT_MEMORY_RX_NONCACHED].prot_sect |=
PMD_SECT_BUFFERED;
} else {
/* For both ARMv6 and non-TEX-remapping ARMv7 */
- mem_types[MT_MEMORY_RWX_NONCACHED].prot_sect |=
+ mem_types[MT_MEMORY_RW_NONCACHED].prot_sect |=
+ PMD_SECT_TEX(1);
+ mem_types[MT_MEMORY_RX_NONCACHED].prot_sect |=
PMD_SECT_TEX(1);
}
} else {
- mem_types[MT_MEMORY_RWX_NONCACHED].prot_sect |= PMD_SECT_BUFFERABLE;
+ mem_types[MT_MEMORY_RW_NONCACHED].prot_sect |= PMD_SECT_BUFFERABLE;
+ mem_types[MT_MEMORY_RX_NONCACHED].prot_sect |= PMD_SECT_BUFFERABLE;
}
#ifdef CONFIG_ARM_LPAE
user_pgprot |= PTE_EXT_PXN;
#endif
+ user_pgprot |= __supported_pte_mask;
+
for (i = 0; i < 16; i++) {
pteval_t v = pgprot_val(protection_map[i]);
protection_map[i] = __pgprot(v | user_pgprot);
mem_types[MT_LOW_VECTORS].prot_l1 |= ecc_mask;
mem_types[MT_HIGH_VECTORS].prot_l1 |= ecc_mask;
- mem_types[MT_MEMORY_RWX].prot_sect |= ecc_mask | cp->pmd;
- mem_types[MT_MEMORY_RWX].prot_pte |= kern_pgprot;
+ mem_types[__MT_MEMORY_RWX].prot_sect |= ecc_mask | cp->pmd;
+ mem_types[__MT_MEMORY_RWX].prot_pte |= kern_pgprot;
mem_types[MT_MEMORY_RW].prot_sect |= ecc_mask | cp->pmd;
mem_types[MT_MEMORY_RW].prot_pte |= kern_pgprot;
+ mem_types[MT_MEMORY_RX].prot_sect |= ecc_mask | cp->pmd;
+ mem_types[MT_MEMORY_RX].prot_pte |= kern_pgprot;
mem_types[MT_MEMORY_DMA_READY].prot_pte |= kern_pgprot;
- mem_types[MT_MEMORY_RWX_NONCACHED].prot_sect |= ecc_mask;
- mem_types[MT_ROM].prot_sect |= cp->pmd;
+ mem_types[MT_MEMORY_RW_NONCACHED].prot_sect |= ecc_mask;
+ mem_types[MT_MEMORY_RX_NONCACHED].prot_sect |= ecc_mask;
+ mem_types[MT_ROM_RX].prot_sect |= cp->pmd;
switch (cp->pmd) {
case PMD_SECT_WT:
- mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_WT;
+ mem_types[MT_CACHECLEAN_RO].prot_sect |= PMD_SECT_WT;
break;
case PMD_SECT_WB:
case PMD_SECT_WBWA:
- mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_WB;
+ mem_types[MT_CACHECLEAN_RO].prot_sect |= PMD_SECT_WB;
break;
}
pr_info("Memory policy: %sData cache %s\n",
return;
}
- if ((md->type == MT_DEVICE || md->type == MT_ROM) &&
+ if ((md->type == MT_DEVICE || md->type == MT_ROM_RX) &&
md->virtual >= PAGE_OFFSET &&
(md->virtual < VMALLOC_START || md->virtual >= VMALLOC_END)) {
pr_warn("BUG: mapping for 0x%08llx at 0x%08lx out of vmalloc space\n",
* called function. This means you can't use any function or debugging
* method which may touch any device, otherwise the kernel _will_ crash.
*/
+
+static char vectors[PAGE_SIZE * 2] __read_only __aligned(PAGE_SIZE);
+
static void __init devicemaps_init(const struct machine_desc *mdesc)
{
struct map_desc map;
unsigned long addr;
- void *vectors;
- /*
- * Allocate the vector page early.
- */
- vectors = early_alloc(PAGE_SIZE * 2);
-
- early_trap_init(vectors);
+ early_trap_init(&vectors);
for (addr = VMALLOC_START; addr; addr += PMD_SIZE)
pmd_clear(pmd_off_k(addr));
map.pfn = __phys_to_pfn(CONFIG_XIP_PHYS_ADDR & SECTION_MASK);
map.virtual = MODULES_VADDR;
map.length = ((unsigned long)_etext - map.virtual + ~SECTION_MASK) & SECTION_MASK;
- map.type = MT_ROM;
+ map.type = MT_ROM_RX;
create_mapping(&map);
#endif
map.pfn = __phys_to_pfn(FLUSH_BASE_PHYS);
map.virtual = FLUSH_BASE;
map.length = SZ_1M;
- map.type = MT_CACHECLEAN;
+ map.type = MT_CACHECLEAN_RO;
create_mapping(&map);
#endif
#ifdef FLUSH_BASE_MINICACHE
map.pfn = __phys_to_pfn(FLUSH_BASE_PHYS + SZ_1M);
map.virtual = FLUSH_BASE_MINICACHE;
map.length = SZ_1M;
- map.type = MT_MINICLEAN;
+ map.type = MT_MINICLEAN_RO;
create_mapping(&map);
#endif
* location (0xffff0000). If we aren't using high-vectors, also
* create a mapping at the low-vectors virtual address.
*/
- map.pfn = __phys_to_pfn(virt_to_phys(vectors));
+ map.pfn = __phys_to_pfn(virt_to_phys(&vectors));
map.virtual = 0xffff0000;
map.length = PAGE_SIZE;
#ifdef CONFIG_KUSER_HELPERS
static void __init map_lowmem(void)
{
struct memblock_region *reg;
+#ifndef CONFIG_PAX_KERNEXEC
phys_addr_t kernel_x_start = round_down(__pa(_stext), SECTION_SIZE);
phys_addr_t kernel_x_end = round_up(__pa(__init_end), SECTION_SIZE);
+#endif
/* Map all the lowmem memory banks. */
for_each_memblock(memory, reg) {
if (start >= end)
break;
+#ifdef CONFIG_PAX_KERNEXEC
+ map.pfn = __phys_to_pfn(start);
+ map.virtual = __phys_to_virt(start);
+ map.length = end - start;
+
+ if (map.virtual <= (unsigned long)_stext && ((unsigned long)_end < (map.virtual + map.length))) {
+ struct map_desc kernel;
+ struct map_desc initmap;
+
+ /* when freeing initmem we will make this RW */
+ initmap.pfn = __phys_to_pfn(__pa(__init_begin));
+ initmap.virtual = (unsigned long)__init_begin;
+ initmap.length = _sdata - __init_begin;
+ initmap.type = __MT_MEMORY_RWX;
+ create_mapping(&initmap);
+
+ /* when freeing initmem we will make this RX */
+ kernel.pfn = __phys_to_pfn(__pa(_stext));
+ kernel.virtual = (unsigned long)_stext;
+ kernel.length = __init_begin - _stext;
+ kernel.type = __MT_MEMORY_RWX;
+ create_mapping(&kernel);
+
+ if (map.virtual < (unsigned long)_stext) {
+ map.length = (unsigned long)_stext - map.virtual;
+ map.type = __MT_MEMORY_RWX;
+ create_mapping(&map);
+ }
+
+ map.pfn = __phys_to_pfn(__pa(_sdata));
+ map.virtual = (unsigned long)_sdata;
+ map.length = end - __pa(_sdata);
+ }
+
+ map.type = MT_MEMORY_RW;
+ create_mapping(&map);
+#else
if (end < kernel_x_start) {
map.pfn = __phys_to_pfn(start);
map.virtual = __phys_to_virt(start);
map.length = end - start;
- map.type = MT_MEMORY_RWX;
+ map.type = __MT_MEMORY_RWX;
create_mapping(&map);
} else if (start >= kernel_x_end) {
map.pfn = __phys_to_pfn(kernel_x_start);
map.virtual = __phys_to_virt(kernel_x_start);
map.length = kernel_x_end - kernel_x_start;
- map.type = MT_MEMORY_RWX;
+ map.type = __MT_MEMORY_RWX;
create_mapping(&map);
create_mapping(&map);
}
}
+#endif
}
}
#include <asm/cacheflush.h>
#include <asm/hwcap.h>
#include <asm/opcodes.h>
+#include <asm/pgtable.h>
#include "bpf_jit_32.h"
#endif
};
+#ifdef CONFIG_GRKERNSEC_BPF_HARDEN
+int bpf_jit_enable __read_only;
+#else
int bpf_jit_enable __read_mostly;
+#endif
static u64 jit_get_skb_b(struct sk_buff *skb, unsigned offset)
{
{
u32 *ptr;
/* We are guaranteed to have aligned memory. */
+ pax_open_kernel();
for (ptr = area; size >= sizeof(u32); size -= sizeof(u32))
*ptr++ = __opcode_to_mem_arm(ARM_INST_UDF);
+ pax_close_kernel();
}
static void build_prologue(struct jit_ctx *ctx)
.virtual = IOP3XX_PERIPHERAL_VIRT_BASE,
.pfn = __phys_to_pfn(IOP3XX_PERIPHERAL_PHYS_BASE),
.length = IOP3XX_PERIPHERAL_SIZE,
- .type = MT_UNCACHED,
+ .type = MT_UNCACHED_RW,
},
};
* Looks like we need to preserve some bootloader code at the
* beginning of SRAM for jumping to flash for reboot to work...
*/
+ pax_open_kernel();
memset_io(omap_sram_base + omap_sram_skip, 0,
omap_sram_size - omap_sram_skip);
+ pax_close_kernel();
}
int (*started)(unsigned ch);
int (*flush)(unsigned ch);
int (*stop)(unsigned ch);
-};
+} __no_const;
extern void *samsung_dmadev_get_ops(void);
extern void *s3c_dma_get_ops(void);
do { \
compiletime_assert_atomic_type(*p); \
barrier(); \
- ACCESS_ONCE(*p) = (v); \
+ ACCESS_ONCE_RW(*p) = (v); \
} while (0)
#define smp_load_acquire(p) \
{
switch (size) {
case 1:
- ACCESS_ONCE(*(u8 *)ptr) = (u8)val;
+ ACCESS_ONCE_RW(*(u8 *)ptr) = (u8)val;
break;
case 2:
- ACCESS_ONCE(*(u16 *)ptr) = (u16)val;
+ ACCESS_ONCE_RW(*(u16 *)ptr) = (u16)val;
break;
case 4:
- ACCESS_ONCE(*(u32 *)ptr) = (u32)val;
+ ACCESS_ONCE_RW(*(u32 *)ptr) = (u32)val;
break;
case 8:
- ACCESS_ONCE(*(u64 *)ptr) = (u64)val;
+ ACCESS_ONCE_RW(*(u64 *)ptr) = (u64)val;
break;
default:
BUILD_BUG();
flag; \
})
+#define access_ok_noprefault(type, addr, size) access_ok((type), (addr), (size))
#define access_ok(type, addr, size) __range_ok(addr, size)
#define user_addr_max get_fs
#ifndef __ASM_AVR32_CACHE_H
#define __ASM_AVR32_CACHE_H
+#include <linux/const.h>
+
#define L1_CACHE_SHIFT 5
-#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT)
+#define L1_CACHE_BYTES (_AC(1,UL) << L1_CACHE_SHIFT)
/*
* Memory returned by kmalloc() may be used for DMA, so we must make
the loader. We need to make sure that it is out of the way of the program
that it will "exec", and that there is sufficient room for the brk. */
-#define ELF_ET_DYN_BASE (2 * TASK_SIZE / 3)
+#define ELF_ET_DYN_BASE (TASK_SIZE / 3 * 2)
+#ifdef CONFIG_PAX_ASLR
+#define PAX_ELF_ET_DYN_BASE 0x00001000UL
+
+#define PAX_DELTA_MMAP_LEN 15
+#define PAX_DELTA_STACK_LEN 15
+#endif
/* This yields a mask that user programs can use to figure out what
instruction set this CPU supports. This could be done in user space,
#define __ASM_AVR32_KMAP_TYPES_H
#ifdef CONFIG_DEBUG_HIGHMEM
-# define KM_TYPE_NR 29
+# define KM_TYPE_NR 30
#else
-# define KM_TYPE_NR 14
+# define KM_TYPE_NR 15
#endif
#endif /* __ASM_AVR32_KMAP_TYPES_H */
int exception_trace = 1;
+#ifdef CONFIG_PAX_PAGEEXEC
+void pax_report_insns(struct pt_regs *regs, void *pc, void *sp)
+{
+ unsigned long i;
+
+ printk(KERN_ERR "PAX: bytes at PC: ");
+ for (i = 0; i < 20; i++) {
+ unsigned char c;
+ if (get_user(c, (unsigned char *)pc+i))
+ printk(KERN_CONT "???????? ");
+ else
+ printk(KERN_CONT "%02x ", c);
+ }
+ printk("\n");
+}
+#endif
+
/*
* This routine handles page faults. It determines the address and the
* problem, and then passes it off to one of the appropriate routines.
up_read(&mm->mmap_sem);
if (user_mode(regs)) {
+
+#ifdef CONFIG_PAX_PAGEEXEC
+ if (mm->pax_flags & MF_PAX_PAGEEXEC) {
+ if (ecr == ECR_PROTECTION_X || ecr == ECR_TLB_MISS_X) {
+ pax_report_fault(regs, (void *)regs->pc, (void *)regs->sp);
+ do_group_exit(SIGKILL);
+ }
+ }
+#endif
+
if (exception_trace && printk_ratelimit())
printk("%s%s[%d]: segfault at %08lx pc %08lx "
"sp %08lx ecr %lu\n",
#ifndef __ARCH_BLACKFIN_CACHE_H
#define __ARCH_BLACKFIN_CACHE_H
+#include <linux/const.h>
#include <linux/linkage.h> /* for asmlinkage */
/*
* Blackfin loads 32 bytes for cache
*/
#define L1_CACHE_SHIFT 5
-#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT)
+#define L1_CACHE_BYTES (_AC(1,UL) << L1_CACHE_SHIFT)
#define SMP_CACHE_BYTES L1_CACHE_BYTES
#define ARCH_DMA_MINALIGN L1_CACHE_BYTES
#ifndef _ASM_ARCH_CACHE_H
#define _ASM_ARCH_CACHE_H
+#include <linux/const.h>
/* Etrax 100LX have 32-byte cache-lines. */
-#define L1_CACHE_BYTES 32
#define L1_CACHE_SHIFT 5
+#define L1_CACHE_BYTES (_AC(1,UL) << L1_CACHE_SHIFT)
#endif /* _ASM_ARCH_CACHE_H */
#ifndef _ASM_CRIS_ARCH_CACHE_H
#define _ASM_CRIS_ARCH_CACHE_H
+#include <linux/const.h>
#include <arch/hwregs/dma.h>
/* A cache-line is 32 bytes. */
-#define L1_CACHE_BYTES 32
#define L1_CACHE_SHIFT 5
+#define L1_CACHE_BYTES (_AC(1,UL) << L1_CACHE_SHIFT)
#define __read_mostly __attribute__((__section__(".data..read_mostly")))
#define atomic64_cmpxchg(v, old, new) (__cmpxchg_64(old, new, &(v)->counter))
#define atomic64_xchg(v, new) (__xchg_64(new, &(v)->counter))
+#define atomic64_read_unchecked(v) atomic64_read(v)
+#define atomic64_set_unchecked(v, i) atomic64_set((v), (i))
+#define atomic64_add_unchecked(a, v) atomic64_add((a), (v))
+#define atomic64_add_return_unchecked(a, v) atomic64_add_return((a), (v))
+#define atomic64_sub_unchecked(a, v) atomic64_sub((a), (v))
+#define atomic64_inc_unchecked(v) atomic64_inc(v)
+#define atomic64_inc_return_unchecked(v) atomic64_inc_return(v)
+#define atomic64_dec_unchecked(v) atomic64_dec(v)
+#define atomic64_cmpxchg_unchecked(v, o, n) atomic64_cmpxchg((v), (o), (n))
+
static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u)
{
int c, old;
#ifndef __ASM_CACHE_H
#define __ASM_CACHE_H
+#include <linux/const.h>
/* bytes per L1 cache line */
#define L1_CACHE_SHIFT (CONFIG_FRV_L1_CACHE_SHIFT)
-#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT)
+#define L1_CACHE_BYTES (_AC(1,UL) << L1_CACHE_SHIFT)
#define __cacheline_aligned __attribute__((aligned(L1_CACHE_BYTES)))
#define ____cacheline_aligned __attribute__((aligned(L1_CACHE_BYTES)))
#ifndef _ASM_KMAP_TYPES_H
#define _ASM_KMAP_TYPES_H
-#define KM_TYPE_NR 17
+#define KM_TYPE_NR 18
#endif
{
struct vm_area_struct *vma;
struct vm_unmapped_area_info info;
+ unsigned long offset = gr_rand_threadstack_offset(current->mm, filp, flags);
if (len > TASK_SIZE)
return -ENOMEM;
if (addr) {
addr = PAGE_ALIGN(addr);
vma = find_vma(current->mm, addr);
- if (TASK_SIZE - len >= addr &&
- (!vma || addr + len <= vma->vm_start))
+ if (TASK_SIZE - len >= addr && check_heap_stack_gap(vma, addr, len, offset))
goto success;
}
info.high_limit = (current->mm->start_stack - 0x00200000);
info.align_mask = 0;
info.align_offset = 0;
+ info.threadstack_offset = offset;
addr = vm_unmapped_area(&info);
if (!(addr & ~PAGE_MASK))
goto success;
#ifndef __ASM_CACHE_H
#define __ASM_CACHE_H
+#include <linux/const.h>
+
/* Bytes per L1 cache line */
-#define L1_CACHE_SHIFT (5)
-#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT)
+#define L1_CACHE_SHIFT 5
+#define L1_CACHE_BYTES (_AC(1,UL) << L1_CACHE_SHIFT)
#define ARCH_DMA_MINALIGN L1_CACHE_BYTES
config KEXEC
bool "kexec system call"
depends on !IA64_HP_SIM && (!SMP || HOTPLUG_CPU)
+ depends on !GRKERNSEC_KMEM
help
kexec is a system call that implements the ability to shutdown your
current kernel, and to start another kernel. It is like a reboot
archprepare: make_nr_irqs_h FORCE
PHONY += make_nr_irqs_h FORCE
+make_nr_irqs_h: KBUILD_CFLAGS := $(filter-out $(GCC_PLUGINS_CFLAGS),$(KBUILD_CFLAGS))
make_nr_irqs_h: FORCE
$(Q)$(MAKE) $(build)=arch/ia64/kernel include/generated/nr-irqs.h
#define atomic64_inc(v) atomic64_add(1, (v))
#define atomic64_dec(v) atomic64_sub(1, (v))
+#define atomic64_read_unchecked(v) atomic64_read(v)
+#define atomic64_set_unchecked(v, i) atomic64_set((v), (i))
+#define atomic64_add_unchecked(a, v) atomic64_add((a), (v))
+#define atomic64_add_return_unchecked(a, v) atomic64_add_return((a), (v))
+#define atomic64_sub_unchecked(a, v) atomic64_sub((a), (v))
+#define atomic64_inc_unchecked(v) atomic64_inc(v)
+#define atomic64_inc_return_unchecked(v) atomic64_inc_return(v)
+#define atomic64_dec_unchecked(v) atomic64_dec(v)
+#define atomic64_cmpxchg_unchecked(v, o, n) atomic64_cmpxchg((v), (o), (n))
+
#endif /* _ASM_IA64_ATOMIC_H */
do { \
compiletime_assert_atomic_type(*p); \
barrier(); \
- ACCESS_ONCE(*p) = (v); \
+ ACCESS_ONCE_RW(*p) = (v); \
} while (0)
#define smp_load_acquire(p) \
#ifndef _ASM_IA64_CACHE_H
#define _ASM_IA64_CACHE_H
+#include <linux/const.h>
/*
* Copyright (C) 1998-2000 Hewlett-Packard Co
/* Bytes per L1 (data) cache line. */
#define L1_CACHE_SHIFT CONFIG_IA64_L1_CACHE_SHIFT
-#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT)
+#define L1_CACHE_BYTES (_AC(1,UL) << L1_CACHE_SHIFT)
#ifdef CONFIG_SMP
# define SMP_CACHE_SHIFT L1_CACHE_SHIFT
*/
#define ELF_ET_DYN_BASE (TASK_UNMAPPED_BASE + 0x800000000UL)
+#ifdef CONFIG_PAX_ASLR
+#define PAX_ELF_ET_DYN_BASE (current->personality == PER_LINUX32 ? 0x08048000UL : 0x4000000000000000UL)
+
+#define PAX_DELTA_MMAP_LEN (current->personality == PER_LINUX32 ? 16 : 3*PAGE_SHIFT - 13)
+#define PAX_DELTA_STACK_LEN (current->personality == PER_LINUX32 ? 16 : 3*PAGE_SHIFT - 13)
+#endif
+
#define PT_IA_64_UNWIND 0x70000001
/* IA-64 relocations: */
pgd_val(*pgd_entry) = __pa(pud);
}
+static inline void
+pgd_populate_kernel(struct mm_struct *mm, pgd_t * pgd_entry, pud_t * pud)
+{
+ pgd_populate(mm, pgd_entry, pud);
+}
+
static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
{
return quicklist_alloc(0, GFP_KERNEL, NULL);
pud_val(*pud_entry) = __pa(pmd);
}
+static inline void
+pud_populate_kernel(struct mm_struct *mm, pud_t * pud_entry, pmd_t * pmd)
+{
+ pud_populate(mm, pud_entry, pmd);
+}
+
static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
{
return quicklist_alloc(0, GFP_KERNEL, NULL);
* David Mosberger-Tang <davidm@hpl.hp.com>
*/
-
+#include <linux/const.h>
#include <asm/mman.h>
#include <asm/page.h>
#include <asm/processor.h>
#define PAGE_READONLY __pgprot(__ACCESS_BITS | _PAGE_PL_3 | _PAGE_AR_R)
#define PAGE_COPY __pgprot(__ACCESS_BITS | _PAGE_PL_3 | _PAGE_AR_R)
#define PAGE_COPY_EXEC __pgprot(__ACCESS_BITS | _PAGE_PL_3 | _PAGE_AR_RX)
+
+#ifdef CONFIG_PAX_PAGEEXEC
+# define PAGE_SHARED_NOEXEC __pgprot(__ACCESS_BITS | _PAGE_PL_3 | _PAGE_AR_RW)
+# define PAGE_READONLY_NOEXEC __pgprot(__ACCESS_BITS | _PAGE_PL_3 | _PAGE_AR_R)
+# define PAGE_COPY_NOEXEC __pgprot(__ACCESS_BITS | _PAGE_PL_3 | _PAGE_AR_R)
+#else
+# define PAGE_SHARED_NOEXEC PAGE_SHARED
+# define PAGE_READONLY_NOEXEC PAGE_READONLY
+# define PAGE_COPY_NOEXEC PAGE_COPY
+#endif
+
#define PAGE_GATE __pgprot(__ACCESS_BITS | _PAGE_PL_0 | _PAGE_AR_X_RX)
#define PAGE_KERNEL __pgprot(__DIRTY_BITS | _PAGE_PL_0 | _PAGE_AR_RWX)
#define PAGE_KERNELRX __pgprot(__ACCESS_BITS | _PAGE_PL_0 | _PAGE_AR_RX)
unsigned short *p = (unsigned short *)&lock->lock + 1, tmp;
asm volatile ("ld2.bias %0=[%1]" : "=r"(tmp) : "r"(p));
- ACCESS_ONCE(*p) = (tmp + 2) & ~1;
+ ACCESS_ONCE_RW(*p) = (tmp + 2) & ~1;
}
static __always_inline void __ticket_spin_unlock_wait(arch_spinlock_t *lock)
&& ((segment).seg == KERNEL_DS.seg \
|| likely(REGION_OFFSET((unsigned long) (addr)) < RGN_MAP_LIMIT))); \
})
+#define access_ok_noprefault(type, addr, size) access_ok((type), (addr), (size))
#define access_ok(type, addr, size) __access_ok((addr), (size), get_fs())
/*
static inline unsigned long
__copy_to_user (void __user *to, const void *from, unsigned long count)
{
+ if (count > INT_MAX)
+ return count;
+
+ if (!__builtin_constant_p(count))
+ check_object_size(from, count, true);
+
return __copy_user(to, (__force void __user *) from, count);
}
static inline unsigned long
__copy_from_user (void *to, const void __user *from, unsigned long count)
{
+ if (count > INT_MAX)
+ return count;
+
+ if (!__builtin_constant_p(count))
+ check_object_size(to, count, false);
+
return __copy_user((__force void __user *) to, from, count);
}
({ \
void __user *__cu_to = (to); \
const void *__cu_from = (from); \
- long __cu_len = (n); \
+ unsigned long __cu_len = (n); \
\
- if (__access_ok(__cu_to, __cu_len, get_fs())) \
+ if (__cu_len <= INT_MAX && __access_ok(__cu_to, __cu_len, get_fs())) { \
+ if (!__builtin_constant_p(n)) \
+ check_object_size(__cu_from, __cu_len, true); \
__cu_len = __copy_user(__cu_to, (__force void __user *) __cu_from, __cu_len); \
+ } \
__cu_len; \
})
({ \
void *__cu_to = (to); \
const void __user *__cu_from = (from); \
- long __cu_len = (n); \
+ unsigned long __cu_len = (n); \
\
__chk_user_ptr(__cu_from); \
- if (__access_ok(__cu_from, __cu_len, get_fs())) \
+ if (__cu_len <= INT_MAX && __access_ok(__cu_from, __cu_len, get_fs())) { \
+ if (!__builtin_constant_p(n)) \
+ check_object_size(__cu_to, __cu_len, false); \
__cu_len = __copy_user((__force void __user *) __cu_to, __cu_from, __cu_len); \
+ } \
__cu_len; \
})
return 0;
}
+static inline int
+in_init_rx (const struct module *mod, uint64_t addr)
+{
+ return addr - (uint64_t) mod->module_init_rx < mod->init_size_rx;
+}
+
+static inline int
+in_init_rw (const struct module *mod, uint64_t addr)
+{
+ return addr - (uint64_t) mod->module_init_rw < mod->init_size_rw;
+}
+
static inline int
in_init (const struct module *mod, uint64_t addr)
{
- return addr - (uint64_t) mod->module_init < mod->init_size;
+ return in_init_rx(mod, addr) || in_init_rw(mod, addr);
+}
+
+static inline int
+in_core_rx (const struct module *mod, uint64_t addr)
+{
+ return addr - (uint64_t) mod->module_core_rx < mod->core_size_rx;
+}
+
+static inline int
+in_core_rw (const struct module *mod, uint64_t addr)
+{
+ return addr - (uint64_t) mod->module_core_rw < mod->core_size_rw;
}
static inline int
in_core (const struct module *mod, uint64_t addr)
{
- return addr - (uint64_t) mod->module_core < mod->core_size;
+ return in_core_rx(mod, addr) || in_core_rw(mod, addr);
}
static inline int
break;
case RV_BDREL:
- val -= (uint64_t) (in_init(mod, val) ? mod->module_init : mod->module_core);
+ if (in_init_rx(mod, val))
+ val -= (uint64_t) mod->module_init_rx;
+ else if (in_init_rw(mod, val))
+ val -= (uint64_t) mod->module_init_rw;
+ else if (in_core_rx(mod, val))
+ val -= (uint64_t) mod->module_core_rx;
+ else if (in_core_rw(mod, val))
+ val -= (uint64_t) mod->module_core_rw;
break;
case RV_LTV:
* addresses have been selected...
*/
uint64_t gp;
- if (mod->core_size > MAX_LTOFF)
+ if (mod->core_size_rx + mod->core_size_rw > MAX_LTOFF)
/*
* This takes advantage of fact that SHF_ARCH_SMALL gets allocated
* at the end of the module.
*/
- gp = mod->core_size - MAX_LTOFF / 2;
+ gp = mod->core_size_rx + mod->core_size_rw - MAX_LTOFF / 2;
else
- gp = mod->core_size / 2;
- gp = (uint64_t) mod->module_core + ((gp + 7) & -8);
+ gp = (mod->core_size_rx + mod->core_size_rw) / 2;
+ gp = (uint64_t) mod->module_core_rx + ((gp + 7) & -8);
mod->arch.gp = gp;
DEBUGP("%s: placing gp at 0x%lx\n", __func__, gp);
}
return NOTIFY_OK;
}
-static struct notifier_block __refdata palinfo_cpu_notifier =
+static struct notifier_block palinfo_cpu_notifier =
{
.notifier_call = palinfo_cpu_callback,
.priority = 0,
unsigned long align_mask = 0;
struct mm_struct *mm = current->mm;
struct vm_unmapped_area_info info;
+ unsigned long offset = gr_rand_threadstack_offset(mm, filp, flags);
if (len > RGN_MAP_LIMIT)
return -ENOMEM;
if (REGION_NUMBER(addr) == RGN_HPAGE)
addr = 0;
#endif
+
+#ifdef CONFIG_PAX_RANDMMAP
+ if (mm->pax_flags & MF_PAX_RANDMMAP)
+ addr = mm->free_area_cache;
+ else
+#endif
+
if (!addr)
addr = TASK_UNMAPPED_BASE;
info.high_limit = TASK_SIZE;
info.align_mask = align_mask;
info.align_offset = 0;
+ info.threadstack_offset = offset;
return vm_unmapped_area(&info);
}
/* Per-cpu data: */
. = ALIGN(PERCPU_PAGE_SIZE);
PERCPU_VADDR(SMP_CACHE_BYTES, PERCPU_ADDR, :percpu)
- __phys_per_cpu_start = __per_cpu_load;
+ __phys_per_cpu_start = per_cpu_load;
/*
* ensure percpu data fits
* into percpu page size
return pte_present(pte);
}
+#ifdef CONFIG_PAX_PAGEEXEC
+void pax_report_insns(struct pt_regs *regs, void *pc, void *sp)
+{
+ unsigned long i;
+
+ printk(KERN_ERR "PAX: bytes at PC: ");
+ for (i = 0; i < 8; i++) {
+ unsigned int c;
+ if (get_user(c, (unsigned int *)pc+i))
+ printk(KERN_CONT "???????? ");
+ else
+ printk(KERN_CONT "%08x ", c);
+ }
+ printk("\n");
+}
+#endif
+
# define VM_READ_BIT 0
# define VM_WRITE_BIT 1
# define VM_EXEC_BIT 2
if (((isr >> IA64_ISR_R_BIT) & 1UL) && (!(vma->vm_flags & (VM_READ | VM_WRITE))))
goto bad_area;
- if ((vma->vm_flags & mask) != mask)
+ if ((vma->vm_flags & mask) != mask) {
+
+#ifdef CONFIG_PAX_PAGEEXEC
+ if (!(vma->vm_flags & VM_EXEC) && (mask & VM_EXEC)) {
+ if (!(mm->pax_flags & MF_PAX_PAGEEXEC) || address != regs->cr_iip)
+ goto bad_area;
+
+ up_read(&mm->mmap_sem);
+ pax_report_fault(regs, (void *)regs->cr_iip, (void *)regs->r12);
+ do_group_exit(SIGKILL);
+ }
+#endif
+
goto bad_area;
+ }
/*
* If for any reason at all we couldn't handle the fault, make
unsigned long pgoff, unsigned long flags)
{
struct vm_unmapped_area_info info;
+ unsigned long offset = gr_rand_threadstack_offset(current->mm, file, flags);
if (len > RGN_MAP_LIMIT)
return -ENOMEM;
info.high_limit = HPAGE_REGION_BASE + RGN_MAP_LIMIT;
info.align_mask = PAGE_MASK & (HPAGE_SIZE - 1);
info.align_offset = 0;
+ info.threadstack_offset = offset;
return vm_unmapped_area(&info);
}
vma->vm_start = current->thread.rbs_bot & PAGE_MASK;
vma->vm_end = vma->vm_start + PAGE_SIZE;
vma->vm_flags = VM_DATA_DEFAULT_FLAGS|VM_GROWSUP|VM_ACCOUNT;
+
+#ifdef CONFIG_PAX_PAGEEXEC
+ if (current->mm->pax_flags & MF_PAX_PAGEEXEC) {
+ vma->vm_flags &= ~VM_EXEC;
+
+#ifdef CONFIG_PAX_MPROTECT
+ if (current->mm->pax_flags & MF_PAX_MPROTECT)
+ vma->vm_flags &= ~VM_MAYEXEC;
+#endif
+
+ }
+#endif
+
vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
down_write(¤t->mm->mmap_sem);
if (insert_vm_struct(current->mm, vma)) {
gate_vma.vm_start = FIXADDR_USER_START;
gate_vma.vm_end = FIXADDR_USER_END;
gate_vma.vm_flags = VM_READ | VM_MAYREAD | VM_EXEC | VM_MAYEXEC;
- gate_vma.vm_page_prot = __P101;
+ gate_vma.vm_page_prot = vm_get_page_prot(gate_vma.vm_flags);
return 0;
}
#ifndef _ASM_M32R_CACHE_H
#define _ASM_M32R_CACHE_H
+#include <linux/const.h>
+
/* L1 cache line size */
#define L1_CACHE_SHIFT 4
-#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT)
+#define L1_CACHE_BYTES (_AC(1,UL) << L1_CACHE_SHIFT)
#endif /* _ASM_M32R_CACHE_H */
unsigned long
__generic_copy_to_user(void __user *to, const void *from, unsigned long n)
{
+ if ((long)n < 0)
+ return n;
+
prefetch(from);
if (access_ok(VERIFY_WRITE, to, n))
__copy_user(to,from,n);
unsigned long
__generic_copy_from_user(void *to, const void __user *from, unsigned long n)
{
+ if ((long)n < 0)
+ return n;
+
prefetchw(to);
if (access_ok(VERIFY_READ, from, n))
__copy_user_zeroing(to,from,n);
#ifndef __ARCH_M68K_CACHE_H
#define __ARCH_M68K_CACHE_H
+#include <linux/const.h>
+
/* bytes per L1 cache line */
#define L1_CACHE_SHIFT 4
-#define L1_CACHE_BYTES (1<< L1_CACHE_SHIFT)
+#define L1_CACHE_BYTES (_AC(1,UL) << L1_CACHE_SHIFT)
#define ARCH_DMA_MINALIGN L1_CACHE_BYTES
do { \
compiletime_assert_atomic_type(*p); \
smp_mb(); \
- ACCESS_ONCE(*p) = (v); \
+ ACCESS_ONCE_RW(*p) = (v); \
} while (0)
#define smp_load_acquire(p) \
info.high_limit = TASK_SIZE;
info.align_mask = PAGE_MASK & HUGEPT_MASK;
info.align_offset = 0;
+ info.threadstack_offset = 0;
return vm_unmapped_area(&info);
}
#ifndef _ASM_MICROBLAZE_CACHE_H
#define _ASM_MICROBLAZE_CACHE_H
+#include <linux/const.h>
#include <asm/registers.h>
#define L1_CACHE_SHIFT 5
/* word-granular cache in microblaze */
-#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT)
+#define L1_CACHE_BYTES (_AC(1,UL) << L1_CACHE_SHIFT)
#define SMP_CACHE_BYTES L1_CACHE_BYTES
config KEXEC
bool "Kexec system call"
+ depends on !GRKERNSEC_KMEM
help
kexec is a system call that implements the ability to shutdown your
current kernel, and to start another kernel. It is like a reboot
if (dma_release_from_coherent(dev, order, vaddr))
return;
- swiotlb_free_coherent(dev, size, vaddr, dma_handle);
+ swiotlb_free_coherent(dev, size, vaddr, dma_handle, attrs);
}
static dma_addr_t octeon_unity_phys_to_dma(struct device *dev, phys_addr_t paddr)
#include <asm/cmpxchg.h>
#include <asm/war.h>
+#ifdef CONFIG_GENERIC_ATOMIC64
+#include <asm-generic/atomic64.h>
+#endif
+
#define ATOMIC_INIT(i) { (i) }
+#ifdef CONFIG_64BIT
+#define _ASM_EXTABLE(from, to) \
+" .section __ex_table,\"a\"\n" \
+" .dword " #from ", " #to"\n" \
+" .previous\n"
+#else
+#define _ASM_EXTABLE(from, to) \
+" .section __ex_table,\"a\"\n" \
+" .word " #from ", " #to"\n" \
+" .previous\n"
+#endif
+
/*
* atomic_read - read atomic variable
* @v: pointer of type atomic_t
*
* Atomically reads the value of @v.
*/
-#define atomic_read(v) ACCESS_ONCE((v)->counter)
+static inline int atomic_read(const atomic_t *v)
+{
+ return ACCESS_ONCE(v->counter);
+}
+
+static inline int atomic_read_unchecked(const atomic_unchecked_t *v)
+{
+ return ACCESS_ONCE(v->counter);
+}
/*
* atomic_set - set atomic variable
*
* Atomically sets the value of @v to @i.
*/
-#define atomic_set(v, i) ((v)->counter = (i))
+static inline void atomic_set(atomic_t *v, int i)
+{
+ v->counter = i;
+}
-#define ATOMIC_OP(op, c_op, asm_op) \
-static __inline__ void atomic_##op(int i, atomic_t * v) \
+static inline void atomic_set_unchecked(atomic_unchecked_t *v, int i)
+{
+ v->counter = i;
+}
+
+#ifdef CONFIG_PAX_REFCOUNT
+#define __OVERFLOW_POST \
+ " b 4f \n" \
+ " .set noreorder \n" \
+ "3: b 5f \n" \
+ " move %0, %1 \n" \
+ " .set reorder \n"
+#define __OVERFLOW_EXTABLE \
+ "3:\n" \
+ _ASM_EXTABLE(2b, 3b)
+#else
+#define __OVERFLOW_POST
+#define __OVERFLOW_EXTABLE
+#endif
+
+#define __ATOMIC_OP(op, suffix, asm_op, extable) \
+static inline void atomic_##op##suffix(int i, atomic##suffix##_t * v) \
{ \
if (kernel_uses_llsc && R10000_LLSC_WAR) { \
int temp; \
\
__asm__ __volatile__( \
- " .set arch=r4000 \n" \
- "1: ll %0, %1 # atomic_" #op " \n" \
- " " #asm_op " %0, %2 \n" \
+ " .set mips3 \n" \
+ "1: ll %0, %1 # atomic_" #op #suffix "\n" \
+ "2: " #asm_op " %0, %2 \n" \
" sc %0, %1 \n" \
" beqzl %0, 1b \n" \
+ extable \
" .set mips0 \n" \
: "=&r" (temp), "+" GCC_OFF12_ASM() (v->counter) \
: "Ir" (i)); \
} else if (kernel_uses_llsc) { \
int temp; \
\
- do { \
- __asm__ __volatile__( \
- " .set arch=r4000 \n" \
- " ll %0, %1 # atomic_" #op "\n" \
- " " #asm_op " %0, %2 \n" \
- " sc %0, %1 \n" \
- " .set mips0 \n" \
- : "=&r" (temp), "+" GCC_OFF12_ASM() (v->counter) \
- : "Ir" (i)); \
- } while (unlikely(!temp)); \
+ __asm__ __volatile__( \
+ " .set mips3 \n" \
+ "1: ll %0, %1 # atomic_" #op #suffix "\n" \
+ "2: " #asm_op " %0, %2 \n" \
+ " sc %0, %1 \n" \
+ " beqz %0, 1b \n" \
+ extable \
+ " .set mips0 \n" \
+ : "=&r" (temp), "+" GCC_OFF12_ASM() (v->counter) \
+ : "Ir" (i)); \
} else { \
unsigned long flags; \
\
raw_local_irq_save(flags); \
- v->counter c_op i; \
+ __asm__ __volatile__( \
+ "2: " #asm_op " %0, %1 \n" \
+ extable \
+ : "+r" (v->counter) : "Ir" (i)); \
raw_local_irq_restore(flags); \
} \
}
-#define ATOMIC_OP_RETURN(op, c_op, asm_op) \
-static __inline__ int atomic_##op##_return(int i, atomic_t * v) \
+#define ATOMIC_OP(op, asm_op) __ATOMIC_OP(op, , asm_op##u) \
+ __ATOMIC_OP(op, _unchecked, asm_op)
+
+#define __ATOMIC_OP_RETURN(op, suffix, asm_op, post_op, extable) \
+static inline int atomic_##op##_return##suffix(int i, atomic##suffix##_t * v) \
{ \
int result; \
\
int temp; \
\
__asm__ __volatile__( \
- " .set arch=r4000 \n" \
- "1: ll %1, %2 # atomic_" #op "_return \n" \
- " " #asm_op " %0, %1, %3 \n" \
+ " .set mips3 \n" \
+ "1: ll %1, %2 # atomic_" #op "_return" #suffix"\n" \
+ "2: " #asm_op " %0, %1, %3 \n" \
" sc %0, %2 \n" \
" beqzl %0, 1b \n" \
- " " #asm_op " %0, %1, %3 \n" \
+ post_op \
+ extable \
+ "4: " #asm_op " %0, %1, %3 \n" \
+ "5: \n" \
" .set mips0 \n" \
: "=&r" (result), "=&r" (temp), \
"+" GCC_OFF12_ASM() (v->counter) \
} else if (kernel_uses_llsc) { \
int temp; \
\
- do { \
- __asm__ __volatile__( \
- " .set arch=r4000 \n" \
- " ll %1, %2 # atomic_" #op "_return \n" \
- " " #asm_op " %0, %1, %3 \n" \
- " sc %0, %2 \n" \
- " .set mips0 \n" \
- : "=&r" (result), "=&r" (temp), \
- "+" GCC_OFF12_ASM() (v->counter) \
- : "Ir" (i)); \
- } while (unlikely(!result)); \
+ __asm__ __volatile__( \
+ " .set mips3 \n" \
+ "1: ll %1, %2 # atomic_" #op "_return" #suffix "\n" \
+ "2: " #asm_op " %0, %1, %3 \n" \
+ " sc %0, %2 \n" \
+ post_op \
+ extable \
+ "4: " #asm_op " %0, %1, %3 \n" \
+ "5: \n" \
+ " .set mips0 \n" \
+ : "=&r" (result), "=&r" (temp), \
+ "+" GCC_OFF12_ASM() (v->counter) \
+ : "Ir" (i)); \
\
result = temp; result c_op i; \
} else { \
unsigned long flags; \
\
raw_local_irq_save(flags); \
- result = v->counter; \
- result c_op i; \
- v->counter = result; \
+ __asm__ __volatile__( \
+ " lw %0, %1 \n" \
+ "2: " #asm_op " %0, %1, %2 \n" \
+ " sw %0, %1 \n" \
+ "3: \n" \
+ extable \
+ : "=&r" (result), "+" GCC_OFF12_ASM() (v->counter) \
+ : "Ir" (i)); \
raw_local_irq_restore(flags); \
} \
\
return result; \
}
-#define ATOMIC_OPS(op, c_op, asm_op) \
- ATOMIC_OP(op, c_op, asm_op) \
- ATOMIC_OP_RETURN(op, c_op, asm_op)
+#define ATOMIC_OP_RETURN(op, asm_op) __ATOMIC_OP_RETURN(op, , asm_op##u, , __OVERFLOW_EXTABLE) \
+ __ATOMIC_OP_RETURN(op, _unchecked, asm_op, __OVERFLOW_POST, __OVERFLOW_EXTABLE)
+
+#define ATOMIC_OPS(op, asm_op) \
+ ATOMIC_OP(op, asm_op) \
+ ATOMIC_OP_RETURN(op, asm_op)
-ATOMIC_OPS(add, +=, addu)
-ATOMIC_OPS(sub, -=, subu)
+ATOMIC_OPS(add, add)
+ATOMIC_OPS(sub, sub)
#undef ATOMIC_OPS
#undef ATOMIC_OP_RETURN
+#undef __ATOMIC_OP_RETURN
#undef ATOMIC_OP
+#undef __ATOMIC_OP
/*
* atomic_sub_if_positive - conditionally subtract integer from atomic variable
* Atomically test @v and subtract @i if @v is greater or equal than @i.
* The function returns the old value of @v minus @i.
*/
-static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
+static __inline__ int atomic_sub_if_positive(int i, atomic_t *v)
{
int result;
return result;
}
-#define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
-#define atomic_xchg(v, new) (xchg(&((v)->counter), (new)))
+static inline int atomic_cmpxchg(atomic_t *v, int old, int new)
+{
+ return cmpxchg(&v->counter, old, new);
+}
+
+static inline int atomic_cmpxchg_unchecked(atomic_unchecked_t *v, int old,
+ int new)
+{
+ return cmpxchg(&(v->counter), old, new);
+}
+
+static inline int atomic_xchg(atomic_t *v, int new)
+{
+ return xchg(&v->counter, new);
+}
+
+static inline int atomic_xchg_unchecked(atomic_unchecked_t *v, int new)
+{
+ return xchg(&(v->counter), new);
+}
/**
* __atomic_add_unless - add unless the number is a given value
#define atomic_dec_return(v) atomic_sub_return(1, (v))
#define atomic_inc_return(v) atomic_add_return(1, (v))
+static __inline__ int atomic_inc_return_unchecked(atomic_unchecked_t *v)
+{
+ return atomic_add_return_unchecked(1, v);
+}
/*
* atomic_sub_and_test - subtract value from variable and test result
* other cases.
*/
#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0)
+static __inline__ int atomic_inc_and_test_unchecked(atomic_unchecked_t *v)
+{
+ return atomic_add_return_unchecked(1, v) == 0;
+}
/*
* atomic_dec_and_test - decrement by 1 and test
* Atomically increments @v by 1.
*/
#define atomic_inc(v) atomic_add(1, (v))
+static __inline__ void atomic_inc_unchecked(atomic_unchecked_t *v)
+{
+ atomic_add_unchecked(1, v);
+}
/*
* atomic_dec - decrement and test
* Atomically decrements @v by 1.
*/
#define atomic_dec(v) atomic_sub(1, (v))
+static __inline__ void atomic_dec_unchecked(atomic_unchecked_t *v)
+{
+ atomic_sub_unchecked(1, v);
+}
/*
* atomic_add_negative - add and test if negative
* @v: pointer of type atomic64_t
*
*/
-#define atomic64_read(v) ACCESS_ONCE((v)->counter)
+static inline long atomic64_read(const atomic64_t *v)
+{
+ return ACCESS_ONCE(v->counter);
+}
+
+static inline long atomic64_read_unchecked(const atomic64_unchecked_t *v)
+{
+ return ACCESS_ONCE(v->counter);
+}
/*
* atomic64_set - set atomic variable
* @v: pointer of type atomic64_t
* @i: required value
*/
-#define atomic64_set(v, i) ((v)->counter = (i))
+static inline void atomic64_set(atomic64_t *v, long i)
+{
+ v->counter = i;
+}
+
+static inline void atomic64_set_unchecked(atomic64_unchecked_t *v, long i)
+{
+ v->counter = i;
+}
-#define ATOMIC64_OP(op, c_op, asm_op) \
-static __inline__ void atomic64_##op(long i, atomic64_t * v) \
+#define __ATOMIC64_OP(op, suffix, asm_op, extable) \
+static inline void atomic64_##op##suffix(long i, atomic64##suffix##_t * v) \
{ \
if (kernel_uses_llsc && R10000_LLSC_WAR) { \
long temp; \
\
__asm__ __volatile__( \
- " .set arch=r4000 \n" \
- "1: lld %0, %1 # atomic64_" #op " \n" \
- " " #asm_op " %0, %2 \n" \
+ " .set mips3 \n" \
+ "1: lld %0, %1 # atomic64_" #op #suffix "\n" \
+ "2: " #asm_op " %0, %2 \n" \
" scd %0, %1 \n" \
" beqzl %0, 1b \n" \
+ extable \
" .set mips0 \n" \
: "=&r" (temp), "+" GCC_OFF12_ASM() (v->counter) \
: "Ir" (i)); \
} else if (kernel_uses_llsc) { \
long temp; \
\
- do { \
- __asm__ __volatile__( \
- " .set arch=r4000 \n" \
- " lld %0, %1 # atomic64_" #op "\n" \
- " " #asm_op " %0, %2 \n" \
- " scd %0, %1 \n" \
- " .set mips0 \n" \
- : "=&r" (temp), "+" GCC_OFF12_ASM() (v->counter) \
- : "Ir" (i)); \
- } while (unlikely(!temp)); \
+ __asm__ __volatile__( \
+ " .set mips3 \n" \
+ "1: lld %0, %1 # atomic64_" #op #suffix "\n" \
+ "2: " #asm_op " %0, %2 \n" \
+ " scd %0, %1 \n" \
+ " beqz %0, 1b \n" \
+ extable \
+ " .set mips0 \n" \
+ : "=&r" (temp), "+" GCC_OFF12_ASM() (v->counter) \
+ : "Ir" (i)); \
} else { \
unsigned long flags; \
\
raw_local_irq_save(flags); \
- v->counter c_op i; \
+ __asm__ __volatile__( \
+ "2: " #asm_op " %0, %1 \n" \
+ extable \
+ : "+" GCC_OFF12_ASM() (v->counter) : "Ir" (i)); \
raw_local_irq_restore(flags); \
} \
}
-#define ATOMIC64_OP_RETURN(op, c_op, asm_op) \
-static __inline__ long atomic64_##op##_return(long i, atomic64_t * v) \
+#define ATOMIC64_OP(op, asm_op) __ATOMIC64_OP(op, , asm_op##u) \
+ __ATOMIC64_OP(op, _unchecked, asm_op)
+
+#define __ATOMIC64_OP_RETURN(op, suffix, asm_op, post_op, extable) \
+static inline long atomic64_##op##_return##suffix(long i, atomic64##suffix##_t * v)\
{ \
long result; \
\
long temp; \
\
__asm__ __volatile__( \
- " .set arch=r4000 \n" \
+ " .set mips3 \n" \
"1: lld %1, %2 # atomic64_" #op "_return\n" \
- " " #asm_op " %0, %1, %3 \n" \
+ "2: " #asm_op " %0, %1, %3 \n" \
" scd %0, %2 \n" \
" beqzl %0, 1b \n" \
- " " #asm_op " %0, %1, %3 \n" \
+ post_op \
+ extable \
+ "4: " #asm_op " %0, %1, %3 \n" \
+ "5: \n" \
" .set mips0 \n" \
: "=&r" (result), "=&r" (temp), \
"+" GCC_OFF12_ASM() (v->counter) \
} else if (kernel_uses_llsc) { \
long temp; \
\
- do { \
- __asm__ __volatile__( \
- " .set arch=r4000 \n" \
- " lld %1, %2 # atomic64_" #op "_return\n" \
- " " #asm_op " %0, %1, %3 \n" \
- " scd %0, %2 \n" \
- " .set mips0 \n" \
- : "=&r" (result), "=&r" (temp), \
- "=" GCC_OFF12_ASM() (v->counter) \
- : "Ir" (i), GCC_OFF12_ASM() (v->counter) \
- : "memory"); \
- } while (unlikely(!result)); \
+ __asm__ __volatile__( \
+ " .set mips3 \n" \
+ "1: lld %1, %2 # atomic64_" #op "_return" #suffix "\n"\
+ "2: " #asm_op " %0, %1, %3 \n" \
+ " scd %0, %2 \n" \
+ " beqz %0, 1b \n" \
+ post_op \
+ extable \
+ "4: " #asm_op " %0, %1, %3 \n" \
+ "5: \n" \
+ " .set mips0 \n" \
+ : "=&r" (result), "=&r" (temp), \
+ "=" GCC_OFF12_ASM() (v->counter) \
+ : "Ir" (i), GCC_OFF12_ASM() (v->counter) \
+ : "memory"); \
\
result = temp; result c_op i; \
} else { \
unsigned long flags; \
\
raw_local_irq_save(flags); \
- result = v->counter; \
- result c_op i; \
- v->counter = result; \
+ __asm__ __volatile__( \
+ " ld %0, %1 \n" \
+ "2: " #asm_op " %0, %1, %2 \n" \
+ " sd %0, %1 \n" \
+ "3: \n" \
+ extable \
+ : "=&r" (result), "+" GCC_OFF12_ASM() (v->counter) \
+ : "Ir" (i)); \
raw_local_irq_restore(flags); \
} \
\
return result; \
}
-#define ATOMIC64_OPS(op, c_op, asm_op) \
- ATOMIC64_OP(op, c_op, asm_op) \
- ATOMIC64_OP_RETURN(op, c_op, asm_op)
+#define ATOMIC64_OP_RETURN(op, asm_op) __ATOMIC64_OP_RETURN(op, , asm_op##u, , __OVERFLOW_EXTABLE) \
+ __ATOMIC64_OP_RETURN(op, _unchecked, asm_op, __OVERFLOW_POST, __OVERFLOW_EXTABLE)
-ATOMIC64_OPS(add, +=, daddu)
-ATOMIC64_OPS(sub, -=, dsubu)
+#define ATOMIC64_OPS(op, asm_op) \
+ ATOMIC64_OP(op, asm_op) \
+ ATOMIC64_OP_RETURN(op, asm_op)
+
+ATOMIC64_OPS(add, dadd)
+ATOMIC64_OPS(sub, dsub)
#undef ATOMIC64_OPS
#undef ATOMIC64_OP_RETURN
+#undef __ATOMIC64_OP_RETURN
#undef ATOMIC64_OP
+#undef __ATOMIC64_OP
+#undef __OVERFLOW_EXTABLE
+#undef __OVERFLOW_POST
/*
* atomic64_sub_if_positive - conditionally subtract integer from atomic
* Atomically test @v and subtract @i if @v is greater or equal than @i.
* The function returns the old value of @v minus @i.
*/
-static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)
+static __inline__ long atomic64_sub_if_positive(long i, atomic64_t *v)
{
long result;
return result;
}
-#define atomic64_cmpxchg(v, o, n) \
- ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n)))
-#define atomic64_xchg(v, new) (xchg(&((v)->counter), (new)))
+static inline long atomic64_cmpxchg(atomic64_t *v, long old, long new)
+{
+ return cmpxchg(&v->counter, old, new);
+}
+
+static inline long atomic64_cmpxchg_unchecked(atomic64_unchecked_t *v, long old,
+ long new)
+{
+ return cmpxchg(&(v->counter), old, new);
+}
+
+static inline long atomic64_xchg(atomic64_t *v, long new)
+{
+ return xchg(&v->counter, new);
+}
+
+static inline long atomic64_xchg_unchecked(atomic64_unchecked_t *v, long new)
+{
+ return xchg(&(v->counter), new);
+}
/**
* atomic64_add_unless - add unless the number is a given value
#define atomic64_dec_return(v) atomic64_sub_return(1, (v))
#define atomic64_inc_return(v) atomic64_add_return(1, (v))
+#define atomic64_inc_return_unchecked(v) atomic64_add_return_unchecked(1, (v))
/*
* atomic64_sub_and_test - subtract value from variable and test result
* other cases.
*/
#define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0)
+#define atomic64_inc_and_test_unchecked(v) atomic64_add_return_unchecked(1, (v)) == 0)
/*
* atomic64_dec_and_test - decrement by 1 and test
* Atomically increments @v by 1.
*/
#define atomic64_inc(v) atomic64_add(1, (v))
+#define atomic64_inc_unchecked(v) atomic64_add_unchecked(1, (v))
/*
* atomic64_dec - decrement and test
* Atomically decrements @v by 1.
*/
#define atomic64_dec(v) atomic64_sub(1, (v))
+#define atomic64_dec_unchecked(v) atomic64_sub_unchecked(1, (v))
/*
* atomic64_add_negative - add and test if negative
do { \
compiletime_assert_atomic_type(*p); \
smp_mb(); \
- ACCESS_ONCE(*p) = (v); \
+ ACCESS_ONCE_RW(*p) = (v); \
} while (0)
#define smp_load_acquire(p) \
#ifndef _ASM_CACHE_H
#define _ASM_CACHE_H
+#include <linux/const.h>
#include <kmalloc.h>
#define L1_CACHE_SHIFT CONFIG_MIPS_L1_CACHE_SHIFT
-#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT)
+#define L1_CACHE_BYTES (_AC(1,UL) << L1_CACHE_SHIFT)
#define SMP_CACHE_SHIFT L1_CACHE_SHIFT
#define SMP_CACHE_BYTES L1_CACHE_BYTES
#define ELF_ET_DYN_BASE (TASK_SIZE / 3 * 2)
#endif
+#ifdef CONFIG_PAX_ASLR
+#define PAX_ELF_ET_DYN_BASE (TASK_IS_32BIT_ADDR ? 0x00400000UL : 0x00400000UL)
+
+#define PAX_DELTA_MMAP_LEN (TASK_IS_32BIT_ADDR ? 27-PAGE_SHIFT : 36-PAGE_SHIFT)
+#define PAX_DELTA_STACK_LEN (TASK_IS_32BIT_ADDR ? 27-PAGE_SHIFT : 36-PAGE_SHIFT)
+#endif
+
#define ARCH_HAS_SETUP_ADDITIONAL_PAGES 1
struct linux_binprm;
extern int arch_setup_additional_pages(struct linux_binprm *bprm,
int uses_interp);
-struct mm_struct;
-extern unsigned long arch_randomize_brk(struct mm_struct *mm);
-#define arch_randomize_brk arch_randomize_brk
-
struct arch_elf_state {
int fp_abi;
int interp_fp_abi;
#ifndef _ASM_EXEC_H
#define _ASM_EXEC_H
-extern unsigned long arch_align_stack(unsigned long sp);
+#define arch_align_stack(x) ((x) & ~0xfUL)
#endif /* _ASM_EXEC_H */
#include <linux/atomic.h>
-extern atomic_t irq_err_count;
+extern atomic_unchecked_t irq_err_count;
/*
* interrupt-retrigger: NOP for now. This may not be appropriate for all
atomic_long_t a;
} local_t;
+typedef struct {
+ atomic_long_unchecked_t a;
+} local_unchecked_t;
+
#define LOCAL_INIT(i) { ATOMIC_LONG_INIT(i) }
#define local_read(l) atomic_long_read(&(l)->a)
+#define local_read_unchecked(l) atomic_long_read_unchecked(&(l)->a)
#define local_set(l, i) atomic_long_set(&(l)->a, (i))
+#define local_set_unchecked(l, i) atomic_long_set_unchecked(&(l)->a, (i))
#define local_add(i, l) atomic_long_add((i), (&(l)->a))
+#define local_add_unchecked(i, l) atomic_long_add_unchecked((i), (&(l)->a))
#define local_sub(i, l) atomic_long_sub((i), (&(l)->a))
+#define local_sub_unchecked(i, l) atomic_long_sub_unchecked((i), (&(l)->a))
#define local_inc(l) atomic_long_inc(&(l)->a)
+#define local_inc_unchecked(l) atomic_long_inc_unchecked(&(l)->a)
#define local_dec(l) atomic_long_dec(&(l)->a)
+#define local_dec_unchecked(l) atomic_long_dec_unchecked(&(l)->a)
/*
* Same as above, but return the result value
return result;
}
+static __inline__ long local_add_return_unchecked(long i, local_unchecked_t * l)
+{
+ unsigned long result;
+
+ if (kernel_uses_llsc && R10000_LLSC_WAR) {
+ unsigned long temp;
+
+ __asm__ __volatile__(
+ " .set mips3 \n"
+ "1:" __LL "%1, %2 # local_add_return \n"
+ " addu %0, %1, %3 \n"
+ __SC "%0, %2 \n"
+ " beqzl %0, 1b \n"
+ " addu %0, %1, %3 \n"
+ " .set mips0 \n"
+ : "=&r" (result), "=&r" (temp), "=m" (l->a.counter)
+ : "Ir" (i), "m" (l->a.counter)
+ : "memory");
+ } else if (kernel_uses_llsc) {
+ unsigned long temp;
+
+ __asm__ __volatile__(
+ " .set mips3 \n"
+ "1:" __LL "%1, %2 # local_add_return \n"
+ " addu %0, %1, %3 \n"
+ __SC "%0, %2 \n"
+ " beqz %0, 1b \n"
+ " addu %0, %1, %3 \n"
+ " .set mips0 \n"
+ : "=&r" (result), "=&r" (temp), "=m" (l->a.counter)
+ : "Ir" (i), "m" (l->a.counter)
+ : "memory");
+ } else {
+ unsigned long flags;
+
+ local_irq_save(flags);
+ result = l->a.counter;
+ result += i;
+ l->a.counter = result;
+ local_irq_restore(flags);
+ }
+
+ return result;
+}
+
static __inline__ long local_sub_return(long i, local_t * l)
{
unsigned long result;
#define local_cmpxchg(l, o, n) \
((long)cmpxchg_local(&((l)->a.counter), (o), (n)))
+#define local_cmpxchg_unchecked(l, o, n) \
+ ((long)cmpxchg_local(&((l)->a.counter), (o), (n)))
#define local_xchg(l, n) (atomic_long_xchg((&(l)->a), (n)))
/**
#ifdef CONFIG_CPU_MIPS32
typedef struct { unsigned long pte_low, pte_high; } pte_t;
#define pte_val(x) ((x).pte_low | ((unsigned long long)(x).pte_high << 32))
- #define __pte(x) ({ pte_t __pte = {(x), ((unsigned long long)(x)) >> 32}; __pte; })
+ #define __pte(x) ({ pte_t __pte = {(x), (x) >> 32}; __pte; })
#else
typedef struct { unsigned long long pte; } pte_t;
#define pte_val(x) ((x).pte)
{
set_pud(pud, __pud((unsigned long)pmd));
}
+
+static inline void pud_populate_kernel(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
+{
+ pud_populate(mm, pud, pmd);
+}
#endif
/*
#include <asm/io.h>
#include <asm/pgtable-bits.h>
+#define ktla_ktva(addr) (addr)
+#define ktva_ktla(addr) (addr)
+
struct mm_struct;
struct vm_area_struct;
#define TIF_SECCOMP 4 /* secure computing */
#define TIF_NOTIFY_RESUME 5 /* callback before returning to user */
#define TIF_RESTORE_SIGMASK 9 /* restore signal mask in do_signal() */
+/* li takes a 32bit immediate */
+#define TIF_GRSEC_SETXID 10 /* update credentials on syscall entry/exit */
+
#define TIF_USEDFPU 16 /* FPU was used by this task this quantum (SMP) */
#define TIF_MEMDIE 18 /* is terminating due to OOM killer */
#define TIF_NOHZ 19 /* in adaptive nohz mode */
#define _TIF_USEDMSA (1<<TIF_USEDMSA)
#define _TIF_MSA_CTX_LIVE (1<<TIF_MSA_CTX_LIVE)
#define _TIF_SYSCALL_TRACEPOINT (1<<TIF_SYSCALL_TRACEPOINT)
+#define _TIF_GRSEC_SETXID (1<<TIF_GRSEC_SETXID)
#define _TIF_WORK_SYSCALL_ENTRY (_TIF_NOHZ | _TIF_SYSCALL_TRACE | \
_TIF_SYSCALL_AUDIT | \
- _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP)
+ _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP | \
+ _TIF_GRSEC_SETXID)
/* work to do in syscall_trace_leave() */
#define _TIF_WORK_SYSCALL_EXIT (_TIF_NOHZ | _TIF_SYSCALL_TRACE | \
- _TIF_SYSCALL_AUDIT | _TIF_SYSCALL_TRACEPOINT)
+ _TIF_SYSCALL_AUDIT | _TIF_SYSCALL_TRACEPOINT | _TIF_GRSEC_SETXID)
/* work to do on interrupt/exception return */
#define _TIF_WORK_MASK \
/* work to do on any return to u-space */
#define _TIF_ALLWORK_MASK (_TIF_NOHZ | _TIF_WORK_MASK | \
_TIF_WORK_SYSCALL_EXIT | \
- _TIF_SYSCALL_TRACEPOINT)
+ _TIF_SYSCALL_TRACEPOINT | _TIF_GRSEC_SETXID)
/*
* We stash processor id into a COP0 register to retrieve it fast
__ok == 0; \
})
+#define access_ok_noprefault(type, addr, size) access_ok((type), (addr), (size))
#define access_ok(type, addr, size) \
likely(__access_ok((addr), (size), __access_mask))
#undef ELF_ET_DYN_BASE
#define ELF_ET_DYN_BASE (TASK32_SIZE / 3 * 2)
+#ifdef CONFIG_PAX_ASLR
+#define PAX_ELF_ET_DYN_BASE (TASK_IS_32BIT_ADDR ? 0x00400000UL : 0x00400000UL)
+
+#define PAX_DELTA_MMAP_LEN (TASK_IS_32BIT_ADDR ? 27-PAGE_SHIFT : 36-PAGE_SHIFT)
+#define PAX_DELTA_STACK_LEN (TASK_IS_32BIT_ADDR ? 27-PAGE_SHIFT : 36-PAGE_SHIFT)
+#endif
+
#include <asm/processor.h>
#include <linux/module.h>
#include <linux/elfcore.h>
#undef ELF_ET_DYN_BASE
#define ELF_ET_DYN_BASE (TASK32_SIZE / 3 * 2)
+#ifdef CONFIG_PAX_ASLR
+#define PAX_ELF_ET_DYN_BASE (TASK_IS_32BIT_ADDR ? 0x00400000UL : 0x00400000UL)
+
+#define PAX_DELTA_MMAP_LEN (TASK_IS_32BIT_ADDR ? 27-PAGE_SHIFT : 36-PAGE_SHIFT)
+#define PAX_DELTA_STACK_LEN (TASK_IS_32BIT_ADDR ? 27-PAGE_SHIFT : 36-PAGE_SHIFT)
+#endif
+
#include <asm/processor.h>
#include <linux/module.h>
printk(KERN_DEBUG "spurious 8259A interrupt: IRQ%d.\n", irq);
spurious_irq_mask |= irqmask;
}
- atomic_inc(&irq_err_count);
+ atomic_inc_unchecked(&irq_err_count);
/*
* Theoretically we do not have to handle this IRQ,
* but in Linux this does not cause problems and is
}
}
- atomic_inc(&irq_err_count);
+ atomic_inc_unchecked(&irq_err_count);
}
void __init gt641xx_irq_init(void)
printk("unexpected IRQ # %d\n", irq);
}
-atomic_t irq_err_count;
+atomic_unchecked_t irq_err_count;
int arch_show_interrupts(struct seq_file *p, int prec)
{
- seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read(&irq_err_count));
+ seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read_unchecked(&irq_err_count));
return 0;
}
asmlinkage void spurious_interrupt(void)
{
- atomic_inc(&irq_err_count);
+ atomic_inc_unchecked(&irq_err_count);
}
void __init init_IRQ(void)
#endif
}
+
#ifdef DEBUG_STACKOVERFLOW
+extern void gr_handle_kernel_exploit(void);
+
static inline void check_stack_overflow(void)
{
unsigned long sp;
printk("do_IRQ: stack overflow: %ld\n",
sp - sizeof(struct thread_info));
dump_stack();
+ gr_handle_kernel_exploit();
}
}
#else
nc_core_ready_count = nc_addr;
/* Ensure ready_count is zero-initialised before the assembly runs */
- ACCESS_ONCE(*nc_core_ready_count) = 0;
+ ACCESS_ONCE_RW(*nc_core_ready_count) = 0;
coupled_barrier(&per_cpu(pm_barrier, core), online);
/* Run the generated entry code */
return pc;
}
-/*
- * Don't forget that the stack pointer must be aligned on a 8 bytes
- * boundary for 32-bits ABI and 16 bytes for 64-bits ABI.
- */
-unsigned long arch_align_stack(unsigned long sp)
-{
- if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
- sp -= get_random_int() & ~PAGE_MASK;
-
- return sp & ALMASK;
-}
-
static void arch_dump_stack(void *info)
{
struct pt_regs *regs;
return ret;
}
+#ifdef CONFIG_GRKERNSEC_SETXID
+extern void gr_delayed_cred_worker(void);
+#endif
+
/*
* Notification of system call entry/exit
* - triggered by current->work.syscall_trace
tracehook_report_syscall_entry(regs))
ret = -1;
+#ifdef CONFIG_GRKERNSEC_SETXID
+ if (unlikely(test_and_clear_thread_flag(TIF_GRSEC_SETXID)))
+ gr_delayed_cred_worker();
+#endif
+
if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT)))
trace_sys_enter(regs, regs->regs[2]);
#include <linux/reboot.h>
#include <asm/reboot.h>
+#include <asm/bug.h>
/*
* Urgs ... Too many MIPS machines to handle this in a generic way.
{
if (_machine_restart)
_machine_restart(command);
+ BUG();
}
void machine_halt(void)
{
if (_machine_halt)
_machine_halt();
+ BUG();
}
void machine_power_off(void)
{
if (pm_power_off)
pm_power_off();
+ BUG();
}
#include <asm/mipsregs.h>
static atomic_t count_start_flag = ATOMIC_INIT(0);
-static atomic_t count_count_start = ATOMIC_INIT(0);
-static atomic_t count_count_stop = ATOMIC_INIT(0);
+static atomic_unchecked_t count_count_start = ATOMIC_INIT(0);
+static atomic_unchecked_t count_count_stop = ATOMIC_INIT(0);
static atomic_t count_reference = ATOMIC_INIT(0);
#define COUNTON 100
for (i = 0; i < NR_LOOPS; i++) {
/* slaves loop on '!= 2' */
- while (atomic_read(&count_count_start) != 1)
+ while (atomic_read_unchecked(&count_count_start) != 1)
mb();
- atomic_set(&count_count_stop, 0);
+ atomic_set_unchecked(&count_count_stop, 0);
smp_wmb();
/* this lets the slaves write their count register */
- atomic_inc(&count_count_start);
+ atomic_inc_unchecked(&count_count_start);
/*
* Everyone initialises count in the last loop:
/*
* Wait for all slaves to leave the synchronization point:
*/
- while (atomic_read(&count_count_stop) != 1)
+ while (atomic_read_unchecked(&count_count_stop) != 1)
mb();
- atomic_set(&count_count_start, 0);
+ atomic_set_unchecked(&count_count_start, 0);
smp_wmb();
- atomic_inc(&count_count_stop);
+ atomic_inc_unchecked(&count_count_stop);
}
/* Arrange for an interrupt in a short while */
write_c0_compare(read_c0_count() + COUNTON);
initcount = atomic_read(&count_reference);
for (i = 0; i < NR_LOOPS; i++) {
- atomic_inc(&count_count_start);
- while (atomic_read(&count_count_start) != 2)
+ atomic_inc_unchecked(&count_count_start);
+ while (atomic_read_unchecked(&count_count_start) != 2)
mb();
/*
if (i == NR_LOOPS-1)
write_c0_count(initcount);
- atomic_inc(&count_count_stop);
- while (atomic_read(&count_count_stop) != 2)
+ atomic_inc_unchecked(&count_count_stop);
+ while (atomic_read_unchecked(&count_count_stop) != 2)
mb();
}
/* Arrange for an interrupt in a short while */
siginfo_t info;
prev_state = exception_enter();
- die_if_kernel("Integer overflow", regs);
+ if (unlikely(!user_mode(regs))) {
+
+#ifdef CONFIG_PAX_REFCOUNT
+ if (fixup_exception(regs)) {
+ pax_report_refcount_overflow(regs);
+ exception_exit(prev_state);
+ return;
+ }
+#endif
+
+ die("Integer overflow", regs);
+ }
info.si_code = FPE_INTOVF;
info.si_signo = SIGFPE;
return r;
}
-int kvm_arch_init(void *opaque)
+int kvm_arch_init(const void *opaque)
{
if (kvm_mips_callbacks) {
kvm_err("kvm: module already exists\n");
#include <asm/highmem.h> /* For VMALLOC_END */
#include <linux/kdebug.h>
+#ifdef CONFIG_PAX_PAGEEXEC
+void pax_report_insns(struct pt_regs *regs, void *pc, void *sp)
+{
+ unsigned long i;
+
+ printk(KERN_ERR "PAX: bytes at PC: ");
+ for (i = 0; i < 5; i++) {
+ unsigned int c;
+ if (get_user(c, (unsigned int *)pc+i))
+ printk(KERN_CONT "???????? ");
+ else
+ printk(KERN_CONT "%08x ", c);
+ }
+ printk("\n");
+}
+#endif
+
/*
* This routine handles page faults. It determines the address,
* and the problem, and then passes it off to one of the appropriate
bad_area_nosemaphore:
/* User mode accesses just cause a SIGSEGV */
if (user_mode(regs)) {
+
+#ifdef CONFIG_PAX_PAGEEXEC
+ if (cpu_has_rixi && (mm->pax_flags & MF_PAX_PAGEEXEC) && !write && address == instruction_pointer(regs)) {
+ pax_report_fault(regs, (void *)address, (void *)user_stack_pointer(regs));
+ do_group_exit(SIGKILL);
+ }
+#endif
+
tsk->thread.cp0_badvaddr = address;
tsk->thread.error_code = write;
#if 0
struct vm_area_struct *vma;
unsigned long addr = addr0;
int do_color_align;
+ unsigned long offset = gr_rand_threadstack_offset(mm, filp, flags);
struct vm_unmapped_area_info info;
if (unlikely(len > TASK_SIZE))
do_color_align = 1;
/* requesting a specific address */
+
+#ifdef CONFIG_PAX_RANDMMAP
+ if (!(current->mm->pax_flags & MF_PAX_RANDMMAP))
+#endif
+
if (addr) {
if (do_color_align)
addr = COLOUR_ALIGN(addr, pgoff);
addr = PAGE_ALIGN(addr);
vma = find_vma(mm, addr);
- if (TASK_SIZE - len >= addr &&
- (!vma || addr + len <= vma->vm_start))
+ if (TASK_SIZE - len >= addr && check_heap_stack_gap(vma, addr, len, offset))
return addr;
}
info.length = len;
info.align_mask = do_color_align ? (PAGE_MASK & shm_align_mask) : 0;
info.align_offset = pgoff << PAGE_SHIFT;
+ info.threadstack_offset = offset;
if (dir == DOWN) {
info.flags = VM_UNMAPPED_AREA_TOPDOWN;
{
unsigned long random_factor = 0UL;
+#ifdef CONFIG_PAX_RANDMMAP
+ if (!(mm->pax_flags & MF_PAX_RANDMMAP))
+#endif
+
if (current->flags & PF_RANDOMIZE) {
random_factor = get_random_int();
random_factor = random_factor << PAGE_SHIFT;
if (mmap_is_legacy()) {
mm->mmap_base = TASK_UNMAPPED_BASE + random_factor;
+
+#ifdef CONFIG_PAX_RANDMMAP
+ if (mm->pax_flags & MF_PAX_RANDMMAP)
+ mm->mmap_base += mm->delta_mmap;
+#endif
+
mm->get_unmapped_area = arch_get_unmapped_area;
} else {
mm->mmap_base = mmap_base(random_factor);
- mm->get_unmapped_area = arch_get_unmapped_area_topdown;
- }
-}
-
-static inline unsigned long brk_rnd(void)
-{
- unsigned long rnd = get_random_int();
-
- rnd = rnd << PAGE_SHIFT;
- /* 8MB for 32bit, 256MB for 64bit */
- if (TASK_IS_32BIT_ADDR)
- rnd = rnd & 0x7ffffful;
- else
- rnd = rnd & 0xffffffful;
- return rnd;
-}
-
-unsigned long arch_randomize_brk(struct mm_struct *mm)
-{
- unsigned long base = mm->brk;
- unsigned long ret;
+#ifdef CONFIG_PAX_RANDMMAP
+ if (mm->pax_flags & MF_PAX_RANDMMAP)
+ mm->mmap_base -= mm->delta_mmap + mm->delta_stack;
+#endif
- ret = PAGE_ALIGN(base + brk_rnd());
-
- if (ret < mm->brk)
- return mm->brk;
-
- return ret;
+ mm->get_unmapped_area = arch_get_unmapped_area_topdown;
+ }
}
int __virt_addr_valid(const volatile void *kaddr)
static struct pci_ops octeon_pci_ops = {
- octeon_read_config,
- octeon_write_config,
+ .read = octeon_read_config,
+ .write = octeon_write_config,
};
static struct resource octeon_pci_mem_resource = {
}
static struct pci_ops octeon_pcie0_ops = {
- octeon_pcie0_read_config,
- octeon_pcie0_write_config,
+ .read = octeon_pcie0_read_config,
+ .write = octeon_pcie0_write_config,
};
static struct resource octeon_pcie0_mem_resource = {
};
static struct pci_ops octeon_pcie1_ops = {
- octeon_pcie1_read_config,
- octeon_pcie1_write_config,
+ .read = octeon_pcie1_read_config,
+ .write = octeon_pcie1_write_config,
};
static struct resource octeon_pcie1_mem_resource = {
};
static struct pci_ops octeon_dummy_ops = {
- octeon_dummy_read_config,
- octeon_dummy_write_config,
+ .read = octeon_dummy_read_config,
+ .write = octeon_dummy_write_config,
};
static struct resource octeon_dummy_mem_resource = {
cont_nmi_dump(void)
{
#ifndef REAL_NMI_SIGNAL
- static atomic_t nmied_cpus = ATOMIC_INIT(0);
+ static atomic_unchecked_t nmied_cpus = ATOMIC_INIT(0);
- atomic_inc(&nmied_cpus);
+ atomic_inc_unchecked(&nmied_cpus);
#endif
/*
* Only allow 1 cpu to proceed
udelay(10000);
}
#else
- while (atomic_read(&nmied_cpus) != num_online_cpus());
+ while (atomic_read_unchecked(&nmied_cpus) != num_online_cpus());
#endif
/*
"spurious RM200 8259A interrupt: IRQ%d.\n", irq);
spurious_irq_mask |= irqmask;
}
- atomic_inc(&irq_err_count);
+ atomic_inc_unchecked(&irq_err_count);
/*
* Theoretically we do not have to handle this IRQ,
* but in Linux this does not cause problems and is
printk(KERN_ERR "spurious ICU interrupt: %04x,%04x\n", pend1, pend2);
- atomic_inc(&irq_err_count);
+ atomic_inc_unchecked(&irq_err_count);
return -1;
}
irq_cascade_t *cascade;
if (irq >= NR_IRQS) {
- atomic_inc(&irq_err_count);
+ atomic_inc_unchecked(&irq_err_count);
return;
}
ret = cascade->get_irq(irq);
irq = ret;
if (ret < 0)
- atomic_inc(&irq_err_count);
+ atomic_inc_unchecked(&irq_err_count);
else
irq_dispatch(irq);
if (!irqd_irq_disabled(idata) && chip->irq_unmask)
#ifndef _ASM_PROC_CACHE_H
#define _ASM_PROC_CACHE_H
+#include <linux/const.h>
+
/* L1 cache */
#define L1_CACHE_NWAYS 4 /* number of ways in caches */
#define L1_CACHE_NENTRIES 256 /* number of entries in each way */
-#define L1_CACHE_BYTES 16 /* bytes per entry */
#define L1_CACHE_SHIFT 4 /* shift for bytes per entry */
+#define L1_CACHE_BYTES (_AC(1,UL) << L1_CACHE_SHIFT) /* bytes per entry */
#define L1_CACHE_WAYDISP 0x1000 /* displacement of one way from the next */
#define L1_CACHE_TAG_VALID 0x00000001 /* cache tag valid bit */
#ifndef _ASM_PROC_CACHE_H
#define _ASM_PROC_CACHE_H
+#include <linux/const.h>
+
/*
* L1 cache
*/
#define L1_CACHE_NWAYS 4 /* number of ways in caches */
#define L1_CACHE_NENTRIES 128 /* number of entries in each way */
-#define L1_CACHE_BYTES 32 /* bytes per entry */
#define L1_CACHE_SHIFT 5 /* shift for bytes per entry */
+#define L1_CACHE_BYTES (_AC(1,UL) << L1_CACHE_SHIFT) /* bytes per entry */
#define L1_CACHE_WAYDISP 0x1000 /* distance from one way to the next */
#define L1_CACHE_TAG_VALID 0x00000001 /* cache tag valid bit */
#ifndef __ASM_OPENRISC_CACHE_H
#define __ASM_OPENRISC_CACHE_H
+#include <linux/const.h>
+
/* FIXME: How can we replace these with values from the CPU...
* they shouldn't be hard-coded!
*/
-#define L1_CACHE_BYTES 16
#define L1_CACHE_SHIFT 4
+#define L1_CACHE_BYTES (_AC(1,UL) << L1_CACHE_SHIFT)
#endif /* __ASM_OPENRISC_CACHE_H */
return dec;
}
+#define atomic64_read_unchecked(v) atomic64_read(v)
+#define atomic64_set_unchecked(v, i) atomic64_set((v), (i))
+#define atomic64_add_unchecked(a, v) atomic64_add((a), (v))
+#define atomic64_add_return_unchecked(a, v) atomic64_add_return((a), (v))
+#define atomic64_sub_unchecked(a, v) atomic64_sub((a), (v))
+#define atomic64_inc_unchecked(v) atomic64_inc(v)
+#define atomic64_inc_return_unchecked(v) atomic64_inc_return(v)
+#define atomic64_dec_unchecked(v) atomic64_dec(v)
+#define atomic64_cmpxchg_unchecked(v, o, n) atomic64_cmpxchg((v), (o), (n))
+
#endif /* !CONFIG_64BIT */
#ifndef __ARCH_PARISC_CACHE_H
#define __ARCH_PARISC_CACHE_H
+#include <linux/const.h>
/*
* PA 2.0 processors have 64-byte cachelines; PA 1.1 processors have
* just ruin performance.
*/
#ifdef CONFIG_PA20
-#define L1_CACHE_BYTES 64
#define L1_CACHE_SHIFT 6
#else
-#define L1_CACHE_BYTES 32
#define L1_CACHE_SHIFT 5
#endif
+#define L1_CACHE_BYTES (_AC(1,UL) << L1_CACHE_SHIFT)
+
#ifndef __ASSEMBLY__
#define SMP_CACHE_BYTES L1_CACHE_BYTES
#define ELF_ET_DYN_BASE (TASK_UNMAPPED_BASE + 0x01000000)
+#ifdef CONFIG_PAX_ASLR
+#define PAX_ELF_ET_DYN_BASE 0x10000UL
+
+#define PAX_DELTA_MMAP_LEN 16
+#define PAX_DELTA_STACK_LEN 16
+#endif
+
/* This yields a mask that user programs can use to figure out what
instruction set this CPU supports. This could be done in user space,
but it's not easy, and we've already done it here. */
(__u32)(__pa((unsigned long)pmd) >> PxD_VALUE_SHIFT));
}
+static inline void pgd_populate_kernel(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmd)
+{
+ pgd_populate(mm, pgd, pmd);
+}
+
static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address)
{
pmd_t *pmd = (pmd_t *)__get_free_pages(GFP_KERNEL|__GFP_REPEAT,
#define pmd_alloc_one(mm, addr) ({ BUG(); ((pmd_t *)2); })
#define pmd_free(mm, x) do { } while (0)
#define pgd_populate(mm, pmd, pte) BUG()
+#define pgd_populate_kernel(mm, pmd, pte) BUG()
#endif
#define PAGE_EXECREAD __pgprot(_PAGE_PRESENT | _PAGE_USER | _PAGE_READ | _PAGE_EXEC |_PAGE_ACCESSED)
#define PAGE_COPY PAGE_EXECREAD
#define PAGE_RWX __pgprot(_PAGE_PRESENT | _PAGE_USER | _PAGE_READ | _PAGE_WRITE | _PAGE_EXEC |_PAGE_ACCESSED)
+
+#ifdef CONFIG_PAX_PAGEEXEC
+# define PAGE_SHARED_NOEXEC __pgprot(_PAGE_PRESENT | _PAGE_USER | _PAGE_READ | _PAGE_WRITE | _PAGE_ACCESSED)
+# define PAGE_COPY_NOEXEC __pgprot(_PAGE_PRESENT | _PAGE_USER | _PAGE_READ | _PAGE_ACCESSED)
+# define PAGE_READONLY_NOEXEC __pgprot(_PAGE_PRESENT | _PAGE_USER | _PAGE_READ | _PAGE_ACCESSED)
+#else
+# define PAGE_SHARED_NOEXEC PAGE_SHARED
+# define PAGE_COPY_NOEXEC PAGE_COPY
+# define PAGE_READONLY_NOEXEC PAGE_READONLY
+#endif
+
#define PAGE_KERNEL __pgprot(_PAGE_KERNEL)
#define PAGE_KERNEL_EXEC __pgprot(_PAGE_KERNEL_EXEC)
#define PAGE_KERNEL_RWX __pgprot(_PAGE_KERNEL_RWX)
const void __user *from,
unsigned long n)
{
- int sz = __compiletime_object_size(to);
+ size_t sz = __compiletime_object_size(to);
int ret = -EFAULT;
- if (likely(sz == -1 || !__builtin_constant_p(n) || sz >= n))
+ if (likely(sz == (size_t)-1 || !__builtin_constant_p(n) || sz >= n))
ret = __copy_from_user(to, from, n);
else
copy_from_user_overflow();
/* three functions to determine where in the module core
* or init pieces the location is */
+static inline int in_init_rx(struct module *me, void *loc)
+{
+ return (loc >= me->module_init_rx &&
+ loc < (me->module_init_rx + me->init_size_rx));
+}
+
+static inline int in_init_rw(struct module *me, void *loc)
+{
+ return (loc >= me->module_init_rw &&
+ loc < (me->module_init_rw + me->init_size_rw));
+}
+
static inline int in_init(struct module *me, void *loc)
{
- return (loc >= me->module_init &&
- loc <= (me->module_init + me->init_size));
+ return in_init_rx(me, loc) || in_init_rw(me, loc);
+}
+
+static inline int in_core_rx(struct module *me, void *loc)
+{
+ return (loc >= me->module_core_rx &&
+ loc < (me->module_core_rx + me->core_size_rx));
+}
+
+static inline int in_core_rw(struct module *me, void *loc)
+{
+ return (loc >= me->module_core_rw &&
+ loc < (me->module_core_rw + me->core_size_rw));
}
static inline int in_core(struct module *me, void *loc)
{
- return (loc >= me->module_core &&
- loc <= (me->module_core + me->core_size));
+ return in_core_rx(me, loc) || in_core_rw(me, loc);
}
static inline int in_local(struct module *me, void *loc)
}
/* align things a bit */
- me->core_size = ALIGN(me->core_size, 16);
- me->arch.got_offset = me->core_size;
- me->core_size += gots * sizeof(struct got_entry);
+ me->core_size_rw = ALIGN(me->core_size_rw, 16);
+ me->arch.got_offset = me->core_size_rw;
+ me->core_size_rw += gots * sizeof(struct got_entry);
- me->core_size = ALIGN(me->core_size, 16);
- me->arch.fdesc_offset = me->core_size;
- me->core_size += fdescs * sizeof(Elf_Fdesc);
+ me->core_size_rw = ALIGN(me->core_size_rw, 16);
+ me->arch.fdesc_offset = me->core_size_rw;
+ me->core_size_rw += fdescs * sizeof(Elf_Fdesc);
me->arch.got_max = gots;
me->arch.fdesc_max = fdescs;
BUG_ON(value == 0);
- got = me->module_core + me->arch.got_offset;
+ got = me->module_core_rw + me->arch.got_offset;
for (i = 0; got[i].addr; i++)
if (got[i].addr == value)
goto out;
#ifdef CONFIG_64BIT
static Elf_Addr get_fdesc(struct module *me, unsigned long value)
{
- Elf_Fdesc *fdesc = me->module_core + me->arch.fdesc_offset;
+ Elf_Fdesc *fdesc = me->module_core_rw + me->arch.fdesc_offset;
if (!value) {
printk(KERN_ERR "%s: zero OPD requested!\n", me->name);
/* Create new one */
fdesc->addr = value;
- fdesc->gp = (Elf_Addr)me->module_core + me->arch.got_offset;
+ fdesc->gp = (Elf_Addr)me->module_core_rw + me->arch.got_offset;
return (Elf_Addr)fdesc;
}
#endif /* CONFIG_64BIT */
table = (unsigned char *)sechdrs[me->arch.unwind_section].sh_addr;
end = table + sechdrs[me->arch.unwind_section].sh_size;
- gp = (Elf_Addr)me->module_core + me->arch.got_offset;
+ gp = (Elf_Addr)me->module_core_rw + me->arch.got_offset;
DEBUGP("register_unwind_table(), sect = %d at 0x%p - 0x%p (gp=0x%lx)\n",
me->arch.unwind_section, table, end, gp);
unsigned long task_size = TASK_SIZE;
int do_color_align, last_mmap;
struct vm_unmapped_area_info info;
+ unsigned long offset = gr_rand_threadstack_offset(current->mm, filp, flags);
if (len > task_size)
return -ENOMEM;
goto found_addr;
}
+#ifdef CONFIG_PAX_RANDMMAP
+ if (!(mm->pax_flags & MF_PAX_RANDMMAP))
+#endif
+
if (addr) {
if (do_color_align && last_mmap)
addr = COLOR_ALIGN(addr, last_mmap, pgoff);
info.high_limit = mmap_upper_limit();
info.align_mask = last_mmap ? (PAGE_MASK & (SHM_COLOUR - 1)) : 0;
info.align_offset = shared_align_offset(last_mmap, pgoff);
+ info.threadstack_offset = offset;
addr = vm_unmapped_area(&info);
found_addr:
unsigned long addr = addr0;
int do_color_align, last_mmap;
struct vm_unmapped_area_info info;
+ unsigned long offset = gr_rand_threadstack_offset(current->mm, filp, flags);
#ifdef CONFIG_64BIT
/* This should only ever run for 32-bit processes. */
}
/* requesting a specific address */
+#ifdef CONFIG_PAX_RANDMMAP
+ if (!(mm->pax_flags & MF_PAX_RANDMMAP))
+#endif
+
if (addr) {
if (do_color_align && last_mmap)
addr = COLOR_ALIGN(addr, last_mmap, pgoff);
info.high_limit = mm->mmap_base;
info.align_mask = last_mmap ? (PAGE_MASK & (SHM_COLOUR - 1)) : 0;
info.align_offset = shared_align_offset(last_mmap, pgoff);
+ info.threadstack_offset = offset;
addr = vm_unmapped_area(&info);
if (!(addr & ~PAGE_MASK))
goto found_addr;
mm->mmap_legacy_base = mmap_legacy_base();
mm->mmap_base = mmap_upper_limit();
+#ifdef CONFIG_PAX_RANDMMAP
+ if (mm->pax_flags & MF_PAX_RANDMMAP) {
+ mm->mmap_legacy_base += mm->delta_mmap;
+ mm->mmap_base -= mm->delta_mmap + mm->delta_stack;
+ }
+#endif
+
if (mmap_is_legacy()) {
mm->mmap_base = mm->mmap_legacy_base;
mm->get_unmapped_area = arch_get_unmapped_area;
down_read(¤t->mm->mmap_sem);
vma = find_vma(current->mm,regs->iaoq[0]);
- if (vma && (regs->iaoq[0] >= vma->vm_start)
- && (vma->vm_flags & VM_EXEC)) {
-
+ if (vma && (regs->iaoq[0] >= vma->vm_start)) {
fault_address = regs->iaoq[0];
fault_space = regs->iasq[0];
#include <linux/sched.h>
#include <linux/interrupt.h>
#include <linux/module.h>
+#include <linux/unistd.h>
#include <asm/uaccess.h>
#include <asm/traps.h>
static unsigned long
parisc_acctyp(unsigned long code, unsigned int inst)
{
- if (code == 6 || code == 16)
+ if (code == 6 || code == 7 || code == 16)
return VM_EXEC;
switch (inst & 0xf0000000) {
}
#endif
+#ifdef CONFIG_PAX_PAGEEXEC
+/*
+ * PaX: decide what to do with offenders (instruction_pointer(regs) = fault address)
+ *
+ * returns 1 when task should be killed
+ * 2 when rt_sigreturn trampoline was detected
+ * 3 when unpatched PLT trampoline was detected
+ */
+static int pax_handle_fetch_fault(struct pt_regs *regs)
+{
+
+#ifdef CONFIG_PAX_EMUPLT
+ int err;
+
+ do { /* PaX: unpatched PLT emulation */
+ unsigned int bl, depwi;
+
+ err = get_user(bl, (unsigned int *)instruction_pointer(regs));
+ err |= get_user(depwi, (unsigned int *)(instruction_pointer(regs)+4));
+
+ if (err)
+ break;
+
+ if (bl == 0xEA9F1FDDU && depwi == 0xD6801C1EU) {
+ unsigned int ldw, bv, ldw2, addr = instruction_pointer(regs)-12;
+
+ err = get_user(ldw, (unsigned int *)addr);
+ err |= get_user(bv, (unsigned int *)(addr+4));
+ err |= get_user(ldw2, (unsigned int *)(addr+8));
+
+ if (err)
+ break;
+
+ if (ldw == 0x0E801096U &&
+ bv == 0xEAC0C000U &&
+ ldw2 == 0x0E881095U)
+ {
+ unsigned int resolver, map;
+
+ err = get_user(resolver, (unsigned int *)(instruction_pointer(regs)+8));
+ err |= get_user(map, (unsigned int *)(instruction_pointer(regs)+12));
+ if (err)
+ break;
+
+ regs->gr[20] = instruction_pointer(regs)+8;
+ regs->gr[21] = map;
+ regs->gr[22] = resolver;
+ regs->iaoq[0] = resolver | 3UL;
+ regs->iaoq[1] = regs->iaoq[0] + 4;
+ return 3;
+ }
+ }
+ } while (0);
+#endif
+
+#ifdef CONFIG_PAX_EMUTRAMP
+
+#ifndef CONFIG_PAX_EMUSIGRT
+ if (!(current->mm->pax_flags & MF_PAX_EMUTRAMP))
+ return 1;
+#endif
+
+ do { /* PaX: rt_sigreturn emulation */
+ unsigned int ldi1, ldi2, bel, nop;
+
+ err = get_user(ldi1, (unsigned int *)instruction_pointer(regs));
+ err |= get_user(ldi2, (unsigned int *)(instruction_pointer(regs)+4));
+ err |= get_user(bel, (unsigned int *)(instruction_pointer(regs)+8));
+ err |= get_user(nop, (unsigned int *)(instruction_pointer(regs)+12));
+
+ if (err)
+ break;
+
+ if ((ldi1 == 0x34190000U || ldi1 == 0x34190002U) &&
+ ldi2 == 0x3414015AU &&
+ bel == 0xE4008200U &&
+ nop == 0x08000240U)
+ {
+ regs->gr[25] = (ldi1 & 2) >> 1;
+ regs->gr[20] = __NR_rt_sigreturn;
+ regs->gr[31] = regs->iaoq[1] + 16;
+ regs->sr[0] = regs->iasq[1];
+ regs->iaoq[0] = 0x100UL;
+ regs->iaoq[1] = regs->iaoq[0] + 4;
+ regs->iasq[0] = regs->sr[2];
+ regs->iasq[1] = regs->sr[2];
+ return 2;
+ }
+ } while (0);
+#endif
+
+ return 1;
+}
+
+void pax_report_insns(struct pt_regs *regs, void *pc, void *sp)
+{
+ unsigned long i;
+
+ printk(KERN_ERR "PAX: bytes at PC: ");
+ for (i = 0; i < 5; i++) {
+ unsigned int c;
+ if (get_user(c, (unsigned int *)pc+i))
+ printk(KERN_CONT "???????? ");
+ else
+ printk(KERN_CONT "%08x ", c);
+ }
+ printk("\n");
+}
+#endif
+
int fixup_exception(struct pt_regs *regs)
{
const struct exception_table_entry *fix;
good_area:
- if ((vma->vm_flags & acc_type) != acc_type)
+ if ((vma->vm_flags & acc_type) != acc_type) {
+
+#ifdef CONFIG_PAX_PAGEEXEC
+ if ((mm->pax_flags & MF_PAX_PAGEEXEC) && (acc_type & VM_EXEC) &&
+ (address & ~3UL) == instruction_pointer(regs))
+ {
+ up_read(&mm->mmap_sem);
+ switch (pax_handle_fetch_fault(regs)) {
+
+#ifdef CONFIG_PAX_EMUPLT
+ case 3:
+ return;
+#endif
+
+#ifdef CONFIG_PAX_EMUTRAMP
+ case 2:
+ return;
+#endif
+
+ }
+ pax_report_fault(regs, (void *)instruction_pointer(regs), (void *)regs->gr[30]);
+ do_group_exit(SIGKILL);
+ }
+#endif
+
goto bad_area;
+ }
/*
* If for any reason at all we couldn't handle the fault, make
config KEXEC
bool "kexec system call"
depends on (PPC_BOOK3S || FSL_BOOKE || (44x && !SMP))
+ depends on !GRKERNSEC_KMEM
help
kexec is a system call that implements the ability to shutdown your
current kernel, and to start another kernel. It is like a reboot
#define ATOMIC_INIT(i) { (i) }
+#define _ASM_EXTABLE(from, to) \
+" .section __ex_table,\"a\"\n" \
+ PPC_LONG" " #from ", " #to"\n" \
+" .previous\n"
+
static __inline__ int atomic_read(const atomic_t *v)
{
int t;
return t;
}
+static __inline__ int atomic_read_unchecked(const atomic_unchecked_t *v)
+{
+ int t;
+
+ __asm__ __volatile__("lwz%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter));
+
+ return t;
+}
+
static __inline__ void atomic_set(atomic_t *v, int i)
{
__asm__ __volatile__("stw%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i));
}
-#define ATOMIC_OP(op, asm_op) \
-static __inline__ void atomic_##op(int a, atomic_t *v) \
+static __inline__ void atomic_set_unchecked(atomic_unchecked_t *v, int i)
+{
+ __asm__ __volatile__("stw%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i));
+}
+
+#ifdef CONFIG_PAX_REFCOUNT
+#define __REFCOUNT_OP(op) op##o.
+#define __OVERFLOW_PRE \
+ " mcrxr cr0\n"
+#define __OVERFLOW_POST \
+ " bf 4*cr0+so, 3f\n" \
+ "2: .long 0x00c00b00\n" \
+ "3:\n"
+#define __OVERFLOW_EXTABLE \
+ "\n4:\n"
+ _ASM_EXTABLE(2b, 4b)
+#else
+#define __REFCOUNT_OP(op) op
+#define __OVERFLOW_PRE
+#define __OVERFLOW_POST
+#define __OVERFLOW_EXTABLE
+#endif
+
+#define __ATOMIC_OP(op, suffix, pre_op, asm_op, post_op, extable) \
+static inline void atomic_##op##suffix(int a, atomic##suffix##_t *v) \
{ \
int t; \
\
__asm__ __volatile__( \
-"1: lwarx %0,0,%3 # atomic_" #op "\n" \
+"1: lwarx %0,0,%3 # atomic_" #op #suffix "\n" \
+ pre_op \
#asm_op " %0,%2,%0\n" \
+ post_op \
PPC405_ERR77(0,%3) \
" stwcx. %0,0,%3 \n" \
" bne- 1b\n" \
+ extable \
: "=&r" (t), "+m" (v->counter) \
: "r" (a), "r" (&v->counter) \
: "cc"); \
} \
-#define ATOMIC_OP_RETURN(op, asm_op) \
-static __inline__ int atomic_##op##_return(int a, atomic_t *v) \
+#define ATOMIC_OP(op, asm_op) __ATOMIC_OP(op, , , asm_op, , ) \
+ __ATOMIC_OP(op, _unchecked, __OVERFLOW_PRE, __REFCOUNT_OP(asm_op), __OVERFLOW_POST, __OVERFLOW_EXTABLE)
+
+#define __ATOMIC_OP_RETURN(op, suffix, pre_op, asm_op, post_op, extable)\
+static inline int atomic_##op##_return##suffix(int a, atomic##suffix##_t *v)\
{ \
int t; \
\
__asm__ __volatile__( \
PPC_ATOMIC_ENTRY_BARRIER \
-"1: lwarx %0,0,%2 # atomic_" #op "_return\n" \
+"1: lwarx %0,0,%2 # atomic_" #op "_return" #suffix "\n" \
+ pre_op \
#asm_op " %0,%1,%0\n" \
+ post_op \
PPC405_ERR77(0,%2) \
" stwcx. %0,0,%2 \n" \
" bne- 1b\n" \
+ extable \
PPC_ATOMIC_EXIT_BARRIER \
: "=&r" (t) \
: "r" (a), "r" (&v->counter) \
return t; \
}
+#define ATOMIC_OP_RETURN(op, asm_op) __ATOMIC_OP_RETURN(op, , , asm_op, , )\
+ __ATOMIC_OP_RETURN(op, _unchecked, __OVERFLOW_PRE, __REFCOUNT_OP(asm_op), __OVERFLOW_POST, __OVERFLOW_EXTABLE)
+
#define ATOMIC_OPS(op, asm_op) ATOMIC_OP(op, asm_op) ATOMIC_OP_RETURN(op, asm_op)
ATOMIC_OPS(add, add)
#undef ATOMIC_OPS
#undef ATOMIC_OP_RETURN
+#undef __ATOMIC_OP_RETURN
#undef ATOMIC_OP
+#undef __ATOMIC_OP
#define atomic_add_negative(a, v) (atomic_add_return((a), (v)) < 0)
-static __inline__ void atomic_inc(atomic_t *v)
-{
- int t;
+/*
+ * atomic_inc - increment atomic variable
+ * @v: pointer of type atomic_t
+ *
+ * Automatically increments @v by 1
+ */
+#define atomic_inc(v) atomic_add(1, (v))
+#define atomic_inc_return(v) atomic_add_return(1, (v))
- __asm__ __volatile__(
-"1: lwarx %0,0,%2 # atomic_inc\n\
- addic %0,%0,1\n"
- PPC405_ERR77(0,%2)
-" stwcx. %0,0,%2 \n\
- bne- 1b"
- : "=&r" (t), "+m" (v->counter)
- : "r" (&v->counter)
- : "cc", "xer");
+static inline void atomic_inc_unchecked(atomic_unchecked_t *v)
+{
+ atomic_add_unchecked(1, v);
}
-static __inline__ int atomic_inc_return(atomic_t *v)
+static inline int atomic_inc_return_unchecked(atomic_unchecked_t *v)
{
- int t;
-
- __asm__ __volatile__(
- PPC_ATOMIC_ENTRY_BARRIER
-"1: lwarx %0,0,%1 # atomic_inc_return\n\
- addic %0,%0,1\n"
- PPC405_ERR77(0,%1)
-" stwcx. %0,0,%1 \n\
- bne- 1b"
- PPC_ATOMIC_EXIT_BARRIER
- : "=&r" (t)
- : "r" (&v->counter)
- : "cc", "xer", "memory");
-
- return t;
+ return atomic_add_return_unchecked(1, v);
}
/*
*/
#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0)
-static __inline__ void atomic_dec(atomic_t *v)
+static __inline__ int atomic_inc_and_test_unchecked(atomic_unchecked_t *v)
{
- int t;
-
- __asm__ __volatile__(
-"1: lwarx %0,0,%2 # atomic_dec\n\
- addic %0,%0,-1\n"
- PPC405_ERR77(0,%2)\
-" stwcx. %0,0,%2\n\
- bne- 1b"
- : "=&r" (t), "+m" (v->counter)
- : "r" (&v->counter)
- : "cc", "xer");
+ return atomic_add_return_unchecked(1, v) == 0;
}
-static __inline__ int atomic_dec_return(atomic_t *v)
-{
- int t;
-
- __asm__ __volatile__(
- PPC_ATOMIC_ENTRY_BARRIER
-"1: lwarx %0,0,%1 # atomic_dec_return\n\
- addic %0,%0,-1\n"
- PPC405_ERR77(0,%1)
-" stwcx. %0,0,%1\n\
- bne- 1b"
- PPC_ATOMIC_EXIT_BARRIER
- : "=&r" (t)
- : "r" (&v->counter)
- : "cc", "xer", "memory");
+/*
+ * atomic_dec - decrement atomic variable
+ * @v: pointer of type atomic_t
+ *
+ * Atomically decrements @v by 1
+ */
+#define atomic_dec(v) atomic_sub(1, (v))
+#define atomic_dec_return(v) atomic_sub_return(1, (v))
- return t;
+static __inline__ void atomic_dec_unchecked(atomic_unchecked_t *v)
+{
+ atomic_sub_unchecked(1, v);
}
#define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
#define atomic_xchg(v, new) (xchg(&((v)->counter), new))
+static inline int atomic_cmpxchg_unchecked(atomic_unchecked_t *v, int old, int new)
+{
+ return cmpxchg(&(v->counter), old, new);
+}
+
+static inline int atomic_xchg_unchecked(atomic_unchecked_t *v, int new)
+{
+ return xchg(&(v->counter), new);
+}
+
/**
* __atomic_add_unless - add unless the number is a given value
* @v: pointer of type atomic_t
PPC_ATOMIC_ENTRY_BARRIER
"1: lwarx %0,0,%1 # __atomic_add_unless\n\
cmpw 0,%0,%3 \n\
- beq- 2f \n\
- add %0,%2,%0 \n"
+ beq- 2f \n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+" mcrxr cr0\n"
+" addo. %0,%2,%0\n"
+" bf 4*cr0+so, 4f\n"
+"3:.long " "0x00c00b00""\n"
+"4:\n"
+#else
+ "add %0,%2,%0 \n"
+#endif
+
PPC405_ERR77(0,%2)
" stwcx. %0,0,%1 \n\
bne- 1b \n"
+"5:"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ _ASM_EXTABLE(3b, 5b)
+#endif
+
PPC_ATOMIC_EXIT_BARRIER
" subf %0,%2,%0 \n\
2:"
}
#define atomic_dec_if_positive atomic_dec_if_positive
+#define smp_mb__before_atomic_dec() smp_mb()
+#define smp_mb__after_atomic_dec() smp_mb()
+#define smp_mb__before_atomic_inc() smp_mb()
+#define smp_mb__after_atomic_inc() smp_mb()
+
#ifdef __powerpc64__
#define ATOMIC64_INIT(i) { (i) }
return t;
}
+static __inline__ long atomic64_read_unchecked(const atomic64_unchecked_t *v)
+{
+ long t;
+
+ __asm__ __volatile__("ld%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter));
+
+ return t;
+}
+
static __inline__ void atomic64_set(atomic64_t *v, long i)
{
__asm__ __volatile__("std%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i));
}
-#define ATOMIC64_OP(op, asm_op) \
-static __inline__ void atomic64_##op(long a, atomic64_t *v) \
+static __inline__ void atomic64_set_unchecked(atomic64_unchecked_t *v, long i)
+{
+ __asm__ __volatile__("std%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i));
+}
+
+#define __ATOMIC64_OP(op, suffix, pre_op, asm_op, post_op, extable) \
+static inline void atomic64_##op##suffix(long a, atomic64##suffix##_t *v)\
{ \
long t; \
\
__asm__ __volatile__( \
"1: ldarx %0,0,%3 # atomic64_" #op "\n" \
+ pre_op \
#asm_op " %0,%2,%0\n" \
+ post_op \
" stdcx. %0,0,%3 \n" \
" bne- 1b\n" \
+ extable \
: "=&r" (t), "+m" (v->counter) \
: "r" (a), "r" (&v->counter) \
: "cc"); \
}
-#define ATOMIC64_OP_RETURN(op, asm_op) \
-static __inline__ long atomic64_##op##_return(long a, atomic64_t *v) \
+#define ATOMIC64_OP(op, asm_op) __ATOMIC64_OP(op, , , asm_op, , ) \
+ __ATOMIC64_OP(op, _unchecked, __OVERFLOW_PRE, __REFCOUNT_OP(asm_op), __OVERFLOW_POST, __OVERFLOW_EXTABLE)
+
+#define __ATOMIC64_OP_RETURN(op, suffix, pre_op, asm_op, post_op, extable)\
+static inline long atomic64_##op##_return##suffix(long a, atomic64##suffix##_t *v)\
{ \
long t; \
\
__asm__ __volatile__( \
PPC_ATOMIC_ENTRY_BARRIER \
"1: ldarx %0,0,%2 # atomic64_" #op "_return\n" \
+ pre_op \
#asm_op " %0,%1,%0\n" \
+ post_op \
" stdcx. %0,0,%2 \n" \
" bne- 1b\n" \
+ extable \
PPC_ATOMIC_EXIT_BARRIER \
: "=&r" (t) \
: "r" (a), "r" (&v->counter) \
return t; \
}
+#define ATOMIC64_OP_RETURN(op, asm_op) __ATOMIC64_OP_RETURN(op, , , asm_op, , )\
+ __ATOMIC64_OP_RETURN(op, _unchecked, __OVERFLOW_PRE, __REFCOUNT_OP(asm_op), __OVERFLOW_POST, __OVERFLOW_EXTABLE)
+
#define ATOMIC64_OPS(op, asm_op) ATOMIC64_OP(op, asm_op) ATOMIC64_OP_RETURN(op, asm_op)
ATOMIC64_OPS(add, add)
#undef ATOMIC64_OPS
#undef ATOMIC64_OP_RETURN
+#undef __ATOMIC64_OP_RETURN
#undef ATOMIC64_OP
+#undef __ATOMIC64_OP
+#undef __OVERFLOW_EXTABLE
+#undef __OVERFLOW_POST
+#undef __OVERFLOW_PRE
+#undef __REFCOUNT_OP
#define atomic64_add_negative(a, v) (atomic64_add_return((a), (v)) < 0)
-static __inline__ void atomic64_inc(atomic64_t *v)
-{
- long t;
+/*
+ * atomic64_inc - increment atomic variable
+ * @v: pointer of type atomic64_t
+ *
+ * Automatically increments @v by 1
+ */
+#define atomic64_inc(v) atomic64_add(1, (v))
+#define atomic64_inc_return(v) atomic64_add_return(1, (v))
- __asm__ __volatile__(
-"1: ldarx %0,0,%2 # atomic64_inc\n\
- addic %0,%0,1\n\
- stdcx. %0,0,%2 \n\
- bne- 1b"
- : "=&r" (t), "+m" (v->counter)
- : "r" (&v->counter)
- : "cc", "xer");
+static inline void atomic64_inc_unchecked(atomic64_unchecked_t *v)
+{
+ atomic64_add_unchecked(1, v);
}
-static __inline__ long atomic64_inc_return(atomic64_t *v)
+static inline long atomic64_inc_return_unchecked(atomic64_unchecked_t *v)
{
- long t;
-
- __asm__ __volatile__(
- PPC_ATOMIC_ENTRY_BARRIER
-"1: ldarx %0,0,%1 # atomic64_inc_return\n\
- addic %0,%0,1\n\
- stdcx. %0,0,%1 \n\
- bne- 1b"
- PPC_ATOMIC_EXIT_BARRIER
- : "=&r" (t)
- : "r" (&v->counter)
- : "cc", "xer", "memory");
-
- return t;
+ return atomic64_add_return_unchecked(1, v);
}
/*
*/
#define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0)
-static __inline__ void atomic64_dec(atomic64_t *v)
-{
- long t;
-
- __asm__ __volatile__(
-"1: ldarx %0,0,%2 # atomic64_dec\n\
- addic %0,%0,-1\n\
- stdcx. %0,0,%2\n\
- bne- 1b"
- : "=&r" (t), "+m" (v->counter)
- : "r" (&v->counter)
- : "cc", "xer");
-}
+/*
+ * atomic64_dec - decrement atomic variable
+ * @v: pointer of type atomic64_t
+ *
+ * Atomically decrements @v by 1
+ */
+#define atomic64_dec(v) atomic64_sub(1, (v))
+#define atomic64_dec_return(v) atomic64_sub_return(1, (v))
-static __inline__ long atomic64_dec_return(atomic64_t *v)
+static __inline__ void atomic64_dec_unchecked(atomic64_unchecked_t *v)
{
- long t;
-
- __asm__ __volatile__(
- PPC_ATOMIC_ENTRY_BARRIER
-"1: ldarx %0,0,%1 # atomic64_dec_return\n\
- addic %0,%0,-1\n\
- stdcx. %0,0,%1\n\
- bne- 1b"
- PPC_ATOMIC_EXIT_BARRIER
- : "=&r" (t)
- : "r" (&v->counter)
- : "cc", "xer", "memory");
-
- return t;
+ atomic64_sub_unchecked(1, v);
}
#define atomic64_sub_and_test(a, v) (atomic64_sub_return((a), (v)) == 0)
#define atomic64_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
#define atomic64_xchg(v, new) (xchg(&((v)->counter), new))
+static inline long atomic64_cmpxchg_unchecked(atomic64_unchecked_t *v, long old, long new)
+{
+ return cmpxchg(&(v->counter), old, new);
+}
+
+static inline long atomic64_xchg_unchecked(atomic64_unchecked_t *v, long new)
+{
+ return xchg(&(v->counter), new);
+}
+
/**
* atomic64_add_unless - add unless the number is a given value
* @v: pointer of type atomic64_t
__asm__ __volatile__ (
PPC_ATOMIC_ENTRY_BARRIER
-"1: ldarx %0,0,%1 # __atomic_add_unless\n\
+"1: ldarx %0,0,%1 # atomic64_add_unless\n\
cmpd 0,%0,%3 \n\
- beq- 2f \n\
- add %0,%2,%0 \n"
+ beq- 2f \n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+" mcrxr cr0\n"
+" addo. %0,%2,%0\n"
+" bf 4*cr0+so, 4f\n"
+"3:.long " "0x00c00b00""\n"
+"4:\n"
+#else
+ "add %0,%2,%0 \n"
+#endif
+
" stdcx. %0,0,%1 \n\
bne- 1b \n"
PPC_ATOMIC_EXIT_BARRIER
+"5:"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ _ASM_EXTABLE(3b, 5b)
+#endif
+
" subf %0,%2,%0 \n\
2:"
: "=&r" (t)
do { \
compiletime_assert_atomic_type(*p); \
smp_lwsync(); \
- ACCESS_ONCE(*p) = (v); \
+ ACCESS_ONCE_RW(*p) = (v); \
} while (0)
#define smp_load_acquire(p) \
#ifdef __KERNEL__
#include <asm/reg.h>
+#include <linux/const.h>
/* bytes per L1 cache line */
#if defined(CONFIG_8xx) || defined(CONFIG_403GCX)
#define L1_CACHE_SHIFT 7
#endif
-#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT)
+#define L1_CACHE_BYTES (_AC(1,UL) << L1_CACHE_SHIFT)
#define SMP_CACHE_BYTES L1_CACHE_BYTES
#define ELF_ET_DYN_BASE 0x20000000
+#ifdef CONFIG_PAX_ASLR
+#define PAX_ELF_ET_DYN_BASE (0x10000000UL)
+
+#ifdef __powerpc64__
+#define PAX_DELTA_MMAP_LEN (is_32bit_task() ? 16 : 28)
+#define PAX_DELTA_STACK_LEN (is_32bit_task() ? 16 : 28)
+#else
+#define PAX_DELTA_MMAP_LEN 15
+#define PAX_DELTA_STACK_LEN 15
+#endif
+#endif
+
#define ELF_CORE_EFLAGS (is_elf2_task() ? 2 : 0)
/*
(0x7ff >> (PAGE_SHIFT - 12)) : \
(0x3ffff >> (PAGE_SHIFT - 12)))
-extern unsigned long arch_randomize_brk(struct mm_struct *mm);
-#define arch_randomize_brk arch_randomize_brk
-
-
#ifdef CONFIG_SPU_BASE
/* Notes used in ET_CORE. Note name is "SPU/<fd>/<filename>". */
#define NT_SPU 1
#ifndef _ASM_POWERPC_EXEC_H
#define _ASM_POWERPC_EXEC_H
-extern unsigned long arch_align_stack(unsigned long sp);
+#define arch_align_stack(x) ((x) & ~0xfUL)
#endif /* _ASM_POWERPC_EXEC_H */
* 2 of the License, or (at your option) any later version.
*/
-#define KM_TYPE_NR 16
+#define KM_TYPE_NR 17
#endif /* __KERNEL__ */
#endif /* _ASM_POWERPC_KMAP_TYPES_H */
atomic_long_t a;
} local_t;
+typedef struct
+{
+ atomic_long_unchecked_t a;
+} local_unchecked_t;
+
#define LOCAL_INIT(i) { ATOMIC_LONG_INIT(i) }
#define local_read(l) atomic_long_read(&(l)->a)
+#define local_read_unchecked(l) atomic_long_read_unchecked(&(l)->a)
#define local_set(l,i) atomic_long_set(&(l)->a, (i))
+#define local_set_unchecked(l,i) atomic_long_set_unchecked(&(l)->a, (i))
#define local_add(i,l) atomic_long_add((i),(&(l)->a))
+#define local_add_unchecked(i,l) atomic_long_add_unchecked((i),(&(l)->a))
#define local_sub(i,l) atomic_long_sub((i),(&(l)->a))
+#define local_sub_unchecked(i,l) atomic_long_sub_unchecked((i),(&(l)->a))
#define local_inc(l) atomic_long_inc(&(l)->a)
+#define local_inc_unchecked(l) atomic_long_inc_unchecked(&(l)->a)
#define local_dec(l) atomic_long_dec(&(l)->a)
+#define local_dec_unchecked(l) atomic_long_dec_unchecked(&(l)->a)
static __inline__ long local_add_return(long a, local_t *l)
{
long t;
+ __asm__ __volatile__(
+"1:" PPC_LLARX(%0,0,%2,0) " # local_add_return\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+" mcrxr cr0\n"
+" addo. %0,%1,%0\n"
+" bf 4*cr0+so, 3f\n"
+"2:.long " "0x00c00b00""\n"
+#else
+" add %0,%1,%0\n"
+#endif
+
+"3:\n"
+ PPC405_ERR77(0,%2)
+ PPC_STLCX "%0,0,%2 \n\
+ bne- 1b"
+
+#ifdef CONFIG_PAX_REFCOUNT
+"\n4:\n"
+ _ASM_EXTABLE(2b, 4b)
+#endif
+
+ : "=&r" (t)
+ : "r" (a), "r" (&(l->a.counter))
+ : "cc", "memory");
+
+ return t;
+}
+
+static __inline__ long local_add_return_unchecked(long a, local_unchecked_t *l)
+{
+ long t;
+
__asm__ __volatile__(
"1:" PPC_LLARX(%0,0,%2,0) " # local_add_return\n\
add %0,%1,%0\n"
#define local_cmpxchg(l, o, n) \
(cmpxchg_local(&((l)->a.counter), (o), (n)))
+#define local_cmpxchg_unchecked(l, o, n) \
+ (cmpxchg_local(&((l)->a.counter), (o), (n)))
#define local_xchg(l, n) (xchg_local(&((l)->a.counter), (n)))
/**
}
#define arch_calc_vm_prot_bits(prot) arch_calc_vm_prot_bits(prot)
-static inline pgprot_t arch_vm_get_page_prot(unsigned long vm_flags)
+static inline pgprot_t arch_vm_get_page_prot(vm_flags_t vm_flags)
{
return (vm_flags & VM_SAO) ? __pgprot(_PAGE_SAO) : __pgprot(0);
}
* and needs to be executable. This means the whole heap ends
* up being executable.
*/
-#define VM_DATA_DEFAULT_FLAGS32 (VM_READ | VM_WRITE | VM_EXEC | \
- VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC)
+#define VM_DATA_DEFAULT_FLAGS32 \
+ (((current->personality & READ_IMPLIES_EXEC) ? VM_EXEC : 0) | \
+ VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC)
#define VM_DATA_DEFAULT_FLAGS64 (VM_READ | VM_WRITE | \
VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC)
#define is_kernel_addr(x) ((x) >= PAGE_OFFSET)
#endif
+#define ktla_ktva(addr) (addr)
+#define ktva_ktla(addr) (addr)
+
#ifndef CONFIG_PPC_BOOK3S_64
/*
* Use the top bit of the higher-level page table entries to indicate whether
* stack by default, so in the absence of a PT_GNU_STACK program header
* we turn execute permission off.
*/
-#define VM_STACK_DEFAULT_FLAGS32 (VM_READ | VM_WRITE | VM_EXEC | \
- VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC)
+#define VM_STACK_DEFAULT_FLAGS32 \
+ (((current->personality & READ_IMPLIES_EXEC) ? VM_EXEC : 0) | \
+ VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC)
#define VM_STACK_DEFAULT_FLAGS64 (VM_READ | VM_WRITE | \
VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC)
+#ifndef CONFIG_PAX_PAGEEXEC
#define VM_STACK_DEFAULT_FLAGS \
(is_32bit_task() ? \
VM_STACK_DEFAULT_FLAGS32 : VM_STACK_DEFAULT_FLAGS64)
+#endif
#include <asm-generic/getorder.h>
#ifndef CONFIG_PPC_64K_PAGES
#define pgd_populate(MM, PGD, PUD) pgd_set(PGD, PUD)
+#define pgd_populate_kernel(MM, PGD, PUD) pgd_populate((MM), (PGD), (PUD))
static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
{
pud_set(pud, (unsigned long)pmd);
}
+static inline void pud_populate_kernel(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
+{
+ pud_populate(mm, pud, pmd);
+}
+
#define pmd_populate(mm, pmd, pte_page) \
pmd_populate_kernel(mm, pmd, page_address(pte_page))
#define pmd_populate_kernel(mm, pmd, pte) pmd_set(pmd, (unsigned long)(pte))
#endif
#define pud_populate(mm, pud, pmd) pud_set(pud, (unsigned long)pmd)
+#define pud_populate_kernel(mm, pud, pmd) pud_populate((mm), (pud), (pmd))
static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd,
pte_t *pte)
#define _ASM_POWERPC_PGTABLE_H
#ifdef __KERNEL__
+#include <linux/const.h>
#ifndef __ASSEMBLY__
#include <linux/mmdebug.h>
#include <linux/mmzone.h>
#define _PAGE_FILE 0x004 /* when !present: nonlinear file mapping */
#define _PAGE_USER 0x004 /* usermode access allowed */
#define _PAGE_GUARDED 0x008 /* G: prohibit speculative access */
+#define _PAGE_EXEC _PAGE_GUARDED
#define _PAGE_COHERENT 0x010 /* M: enforce memory coherence (SMP systems) */
#define _PAGE_NO_CACHE 0x020 /* I: cache inhibit */
#define _PAGE_WRITETHRU 0x040 /* W: cache write-through */
#define SPRN_DBCR 0x136 /* e300 Data Breakpoint Control Reg */
#define SPRN_DSISR 0x012 /* Data Storage Interrupt Status Register */
#define DSISR_NOHPTE 0x40000000 /* no translation found */
+#define DSISR_GUARDED 0x10000000 /* fetch from guarded storage */
#define DSISR_PROTFAULT 0x08000000 /* protection fault */
#define DSISR_ISSTORE 0x02000000 /* access was a store */
#define DSISR_DABRMATCH 0x00400000 /* hit data breakpoint */
int (*cpu_disable)(void);
void (*cpu_die)(unsigned int nr);
int (*cpu_bootable)(unsigned int nr);
-};
+} __no_const;
extern void smp_send_debugger_break(void);
extern void start_secondary_resume(void);
__asm__ __volatile__(
"1: " PPC_LWARX(%0,0,%1,1) "\n"
__DO_SIGN_EXTEND
-" addic. %0,%0,1\n\
- ble- 2f\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+" mcrxr cr0\n"
+" addico. %0,%0,1\n"
+" bf 4*cr0+so, 3f\n"
+"2:.long " "0x00c00b00""\n"
+#else
+" addic. %0,%0,1\n"
+#endif
+
+"3:\n"
+ "ble- 4f\n"
PPC405_ERR77(0,%1)
" stwcx. %0,0,%1\n\
bne- 1b\n"
PPC_ACQUIRE_BARRIER
-"2:" : "=&r" (tmp)
+"4:"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ _ASM_EXTABLE(2b,4b)
+#endif
+
+ : "=&r" (tmp)
: "r" (&rw->lock)
: "cr0", "xer", "memory");
__asm__ __volatile__(
"# read_unlock\n\t"
PPC_RELEASE_BARRIER
-"1: lwarx %0,0,%1\n\
- addic %0,%0,-1\n"
+"1: lwarx %0,0,%1\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+" mcrxr cr0\n"
+" addico. %0,%0,-1\n"
+" bf 4*cr0+so, 3f\n"
+"2:.long " "0x00c00b00""\n"
+#else
+" addic. %0,%0,-1\n"
+#endif
+
+"3:\n"
PPC405_ERR77(0,%1)
" stwcx. %0,0,%1\n\
bne- 1b"
+
+#ifdef CONFIG_PAX_REFCOUNT
+"\n4:\n"
+ _ASM_EXTABLE(2b, 4b)
+#endif
+
: "=&r"(tmp)
: "r"(&rw->lock)
: "cr0", "xer", "memory");
#if defined(CONFIG_PPC64)
#define TIF_ELF2ABI 18 /* function descriptors must die! */
#endif
+/* mask must be expressable within 16 bits to satisfy 'andi' instruction reqs */
+#define TIF_GRSEC_SETXID 6 /* update credentials on syscall entry/exit */
/* as above, but as bit values */
#define _TIF_SYSCALL_TRACE (1<<TIF_SYSCALL_TRACE)
#define _TIF_SYSCALL_TRACEPOINT (1<<TIF_SYSCALL_TRACEPOINT)
#define _TIF_EMULATE_STACK_STORE (1<<TIF_EMULATE_STACK_STORE)
#define _TIF_NOHZ (1<<TIF_NOHZ)
+#define _TIF_GRSEC_SETXID (1<<TIF_GRSEC_SETXID)
#define _TIF_SYSCALL_T_OR_A (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \
_TIF_SECCOMP | _TIF_SYSCALL_TRACEPOINT | \
- _TIF_NOHZ)
+ _TIF_NOHZ | _TIF_GRSEC_SETXID)
#define _TIF_USER_WORK_MASK (_TIF_SIGPENDING | _TIF_NEED_RESCHED | \
_TIF_NOTIFY_RESUME | _TIF_UPROBE | \
#endif
+#define access_ok_noprefault(type, addr, size) access_ok((type), (addr), (size))
#define access_ok(type, addr, size) \
(__chk_user_ptr(addr), \
__access_ok((__force unsigned long)(addr), (size), get_fs()))
extern unsigned long __copy_tofrom_user(void __user *to,
const void __user *from, unsigned long size);
-#ifndef __powerpc64__
-
-static inline unsigned long copy_from_user(void *to,
- const void __user *from, unsigned long n)
-{
- unsigned long over;
-
- if (access_ok(VERIFY_READ, from, n))
- return __copy_tofrom_user((__force void __user *)to, from, n);
- if ((unsigned long)from < TASK_SIZE) {
- over = (unsigned long)from + n - TASK_SIZE;
- return __copy_tofrom_user((__force void __user *)to, from,
- n - over) + over;
- }
- return n;
-}
-
-static inline unsigned long copy_to_user(void __user *to,
- const void *from, unsigned long n)
-{
- unsigned long over;
-
- if (access_ok(VERIFY_WRITE, to, n))
- return __copy_tofrom_user(to, (__force void __user *)from, n);
- if ((unsigned long)to < TASK_SIZE) {
- over = (unsigned long)to + n - TASK_SIZE;
- return __copy_tofrom_user(to, (__force void __user *)from,
- n - over) + over;
- }
- return n;
-}
-
-#else /* __powerpc64__ */
-
-#define __copy_in_user(to, from, size) \
- __copy_tofrom_user((to), (from), (size))
-
-extern unsigned long copy_from_user(void *to, const void __user *from,
- unsigned long n);
-extern unsigned long copy_to_user(void __user *to, const void *from,
- unsigned long n);
-extern unsigned long copy_in_user(void __user *to, const void __user *from,
- unsigned long n);
-
-#endif /* __powerpc64__ */
-
static inline unsigned long __copy_from_user_inatomic(void *to,
const void __user *from, unsigned long n)
{
if (ret == 0)
return 0;
}
+
+ if (!__builtin_constant_p(n))
+ check_object_size(to, n, false);
+
return __copy_tofrom_user((__force void __user *)to, from, n);
}
if (ret == 0)
return 0;
}
+
+ if (!__builtin_constant_p(n))
+ check_object_size(from, n, true);
+
return __copy_tofrom_user(to, (__force const void __user *)from, n);
}
return __copy_to_user_inatomic(to, from, size);
}
+#ifndef __powerpc64__
+
+static inline unsigned long __must_check copy_from_user(void *to,
+ const void __user *from, unsigned long n)
+{
+ unsigned long over;
+
+ if ((long)n < 0)
+ return n;
+
+ if (access_ok(VERIFY_READ, from, n)) {
+ if (!__builtin_constant_p(n))
+ check_object_size(to, n, false);
+ return __copy_tofrom_user((__force void __user *)to, from, n);
+ }
+ if ((unsigned long)from < TASK_SIZE) {
+ over = (unsigned long)from + n - TASK_SIZE;
+ if (!__builtin_constant_p(n - over))
+ check_object_size(to, n - over, false);
+ return __copy_tofrom_user((__force void __user *)to, from,
+ n - over) + over;
+ }
+ return n;
+}
+
+static inline unsigned long __must_check copy_to_user(void __user *to,
+ const void *from, unsigned long n)
+{
+ unsigned long over;
+
+ if ((long)n < 0)
+ return n;
+
+ if (access_ok(VERIFY_WRITE, to, n)) {
+ if (!__builtin_constant_p(n))
+ check_object_size(from, n, true);
+ return __copy_tofrom_user(to, (__force void __user *)from, n);
+ }
+ if ((unsigned long)to < TASK_SIZE) {
+ over = (unsigned long)to + n - TASK_SIZE;
+ if (!__builtin_constant_p(n))
+ check_object_size(from, n - over, true);
+ return __copy_tofrom_user(to, (__force void __user *)from,
+ n - over) + over;
+ }
+ return n;
+}
+
+#else /* __powerpc64__ */
+
+#define __copy_in_user(to, from, size) \
+ __copy_tofrom_user((to), (from), (size))
+
+static inline unsigned long __must_check copy_from_user(void *to, const void __user *from, unsigned long n)
+{
+ if ((long)n < 0 || n > INT_MAX)
+ return n;
+
+ if (!__builtin_constant_p(n))
+ check_object_size(to, n, false);
+
+ if (likely(access_ok(VERIFY_READ, from, n)))
+ n = __copy_from_user(to, from, n);
+ else
+ memset(to, 0, n);
+ return n;
+}
+
+static inline unsigned long __must_check copy_to_user(void __user *to, const void *from, unsigned long n)
+{
+ if ((long)n < 0 || n > INT_MAX)
+ return n;
+
+ if (likely(access_ok(VERIFY_WRITE, to, n))) {
+ if (!__builtin_constant_p(n))
+ check_object_size(from, n, true);
+ n = __copy_to_user(to, from, n);
+ }
+ return n;
+}
+
+extern unsigned long copy_in_user(void __user *to, const void __user *from,
+ unsigned long n);
+
+#endif /* __powerpc64__ */
+
extern unsigned long __clear_user(void __user *addr, unsigned long size);
static inline unsigned long clear_user(void __user *addr, unsigned long size)
CFLAGS_btext.o += -fPIC
endif
+CFLAGS_REMOVE_cputable.o = $(LATENT_ENTROPY_PLUGIN_CFLAGS)
+CFLAGS_REMOVE_prom_init.o = $(LATENT_ENTROPY_PLUGIN_CFLAGS)
+CFLAGS_REMOVE_btext.o = $(LATENT_ENTROPY_PLUGIN_CFLAGS)
+CFLAGS_REMOVE_prom.o = $(LATENT_ENTROPY_PLUGIN_CFLAGS)
+
ifdef CONFIG_FUNCTION_TRACER
# Do not trace early boot code
CFLAGS_REMOVE_cputable.o = -pg -mno-sched-epilog
CFLAGS_REMOVE_time.o = -pg -mno-sched-epilog
endif
+CFLAGS_REMOVE_prom_init.o += $(LATENT_ENTROPY_PLUGIN_CFLAGS)
+
obj-y := cputable.o ptrace.o syscalls.o \
irq.o align.o signal_32.o pmc.o vdso.o \
process.o systbl.o idle.o \
std r14,_DAR(r1)
std r15,_DSISR(r1)
addi r3,r1,STACK_FRAME_OVERHEAD
+ bl save_nvgprs
mr r4,r14
mr r5,r15
ld r14,PACA_EXGEN+EX_R14(r13)
cmpdi r3,0
bne- 1f
b ret_from_except_lite
-1: bl save_nvgprs
- mr r5,r3
+1: mr r5,r3
addi r3,r1,STACK_FRAME_OVERHEAD
ld r4,_DAR(r1)
bl bad_page_fault
11: ld r4,_DAR(r1)
ld r5,_DSISR(r1)
addi r3,r1,STACK_FRAME_OVERHEAD
+ bl save_nvgprs
bl do_page_fault
cmpdi r3,0
beq+ 12f
- bl save_nvgprs
mr r5,r3
addi r3,r1,STACK_FRAME_OVERHEAD
lwz r4,_DAR(r1)
}
#endif
+extern void gr_handle_kernel_exploit(void);
+
static inline void check_stack_overflow(void)
{
#ifdef CONFIG_DEBUG_STACKOVERFLOW
pr_err("do_IRQ: stack overflow: %ld\n",
sp - sizeof(struct thread_info));
dump_stack();
+ gr_handle_kernel_exploit();
}
#endif
}
me->arch.core_plt_section = i;
}
if (!me->arch.core_plt_section || !me->arch.init_plt_section) {
- pr_err("Module doesn't contain .plt or .init.plt sections.\n");
+ pr_err("Module $s doesn't contain .plt or .init.plt sections.\n", me->name);
return -ENOEXEC;
}
pr_debug("Doing plt for call to 0x%x at 0x%x\n", val, (unsigned int)location);
/* Init, or core PLT? */
- if (location >= mod->module_core
- && location < mod->module_core + mod->core_size)
+ if ((location >= mod->module_core_rx && location < mod->module_core_rx + mod->core_size_rx) ||
+ (location >= mod->module_core_rw && location < mod->module_core_rw + mod->core_size_rw))
entry = (void *)sechdrs[mod->arch.core_plt_section].sh_addr;
- else
+ else if ((location >= mod->module_init_rx && location < mod->module_init_rx + mod->init_size_rx) ||
+ (location >= mod->module_init_rw && location < mod->module_init_rw + mod->init_size_rw))
entry = (void *)sechdrs[mod->arch.init_plt_section].sh_addr;
+ else {
+ printk(KERN_ERR "%s: invalid R_PPC_REL24 entry found\n", mod->name);
+ return ~0UL;
+ }
/* Find this entry, or if that fails, the next avail. entry */
while (entry->jump[0]) {
}
#ifdef CONFIG_DYNAMIC_FTRACE
module->arch.tramp =
- do_plt_call(module->module_core,
+ do_plt_call(module->module_core_rx,
(unsigned long)ftrace_caller,
sechdrs, module);
#endif
* Lookup NIP late so we have the best change of getting the
* above info out without failing
*/
- printk("NIP ["REG"] %pS\n", regs->nip, (void *)regs->nip);
- printk("LR ["REG"] %pS\n", regs->link, (void *)regs->link);
+ printk("NIP ["REG"] %pA\n", regs->nip, (void *)regs->nip);
+ printk("LR ["REG"] %pA\n", regs->link, (void *)regs->link);
#endif
show_stack(current, (unsigned long *) regs->gpr[1]);
if (!user_mode(regs))
newsp = stack[0];
ip = stack[STACK_FRAME_LR_SAVE];
if (!firstframe || ip != lr) {
- printk("["REG"] ["REG"] %pS", sp, ip, (void *)ip);
+ printk("["REG"] ["REG"] %pA", sp, ip, (void *)ip);
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
if ((ip == rth) && curr_frame >= 0) {
- printk(" (%pS)",
+ printk(" (%pA)",
(void *)current->ret_stack[curr_frame].ret);
curr_frame--;
}
struct pt_regs *regs = (struct pt_regs *)
(sp + STACK_FRAME_OVERHEAD);
lr = regs->link;
- printk("--- interrupt: %lx at %pS\n LR = %pS\n",
+ printk("--- interrupt: %lx at %pA\n LR = %pA\n",
regs->trap, (void *)regs->nip, (void *)lr);
firstframe = 1;
}
mtspr(SPRN_CTRLT, ctrl);
}
#endif /* CONFIG_PPC64 */
-
-unsigned long arch_align_stack(unsigned long sp)
-{
- if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
- sp -= get_random_int() & ~PAGE_MASK;
- return sp & ~0xf;
-}
-
-static inline unsigned long brk_rnd(void)
-{
- unsigned long rnd = 0;
-
- /* 8MB for 32bit, 1GB for 64bit */
- if (is_32bit_task())
- rnd = (long)(get_random_int() % (1<<(23-PAGE_SHIFT)));
- else
- rnd = (long)(get_random_int() % (1<<(30-PAGE_SHIFT)));
-
- return rnd << PAGE_SHIFT;
-}
-
-unsigned long arch_randomize_brk(struct mm_struct *mm)
-{
- unsigned long base = mm->brk;
- unsigned long ret;
-
-#ifdef CONFIG_PPC_STD_MMU_64
- /*
- * If we are using 1TB segments and we are allowed to randomise
- * the heap, we can put it above 1TB so it is backed by a 1TB
- * segment. Otherwise the heap will be in the bottom 1TB
- * which always uses 256MB segments and this may result in a
- * performance penalty.
- */
- if (!is_32bit_task() && (mmu_highuser_ssize == MMU_SEGSIZE_1T))
- base = max_t(unsigned long, mm->brk, 1UL << SID_SHIFT_1T);
-#endif
-
- ret = PAGE_ALIGN(base + brk_rnd());
-
- if (ret < mm->brk)
- return mm->brk;
-
- return ret;
-}
-
return ret;
}
+#ifdef CONFIG_GRKERNSEC_SETXID
+extern void gr_delayed_cred_worker(void);
+#endif
+
/*
* We must return the syscall number to actually look up in the table.
* This can be -1L to skip running any syscall at all.
secure_computing_strict(regs->gpr[0]);
+#ifdef CONFIG_GRKERNSEC_SETXID
+ if (unlikely(test_and_clear_thread_flag(TIF_GRSEC_SETXID)))
+ gr_delayed_cred_worker();
+#endif
+
if (test_thread_flag(TIF_SYSCALL_TRACE) &&
tracehook_report_syscall_entry(regs))
/*
{
int step;
+#ifdef CONFIG_GRKERNSEC_SETXID
+ if (unlikely(test_and_clear_thread_flag(TIF_GRSEC_SETXID)))
+ gr_delayed_cred_worker();
+#endif
+
audit_syscall_exit(regs);
if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT)))
/* Save user registers on the stack */
frame = &rt_sf->uc.uc_mcontext;
addr = frame;
- if (vdso32_rt_sigtramp && current->mm->context.vdso_base) {
+ if (vdso32_rt_sigtramp && current->mm->context.vdso_base != ~0UL) {
sigret = 0;
tramp = current->mm->context.vdso_base + vdso32_rt_sigtramp;
} else {
current->thread.fp_state.fpscr = 0;
/* Set up to return from userspace. */
- if (vdso64_rt_sigtramp && current->mm->context.vdso_base) {
+ if (vdso64_rt_sigtramp && current->mm->context.vdso_base != ~0UL) {
regs->link = current->mm->context.vdso_base + vdso64_rt_sigtramp;
} else {
err |= setup_trampoline(__NR_rt_sigreturn, &frame->tramp[0]);
#include <linux/debugfs.h>
#include <linux/ratelimit.h>
#include <linux/context_tracking.h>
+#include <linux/uaccess.h>
#include <asm/emulated_ops.h>
#include <asm/pgtable.h>
return flags;
}
+extern void gr_handle_kernel_exploit(void);
+
static void __kprobes oops_end(unsigned long flags, struct pt_regs *regs,
int signr)
{
panic("Fatal exception in interrupt");
if (panic_on_oops)
panic("Fatal exception");
+
+ gr_handle_kernel_exploit();
+
do_exit(signr);
}
enum ctx_state prev_state = exception_enter();
unsigned int reason = get_reason(regs);
+#ifdef CONFIG_PAX_REFCOUNT
+ unsigned int bkpt;
+ const struct exception_table_entry *entry;
+
+ if (reason & REASON_ILLEGAL) {
+ /* Check if PaX bad instruction */
+ if (!probe_kernel_address(regs->nip, bkpt) && bkpt == 0xc00b00) {
+ current->thread.trap_nr = 0;
+ pax_report_refcount_overflow(regs);
+ /* fixup_exception() for PowerPC does not exist, simulate its job */
+ if ((entry = search_exception_tables(regs->nip)) != NULL) {
+ regs->nip = entry->fixup;
+ return;
+ }
+ /* fixup_exception() could not handle */
+ goto bail;
+ }
+ }
+#endif
+
/* We can now get here via a FP Unavailable exception if the core
* has no FPU, in that case the reason flags will be 0 */
#include <asm/vdso.h>
#include <asm/vdso_datapage.h>
#include <asm/setup.h>
+#include <asm/mman.h>
#undef DEBUG
vdso_base = VDSO32_MBASE;
#endif
- current->mm->context.vdso_base = 0;
+ current->mm->context.vdso_base = ~0UL;
/* vDSO has a problem and was disabled, just don't "enable" it for the
* process
vdso_base = get_unmapped_area(NULL, vdso_base,
(vdso_pages << PAGE_SHIFT) +
((VDSO_ALIGNMENT - 1) & PAGE_MASK),
- 0, 0);
+ 0, MAP_PRIVATE | MAP_EXECUTABLE);
if (IS_ERR_VALUE(vdso_base)) {
rc = vdso_base;
goto fail_mmapsem;
}
EXPORT_SYMBOL_GPL(kvmppc_init_lpid);
-int kvm_arch_init(void *opaque)
+int kvm_arch_init(const void *opaque)
{
return 0;
}
#include <linux/module.h>
#include <asm/uaccess.h>
-unsigned long copy_from_user(void *to, const void __user *from, unsigned long n)
-{
- if (likely(access_ok(VERIFY_READ, from, n)))
- n = __copy_from_user(to, from, n);
- else
- memset(to, 0, n);
- return n;
-}
-
-unsigned long copy_to_user(void __user *to, const void *from, unsigned long n)
-{
- if (likely(access_ok(VERIFY_WRITE, to, n)))
- n = __copy_to_user(to, from, n);
- return n;
-}
-
unsigned long copy_in_user(void __user *to, const void __user *from,
unsigned long n)
{
return n;
}
-EXPORT_SYMBOL(copy_from_user);
-EXPORT_SYMBOL(copy_to_user);
EXPORT_SYMBOL(copy_in_user);
#include <linux/ratelimit.h>
#include <linux/context_tracking.h>
#include <linux/hugetlb.h>
+#include <linux/slab.h>
+#include <linux/pagemap.h>
+#include <linux/compiler.h>
+#include <linux/unistd.h>
#include <asm/firmware.h>
#include <asm/page.h>
}
#endif
+#ifdef CONFIG_PAX_PAGEEXEC
+/*
+ * PaX: decide what to do with offenders (regs->nip = fault address)
+ *
+ * returns 1 when task should be killed
+ */
+static int pax_handle_fetch_fault(struct pt_regs *regs)
+{
+ return 1;
+}
+
+void pax_report_insns(struct pt_regs *regs, void *pc, void *sp)
+{
+ unsigned long i;
+
+ printk(KERN_ERR "PAX: bytes at PC: ");
+ for (i = 0; i < 5; i++) {
+ unsigned int c;
+ if (get_user(c, (unsigned int __user *)pc+i))
+ printk(KERN_CONT "???????? ");
+ else
+ printk(KERN_CONT "%08x ", c);
+ }
+ printk("\n");
+}
+#endif
+
/*
* Check whether the instruction at regs->nip is a store using
* an update addressing form which will update r1.
* indicate errors in DSISR but can validly be set in SRR1.
*/
if (trap == 0x400)
- error_code &= 0x48200000;
+ error_code &= 0x58200000;
else
is_write = error_code & DSISR_ISSTORE;
#else
* "undefined". Of those that can be set, this is the only
* one which seems bad.
*/
- if (error_code & 0x10000000)
+ if (error_code & DSISR_GUARDED)
/* Guarded storage error. */
goto bad_area;
#endif /* CONFIG_8xx */
* processors use the same I/D cache coherency mechanism
* as embedded.
*/
- if (error_code & DSISR_PROTFAULT)
+ if (error_code & (DSISR_PROTFAULT | DSISR_GUARDED))
goto bad_area;
#endif /* CONFIG_PPC_STD_MMU */
bad_area_nosemaphore:
/* User mode accesses cause a SIGSEGV */
if (user_mode(regs)) {
+
+#ifdef CONFIG_PAX_PAGEEXEC
+ if (mm->pax_flags & MF_PAX_PAGEEXEC) {
+#ifdef CONFIG_PPC_STD_MMU
+ if (is_exec && (error_code & (DSISR_PROTFAULT | DSISR_GUARDED))) {
+#else
+ if (is_exec && regs->nip == address) {
+#endif
+ switch (pax_handle_fetch_fault(regs)) {
+ }
+
+ pax_report_fault(regs, (void *)regs->nip, (void *)regs->gpr[PT_R1]);
+ do_group_exit(SIGKILL);
+ }
+ }
+#endif
+
_exception(SIGSEGV, regs, code, address);
goto bail;
}
return sysctl_legacy_va_layout;
}
-static unsigned long mmap_rnd(void)
+static unsigned long mmap_rnd(struct mm_struct *mm)
{
unsigned long rnd = 0;
+#ifdef CONFIG_PAX_RANDMMAP
+ if (!(mm->pax_flags & MF_PAX_RANDMMAP))
+#endif
+
if (current->flags & PF_RANDOMIZE) {
/* 8MB for 32bit, 1GB for 64bit */
if (is_32bit_task())
return rnd << PAGE_SHIFT;
}
-static inline unsigned long mmap_base(void)
+static inline unsigned long mmap_base(struct mm_struct *mm)
{
unsigned long gap = rlimit(RLIMIT_STACK);
else if (gap > MAX_GAP)
gap = MAX_GAP;
- return PAGE_ALIGN(TASK_SIZE - gap - mmap_rnd());
+ return PAGE_ALIGN(TASK_SIZE - gap - mmap_rnd(mm));
}
/*
*/
if (mmap_is_legacy()) {
mm->mmap_base = TASK_UNMAPPED_BASE;
+
+#ifdef CONFIG_PAX_RANDMMAP
+ if (mm->pax_flags & MF_PAX_RANDMMAP)
+ mm->mmap_base += mm->delta_mmap;
+#endif
+
mm->get_unmapped_area = arch_get_unmapped_area;
} else {
- mm->mmap_base = mmap_base();
+ mm->mmap_base = mmap_base(mm);
+
+#ifdef CONFIG_PAX_RANDMMAP
+ if (mm->pax_flags & MF_PAX_RANDMMAP)
+ mm->mmap_base -= mm->delta_mmap + mm->delta_stack;
+#endif
+
mm->get_unmapped_area = arch_get_unmapped_area_topdown;
}
}
if ((mm->task_size - len) < addr)
return 0;
vma = find_vma(mm, addr);
- return (!vma || (addr + len) <= vma->vm_start);
+ return check_heap_stack_gap(vma, addr, len, 0);
}
static int slice_low_has_vma(struct mm_struct *mm, unsigned long slice)
info.align_offset = 0;
addr = TASK_UNMAPPED_BASE;
+
+#ifdef CONFIG_PAX_RANDMMAP
+ if (mm->pax_flags & MF_PAX_RANDMMAP)
+ addr += mm->delta_mmap;
+#endif
+
while (addr < TASK_SIZE) {
info.low_limit = addr;
if (!slice_scan_available(addr, available, 1, &addr))
if (fixed && addr > (mm->task_size - len))
return -ENOMEM;
+#ifdef CONFIG_PAX_RANDMMAP
+ if (!fixed && (mm->pax_flags & MF_PAX_RANDMMAP))
+ addr = 0;
+#endif
+
/* If hint, make sure it matches our alignment restrictions */
if (!fixed && addr) {
addr = _ALIGN_UP(addr, 1ul << pshift);
}
static struct pci_ops scc_pciex_pci_ops = {
- scc_pciex_read_config,
- scc_pciex_write_config,
+ .read = scc_pciex_read_config,
+ .write = scc_pciex_write_config,
};
static void pciex_clear_intr_all(unsigned int __iomem *base)
return VM_FAULT_NOPAGE;
}
-static int spufs_mem_mmap_access(struct vm_area_struct *vma,
+static ssize_t spufs_mem_mmap_access(struct vm_area_struct *vma,
unsigned long address,
- void *buf, int len, int write)
+ void *buf, size_t len, int write)
{
struct spu_context *ctx = vma->vm_file->private_data;
unsigned long offset = address - vma->vm_start;
#define atomic64_dec_and_test(_v) (atomic64_sub_return(1, _v) == 0)
#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0)
+#define atomic64_read_unchecked(v) atomic64_read(v)
+#define atomic64_set_unchecked(v, i) atomic64_set((v), (i))
+#define atomic64_add_unchecked(a, v) atomic64_add((a), (v))
+#define atomic64_add_return_unchecked(a, v) atomic64_add_return((a), (v))
+#define atomic64_sub_unchecked(a, v) atomic64_sub((a), (v))
+#define atomic64_inc_unchecked(v) atomic64_inc(v)
+#define atomic64_inc_return_unchecked(v) atomic64_inc_return(v)
+#define atomic64_dec_unchecked(v) atomic64_dec(v)
+#define atomic64_cmpxchg_unchecked(v, o, n) atomic64_cmpxchg((v), (o), (n))
+
#endif /* __ARCH_S390_ATOMIC__ */
do { \
compiletime_assert_atomic_type(*p); \
barrier(); \
- ACCESS_ONCE(*p) = (v); \
+ ACCESS_ONCE_RW(*p) = (v); \
} while (0)
#define smp_load_acquire(p) \
#ifndef __ARCH_S390_CACHE_H
#define __ARCH_S390_CACHE_H
-#define L1_CACHE_BYTES 256
+#include <linux/const.h>
+
#define L1_CACHE_SHIFT 8
+#define L1_CACHE_BYTES (_AC(1,UL) << L1_CACHE_SHIFT)
#define NET_SKB_PAD 32
#define __read_mostly __attribute__((__section__(".data..read_mostly")))
the loader. We need to make sure that it is out of the way of the program
that it will "exec", and that there is sufficient room for the brk. */
-extern unsigned long randomize_et_dyn(unsigned long base);
-#define ELF_ET_DYN_BASE (randomize_et_dyn(STACK_TOP / 3 * 2))
+#define ELF_ET_DYN_BASE (STACK_TOP / 3 * 2)
+
+#ifdef CONFIG_PAX_ASLR
+#define PAX_ELF_ET_DYN_BASE (test_thread_flag(TIF_31BIT) ? 0x10000UL : 0x80000000UL)
+
+#define PAX_DELTA_MMAP_LEN (test_thread_flag(TIF_31BIT) ? 15 : 26)
+#define PAX_DELTA_STACK_LEN (test_thread_flag(TIF_31BIT) ? 15 : 26)
+#endif
/* This yields a mask that user programs can use to figure out what
instruction set this CPU supports. */
#define ARCH_HAS_SETUP_ADDITIONAL_PAGES 1
int arch_setup_additional_pages(struct linux_binprm *, int);
-extern unsigned long arch_randomize_brk(struct mm_struct *mm);
-#define arch_randomize_brk arch_randomize_brk
-
void *fill_cpu_elf_notes(void *ptr, struct save_area *sa, __vector128 *vxrs);
#endif
#ifndef __ASM_EXEC_H
#define __ASM_EXEC_H
-extern unsigned long arch_align_stack(unsigned long sp);
+#define arch_align_stack(x) ((x) & ~0xfUL)
#endif /* __ASM_EXEC_H */
__range_ok((unsigned long)(addr), (size)); \
})
+#define access_ok_noprefault(type, addr, size) access_ok((type), (addr), (size))
#define access_ok(type, addr, size) __access_ok(addr, size)
/*
copy_to_user(void __user *to, const void *from, unsigned long n)
{
might_fault();
+
+ if ((long)n < 0)
+ return n;
+
return __copy_to_user(to, from, n);
}
static inline unsigned long __must_check
copy_from_user(void *to, const void __user *from, unsigned long n)
{
- unsigned int sz = __compiletime_object_size(to);
+ size_t sz = __compiletime_object_size(to);
might_fault();
- if (unlikely(sz != -1 && sz < n)) {
+
+ if ((long)n < 0)
+ return n;
+
+ if (unlikely(sz != (size_t)-1 && sz < n)) {
copy_from_user_overflow();
return n;
}
/* Increase core size by size of got & plt and set start
offsets for got and plt. */
- me->core_size = ALIGN(me->core_size, 4);
- me->arch.got_offset = me->core_size;
- me->core_size += me->arch.got_size;
- me->arch.plt_offset = me->core_size;
- me->core_size += me->arch.plt_size;
+ me->core_size_rw = ALIGN(me->core_size_rw, 4);
+ me->arch.got_offset = me->core_size_rw;
+ me->core_size_rw += me->arch.got_size;
+ me->arch.plt_offset = me->core_size_rx;
+ me->core_size_rx += me->arch.plt_size;
return 0;
}
if (info->got_initialized == 0) {
Elf_Addr *gotent;
- gotent = me->module_core + me->arch.got_offset +
+ gotent = me->module_core_rw + me->arch.got_offset +
info->got_offset;
*gotent = val;
info->got_initialized = 1;
rc = apply_rela_bits(loc, val, 0, 64, 0);
else if (r_type == R_390_GOTENT ||
r_type == R_390_GOTPLTENT) {
- val += (Elf_Addr) me->module_core - loc;
+ val += (Elf_Addr) me->module_core_rw - loc;
rc = apply_rela_bits(loc, val, 1, 32, 1);
}
break;
case R_390_PLTOFF64: /* 16 bit offset from GOT to PLT. */
if (info->plt_initialized == 0) {
unsigned int *ip;
- ip = me->module_core + me->arch.plt_offset +
+ ip = me->module_core_rx + me->arch.plt_offset +
info->plt_offset;
#ifndef CONFIG_64BIT
ip[0] = 0x0d105810; /* basr 1,0; l 1,6(1); br 1 */
val - loc + 0xffffUL < 0x1ffffeUL) ||
(r_type == R_390_PLT32DBL &&
val - loc + 0xffffffffULL < 0x1fffffffeULL)))
- val = (Elf_Addr) me->module_core +
+ val = (Elf_Addr) me->module_core_rx +
me->arch.plt_offset +
info->plt_offset;
val += rela->r_addend - loc;
case R_390_GOTOFF32: /* 32 bit offset to GOT. */
case R_390_GOTOFF64: /* 64 bit offset to GOT. */
val = val + rela->r_addend -
- ((Elf_Addr) me->module_core + me->arch.got_offset);
+ ((Elf_Addr) me->module_core_rw + me->arch.got_offset);
if (r_type == R_390_GOTOFF16)
rc = apply_rela_bits(loc, val, 0, 16, 0);
else if (r_type == R_390_GOTOFF32)
break;
case R_390_GOTPC: /* 32 bit PC relative offset to GOT. */
case R_390_GOTPCDBL: /* 32 bit PC rel. off. to GOT shifted by 1. */
- val = (Elf_Addr) me->module_core + me->arch.got_offset +
+ val = (Elf_Addr) me->module_core_rw + me->arch.got_offset +
rela->r_addend - loc;
if (r_type == R_390_GOTPC)
rc = apply_rela_bits(loc, val, 1, 32, 0);
}
return 0;
}
-
-unsigned long arch_align_stack(unsigned long sp)
-{
- if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
- sp -= get_random_int() & ~PAGE_MASK;
- return sp & ~0xf;
-}
-
-static inline unsigned long brk_rnd(void)
-{
- /* 8MB for 32bit, 1GB for 64bit */
- if (is_32bit_task())
- return (get_random_int() & 0x7ffUL) << PAGE_SHIFT;
- else
- return (get_random_int() & 0x3ffffUL) << PAGE_SHIFT;
-}
-
-unsigned long arch_randomize_brk(struct mm_struct *mm)
-{
- unsigned long ret;
-
- ret = PAGE_ALIGN(mm->brk + brk_rnd());
- return (ret > mm->brk) ? ret : mm->brk;
-}
-
-unsigned long randomize_et_dyn(unsigned long base)
-{
- unsigned long ret;
-
- if (!(current->flags & PF_RANDOMIZE))
- return base;
- ret = PAGE_ALIGN(base + brk_rnd());
- return (ret > base) ? ret : base;
-}
*/
if (mmap_is_legacy()) {
mm->mmap_base = mmap_base_legacy();
+
+#ifdef CONFIG_PAX_RANDMMAP
+ if (mm->pax_flags & MF_PAX_RANDMMAP)
+ mm->mmap_base += mm->delta_mmap;
+#endif
+
mm->get_unmapped_area = arch_get_unmapped_area;
} else {
mm->mmap_base = mmap_base();
+
+#ifdef CONFIG_PAX_RANDMMAP
+ if (mm->pax_flags & MF_PAX_RANDMMAP)
+ mm->mmap_base -= mm->delta_mmap + mm->delta_stack;
+#endif
+
mm->get_unmapped_area = arch_get_unmapped_area_topdown;
}
}
*/
if (mmap_is_legacy()) {
mm->mmap_base = mmap_base_legacy();
+
+#ifdef CONFIG_PAX_RANDMMAP
+ if (mm->pax_flags & MF_PAX_RANDMMAP)
+ mm->mmap_base += mm->delta_mmap;
+#endif
+
mm->get_unmapped_area = s390_get_unmapped_area;
} else {
mm->mmap_base = mmap_base();
+
+#ifdef CONFIG_PAX_RANDMMAP
+ if (mm->pax_flags & MF_PAX_RANDMMAP)
+ mm->mmap_base -= mm->delta_mmap + mm->delta_stack;
+#endif
+
mm->get_unmapped_area = s390_get_unmapped_area_topdown;
}
}
#ifndef _ASM_SCORE_CACHE_H
#define _ASM_SCORE_CACHE_H
+#include <linux/const.h>
+
#define L1_CACHE_SHIFT 4
-#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT)
+#define L1_CACHE_BYTES (_AC(1,UL) << L1_CACHE_SHIFT)
#endif /* _ASM_SCORE_CACHE_H */
#ifndef _ASM_SCORE_EXEC_H
#define _ASM_SCORE_EXEC_H
-extern unsigned long arch_align_stack(unsigned long sp);
+#define arch_align_stack(x) (x)
#endif /* _ASM_SCORE_EXEC_H */
return task_pt_regs(task)->cp0_epc;
}
-
-unsigned long arch_align_stack(unsigned long sp)
-{
- return sp;
-}
#define __ASM_SH_CACHE_H
#ifdef __KERNEL__
+#include <linux/const.h>
#include <linux/init.h>
#include <cpu/cache.h>
-#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT)
+#define L1_CACHE_BYTES (_AC(1,UL) << L1_CACHE_SHIFT)
#define __read_mostly __attribute__((__section__(".data..read_mostly")))
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
int do_colour_align;
+ unsigned long offset = gr_rand_threadstack_offset(mm, filp, flags);
struct vm_unmapped_area_info info;
if (flags & MAP_FIXED) {
if (filp || (flags & MAP_SHARED))
do_colour_align = 1;
+#ifdef CONFIG_PAX_RANDMMAP
+ if (!(mm->pax_flags & MF_PAX_RANDMMAP))
+#endif
+
if (addr) {
if (do_colour_align)
addr = COLOUR_ALIGN(addr, pgoff);
addr = PAGE_ALIGN(addr);
vma = find_vma(mm, addr);
- if (TASK_SIZE - len >= addr &&
- (!vma || addr + len <= vma->vm_start))
+ if (TASK_SIZE - len >= addr && check_heap_stack_gap(vma, addr, len, offset))
return addr;
}
info.flags = 0;
info.length = len;
- info.low_limit = TASK_UNMAPPED_BASE;
+ info.low_limit = mm->mmap_base;
info.high_limit = TASK_SIZE;
info.align_mask = do_colour_align ? (PAGE_MASK & shm_align_mask) : 0;
info.align_offset = pgoff << PAGE_SHIFT;
struct mm_struct *mm = current->mm;
unsigned long addr = addr0;
int do_colour_align;
+ unsigned long offset = gr_rand_threadstack_offset(mm, filp, flags);
struct vm_unmapped_area_info info;
if (flags & MAP_FIXED) {
if (filp || (flags & MAP_SHARED))
do_colour_align = 1;
+#ifdef CONFIG_PAX_RANDMMAP
+ if (!(mm->pax_flags & MF_PAX_RANDMMAP))
+#endif
+
/* requesting a specific address */
if (addr) {
if (do_colour_align)
addr = PAGE_ALIGN(addr);
vma = find_vma(mm, addr);
- if (TASK_SIZE - len >= addr &&
- (!vma || addr + len <= vma->vm_start))
+ if (TASK_SIZE - len >= addr && check_heap_stack_gap(vma, addr, len, offset))
return addr;
}
VM_BUG_ON(addr != -ENOMEM);
info.flags = 0;
info.low_limit = TASK_UNMAPPED_BASE;
+
+#ifdef CONFIG_PAX_RANDMMAP
+ if (mm->pax_flags & MF_PAX_RANDMMAP)
+ info.low_limit += mm->delta_mmap;
+#endif
+
info.high_limit = TASK_SIZE;
addr = vm_unmapped_area(&info);
}
#define ATOMIC64_INIT(i) { (i) }
#define atomic_read(v) ACCESS_ONCE((v)->counter)
+static inline int atomic_read_unchecked(const atomic_unchecked_t *v)
+{
+ return ACCESS_ONCE(v->counter);
+}
#define atomic64_read(v) ACCESS_ONCE((v)->counter)
+static inline long atomic64_read_unchecked(const atomic64_unchecked_t *v)
+{
+ return ACCESS_ONCE(v->counter);
+}
#define atomic_set(v, i) (((v)->counter) = i)
+static inline void atomic_set_unchecked(atomic_unchecked_t *v, int i)
+{
+ v->counter = i;
+}
#define atomic64_set(v, i) (((v)->counter) = i)
+static inline void atomic64_set_unchecked(atomic64_unchecked_t *v, long i)
+{
+ v->counter = i;
+}
+
+#define __ATOMIC_OP(op, suffix) \
+void atomic_##op##suffix(int, atomic##suffix##_t *); \
+void atomic64_##op##suffix(long, atomic64##suffix##_t *);
+
+#define ATOMIC_OP(op) __ATOMIC_OP(op, ) __ATOMIC_OP(op, _unchecked)
-#define ATOMIC_OP(op) \
-void atomic_##op(int, atomic_t *); \
-void atomic64_##op(long, atomic64_t *);
+#define __ATOMIC_OP_RETURN(op, suffix) \
+int atomic_##op##_return##suffix(int, atomic##suffix##_t *); \
+long atomic64_##op##_return##suffix(long, atomic64##suffix##_t *);
-#define ATOMIC_OP_RETURN(op) \
-int atomic_##op##_return(int, atomic_t *); \
-long atomic64_##op##_return(long, atomic64_t *);
+#define ATOMIC_OP_RETURN(op) __ATOMIC_OP_RETURN(op, ) __ATOMIC_OP_RETURN(op, _unchecked)
#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op)
#undef ATOMIC_OPS
#undef ATOMIC_OP_RETURN
+#undef __ATOMIC_OP_RETURN
#undef ATOMIC_OP
+#undef __ATOMIC_OP
#define atomic_dec_return(v) atomic_sub_return(1, v)
#define atomic64_dec_return(v) atomic64_sub_return(1, v)
#define atomic_inc_return(v) atomic_add_return(1, v)
+static inline int atomic_inc_return_unchecked(atomic_unchecked_t *v)
+{
+ return atomic_add_return_unchecked(1, v);
+}
#define atomic64_inc_return(v) atomic64_add_return(1, v)
+static inline long atomic64_inc_return_unchecked(atomic64_unchecked_t *v)
+{
+ return atomic64_add_return_unchecked(1, v);
+}
/*
* atomic_inc_and_test - increment and test
* other cases.
*/
#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0)
+static inline int atomic_inc_and_test_unchecked(atomic_unchecked_t *v)
+{
+ return atomic_inc_return_unchecked(v) == 0;
+}
#define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0)
#define atomic_sub_and_test(i, v) (atomic_sub_return(i, v) == 0)
#define atomic64_dec_and_test(v) (atomic64_sub_return(1, v) == 0)
#define atomic_inc(v) atomic_add(1, v)
+static inline void atomic_inc_unchecked(atomic_unchecked_t *v)
+{
+ atomic_add_unchecked(1, v);
+}
#define atomic64_inc(v) atomic64_add(1, v)
+static inline void atomic64_inc_unchecked(atomic64_unchecked_t *v)
+{
+ atomic64_add_unchecked(1, v);
+}
#define atomic_dec(v) atomic_sub(1, v)
+static inline void atomic_dec_unchecked(atomic_unchecked_t *v)
+{
+ atomic_sub_unchecked(1, v);
+}
#define atomic64_dec(v) atomic64_sub(1, v)
+static inline void atomic64_dec_unchecked(atomic64_unchecked_t *v)
+{
+ atomic64_sub_unchecked(1, v);
+}
#define atomic_add_negative(i, v) (atomic_add_return(i, v) < 0)
#define atomic64_add_negative(i, v) (atomic64_add_return(i, v) < 0)
#define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
+static inline int atomic_cmpxchg_unchecked(atomic_unchecked_t *v, int old, int new)
+{
+ return cmpxchg(&v->counter, old, new);
+}
#define atomic_xchg(v, new) (xchg(&((v)->counter), new))
+static inline int atomic_xchg_unchecked(atomic_unchecked_t *v, int new)
+{
+ return xchg(&v->counter, new);
+}
static inline int __atomic_add_unless(atomic_t *v, int a, int u)
{
- int c, old;
+ int c, old, new;
c = atomic_read(v);
for (;;) {
- if (unlikely(c == (u)))
+ if (unlikely(c == u))
break;
- old = atomic_cmpxchg((v), c, c + (a));
+
+ asm volatile("addcc %2, %0, %0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "tvs %%icc, 6\n"
+#endif
+
+ : "=r" (new)
+ : "0" (c), "ir" (a)
+ : "cc");
+
+ old = atomic_cmpxchg(v, c, new);
if (likely(old == c))
break;
c = old;
#define atomic64_cmpxchg(v, o, n) \
((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n)))
#define atomic64_xchg(v, new) (xchg(&((v)->counter), new))
+static inline long atomic64_xchg_unchecked(atomic64_unchecked_t *v, long new)
+{
+ return xchg(&v->counter, new);
+}
static inline long atomic64_add_unless(atomic64_t *v, long a, long u)
{
- long c, old;
+ long c, old, new;
c = atomic64_read(v);
for (;;) {
- if (unlikely(c == (u)))
+ if (unlikely(c == u))
break;
- old = atomic64_cmpxchg((v), c, c + (a));
+
+ asm volatile("addcc %2, %0, %0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "tvs %%xcc, 6\n"
+#endif
+
+ : "=r" (new)
+ : "0" (c), "ir" (a)
+ : "cc");
+
+ old = atomic64_cmpxchg(v, c, new);
if (likely(old == c))
break;
c = old;
}
- return c != (u);
+ return c != u;
}
#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0)
do { \
compiletime_assert_atomic_type(*p); \
barrier(); \
- ACCESS_ONCE(*p) = (v); \
+ ACCESS_ONCE_RW(*p) = (v); \
} while (0)
#define smp_load_acquire(p) \
#ifndef _SPARC_CACHE_H
#define _SPARC_CACHE_H
+#include <linux/const.h>
+
#define ARCH_SLAB_MINALIGN __alignof__(unsigned long long)
#define L1_CACHE_SHIFT 5
-#define L1_CACHE_BYTES 32
+#define L1_CACHE_BYTES (_AC(1,UL) << L1_CACHE_SHIFT)
#ifdef CONFIG_SPARC32
#define SMP_CACHE_BYTES_SHIFT 5
#define ELF_ET_DYN_BASE (TASK_UNMAPPED_BASE)
+#ifdef CONFIG_PAX_ASLR
+#define PAX_ELF_ET_DYN_BASE 0x10000UL
+
+#define PAX_DELTA_MMAP_LEN 16
+#define PAX_DELTA_STACK_LEN 16
+#endif
+
/* This yields a mask that user programs can use to figure out what
instruction set this cpu supports. This can NOT be done in userspace
on Sparc. */
#define ELF_ET_DYN_BASE 0x0000010000000000UL
#define COMPAT_ELF_ET_DYN_BASE 0x0000000070000000UL
+#ifdef CONFIG_PAX_ASLR
+#define PAX_ELF_ET_DYN_BASE (test_thread_flag(TIF_32BIT) ? 0x10000UL : 0x100000UL)
+
+#define PAX_DELTA_MMAP_LEN (test_thread_flag(TIF_32BIT) ? 14 : 28)
+#define PAX_DELTA_STACK_LEN (test_thread_flag(TIF_32BIT) ? 15 : 29)
+#endif
+
extern unsigned long sparc64_elf_hwcap;
#define ELF_HWCAP sparc64_elf_hwcap
}
#define pgd_populate(MM, PGD, PMD) pgd_set(PGD, PMD)
+#define pgd_populate_kernel(MM, PGD, PMD) pgd_populate((MM), (PGD), (PMD))
static inline pmd_t *pmd_alloc_one(struct mm_struct *mm,
unsigned long address)
}
#define pgd_populate(MM, PGD, PUD) __pgd_populate(PGD, PUD)
+#define pgd_populate_kernel(MM, PGD, PMD) pgd_populate((MM), (PGD), (PMD))
static inline pgd_t *pgd_alloc(struct mm_struct *mm)
{
}
#define pud_populate(MM, PUD, PMD) __pud_populate(PUD, PMD)
+#define pud_populate_kernel(MM, PUD, PMD) pud_populate((MM), (PUD), (PMD))
static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
{
#else
#include <asm/pgtable_32.h>
#endif
+
+#define ktla_ktva(addr) (addr)
+#define ktva_ktla(addr) (addr)
+
#endif
#define PAGE_SHARED SRMMU_PAGE_SHARED
#define PAGE_COPY SRMMU_PAGE_COPY
#define PAGE_READONLY SRMMU_PAGE_RDONLY
+#define PAGE_SHARED_NOEXEC SRMMU_PAGE_SHARED_NOEXEC
+#define PAGE_COPY_NOEXEC SRMMU_PAGE_COPY_NOEXEC
+#define PAGE_READONLY_NOEXEC SRMMU_PAGE_RDONLY_NOEXEC
#define PAGE_KERNEL SRMMU_PAGE_KERNEL
/* Top-level page directory - dummy used by init-mm.
/* xwr */
#define __P000 PAGE_NONE
-#define __P001 PAGE_READONLY
-#define __P010 PAGE_COPY
-#define __P011 PAGE_COPY
+#define __P001 PAGE_READONLY_NOEXEC
+#define __P010 PAGE_COPY_NOEXEC
+#define __P011 PAGE_COPY_NOEXEC
#define __P100 PAGE_READONLY
#define __P101 PAGE_READONLY
#define __P110 PAGE_COPY
#define __P111 PAGE_COPY
#define __S000 PAGE_NONE
-#define __S001 PAGE_READONLY
-#define __S010 PAGE_SHARED
-#define __S011 PAGE_SHARED
+#define __S001 PAGE_READONLY_NOEXEC
+#define __S010 PAGE_SHARED_NOEXEC
+#define __S011 PAGE_SHARED_NOEXEC
#define __S100 PAGE_READONLY
#define __S101 PAGE_READONLY
#define __S110 PAGE_SHARED
SRMMU_EXEC | SRMMU_REF)
#define SRMMU_PAGE_RDONLY __pgprot(SRMMU_VALID | SRMMU_CACHE | \
SRMMU_EXEC | SRMMU_REF)
+
+#define SRMMU_PAGE_SHARED_NOEXEC __pgprot(SRMMU_VALID | SRMMU_CACHE | SRMMU_WRITE | SRMMU_REF)
+#define SRMMU_PAGE_COPY_NOEXEC __pgprot(SRMMU_VALID | SRMMU_CACHE | SRMMU_REF)
+#define SRMMU_PAGE_RDONLY_NOEXEC __pgprot(SRMMU_VALID | SRMMU_CACHE | SRMMU_REF)
+
#define SRMMU_PAGE_KERNEL __pgprot(SRMMU_VALID | SRMMU_CACHE | SRMMU_PRIV | \
SRMMU_DIRTY | SRMMU_REF)
void handle_ld_nf(u32 insn, struct pt_regs *regs);
/* init_64.c */
-extern atomic_t dcpage_flushes;
-extern atomic_t dcpage_flushes_xcall;
+extern atomic_unchecked_t dcpage_flushes;
+extern atomic_unchecked_t dcpage_flushes_xcall;
extern int sysctl_tsb_ratio;
#endif
/* Multi-reader locks, these are much saner than the 32-bit Sparc ones... */
-static void inline arch_read_lock(arch_rwlock_t *lock)
+static inline void arch_read_lock(arch_rwlock_t *lock)
{
unsigned long tmp1, tmp2;
__asm__ __volatile__ (
"1: ldsw [%2], %0\n"
" brlz,pn %0, 2f\n"
-"4: add %0, 1, %1\n"
+"4: addcc %0, 1, %1\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+" tvs %%icc, 6\n"
+#endif
+
" cas [%2], %0, %1\n"
" cmp %0, %1\n"
" bne,pn %%icc, 1b\n"
" .previous"
: "=&r" (tmp1), "=&r" (tmp2)
: "r" (lock)
- : "memory");
+ : "memory", "cc");
}
-static int inline arch_read_trylock(arch_rwlock_t *lock)
+static inline int arch_read_trylock(arch_rwlock_t *lock)
{
int tmp1, tmp2;
"1: ldsw [%2], %0\n"
" brlz,a,pn %0, 2f\n"
" mov 0, %0\n"
-" add %0, 1, %1\n"
+" addcc %0, 1, %1\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+" tvs %%icc, 6\n"
+#endif
+
" cas [%2], %0, %1\n"
" cmp %0, %1\n"
" bne,pn %%icc, 1b\n"
return tmp1;
}
-static void inline arch_read_unlock(arch_rwlock_t *lock)
+static inline void arch_read_unlock(arch_rwlock_t *lock)
{
unsigned long tmp1, tmp2;
__asm__ __volatile__(
"1: lduw [%2], %0\n"
-" sub %0, 1, %1\n"
+" subcc %0, 1, %1\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+" tvs %%icc, 6\n"
+#endif
+
" cas [%2], %0, %1\n"
" cmp %0, %1\n"
" bne,pn %%xcc, 1b\n"
: "memory");
}
-static void inline arch_write_lock(arch_rwlock_t *lock)
+static inline void arch_write_lock(arch_rwlock_t *lock)
{
unsigned long mask, tmp1, tmp2;
: "memory");
}
-static void inline arch_write_unlock(arch_rwlock_t *lock)
+static inline void arch_write_unlock(arch_rwlock_t *lock)
{
__asm__ __volatile__(
" stw %%g0, [%0]"
: "memory");
}
-static int inline arch_write_trylock(arch_rwlock_t *lock)
+static inline int arch_write_trylock(arch_rwlock_t *lock)
{
unsigned long mask, tmp1, tmp2, result;
unsigned long w_saved;
struct restart_block restart_block;
+
+ unsigned long lowest_stack;
};
/*
struct pt_regs *kern_una_regs;
unsigned int kern_una_insn;
+ unsigned long lowest_stack;
+
unsigned long fpregs[(7 * 256) / sizeof(unsigned long)]
__attribute__ ((aligned(64)));
};
#define TIF_NEED_RESCHED 3 /* rescheduling necessary */
/* flag bit 4 is available */
#define TIF_UNALIGNED 5 /* allowed to do unaligned accesses */
-/* flag bit 6 is available */
+#define TIF_GRSEC_SETXID 6 /* update credentials on syscall entry/exit */
#define TIF_32BIT 7 /* 32-bit binary */
#define TIF_NOHZ 8 /* in adaptive nohz mode */
#define TIF_SECCOMP 9 /* secure computing */
#define TIF_SYSCALL_AUDIT 10 /* syscall auditing active */
#define TIF_SYSCALL_TRACEPOINT 11 /* syscall tracepoint instrumentation */
+
/* NOTE: Thread flags >= 12 should be ones we have no interest
* in using in assembly, else we can't use the mask as
* an immediate value in instructions such as andcc.
#define _TIF_SYSCALL_AUDIT (1<<TIF_SYSCALL_AUDIT)
#define _TIF_SYSCALL_TRACEPOINT (1<<TIF_SYSCALL_TRACEPOINT)
#define _TIF_POLLING_NRFLAG (1<<TIF_POLLING_NRFLAG)
+#define _TIF_GRSEC_SETXID (1<<TIF_GRSEC_SETXID)
#define _TIF_USER_WORK_MASK ((0xff << TI_FLAG_WSAVED_SHIFT) | \
_TIF_DO_NOTIFY_RESUME_MASK | \
_TIF_NEED_RESCHED)
#define _TIF_DO_NOTIFY_RESUME_MASK (_TIF_NOTIFY_RESUME | _TIF_SIGPENDING)
+#define _TIF_WORK_SYSCALL \
+ (_TIF_SYSCALL_TRACE | _TIF_SECCOMP | _TIF_SYSCALL_AUDIT | \
+ _TIF_SYSCALL_TRACEPOINT | _TIF_NOHZ | _TIF_GRSEC_SETXID)
+
#define is_32bit_task() (test_thread_flag(TIF_32BIT))
/*
#ifndef ___ASM_SPARC_UACCESS_H
#define ___ASM_SPARC_UACCESS_H
+
#if defined(__sparc__) && defined(__arch64__)
#include <asm/uaccess_64.h>
#else
static inline unsigned long copy_to_user(void __user *to, const void *from, unsigned long n)
{
- if (n && __access_ok((unsigned long) to, n))
+ if ((long)n < 0)
+ return n;
+
+ if (n && __access_ok((unsigned long) to, n)) {
+ if (!__builtin_constant_p(n))
+ check_object_size(from, n, true);
return __copy_user(to, (__force void __user *) from, n);
- else
+ } else
return n;
}
static inline unsigned long __copy_to_user(void __user *to, const void *from, unsigned long n)
{
+ if ((long)n < 0)
+ return n;
+
+ if (!__builtin_constant_p(n))
+ check_object_size(from, n, true);
+
return __copy_user(to, (__force void __user *) from, n);
}
static inline unsigned long copy_from_user(void *to, const void __user *from, unsigned long n)
{
- if (n && __access_ok((unsigned long) from, n))
+ if ((long)n < 0)
+ return n;
+
+ if (n && __access_ok((unsigned long) from, n)) {
+ if (!__builtin_constant_p(n))
+ check_object_size(to, n, false);
return __copy_user((__force void __user *) to, from, n);
- else
+ } else
return n;
}
static inline unsigned long __copy_from_user(void *to, const void __user *from, unsigned long n)
{
+ if ((long)n < 0)
+ return n;
+
return __copy_user((__force void __user *) to, from, n);
}
#include <linux/compiler.h>
#include <linux/string.h>
#include <linux/thread_info.h>
+#include <linux/kernel.h>
#include <asm/asi.h>
#include <asm/spitfire.h>
#include <asm-generic/uaccess-unaligned.h>
static inline unsigned long __must_check
copy_from_user(void *to, const void __user *from, unsigned long size)
{
- unsigned long ret = ___copy_from_user(to, from, size);
+ unsigned long ret;
+ if ((long)size < 0 || size > INT_MAX)
+ return size;
+
+ if (!__builtin_constant_p(size))
+ check_object_size(to, size, false);
+
+ ret = ___copy_from_user(to, from, size);
if (unlikely(ret))
ret = copy_from_user_fixup(to, from, size);
static inline unsigned long __must_check
copy_to_user(void __user *to, const void *from, unsigned long size)
{
- unsigned long ret = ___copy_to_user(to, from, size);
+ unsigned long ret;
+
+ if ((long)size < 0 || size > INT_MAX)
+ return size;
+
+ if (!__builtin_constant_p(size))
+ check_object_size(from, size, true);
+ ret = ___copy_to_user(to, from, size);
if (unlikely(ret))
ret = copy_to_user_fixup(to, from, size);
return ret;
#
asflags-y := -ansi
-ccflags-y := -Werror
+#ccflags-y := -Werror
extra-y := head_$(BITS).o
printk("PSR: %08lx PC: %08lx NPC: %08lx Y: %08lx %s\n",
r->psr, r->pc, r->npc, r->y, print_tainted());
- printk("PC: <%pS>\n", (void *) r->pc);
+ printk("PC: <%pA>\n", (void *) r->pc);
printk("%%G: %08lx %08lx %08lx %08lx %08lx %08lx %08lx %08lx\n",
r->u_regs[0], r->u_regs[1], r->u_regs[2], r->u_regs[3],
r->u_regs[4], r->u_regs[5], r->u_regs[6], r->u_regs[7]);
printk("%%O: %08lx %08lx %08lx %08lx %08lx %08lx %08lx %08lx\n",
r->u_regs[8], r->u_regs[9], r->u_regs[10], r->u_regs[11],
r->u_regs[12], r->u_regs[13], r->u_regs[14], r->u_regs[15]);
- printk("RPC: <%pS>\n", (void *) r->u_regs[15]);
+ printk("RPC: <%pA>\n", (void *) r->u_regs[15]);
printk("%%L: %08lx %08lx %08lx %08lx %08lx %08lx %08lx %08lx\n",
rw->locals[0], rw->locals[1], rw->locals[2], rw->locals[3],
rw = (struct reg_window32 *) fp;
pc = rw->ins[7];
printk("[%08lx : ", pc);
- printk("%pS ] ", (void *) pc);
+ printk("%pA ] ", (void *) pc);
fp = rw->ins[6];
} while (++count < 16);
printk("\n");
printk("i4: %016lx i5: %016lx i6: %016lx i7: %016lx\n",
rwk->ins[4], rwk->ins[5], rwk->ins[6], rwk->ins[7]);
if (regs->tstate & TSTATE_PRIV)
- printk("I7: <%pS>\n", (void *) rwk->ins[7]);
+ printk("I7: <%pA>\n", (void *) rwk->ins[7]);
}
void show_regs(struct pt_regs *regs)
printk("TSTATE: %016lx TPC: %016lx TNPC: %016lx Y: %08x %s\n", regs->tstate,
regs->tpc, regs->tnpc, regs->y, print_tainted());
- printk("TPC: <%pS>\n", (void *) regs->tpc);
+ printk("TPC: <%pA>\n", (void *) regs->tpc);
printk("g0: %016lx g1: %016lx g2: %016lx g3: %016lx\n",
regs->u_regs[0], regs->u_regs[1], regs->u_regs[2],
regs->u_regs[3]);
printk("o4: %016lx o5: %016lx sp: %016lx ret_pc: %016lx\n",
regs->u_regs[12], regs->u_regs[13], regs->u_regs[14],
regs->u_regs[15]);
- printk("RPC: <%pS>\n", (void *) regs->u_regs[15]);
+ printk("RPC: <%pA>\n", (void *) regs->u_regs[15]);
show_regwindow(regs);
show_stack(current, (unsigned long *) regs->u_regs[UREG_FP]);
}
((tp && tp->task) ? tp->task->pid : -1));
if (gp->tstate & TSTATE_PRIV) {
- printk(" TPC[%pS] O7[%pS] I7[%pS] RPC[%pS]\n",
+ printk(" TPC[%pA] O7[%pA] I7[%pA] RPC[%pA]\n",
(void *) gp->tpc,
(void *) gp->o7,
(void *) gp->i7,
unsigned int prom_early_allocated __initdata;
-static struct of_pdt_ops prom_sparc_ops __initdata = {
+static struct of_pdt_ops prom_sparc_ops __initconst = {
.nextprop = prom_common_nextprop,
.getproplen = prom_getproplen,
.getproperty = prom_getproperty,
return ret;
}
+#ifdef CONFIG_GRKERNSEC_SETXID
+extern void gr_delayed_cred_worker(void);
+#endif
+
asmlinkage int syscall_trace_enter(struct pt_regs *regs)
{
int ret = 0;
if (test_thread_flag(TIF_NOHZ))
user_exit();
+#ifdef CONFIG_GRKERNSEC_SETXID
+ if (unlikely(test_and_clear_thread_flag(TIF_GRSEC_SETXID)))
+ gr_delayed_cred_worker();
+#endif
+
if (test_thread_flag(TIF_SYSCALL_TRACE))
ret = tracehook_report_syscall_entry(regs);
if (test_thread_flag(TIF_NOHZ))
user_exit();
+#ifdef CONFIG_GRKERNSEC_SETXID
+ if (unlikely(test_and_clear_thread_flag(TIF_GRSEC_SETXID)))
+ gr_delayed_cred_worker();
+#endif
+
audit_syscall_exit(regs);
if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT)))
return;
#ifdef CONFIG_DEBUG_DCFLUSH
- atomic_inc(&dcpage_flushes);
+ atomic_inc_unchecked(&dcpage_flushes);
#endif
this_cpu = get_cpu();
xcall_deliver(data0, __pa(pg_addr),
(u64) pg_addr, cpumask_of(cpu));
#ifdef CONFIG_DEBUG_DCFLUSH
- atomic_inc(&dcpage_flushes_xcall);
+ atomic_inc_unchecked(&dcpage_flushes_xcall);
#endif
}
}
preempt_disable();
#ifdef CONFIG_DEBUG_DCFLUSH
- atomic_inc(&dcpage_flushes);
+ atomic_inc_unchecked(&dcpage_flushes);
#endif
data0 = 0;
pg_addr = page_address(page);
xcall_deliver(data0, __pa(pg_addr),
(u64) pg_addr, cpu_online_mask);
#ifdef CONFIG_DEBUG_DCFLUSH
- atomic_inc(&dcpage_flushes_xcall);
+ atomic_inc_unchecked(&dcpage_flushes_xcall);
#endif
}
__local_flush_dcache_page(page);
if (len > TASK_SIZE - PAGE_SIZE)
return -ENOMEM;
if (!addr)
- addr = TASK_UNMAPPED_BASE;
+ addr = current->mm->mmap_base;
info.flags = 0;
info.length = len;
struct vm_area_struct * vma;
unsigned long task_size = TASK_SIZE;
int do_color_align;
+ unsigned long offset = gr_rand_threadstack_offset(mm, filp, flags);
struct vm_unmapped_area_info info;
if (flags & MAP_FIXED) {
/* We do not accept a shared mapping if it would violate
* cache aliasing constraints.
*/
- if ((flags & MAP_SHARED) &&
+ if ((filp || (flags & MAP_SHARED)) &&
((addr - (pgoff << PAGE_SHIFT)) & (SHMLBA - 1)))
return -EINVAL;
return addr;
if (filp || (flags & MAP_SHARED))
do_color_align = 1;
+#ifdef CONFIG_PAX_RANDMMAP
+ if (!(mm->pax_flags & MF_PAX_RANDMMAP))
+#endif
+
if (addr) {
if (do_color_align)
addr = COLOR_ALIGN(addr, pgoff);
addr = PAGE_ALIGN(addr);
vma = find_vma(mm, addr);
- if (task_size - len >= addr &&
- (!vma || addr + len <= vma->vm_start))
+ if (task_size - len >= addr && check_heap_stack_gap(vma, addr, len, offset))
return addr;
}
info.flags = 0;
info.length = len;
- info.low_limit = TASK_UNMAPPED_BASE;
+ info.low_limit = mm->mmap_base;
info.high_limit = min(task_size, VA_EXCLUDE_START);
info.align_mask = do_color_align ? (PAGE_MASK & (SHMLBA - 1)) : 0;
info.align_offset = pgoff << PAGE_SHIFT;
+ info.threadstack_offset = offset;
addr = vm_unmapped_area(&info);
if ((addr & ~PAGE_MASK) && task_size > VA_EXCLUDE_END) {
VM_BUG_ON(addr != -ENOMEM);
info.low_limit = VA_EXCLUDE_END;
+
+#ifdef CONFIG_PAX_RANDMMAP
+ if (mm->pax_flags & MF_PAX_RANDMMAP)
+ info.low_limit += mm->delta_mmap;
+#endif
+
info.high_limit = task_size;
addr = vm_unmapped_area(&info);
}
unsigned long task_size = STACK_TOP32;
unsigned long addr = addr0;
int do_color_align;
+ unsigned long offset = gr_rand_threadstack_offset(mm, filp, flags);
struct vm_unmapped_area_info info;
/* This should only ever run for 32-bit processes. */
/* We do not accept a shared mapping if it would violate
* cache aliasing constraints.
*/
- if ((flags & MAP_SHARED) &&
+ if ((filp || (flags & MAP_SHARED)) &&
((addr - (pgoff << PAGE_SHIFT)) & (SHMLBA - 1)))
return -EINVAL;
return addr;
if (filp || (flags & MAP_SHARED))
do_color_align = 1;
+#ifdef CONFIG_PAX_RANDMMAP
+ if (!(mm->pax_flags & MF_PAX_RANDMMAP))
+#endif
+
/* requesting a specific address */
if (addr) {
if (do_color_align)
addr = PAGE_ALIGN(addr);
vma = find_vma(mm, addr);
- if (task_size - len >= addr &&
- (!vma || addr + len <= vma->vm_start))
+ if (task_size - len >= addr && check_heap_stack_gap(vma, addr, len, offset))
return addr;
}
info.high_limit = mm->mmap_base;
info.align_mask = do_color_align ? (PAGE_MASK & (SHMLBA - 1)) : 0;
info.align_offset = pgoff << PAGE_SHIFT;
+ info.threadstack_offset = offset;
addr = vm_unmapped_area(&info);
/*
VM_BUG_ON(addr != -ENOMEM);
info.flags = 0;
info.low_limit = TASK_UNMAPPED_BASE;
+
+#ifdef CONFIG_PAX_RANDMMAP
+ if (mm->pax_flags & MF_PAX_RANDMMAP)
+ info.low_limit += mm->delta_mmap;
+#endif
+
info.high_limit = STACK_TOP32;
addr = vm_unmapped_area(&info);
}
EXPORT_SYMBOL(get_fb_unmapped_area);
/* Essentially the same as PowerPC. */
-static unsigned long mmap_rnd(void)
+static unsigned long mmap_rnd(struct mm_struct *mm)
{
unsigned long rnd = 0UL;
+#ifdef CONFIG_PAX_RANDMMAP
+ if (!(mm->pax_flags & MF_PAX_RANDMMAP))
+#endif
+
if (current->flags & PF_RANDOMIZE) {
unsigned long val = get_random_int();
if (test_thread_flag(TIF_32BIT))
void arch_pick_mmap_layout(struct mm_struct *mm)
{
- unsigned long random_factor = mmap_rnd();
+ unsigned long random_factor = mmap_rnd(mm);
unsigned long gap;
/*
gap == RLIM_INFINITY ||
sysctl_legacy_va_layout) {
mm->mmap_base = TASK_UNMAPPED_BASE + random_factor;
+
+#ifdef CONFIG_PAX_RANDMMAP
+ if (mm->pax_flags & MF_PAX_RANDMMAP)
+ mm->mmap_base += mm->delta_mmap;
+#endif
+
mm->get_unmapped_area = arch_get_unmapped_area;
} else {
/* We know it's 32-bit */
gap = (task_size / 6 * 5);
mm->mmap_base = PAGE_ALIGN(task_size - gap - random_factor);
+
+#ifdef CONFIG_PAX_RANDMMAP
+ if (mm->pax_flags & MF_PAX_RANDMMAP)
+ mm->mmap_base -= mm->delta_mmap + mm->delta_stack;
+#endif
+
mm->get_unmapped_area = arch_get_unmapped_area_topdown;
}
}
#endif
.align 32
1: ldx [%g6 + TI_FLAGS], %l5
- andcc %l5, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT|_TIF_NOHZ), %g0
+ andcc %l5, _TIF_WORK_SYSCALL, %g0
be,pt %icc, rtrap
nop
call syscall_trace_leave
srl %i3, 0, %o3 ! IEU0
srl %i2, 0, %o2 ! IEU0 Group
- andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT|_TIF_NOHZ), %g0
+ andcc %l0, _TIF_WORK_SYSCALL, %g0
bne,pn %icc, linux_syscall_trace32 ! CTI
mov %i0, %l5 ! IEU1
5: call %l7 ! CTI Group brk forced
mov %i3, %o3 ! IEU1
mov %i4, %o4 ! IEU0 Group
- andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT|_TIF_NOHZ), %g0
+ andcc %l0, _TIF_WORK_SYSCALL, %g0
bne,pn %icc, linux_syscall_trace ! CTI Group
mov %i0, %l5 ! IEU0
2: call %l7 ! CTI Group brk forced
cmp %o0, -ERESTART_RESTARTBLOCK
bgeu,pn %xcc, 1f
- andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT|_TIF_NOHZ), %g0
+ andcc %l0, _TIF_WORK_SYSCALL, %g0
ldx [%sp + PTREGS_OFF + PT_V9_TNPC], %l1 ! pc = npc
2:
#define __SAVE __asm__ __volatile__("save %sp, -0x40, %sp\n\t")
#define __RESTORE __asm__ __volatile__("restore %g0, %g0, %g0\n\t")
+extern void gr_handle_kernel_exploit(void);
+
void __noreturn die_if_kernel(char *str, struct pt_regs *regs)
{
static int die_counter;
count++ < 30 &&
(((unsigned long) rw) >= PAGE_OFFSET) &&
!(((unsigned long) rw) & 0x7)) {
- printk("Caller[%08lx]: %pS\n", rw->ins[7],
+ printk("Caller[%08lx]: %pA\n", rw->ins[7],
(void *) rw->ins[7]);
rw = (struct reg_window32 *)rw->ins[6];
}
}
printk("Instruction DUMP:");
instruction_dump ((unsigned long *) regs->pc);
- if(regs->psr & PSR_PS)
+ if(regs->psr & PSR_PS) {
+ gr_handle_kernel_exploit();
do_exit(SIGKILL);
+ }
do_exit(SIGSEGV);
}
i + 1,
p->trapstack[i].tstate, p->trapstack[i].tpc,
p->trapstack[i].tnpc, p->trapstack[i].tt);
- printk("TRAPLOG: TPC<%pS>\n", (void *) p->trapstack[i].tpc);
+ printk("TRAPLOG: TPC<%pA>\n", (void *) p->trapstack[i].tpc);
}
}
lvl -= 0x100;
if (regs->tstate & TSTATE_PRIV) {
+
+#ifdef CONFIG_PAX_REFCOUNT
+ if (lvl == 6)
+ pax_report_refcount_overflow(regs);
+#endif
+
sprintf(buffer, "Kernel bad sw trap %lx", lvl);
die_if_kernel(buffer, regs);
}
void bad_trap_tl1(struct pt_regs *regs, long lvl)
{
char buffer[32];
-
+
if (notify_die(DIE_TRAP_TL1, "bad trap tl1", regs,
0, lvl, SIGTRAP) == NOTIFY_STOP)
return;
+#ifdef CONFIG_PAX_REFCOUNT
+ if (lvl == 6)
+ pax_report_refcount_overflow(regs);
+#endif
+
dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
sprintf (buffer, "Bad trap %lx at tl>0", lvl);
regs->tpc, regs->tnpc, regs->u_regs[UREG_I7], regs->tstate);
printk("%s" "ERROR(%d): ",
(recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id());
- printk("TPC<%pS>\n", (void *) regs->tpc);
+ printk("TPC<%pA>\n", (void *) regs->tpc);
printk("%s" "ERROR(%d): M_SYND(%lx), E_SYND(%lx)%s%s\n",
(recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
(afsr & CHAFSR_M_SYNDROME) >> CHAFSR_M_SYNDROME_SHIFT,
smp_processor_id(),
(type & 0x1) ? 'I' : 'D',
regs->tpc);
- printk(KERN_EMERG "TPC<%pS>\n", (void *) regs->tpc);
+ printk(KERN_EMERG "TPC<%pA>\n", (void *) regs->tpc);
panic("Irrecoverable Cheetah+ parity error.");
}
smp_processor_id(),
(type & 0x1) ? 'I' : 'D',
regs->tpc);
- printk(KERN_WARNING "TPC<%pS>\n", (void *) regs->tpc);
+ printk(KERN_WARNING "TPC<%pA>\n", (void *) regs->tpc);
}
struct sun4v_error_entry {
/*0x38*/u64 reserved_5;
};
-static atomic_t sun4v_resum_oflow_cnt = ATOMIC_INIT(0);
-static atomic_t sun4v_nonresum_oflow_cnt = ATOMIC_INIT(0);
+static atomic_unchecked_t sun4v_resum_oflow_cnt = ATOMIC_INIT(0);
+static atomic_unchecked_t sun4v_nonresum_oflow_cnt = ATOMIC_INIT(0);
static const char *sun4v_err_type_to_str(u8 type)
{
}
static void sun4v_log_error(struct pt_regs *regs, struct sun4v_error_entry *ent,
- int cpu, const char *pfx, atomic_t *ocnt)
+ int cpu, const char *pfx, atomic_unchecked_t *ocnt)
{
u64 *raw_ptr = (u64 *) ent;
u32 attrs;
show_regs(regs);
- if ((cnt = atomic_read(ocnt)) != 0) {
- atomic_set(ocnt, 0);
+ if ((cnt = atomic_read_unchecked(ocnt)) != 0) {
+ atomic_set_unchecked(ocnt, 0);
wmb();
printk("%s: Queue overflowed %d times.\n",
pfx, cnt);
*/
void sun4v_resum_overflow(struct pt_regs *regs)
{
- atomic_inc(&sun4v_resum_oflow_cnt);
+ atomic_inc_unchecked(&sun4v_resum_oflow_cnt);
}
/* We run with %pil set to PIL_NORMAL_MAX and PSTATE_IE enabled in %pstate.
/* XXX Actually even this can make not that much sense. Perhaps
* XXX we should just pull the plug and panic directly from here?
*/
- atomic_inc(&sun4v_nonresum_oflow_cnt);
+ atomic_inc_unchecked(&sun4v_nonresum_oflow_cnt);
}
static void sun4v_tlb_error(struct pt_regs *regs)
printk(KERN_EMERG "SUN4V-ITLB: Error at TPC[%lx], tl %d\n",
regs->tpc, tl);
- printk(KERN_EMERG "SUN4V-ITLB: TPC<%pS>\n", (void *) regs->tpc);
+ printk(KERN_EMERG "SUN4V-ITLB: TPC<%pA>\n", (void *) regs->tpc);
printk(KERN_EMERG "SUN4V-ITLB: O7[%lx]\n", regs->u_regs[UREG_I7]);
- printk(KERN_EMERG "SUN4V-ITLB: O7<%pS>\n",
+ printk(KERN_EMERG "SUN4V-ITLB: O7<%pA>\n",
(void *) regs->u_regs[UREG_I7]);
printk(KERN_EMERG "SUN4V-ITLB: vaddr[%lx] ctx[%lx] "
"pte[%lx] error[%lx]\n",
printk(KERN_EMERG "SUN4V-DTLB: Error at TPC[%lx], tl %d\n",
regs->tpc, tl);
- printk(KERN_EMERG "SUN4V-DTLB: TPC<%pS>\n", (void *) regs->tpc);
+ printk(KERN_EMERG "SUN4V-DTLB: TPC<%pA>\n", (void *) regs->tpc);
printk(KERN_EMERG "SUN4V-DTLB: O7[%lx]\n", regs->u_regs[UREG_I7]);
- printk(KERN_EMERG "SUN4V-DTLB: O7<%pS>\n",
+ printk(KERN_EMERG "SUN4V-DTLB: O7<%pA>\n",
(void *) regs->u_regs[UREG_I7]);
printk(KERN_EMERG "SUN4V-DTLB: vaddr[%lx] ctx[%lx] "
"pte[%lx] error[%lx]\n",
fp = (unsigned long)sf->fp + STACK_BIAS;
}
- printk(" [%016lx] %pS\n", pc, (void *) pc);
+ printk(" [%016lx] %pA\n", pc, (void *) pc);
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
if ((pc + 8UL) == (unsigned long) &return_to_handler) {
int index = tsk->curr_ret_stack;
if (tsk->ret_stack && index >= graph) {
pc = tsk->ret_stack[index - graph].ret;
- printk(" [%016lx] %pS\n", pc, (void *) pc);
+ printk(" [%016lx] %pA\n", pc, (void *) pc);
graph++;
}
}
return (struct reg_window *) (fp + STACK_BIAS);
}
+extern void gr_handle_kernel_exploit(void);
+
void __noreturn die_if_kernel(char *str, struct pt_regs *regs)
{
static int die_counter;
while (rw &&
count++ < 30 &&
kstack_valid(tp, (unsigned long) rw)) {
- printk("Caller[%016lx]: %pS\n", rw->ins[7],
+ printk("Caller[%016lx]: %pA\n", rw->ins[7],
(void *) rw->ins[7]);
rw = kernel_stack_up(rw);
}
user_instruction_dump ((unsigned int __user *) regs->tpc);
}
- if (regs->tstate & TSTATE_PRIV)
+ if (regs->tstate & TSTATE_PRIV) {
+ gr_handle_kernel_exploit();
do_exit(SIGKILL);
+ }
do_exit(SIGSEGV);
}
EXPORT_SYMBOL(die_if_kernel);
static DEFINE_RATELIMIT_STATE(ratelimit, 5 * HZ, 5);
if (__ratelimit(&ratelimit)) {
- printk("Kernel unaligned access at TPC[%lx] %pS\n",
+ printk("Kernel unaligned access at TPC[%lx] %pA\n",
regs->tpc, (void *) regs->tpc);
}
}
#
asflags-y := -ansi -DST_DIV0=0x02
-ccflags-y := -Werror
+#ccflags-y := -Werror
lib-$(CONFIG_SPARC32) += ashrdi3.o
lib-$(CONFIG_SPARC32) += memcpy.o memset.o
* a value and does the barriers.
*/
-#define ATOMIC_OP(op) \
-ENTRY(atomic_##op) /* %o0 = increment, %o1 = atomic_ptr */ \
+#ifdef CONFIG_PAX_REFCOUNT
+#define __REFCOUNT_OP(op) op##cc
+#define __OVERFLOW_IOP tvs %icc, 6;
+#define __OVERFLOW_XOP tvs %xcc, 6;
+#else
+#define __REFCOUNT_OP(op) op
+#define __OVERFLOW_IOP
+#define __OVERFLOW_XOP
+#endif
+
+#define __ATOMIC_OP(op, suffix, asm_op, post_op) \
+ENTRY(atomic_##op##suffix) /* %o0 = increment, %o1 = atomic_ptr */ \
BACKOFF_SETUP(%o2); \
1: lduw [%o1], %g1; \
- op %g1, %o0, %g7; \
+ asm_op %g1, %o0, %g7; \
+ post_op \
cas [%o1], %g1, %g7; \
cmp %g1, %g7; \
bne,pn %icc, BACKOFF_LABEL(2f, 1b); \
2: BACKOFF_SPIN(%o2, %o3, 1b); \
ENDPROC(atomic_##op); \
-#define ATOMIC_OP_RETURN(op) \
-ENTRY(atomic_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */ \
+#define ATOMIC_OP(op) __ATOMIC_OP(op, , op, ) \
+ __ATOMIC_OP(op, _unchecked, __REFCOUNT_OP(op), __OVERFLOW_IOP)
+
+#define __ATOMIC_OP_RETURN(op, suffix, asm_op, post_op) \
+ENTRY(atomic_##op##_return##suffix) /* %o0 = increment, %o1 = atomic_ptr */\
BACKOFF_SETUP(%o2); \
1: lduw [%o1], %g1; \
- op %g1, %o0, %g7; \
+ asm_op %g1, %o0, %g7; \
+ post_op \
cas [%o1], %g1, %g7; \
cmp %g1, %g7; \
bne,pn %icc, BACKOFF_LABEL(2f, 1b); \
2: BACKOFF_SPIN(%o2, %o3, 1b); \
ENDPROC(atomic_##op##_return);
+#define ATOMIC_OP_RETURN(op) __ATOMIC_OP_RETURN(op, , op, ) \
+ __ATOMIC_OP_RETURN(op, _unchecked, __REFCOUNT_OP(op), __OVERFLOW_IOP)
+
#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op)
ATOMIC_OPS(add)
#undef ATOMIC_OPS
#undef ATOMIC_OP_RETURN
+#undef __ATOMIC_OP_RETURN
#undef ATOMIC_OP
+#undef __ATOMIC_OP
-#define ATOMIC64_OP(op) \
-ENTRY(atomic64_##op) /* %o0 = increment, %o1 = atomic_ptr */ \
+#define __ATOMIC64_OP(op, suffix, asm_op, post_op) \
+ENTRY(atomic64_##op##suffix) /* %o0 = increment, %o1 = atomic_ptr */ \
BACKOFF_SETUP(%o2); \
1: ldx [%o1], %g1; \
- op %g1, %o0, %g7; \
+ asm_op %g1, %o0, %g7; \
+ post_op \
casx [%o1], %g1, %g7; \
cmp %g1, %g7; \
bne,pn %xcc, BACKOFF_LABEL(2f, 1b); \
2: BACKOFF_SPIN(%o2, %o3, 1b); \
ENDPROC(atomic64_##op); \
-#define ATOMIC64_OP_RETURN(op) \
-ENTRY(atomic64_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */ \
+#define ATOMIC64_OP(op) __ATOMIC64_OP(op, , op, ) \
+ __ATOMIC64_OP(op, _unchecked, __REFCOUNT_OP(op), __OVERFLOW_XOP)
+
+#define __ATOMIC64_OP_RETURN(op, suffix, asm_op, post_op) \
+ENTRY(atomic64_##op##_return##suffix) /* %o0 = increment, %o1 = atomic_ptr */\
BACKOFF_SETUP(%o2); \
1: ldx [%o1], %g1; \
- op %g1, %o0, %g7; \
+ asm_op %g1, %o0, %g7; \
+ post_op \
casx [%o1], %g1, %g7; \
cmp %g1, %g7; \
bne,pn %xcc, BACKOFF_LABEL(2f, 1b); \
2: BACKOFF_SPIN(%o2, %o3, 1b); \
ENDPROC(atomic64_##op##_return);
+#define ATOMIC64_OP_RETURN(op) __ATOMIC64_OP_RETURN(op, , op, ) \
+i __ATOMIC64_OP_RETURN(op, _unchecked, __REFCOUNT_OP(op), __OVERFLOW_XOP)
+
#define ATOMIC64_OPS(op) ATOMIC64_OP(op) ATOMIC64_OP_RETURN(op)
ATOMIC64_OPS(add)
#undef ATOMIC64_OPS
#undef ATOMIC64_OP_RETURN
+#undef __ATOMIC64_OP_RETURN
#undef ATOMIC64_OP
+#undef __ATOMIC64_OP
+#undef __OVERFLOW_XOP
+#undef __OVERFLOW_IOP
+#undef __REFCOUNT_OP
ENTRY(atomic64_dec_if_positive) /* %o0 = atomic_ptr */
BACKOFF_SETUP(%o2)
/* Atomic counter implementation. */
#define ATOMIC_OP(op) \
EXPORT_SYMBOL(atomic_##op); \
-EXPORT_SYMBOL(atomic64_##op);
+EXPORT_SYMBOL(atomic_##op##_unchecked); \
+EXPORT_SYMBOL(atomic64_##op); \
+EXPORT_SYMBOL(atomic64_##op##_unchecked);
#define ATOMIC_OP_RETURN(op) \
EXPORT_SYMBOL(atomic_##op##_return); \
#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op)
ATOMIC_OPS(add)
+EXPORT_SYMBOL(atomic_add_ret_unchecked);
+EXPORT_SYMBOL(atomic64_add_ret_unchecked);
ATOMIC_OPS(sub)
#undef ATOMIC_OPS
#
asflags-y := -ansi
-ccflags-y := -Werror
+#ccflags-y := -Werror
obj-$(CONFIG_SPARC64) += ultra.o tlb.o tsb.o gup.o
obj-y += fault_$(BITS).o
#include <linux/perf_event.h>
#include <linux/interrupt.h>
#include <linux/kdebug.h>
+#include <linux/slab.h>
+#include <linux/pagemap.h>
+#include <linux/compiler.h>
#include <asm/page.h>
#include <asm/pgtable.h>
return safe_compute_effective_address(regs, insn);
}
+#ifdef CONFIG_PAX_PAGEEXEC
+#ifdef CONFIG_PAX_DLRESOLVE
+static void pax_emuplt_close(struct vm_area_struct *vma)
+{
+ vma->vm_mm->call_dl_resolve = 0UL;
+}
+
+static int pax_emuplt_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
+{
+ unsigned int *kaddr;
+
+ vmf->page = alloc_page(GFP_HIGHUSER);
+ if (!vmf->page)
+ return VM_FAULT_OOM;
+
+ kaddr = kmap(vmf->page);
+ memset(kaddr, 0, PAGE_SIZE);
+ kaddr[0] = 0x9DE3BFA8U; /* save */
+ flush_dcache_page(vmf->page);
+ kunmap(vmf->page);
+ return VM_FAULT_MAJOR;
+}
+
+static const struct vm_operations_struct pax_vm_ops = {
+ .close = pax_emuplt_close,
+ .fault = pax_emuplt_fault
+};
+
+static int pax_insert_vma(struct vm_area_struct *vma, unsigned long addr)
+{
+ int ret;
+
+ INIT_LIST_HEAD(&vma->anon_vma_chain);
+ vma->vm_mm = current->mm;
+ vma->vm_start = addr;
+ vma->vm_end = addr + PAGE_SIZE;
+ vma->vm_flags = VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYEXEC;
+ vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
+ vma->vm_ops = &pax_vm_ops;
+
+ ret = insert_vm_struct(current->mm, vma);
+ if (ret)
+ return ret;
+
+ ++current->mm->total_vm;
+ return 0;
+}
+#endif
+
+/*
+ * PaX: decide what to do with offenders (regs->pc = fault address)
+ *
+ * returns 1 when task should be killed
+ * 2 when patched PLT trampoline was detected
+ * 3 when unpatched PLT trampoline was detected
+ */
+static int pax_handle_fetch_fault(struct pt_regs *regs)
+{
+
+#ifdef CONFIG_PAX_EMUPLT
+ int err;
+
+ do { /* PaX: patched PLT emulation #1 */
+ unsigned int sethi1, sethi2, jmpl;
+
+ err = get_user(sethi1, (unsigned int *)regs->pc);
+ err |= get_user(sethi2, (unsigned int *)(regs->pc+4));
+ err |= get_user(jmpl, (unsigned int *)(regs->pc+8));
+
+ if (err)
+ break;
+
+ if ((sethi1 & 0xFFC00000U) == 0x03000000U &&
+ (sethi2 & 0xFFC00000U) == 0x03000000U &&
+ (jmpl & 0xFFFFE000U) == 0x81C06000U)
+ {
+ unsigned int addr;
+
+ regs->u_regs[UREG_G1] = (sethi2 & 0x003FFFFFU) << 10;
+ addr = regs->u_regs[UREG_G1];
+ addr += (((jmpl | 0xFFFFE000U) ^ 0x00001000U) + 0x00001000U);
+ regs->pc = addr;
+ regs->npc = addr+4;
+ return 2;
+ }
+ } while (0);
+
+ do { /* PaX: patched PLT emulation #2 */
+ unsigned int ba;
+
+ err = get_user(ba, (unsigned int *)regs->pc);
+
+ if (err)
+ break;
+
+ if ((ba & 0xFFC00000U) == 0x30800000U || (ba & 0xFFF80000U) == 0x30480000U) {
+ unsigned int addr;
+
+ if ((ba & 0xFFC00000U) == 0x30800000U)
+ addr = regs->pc + ((((ba | 0xFFC00000U) ^ 0x00200000U) + 0x00200000U) << 2);
+ else
+ addr = regs->pc + ((((ba | 0xFFF80000U) ^ 0x00040000U) + 0x00040000U) << 2);
+ regs->pc = addr;
+ regs->npc = addr+4;
+ return 2;
+ }
+ } while (0);
+
+ do { /* PaX: patched PLT emulation #3 */
+ unsigned int sethi, bajmpl, nop;
+
+ err = get_user(sethi, (unsigned int *)regs->pc);
+ err |= get_user(bajmpl, (unsigned int *)(regs->pc+4));
+ err |= get_user(nop, (unsigned int *)(regs->pc+8));
+
+ if (err)
+ break;
+
+ if ((sethi & 0xFFC00000U) == 0x03000000U &&
+ ((bajmpl & 0xFFFFE000U) == 0x81C06000U || (bajmpl & 0xFFF80000U) == 0x30480000U) &&
+ nop == 0x01000000U)
+ {
+ unsigned int addr;
+
+ addr = (sethi & 0x003FFFFFU) << 10;
+ regs->u_regs[UREG_G1] = addr;
+ if ((bajmpl & 0xFFFFE000U) == 0x81C06000U)
+ addr += (((bajmpl | 0xFFFFE000U) ^ 0x00001000U) + 0x00001000U);
+ else
+ addr = regs->pc + ((((bajmpl | 0xFFF80000U) ^ 0x00040000U) + 0x00040000U) << 2);
+ regs->pc = addr;
+ regs->npc = addr+4;
+ return 2;
+ }
+ } while (0);
+
+ do { /* PaX: unpatched PLT emulation step 1 */
+ unsigned int sethi, ba, nop;
+
+ err = get_user(sethi, (unsigned int *)regs->pc);
+ err |= get_user(ba, (unsigned int *)(regs->pc+4));
+ err |= get_user(nop, (unsigned int *)(regs->pc+8));
+
+ if (err)
+ break;
+
+ if ((sethi & 0xFFC00000U) == 0x03000000U &&
+ ((ba & 0xFFC00000U) == 0x30800000U || (ba & 0xFFF80000U) == 0x30680000U) &&
+ nop == 0x01000000U)
+ {
+ unsigned int addr, save, call;
+
+ if ((ba & 0xFFC00000U) == 0x30800000U)
+ addr = regs->pc + 4 + ((((ba | 0xFFC00000U) ^ 0x00200000U) + 0x00200000U) << 2);
+ else
+ addr = regs->pc + 4 + ((((ba | 0xFFF80000U) ^ 0x00040000U) + 0x00040000U) << 2);
+
+ err = get_user(save, (unsigned int *)addr);
+ err |= get_user(call, (unsigned int *)(addr+4));
+ err |= get_user(nop, (unsigned int *)(addr+8));
+ if (err)
+ break;
+
+#ifdef CONFIG_PAX_DLRESOLVE
+ if (save == 0x9DE3BFA8U &&
+ (call & 0xC0000000U) == 0x40000000U &&
+ nop == 0x01000000U)
+ {
+ struct vm_area_struct *vma;
+ unsigned long call_dl_resolve;
+
+ down_read(¤t->mm->mmap_sem);
+ call_dl_resolve = current->mm->call_dl_resolve;
+ up_read(¤t->mm->mmap_sem);
+ if (likely(call_dl_resolve))
+ goto emulate;
+
+ vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
+
+ down_write(¤t->mm->mmap_sem);
+ if (current->mm->call_dl_resolve) {
+ call_dl_resolve = current->mm->call_dl_resolve;
+ up_write(¤t->mm->mmap_sem);
+ if (vma)
+ kmem_cache_free(vm_area_cachep, vma);
+ goto emulate;
+ }
+
+ call_dl_resolve = get_unmapped_area(NULL, 0UL, PAGE_SIZE, 0UL, MAP_PRIVATE);
+ if (!vma || (call_dl_resolve & ~PAGE_MASK)) {
+ up_write(¤t->mm->mmap_sem);
+ if (vma)
+ kmem_cache_free(vm_area_cachep, vma);
+ return 1;
+ }
+
+ if (pax_insert_vma(vma, call_dl_resolve)) {
+ up_write(¤t->mm->mmap_sem);
+ kmem_cache_free(vm_area_cachep, vma);
+ return 1;
+ }
+
+ current->mm->call_dl_resolve = call_dl_resolve;
+ up_write(¤t->mm->mmap_sem);
+
+emulate:
+ regs->u_regs[UREG_G1] = (sethi & 0x003FFFFFU) << 10;
+ regs->pc = call_dl_resolve;
+ regs->npc = addr+4;
+ return 3;
+ }
+#endif
+
+ /* PaX: glibc 2.4+ generates sethi/jmpl instead of save/call */
+ if ((save & 0xFFC00000U) == 0x05000000U &&
+ (call & 0xFFFFE000U) == 0x85C0A000U &&
+ nop == 0x01000000U)
+ {
+ regs->u_regs[UREG_G1] = (sethi & 0x003FFFFFU) << 10;
+ regs->u_regs[UREG_G2] = addr + 4;
+ addr = (save & 0x003FFFFFU) << 10;
+ addr += (((call | 0xFFFFE000U) ^ 0x00001000U) + 0x00001000U);
+ regs->pc = addr;
+ regs->npc = addr+4;
+ return 3;
+ }
+ }
+ } while (0);
+
+ do { /* PaX: unpatched PLT emulation step 2 */
+ unsigned int save, call, nop;
+
+ err = get_user(save, (unsigned int *)(regs->pc-4));
+ err |= get_user(call, (unsigned int *)regs->pc);
+ err |= get_user(nop, (unsigned int *)(regs->pc+4));
+ if (err)
+ break;
+
+ if (save == 0x9DE3BFA8U &&
+ (call & 0xC0000000U) == 0x40000000U &&
+ nop == 0x01000000U)
+ {
+ unsigned int dl_resolve = regs->pc + ((((call | 0xC0000000U) ^ 0x20000000U) + 0x20000000U) << 2);
+
+ regs->u_regs[UREG_RETPC] = regs->pc;
+ regs->pc = dl_resolve;
+ regs->npc = dl_resolve+4;
+ return 3;
+ }
+ } while (0);
+#endif
+
+ return 1;
+}
+
+void pax_report_insns(struct pt_regs *regs, void *pc, void *sp)
+{
+ unsigned long i;
+
+ printk(KERN_ERR "PAX: bytes at PC: ");
+ for (i = 0; i < 8; i++) {
+ unsigned int c;
+ if (get_user(c, (unsigned int *)pc+i))
+ printk(KERN_CONT "???????? ");
+ else
+ printk(KERN_CONT "%08x ", c);
+ }
+ printk("\n");
+}
+#endif
+
static noinline void do_fault_siginfo(int code, int sig, struct pt_regs *regs,
int text_fault)
{
if (!(vma->vm_flags & VM_WRITE))
goto bad_area;
} else {
+
+#ifdef CONFIG_PAX_PAGEEXEC
+ if ((mm->pax_flags & MF_PAX_PAGEEXEC) && text_fault && !(vma->vm_flags & VM_EXEC)) {
+ up_read(&mm->mmap_sem);
+ switch (pax_handle_fetch_fault(regs)) {
+
+#ifdef CONFIG_PAX_EMUPLT
+ case 2:
+ case 3:
+ return;
+#endif
+
+ }
+ pax_report_fault(regs, (void *)regs->pc, (void *)regs->u_regs[UREG_FP]);
+ do_group_exit(SIGKILL);
+ }
+#endif
+
/* Allow reads even for write-only mappings */
if (!(vma->vm_flags & (VM_READ | VM_EXEC)))
goto bad_area;
#include <linux/kdebug.h>
#include <linux/percpu.h>
#include <linux/context_tracking.h>
+#include <linux/slab.h>
+#include <linux/pagemap.h>
+#include <linux/compiler.h>
#include <asm/page.h>
#include <asm/pgtable.h>
printk(KERN_CRIT "OOPS: Bogus kernel PC [%016lx] in fault handler\n",
regs->tpc);
printk(KERN_CRIT "OOPS: RPC [%016lx]\n", regs->u_regs[15]);
- printk("OOPS: RPC <%pS>\n", (void *) regs->u_regs[15]);
+ printk("OOPS: RPC <%pA>\n", (void *) regs->u_regs[15]);
printk(KERN_CRIT "OOPS: Fault was to vaddr[%lx]\n", vaddr);
dump_stack();
unhandled_fault(regs->tpc, current, regs);
show_regs(regs);
}
+#ifdef CONFIG_PAX_PAGEEXEC
+#ifdef CONFIG_PAX_DLRESOLVE
+static void pax_emuplt_close(struct vm_area_struct *vma)
+{
+ vma->vm_mm->call_dl_resolve = 0UL;
+}
+
+static int pax_emuplt_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
+{
+ unsigned int *kaddr;
+
+ vmf->page = alloc_page(GFP_HIGHUSER);
+ if (!vmf->page)
+ return VM_FAULT_OOM;
+
+ kaddr = kmap(vmf->page);
+ memset(kaddr, 0, PAGE_SIZE);
+ kaddr[0] = 0x9DE3BFA8U; /* save */
+ flush_dcache_page(vmf->page);
+ kunmap(vmf->page);
+ return VM_FAULT_MAJOR;
+}
+
+static const struct vm_operations_struct pax_vm_ops = {
+ .close = pax_emuplt_close,
+ .fault = pax_emuplt_fault
+};
+
+static int pax_insert_vma(struct vm_area_struct *vma, unsigned long addr)
+{
+ int ret;
+
+ INIT_LIST_HEAD(&vma->anon_vma_chain);
+ vma->vm_mm = current->mm;
+ vma->vm_start = addr;
+ vma->vm_end = addr + PAGE_SIZE;
+ vma->vm_flags = VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYEXEC;
+ vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
+ vma->vm_ops = &pax_vm_ops;
+
+ ret = insert_vm_struct(current->mm, vma);
+ if (ret)
+ return ret;
+
+ ++current->mm->total_vm;
+ return 0;
+}
+#endif
+
+/*
+ * PaX: decide what to do with offenders (regs->tpc = fault address)
+ *
+ * returns 1 when task should be killed
+ * 2 when patched PLT trampoline was detected
+ * 3 when unpatched PLT trampoline was detected
+ */
+static int pax_handle_fetch_fault(struct pt_regs *regs)
+{
+
+#ifdef CONFIG_PAX_EMUPLT
+ int err;
+
+ do { /* PaX: patched PLT emulation #1 */
+ unsigned int sethi1, sethi2, jmpl;
+
+ err = get_user(sethi1, (unsigned int *)regs->tpc);
+ err |= get_user(sethi2, (unsigned int *)(regs->tpc+4));
+ err |= get_user(jmpl, (unsigned int *)(regs->tpc+8));
+
+ if (err)
+ break;
+
+ if ((sethi1 & 0xFFC00000U) == 0x03000000U &&
+ (sethi2 & 0xFFC00000U) == 0x03000000U &&
+ (jmpl & 0xFFFFE000U) == 0x81C06000U)
+ {
+ unsigned long addr;
+
+ regs->u_regs[UREG_G1] = (sethi2 & 0x003FFFFFU) << 10;
+ addr = regs->u_regs[UREG_G1];
+ addr += (((jmpl | 0xFFFFFFFFFFFFE000UL) ^ 0x00001000UL) + 0x00001000UL);
+
+ if (test_thread_flag(TIF_32BIT))
+ addr &= 0xFFFFFFFFUL;
+
+ regs->tpc = addr;
+ regs->tnpc = addr+4;
+ return 2;
+ }
+ } while (0);
+
+ do { /* PaX: patched PLT emulation #2 */
+ unsigned int ba;
+
+ err = get_user(ba, (unsigned int *)regs->tpc);
+
+ if (err)
+ break;
+
+ if ((ba & 0xFFC00000U) == 0x30800000U || (ba & 0xFFF80000U) == 0x30480000U) {
+ unsigned long addr;
+
+ if ((ba & 0xFFC00000U) == 0x30800000U)
+ addr = regs->tpc + ((((ba | 0xFFFFFFFFFFC00000UL) ^ 0x00200000UL) + 0x00200000UL) << 2);
+ else
+ addr = regs->tpc + ((((ba | 0xFFFFFFFFFFF80000UL) ^ 0x00040000UL) + 0x00040000UL) << 2);
+
+ if (test_thread_flag(TIF_32BIT))
+ addr &= 0xFFFFFFFFUL;
+
+ regs->tpc = addr;
+ regs->tnpc = addr+4;
+ return 2;
+ }
+ } while (0);
+
+ do { /* PaX: patched PLT emulation #3 */
+ unsigned int sethi, bajmpl, nop;
+
+ err = get_user(sethi, (unsigned int *)regs->tpc);
+ err |= get_user(bajmpl, (unsigned int *)(regs->tpc+4));
+ err |= get_user(nop, (unsigned int *)(regs->tpc+8));
+
+ if (err)
+ break;
+
+ if ((sethi & 0xFFC00000U) == 0x03000000U &&
+ ((bajmpl & 0xFFFFE000U) == 0x81C06000U || (bajmpl & 0xFFF80000U) == 0x30480000U) &&
+ nop == 0x01000000U)
+ {
+ unsigned long addr;
+
+ addr = (sethi & 0x003FFFFFU) << 10;
+ regs->u_regs[UREG_G1] = addr;
+ if ((bajmpl & 0xFFFFE000U) == 0x81C06000U)
+ addr += (((bajmpl | 0xFFFFFFFFFFFFE000UL) ^ 0x00001000UL) + 0x00001000UL);
+ else
+ addr = regs->tpc + ((((bajmpl | 0xFFFFFFFFFFF80000UL) ^ 0x00040000UL) + 0x00040000UL) << 2);
+
+ if (test_thread_flag(TIF_32BIT))
+ addr &= 0xFFFFFFFFUL;
+
+ regs->tpc = addr;
+ regs->tnpc = addr+4;
+ return 2;
+ }
+ } while (0);
+
+ do { /* PaX: patched PLT emulation #4 */
+ unsigned int sethi, mov1, call, mov2;
+
+ err = get_user(sethi, (unsigned int *)regs->tpc);
+ err |= get_user(mov1, (unsigned int *)(regs->tpc+4));
+ err |= get_user(call, (unsigned int *)(regs->tpc+8));
+ err |= get_user(mov2, (unsigned int *)(regs->tpc+12));
+
+ if (err)
+ break;
+
+ if ((sethi & 0xFFC00000U) == 0x03000000U &&
+ mov1 == 0x8210000FU &&
+ (call & 0xC0000000U) == 0x40000000U &&
+ mov2 == 0x9E100001U)
+ {
+ unsigned long addr;
+
+ regs->u_regs[UREG_G1] = regs->u_regs[UREG_RETPC];
+ addr = regs->tpc + 4 + ((((call | 0xFFFFFFFFC0000000UL) ^ 0x20000000UL) + 0x20000000UL) << 2);
+
+ if (test_thread_flag(TIF_32BIT))
+ addr &= 0xFFFFFFFFUL;
+
+ regs->tpc = addr;
+ regs->tnpc = addr+4;
+ return 2;
+ }
+ } while (0);
+
+ do { /* PaX: patched PLT emulation #5 */
+ unsigned int sethi, sethi1, sethi2, or1, or2, sllx, jmpl, nop;
+
+ err = get_user(sethi, (unsigned int *)regs->tpc);
+ err |= get_user(sethi1, (unsigned int *)(regs->tpc+4));
+ err |= get_user(sethi2, (unsigned int *)(regs->tpc+8));
+ err |= get_user(or1, (unsigned int *)(regs->tpc+12));
+ err |= get_user(or2, (unsigned int *)(regs->tpc+16));
+ err |= get_user(sllx, (unsigned int *)(regs->tpc+20));
+ err |= get_user(jmpl, (unsigned int *)(regs->tpc+24));
+ err |= get_user(nop, (unsigned int *)(regs->tpc+28));
+
+ if (err)
+ break;
+
+ if ((sethi & 0xFFC00000U) == 0x03000000U &&
+ (sethi1 & 0xFFC00000U) == 0x03000000U &&
+ (sethi2 & 0xFFC00000U) == 0x0B000000U &&
+ (or1 & 0xFFFFE000U) == 0x82106000U &&
+ (or2 & 0xFFFFE000U) == 0x8A116000U &&
+ sllx == 0x83287020U &&
+ jmpl == 0x81C04005U &&
+ nop == 0x01000000U)
+ {
+ unsigned long addr;
+
+ regs->u_regs[UREG_G1] = ((sethi1 & 0x003FFFFFU) << 10) | (or1 & 0x000003FFU);
+ regs->u_regs[UREG_G1] <<= 32;
+ regs->u_regs[UREG_G5] = ((sethi2 & 0x003FFFFFU) << 10) | (or2 & 0x000003FFU);
+ addr = regs->u_regs[UREG_G1] + regs->u_regs[UREG_G5];
+ regs->tpc = addr;
+ regs->tnpc = addr+4;
+ return 2;
+ }
+ } while (0);
+
+ do { /* PaX: patched PLT emulation #6 */
+ unsigned int sethi, sethi1, sethi2, sllx, or, jmpl, nop;
+
+ err = get_user(sethi, (unsigned int *)regs->tpc);
+ err |= get_user(sethi1, (unsigned int *)(regs->tpc+4));
+ err |= get_user(sethi2, (unsigned int *)(regs->tpc+8));
+ err |= get_user(sllx, (unsigned int *)(regs->tpc+12));
+ err |= get_user(or, (unsigned int *)(regs->tpc+16));
+ err |= get_user(jmpl, (unsigned int *)(regs->tpc+20));
+ err |= get_user(nop, (unsigned int *)(regs->tpc+24));
+
+ if (err)
+ break;
+
+ if ((sethi & 0xFFC00000U) == 0x03000000U &&
+ (sethi1 & 0xFFC00000U) == 0x03000000U &&
+ (sethi2 & 0xFFC00000U) == 0x0B000000U &&
+ sllx == 0x83287020U &&
+ (or & 0xFFFFE000U) == 0x8A116000U &&
+ jmpl == 0x81C04005U &&
+ nop == 0x01000000U)
+ {
+ unsigned long addr;
+
+ regs->u_regs[UREG_G1] = (sethi1 & 0x003FFFFFU) << 10;
+ regs->u_regs[UREG_G1] <<= 32;
+ regs->u_regs[UREG_G5] = ((sethi2 & 0x003FFFFFU) << 10) | (or & 0x3FFU);
+ addr = regs->u_regs[UREG_G1] + regs->u_regs[UREG_G5];
+ regs->tpc = addr;
+ regs->tnpc = addr+4;
+ return 2;
+ }
+ } while (0);
+
+ do { /* PaX: unpatched PLT emulation step 1 */
+ unsigned int sethi, ba, nop;
+
+ err = get_user(sethi, (unsigned int *)regs->tpc);
+ err |= get_user(ba, (unsigned int *)(regs->tpc+4));
+ err |= get_user(nop, (unsigned int *)(regs->tpc+8));
+
+ if (err)
+ break;
+
+ if ((sethi & 0xFFC00000U) == 0x03000000U &&
+ ((ba & 0xFFC00000U) == 0x30800000U || (ba & 0xFFF80000U) == 0x30680000U) &&
+ nop == 0x01000000U)
+ {
+ unsigned long addr;
+ unsigned int save, call;
+ unsigned int sethi1, sethi2, or1, or2, sllx, add, jmpl;
+
+ if ((ba & 0xFFC00000U) == 0x30800000U)
+ addr = regs->tpc + 4 + ((((ba | 0xFFFFFFFFFFC00000UL) ^ 0x00200000UL) + 0x00200000UL) << 2);
+ else
+ addr = regs->tpc + 4 + ((((ba | 0xFFFFFFFFFFF80000UL) ^ 0x00040000UL) + 0x00040000UL) << 2);
+
+ if (test_thread_flag(TIF_32BIT))
+ addr &= 0xFFFFFFFFUL;
+
+ err = get_user(save, (unsigned int *)addr);
+ err |= get_user(call, (unsigned int *)(addr+4));
+ err |= get_user(nop, (unsigned int *)(addr+8));
+ if (err)
+ break;
+
+#ifdef CONFIG_PAX_DLRESOLVE
+ if (save == 0x9DE3BFA8U &&
+ (call & 0xC0000000U) == 0x40000000U &&
+ nop == 0x01000000U)
+ {
+ struct vm_area_struct *vma;
+ unsigned long call_dl_resolve;
+
+ down_read(¤t->mm->mmap_sem);
+ call_dl_resolve = current->mm->call_dl_resolve;
+ up_read(¤t->mm->mmap_sem);
+ if (likely(call_dl_resolve))
+ goto emulate;
+
+ vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
+
+ down_write(¤t->mm->mmap_sem);
+ if (current->mm->call_dl_resolve) {
+ call_dl_resolve = current->mm->call_dl_resolve;
+ up_write(¤t->mm->mmap_sem);
+ if (vma)
+ kmem_cache_free(vm_area_cachep, vma);
+ goto emulate;
+ }
+
+ call_dl_resolve = get_unmapped_area(NULL, 0UL, PAGE_SIZE, 0UL, MAP_PRIVATE);
+ if (!vma || (call_dl_resolve & ~PAGE_MASK)) {
+ up_write(¤t->mm->mmap_sem);
+ if (vma)
+ kmem_cache_free(vm_area_cachep, vma);
+ return 1;
+ }
+
+ if (pax_insert_vma(vma, call_dl_resolve)) {
+ up_write(¤t->mm->mmap_sem);
+ kmem_cache_free(vm_area_cachep, vma);
+ return 1;
+ }
+
+ current->mm->call_dl_resolve = call_dl_resolve;
+ up_write(¤t->mm->mmap_sem);
+
+emulate:
+ regs->u_regs[UREG_G1] = (sethi & 0x003FFFFFU) << 10;
+ regs->tpc = call_dl_resolve;
+ regs->tnpc = addr+4;
+ return 3;
+ }
+#endif
+
+ /* PaX: glibc 2.4+ generates sethi/jmpl instead of save/call */
+ if ((save & 0xFFC00000U) == 0x05000000U &&
+ (call & 0xFFFFE000U) == 0x85C0A000U &&
+ nop == 0x01000000U)
+ {
+ regs->u_regs[UREG_G1] = (sethi & 0x003FFFFFU) << 10;
+ regs->u_regs[UREG_G2] = addr + 4;
+ addr = (save & 0x003FFFFFU) << 10;
+ addr += (((call | 0xFFFFFFFFFFFFE000UL) ^ 0x00001000UL) + 0x00001000UL);
+
+ if (test_thread_flag(TIF_32BIT))
+ addr &= 0xFFFFFFFFUL;
+
+ regs->tpc = addr;
+ regs->tnpc = addr+4;
+ return 3;
+ }
+
+ /* PaX: 64-bit PLT stub */
+ err = get_user(sethi1, (unsigned int *)addr);
+ err |= get_user(sethi2, (unsigned int *)(addr+4));
+ err |= get_user(or1, (unsigned int *)(addr+8));
+ err |= get_user(or2, (unsigned int *)(addr+12));
+ err |= get_user(sllx, (unsigned int *)(addr+16));
+ err |= get_user(add, (unsigned int *)(addr+20));
+ err |= get_user(jmpl, (unsigned int *)(addr+24));
+ err |= get_user(nop, (unsigned int *)(addr+28));
+ if (err)
+ break;
+
+ if ((sethi1 & 0xFFC00000U) == 0x09000000U &&
+ (sethi2 & 0xFFC00000U) == 0x0B000000U &&
+ (or1 & 0xFFFFE000U) == 0x88112000U &&
+ (or2 & 0xFFFFE000U) == 0x8A116000U &&
+ sllx == 0x89293020U &&
+ add == 0x8A010005U &&
+ jmpl == 0x89C14000U &&
+ nop == 0x01000000U)
+ {
+ regs->u_regs[UREG_G1] = (sethi & 0x003FFFFFU) << 10;
+ regs->u_regs[UREG_G4] = ((sethi1 & 0x003FFFFFU) << 10) | (or1 & 0x000003FFU);
+ regs->u_regs[UREG_G4] <<= 32;
+ regs->u_regs[UREG_G5] = ((sethi2 & 0x003FFFFFU) << 10) | (or2 & 0x000003FFU);
+ regs->u_regs[UREG_G5] += regs->u_regs[UREG_G4];
+ regs->u_regs[UREG_G4] = addr + 24;
+ addr = regs->u_regs[UREG_G5];
+ regs->tpc = addr;
+ regs->tnpc = addr+4;
+ return 3;
+ }
+ }
+ } while (0);
+
+#ifdef CONFIG_PAX_DLRESOLVE
+ do { /* PaX: unpatched PLT emulation step 2 */
+ unsigned int save, call, nop;
+
+ err = get_user(save, (unsigned int *)(regs->tpc-4));
+ err |= get_user(call, (unsigned int *)regs->tpc);
+ err |= get_user(nop, (unsigned int *)(regs->tpc+4));
+ if (err)
+ break;
+
+ if (save == 0x9DE3BFA8U &&
+ (call & 0xC0000000U) == 0x40000000U &&
+ nop == 0x01000000U)
+ {
+ unsigned long dl_resolve = regs->tpc + ((((call | 0xFFFFFFFFC0000000UL) ^ 0x20000000UL) + 0x20000000UL) << 2);
+
+ if (test_thread_flag(TIF_32BIT))
+ dl_resolve &= 0xFFFFFFFFUL;
+
+ regs->u_regs[UREG_RETPC] = regs->tpc;
+ regs->tpc = dl_resolve;
+ regs->tnpc = dl_resolve+4;
+ return 3;
+ }
+ } while (0);
+#endif
+
+ do { /* PaX: patched PLT emulation #7, must be AFTER the unpatched PLT emulation */
+ unsigned int sethi, ba, nop;
+
+ err = get_user(sethi, (unsigned int *)regs->tpc);
+ err |= get_user(ba, (unsigned int *)(regs->tpc+4));
+ err |= get_user(nop, (unsigned int *)(regs->tpc+8));
+
+ if (err)
+ break;
+
+ if ((sethi & 0xFFC00000U) == 0x03000000U &&
+ (ba & 0xFFF00000U) == 0x30600000U &&
+ nop == 0x01000000U)
+ {
+ unsigned long addr;
+
+ addr = (sethi & 0x003FFFFFU) << 10;
+ regs->u_regs[UREG_G1] = addr;
+ addr = regs->tpc + ((((ba | 0xFFFFFFFFFFF80000UL) ^ 0x00040000UL) + 0x00040000UL) << 2);
+
+ if (test_thread_flag(TIF_32BIT))
+ addr &= 0xFFFFFFFFUL;
+
+ regs->tpc = addr;
+ regs->tnpc = addr+4;
+ return 2;
+ }
+ } while (0);
+
+#endif
+
+ return 1;
+}
+
+void pax_report_insns(struct pt_regs *regs, void *pc, void *sp)
+{
+ unsigned long i;
+
+ printk(KERN_ERR "PAX: bytes at PC: ");
+ for (i = 0; i < 8; i++) {
+ unsigned int c;
+ if (get_user(c, (unsigned int *)pc+i))
+ printk(KERN_CONT "???????? ");
+ else
+ printk(KERN_CONT "%08x ", c);
+ }
+ printk("\n");
+}
+#endif
+
asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs)
{
enum ctx_state prev_state = exception_enter();
if (!vma)
goto bad_area;
+#ifdef CONFIG_PAX_PAGEEXEC
+ /* PaX: detect ITLB misses on non-exec pages */
+ if ((mm->pax_flags & MF_PAX_PAGEEXEC) && vma->vm_start <= address &&
+ !(vma->vm_flags & VM_EXEC) && (fault_code & FAULT_CODE_ITLB))
+ {
+ if (address != regs->tpc)
+ goto good_area;
+
+ up_read(&mm->mmap_sem);
+ switch (pax_handle_fetch_fault(regs)) {
+
+#ifdef CONFIG_PAX_EMUPLT
+ case 2:
+ case 3:
+ return;
+#endif
+
+ }
+ pax_report_fault(regs, (void *)regs->tpc, (void *)(regs->u_regs[UREG_FP] + STACK_BIAS));
+ do_group_exit(SIGKILL);
+ }
+#endif
+
/* Pure DTLB misses do not tell us whether the fault causing
* load/store/atomic was a write or not, it only says that there
* was no match. So in such a case we (carefully) read the
unsigned long addr,
unsigned long len,
unsigned long pgoff,
- unsigned long flags)
+ unsigned long flags,
+ unsigned long offset)
{
+ struct mm_struct *mm = current->mm;
unsigned long task_size = TASK_SIZE;
struct vm_unmapped_area_info info;
info.flags = 0;
info.length = len;
- info.low_limit = TASK_UNMAPPED_BASE;
+ info.low_limit = mm->mmap_base;
info.high_limit = min(task_size, VA_EXCLUDE_START);
info.align_mask = PAGE_MASK & ~HPAGE_MASK;
info.align_offset = 0;
+ info.threadstack_offset = offset;
addr = vm_unmapped_area(&info);
if ((addr & ~PAGE_MASK) && task_size > VA_EXCLUDE_END) {
VM_BUG_ON(addr != -ENOMEM);
info.low_limit = VA_EXCLUDE_END;
+
+#ifdef CONFIG_PAX_RANDMMAP
+ if (mm->pax_flags & MF_PAX_RANDMMAP)
+ info.low_limit += mm->delta_mmap;
+#endif
+
info.high_limit = task_size;
addr = vm_unmapped_area(&info);
}
hugetlb_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
const unsigned long len,
const unsigned long pgoff,
- const unsigned long flags)
+ const unsigned long flags,
+ const unsigned long offset)
{
struct mm_struct *mm = current->mm;
unsigned long addr = addr0;
info.high_limit = mm->mmap_base;
info.align_mask = PAGE_MASK & ~HPAGE_MASK;
info.align_offset = 0;
+ info.threadstack_offset = offset;
addr = vm_unmapped_area(&info);
/*
VM_BUG_ON(addr != -ENOMEM);
info.flags = 0;
info.low_limit = TASK_UNMAPPED_BASE;
+
+#ifdef CONFIG_PAX_RANDMMAP
+ if (mm->pax_flags & MF_PAX_RANDMMAP)
+ info.low_limit += mm->delta_mmap;
+#endif
+
info.high_limit = STACK_TOP32;
addr = vm_unmapped_area(&info);
}
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
unsigned long task_size = TASK_SIZE;
+ unsigned long offset = gr_rand_threadstack_offset(mm, file, flags);
if (test_thread_flag(TIF_32BIT))
task_size = STACK_TOP32;
return addr;
}
+#ifdef CONFIG_PAX_RANDMMAP
+ if (!(mm->pax_flags & MF_PAX_RANDMMAP))
+#endif
+
if (addr) {
addr = ALIGN(addr, HPAGE_SIZE);
vma = find_vma(mm, addr);
- if (task_size - len >= addr &&
- (!vma || addr + len <= vma->vm_start))
+ if (task_size - len >= addr && check_heap_stack_gap(vma, addr, len, offset))
return addr;
}
if (mm->get_unmapped_area == arch_get_unmapped_area)
return hugetlb_get_unmapped_area_bottomup(file, addr, len,
- pgoff, flags);
+ pgoff, flags, offset);
else
return hugetlb_get_unmapped_area_topdown(file, addr, len,
- pgoff, flags);
+ pgoff, flags, offset);
}
pte_t *huge_pte_alloc(struct mm_struct *mm,
int num_kernel_image_mappings;
#ifdef CONFIG_DEBUG_DCFLUSH
-atomic_t dcpage_flushes = ATOMIC_INIT(0);
+atomic_unchecked_t dcpage_flushes = ATOMIC_INIT(0);
#ifdef CONFIG_SMP
-atomic_t dcpage_flushes_xcall = ATOMIC_INIT(0);
+atomic_unchecked_t dcpage_flushes_xcall = ATOMIC_INIT(0);
#endif
#endif
{
BUG_ON(tlb_type == hypervisor);
#ifdef CONFIG_DEBUG_DCFLUSH
- atomic_inc(&dcpage_flushes);
+ atomic_inc_unchecked(&dcpage_flushes);
#endif
#ifdef DCACHE_ALIASING_POSSIBLE
#ifdef CONFIG_DEBUG_DCFLUSH
seq_printf(m, "DCPageFlushes\t: %d\n",
- atomic_read(&dcpage_flushes));
+ atomic_read_unchecked(&dcpage_flushes));
#ifdef CONFIG_SMP
seq_printf(m, "DCPageFlushesXC\t: %d\n",
- atomic_read(&dcpage_flushes_xcall));
+ atomic_read_unchecked(&dcpage_flushes_xcall));
#endif /* CONFIG_SMP */
#endif /* CONFIG_DEBUG_DCFLUSH */
}
config KEXEC
bool "kexec system call"
+ depends on !GRKERNSEC_KMEM
---help---
kexec is a system call that implements the ability to shutdown your
current kernel, and to start another kernel. It is like a reboot
#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0)
+#define atomic64_read_unchecked(v) atomic64_read(v)
+#define atomic64_set_unchecked(v, i) atomic64_set((v), (i))
+#define atomic64_add_unchecked(a, v) atomic64_add((a), (v))
+#define atomic64_add_return_unchecked(a, v) atomic64_add_return((a), (v))
+#define atomic64_sub_unchecked(a, v) atomic64_sub((a), (v))
+#define atomic64_inc_unchecked(v) atomic64_inc(v)
+#define atomic64_inc_return_unchecked(v) atomic64_inc_return(v)
+#define atomic64_dec_unchecked(v) atomic64_dec(v)
+#define atomic64_cmpxchg_unchecked(v, o, n) atomic64_cmpxchg((v), (o), (n))
+
/* Define this to indicate that cmpxchg is an efficient operation. */
#define __HAVE_ARCH_CMPXCHG
#ifndef _ASM_TILE_CACHE_H
#define _ASM_TILE_CACHE_H
+#include <linux/const.h>
#include <arch/chip.h>
/* bytes per L1 data cache line */
#define L1_CACHE_SHIFT CHIP_L1D_LOG_LINE_SIZE()
-#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT)
+#define L1_CACHE_BYTES (_AC(1,UL) << L1_CACHE_SHIFT)
/* bytes per L2 cache line */
#define L2_CACHE_SHIFT CHIP_L2_LOG_LINE_SIZE()
const void __user *from,
unsigned long n)
{
- int sz = __compiletime_object_size(to);
+ size_t sz = __compiletime_object_size(to);
- if (likely(sz == -1 || sz >= n))
+ if (likely(sz == (size_t)-1 || sz >= n))
n = _copy_from_user(to, from, n);
else
copy_from_user_overflow();
info.high_limit = TASK_SIZE;
info.align_mask = PAGE_MASK & ~huge_page_mask(h);
info.align_offset = 0;
+ info.threadstack_offset = 0;
return vm_unmapped_area(&info);
}
info.high_limit = current->mm->mmap_base;
info.align_mask = PAGE_MASK & ~huge_page_mask(h);
info.align_offset = 0;
+ info.threadstack_offset = 0;
addr = vm_unmapped_area(&info);
/*
$(patsubst -I%,,$(KBUILD_CFLAGS)))) $(ARCH_INCLUDE) $(MODE_INCLUDE) \
$(filter -I%,$(CFLAGS)) -D_FILE_OFFSET_BITS=64 -idirafter include
+ifdef CONSTIFY_PLUGIN
+USER_CFLAGS += -fplugin-arg-constify_plugin-no-constify
+endif
+
#This will adjust *FLAGS accordingly to the platform.
include $(srctree)/$(ARCH_DIR)/Makefile-os-$(OS)
#ifndef __UM_CACHE_H
#define __UM_CACHE_H
+#include <linux/const.h>
#if defined(CONFIG_UML_X86) && !defined(CONFIG_64BIT)
# define L1_CACHE_SHIFT (CONFIG_X86_L1_CACHE_SHIFT)
# define L1_CACHE_SHIFT 5
#endif
-#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT)
+#define L1_CACHE_BYTES (_AC(1,UL) << L1_CACHE_SHIFT)
#endif
/* No more #include "asm/arch/kmap_types.h" ! */
-#define KM_TYPE_NR 14
+#define KM_TYPE_NR 15
#endif
#define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT)
#define PAGE_MASK (~(PAGE_SIZE-1))
+#define ktla_ktva(addr) (addr)
+#define ktva_ktla(addr) (addr)
+
#ifndef __ASSEMBLY__
struct page;
#define pud_present(x) (pud_val(x) & _PAGE_PRESENT)
#define pud_populate(mm, pud, pmd) \
set_pud(pud, __pud(_PAGE_TABLE + __pa(pmd)))
+#define pud_populate_kernel(mm, pud, pmd) pud_populate((mm), (pud), (pmd))
#ifdef CONFIG_64BIT
#define set_pud(pudptr, pudval) set_64bit((u64 *) (pudptr), pud_val(pudval))
return 2;
}
-/*
- * Only x86 and x86_64 have an arch_align_stack().
- * All other arches have "#define arch_align_stack(x) (x)"
- * in their asm/exec.h
- * As this is included in UML from asm-um/system-generic.h,
- * we can use it to behave as the subarch does.
- */
-#ifndef arch_align_stack
-unsigned long arch_align_stack(unsigned long sp)
-{
- if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
- sp -= get_random_int() % 8192;
- return sp & ~0xf;
-}
-#endif
-
unsigned long get_wchan(struct task_struct *p)
{
unsigned long stack_page, sp, ip;
#ifndef __UNICORE_CACHE_H__
#define __UNICORE_CACHE_H__
-#define L1_CACHE_SHIFT (5)
-#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT)
+#include <linux/const.h>
+
+#define L1_CACHE_SHIFT 5
+#define L1_CACHE_BYTES (_AC(1,UL) << L1_CACHE_SHIFT)
/*
* Memory returned by kmalloc() may be used for DMA, so we must make
select RTC_LIB
select HAVE_DEBUG_STACKOVERFLOW
select HAVE_IRQ_EXIT_ON_IRQ_STACK if X86_64
- select HAVE_CC_STACKPROTECTOR
+ select HAVE_CC_STACKPROTECTOR if X86_64 || !PAX_MEMORY_UDEREF
select GENERIC_CPU_AUTOPROBE
select HAVE_ARCH_AUDITSYSCALL
select ARCH_SUPPORTS_ATOMIC_RMW
config X86_32_LAZY_GS
def_bool y
- depends on X86_32 && !CC_STACKPROTECTOR
+ depends on X86_32 && !CC_STACKPROTECTOR && !PAX_MEMORY_UDEREF
config ARCH_HWEIGHT_CFLAGS
string
menuconfig HYPERVISOR_GUEST
bool "Linux guest support"
+ depends on !GRKERNSEC_CONFIG_AUTO || GRKERNSEC_CONFIG_VIRT_GUEST || (GRKERNSEC_CONFIG_VIRT_HOST && GRKERNSEC_CONFIG_VIRT_XEN)
---help---
Say Y here to enable options for running Linux under various hyper-
visors. This option enables basic hypervisor detection and platform
config X86_16BIT
bool "Enable support for 16-bit segments" if EXPERT
+ depends on !GRKERNSEC
default y
---help---
This option is required by programs like Wine to run 16-bit
config NOHIGHMEM
bool "off"
+ depends on !(PAX_PAGEEXEC && PAX_ENABLE_PAE)
---help---
Linux can use up to 64 Gigabytes of physical memory on x86 systems.
However, the address space of 32-bit x86 processors is only 4
config HIGHMEM4G
bool "4GB"
+ depends on !(PAX_PAGEEXEC && PAX_ENABLE_PAE)
---help---
Select this if you have a 32-bit processor and between 1 and 4
gigabytes of physical RAM.
hex
default 0xB0000000 if VMSPLIT_3G_OPT
default 0x80000000 if VMSPLIT_2G
- default 0x78000000 if VMSPLIT_2G_OPT
+ default 0x70000000 if VMSPLIT_2G_OPT
default 0x40000000 if VMSPLIT_1G
default 0xC0000000
depends on X86_32
config KEXEC
bool "kexec system call"
+ depends on !GRKERNSEC_KMEM
---help---
kexec is a system call that implements the ability to shutdown your
current kernel, and to start another kernel. It is like a reboot
config PHYSICAL_ALIGN
hex "Alignment value to which kernel should be aligned"
- default "0x200000"
+ default "0x1000000"
+ range 0x200000 0x1000000 if PAX_KERNEXEC && X86_PAE
+ range 0x400000 0x1000000 if PAX_KERNEXEC && !X86_PAE
range 0x2000 0x1000000 if X86_32
range 0x200000 0x1000000 if X86_64
---help---
def_bool n
prompt "Disable the 32-bit vDSO (needed for glibc 2.3.3)"
depends on X86_32 || IA32_EMULATION
+ depends on !PAX_PAGEEXEC && !PAX_SEGMEXEC && !PAX_KERNEXEC && !PAX_MEMORY_UDEREF
---help---
Certain buggy versions of glibc will crash if they are
presented with a 32-bit vDSO that is not mapped at the address
config X86_F00F_BUG
def_bool y
- depends on M586MMX || M586TSC || M586 || M486
+ depends on (M586MMX || M586TSC || M586 || M486) && !PAX_KERNEXEC
config X86_INVD_BUG
def_bool y
config X86_ALIGNMENT_16
def_bool y
- depends on MWINCHIP3D || MWINCHIPC6 || MCYRIXIII || MELAN || MK6 || M586MMX || M586TSC || M586 || M486 || MVIAC3_2 || MGEODEGX1
+ depends on MWINCHIP3D || MWINCHIPC6 || MCYRIXIII || X86_ELAN || MK8 || MK7 || MK6 || MCORE2 || MPENTIUM4 || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || M586 || M486 || MVIAC3_2 || MGEODEGX1
config X86_INTEL_USERCOPY
def_bool y
# generates cmov.
config X86_CMOV
def_bool y
- depends on (MK8 || MK7 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MCRUSOE || MEFFICEON || X86_64 || MATOM || MGEODE_LX)
+ depends on (MK8 || MK7 || MCORE2 || MPSC || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MCRUSOE || MEFFICEON || X86_64 || MATOM || MGEODE_LX)
config X86_MINIMUM_CPU_FAMILY
int
config DEBUG_RODATA
bool "Write protect kernel read-only data structures"
default y
- depends on DEBUG_KERNEL
+ depends on DEBUG_KERNEL && BROKEN
---help---
Mark the kernel read-only data as write-protected in the pagetables,
in order to catch accidental (and incorrect) writes to such const
config DEBUG_SET_MODULE_RONX
bool "Set loadable kernel module data as NX and text as RO"
- depends on MODULES
+ depends on MODULES && BROKEN
---help---
This option helps catch unintended modifications to loadable
kernel module's text and read-only data. It also prevents execution
# CPU-specific tuning. Anything which can be shared with UML should go here.
include $(srctree)/arch/x86/Makefile_32.cpu
KBUILD_CFLAGS += $(cflags-y)
-
- # temporary until string.h is fixed
- KBUILD_CFLAGS += -ffreestanding
else
BITS := 64
UTS_MACHINE := x86_64
KBUILD_CFLAGS += $(call cc-option,-maccumulate-outgoing-args)
endif
+# temporary until string.h is fixed
+KBUILD_CFLAGS += -ffreestanding
+
# Make sure compiler does not have buggy stack-protector support.
ifdef CONFIG_CC_STACKPROTECTOR
cc_has_sp := $(srctree)/scripts/gcc-x86_$(BITS)-has-stack-protector.sh
$(Q)$(MAKE) $(build)=arch/x86/syscalls all
archprepare:
+ $(if $(LDFLAGS_BUILD_ID),,$(error $(OLD_LD)))
ifeq ($(CONFIG_KEXEC_FILE),y)
$(Q)$(MAKE) $(build)=arch/x86/purgatory arch/x86/purgatory/kexec-purgatory.c
endif
echo ' FDARGS="..." arguments for the booted kernel'
echo ' FDINITRD=file initrd for the booted kernel'
endef
+
+define OLD_LD
+
+*** ${VERSION}.${PATCHLEVEL} PaX kernels no longer build correctly with old versions of binutils.
+*** Please upgrade your binutils to 2.18 or newer
+endef
# ---------------------------------------------------------------------------
KBUILD_CFLAGS := $(USERINCLUDE) $(REALMODE_CFLAGS) -D_SETUP
+ifdef CONSTIFY_PLUGIN
+KBUILD_CFLAGS += -fplugin-arg-constify_plugin-no-constify
+endif
KBUILD_AFLAGS := $(KBUILD_CFLAGS) -D__ASSEMBLY__
GCOV_PROFILE := n
u8 v;
const u32 *p = (const u32 *)addr;
- asm("btl %2,%1; setc %0" : "=qm" (v) : "m" (*p), "Ir" (nr));
+ asm volatile("btl %2,%1; setc %0" : "=qm" (v) : "m" (*p), "Ir" (nr));
return v;
}
static inline void set_bit(int nr, void *addr)
{
- asm("btsl %1,%0" : "+m" (*(u32 *)addr) : "Ir" (nr));
+ asm volatile("btsl %1,%0" : "+m" (*(u32 *)addr) : "Ir" (nr));
}
#endif /* BOOT_BITOPS_H */
static inline u16 ds(void)
{
u16 seg;
- asm("movw %%ds,%0" : "=rm" (seg));
+ asm volatile("movw %%ds,%0" : "=rm" (seg));
return seg;
}
KBUILD_CFLAGS += -mno-mmx -mno-sse
KBUILD_CFLAGS += $(call cc-option,-ffreestanding)
KBUILD_CFLAGS += $(call cc-option,-fno-stack-protector)
+ifdef CONSTIFY_PLUGIN
+KBUILD_CFLAGS += -fplugin-arg-constify_plugin-no-constify
+endif
KBUILD_AFLAGS := $(KBUILD_CFLAGS) -D__ASSEMBLY__
GCOV_PROFILE := n
* parameter 2, ..., param n. To make things easy, we save the return
* address of efi_call_phys in a global variable.
*/
- popl %ecx
- movl %ecx, saved_return_addr(%edx)
- /* get the function pointer into ECX*/
- popl %ecx
- movl %ecx, efi_rt_function_ptr(%edx)
+ popl saved_return_addr(%edx)
+ popl efi_rt_function_ptr(%edx)
/*
* 3. Call the physical function.
*/
- call *%ecx
+ call *efi_rt_function_ptr(%edx)
/*
* 4. Balance the stack. And because EAX contain the return value,
1: popl %edx
subl $1b, %edx
- movl efi_rt_function_ptr(%edx), %ecx
- pushl %ecx
+ pushl efi_rt_function_ptr(%edx)
/*
* 10. Push the saved return address onto the stack and return.
*/
- movl saved_return_addr(%edx), %ecx
- pushl %ecx
- ret
+ jmpl *saved_return_addr(%edx)
ENDPROC(efi_call_phys)
.previous
.long 0 /* Filled out by user */
.word 0
.quad 0x0000000000000000 /* NULL descriptor */
- .quad 0x00af9a000000ffff /* __KERNEL_CS */
- .quad 0x00cf92000000ffff /* __KERNEL_DS */
+ .quad 0x00af9b000000ffff /* __KERNEL_CS */
+ .quad 0x00cf93000000ffff /* __KERNEL_DS */
.quad 0x0080890000000000 /* TS descriptor */
.quad 0x0000000000000000 /* TS continued */
efi_gdt64_end:
addl %eax, %ebx
notl %eax
andl %eax, %ebx
- cmpl $LOAD_PHYSICAL_ADDR, %ebx
+ cmpl $____LOAD_PHYSICAL_ADDR, %ebx
jge 1f
#endif
- movl $LOAD_PHYSICAL_ADDR, %ebx
+ movl $____LOAD_PHYSICAL_ADDR, %ebx
1:
/* Target address to relocate to for decompression */
addl %eax, %ebx
notl %eax
andl %eax, %ebx
- cmpl $LOAD_PHYSICAL_ADDR, %ebx
+ cmpl $____LOAD_PHYSICAL_ADDR, %ebx
jge 1f
#endif
- movl $LOAD_PHYSICAL_ADDR, %ebx
+ movl $____LOAD_PHYSICAL_ADDR, %ebx
1:
/* Target address to relocate to for decompression */
addq %rax, %rbp
notq %rax
andq %rax, %rbp
- cmpq $LOAD_PHYSICAL_ADDR, %rbp
+ cmpq $____LOAD_PHYSICAL_ADDR, %rbp
jge 1f
#endif
- movq $LOAD_PHYSICAL_ADDR, %rbp
+ movq $____LOAD_PHYSICAL_ADDR, %rbp
1:
/* Target address to relocate to for decompression */
.long gdt
.word 0
.quad 0x0000000000000000 /* NULL descriptor */
- .quad 0x00af9a000000ffff /* __KERNEL_CS */
- .quad 0x00cf92000000ffff /* __KERNEL_DS */
+ .quad 0x00af9b000000ffff /* __KERNEL_CS */
+ .quad 0x00cf93000000ffff /* __KERNEL_DS */
.quad 0x0080890000000000 /* TS descriptor */
.quad 0x0000000000000000 /* TS continued */
gdt_end:
* Calculate the delta between where vmlinux was linked to load
* and where it was actually loaded.
*/
- delta = min_addr - LOAD_PHYSICAL_ADDR;
+ delta = min_addr - ____LOAD_PHYSICAL_ADDR;
if (!delta) {
debug_putstr("No relocation needed... ");
return;
Elf32_Ehdr ehdr;
Elf32_Phdr *phdrs, *phdr;
#endif
- void *dest;
+ void *dest, *prev;
int i;
memcpy(&ehdr, output, sizeof(ehdr));
case PT_LOAD:
#ifdef CONFIG_RELOCATABLE
dest = output;
- dest += (phdr->p_paddr - LOAD_PHYSICAL_ADDR);
+ dest += (phdr->p_paddr - ____LOAD_PHYSICAL_ADDR);
#else
dest = (void *)(phdr->p_paddr);
#endif
memcpy(dest,
output + phdr->p_offset,
phdr->p_filesz);
+ if (i)
+ memset(prev, 0xff, dest - prev);
+ prev = dest + phdr->p_filesz;
break;
default: /* Ignore other PT_* */ break;
}
error("Destination address too large");
#endif
#ifndef CONFIG_RELOCATABLE
- if ((unsigned long)output != LOAD_PHYSICAL_ADDR)
+ if ((unsigned long)output != ____LOAD_PHYSICAL_ADDR)
error("Wrong destination address");
#endif
u32 ecx = MSR_K7_HWCR;
u32 eax, edx;
- asm("rdmsr" : "=a" (eax), "=d" (edx) : "c" (ecx));
+ asm volatile("rdmsr" : "=a" (eax), "=d" (edx) : "c" (ecx));
eax &= ~(1 << 15);
- asm("wrmsr" : : "a" (eax), "d" (edx), "c" (ecx));
+ asm volatile("wrmsr" : : "a" (eax), "d" (edx), "c" (ecx));
get_cpuflags(); /* Make sure it really did something */
err = check_cpuflags();
u32 ecx = MSR_VIA_FCR;
u32 eax, edx;
- asm("rdmsr" : "=a" (eax), "=d" (edx) : "c" (ecx));
+ asm volatile("rdmsr" : "=a" (eax), "=d" (edx) : "c" (ecx));
eax |= (1<<1)|(1<<7);
- asm("wrmsr" : : "a" (eax), "d" (edx), "c" (ecx));
+ asm volatile("wrmsr" : : "a" (eax), "d" (edx), "c" (ecx));
set_bit(X86_FEATURE_CX8, cpu.flags);
err = check_cpuflags();
u32 eax, edx;
u32 level = 1;
- asm("rdmsr" : "=a" (eax), "=d" (edx) : "c" (ecx));
- asm("wrmsr" : : "a" (~0), "d" (edx), "c" (ecx));
- asm("cpuid"
+ asm volatile("rdmsr" : "=a" (eax), "=d" (edx) : "c" (ecx));
+ asm volatile("wrmsr" : : "a" (~0), "d" (edx), "c" (ecx));
+ asm volatile("cpuid"
: "+a" (level), "=d" (cpu.flags[0])
: : "ecx", "ebx");
- asm("wrmsr" : : "a" (eax), "d" (edx), "c" (ecx));
+ asm volatile("wrmsr" : : "a" (eax), "d" (edx), "c" (ecx));
err = check_cpuflags();
} else if (err == 0x01 &&
# single linked list of
# struct setup_data
-pref_address: .quad LOAD_PHYSICAL_ADDR # preferred load addr
+pref_address: .quad ____LOAD_PHYSICAL_ADDR # preferred load addr
#define ZO_INIT_SIZE (ZO__end - ZO_startup_32 + ZO_z_extract_offset)
+#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_KERNEXEC)
+#define VO_INIT_SIZE (VO__end - VO__text - __PAGE_OFFSET - ____LOAD_PHYSICAL_ADDR)
+#else
#define VO_INIT_SIZE (VO__end - VO__text)
+#endif
#if ZO_INIT_SIZE > VO_INIT_SIZE
#define INIT_SIZE ZO_INIT_SIZE
#else
static int detect_memory_e820(void)
{
- int count = 0;
+ unsigned int count = 0;
struct biosregs ireg, oreg;
struct e820entry *desc = boot_params.e820_map;
static struct e820entry buf; /* static so it is zeroed */
boot_params.screen_info.vesapm_seg = oreg.es;
boot_params.screen_info.vesapm_off = oreg.di;
+ boot_params.screen_info.vesapm_size = oreg.cx;
}
/*
static unsigned int get_entry(void)
{
char entry_buf[4];
- int i, len = 0;
+ unsigned int i, len = 0;
int key;
unsigned int v;
* including this sentence is retained in full.
*/
+#include <asm/alternative-asm.h>
+
.extern crypto_ft_tab
.extern crypto_it_tab
.extern crypto_fl_tab
je B192; \
leaq 32(r9),r9;
+#define ret pax_force_retaddr; ret
+
#define epilogue(FUNC,r1,r2,r3,r4,r5,r6,r7,r8,r9) \
movq r1,r2; \
movq r3,r4; \
#include <linux/linkage.h>
#include <asm/inst.h>
+#include <asm/alternative-asm.h>
#ifdef __x86_64__
.data
* num_initial_blocks = b mod 4
* encrypt the initial num_initial_blocks blocks and apply ghash on
* the ciphertext
-* %r10, %r11, %r12, %rax, %xmm5, %xmm6, %xmm7, %xmm8, %xmm9 registers
+* %r10, %r11, %r15, %rax, %xmm5, %xmm6, %xmm7, %xmm8, %xmm9 registers
* are clobbered
* arg1, %arg2, %arg3, %r14 are used as a pointer only, not modified
*/
.macro INITIAL_BLOCKS_DEC num_initial_blocks TMP1 TMP2 TMP3 TMP4 TMP5 XMM0 XMM1 \
XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation
mov arg7, %r10 # %r10 = AAD
- mov arg8, %r12 # %r12 = aadLen
- mov %r12, %r11
+ mov arg8, %r15 # %r15 = aadLen
+ mov %r15, %r11
pxor %xmm\i, %xmm\i
_get_AAD_loop\num_initial_blocks\operation:
movd (%r10), \TMP1
psrldq $4, %xmm\i
pxor \TMP1, %xmm\i
add $4, %r10
- sub $4, %r12
+ sub $4, %r15
jne _get_AAD_loop\num_initial_blocks\operation
cmp $16, %r11
je _get_AAD_loop2_done\num_initial_blocks\operation
- mov $16, %r12
+ mov $16, %r15
_get_AAD_loop2\num_initial_blocks\operation:
psrldq $4, %xmm\i
- sub $4, %r12
- cmp %r11, %r12
+ sub $4, %r15
+ cmp %r11, %r15
jne _get_AAD_loop2\num_initial_blocks\operation
_get_AAD_loop2_done\num_initial_blocks\operation:
movdqa SHUF_MASK(%rip), %xmm14
* num_initial_blocks = b mod 4
* encrypt the initial num_initial_blocks blocks and apply ghash on
* the ciphertext
-* %r10, %r11, %r12, %rax, %xmm5, %xmm6, %xmm7, %xmm8, %xmm9 registers
+* %r10, %r11, %r15, %rax, %xmm5, %xmm6, %xmm7, %xmm8, %xmm9 registers
* are clobbered
* arg1, %arg2, %arg3, %r14 are used as a pointer only, not modified
*/
.macro INITIAL_BLOCKS_ENC num_initial_blocks TMP1 TMP2 TMP3 TMP4 TMP5 XMM0 XMM1 \
XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation
mov arg7, %r10 # %r10 = AAD
- mov arg8, %r12 # %r12 = aadLen
- mov %r12, %r11
+ mov arg8, %r15 # %r15 = aadLen
+ mov %r15, %r11
pxor %xmm\i, %xmm\i
_get_AAD_loop\num_initial_blocks\operation:
movd (%r10), \TMP1
psrldq $4, %xmm\i
pxor \TMP1, %xmm\i
add $4, %r10
- sub $4, %r12
+ sub $4, %r15
jne _get_AAD_loop\num_initial_blocks\operation
cmp $16, %r11
je _get_AAD_loop2_done\num_initial_blocks\operation
- mov $16, %r12
+ mov $16, %r15
_get_AAD_loop2\num_initial_blocks\operation:
psrldq $4, %xmm\i
- sub $4, %r12
- cmp %r11, %r12
+ sub $4, %r15
+ cmp %r11, %r15
jne _get_AAD_loop2\num_initial_blocks\operation
_get_AAD_loop2_done\num_initial_blocks\operation:
movdqa SHUF_MASK(%rip), %xmm14
*
*****************************************************************************/
ENTRY(aesni_gcm_dec)
- push %r12
+ push %r15
push %r13
push %r14
mov %rsp, %r14
*/
sub $VARIABLE_OFFSET, %rsp
and $~63, %rsp # align rsp to 64 bytes
- mov %arg6, %r12
- movdqu (%r12), %xmm13 # %xmm13 = HashKey
+ mov %arg6, %r15
+ movdqu (%r15), %xmm13 # %xmm13 = HashKey
movdqa SHUF_MASK(%rip), %xmm2
PSHUFB_XMM %xmm2, %xmm13
movdqa %xmm13, HashKey(%rsp) # store HashKey<<1 (mod poly)
mov %arg4, %r13 # save the number of bytes of plaintext/ciphertext
and $-16, %r13 # %r13 = %r13 - (%r13 mod 16)
- mov %r13, %r12
- and $(3<<4), %r12
+ mov %r13, %r15
+ and $(3<<4), %r15
jz _initial_num_blocks_is_0_decrypt
- cmp $(2<<4), %r12
+ cmp $(2<<4), %r15
jb _initial_num_blocks_is_1_decrypt
je _initial_num_blocks_is_2_decrypt
_initial_num_blocks_is_3_decrypt:
sub $16, %r11
add %r13, %r11
movdqu (%arg3,%r11,1), %xmm1 # receive the last <16 byte block
- lea SHIFT_MASK+16(%rip), %r12
- sub %r13, %r12
+ lea SHIFT_MASK+16(%rip), %r15
+ sub %r13, %r15
# adjust the shuffle mask pointer to be able to shift 16-%r13 bytes
# (%r13 is the number of bytes in plaintext mod 16)
- movdqu (%r12), %xmm2 # get the appropriate shuffle mask
+ movdqu (%r15), %xmm2 # get the appropriate shuffle mask
PSHUFB_XMM %xmm2, %xmm1 # right shift 16-%r13 butes
movdqa %xmm1, %xmm2
pxor %xmm1, %xmm0 # Ciphertext XOR E(K, Yn)
- movdqu ALL_F-SHIFT_MASK(%r12), %xmm1
+ movdqu ALL_F-SHIFT_MASK(%r15), %xmm1
# get the appropriate mask to mask out top 16-%r13 bytes of %xmm0
pand %xmm1, %xmm0 # mask out top 16-%r13 bytes of %xmm0
pand %xmm1, %xmm2
sub $1, %r13
jne _less_than_8_bytes_left_decrypt
_multiple_of_16_bytes_decrypt:
- mov arg8, %r12 # %r13 = aadLen (number of bytes)
- shl $3, %r12 # convert into number of bits
- movd %r12d, %xmm15 # len(A) in %xmm15
+ mov arg8, %r15 # %r13 = aadLen (number of bytes)
+ shl $3, %r15 # convert into number of bits
+ movd %r15d, %xmm15 # len(A) in %xmm15
shl $3, %arg4 # len(C) in bits (*128)
MOVQ_R64_XMM %arg4, %xmm1
pslldq $8, %xmm15 # %xmm15 = len(A)||0x0000000000000000
mov %r14, %rsp
pop %r14
pop %r13
- pop %r12
+ pop %r15
+ pax_force_retaddr
ret
ENDPROC(aesni_gcm_dec)
* poly = x^128 + x^127 + x^126 + x^121 + 1
***************************************************************************/
ENTRY(aesni_gcm_enc)
- push %r12
+ push %r15
push %r13
push %r14
mov %rsp, %r14
#
sub $VARIABLE_OFFSET, %rsp
and $~63, %rsp
- mov %arg6, %r12
- movdqu (%r12), %xmm13
+ mov %arg6, %r15
+ movdqu (%r15), %xmm13
movdqa SHUF_MASK(%rip), %xmm2
PSHUFB_XMM %xmm2, %xmm13
movdqa %xmm13, HashKey(%rsp)
mov %arg4, %r13 # %xmm13 holds HashKey<<1 (mod poly)
and $-16, %r13
- mov %r13, %r12
+ mov %r13, %r15
# Encrypt first few blocks
- and $(3<<4), %r12
+ and $(3<<4), %r15
jz _initial_num_blocks_is_0_encrypt
- cmp $(2<<4), %r12
+ cmp $(2<<4), %r15
jb _initial_num_blocks_is_1_encrypt
je _initial_num_blocks_is_2_encrypt
_initial_num_blocks_is_3_encrypt:
sub $16, %r11
add %r13, %r11
movdqu (%arg3,%r11,1), %xmm1 # receive the last <16 byte blocks
- lea SHIFT_MASK+16(%rip), %r12
- sub %r13, %r12
+ lea SHIFT_MASK+16(%rip), %r15
+ sub %r13, %r15
# adjust the shuffle mask pointer to be able to shift 16-r13 bytes
# (%r13 is the number of bytes in plaintext mod 16)
- movdqu (%r12), %xmm2 # get the appropriate shuffle mask
+ movdqu (%r15), %xmm2 # get the appropriate shuffle mask
PSHUFB_XMM %xmm2, %xmm1 # shift right 16-r13 byte
pxor %xmm1, %xmm0 # Plaintext XOR Encrypt(K, Yn)
- movdqu ALL_F-SHIFT_MASK(%r12), %xmm1
+ movdqu ALL_F-SHIFT_MASK(%r15), %xmm1
# get the appropriate mask to mask out top 16-r13 bytes of xmm0
pand %xmm1, %xmm0 # mask out top 16-r13 bytes of xmm0
movdqa SHUF_MASK(%rip), %xmm10
sub $1, %r13
jne _less_than_8_bytes_left_encrypt
_multiple_of_16_bytes_encrypt:
- mov arg8, %r12 # %r12 = addLen (number of bytes)
- shl $3, %r12
- movd %r12d, %xmm15 # len(A) in %xmm15
+ mov arg8, %r15 # %r15 = addLen (number of bytes)
+ shl $3, %r15
+ movd %r15d, %xmm15 # len(A) in %xmm15
shl $3, %arg4 # len(C) in bits (*128)
MOVQ_R64_XMM %arg4, %xmm1
pslldq $8, %xmm15 # %xmm15 = len(A)||0x0000000000000000
mov %r14, %rsp
pop %r14
pop %r13
- pop %r12
+ pop %r15
+ pax_force_retaddr
ret
ENDPROC(aesni_gcm_enc)
pxor %xmm1, %xmm0
movaps %xmm0, (TKEYP)
add $0x10, TKEYP
+ pax_force_retaddr
ret
ENDPROC(_key_expansion_128)
ENDPROC(_key_expansion_256a)
shufps $0b01001110, %xmm2, %xmm1
movaps %xmm1, 0x10(TKEYP)
add $0x20, TKEYP
+ pax_force_retaddr
ret
ENDPROC(_key_expansion_192a)
movaps %xmm0, (TKEYP)
add $0x10, TKEYP
+ pax_force_retaddr
ret
ENDPROC(_key_expansion_192b)
pxor %xmm1, %xmm2
movaps %xmm2, (TKEYP)
add $0x10, TKEYP
+ pax_force_retaddr
ret
ENDPROC(_key_expansion_256b)
#ifndef __x86_64__
popl KEYP
#endif
+ pax_force_retaddr
ret
ENDPROC(aesni_set_key)
popl KLEN
popl KEYP
#endif
+ pax_force_retaddr
ret
ENDPROC(aesni_enc)
AESENC KEY STATE
movaps 0x70(TKEYP), KEY
AESENCLAST KEY STATE
+ pax_force_retaddr
ret
ENDPROC(_aesni_enc1)
AESENCLAST KEY STATE2
AESENCLAST KEY STATE3
AESENCLAST KEY STATE4
+ pax_force_retaddr
ret
ENDPROC(_aesni_enc4)
popl KLEN
popl KEYP
#endif
+ pax_force_retaddr
ret
ENDPROC(aesni_dec)
AESDEC KEY STATE
movaps 0x70(TKEYP), KEY
AESDECLAST KEY STATE
+ pax_force_retaddr
ret
ENDPROC(_aesni_dec1)
AESDECLAST KEY STATE2
AESDECLAST KEY STATE3
AESDECLAST KEY STATE4
+ pax_force_retaddr
ret
ENDPROC(_aesni_dec4)
popl KEYP
popl LEN
#endif
+ pax_force_retaddr
ret
ENDPROC(aesni_ecb_enc)
popl KEYP
popl LEN
#endif
+ pax_force_retaddr
ret
ENDPROC(aesni_ecb_dec)
popl LEN
popl IVP
#endif
+ pax_force_retaddr
ret
ENDPROC(aesni_cbc_enc)
popl LEN
popl IVP
#endif
+ pax_force_retaddr
ret
ENDPROC(aesni_cbc_dec)
mov $1, TCTR_LOW
MOVQ_R64_XMM TCTR_LOW INC
MOVQ_R64_XMM CTR TCTR_LOW
+ pax_force_retaddr
ret
ENDPROC(_aesni_inc_init)
.Linc_low:
movaps CTR, IV
PSHUFB_XMM BSWAP_MASK IV
+ pax_force_retaddr
ret
ENDPROC(_aesni_inc)
.Lctr_enc_ret:
movups IV, (IVP)
.Lctr_enc_just_ret:
+ pax_force_retaddr
ret
ENDPROC(aesni_ctr_enc)
pxor INC, STATE4
movdqu STATE4, 0x70(OUTP)
+ pax_force_retaddr
ret
ENDPROC(aesni_xts_crypt8)
*/
#include <linux/linkage.h>
+#include <asm/alternative-asm.h>
.file "blowfish-x86_64-asm.S"
.text
jnz .L__enc_xor;
write_block();
+ pax_force_retaddr
ret;
.L__enc_xor:
xor_block();
+ pax_force_retaddr
ret;
ENDPROC(__blowfish_enc_blk)
movq %r11, %rbp;
+ pax_force_retaddr
ret;
ENDPROC(blowfish_dec_blk)
popq %rbx;
popq %rbp;
+ pax_force_retaddr
ret;
.L__enc_xor4:
popq %rbx;
popq %rbp;
+ pax_force_retaddr
ret;
ENDPROC(__blowfish_enc_blk_4way)
popq %rbx;
popq %rbp;
+ pax_force_retaddr
ret;
ENDPROC(blowfish_dec_blk_4way)
*/
#include <linux/linkage.h>
+#include <asm/alternative-asm.h>
#define CAMELLIA_TABLE_BYTE_LEN 272
roundsm16(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm15,
%rcx, (%r9));
+ pax_force_retaddr
ret;
ENDPROC(roundsm16_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd)
roundsm16(%xmm4, %xmm5, %xmm6, %xmm7, %xmm0, %xmm1, %xmm2, %xmm3,
%xmm12, %xmm13, %xmm14, %xmm15, %xmm8, %xmm9, %xmm10, %xmm11,
%rax, (%r9));
+ pax_force_retaddr
ret;
ENDPROC(roundsm16_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab)
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
%xmm15, (key_table)(CTX, %r8, 8), (%rax), 1 * 16(%rax));
+ pax_force_retaddr
ret;
.align 8
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
%xmm15, (key_table)(CTX), (%rax), 1 * 16(%rax));
+ pax_force_retaddr
ret;
.align 8
%xmm15, %xmm14, %xmm13, %xmm12, %xmm11, %xmm10, %xmm9,
%xmm8, %rsi);
+ pax_force_retaddr
ret;
ENDPROC(camellia_ecb_enc_16way)
%xmm15, %xmm14, %xmm13, %xmm12, %xmm11, %xmm10, %xmm9,
%xmm8, %rsi);
+ pax_force_retaddr
ret;
ENDPROC(camellia_ecb_dec_16way)
%xmm15, %xmm14, %xmm13, %xmm12, %xmm11, %xmm10, %xmm9,
%xmm8, %rsi);
+ pax_force_retaddr
ret;
ENDPROC(camellia_cbc_dec_16way)
%xmm15, %xmm14, %xmm13, %xmm12, %xmm11, %xmm10, %xmm9,
%xmm8, %rsi);
+ pax_force_retaddr
ret;
ENDPROC(camellia_ctr_16way)
%xmm15, %xmm14, %xmm13, %xmm12, %xmm11, %xmm10, %xmm9,
%xmm8, %rsi);
+ pax_force_retaddr
ret;
ENDPROC(camellia_xts_crypt_16way)
*/
#include <linux/linkage.h>
+#include <asm/alternative-asm.h>
#define CAMELLIA_TABLE_BYTE_LEN 272
roundsm32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
%ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, %ymm15,
%rcx, (%r9));
+ pax_force_retaddr
ret;
ENDPROC(roundsm32_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd)
roundsm32(%ymm4, %ymm5, %ymm6, %ymm7, %ymm0, %ymm1, %ymm2, %ymm3,
%ymm12, %ymm13, %ymm14, %ymm15, %ymm8, %ymm9, %ymm10, %ymm11,
%rax, (%r9));
+ pax_force_retaddr
ret;
ENDPROC(roundsm32_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab)
%ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
%ymm15, (key_table)(CTX, %r8, 8), (%rax), 1 * 32(%rax));
+ pax_force_retaddr
ret;
.align 8
%ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
%ymm15, (key_table)(CTX), (%rax), 1 * 32(%rax));
+ pax_force_retaddr
ret;
.align 8
vzeroupper;
+ pax_force_retaddr
ret;
ENDPROC(camellia_ecb_enc_32way)
vzeroupper;
+ pax_force_retaddr
ret;
ENDPROC(camellia_ecb_dec_32way)
vzeroupper;
+ pax_force_retaddr
ret;
ENDPROC(camellia_cbc_dec_32way)
vzeroupper;
+ pax_force_retaddr
ret;
ENDPROC(camellia_ctr_32way)
vzeroupper;
+ pax_force_retaddr
ret;
ENDPROC(camellia_xts_crypt_32way)
*/
#include <linux/linkage.h>
+#include <asm/alternative-asm.h>
.file "camellia-x86_64-asm_64.S"
.text
enc_outunpack(mov, RT1);
movq RRBP, %rbp;
+ pax_force_retaddr
ret;
.L__enc_xor:
enc_outunpack(xor, RT1);
movq RRBP, %rbp;
+ pax_force_retaddr
ret;
ENDPROC(__camellia_enc_blk)
dec_outunpack();
movq RRBP, %rbp;
+ pax_force_retaddr
ret;
ENDPROC(camellia_dec_blk)
movq RRBP, %rbp;
popq %rbx;
+ pax_force_retaddr
ret;
.L__enc2_xor:
movq RRBP, %rbp;
popq %rbx;
+ pax_force_retaddr
ret;
ENDPROC(__camellia_enc_blk_2way)
movq RRBP, %rbp;
movq RXOR, %rbx;
+ pax_force_retaddr
ret;
ENDPROC(camellia_dec_blk_2way)
*/
#include <linux/linkage.h>
+#include <asm/alternative-asm.h>
.file "cast5-avx-x86_64-asm_64.S"
outunpack_blocks(RR3, RL3, RTMP, RX, RKM);
outunpack_blocks(RR4, RL4, RTMP, RX, RKM);
+ pax_force_retaddr
ret;
ENDPROC(__cast5_enc_blk16)
outunpack_blocks(RR3, RL3, RTMP, RX, RKM);
outunpack_blocks(RR4, RL4, RTMP, RX, RKM);
+ pax_force_retaddr
ret;
.L__skip_dec:
vmovdqu RR4, (6*4*4)(%r11);
vmovdqu RL4, (7*4*4)(%r11);
+ pax_force_retaddr
ret;
ENDPROC(cast5_ecb_enc_16way)
vmovdqu RR4, (6*4*4)(%r11);
vmovdqu RL4, (7*4*4)(%r11);
+ pax_force_retaddr
ret;
ENDPROC(cast5_ecb_dec_16way)
* %rdx: src
*/
- pushq %r12;
+ pushq %r14;
movq %rsi, %r11;
- movq %rdx, %r12;
+ movq %rdx, %r14;
vmovdqu (0*16)(%rdx), RL1;
vmovdqu (1*16)(%rdx), RR1;
call __cast5_dec_blk16;
/* xor with src */
- vmovq (%r12), RX;
+ vmovq (%r14), RX;
vpshufd $0x4f, RX, RX;
vpxor RX, RR1, RR1;
- vpxor 0*16+8(%r12), RL1, RL1;
- vpxor 1*16+8(%r12), RR2, RR2;
- vpxor 2*16+8(%r12), RL2, RL2;
- vpxor 3*16+8(%r12), RR3, RR3;
- vpxor 4*16+8(%r12), RL3, RL3;
- vpxor 5*16+8(%r12), RR4, RR4;
- vpxor 6*16+8(%r12), RL4, RL4;
+ vpxor 0*16+8(%r14), RL1, RL1;
+ vpxor 1*16+8(%r14), RR2, RR2;
+ vpxor 2*16+8(%r14), RL2, RL2;
+ vpxor 3*16+8(%r14), RR3, RR3;
+ vpxor 4*16+8(%r14), RL3, RL3;
+ vpxor 5*16+8(%r14), RR4, RR4;
+ vpxor 6*16+8(%r14), RL4, RL4;
vmovdqu RR1, (0*16)(%r11);
vmovdqu RL1, (1*16)(%r11);
vmovdqu RR4, (6*16)(%r11);
vmovdqu RL4, (7*16)(%r11);
- popq %r12;
+ popq %r14;
+ pax_force_retaddr
ret;
ENDPROC(cast5_cbc_dec_16way)
* %rcx: iv (big endian, 64bit)
*/
- pushq %r12;
+ pushq %r14;
movq %rsi, %r11;
- movq %rdx, %r12;
+ movq %rdx, %r14;
vpcmpeqd RTMP, RTMP, RTMP;
vpsrldq $8, RTMP, RTMP; /* low: -1, high: 0 */
call __cast5_enc_blk16;
/* dst = src ^ iv */
- vpxor (0*16)(%r12), RR1, RR1;
- vpxor (1*16)(%r12), RL1, RL1;
- vpxor (2*16)(%r12), RR2, RR2;
- vpxor (3*16)(%r12), RL2, RL2;
- vpxor (4*16)(%r12), RR3, RR3;
- vpxor (5*16)(%r12), RL3, RL3;
- vpxor (6*16)(%r12), RR4, RR4;
- vpxor (7*16)(%r12), RL4, RL4;
+ vpxor (0*16)(%r14), RR1, RR1;
+ vpxor (1*16)(%r14), RL1, RL1;
+ vpxor (2*16)(%r14), RR2, RR2;
+ vpxor (3*16)(%r14), RL2, RL2;
+ vpxor (4*16)(%r14), RR3, RR3;
+ vpxor (5*16)(%r14), RL3, RL3;
+ vpxor (6*16)(%r14), RR4, RR4;
+ vpxor (7*16)(%r14), RL4, RL4;
vmovdqu RR1, (0*16)(%r11);
vmovdqu RL1, (1*16)(%r11);
vmovdqu RR2, (2*16)(%r11);
vmovdqu RR4, (6*16)(%r11);
vmovdqu RL4, (7*16)(%r11);
- popq %r12;
+ popq %r14;
+ pax_force_retaddr
ret;
ENDPROC(cast5_ctr_16way)
*/
#include <linux/linkage.h>
+#include <asm/alternative-asm.h>
#include "glue_helper-asm-avx.S"
.file "cast6-avx-x86_64-asm_64.S"
outunpack_blocks(RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM);
outunpack_blocks(RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM);
+ pax_force_retaddr
ret;
ENDPROC(__cast6_enc_blk8)
outunpack_blocks(RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM);
outunpack_blocks(RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM);
+ pax_force_retaddr
ret;
ENDPROC(__cast6_dec_blk8)
store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
+ pax_force_retaddr
ret;
ENDPROC(cast6_ecb_enc_8way)
store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
+ pax_force_retaddr
ret;
ENDPROC(cast6_ecb_dec_8way)
* %rdx: src
*/
- pushq %r12;
+ pushq %r14;
movq %rsi, %r11;
- movq %rdx, %r12;
+ movq %rdx, %r14;
load_8way(%rdx, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
call __cast6_dec_blk8;
- store_cbc_8way(%r12, %r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
+ store_cbc_8way(%r14, %r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
- popq %r12;
+ popq %r14;
+ pax_force_retaddr
ret;
ENDPROC(cast6_cbc_dec_8way)
* %rcx: iv (little endian, 128bit)
*/
- pushq %r12;
+ pushq %r14;
movq %rsi, %r11;
- movq %rdx, %r12;
+ movq %rdx, %r14;
load_ctr_8way(%rcx, .Lbswap128_mask, RA1, RB1, RC1, RD1, RA2, RB2, RC2,
RD2, RX, RKR, RKM);
call __cast6_enc_blk8;
- store_ctr_8way(%r12, %r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
+ store_ctr_8way(%r14, %r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
- popq %r12;
+ popq %r14;
+ pax_force_retaddr
ret;
ENDPROC(cast6_ctr_8way)
/* dst <= regs xor IVs(in dst) */
store_xts_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
+ pax_force_retaddr
ret;
ENDPROC(cast6_xts_enc_8way)
/* dst <= regs xor IVs(in dst) */
store_xts_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
+ pax_force_retaddr
ret;
ENDPROC(cast6_xts_dec_8way)
#include <asm/inst.h>
#include <linux/linkage.h>
+#include <asm/alternative-asm.h>
## ISCSI CRC 32 Implementation with crc32 and pclmulqdq Instruction
popq %rsi
popq %rdi
popq %rbx
+ pax_force_retaddr
ret
################################################################
#include <linux/linkage.h>
#include <asm/inst.h>
+#include <asm/alternative-asm.h>
.data
psrlq $1, T2
pxor T2, T1
pxor T1, DATA
+ pax_force_retaddr
ret
ENDPROC(__clmul_gf128mul_ble)
call __clmul_gf128mul_ble
PSHUFB_XMM BSWAP DATA
movups DATA, (%rdi)
+ pax_force_retaddr
ret
ENDPROC(clmul_ghash_mul)
PSHUFB_XMM BSWAP DATA
movups DATA, (%rdi)
.Lupdate_just_ret:
+ pax_force_retaddr
ret
ENDPROC(clmul_ghash_update)
#include <linux/linkage.h>
+#include <asm/alternative-asm.h>
# enter salsa20_encrypt_bytes
ENTRY(salsa20_encrypt_bytes)
add %r11,%rsp
mov %rdi,%rax
mov %rsi,%rdx
+ pax_force_retaddr
ret
# bytesatleast65:
._bytesatleast65:
add %r11,%rsp
mov %rdi,%rax
mov %rsi,%rdx
+ pax_force_retaddr
ret
ENDPROC(salsa20_keysetup)
add %r11,%rsp
mov %rdi,%rax
mov %rsi,%rdx
+ pax_force_retaddr
ret
ENDPROC(salsa20_ivsetup)
*/
#include <linux/linkage.h>
+#include <asm/alternative-asm.h>
#include "glue_helper-asm-avx.S"
.file "serpent-avx-x86_64-asm_64.S"
write_blocks(RA1, RB1, RC1, RD1, RK0, RK1, RK2);
write_blocks(RA2, RB2, RC2, RD2, RK0, RK1, RK2);
+ pax_force_retaddr
ret;
ENDPROC(__serpent_enc_blk8_avx)
write_blocks(RC1, RD1, RB1, RE1, RK0, RK1, RK2);
write_blocks(RC2, RD2, RB2, RE2, RK0, RK1, RK2);
+ pax_force_retaddr
ret;
ENDPROC(__serpent_dec_blk8_avx)
store_8way(%rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
+ pax_force_retaddr
ret;
ENDPROC(serpent_ecb_enc_8way_avx)
store_8way(%rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2);
+ pax_force_retaddr
ret;
ENDPROC(serpent_ecb_dec_8way_avx)
store_cbc_8way(%rdx, %rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2);
+ pax_force_retaddr
ret;
ENDPROC(serpent_cbc_dec_8way_avx)
store_ctr_8way(%rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
+ pax_force_retaddr
ret;
ENDPROC(serpent_ctr_8way_avx)
/* dst <= regs xor IVs(in dst) */
store_xts_8way(%rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
+ pax_force_retaddr
ret;
ENDPROC(serpent_xts_enc_8way_avx)
/* dst <= regs xor IVs(in dst) */
store_xts_8way(%rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2);
+ pax_force_retaddr
ret;
ENDPROC(serpent_xts_dec_8way_avx)
*/
#include <linux/linkage.h>
+#include <asm/alternative-asm.h>
#include "glue_helper-asm-avx2.S"
.file "serpent-avx2-asm_64.S"
write_blocks(RA1, RB1, RC1, RD1, RK0, RK1, RK2);
write_blocks(RA2, RB2, RC2, RD2, RK0, RK1, RK2);
+ pax_force_retaddr
ret;
ENDPROC(__serpent_enc_blk16)
write_blocks(RC1, RD1, RB1, RE1, RK0, RK1, RK2);
write_blocks(RC2, RD2, RB2, RE2, RK0, RK1, RK2);
+ pax_force_retaddr
ret;
ENDPROC(__serpent_dec_blk16)
vzeroupper;
+ pax_force_retaddr
ret;
ENDPROC(serpent_ecb_enc_16way)
vzeroupper;
+ pax_force_retaddr
ret;
ENDPROC(serpent_ecb_dec_16way)
vzeroupper;
+ pax_force_retaddr
ret;
ENDPROC(serpent_cbc_dec_16way)
vzeroupper;
+ pax_force_retaddr
ret;
ENDPROC(serpent_ctr_16way)
vzeroupper;
+ pax_force_retaddr
ret;
ENDPROC(serpent_xts_enc_16way)
vzeroupper;
+ pax_force_retaddr
ret;
ENDPROC(serpent_xts_dec_16way)
*/
#include <linux/linkage.h>
+#include <asm/alternative-asm.h>
.file "serpent-sse2-x86_64-asm_64.S"
.text
write_blocks(%rsi, RA1, RB1, RC1, RD1, RK0, RK1, RK2);
write_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2);
+ pax_force_retaddr
ret;
.L__enc_xor8:
xor_blocks(%rsi, RA1, RB1, RC1, RD1, RK0, RK1, RK2);
xor_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2);
+ pax_force_retaddr
ret;
ENDPROC(__serpent_enc_blk_8way)
write_blocks(%rsi, RC1, RD1, RB1, RE1, RK0, RK1, RK2);
write_blocks(%rax, RC2, RD2, RB2, RE2, RK0, RK1, RK2);
+ pax_force_retaddr
ret;
ENDPROC(serpent_dec_blk_8way)
*/
#include <linux/linkage.h>
+#include <asm/alternative-asm.h>
#define CTX %rdi // arg1
#define BUF %rsi // arg2
push %rbx
push %rbp
- push %r12
+ push %r14
- mov %rsp, %r12
+ mov %rsp, %r14
sub $64, %rsp # allocate workspace
and $~15, %rsp # align stack
xor %rax, %rax
rep stosq
- mov %r12, %rsp # deallocate workspace
+ mov %r14, %rsp # deallocate workspace
- pop %r12
+ pop %r14
pop %rbp
pop %rbx
+ pax_force_retaddr
ret
ENDPROC(\name)
#ifdef CONFIG_AS_AVX
#include <linux/linkage.h>
+#include <asm/alternative-asm.h>
## assume buffers not aligned
#define VMOVDQ vmovdqu
popq %r13
popq %rbp
popq %rbx
+ pax_force_retaddr
ret
ENDPROC(sha256_transform_avx)
#ifdef CONFIG_AS_AVX2
#include <linux/linkage.h>
+#include <asm/alternative-asm.h>
## assume buffers not aligned
#define VMOVDQ vmovdqu
popq %r12
popq %rbp
popq %rbx
+ pax_force_retaddr
ret
ENDPROC(sha256_transform_rorx)
########################################################################
#include <linux/linkage.h>
+#include <asm/alternative-asm.h>
## assume buffers not aligned
#define MOVDQ movdqu
popq %rbp
popq %rbx
+ pax_force_retaddr
ret
ENDPROC(sha256_transform_ssse3)
#ifdef CONFIG_AS_AVX
#include <linux/linkage.h>
+#include <asm/alternative-asm.h>
.text
mov frame_RSPSAVE(%rsp), %rsp
nowork:
+ pax_force_retaddr
ret
ENDPROC(sha512_transform_avx)
#ifdef CONFIG_AS_AVX2
#include <linux/linkage.h>
+#include <asm/alternative-asm.h>
.text
# Restore Stack Pointer
mov frame_RSPSAVE(%rsp), %rsp
+ pax_force_retaddr
ret
ENDPROC(sha512_transform_rorx)
########################################################################
#include <linux/linkage.h>
+#include <asm/alternative-asm.h>
.text
mov frame_RSPSAVE(%rsp), %rsp
nowork:
+ pax_force_retaddr
ret
ENDPROC(sha512_transform_ssse3)
*/
#include <linux/linkage.h>
+#include <asm/alternative-asm.h>
#include "glue_helper-asm-avx.S"
.file "twofish-avx-x86_64-asm_64.S"
outunpack_blocks(RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2);
outunpack_blocks(RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2);
+ pax_force_retaddr
ret;
ENDPROC(__twofish_enc_blk8)
outunpack_blocks(RA1, RB1, RC1, RD1, RK1, RX0, RY0, RK2);
outunpack_blocks(RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2);
+ pax_force_retaddr
ret;
ENDPROC(__twofish_dec_blk8)
store_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
+ pax_force_retaddr
ret;
ENDPROC(twofish_ecb_enc_8way)
store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
+ pax_force_retaddr
ret;
ENDPROC(twofish_ecb_dec_8way)
* %rdx: src
*/
- pushq %r12;
+ pushq %r14;
movq %rsi, %r11;
- movq %rdx, %r12;
+ movq %rdx, %r14;
load_8way(%rdx, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
call __twofish_dec_blk8;
- store_cbc_8way(%r12, %r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
+ store_cbc_8way(%r14, %r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
- popq %r12;
+ popq %r14;
+ pax_force_retaddr
ret;
ENDPROC(twofish_cbc_dec_8way)
* %rcx: iv (little endian, 128bit)
*/
- pushq %r12;
+ pushq %r14;
movq %rsi, %r11;
- movq %rdx, %r12;
+ movq %rdx, %r14;
load_ctr_8way(%rcx, .Lbswap128_mask, RA1, RB1, RC1, RD1, RA2, RB2, RC2,
RD2, RX0, RX1, RY0);
call __twofish_enc_blk8;
- store_ctr_8way(%r12, %r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
+ store_ctr_8way(%r14, %r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
- popq %r12;
+ popq %r14;
+ pax_force_retaddr
ret;
ENDPROC(twofish_ctr_8way)
/* dst <= regs xor IVs(in dst) */
store_xts_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
+ pax_force_retaddr
ret;
ENDPROC(twofish_xts_enc_8way)
/* dst <= regs xor IVs(in dst) */
store_xts_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
+ pax_force_retaddr
ret;
ENDPROC(twofish_xts_dec_8way)
*/
#include <linux/linkage.h>
+#include <asm/alternative-asm.h>
.file "twofish-x86_64-asm-3way.S"
.text
popq %r13;
popq %r14;
popq %r15;
+ pax_force_retaddr
ret;
.L__enc_xor3:
popq %r13;
popq %r14;
popq %r15;
+ pax_force_retaddr
ret;
ENDPROC(__twofish_enc_blk_3way)
popq %r13;
popq %r14;
popq %r15;
+ pax_force_retaddr
ret;
ENDPROC(twofish_dec_blk_3way)
#include <linux/linkage.h>
#include <asm/asm-offsets.h>
+#include <asm/alternative-asm.h>
#define a_offset 0
#define b_offset 4
popq R1
movq $1,%rax
+ pax_force_retaddr
ret
ENDPROC(twofish_enc_blk)
popq R1
movq $1,%rax
+ pax_force_retaddr
ret
ENDPROC(twofish_dec_blk)
unsigned long dump_start, dump_size;
struct user32 dump;
+ memset(&dump, 0, sizeof(dump));
+
fs = get_fs();
set_fs(KERNEL_DS);
has_dumped = 1;
if (__get_user(set.sig[0], &frame->sc.oldmask)
|| (_COMPAT_NSIG_WORDS > 1
&& __copy_from_user((((char *) &set.sig) + 4),
- &frame->extramask,
+ frame->extramask,
sizeof(frame->extramask))))
goto badframe;
sp -= frame_size;
/* Align the stack pointer according to the i386 ABI,
* i.e. so that on function entry ((sp + 4) & 15) == 0. */
- sp = ((sp + 4) & -16ul) - 4;
+ sp = ((sp - 12) & -16ul) - 4;
return (void __user *) sp;
}
} else {
/* Return stub is in 32bit vsyscall page */
if (current->mm->context.vdso)
- restorer = current->mm->context.vdso +
- selected_vdso32->sym___kernel_sigreturn;
+ restorer = (void __force_user *)(current->mm->context.vdso +
+ selected_vdso32->sym___kernel_sigreturn);
else
- restorer = &frame->retcode;
+ restorer = frame->retcode;
}
put_user_try {
* These are actually not used anymore, but left because some
* gdb versions depend on them as a marker.
*/
- put_user_ex(*((u64 *)&code), (u64 __user *)frame->retcode);
+ put_user_ex(*((const u64 *)&code), (u64 __user *)frame->retcode);
} put_user_catch(err);
if (err)
0xb8,
__NR_ia32_rt_sigreturn,
0x80cd,
- 0,
+ 0
};
frame = get_sigframe(ksig, regs, sizeof(*frame), &fpstate);
if (ksig->ka.sa.sa_flags & SA_RESTORER)
restorer = ksig->ka.sa.sa_restorer;
+ else if (current->mm->context.vdso)
+ /* Return stub is in 32bit vsyscall page */
+ restorer = (void __force_user *)(current->mm->context.vdso +
+ selected_vdso32->sym___kernel_rt_sigreturn);
else
- restorer = current->mm->context.vdso +
- selected_vdso32->sym___kernel_rt_sigreturn;
+ restorer = frame->retcode;
put_user_ex(ptr_to_compat(restorer), &frame->pretcode);
/*
* Not actually used anymore, but left because some gdb
* versions need it.
*/
- put_user_ex(*((u64 *)&code), (u64 __user *)frame->retcode);
+ put_user_ex(*((const u64 *)&code), (u64 __user *)frame->retcode);
} put_user_catch(err);
err |= copy_siginfo_to_user32(&frame->info, &ksig->info);
#include <asm/irqflags.h>
#include <asm/asm.h>
#include <asm/smap.h>
+#include <asm/pgtable.h>
#include <linux/linkage.h>
#include <linux/err.h>
+#include <asm/alternative-asm.h>
/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */
#include <linux/elf-em.h>
*/
.macro LOAD_ARGS32 offset, _r9=0
.if \_r9
- movl \offset+16(%rsp),%r9d
+ movl \offset+R9(%rsp),%r9d
.endif
- movl \offset+40(%rsp),%ecx
- movl \offset+48(%rsp),%edx
- movl \offset+56(%rsp),%esi
- movl \offset+64(%rsp),%edi
+ movl \offset+RCX(%rsp),%ecx
+ movl \offset+RDX(%rsp),%edx
+ movl \offset+RSI(%rsp),%esi
+ movl \offset+RDI(%rsp),%edi
movl %eax,%eax /* zero extension */
.endm
ENDPROC(native_irq_enable_sysexit)
#endif
+ .macro pax_enter_kernel_user
+ pax_set_fptr_mask
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ call pax_enter_kernel_user
+#endif
+ .endm
+
+ .macro pax_exit_kernel_user
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ call pax_exit_kernel_user
+#endif
+#ifdef CONFIG_PAX_RANDKSTACK
+ pushq %rax
+ pushq %r11
+ call pax_randomize_kstack
+ popq %r11
+ popq %rax
+#endif
+ .endm
+
+ .macro pax_erase_kstack
+#ifdef CONFIG_PAX_MEMORY_STACKLEAK
+ call pax_erase_kstack
+#endif
+ .endm
+
/*
* 32bit SYSENTER instruction entry.
*
CFI_REGISTER rsp,rbp
SWAPGS_UNSAFE_STACK
movq PER_CPU_VAR(kernel_stack), %rsp
- addq $(KERNEL_STACK_OFFSET),%rsp
- /*
- * No need to follow this irqs on/off section: the syscall
- * disabled irqs, here we enable it straight after entry:
- */
- ENABLE_INTERRUPTS(CLBR_NONE)
movl %ebp,%ebp /* zero extension */
pushq_cfi $__USER32_DS
/*CFI_REL_OFFSET ss,0*/
CFI_REL_OFFSET rsp,0
pushfq_cfi
/*CFI_REL_OFFSET rflags,0*/
- movl TI_sysenter_return+THREAD_INFO(%rsp,3*8-KERNEL_STACK_OFFSET),%r10d
- CFI_REGISTER rip,r10
+ orl $X86_EFLAGS_IF,(%rsp)
+ GET_THREAD_INFO(%r11)
+ movl TI_sysenter_return(%r11), %r11d
+ CFI_REGISTER rip,r11
pushq_cfi $__USER32_CS
/*CFI_REL_OFFSET cs,0*/
movl %eax, %eax
- pushq_cfi %r10
+ pushq_cfi %r11
CFI_REL_OFFSET rip,0
pushq_cfi %rax
cld
SAVE_ARGS 0,1,0
+ pax_enter_kernel_user
+
+#ifdef CONFIG_PAX_RANDKSTACK
+ pax_erase_kstack
+#endif
+
+ /*
+ * No need to follow this irqs on/off section: the syscall
+ * disabled irqs, here we enable it straight after entry:
+ */
+ ENABLE_INTERRUPTS(CLBR_NONE)
/* no need to do an access_ok check here because rbp has been
32bit zero extended */
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ addq pax_user_shadow_base,%rbp
+ ASM_PAX_OPEN_USERLAND
+#endif
+
ASM_STAC
1: movl (%rbp),%ebp
_ASM_EXTABLE(1b,ia32_badarg)
ASM_CLAC
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ ASM_PAX_CLOSE_USERLAND
+#endif
+
/*
* Sysenter doesn't filter flags, so we need to clear NT
* ourselves. To save a few cycles, we can check whether
jnz sysenter_fix_flags
sysenter_flags_fixed:
- orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
- testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
+ GET_THREAD_INFO(%r11)
+ orl $TS_COMPAT,TI_status(%r11)
+ testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%r11)
CFI_REMEMBER_STATE
jnz sysenter_tracesys
cmpq $(IA32_NR_syscalls-1),%rax
sysenter_dispatch:
call *ia32_sys_call_table(,%rax,8)
movq %rax,RAX-ARGOFFSET(%rsp)
+ GET_THREAD_INFO(%r11)
DISABLE_INTERRUPTS(CLBR_NONE)
TRACE_IRQS_OFF
- testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
+ testl $_TIF_ALLWORK_MASK,TI_flags(%r11)
jnz sysexit_audit
sysexit_from_sys_call:
- andl $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
+ pax_exit_kernel_user
+ pax_erase_kstack
+ andl $~TS_COMPAT,TI_status(%r11)
/* clear IF, that popfq doesn't enable interrupts early */
- andl $~0x200,EFLAGS-R11(%rsp)
- movl RIP-R11(%rsp),%edx /* User %eip */
+ andl $~X86_EFLAGS_IF,EFLAGS(%rsp)
+ movl RIP(%rsp),%edx /* User %eip */
CFI_REGISTER rip,rdx
RESTORE_ARGS 0,24,0,0,0,0
xorq %r8,%r8
movl %ebx,%esi /* 2nd arg: 1st syscall arg */
movl %eax,%edi /* 1st arg: syscall number */
call __audit_syscall_entry
+
+ pax_erase_kstack
+
movl RAX-ARGOFFSET(%rsp),%eax /* reload syscall number */
cmpq $(IA32_NR_syscalls-1),%rax
ja ia32_badsys
.endm
.macro auditsys_exit exit
- testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
+ testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),TI_flags(%r11)
jnz ia32_ret_from_sys_call
TRACE_IRQS_ON
ENABLE_INTERRUPTS(CLBR_NONE)
1: setbe %al /* 1 if error, 0 if not */
movzbl %al,%edi /* zero-extend that into %edi */
call __audit_syscall_exit
+ GET_THREAD_INFO(%r11)
movq RAX-ARGOFFSET(%rsp),%rax /* reload syscall return value */
movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi
DISABLE_INTERRUPTS(CLBR_NONE)
TRACE_IRQS_OFF
- testl %edi,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
+ testl %edi,TI_flags(%r11)
jz \exit
CLEAR_RREGS -ARGOFFSET
jmp int_with_check
sysenter_tracesys:
#ifdef CONFIG_AUDITSYSCALL
- testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
+ testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%r11)
jz sysenter_auditsys
#endif
SAVE_REST
RESTORE_REST
cmpq $(IA32_NR_syscalls-1),%rax
ja int_ret_from_sys_call /* sysenter_tracesys has set RAX(%rsp) */
+
+ pax_erase_kstack
+
jmp sysenter_do_call
CFI_ENDPROC
ENDPROC(ia32_sysenter_target)
ENTRY(ia32_cstar_target)
CFI_STARTPROC32 simple
CFI_SIGNAL_FRAME
- CFI_DEF_CFA rsp,KERNEL_STACK_OFFSET
+ CFI_DEF_CFA rsp,0
CFI_REGISTER rip,rcx
/*CFI_REGISTER rflags,r11*/
SWAPGS_UNSAFE_STACK
movl %esp,%r8d
CFI_REGISTER rsp,r8
movq PER_CPU_VAR(kernel_stack),%rsp
+ SAVE_ARGS 8*6,0,0
+ pax_enter_kernel_user
+
+#ifdef CONFIG_PAX_RANDKSTACK
+ pax_erase_kstack
+#endif
+
/*
* No need to follow this irqs on/off section: the syscall
* disabled irqs and here we enable it straight after entry:
*/
ENABLE_INTERRUPTS(CLBR_NONE)
- SAVE_ARGS 8,0,0
movl %eax,%eax /* zero extension */
movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
movq %rcx,RIP-ARGOFFSET(%rsp)
/* no need to do an access_ok check here because r8 has been
32bit zero extended */
/* hardware stack frame is complete now */
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ ASM_PAX_OPEN_USERLAND
+ movq pax_user_shadow_base,%r8
+ addq RSP-ARGOFFSET(%rsp),%r8
+#endif
+
ASM_STAC
1: movl (%r8),%r9d
_ASM_EXTABLE(1b,ia32_badarg)
ASM_CLAC
- orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
- testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ ASM_PAX_CLOSE_USERLAND
+#endif
+
+ GET_THREAD_INFO(%r11)
+ orl $TS_COMPAT,TI_status(%r11)
+ testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%r11)
CFI_REMEMBER_STATE
jnz cstar_tracesys
cmpq $IA32_NR_syscalls-1,%rax
cstar_dispatch:
call *ia32_sys_call_table(,%rax,8)
movq %rax,RAX-ARGOFFSET(%rsp)
+ GET_THREAD_INFO(%r11)
DISABLE_INTERRUPTS(CLBR_NONE)
TRACE_IRQS_OFF
- testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
+ testl $_TIF_ALLWORK_MASK,TI_flags(%r11)
jnz sysretl_audit
sysretl_from_sys_call:
- andl $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
- RESTORE_ARGS 0,-ARG_SKIP,0,0,0
+ pax_exit_kernel_user
+ pax_erase_kstack
+ andl $~TS_COMPAT,TI_status(%r11)
+ RESTORE_ARGS 0,-ORIG_RAX,0,0,0
movl RIP-ARGOFFSET(%rsp),%ecx
CFI_REGISTER rip,rcx
movl EFLAGS-ARGOFFSET(%rsp),%r11d
cstar_tracesys:
#ifdef CONFIG_AUDITSYSCALL
- testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
+ testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%r11)
jz cstar_auditsys
#endif
xchgl %r9d,%ebp
xchgl %ebp,%r9d
cmpq $(IA32_NR_syscalls-1),%rax
ja int_ret_from_sys_call /* cstar_tracesys has set RAX(%rsp) */
+
+ pax_erase_kstack
+
jmp cstar_do_call
END(ia32_cstar_target)
ia32_badarg:
ASM_CLAC
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ ASM_PAX_CLOSE_USERLAND
+#endif
+
movq $-EFAULT,%rax
jmp ia32_sysret
CFI_ENDPROC
CFI_REL_OFFSET rip,RIP-RIP
PARAVIRT_ADJUST_EXCEPTION_FRAME
SWAPGS
- /*
- * No need to follow this irqs on/off section: the syscall
- * disabled irqs and here we enable it straight after entry:
- */
- ENABLE_INTERRUPTS(CLBR_NONE)
movl %eax,%eax
pushq_cfi %rax
cld
/* note the registers are not zero extended to the sf.
this could be a problem. */
SAVE_ARGS 0,1,0
- orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
- testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
+ pax_enter_kernel_user
+
+#ifdef CONFIG_PAX_RANDKSTACK
+ pax_erase_kstack
+#endif
+
+ /*
+ * No need to follow this irqs on/off section: the syscall
+ * disabled irqs and here we enable it straight after entry:
+ */
+ ENABLE_INTERRUPTS(CLBR_NONE)
+ GET_THREAD_INFO(%r11)
+ orl $TS_COMPAT,TI_status(%r11)
+ testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%r11)
jnz ia32_tracesys
cmpq $(IA32_NR_syscalls-1),%rax
ja ia32_badsys
RESTORE_REST
cmpq $(IA32_NR_syscalls-1),%rax
ja int_ret_from_sys_call /* ia32_tracesys has set RAX(%rsp) */
+
+ pax_erase_kstack
+
jmp ia32_do_call
END(ia32_syscall)
*/
static int cp_stat64(struct stat64 __user *ubuf, struct kstat *stat)
{
- typeof(ubuf->st_uid) uid = 0;
- typeof(ubuf->st_gid) gid = 0;
+ typeof(((struct stat64 *)0)->st_uid) uid = 0;
+ typeof(((struct stat64 *)0)->st_gid) gid = 0;
SET_UID(uid, from_kuid_munged(current_user_ns(), stat->uid));
SET_GID(gid, from_kgid_munged(current_user_ns(), stat->gid));
if (!access_ok(VERIFY_WRITE, ubuf, sizeof(struct stat64)) ||
.endm
#endif
+#ifdef KERNEXEC_PLUGIN
+ .macro pax_force_retaddr_bts rip=0
+ btsq $63,\rip(%rsp)
+ .endm
+#ifdef CONFIG_PAX_KERNEXEC_PLUGIN_METHOD_BTS
+ .macro pax_force_retaddr rip=0, reload=0
+ btsq $63,\rip(%rsp)
+ .endm
+ .macro pax_force_fptr ptr
+ btsq $63,\ptr
+ .endm
+ .macro pax_set_fptr_mask
+ .endm
+#endif
+#ifdef CONFIG_PAX_KERNEXEC_PLUGIN_METHOD_OR
+ .macro pax_force_retaddr rip=0, reload=0
+ .if \reload
+ pax_set_fptr_mask
+ .endif
+ orq %r12,\rip(%rsp)
+ .endm
+ .macro pax_force_fptr ptr
+ orq %r12,\ptr
+ .endm
+ .macro pax_set_fptr_mask
+ movabs $0x8000000000000000,%r12
+ .endm
+#endif
+#else
+ .macro pax_force_retaddr rip=0, reload=0
+ .endm
+ .macro pax_force_fptr ptr
+ .endm
+ .macro pax_force_retaddr_bts rip=0
+ .endm
+ .macro pax_set_fptr_mask
+ .endm
+#endif
+
.macro altinstruction_entry orig alt feature orig_len alt_len
.long \orig - .
.long \alt - .
".pushsection .discard,\"aw\",@progbits\n" \
DISCARD_ENTRY(1) \
".popsection\n" \
- ".pushsection .altinstr_replacement, \"ax\"\n" \
+ ".pushsection .altinstr_replacement, \"a\"\n" \
ALTINSTR_REPLACEMENT(newinstr, feature, 1) \
".popsection"
DISCARD_ENTRY(1) \
DISCARD_ENTRY(2) \
".popsection\n" \
- ".pushsection .altinstr_replacement, \"ax\"\n" \
+ ".pushsection .altinstr_replacement, \"a\"\n" \
ALTINSTR_REPLACEMENT(newinstr1, feature1, 1) \
ALTINSTR_REPLACEMENT(newinstr2, feature2, 2) \
".popsection"
#ifdef CONFIG_X86_LOCAL_APIC
-extern unsigned int apic_verbosity;
+extern int apic_verbosity;
extern int local_apic_timer_c2_ok;
extern int disable_apic;
__asm__ __volatile__(APM_DO_ZERO_SEGS
"pushl %%edi\n\t"
"pushl %%ebp\n\t"
- "lcall *%%cs:apm_bios_entry\n\t"
+ "lcall *%%ss:apm_bios_entry\n\t"
"setc %%al\n\t"
"popl %%ebp\n\t"
"popl %%edi\n\t"
__asm__ __volatile__(APM_DO_ZERO_SEGS
"pushl %%edi\n\t"
"pushl %%ebp\n\t"
- "lcall *%%cs:apm_bios_entry\n\t"
+ "lcall *%%ss:apm_bios_entry\n\t"
"setc %%bl\n\t"
"popl %%ebp\n\t"
"popl %%edi\n\t"
return ACCESS_ONCE((v)->counter);
}
+/**
+ * atomic_read_unchecked - read atomic variable
+ * @v: pointer of type atomic_unchecked_t
+ *
+ * Atomically reads the value of @v.
+ */
+static inline int __intentional_overflow(-1) atomic_read_unchecked(const atomic_unchecked_t *v)
+{
+ return ACCESS_ONCE((v)->counter);
+}
+
/**
* atomic_set - set atomic variable
* @v: pointer of type atomic_t
v->counter = i;
}
+/**
+ * atomic_set_unchecked - set atomic variable
+ * @v: pointer of type atomic_unchecked_t
+ * @i: required value
+ *
+ * Atomically sets the value of @v to @i.
+ */
+static inline void atomic_set_unchecked(atomic_unchecked_t *v, int i)
+{
+ v->counter = i;
+}
+
/**
* atomic_add - add integer to atomic variable
* @i: integer value to add
*/
static inline void atomic_add(int i, atomic_t *v)
{
- asm volatile(LOCK_PREFIX "addl %1,%0"
+ asm volatile(LOCK_PREFIX "addl %1,%0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ LOCK_PREFIX "subl %1,%0\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+ : "+m" (v->counter)
+ : "ir" (i));
+}
+
+/**
+ * atomic_add_unchecked - add integer to atomic variable
+ * @i: integer value to add
+ * @v: pointer of type atomic_unchecked_t
+ *
+ * Atomically adds @i to @v.
+ */
+static inline void atomic_add_unchecked(int i, atomic_unchecked_t *v)
+{
+ asm volatile(LOCK_PREFIX "addl %1,%0\n"
: "+m" (v->counter)
: "ir" (i));
}
*/
static inline void atomic_sub(int i, atomic_t *v)
{
- asm volatile(LOCK_PREFIX "subl %1,%0"
+ asm volatile(LOCK_PREFIX "subl %1,%0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ LOCK_PREFIX "addl %1,%0\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+ : "+m" (v->counter)
+ : "ir" (i));
+}
+
+/**
+ * atomic_sub_unchecked - subtract integer from atomic variable
+ * @i: integer value to subtract
+ * @v: pointer of type atomic_unchecked_t
+ *
+ * Atomically subtracts @i from @v.
+ */
+static inline void atomic_sub_unchecked(int i, atomic_unchecked_t *v)
+{
+ asm volatile(LOCK_PREFIX "subl %1,%0\n"
: "+m" (v->counter)
: "ir" (i));
}
*/
static inline int atomic_sub_and_test(int i, atomic_t *v)
{
- GEN_BINARY_RMWcc(LOCK_PREFIX "subl", v->counter, "er", i, "%0", "e");
+ GEN_BINARY_RMWcc(LOCK_PREFIX "subl", LOCK_PREFIX "addl", v->counter, "er", i, "%0", "e");
}
/**
*/
static inline void atomic_inc(atomic_t *v)
{
- asm volatile(LOCK_PREFIX "incl %0"
+ asm volatile(LOCK_PREFIX "incl %0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ LOCK_PREFIX "decl %0\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+ : "+m" (v->counter));
+}
+
+/**
+ * atomic_inc_unchecked - increment atomic variable
+ * @v: pointer of type atomic_unchecked_t
+ *
+ * Atomically increments @v by 1.
+ */
+static inline void atomic_inc_unchecked(atomic_unchecked_t *v)
+{
+ asm volatile(LOCK_PREFIX "incl %0\n"
: "+m" (v->counter));
}
*/
static inline void atomic_dec(atomic_t *v)
{
- asm volatile(LOCK_PREFIX "decl %0"
+ asm volatile(LOCK_PREFIX "decl %0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ LOCK_PREFIX "incl %0\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+ : "+m" (v->counter));
+}
+
+/**
+ * atomic_dec_unchecked - decrement atomic variable
+ * @v: pointer of type atomic_unchecked_t
+ *
+ * Atomically decrements @v by 1.
+ */
+static inline void atomic_dec_unchecked(atomic_unchecked_t *v)
+{
+ asm volatile(LOCK_PREFIX "decl %0\n"
: "+m" (v->counter));
}
*/
static inline int atomic_dec_and_test(atomic_t *v)
{
- GEN_UNARY_RMWcc(LOCK_PREFIX "decl", v->counter, "%0", "e");
+ GEN_UNARY_RMWcc(LOCK_PREFIX "decl", LOCK_PREFIX "incl", v->counter, "%0", "e");
}
/**
*/
static inline int atomic_inc_and_test(atomic_t *v)
{
- GEN_UNARY_RMWcc(LOCK_PREFIX "incl", v->counter, "%0", "e");
+ GEN_UNARY_RMWcc(LOCK_PREFIX "incl", LOCK_PREFIX "decl", v->counter, "%0", "e");
+}
+
+/**
+ * atomic_inc_and_test_unchecked - increment and test
+ * @v: pointer of type atomic_unchecked_t
+ *
+ * Atomically increments @v by 1
+ * and returns true if the result is zero, or false for all
+ * other cases.
+ */
+static inline int atomic_inc_and_test_unchecked(atomic_unchecked_t *v)
+{
+ GEN_UNARY_RMWcc_unchecked(LOCK_PREFIX "incl", v->counter, "%0", "e");
}
/**
*/
static inline int atomic_add_negative(int i, atomic_t *v)
{
- GEN_BINARY_RMWcc(LOCK_PREFIX "addl", v->counter, "er", i, "%0", "s");
+ GEN_BINARY_RMWcc(LOCK_PREFIX "addl", LOCK_PREFIX "subl", v->counter, "er", i, "%0", "s");
}
/**
*
* Atomically adds @i to @v and returns @i + @v
*/
-static inline int atomic_add_return(int i, atomic_t *v)
+static inline int __intentional_overflow(-1) atomic_add_return(int i, atomic_t *v)
+{
+ return i + xadd_check_overflow(&v->counter, i);
+}
+
+/**
+ * atomic_add_return_unchecked - add integer and return
+ * @i: integer value to add
+ * @v: pointer of type atomic_unchecked_t
+ *
+ * Atomically adds @i to @v and returns @i + @v
+ */
+static inline int atomic_add_return_unchecked(int i, atomic_unchecked_t *v)
{
return i + xadd(&v->counter, i);
}
*
* Atomically subtracts @i from @v and returns @v - @i
*/
-static inline int atomic_sub_return(int i, atomic_t *v)
+static inline int __intentional_overflow(-1) atomic_sub_return(int i, atomic_t *v)
{
return atomic_add_return(-i, v);
}
#define atomic_inc_return(v) (atomic_add_return(1, v))
+static inline int atomic_inc_return_unchecked(atomic_unchecked_t *v)
+{
+ return atomic_add_return_unchecked(1, v);
+}
#define atomic_dec_return(v) (atomic_sub_return(1, v))
-static inline int atomic_cmpxchg(atomic_t *v, int old, int new)
+static inline int __intentional_overflow(-1) atomic_cmpxchg(atomic_t *v, int old, int new)
+{
+ return cmpxchg(&v->counter, old, new);
+}
+
+static inline int atomic_cmpxchg_unchecked(atomic_unchecked_t *v, int old, int new)
{
return cmpxchg(&v->counter, old, new);
}
return xchg(&v->counter, new);
}
+static inline int atomic_xchg_unchecked(atomic_unchecked_t *v, int new)
+{
+ return xchg(&v->counter, new);
+}
+
/**
* __atomic_add_unless - add unless the number is already a given value
* @v: pointer of type atomic_t
*/
static inline int __atomic_add_unless(atomic_t *v, int a, int u)
{
- int c, old;
+ int c, old, new;
c = atomic_read(v);
for (;;) {
- if (unlikely(c == (u)))
+ if (unlikely(c == u))
break;
- old = atomic_cmpxchg((v), c, c + (a));
+
+ asm volatile("addl %2,%0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ "subl %2,%0\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+ : "=r" (new)
+ : "0" (c), "ir" (a));
+
+ old = atomic_cmpxchg(v, c, new);
if (likely(old == c))
break;
c = old;
return c;
}
+/**
+ * atomic_inc_not_zero_hint - increment if not null
+ * @v: pointer of type atomic_t
+ * @hint: probable value of the atomic before the increment
+ *
+ * This version of atomic_inc_not_zero() gives a hint of probable
+ * value of the atomic. This helps processor to not read the memory
+ * before doing the atomic read/modify/write cycle, lowering
+ * number of bus transactions on some arches.
+ *
+ * Returns: 0 if increment was not done, 1 otherwise.
+ */
+#define atomic_inc_not_zero_hint atomic_inc_not_zero_hint
+static inline int atomic_inc_not_zero_hint(atomic_t *v, int hint)
+{
+ int val, c = hint, new;
+
+ /* sanity test, should be removed by compiler if hint is a constant */
+ if (!hint)
+ return __atomic_add_unless(v, 1, 0);
+
+ do {
+ asm volatile("incl %0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ "decl %0\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+ : "=r" (new)
+ : "0" (c));
+
+ val = atomic_cmpxchg(v, c, new);
+ if (val == c)
+ return 1;
+ c = val;
+ } while (c);
+
+ return 0;
+}
+
/**
* atomic_inc_short - increment of a short integer
* @v: pointer to type int
}
/* These are x86-specific, used by some header files */
-#define atomic_clear_mask(mask, addr) \
- asm volatile(LOCK_PREFIX "andl %0,%1" \
- : : "r" (~(mask)), "m" (*(addr)) : "memory")
-
-#define atomic_set_mask(mask, addr) \
- asm volatile(LOCK_PREFIX "orl %0,%1" \
- : : "r" ((unsigned)(mask)), "m" (*(addr)) \
- : "memory")
+static inline void atomic_clear_mask(unsigned int mask, atomic_t *v)
+{
+ asm volatile(LOCK_PREFIX "andl %1,%0"
+ : "+m" (v->counter)
+ : "r" (~(mask))
+ : "memory");
+}
+
+static inline void atomic_clear_mask_unchecked(unsigned int mask, atomic_unchecked_t *v)
+{
+ asm volatile(LOCK_PREFIX "andl %1,%0"
+ : "+m" (v->counter)
+ : "r" (~(mask))
+ : "memory");
+}
+
+static inline void atomic_set_mask(unsigned int mask, atomic_t *v)
+{
+ asm volatile(LOCK_PREFIX "orl %1,%0"
+ : "+m" (v->counter)
+ : "r" (mask)
+ : "memory");
+}
+
+static inline void atomic_set_mask_unchecked(unsigned int mask, atomic_unchecked_t *v)
+{
+ asm volatile(LOCK_PREFIX "orl %1,%0"
+ : "+m" (v->counter)
+ : "r" (mask)
+ : "memory");
+}
#ifdef CONFIG_X86_32
# include <asm/atomic64_32.h>
u64 __aligned(8) counter;
} atomic64_t;
+#ifdef CONFIG_PAX_REFCOUNT
+typedef struct {
+ u64 __aligned(8) counter;
+} atomic64_unchecked_t;
+#else
+typedef atomic64_t atomic64_unchecked_t;
+#endif
+
#define ATOMIC64_INIT(val) { (val) }
#define __ATOMIC64_DECL(sym) void atomic64_##sym(atomic64_t *, ...)
ATOMIC64_DECL_ONE(sym##_386)
ATOMIC64_DECL_ONE(add_386);
+ATOMIC64_DECL_ONE(add_unchecked_386);
ATOMIC64_DECL_ONE(sub_386);
+ATOMIC64_DECL_ONE(sub_unchecked_386);
ATOMIC64_DECL_ONE(inc_386);
+ATOMIC64_DECL_ONE(inc_unchecked_386);
ATOMIC64_DECL_ONE(dec_386);
+ATOMIC64_DECL_ONE(dec_unchecked_386);
#endif
#define alternative_atomic64(f, out, in...) \
__alternative_atomic64(f, f, ASM_OUTPUT2(out), ## in)
ATOMIC64_DECL(read);
+ATOMIC64_DECL(read_unchecked);
ATOMIC64_DECL(set);
+ATOMIC64_DECL(set_unchecked);
ATOMIC64_DECL(xchg);
ATOMIC64_DECL(add_return);
+ATOMIC64_DECL(add_return_unchecked);
ATOMIC64_DECL(sub_return);
+ATOMIC64_DECL(sub_return_unchecked);
ATOMIC64_DECL(inc_return);
+ATOMIC64_DECL(inc_return_unchecked);
ATOMIC64_DECL(dec_return);
+ATOMIC64_DECL(dec_return_unchecked);
ATOMIC64_DECL(dec_if_positive);
ATOMIC64_DECL(inc_not_zero);
ATOMIC64_DECL(add_unless);
return cmpxchg64(&v->counter, o, n);
}
+/**
+ * atomic64_cmpxchg_unchecked - cmpxchg atomic64 variable
+ * @p: pointer to type atomic64_unchecked_t
+ * @o: expected value
+ * @n: new value
+ *
+ * Atomically sets @v to @n if it was equal to @o and returns
+ * the old value.
+ */
+
+static inline long long atomic64_cmpxchg_unchecked(atomic64_unchecked_t *v, long long o, long long n)
+{
+ return cmpxchg64(&v->counter, o, n);
+}
+
/**
* atomic64_xchg - xchg atomic64 variable
* @v: pointer to type atomic64_t
: "eax", "edx", "memory");
}
+/**
+ * atomic64_set_unchecked - set atomic64 variable
+ * @v: pointer to type atomic64_unchecked_t
+ * @n: value to assign
+ *
+ * Atomically sets the value of @v to @n.
+ */
+static inline void atomic64_set_unchecked(atomic64_unchecked_t *v, long long i)
+{
+ unsigned high = (unsigned)(i >> 32);
+ unsigned low = (unsigned)i;
+ alternative_atomic64(set, /* no output */,
+ "S" (v), "b" (low), "c" (high)
+ : "eax", "edx", "memory");
+}
+
/**
* atomic64_read - read atomic64 variable
* @v: pointer to type atomic64_t
return r;
}
+/**
+ * atomic64_read_unchecked - read atomic64 variable
+ * @v: pointer to type atomic64_unchecked_t
+ *
+ * Atomically reads the value of @v and returns it.
+ */
+static inline long long __intentional_overflow(-1) atomic64_read_unchecked(atomic64_unchecked_t *v)
+{
+ long long r;
+ alternative_atomic64(read, "=&A" (r), "c" (v) : "memory");
+ return r;
+ }
+
/**
* atomic64_add_return - add and return
* @i: integer value to add
return i;
}
+/**
+ * atomic64_add_return_unchecked - add and return
+ * @i: integer value to add
+ * @v: pointer to type atomic64_unchecked_t
+ *
+ * Atomically adds @i to @v and returns @i + *@v
+ */
+static inline long long atomic64_add_return_unchecked(long long i, atomic64_unchecked_t *v)
+{
+ alternative_atomic64(add_return_unchecked,
+ ASM_OUTPUT2("+A" (i), "+c" (v)),
+ ASM_NO_INPUT_CLOBBER("memory"));
+ return i;
+}
+
/*
* Other variants with different arithmetic operators:
*/
return a;
}
+static inline long long atomic64_inc_return_unchecked(atomic64_unchecked_t *v)
+{
+ long long a;
+ alternative_atomic64(inc_return_unchecked, "=&A" (a),
+ "S" (v) : "memory", "ecx");
+ return a;
+}
+
static inline long long atomic64_dec_return(atomic64_t *v)
{
long long a;
return i;
}
+/**
+ * atomic64_add_unchecked - add integer to atomic64 variable
+ * @i: integer value to add
+ * @v: pointer to type atomic64_unchecked_t
+ *
+ * Atomically adds @i to @v.
+ */
+static inline long long atomic64_add_unchecked(long long i, atomic64_unchecked_t *v)
+{
+ __alternative_atomic64(add_unchecked, add_return_unchecked,
+ ASM_OUTPUT2("+A" (i), "+c" (v)),
+ ASM_NO_INPUT_CLOBBER("memory"));
+ return i;
+}
+
/**
* atomic64_sub - subtract the atomic64 variable
* @i: integer value to subtract
return ACCESS_ONCE((v)->counter);
}
+/**
+ * atomic64_read_unchecked - read atomic64 variable
+ * @v: pointer of type atomic64_unchecked_t
+ *
+ * Atomically reads the value of @v.
+ * Doesn't imply a read memory barrier.
+ */
+static inline long __intentional_overflow(-1) atomic64_read_unchecked(const atomic64_unchecked_t *v)
+{
+ return ACCESS_ONCE((v)->counter);
+}
+
/**
* atomic64_set - set atomic64 variable
* @v: pointer to type atomic64_t
v->counter = i;
}
+/**
+ * atomic64_set_unchecked - set atomic64 variable
+ * @v: pointer to type atomic64_unchecked_t
+ * @i: required value
+ *
+ * Atomically sets the value of @v to @i.
+ */
+static inline void atomic64_set_unchecked(atomic64_unchecked_t *v, long i)
+{
+ v->counter = i;
+}
+
/**
* atomic64_add - add integer to atomic64 variable
* @i: integer value to add
* Atomically adds @i to @v.
*/
static inline void atomic64_add(long i, atomic64_t *v)
+{
+ asm volatile(LOCK_PREFIX "addq %1,%0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ LOCK_PREFIX "subq %1,%0\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+ : "=m" (v->counter)
+ : "er" (i), "m" (v->counter));
+}
+
+/**
+ * atomic64_add_unchecked - add integer to atomic64 variable
+ * @i: integer value to add
+ * @v: pointer to type atomic64_unchecked_t
+ *
+ * Atomically adds @i to @v.
+ */
+static inline void atomic64_add_unchecked(long i, atomic64_unchecked_t *v)
{
asm volatile(LOCK_PREFIX "addq %1,%0"
: "=m" (v->counter)
*/
static inline void atomic64_sub(long i, atomic64_t *v)
{
- asm volatile(LOCK_PREFIX "subq %1,%0"
+ asm volatile(LOCK_PREFIX "subq %1,%0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ LOCK_PREFIX "addq %1,%0\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+ : "=m" (v->counter)
+ : "er" (i), "m" (v->counter));
+}
+
+/**
+ * atomic64_sub_unchecked - subtract the atomic64 variable
+ * @i: integer value to subtract
+ * @v: pointer to type atomic64_unchecked_t
+ *
+ * Atomically subtracts @i from @v.
+ */
+static inline void atomic64_sub_unchecked(long i, atomic64_unchecked_t *v)
+{
+ asm volatile(LOCK_PREFIX "subq %1,%0\n"
: "=m" (v->counter)
: "er" (i), "m" (v->counter));
}
*/
static inline int atomic64_sub_and_test(long i, atomic64_t *v)
{
- GEN_BINARY_RMWcc(LOCK_PREFIX "subq", v->counter, "er", i, "%0", "e");
+ GEN_BINARY_RMWcc(LOCK_PREFIX "subq", LOCK_PREFIX "addq", v->counter, "er", i, "%0", "e");
}
/**
* Atomically increments @v by 1.
*/
static inline void atomic64_inc(atomic64_t *v)
+{
+ asm volatile(LOCK_PREFIX "incq %0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ LOCK_PREFIX "decq %0\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+ : "=m" (v->counter)
+ : "m" (v->counter));
+}
+
+/**
+ * atomic64_inc_unchecked - increment atomic64 variable
+ * @v: pointer to type atomic64_unchecked_t
+ *
+ * Atomically increments @v by 1.
+ */
+static inline void atomic64_inc_unchecked(atomic64_unchecked_t *v)
{
asm volatile(LOCK_PREFIX "incq %0"
: "=m" (v->counter)
*/
static inline void atomic64_dec(atomic64_t *v)
{
- asm volatile(LOCK_PREFIX "decq %0"
+ asm volatile(LOCK_PREFIX "decq %0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ LOCK_PREFIX "incq %0\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+ : "=m" (v->counter)
+ : "m" (v->counter));
+}
+
+/**
+ * atomic64_dec_unchecked - decrement atomic64 variable
+ * @v: pointer to type atomic64_t
+ *
+ * Atomically decrements @v by 1.
+ */
+static inline void atomic64_dec_unchecked(atomic64_unchecked_t *v)
+{
+ asm volatile(LOCK_PREFIX "decq %0\n"
: "=m" (v->counter)
: "m" (v->counter));
}
*/
static inline int atomic64_dec_and_test(atomic64_t *v)
{
- GEN_UNARY_RMWcc(LOCK_PREFIX "decq", v->counter, "%0", "e");
+ GEN_UNARY_RMWcc(LOCK_PREFIX "decq", LOCK_PREFIX "incq", v->counter, "%0", "e");
}
/**
*/
static inline int atomic64_inc_and_test(atomic64_t *v)
{
- GEN_UNARY_RMWcc(LOCK_PREFIX "incq", v->counter, "%0", "e");
+ GEN_UNARY_RMWcc(LOCK_PREFIX "incq", LOCK_PREFIX "decq", v->counter, "%0", "e");
}
/**
*/
static inline int atomic64_add_negative(long i, atomic64_t *v)
{
- GEN_BINARY_RMWcc(LOCK_PREFIX "addq", v->counter, "er", i, "%0", "s");
+ GEN_BINARY_RMWcc(LOCK_PREFIX "addq", LOCK_PREFIX "subq", v->counter, "er", i, "%0", "s");
}
/**
* Atomically adds @i to @v and returns @i + @v
*/
static inline long atomic64_add_return(long i, atomic64_t *v)
+{
+ return i + xadd_check_overflow(&v->counter, i);
+}
+
+/**
+ * atomic64_add_return_unchecked - add and return
+ * @i: integer value to add
+ * @v: pointer to type atomic64_unchecked_t
+ *
+ * Atomically adds @i to @v and returns @i + @v
+ */
+static inline long atomic64_add_return_unchecked(long i, atomic64_unchecked_t *v)
{
return i + xadd(&v->counter, i);
}
}
#define atomic64_inc_return(v) (atomic64_add_return(1, (v)))
+static inline long atomic64_inc_return_unchecked(atomic64_unchecked_t *v)
+{
+ return atomic64_add_return_unchecked(1, v);
+}
#define atomic64_dec_return(v) (atomic64_sub_return(1, (v)))
static inline long atomic64_cmpxchg(atomic64_t *v, long old, long new)
return cmpxchg(&v->counter, old, new);
}
+static inline long atomic64_cmpxchg_unchecked(atomic64_unchecked_t *v, long old, long new)
+{
+ return cmpxchg(&v->counter, old, new);
+}
+
static inline long atomic64_xchg(atomic64_t *v, long new)
{
return xchg(&v->counter, new);
*/
static inline int atomic64_add_unless(atomic64_t *v, long a, long u)
{
- long c, old;
+ long c, old, new;
c = atomic64_read(v);
for (;;) {
- if (unlikely(c == (u)))
+ if (unlikely(c == u))
break;
- old = atomic64_cmpxchg((v), c, c + (a));
+
+ asm volatile("add %2,%0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ "sub %2,%0\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+ : "=r" (new)
+ : "0" (c), "ir" (a));
+
+ old = atomic64_cmpxchg(v, c, new);
if (likely(old == c))
break;
c = old;
}
- return c != (u);
+ return c != u;
}
#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0)
do { \
compiletime_assert_atomic_type(*p); \
smp_mb(); \
- ACCESS_ONCE(*p) = (v); \
+ ACCESS_ONCE_RW(*p) = (v); \
} while (0)
#define smp_load_acquire(p) \
do { \
compiletime_assert_atomic_type(*p); \
barrier(); \
- ACCESS_ONCE(*p) = (v); \
+ ACCESS_ONCE_RW(*p) = (v); \
} while (0)
#define smp_load_acquire(p) \
* a mask operation on a byte.
*/
#define IS_IMMEDIATE(nr) (__builtin_constant_p(nr))
-#define CONST_MASK_ADDR(nr, addr) BITOP_ADDR((void *)(addr) + ((nr)>>3))
+#define CONST_MASK_ADDR(nr, addr) BITOP_ADDR((volatile void *)(addr) + ((nr)>>3))
#define CONST_MASK(nr) (1 << ((nr) & 7))
/**
*/
static inline int test_and_set_bit(long nr, volatile unsigned long *addr)
{
- GEN_BINARY_RMWcc(LOCK_PREFIX "bts", *addr, "Ir", nr, "%0", "c");
+ GEN_BINARY_RMWcc_unchecked(LOCK_PREFIX "bts", *addr, "Ir", nr, "%0", "c");
}
/**
*/
static inline int test_and_clear_bit(long nr, volatile unsigned long *addr)
{
- GEN_BINARY_RMWcc(LOCK_PREFIX "btr", *addr, "Ir", nr, "%0", "c");
+ GEN_BINARY_RMWcc_unchecked(LOCK_PREFIX "btr", *addr, "Ir", nr, "%0", "c");
}
/**
*/
static inline int test_and_change_bit(long nr, volatile unsigned long *addr)
{
- GEN_BINARY_RMWcc(LOCK_PREFIX "btc", *addr, "Ir", nr, "%0", "c");
+ GEN_BINARY_RMWcc_unchecked(LOCK_PREFIX "btc", *addr, "Ir", nr, "%0", "c");
}
static __always_inline int constant_test_bit(long nr, const volatile unsigned long *addr)
*
* Undefined if no bit exists, so code should check against 0 first.
*/
-static inline unsigned long __ffs(unsigned long word)
+static inline unsigned long __intentional_overflow(-1) __ffs(unsigned long word)
{
asm("rep; bsf %1,%0"
: "=r" (word)
*
* Undefined if no zero exists, so code should check against ~0UL first.
*/
-static inline unsigned long ffz(unsigned long word)
+static inline unsigned long __intentional_overflow(-1) ffz(unsigned long word)
{
asm("rep; bsf %1,%0"
: "=r" (word)
*
* Undefined if no set bit exists, so code should check against 0 first.
*/
-static inline unsigned long __fls(unsigned long word)
+static inline unsigned long __intentional_overflow(-1) __fls(unsigned long word)
{
asm("bsr %1,%0"
: "=r" (word)
* set bit if value is nonzero. The last (most significant) bit is
* at position 32.
*/
-static inline int fls(int x)
+static inline int __intentional_overflow(-1) fls(int x)
{
int r;
* at position 64.
*/
#ifdef CONFIG_X86_64
-static __always_inline int fls64(__u64 x)
+static __always_inline __intentional_overflow(-1) int fls64(__u64 x)
{
int bitpos = -1;
/*
#include <uapi/asm/boot.h>
/* Physical address where kernel should be loaded. */
-#define LOAD_PHYSICAL_ADDR ((CONFIG_PHYSICAL_START \
+#define ____LOAD_PHYSICAL_ADDR ((CONFIG_PHYSICAL_START \
+ (CONFIG_PHYSICAL_ALIGN - 1)) \
& ~(CONFIG_PHYSICAL_ALIGN - 1))
+#ifndef __ASSEMBLY__
+extern unsigned char __LOAD_PHYSICAL_ADDR[];
+#define LOAD_PHYSICAL_ADDR ((unsigned long)__LOAD_PHYSICAL_ADDR)
+#endif
+
/* Minimum kernel alignment, as a power of two */
#ifdef CONFIG_X86_64
#define MIN_KERNEL_ALIGN_LG2 PMD_SHIFT
/* L1 cache line size */
#define L1_CACHE_SHIFT (CONFIG_X86_L1_CACHE_SHIFT)
-#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT)
+#define L1_CACHE_BYTES (_AC(1,UL) << L1_CACHE_SHIFT)
#define __read_mostly __attribute__((__section__(".data..read_mostly")))
+#define __read_only __attribute__((__section__(".data..read_only")))
#define INTERNODE_CACHE_SHIFT CONFIG_X86_INTERNODE_CACHE_SHIFT
-#define INTERNODE_CACHE_BYTES (1 << INTERNODE_CACHE_SHIFT)
+#define INTERNODE_CACHE_BYTES (_AC(1,UL) << INTERNODE_CACHE_SHIFT)
#ifdef CONFIG_X86_VSMP
#ifdef CONFIG_SMP
#define RSP 152
#define SS 160
-#define ARGOFFSET R11
-#define SWFRAME ORIG_RAX
+#define ARGOFFSET R15
.macro SAVE_ARGS addskip=0, save_rcx=1, save_r891011=1, rax_enosys=0
- subq $9*8+\addskip, %rsp
- CFI_ADJUST_CFA_OFFSET 9*8+\addskip
- movq_cfi rdi, 8*8
- movq_cfi rsi, 7*8
- movq_cfi rdx, 6*8
+ subq $ORIG_RAX-ARGOFFSET+\addskip, %rsp
+ CFI_ADJUST_CFA_OFFSET ORIG_RAX-ARGOFFSET+\addskip
+ movq_cfi rdi, RDI
+ movq_cfi rsi, RSI
+ movq_cfi rdx, RDX
.if \save_rcx
- movq_cfi rcx, 5*8
+ movq_cfi rcx, RCX
.endif
.if \rax_enosys
- movq $-ENOSYS, 4*8(%rsp)
+ movq $-ENOSYS, RAX(%rsp)
.else
- movq_cfi rax, 4*8
+ movq_cfi rax, RAX
.endif
.if \save_r891011
- movq_cfi r8, 3*8
- movq_cfi r9, 2*8
- movq_cfi r10, 1*8
- movq_cfi r11, 0*8
+ movq_cfi r8, R8
+ movq_cfi r9, R9
+ movq_cfi r10, R10
+ movq_cfi r11, R11
.endif
+#ifdef CONFIG_PAX_KERNEXEC_PLUGIN_METHOD_OR
+ movq_cfi r12, R12
+#endif
+
.endm
-#define ARG_SKIP (9*8)
+#define ARG_SKIP ORIG_RAX
.macro RESTORE_ARGS rstor_rax=1, addskip=0, rstor_rcx=1, rstor_r11=1, \
rstor_r8910=1, rstor_rdx=1
+
+#ifdef CONFIG_PAX_KERNEXEC_PLUGIN_METHOD_OR
+ movq_cfi_restore R12, r12
+#endif
+
.if \rstor_r11
- movq_cfi_restore 0*8, r11
+ movq_cfi_restore R11, r11
.endif
.if \rstor_r8910
- movq_cfi_restore 1*8, r10
- movq_cfi_restore 2*8, r9
- movq_cfi_restore 3*8, r8
+ movq_cfi_restore R10, r10
+ movq_cfi_restore R9, r9
+ movq_cfi_restore R8, r8
.endif
.if \rstor_rax
- movq_cfi_restore 4*8, rax
+ movq_cfi_restore RAX, rax
.endif
.if \rstor_rcx
- movq_cfi_restore 5*8, rcx
+ movq_cfi_restore RCX, rcx
.endif
.if \rstor_rdx
- movq_cfi_restore 6*8, rdx
+ movq_cfi_restore RDX, rdx
.endif
- movq_cfi_restore 7*8, rsi
- movq_cfi_restore 8*8, rdi
+ movq_cfi_restore RSI, rsi
+ movq_cfi_restore RDI, rdi
- .if ARG_SKIP+\addskip > 0
- addq $ARG_SKIP+\addskip, %rsp
- CFI_ADJUST_CFA_OFFSET -(ARG_SKIP+\addskip)
+ .if ORIG_RAX+\addskip > 0
+ addq $ORIG_RAX+\addskip, %rsp
+ CFI_ADJUST_CFA_OFFSET -(ORIG_RAX+\addskip)
.endif
.endm
- .macro LOAD_ARGS offset, skiprax=0
- movq \offset(%rsp), %r11
- movq \offset+8(%rsp), %r10
- movq \offset+16(%rsp), %r9
- movq \offset+24(%rsp), %r8
- movq \offset+40(%rsp), %rcx
- movq \offset+48(%rsp), %rdx
- movq \offset+56(%rsp), %rsi
- movq \offset+64(%rsp), %rdi
+ .macro LOAD_ARGS skiprax=0
+ movq R11(%rsp), %r11
+ movq R10(%rsp), %r10
+ movq R9(%rsp), %r9
+ movq R8(%rsp), %r8
+ movq RCX(%rsp), %rcx
+ movq RDX(%rsp), %rdx
+ movq RSI(%rsp), %rsi
+ movq RDI(%rsp), %rdi
.if \skiprax
.else
- movq \offset+72(%rsp), %rax
+ movq ORIG_RAX(%rsp), %rax
.endif
.endm
-#define REST_SKIP (6*8)
-
.macro SAVE_REST
- subq $REST_SKIP, %rsp
- CFI_ADJUST_CFA_OFFSET REST_SKIP
- movq_cfi rbx, 5*8
- movq_cfi rbp, 4*8
- movq_cfi r12, 3*8
- movq_cfi r13, 2*8
- movq_cfi r14, 1*8
- movq_cfi r15, 0*8
+ movq_cfi rbx, RBX
+ movq_cfi rbp, RBP
+
+#ifndef CONFIG_PAX_KERNEXEC_PLUGIN_METHOD_OR
+ movq_cfi r12, R12
+#endif
+
+ movq_cfi r13, R13
+ movq_cfi r14, R14
+ movq_cfi r15, R15
.endm
.macro RESTORE_REST
- movq_cfi_restore 0*8, r15
- movq_cfi_restore 1*8, r14
- movq_cfi_restore 2*8, r13
- movq_cfi_restore 3*8, r12
- movq_cfi_restore 4*8, rbp
- movq_cfi_restore 5*8, rbx
- addq $REST_SKIP, %rsp
- CFI_ADJUST_CFA_OFFSET -(REST_SKIP)
+ movq_cfi_restore R15, r15
+ movq_cfi_restore R14, r14
+ movq_cfi_restore R13, r13
+
+#ifndef CONFIG_PAX_KERNEXEC_PLUGIN_METHOD_OR
+ movq_cfi_restore R12, r12
+#endif
+
+ movq_cfi_restore RBP, rbp
+ movq_cfi_restore RBX, rbx
.endm
.macro SAVE_ALL
int len, __wsum sum,
int *src_err_ptr, int *dst_err_ptr);
+asmlinkage __wsum csum_partial_copy_generic_to_user(const void *src, void *dst,
+ int len, __wsum sum,
+ int *src_err_ptr, int *dst_err_ptr);
+
+asmlinkage __wsum csum_partial_copy_generic_from_user(const void *src, void *dst,
+ int len, __wsum sum,
+ int *src_err_ptr, int *dst_err_ptr);
+
/*
* Note: when you get a NULL pointer exception here this means someone
* passed in an incorrect kernel address to one of these functions.
might_sleep();
stac();
- ret = csum_partial_copy_generic((__force void *)src, dst,
+ ret = csum_partial_copy_generic_from_user((__force void *)src, dst,
len, sum, err_ptr, NULL);
clac();
might_sleep();
if (access_ok(VERIFY_WRITE, dst, len)) {
stac();
- ret = csum_partial_copy_generic(src, (__force void *)dst,
+ ret = csum_partial_copy_generic_to_user(src, (__force void *)dst,
len, sum, NULL, err_ptr);
clac();
return ret;
__compiletime_error("Bad argument size for cmpxchg");
extern void __xadd_wrong_size(void)
__compiletime_error("Bad argument size for xadd");
+extern void __xadd_check_overflow_wrong_size(void)
+ __compiletime_error("Bad argument size for xadd_check_overflow");
extern void __add_wrong_size(void)
__compiletime_error("Bad argument size for add");
+extern void __add_check_overflow_wrong_size(void)
+ __compiletime_error("Bad argument size for add_check_overflow");
/*
* Constants for operation sizes. On 32-bit, the 64-bit size it set to
__ret; \
})
+#ifdef CONFIG_PAX_REFCOUNT
+#define __xchg_op_check_overflow(ptr, arg, op, lock) \
+ ({ \
+ __typeof__ (*(ptr)) __ret = (arg); \
+ switch (sizeof(*(ptr))) { \
+ case __X86_CASE_L: \
+ asm volatile (lock #op "l %0, %1\n" \
+ "jno 0f\n" \
+ "mov %0,%1\n" \
+ "int $4\n0:\n" \
+ _ASM_EXTABLE(0b, 0b) \
+ : "+r" (__ret), "+m" (*(ptr)) \
+ : : "memory", "cc"); \
+ break; \
+ case __X86_CASE_Q: \
+ asm volatile (lock #op "q %q0, %1\n" \
+ "jno 0f\n" \
+ "mov %0,%1\n" \
+ "int $4\n0:\n" \
+ _ASM_EXTABLE(0b, 0b) \
+ : "+r" (__ret), "+m" (*(ptr)) \
+ : : "memory", "cc"); \
+ break; \
+ default: \
+ __ ## op ## _check_overflow_wrong_size(); \
+ } \
+ __ret; \
+ })
+#else
+#define __xchg_op_check_overflow(ptr, arg, op, lock) __xchg_op(ptr, arg, op, lock)
+#endif
+
/*
* Note: no "lock" prefix even on SMP: xchg always implies lock anyway.
* Since this is generally used to protect other memory information, we
#define xadd_sync(ptr, inc) __xadd((ptr), (inc), "lock; ")
#define xadd_local(ptr, inc) __xadd((ptr), (inc), "")
+#define __xadd_check_overflow(ptr, inc, lock) __xchg_op_check_overflow((ptr), (inc), xadd, lock)
+#define xadd_check_overflow(ptr, inc) __xadd_check_overflow((ptr), (inc), LOCK_PREFIX)
+
#define __add(ptr, inc, lock) \
({ \
__typeof__ (*(ptr)) __ret = (inc); \
typedef u32 compat_uint_t;
typedef u32 compat_ulong_t;
typedef u64 __attribute__((aligned(4))) compat_u64;
-typedef u32 compat_uptr_t;
+typedef u32 __user compat_uptr_t;
struct compat_timespec {
compat_time_t tv_sec;
#define X86_FEATURE_PAUSEFILTER ( 8*32+13) /* AMD filtered pause intercept */
#define X86_FEATURE_PFTHRESHOLD ( 8*32+14) /* AMD pause filter threshold */
#define X86_FEATURE_VMMCALL ( 8*32+15) /* Prefer vmmcall to vmcall */
-
+#define X86_FEATURE_STRONGUDEREF (8*32+31) /* PaX PCID based strong UDEREF */
/* Intel-defined CPU features, CPUID level 0x00000007:0 (ebx), word 9 */
#define X86_FEATURE_FSGSBASE ( 9*32+ 0) /* {RD/WR}{FS/GS}BASE instructions*/
#define X86_FEATURE_BMI1 ( 9*32+ 3) /* 1st group bit manipulation extensions */
#define X86_FEATURE_HLE ( 9*32+ 4) /* Hardware Lock Elision */
#define X86_FEATURE_AVX2 ( 9*32+ 5) /* AVX2 instructions */
-#define X86_FEATURE_SMEP ( 9*32+ 7) /* Supervisor Mode Execution Protection */
+#define X86_FEATURE_SMEP ( 9*32+ 7) /* Supervisor Mode Execution Prevention */
#define X86_FEATURE_BMI2 ( 9*32+ 8) /* 2nd group bit manipulation extensions */
#define X86_FEATURE_ERMS ( 9*32+ 9) /* Enhanced REP MOVSB/STOSB */
#define X86_FEATURE_INVPCID ( 9*32+10) /* Invalidate Processor Context ID */
#define cpu_has_cx16 boot_cpu_has(X86_FEATURE_CX16)
#define cpu_has_eager_fpu boot_cpu_has(X86_FEATURE_EAGER_FPU)
#define cpu_has_topoext boot_cpu_has(X86_FEATURE_TOPOEXT)
+#define cpu_has_pcid boot_cpu_has(X86_FEATURE_PCID)
#if __GNUC__ >= 4
extern void warn_pre_alternatives(void);
#ifdef CONFIG_X86_DEBUG_STATIC_CPU_HAS
t_warn:
- warn_pre_alternatives();
+ if (bit != X86_FEATURE_PCID && bit != X86_FEATURE_INVPCID)
+ warn_pre_alternatives();
return false;
#endif
".section .discard,\"aw\",@progbits\n"
" .byte 0xff + (4f-3f) - (2b-1b)\n" /* size check */
".previous\n"
- ".section .altinstr_replacement,\"ax\"\n"
+ ".section .altinstr_replacement,\"a\"\n"
"3: movb $1,%0\n"
"4:\n"
".previous\n"
" .byte 2b - 1b\n" /* src len */
" .byte 4f - 3f\n" /* repl len */
".previous\n"
- ".section .altinstr_replacement,\"ax\"\n"
+ ".section .altinstr_replacement,\"a\"\n"
"3: .byte 0xe9\n .long %l[t_no] - 2b\n"
"4:\n"
".previous\n"
".section .discard,\"aw\",@progbits\n"
" .byte 0xff + (4f-3f) - (2b-1b)\n" /* size check */
".previous\n"
- ".section .altinstr_replacement,\"ax\"\n"
+ ".section .altinstr_replacement,\"a\"\n"
"3: movb $0,%0\n"
"4:\n"
".previous\n"
".section .discard,\"aw\",@progbits\n"
" .byte 0xff + (6f-5f) - (4b-3b)\n" /* size check */
".previous\n"
- ".section .altinstr_replacement,\"ax\"\n"
+ ".section .altinstr_replacement,\"a\"\n"
"5: movb $1,%0\n"
"6:\n"
".previous\n"
#include <asm/desc_defs.h>
#include <asm/ldt.h>
#include <asm/mmu.h>
+#include <asm/pgtable.h>
#include <linux/smp.h>
#include <linux/percpu.h>
desc->type = (info->read_exec_only ^ 1) << 1;
desc->type |= info->contents << 2;
+ desc->type |= info->seg_not_present ^ 1;
desc->s = 1;
desc->dpl = 0x3;
}
extern struct desc_ptr idt_descr;
-extern gate_desc idt_table[];
-extern struct desc_ptr debug_idt_descr;
-extern gate_desc debug_idt_table[];
-
-struct gdt_page {
- struct desc_struct gdt[GDT_ENTRIES];
-} __attribute__((aligned(PAGE_SIZE)));
-
-DECLARE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page);
+extern gate_desc idt_table[IDT_ENTRIES];
+extern const struct desc_ptr debug_idt_descr;
+extern gate_desc debug_idt_table[IDT_ENTRIES];
+extern struct desc_struct cpu_gdt_table[NR_CPUS][PAGE_SIZE / sizeof(struct desc_struct)];
static inline struct desc_struct *get_cpu_gdt_table(unsigned int cpu)
{
- return per_cpu(gdt_page, cpu).gdt;
+ return cpu_gdt_table[cpu];
}
#ifdef CONFIG_X86_64
unsigned long base, unsigned dpl, unsigned flags,
unsigned short seg)
{
- gate->a = (seg << 16) | (base & 0xffff);
- gate->b = (base & 0xffff0000) | (((0x80 | type | (dpl << 5)) & 0xff) << 8);
+ gate->gate.offset_low = base;
+ gate->gate.seg = seg;
+ gate->gate.reserved = 0;
+ gate->gate.type = type;
+ gate->gate.s = 0;
+ gate->gate.dpl = dpl;
+ gate->gate.p = 1;
+ gate->gate.offset_high = base >> 16;
}
#endif
static inline void native_write_idt_entry(gate_desc *idt, int entry, const gate_desc *gate)
{
+ pax_open_kernel();
memcpy(&idt[entry], gate, sizeof(*gate));
+ pax_close_kernel();
}
static inline void native_write_ldt_entry(struct desc_struct *ldt, int entry, const void *desc)
{
+ pax_open_kernel();
memcpy(&ldt[entry], desc, 8);
+ pax_close_kernel();
}
static inline void
default: size = sizeof(*gdt); break;
}
+ pax_open_kernel();
memcpy(&gdt[entry], desc, size);
+ pax_close_kernel();
}
static inline void pack_descriptor(struct desc_struct *desc, unsigned long base,
static inline void native_load_tr_desc(void)
{
+ pax_open_kernel();
asm volatile("ltr %w0"::"q" (GDT_ENTRY_TSS*8));
+ pax_close_kernel();
}
static inline void native_load_gdt(const struct desc_ptr *dtr)
struct desc_struct *gdt = get_cpu_gdt_table(cpu);
unsigned int i;
+ pax_open_kernel();
for (i = 0; i < GDT_ENTRY_TLS_ENTRIES; i++)
gdt[GDT_ENTRY_TLS_MIN + i] = t->tls_array[i];
+ pax_close_kernel();
}
/* This intentionally ignores lm, since 32-bit apps don't have that field. */
preempt_enable();
}
-static inline unsigned long get_desc_base(const struct desc_struct *desc)
+static inline unsigned long __intentional_overflow(-1) get_desc_base(const struct desc_struct *desc)
{
return (unsigned)(desc->base0 | ((desc->base1) << 16) | ((desc->base2) << 24));
}
}
#ifdef CONFIG_X86_64
-static inline void set_nmi_gate(int gate, void *addr)
+static inline void set_nmi_gate(int gate, const void *addr)
{
gate_desc s;
#endif
#ifdef CONFIG_TRACING
-extern struct desc_ptr trace_idt_descr;
-extern gate_desc trace_idt_table[];
+extern const struct desc_ptr trace_idt_descr;
+extern gate_desc trace_idt_table[IDT_ENTRIES];
static inline void write_trace_idt_entry(int entry, const gate_desc *gate)
{
write_idt_entry(trace_idt_table, entry, gate);
}
-static inline void _trace_set_gate(int gate, unsigned type, void *addr,
+static inline void _trace_set_gate(int gate, unsigned type, const void *addr,
unsigned dpl, unsigned ist, unsigned seg)
{
gate_desc s;
#define _trace_set_gate(gate, type, addr, dpl, ist, seg)
#endif
-static inline void _set_gate(int gate, unsigned type, void *addr,
+static inline void _set_gate(int gate, unsigned type, const void *addr,
unsigned dpl, unsigned ist, unsigned seg)
{
gate_desc s;
#define set_intr_gate(n, addr) \
do { \
BUG_ON((unsigned)n > 0xFF); \
- _set_gate(n, GATE_INTERRUPT, (void *)addr, 0, 0, \
+ _set_gate(n, GATE_INTERRUPT, (const void *)addr, 0, 0, \
__KERNEL_CS); \
- _trace_set_gate(n, GATE_INTERRUPT, (void *)trace_##addr,\
+ _trace_set_gate(n, GATE_INTERRUPT, (const void *)trace_##addr,\
0, 0, __KERNEL_CS); \
} while (0)
/*
* This routine sets up an interrupt gate at directory privilege level 3.
*/
-static inline void set_system_intr_gate(unsigned int n, void *addr)
+static inline void set_system_intr_gate(unsigned int n, const void *addr)
{
BUG_ON((unsigned)n > 0xFF);
_set_gate(n, GATE_INTERRUPT, addr, 0x3, 0, __KERNEL_CS);
}
-static inline void set_system_trap_gate(unsigned int n, void *addr)
+static inline void set_system_trap_gate(unsigned int n, const void *addr)
{
BUG_ON((unsigned)n > 0xFF);
_set_gate(n, GATE_TRAP, addr, 0x3, 0, __KERNEL_CS);
}
-static inline void set_trap_gate(unsigned int n, void *addr)
+static inline void set_trap_gate(unsigned int n, const void *addr)
{
BUG_ON((unsigned)n > 0xFF);
_set_gate(n, GATE_TRAP, addr, 0, 0, __KERNEL_CS);
static inline void set_task_gate(unsigned int n, unsigned int gdt_entry)
{
BUG_ON((unsigned)n > 0xFF);
- _set_gate(n, GATE_TASK, (void *)0, 0, 0, (gdt_entry<<3));
+ _set_gate(n, GATE_TASK, (const void *)0, 0, 0, (gdt_entry<<3));
}
-static inline void set_intr_gate_ist(int n, void *addr, unsigned ist)
+static inline void set_intr_gate_ist(int n, const void *addr, unsigned ist)
{
BUG_ON((unsigned)n > 0xFF);
_set_gate(n, GATE_INTERRUPT, addr, 0, ist, __KERNEL_CS);
}
-static inline void set_system_intr_gate_ist(int n, void *addr, unsigned ist)
+static inline void set_system_intr_gate_ist(int n, const void *addr, unsigned ist)
{
BUG_ON((unsigned)n > 0xFF);
_set_gate(n, GATE_INTERRUPT, addr, 0x3, ist, __KERNEL_CS);
else
load_idt((const struct desc_ptr *)&idt_descr);
}
+
+#ifdef CONFIG_X86_32
+static inline void set_user_cs(unsigned long base, unsigned long limit, int cpu)
+{
+ struct desc_struct d;
+
+ if (likely(limit))
+ limit = (limit - 1UL) >> PAGE_SHIFT;
+ pack_descriptor(&d, base, limit, 0xFB, 0xC);
+ write_gdt_entry(get_cpu_gdt_table(cpu), GDT_ENTRY_DEFAULT_USER_CS, &d, DESCTYPE_S);
+}
+#endif
+
#endif /* _ASM_X86_DESC_H */
unsigned base1: 8, type: 4, s: 1, dpl: 2, p: 1;
unsigned limit: 4, avl: 1, l: 1, d: 1, g: 1, base2: 8;
};
+ struct {
+ u16 offset_low;
+ u16 seg;
+ unsigned reserved: 8, type: 4, s: 1, dpl: 2, p: 1;
+ unsigned offset_high: 16;
+ } gate;
};
} __attribute__((packed));
__mod; \
})
-static inline u64 div_u64_rem(u64 dividend, u32 divisor, u32 *remainder)
+static inline u64 __intentional_overflow(-1) div_u64_rem(u64 dividend, u32 divisor, u32 *remainder)
{
union {
u64 v64;
#include <asm/vdso.h>
-#ifdef CONFIG_X86_64
-extern unsigned int vdso64_enabled;
-#endif
#if defined(CONFIG_X86_32) || defined(CONFIG_COMPAT)
extern unsigned int vdso32_enabled;
#endif
the loader. We need to make sure that it is out of the way of the program
that it will "exec", and that there is sufficient room for the brk. */
+#ifdef CONFIG_PAX_SEGMEXEC
+#define ELF_ET_DYN_BASE ((current->mm->pax_flags & MF_PAX_SEGMEXEC) ? SEGMEXEC_TASK_SIZE/3*2 : TASK_SIZE/3*2)
+#else
#define ELF_ET_DYN_BASE (TASK_SIZE / 3 * 2)
+#endif
+
+#ifdef CONFIG_PAX_ASLR
+#ifdef CONFIG_X86_32
+#define PAX_ELF_ET_DYN_BASE 0x10000000UL
+
+#define PAX_DELTA_MMAP_LEN (current->mm->pax_flags & MF_PAX_SEGMEXEC ? 15 : 16)
+#define PAX_DELTA_STACK_LEN (current->mm->pax_flags & MF_PAX_SEGMEXEC ? 15 : 16)
+#else
+#define PAX_ELF_ET_DYN_BASE 0x400000UL
+
+#define PAX_DELTA_MMAP_LEN ((test_thread_flag(TIF_ADDR32)) ? 16 : TASK_SIZE_MAX_SHIFT - PAGE_SHIFT - 3)
+#define PAX_DELTA_STACK_LEN ((test_thread_flag(TIF_ADDR32)) ? 16 : TASK_SIZE_MAX_SHIFT - PAGE_SHIFT - 3)
+#endif
+#endif
/* This yields a mask that user programs can use to figure out what
instruction set this CPU supports. This could be done in user space,
#define ARCH_DLINFO \
do { \
- if (vdso64_enabled) \
- NEW_AUX_ENT(AT_SYSINFO_EHDR, \
- (unsigned long __force)current->mm->context.vdso); \
+ NEW_AUX_ENT(AT_SYSINFO_EHDR, current->mm->context.vdso); \
} while (0)
/* As a historical oddity, the x32 and x86_64 vDSOs are controlled together. */
#define ARCH_DLINFO_X32 \
do { \
- if (vdso64_enabled) \
- NEW_AUX_ENT(AT_SYSINFO_EHDR, \
- (unsigned long __force)current->mm->context.vdso); \
+ NEW_AUX_ENT(AT_SYSINFO_EHDR, current->mm->context.vdso); \
} while (0)
#define AT_SYSINFO 32
#endif /* !CONFIG_X86_32 */
-#define VDSO_CURRENT_BASE ((unsigned long)current->mm->context.vdso)
+#define VDSO_CURRENT_BASE (current->mm->context.vdso)
#define VDSO_ENTRY \
- ((unsigned long)current->mm->context.vdso + \
+ (current->mm->context.vdso + \
selected_vdso32->sym___kernel_vsyscall)
struct linux_binprm;
int uses_interp);
#define compat_arch_setup_additional_pages compat_arch_setup_additional_pages
-extern unsigned long arch_randomize_brk(struct mm_struct *mm);
-#define arch_randomize_brk arch_randomize_brk
-
/*
* True on X86_32 or when emulating IA32 on X86_64
*/
#ifndef _ASM_X86_EMERGENCY_RESTART_H
#define _ASM_X86_EMERGENCY_RESTART_H
-extern void machine_emergency_restart(void);
+extern void machine_emergency_restart(void) __noreturn;
#endif /* _ASM_X86_EMERGENCY_RESTART_H */
int (*_dma_setup)(char *addr, unsigned long size, int mode, int io);
} fd_routine[] = {
{
- request_dma,
- free_dma,
- get_dma_residue,
- dma_mem_alloc,
- hard_dma_setup
+ ._request_dma = request_dma,
+ ._free_dma = free_dma,
+ ._get_dma_residue = get_dma_residue,
+ ._dma_mem_alloc = dma_mem_alloc,
+ ._dma_setup = hard_dma_setup
},
{
- vdma_request_dma,
- vdma_nop,
- vdma_get_dma_residue,
- vdma_mem_alloc,
- vdma_dma_setup
+ ._request_dma = vdma_request_dma,
+ ._free_dma = vdma_nop,
+ ._get_dma_residue = vdma_get_dma_residue,
+ ._dma_mem_alloc = vdma_mem_alloc,
+ ._dma_setup = vdma_dma_setup
}
};
#define user_insn(insn, output, input...) \
({ \
int err; \
+ pax_open_userland(); \
asm volatile(ASM_STAC "\n" \
- "1:" #insn "\n\t" \
+ "1:" \
+ __copyuser_seg \
+ #insn "\n\t" \
"2: " ASM_CLAC "\n" \
".section .fixup,\"ax\"\n" \
"3: movl $-1,%[err]\n" \
_ASM_EXTABLE(1b, 3b) \
: [err] "=r" (err), output \
: "0"(0), input); \
+ pax_close_userland(); \
err; \
})
"fnclex\n\t"
"emms\n\t"
"fildl %P[addr]" /* set F?P to defined value */
- : : [addr] "m" (tsk->thread.fpu.has_fpu));
+ : : [addr] "m" (init_tss[raw_smp_processor_id()].x86_tss.sp0));
}
return fpu_restore_checking(&tsk->thread.fpu);
#include <asm/smap.h>
#define __futex_atomic_op1(insn, ret, oldval, uaddr, oparg) \
+ typecheck(u32 __user *, uaddr); \
asm volatile("\t" ASM_STAC "\n" \
"1:\t" insn "\n" \
"2:\t" ASM_CLAC "\n" \
"\tjmp\t2b\n" \
"\t.previous\n" \
_ASM_EXTABLE(1b, 3b) \
- : "=r" (oldval), "=r" (ret), "+m" (*uaddr) \
+ : "=r" (oldval), "=r" (ret), "+m" (*(u32 __user *)____m(uaddr)) \
: "i" (-EFAULT), "0" (oparg), "1" (0))
#define __futex_atomic_op2(insn, ret, oldval, uaddr, oparg) \
+ typecheck(u32 __user *, uaddr); \
asm volatile("\t" ASM_STAC "\n" \
"1:\tmovl %2, %0\n" \
"\tmovl\t%0, %3\n" \
"\t" insn "\n" \
- "2:\t" LOCK_PREFIX "cmpxchgl %3, %2\n" \
+ "2:\t" LOCK_PREFIX __copyuser_seg"cmpxchgl %3, %2\n" \
"\tjnz\t1b\n" \
"3:\t" ASM_CLAC "\n" \
"\t.section .fixup,\"ax\"\n" \
_ASM_EXTABLE(1b, 4b) \
_ASM_EXTABLE(2b, 4b) \
: "=&a" (oldval), "=&r" (ret), \
- "+m" (*uaddr), "=&r" (tem) \
+ "+m" (*(u32 __user *)____m(uaddr)), "=&r" (tem) \
: "r" (oparg), "i" (-EFAULT), "1" (0))
static inline int futex_atomic_op_inuser(int encoded_op, u32 __user *uaddr)
pagefault_disable();
+ pax_open_userland();
switch (op) {
case FUTEX_OP_SET:
- __futex_atomic_op1("xchgl %0, %2", ret, oldval, uaddr, oparg);
+ __futex_atomic_op1(__copyuser_seg"xchgl %0, %2", ret, oldval, uaddr, oparg);
break;
case FUTEX_OP_ADD:
- __futex_atomic_op1(LOCK_PREFIX "xaddl %0, %2", ret, oldval,
+ __futex_atomic_op1(LOCK_PREFIX __copyuser_seg"xaddl %0, %2", ret, oldval,
uaddr, oparg);
break;
case FUTEX_OP_OR:
default:
ret = -ENOSYS;
}
+ pax_close_userland();
pagefault_enable();
#endif /* CONFIG_X86_LOCAL_APIC */
/* Statistics */
-extern atomic_t irq_err_count;
-extern atomic_t irq_mis_count;
+extern atomic_unchecked_t irq_err_count;
+extern atomic_unchecked_t irq_mis_count;
/* EISA */
extern void eisa_set_level_irq(unsigned int irq);
void (*init)(int auto_eoi);
int (*irq_pending)(unsigned int irq);
void (*make_irq)(unsigned int irq);
-};
+} __do_const;
extern struct legacy_pic *legacy_pic;
extern struct legacy_pic null_legacy_pic;
"m" (*(volatile type __force *)addr) barrier); }
build_mmio_read(readb, "b", unsigned char, "=q", :"memory")
-build_mmio_read(readw, "w", unsigned short, "=r", :"memory")
-build_mmio_read(readl, "l", unsigned int, "=r", :"memory")
+build_mmio_read(__intentional_overflow(-1) readw, "w", unsigned short, "=r", :"memory")
+build_mmio_read(__intentional_overflow(-1) readl, "l", unsigned int, "=r", :"memory")
build_mmio_read(__readb, "b", unsigned char, "=q", )
-build_mmio_read(__readw, "w", unsigned short, "=r", )
-build_mmio_read(__readl, "l", unsigned int, "=r", )
+build_mmio_read(__intentional_overflow(-1) __readw, "w", unsigned short, "=r", )
+build_mmio_read(__intentional_overflow(-1) __readl, "l", unsigned int, "=r", )
build_mmio_write(writeb, "b", unsigned char, "q", :"memory")
build_mmio_write(writew, "w", unsigned short, "r", :"memory")
* this function
*/
-static inline phys_addr_t virt_to_phys(volatile void *address)
+static inline phys_addr_t __intentional_overflow(-1) virt_to_phys(volatile void *address)
{
return __pa(address);
}
return ioremap_nocache(offset, size);
}
-extern void iounmap(volatile void __iomem *addr);
+extern void iounmap(const volatile void __iomem *addr);
extern void set_iounmap_nonlazy(void);
#include <linux/vmalloc.h>
+#define ARCH_HAS_VALID_PHYS_ADDR_RANGE
+static inline int valid_phys_addr_range(unsigned long addr, size_t count)
+{
+ return ((addr + count + PAGE_SIZE - 1) >> PAGE_SHIFT) < (1ULL << (boot_cpu_data.x86_phys_bits - PAGE_SHIFT)) ? 1 : 0;
+}
+
+static inline int valid_mmap_phys_addr_range(unsigned long pfn, size_t count)
+{
+ return (pfn + (count >> PAGE_SHIFT)) < (1ULL << (boot_cpu_data.x86_phys_bits - PAGE_SHIFT)) ? 1 : 0;
+}
+
/*
* Convert a virtual cached pointer to an uncached pointer
*/
sti; \
sysexit
+#define GET_CR0_INTO_RDI mov %cr0, %rdi
+#define SET_RDI_INTO_CR0 mov %rdi, %cr0
+#define GET_CR3_INTO_RDI mov %cr3, %rdi
+#define SET_RDI_INTO_CR3 mov %rdi, %cr3
+
#else
#define INTERRUPT_RETURN iret
#define ENABLE_INTERRUPTS_SYSEXIT sti; sysexit
#define RELATIVEJUMP_SIZE 5
#define RELATIVECALL_OPCODE 0xe8
#define RELATIVE_ADDR_SIZE 4
-#define MAX_STACK_SIZE 64
-#define MIN_STACK_SIZE(ADDR) \
- (((MAX_STACK_SIZE) < (((unsigned long)current_thread_info()) + \
- THREAD_SIZE - (unsigned long)(ADDR))) \
- ? (MAX_STACK_SIZE) \
- : (((unsigned long)current_thread_info()) + \
- THREAD_SIZE - (unsigned long)(ADDR)))
+#define MAX_STACK_SIZE 64UL
+#define MIN_STACK_SIZE(ADDR) min(MAX_STACK_SIZE, current->thread.sp0 - (unsigned long)(ADDR))
#define flush_insn_slot(p) do { } while (0)
| X86_CR0_NW | X86_CR0_CD | X86_CR0_PG))
#define CR3_L_MODE_RESERVED_BITS 0xFFFFFF0000000000ULL
-#define CR3_PCID_INVD (1UL << 63)
+#define CR3_PCID_INVD (1ULL << 63)
#define CR4_RESERVED_BITS \
(~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\
| X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE \
atomic_long_t a;
} local_t;
+typedef struct {
+ atomic_long_unchecked_t a;
+} local_unchecked_t;
+
#define LOCAL_INIT(i) { ATOMIC_LONG_INIT(i) }
#define local_read(l) atomic_long_read(&(l)->a)
+#define local_read_unchecked(l) atomic_long_read_unchecked(&(l)->a)
#define local_set(l, i) atomic_long_set(&(l)->a, (i))
+#define local_set_unchecked(l, i) atomic_long_set_unchecked(&(l)->a, (i))
static inline void local_inc(local_t *l)
{
- asm volatile(_ASM_INC "%0"
+ asm volatile(_ASM_INC "%0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ _ASM_DEC "%0\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+ : "+m" (l->a.counter));
+}
+
+static inline void local_inc_unchecked(local_unchecked_t *l)
+{
+ asm volatile(_ASM_INC "%0\n"
: "+m" (l->a.counter));
}
static inline void local_dec(local_t *l)
{
- asm volatile(_ASM_DEC "%0"
+ asm volatile(_ASM_DEC "%0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ _ASM_INC "%0\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+ : "+m" (l->a.counter));
+}
+
+static inline void local_dec_unchecked(local_unchecked_t *l)
+{
+ asm volatile(_ASM_DEC "%0\n"
: "+m" (l->a.counter));
}
static inline void local_add(long i, local_t *l)
{
- asm volatile(_ASM_ADD "%1,%0"
+ asm volatile(_ASM_ADD "%1,%0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ _ASM_SUB "%1,%0\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+ : "+m" (l->a.counter)
+ : "ir" (i));
+}
+
+static inline void local_add_unchecked(long i, local_unchecked_t *l)
+{
+ asm volatile(_ASM_ADD "%1,%0\n"
: "+m" (l->a.counter)
: "ir" (i));
}
static inline void local_sub(long i, local_t *l)
{
- asm volatile(_ASM_SUB "%1,%0"
+ asm volatile(_ASM_SUB "%1,%0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ _ASM_ADD "%1,%0\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+ : "+m" (l->a.counter)
+ : "ir" (i));
+}
+
+static inline void local_sub_unchecked(long i, local_unchecked_t *l)
+{
+ asm volatile(_ASM_SUB "%1,%0\n"
: "+m" (l->a.counter)
: "ir" (i));
}
*/
static inline int local_sub_and_test(long i, local_t *l)
{
- GEN_BINARY_RMWcc(_ASM_SUB, l->a.counter, "er", i, "%0", "e");
+ GEN_BINARY_RMWcc(_ASM_SUB, _ASM_ADD, l->a.counter, "er", i, "%0", "e");
}
/**
*/
static inline int local_dec_and_test(local_t *l)
{
- GEN_UNARY_RMWcc(_ASM_DEC, l->a.counter, "%0", "e");
+ GEN_UNARY_RMWcc(_ASM_DEC, _ASM_INC, l->a.counter, "%0", "e");
}
/**
*/
static inline int local_inc_and_test(local_t *l)
{
- GEN_UNARY_RMWcc(_ASM_INC, l->a.counter, "%0", "e");
+ GEN_UNARY_RMWcc(_ASM_INC, _ASM_DEC, l->a.counter, "%0", "e");
}
/**
*/
static inline int local_add_negative(long i, local_t *l)
{
- GEN_BINARY_RMWcc(_ASM_ADD, l->a.counter, "er", i, "%0", "s");
+ GEN_BINARY_RMWcc(_ASM_ADD, _ASM_SUB, l->a.counter, "er", i, "%0", "s");
}
/**
* Atomically adds @i to @l and returns @i + @l
*/
static inline long local_add_return(long i, local_t *l)
+{
+ long __i = i;
+ asm volatile(_ASM_XADD "%0, %1\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ _ASM_MOV "%0,%1\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+ : "+r" (i), "+m" (l->a.counter)
+ : : "memory");
+ return i + __i;
+}
+
+/**
+ * local_add_return_unchecked - add and return
+ * @i: integer value to add
+ * @l: pointer to type local_unchecked_t
+ *
+ * Atomically adds @i to @l and returns @i + @l
+ */
+static inline long local_add_return_unchecked(long i, local_unchecked_t *l)
{
long __i = i;
asm volatile(_ASM_XADD "%0, %1;"
#define local_cmpxchg(l, o, n) \
(cmpxchg_local(&((l)->a.counter), (o), (n)))
+#define local_cmpxchg_unchecked(l, o, n) \
+ (cmpxchg_local(&((l)->a.counter), (o), (n)))
/* Always has a lock prefix */
#define local_xchg(l, n) (xchg(&((l)->a.counter), (n)))
* we put the segment information here.
*/
typedef struct {
- void *ldt;
+ struct desc_struct *ldt;
int size;
#ifdef CONFIG_X86_64
#endif
struct mutex lock;
- void __user *vdso;
+ unsigned long vdso;
+
+#ifdef CONFIG_X86_32
+#if defined(CONFIG_PAX_PAGEEXEC) || defined(CONFIG_PAX_SEGMEXEC)
+ unsigned long user_cs_base;
+ unsigned long user_cs_limit;
+
+#if defined(CONFIG_PAX_PAGEEXEC) && defined(CONFIG_SMP)
+ cpumask_t cpu_user_cs_mask;
+#endif
+
+#endif
+#endif
} mm_context_t;
#ifdef CONFIG_SMP
static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
{
+
+#if defined(CONFIG_X86_64) && defined(CONFIG_PAX_MEMORY_UDEREF)
+ if (!(static_cpu_has(X86_FEATURE_PCID))) {
+ unsigned int i;
+ pgd_t *pgd;
+
+ pax_open_kernel();
+ pgd = get_cpu_pgd(smp_processor_id(), kernel);
+ for (i = USER_PGD_PTRS; i < 2 * USER_PGD_PTRS; ++i)
+ set_pgd_batched(pgd+i, native_make_pgd(0));
+ pax_close_kernel();
+ }
+#endif
+
#ifdef CONFIG_SMP
if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
this_cpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
struct task_struct *tsk)
{
unsigned cpu = smp_processor_id();
+#if defined(CONFIG_X86_32) && defined(CONFIG_SMP) && (defined(CONFIG_PAX_PAGEEXEC) || defined(CONFIG_PAX_SEGMEXEC))
+ int tlbstate = TLBSTATE_OK;
+#endif
if (likely(prev != next)) {
#ifdef CONFIG_SMP
+#if defined(CONFIG_X86_32) && (defined(CONFIG_PAX_PAGEEXEC) || defined(CONFIG_PAX_SEGMEXEC))
+ tlbstate = this_cpu_read(cpu_tlbstate.state);
+#endif
this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
this_cpu_write(cpu_tlbstate.active_mm, next);
#endif
cpumask_set_cpu(cpu, mm_cpumask(next));
/* Re-load page tables */
+#ifdef CONFIG_PAX_PER_CPU_PGD
+ pax_open_kernel();
+
+#if defined(CONFIG_X86_64) && defined(CONFIG_PAX_MEMORY_UDEREF)
+ if (static_cpu_has(X86_FEATURE_PCID))
+ __clone_user_pgds(get_cpu_pgd(cpu, user), next->pgd);
+ else
+#endif
+
+ __clone_user_pgds(get_cpu_pgd(cpu, kernel), next->pgd);
+ __shadow_user_pgds(get_cpu_pgd(cpu, kernel) + USER_PGD_PTRS, next->pgd);
+ pax_close_kernel();
+ BUG_ON((__pa(get_cpu_pgd(cpu, kernel)) | PCID_KERNEL) != (read_cr3() & __PHYSICAL_MASK) && (__pa(get_cpu_pgd(cpu, user)) | PCID_USER) != (read_cr3() & __PHYSICAL_MASK));
+
+#if defined(CONFIG_X86_64) && defined(CONFIG_PAX_MEMORY_UDEREF)
+ if (static_cpu_has(X86_FEATURE_PCID)) {
+ if (static_cpu_has(X86_FEATURE_INVPCID)) {
+ u64 descriptor[2];
+ descriptor[0] = PCID_USER;
+ asm volatile(__ASM_INVPCID : : "d"(&descriptor), "a"(INVPCID_SINGLE_CONTEXT) : "memory");
+ if (!static_cpu_has(X86_FEATURE_STRONGUDEREF)) {
+ descriptor[0] = PCID_KERNEL;
+ asm volatile(__ASM_INVPCID : : "d"(&descriptor), "a"(INVPCID_SINGLE_CONTEXT) : "memory");
+ }
+ } else {
+ write_cr3(__pa(get_cpu_pgd(cpu, user)) | PCID_USER);
+ if (static_cpu_has(X86_FEATURE_STRONGUDEREF))
+ write_cr3(__pa(get_cpu_pgd(cpu, kernel)) | PCID_KERNEL | PCID_NOFLUSH);
+ else
+ write_cr3(__pa(get_cpu_pgd(cpu, kernel)) | PCID_KERNEL);
+ }
+ } else
+#endif
+
+ load_cr3(get_cpu_pgd(cpu, kernel));
+#else
load_cr3(next->pgd);
+#endif
trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL);
/* Stop flush ipis for the previous mm */
*/
if (unlikely(prev->context.ldt != next->context.ldt))
load_LDT_nolock(&next->context);
+
+#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_PAGEEXEC) && defined(CONFIG_SMP)
+ if (!(__supported_pte_mask & _PAGE_NX)) {
+ smp_mb__before_atomic();
+ cpu_clear(cpu, prev->context.cpu_user_cs_mask);
+ smp_mb__after_atomic();
+ cpu_set(cpu, next->context.cpu_user_cs_mask);
+ }
+#endif
+
+#if defined(CONFIG_X86_32) && (defined(CONFIG_PAX_PAGEEXEC) || defined(CONFIG_PAX_SEGMEXEC))
+ if (unlikely(prev->context.user_cs_base != next->context.user_cs_base ||
+ prev->context.user_cs_limit != next->context.user_cs_limit))
+ set_user_cs(next->context.user_cs_base, next->context.user_cs_limit, cpu);
+#ifdef CONFIG_SMP
+ else if (unlikely(tlbstate != TLBSTATE_OK))
+ set_user_cs(next->context.user_cs_base, next->context.user_cs_limit, cpu);
+#endif
+#endif
+
}
+ else {
+
+#ifdef CONFIG_PAX_PER_CPU_PGD
+ pax_open_kernel();
+
+#if defined(CONFIG_X86_64) && defined(CONFIG_PAX_MEMORY_UDEREF)
+ if (static_cpu_has(X86_FEATURE_PCID))
+ __clone_user_pgds(get_cpu_pgd(cpu, user), next->pgd);
+ else
+#endif
+
+ __clone_user_pgds(get_cpu_pgd(cpu, kernel), next->pgd);
+ __shadow_user_pgds(get_cpu_pgd(cpu, kernel) + USER_PGD_PTRS, next->pgd);
+ pax_close_kernel();
+ BUG_ON((__pa(get_cpu_pgd(cpu, kernel)) | PCID_KERNEL) != (read_cr3() & __PHYSICAL_MASK) && (__pa(get_cpu_pgd(cpu, user)) | PCID_USER) != (read_cr3() & __PHYSICAL_MASK));
+
+#if defined(CONFIG_X86_64) && defined(CONFIG_PAX_MEMORY_UDEREF)
+ if (static_cpu_has(X86_FEATURE_PCID)) {
+ if (static_cpu_has(X86_FEATURE_INVPCID)) {
+ u64 descriptor[2];
+ descriptor[0] = PCID_USER;
+ asm volatile(__ASM_INVPCID : : "d"(&descriptor), "a"(INVPCID_SINGLE_CONTEXT) : "memory");
+ if (!static_cpu_has(X86_FEATURE_STRONGUDEREF)) {
+ descriptor[0] = PCID_KERNEL;
+ asm volatile(__ASM_INVPCID : : "d"(&descriptor), "a"(INVPCID_SINGLE_CONTEXT) : "memory");
+ }
+ } else {
+ write_cr3(__pa(get_cpu_pgd(cpu, user)) | PCID_USER);
+ if (static_cpu_has(X86_FEATURE_STRONGUDEREF))
+ write_cr3(__pa(get_cpu_pgd(cpu, kernel)) | PCID_KERNEL | PCID_NOFLUSH);
+ else
+ write_cr3(__pa(get_cpu_pgd(cpu, kernel)) | PCID_KERNEL);
+ }
+ } else
+#endif
+
+ load_cr3(get_cpu_pgd(cpu, kernel));
+#endif
+
#ifdef CONFIG_SMP
- else {
this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
BUG_ON(this_cpu_read(cpu_tlbstate.active_mm) != next);
* tlb flush IPI delivery. We must reload CR3
* to make sure to use no freed page tables.
*/
+
+#ifndef CONFIG_PAX_PER_CPU_PGD
load_cr3(next->pgd);
trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL);
+#endif
+
load_LDT_nolock(&next->context);
+
+#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_PAGEEXEC)
+ if (!(__supported_pte_mask & _PAGE_NX))
+ cpu_set(cpu, next->context.cpu_user_cs_mask);
+#endif
+
+#if defined(CONFIG_X86_32) && (defined(CONFIG_PAX_PAGEEXEC) || defined(CONFIG_PAX_SEGMEXEC))
+#ifdef CONFIG_PAX_PAGEEXEC
+ if (!((next->pax_flags & MF_PAX_PAGEEXEC) && (__supported_pte_mask & _PAGE_NX)))
+#endif
+ set_user_cs(next->context.user_cs_base, next->context.user_cs_limit, cpu);
+#endif
+
}
- }
#endif
+ }
}
#define activate_mm(prev, next) \
#ifdef CONFIG_X86_64
/* X86_64 does not define MODULE_PROC_FAMILY */
+#define MODULE_PROC_FAMILY ""
#elif defined CONFIG_M486
#define MODULE_PROC_FAMILY "486 "
#elif defined CONFIG_M586
#error unknown processor family
#endif
-#ifdef CONFIG_X86_32
-# define MODULE_ARCH_VERMAGIC MODULE_PROC_FAMILY
+#ifdef CONFIG_PAX_KERNEXEC_PLUGIN_METHOD_BTS
+#define MODULE_PAX_KERNEXEC "KERNEXEC_BTS "
+#elif defined(CONFIG_PAX_KERNEXEC_PLUGIN_METHOD_OR)
+#define MODULE_PAX_KERNEXEC "KERNEXEC_OR "
+#else
+#define MODULE_PAX_KERNEXEC ""
#endif
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+#define MODULE_PAX_UDEREF "UDEREF "
+#else
+#define MODULE_PAX_UDEREF ""
+#endif
+
+#define MODULE_ARCH_VERMAGIC MODULE_PROC_FAMILY MODULE_PAX_KERNEXEC MODULE_PAX_UDEREF
+
#endif /* _ASM_X86_MODULE_H */
typedef int (*nmi_handler_t)(unsigned int, struct pt_regs *);
+struct nmiaction;
+
+struct nmiwork {
+ const struct nmiaction *action;
+ u64 max_duration;
+ struct irq_work irq_work;
+};
+
struct nmiaction {
struct list_head list;
nmi_handler_t handler;
- u64 max_duration;
- struct irq_work irq_work;
unsigned long flags;
const char *name;
-};
+ struct nmiwork *work;
+} __do_const;
#define register_nmi_handler(t, fn, fg, n, init...) \
({ \
- static struct nmiaction init fn##_na = { \
+ static struct nmiwork fn##_nw; \
+ static const struct nmiaction init fn##_na = { \
.handler = (fn), \
.name = (n), \
.flags = (fg), \
+ .work = &fn##_nw, \
}; \
__register_nmi_handler((t), &fn##_na); \
})
-int __register_nmi_handler(unsigned int, struct nmiaction *);
+int __register_nmi_handler(unsigned int, const struct nmiaction *);
void unregister_nmi_handler(unsigned int, const char *);
__phys_addr_symbol(__phys_reloc_hide((unsigned long)(x)))
#define __va(x) ((void *)((unsigned long)(x)+PAGE_OFFSET))
+#define __early_va(x) ((void *)((unsigned long)(x)+__START_KERNEL_map - phys_base))
#define __boot_va(x) __va(x)
#define __boot_pa(x) __pa(x)
* virt_to_page(kaddr) returns a valid pointer if and only if
* virt_addr_valid(kaddr) returns true.
*/
-#define virt_to_page(kaddr) pfn_to_page(__pa(kaddr) >> PAGE_SHIFT)
#define pfn_to_kaddr(pfn) __va((pfn) << PAGE_SHIFT)
extern bool __virt_addr_valid(unsigned long kaddr);
#define virt_addr_valid(kaddr) __virt_addr_valid((unsigned long) (kaddr))
+#ifdef CONFIG_GRKERNSEC_KSTACKOVERFLOW
+#define virt_to_page(kaddr) \
+ ({ \
+ const void *__kaddr = (const void *)(kaddr); \
+ BUG_ON(!virt_addr_valid(__kaddr)); \
+ pfn_to_page(__pa(__kaddr) >> PAGE_SHIFT); \
+ })
+#else
+#define virt_to_page(kaddr) pfn_to_page(__pa(kaddr) >> PAGE_SHIFT)
+#endif
+
#endif /* __ASSEMBLY__ */
#include <asm-generic/memory_model.h>
/* duplicated to the one in bootmem.h */
extern unsigned long max_pfn;
-extern unsigned long phys_base;
+extern const unsigned long phys_base;
-static inline unsigned long __phys_addr_nodebug(unsigned long x)
+static inline unsigned long __intentional_overflow(-1) __phys_addr_nodebug(unsigned long x)
{
unsigned long y = x - __START_KERNEL_map;
return (pmd_t) { ret };
}
-static inline pmdval_t pmd_val(pmd_t pmd)
+static inline __intentional_overflow(-1) pmdval_t pmd_val(pmd_t pmd)
{
pmdval_t ret;
val);
}
+static inline void set_pgd_batched(pgd_t *pgdp, pgd_t pgd)
+{
+ pgdval_t val = native_pgd_val(pgd);
+
+ if (sizeof(pgdval_t) > sizeof(long))
+ PVOP_VCALL3(pv_mmu_ops.set_pgd_batched, pgdp,
+ val, (u64)val >> 32);
+ else
+ PVOP_VCALL2(pv_mmu_ops.set_pgd_batched, pgdp,
+ val);
+}
+
static inline void pgd_clear(pgd_t *pgdp)
{
set_pgd(pgdp, __pgd(0));
pv_mmu_ops.set_fixmap(idx, phys, flags);
}
+#ifdef CONFIG_PAX_KERNEXEC
+static inline unsigned long pax_open_kernel(void)
+{
+ return PVOP_CALL0(unsigned long, pv_mmu_ops.pax_open_kernel);
+}
+
+static inline unsigned long pax_close_kernel(void)
+{
+ return PVOP_CALL0(unsigned long, pv_mmu_ops.pax_close_kernel);
+}
+#else
+static inline unsigned long pax_open_kernel(void) { return 0; }
+static inline unsigned long pax_close_kernel(void) { return 0; }
+#endif
+
#if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS)
static __always_inline void __ticket_lock_spinning(struct arch_spinlock *lock,
#define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 4)
#define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .long, 4)
-#define PARA_INDIRECT(addr) *%cs:addr
+#define PARA_INDIRECT(addr) *%ss:addr
#endif
#define INTERRUPT_RETURN \
PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_irq_enable_sysexit), \
CLBR_NONE, \
jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_irq_enable_sysexit))
+
+#define GET_CR0_INTO_RDI \
+ call PARA_INDIRECT(pv_cpu_ops+PV_CPU_read_cr0); \
+ mov %rax,%rdi
+
+#define SET_RDI_INTO_CR0 \
+ call PARA_INDIRECT(pv_cpu_ops+PV_CPU_write_cr0)
+
+#define GET_CR3_INTO_RDI \
+ call PARA_INDIRECT(pv_mmu_ops+PV_MMU_read_cr3); \
+ mov %rax,%rdi
+
+#define SET_RDI_INTO_CR3 \
+ call PARA_INDIRECT(pv_mmu_ops+PV_MMU_write_cr3)
+
#endif /* CONFIG_X86_32 */
#endif /* __ASSEMBLY__ */
*/
unsigned (*patch)(u8 type, u16 clobber, void *insnbuf,
unsigned long addr, unsigned len);
-};
+} __no_const __no_randomize_layout;
struct pv_lazy_ops {
void (*enter)(void);
void (*leave)(void);
void (*flush)(void);
-};
+} __no_randomize_layout;
struct pv_time_ops {
unsigned long long (*sched_clock)(void);
unsigned long long (*steal_clock)(int cpu);
unsigned long (*get_tsc_khz)(void);
-};
+} __no_const __no_randomize_layout;
struct pv_cpu_ops {
/* hooks for various privileged instructions */
void (*start_context_switch)(struct task_struct *prev);
void (*end_context_switch)(struct task_struct *next);
-};
+} __no_const __no_randomize_layout;
struct pv_irq_ops {
/*
#ifdef CONFIG_X86_64
void (*adjust_exception_frame)(void);
#endif
-};
+} __no_randomize_layout;
struct pv_apic_ops {
#ifdef CONFIG_X86_LOCAL_APIC
unsigned long start_eip,
unsigned long start_esp);
#endif
-};
+} __no_const __no_randomize_layout;
struct pv_mmu_ops {
unsigned long (*read_cr2)(void);
struct paravirt_callee_save make_pud;
void (*set_pgd)(pgd_t *pudp, pgd_t pgdval);
+ void (*set_pgd_batched)(pgd_t *pudp, pgd_t pgdval);
#endif /* PAGETABLE_LEVELS == 4 */
#endif /* PAGETABLE_LEVELS >= 3 */
an mfn. We can tell which is which from the index. */
void (*set_fixmap)(unsigned /* enum fixed_addresses */ idx,
phys_addr_t phys, pgprot_t flags);
-};
+
+#ifdef CONFIG_PAX_KERNEXEC
+ unsigned long (*pax_open_kernel)(void);
+ unsigned long (*pax_close_kernel)(void);
+#endif
+
+} __no_randomize_layout;
struct arch_spinlock;
#ifdef CONFIG_SMP
struct pv_lock_ops {
struct paravirt_callee_save lock_spinning;
void (*unlock_kick)(struct arch_spinlock *lock, __ticket_t ticket);
-};
+} __no_randomize_layout;
/* This contains all the paravirt structures: we get a convenient
* number for each function using the offset which we use to indicate
- * what to patch. */
+ * what to patch.
+ * shouldn't be randomized due to the "NEAT TRICK" in paravirt.c
+ */
+
struct paravirt_patch_template {
struct pv_init_ops pv_init_ops;
struct pv_time_ops pv_time_ops;
struct pv_apic_ops pv_apic_ops;
struct pv_mmu_ops pv_mmu_ops;
struct pv_lock_ops pv_lock_ops;
-};
+} __no_randomize_layout;
extern struct pv_info pv_info;
extern struct pv_init_ops pv_init_ops;
static inline void pmd_populate_kernel(struct mm_struct *mm,
pmd_t *pmd, pte_t *pte)
+{
+ paravirt_alloc_pte(mm, __pa(pte) >> PAGE_SHIFT);
+ set_pmd(pmd, __pmd(__pa(pte) | _KERNPG_TABLE));
+}
+
+static inline void pmd_populate_user(struct mm_struct *mm,
+ pmd_t *pmd, pte_t *pte)
{
paravirt_alloc_pte(mm, __pa(pte) >> PAGE_SHIFT);
set_pmd(pmd, __pmd(__pa(pte) | _PAGE_TABLE));
#ifdef CONFIG_X86_PAE
extern void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd);
+static inline void pud_populate_kernel(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd)
+{
+ pud_populate(mm, pudp, pmd);
+}
#else /* !CONFIG_X86_PAE */
static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
{
paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT);
set_pud(pud, __pud(_PAGE_TABLE | __pa(pmd)));
}
+
+static inline void pud_populate_kernel(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
+{
+ paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT);
+ set_pud(pud, __pud(_KERNPG_TABLE | __pa(pmd)));
+}
#endif /* CONFIG_X86_PAE */
#if PAGETABLE_LEVELS > 3
set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(pud)));
}
+static inline void pgd_populate_kernel(struct mm_struct *mm, pgd_t *pgd, pud_t *pud)
+{
+ paravirt_alloc_pud(mm, __pa(pud) >> PAGE_SHIFT);
+ set_pgd(pgd, __pgd(_KERNPG_TABLE | __pa(pud)));
+}
+
static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
{
return (pud_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT);
static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd)
{
+ pax_open_kernel();
*pmdp = pmd;
+ pax_close_kernel();
}
static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte)
static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd)
{
+ pax_open_kernel();
set_64bit((unsigned long long *)(pmdp), native_pmd_val(pmd));
+ pax_close_kernel();
}
static inline void native_set_pud(pud_t *pudp, pud_t pud)
{
+ pax_open_kernel();
set_64bit((unsigned long long *)(pudp), native_pud_val(pud));
+ pax_close_kernel();
}
/*
#ifndef __PAGETABLE_PUD_FOLDED
#define set_pgd(pgdp, pgd) native_set_pgd(pgdp, pgd)
+#define set_pgd_batched(pgdp, pgd) native_set_pgd_batched(pgdp, pgd)
#define pgd_clear(pgd) native_pgd_clear(pgd)
#endif
#define arch_end_context_switch(prev) do {} while(0)
+#define pax_open_kernel() native_pax_open_kernel()
+#define pax_close_kernel() native_pax_close_kernel()
#endif /* CONFIG_PARAVIRT */
+#define __HAVE_ARCH_PAX_OPEN_KERNEL
+#define __HAVE_ARCH_PAX_CLOSE_KERNEL
+
+#ifdef CONFIG_PAX_KERNEXEC
+static inline unsigned long native_pax_open_kernel(void)
+{
+ unsigned long cr0;
+
+ preempt_disable();
+ barrier();
+ cr0 = read_cr0() ^ X86_CR0_WP;
+ BUG_ON(cr0 & X86_CR0_WP);
+ write_cr0(cr0);
+ barrier();
+ return cr0 ^ X86_CR0_WP;
+}
+
+static inline unsigned long native_pax_close_kernel(void)
+{
+ unsigned long cr0;
+
+ barrier();
+ cr0 = read_cr0() ^ X86_CR0_WP;
+ BUG_ON(!(cr0 & X86_CR0_WP));
+ write_cr0(cr0);
+ barrier();
+ preempt_enable_no_resched();
+ return cr0 ^ X86_CR0_WP;
+}
+#else
+static inline unsigned long native_pax_open_kernel(void) { return 0; }
+static inline unsigned long native_pax_close_kernel(void) { return 0; }
+#endif
+
/*
* The following only work if pte_present() is true.
* Undefined behaviour if not..
*/
+static inline int pte_user(pte_t pte)
+{
+ return pte_val(pte) & _PAGE_USER;
+}
+
static inline int pte_dirty(pte_t pte)
{
return pte_flags(pte) & _PAGE_DIRTY;
return (pud_val(pud) & PTE_PFN_MASK) >> PAGE_SHIFT;
}
+static inline unsigned long pgd_pfn(pgd_t pgd)
+{
+ return (pgd_val(pgd) & PTE_PFN_MASK) >> PAGE_SHIFT;
+}
+
#define pte_page(pte) pfn_to_page(pte_pfn(pte))
static inline int pmd_large(pmd_t pte)
return pte_clear_flags(pte, _PAGE_RW);
}
+static inline pte_t pte_mkread(pte_t pte)
+{
+ return __pte(pte_val(pte) | _PAGE_USER);
+}
+
static inline pte_t pte_mkexec(pte_t pte)
{
- return pte_clear_flags(pte, _PAGE_NX);
+#ifdef CONFIG_X86_PAE
+ if (__supported_pte_mask & _PAGE_NX)
+ return pte_clear_flags(pte, _PAGE_NX);
+ else
+#endif
+ return pte_set_flags(pte, _PAGE_USER);
+}
+
+static inline pte_t pte_exprotect(pte_t pte)
+{
+#ifdef CONFIG_X86_PAE
+ if (__supported_pte_mask & _PAGE_NX)
+ return pte_set_flags(pte, _PAGE_NX);
+ else
+#endif
+ return pte_clear_flags(pte, _PAGE_USER);
}
static inline pte_t pte_mkdirty(pte_t pte)
#endif
#ifndef __ASSEMBLY__
+
+#ifdef CONFIG_PAX_PER_CPU_PGD
+extern pgd_t cpu_pgd[NR_CPUS][2][PTRS_PER_PGD];
+enum cpu_pgd_type {kernel = 0, user = 1};
+static inline pgd_t *get_cpu_pgd(unsigned int cpu, enum cpu_pgd_type type)
+{
+ return cpu_pgd[cpu][type];
+}
+#endif
+
#include <linux/mm_types.h>
#include <linux/mmdebug.h>
#include <linux/log2.h>
* Currently stuck as a macro due to indirect forward reference to
* linux/mmzone.h's __section_mem_map_addr() definition:
*/
-#define pud_page(pud) pfn_to_page(pud_val(pud) >> PAGE_SHIFT)
+#define pud_page(pud) pfn_to_page((pud_val(pud) & PTE_PFN_MASK) >> PAGE_SHIFT)
/* Find an entry in the second-level page table.. */
static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address)
* Currently stuck as a macro due to indirect forward reference to
* linux/mmzone.h's __section_mem_map_addr() definition:
*/
-#define pgd_page(pgd) pfn_to_page(pgd_val(pgd) >> PAGE_SHIFT)
+#define pgd_page(pgd) pfn_to_page((pgd_val(pgd) & PTE_PFN_MASK) >> PAGE_SHIFT)
/* to find an entry in a page-table-directory. */
static inline unsigned long pud_index(unsigned long address)
static inline int pgd_bad(pgd_t pgd)
{
- return (pgd_flags(pgd) & ~_PAGE_USER) != _KERNPG_TABLE;
+ return (pgd_flags(pgd) & ~(_PAGE_USER | _PAGE_NX)) != _KERNPG_TABLE;
}
static inline int pgd_none(pgd_t pgd)
* pgd_offset() returns a (pgd_t *)
* pgd_index() is used get the offset into the pgd page's array of pgd_t's;
*/
-#define pgd_offset(mm, address) ((mm)->pgd + pgd_index((address)))
+#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address))
+
+#ifdef CONFIG_PAX_PER_CPU_PGD
+#define pgd_offset_cpu(cpu, type, address) (get_cpu_pgd(cpu, type) + pgd_index(address))
+#endif
+
/*
* a shortcut which implies the use of the kernel's pgd, instead
* of a process's
#define KERNEL_PGD_BOUNDARY pgd_index(PAGE_OFFSET)
#define KERNEL_PGD_PTRS (PTRS_PER_PGD - KERNEL_PGD_BOUNDARY)
+#ifdef CONFIG_X86_32
+#define USER_PGD_PTRS KERNEL_PGD_BOUNDARY
+#else
+#define TASK_SIZE_MAX_SHIFT CONFIG_TASK_SIZE_MAX_SHIFT
+#define USER_PGD_PTRS (_AC(1,UL) << (TASK_SIZE_MAX_SHIFT - PGDIR_SHIFT))
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+#ifdef __ASSEMBLY__
+#define pax_user_shadow_base pax_user_shadow_base(%rip)
+#else
+extern unsigned long pax_user_shadow_base;
+extern pgdval_t clone_pgd_mask;
+#endif
+#endif
+
+#endif
+
#ifndef __ASSEMBLY__
extern int direct_gbpages;
* dst and src can be on the same page, but the range must not overlap,
* and must not cross a page boundary.
*/
-static inline void clone_pgd_range(pgd_t *dst, pgd_t *src, int count)
+static inline void clone_pgd_range(pgd_t *dst, const pgd_t *src, int count)
{
- memcpy(dst, src, count * sizeof(pgd_t));
+ pax_open_kernel();
+ while (count--)
+ *dst++ = *src++;
+ pax_close_kernel();
}
+#ifdef CONFIG_PAX_PER_CPU_PGD
+extern void __clone_user_pgds(pgd_t *dst, const pgd_t *src);
+#endif
+
+#if defined(CONFIG_X86_64) && defined(CONFIG_PAX_MEMORY_UDEREF)
+extern void __shadow_user_pgds(pgd_t *dst, const pgd_t *src);
+#else
+static inline void __shadow_user_pgds(pgd_t *dst, const pgd_t *src) {}
+#endif
+
#define PTE_SHIFT ilog2(PTRS_PER_PTE)
static inline int page_level_shift(enum pg_level level)
{
struct mm_struct;
struct vm_area_struct;
-extern pgd_t swapper_pg_dir[1024];
-extern pgd_t initial_page_table[1024];
-
static inline void pgtable_cache_init(void) { }
static inline void check_pgt_cache(void) { }
void paging_init(void);
# include <asm/pgtable-2level.h>
#endif
+extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
+extern pgd_t initial_page_table[PTRS_PER_PGD];
+#ifdef CONFIG_X86_PAE
+extern pmd_t swapper_pm_dir[PTRS_PER_PGD][PTRS_PER_PMD];
+#endif
+
#if defined(CONFIG_HIGHPTE)
#define pte_offset_map(dir, address) \
((pte_t *)kmap_atomic(pmd_page(*(dir))) + \
/* Clear a kernel PTE and flush it from the TLB */
#define kpte_clear_flush(ptep, vaddr) \
do { \
+ pax_open_kernel(); \
pte_clear(&init_mm, (vaddr), (ptep)); \
+ pax_close_kernel(); \
__flush_tlb_one((vaddr)); \
} while (0)
#endif /* !__ASSEMBLY__ */
+#define HAVE_ARCH_UNMAPPED_AREA
+#define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
+
/*
* kern_addr_valid() is (1) for FLATMEM and (0) for
* SPARSEMEM and DISCONTIGMEM
*/
#ifdef CONFIG_X86_PAE
# include <asm/pgtable-3level_types.h>
-# define PMD_SIZE (1UL << PMD_SHIFT)
+# define PMD_SIZE (_AC(1, UL) << PMD_SHIFT)
# define PMD_MASK (~(PMD_SIZE - 1))
#else
# include <asm/pgtable-2level_types.h>
# define VMALLOC_END (FIXADDR_START - 2 * PAGE_SIZE)
#endif
+#ifdef CONFIG_PAX_KERNEXEC
+#ifndef __ASSEMBLY__
+extern unsigned char MODULES_EXEC_VADDR[];
+extern unsigned char MODULES_EXEC_END[];
+#endif
+#include <asm/boot.h>
+#define ktla_ktva(addr) (addr + LOAD_PHYSICAL_ADDR + PAGE_OFFSET)
+#define ktva_ktla(addr) (addr - LOAD_PHYSICAL_ADDR - PAGE_OFFSET)
+#else
+#define ktla_ktva(addr) (addr)
+#define ktva_ktla(addr) (addr)
+#endif
+
#define MODULES_VADDR VMALLOC_START
#define MODULES_END VMALLOC_END
#define MODULES_LEN (MODULES_VADDR - MODULES_END)
extern pud_t level3_kernel_pgt[512];
extern pud_t level3_ident_pgt[512];
+extern pud_t level3_vmalloc_start_pgt[512];
+extern pud_t level3_vmalloc_end_pgt[512];
+extern pud_t level3_vmemmap_pgt[512];
+extern pud_t level2_vmemmap_pgt[512];
extern pmd_t level2_kernel_pgt[512];
extern pmd_t level2_fixmap_pgt[512];
-extern pmd_t level2_ident_pgt[512];
+extern pmd_t level2_ident_pgt[512*2];
extern pte_t level1_fixmap_pgt[512];
-extern pgd_t init_level4_pgt[];
+extern pte_t level1_vsyscall_pgt[512];
+extern pgd_t init_level4_pgt[512];
#define swapper_pg_dir init_level4_pgt
static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd)
{
+ pax_open_kernel();
*pmdp = pmd;
+ pax_close_kernel();
}
static inline void native_pmd_clear(pmd_t *pmd)
static inline void native_set_pud(pud_t *pudp, pud_t pud)
{
+ pax_open_kernel();
*pudp = pud;
+ pax_close_kernel();
}
static inline void native_pud_clear(pud_t *pud)
}
static inline void native_set_pgd(pgd_t *pgdp, pgd_t pgd)
+{
+ pax_open_kernel();
+ *pgdp = pgd;
+ pax_close_kernel();
+}
+
+static inline void native_set_pgd_batched(pgd_t *pgdp, pgd_t pgd)
{
*pgdp = pgd;
}
#define MODULES_VADDR (__START_KERNEL_map + KERNEL_IMAGE_SIZE)
#define MODULES_END _AC(0xffffffffff000000, UL)
#define MODULES_LEN (MODULES_END - MODULES_VADDR)
+#define MODULES_EXEC_VADDR MODULES_VADDR
+#define MODULES_EXEC_END MODULES_END
#define ESPFIX_PGD_ENTRY _AC(-2, UL)
#define ESPFIX_BASE_ADDR (ESPFIX_PGD_ENTRY << PGDIR_SHIFT)
#define EFI_VA_START ( -4 * (_AC(1, UL) << 30))
#define EFI_VA_END (-68 * (_AC(1, UL) << 30))
+#define ktla_ktva(addr) (addr)
+#define ktva_ktla(addr) (addr)
+
#define EARLY_DYNAMIC_PAGE_TABLES 64
#endif /* _ASM_X86_PGTABLE_64_DEFS_H */
#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
#define _PAGE_NX (_AT(pteval_t, 1) << _PAGE_BIT_NX)
-#else
+#elif defined(CONFIG_KMEMCHECK) || defined(CONFIG_MEM_SOFT_DIRTY)
#define _PAGE_NX (_AT(pteval_t, 0))
+#else
+#define _PAGE_NX (_AT(pteval_t, 1) << _PAGE_BIT_HIDDEN)
#endif
#define _PAGE_FILE (_AT(pteval_t, 1) << _PAGE_BIT_FILE)
#define PAGE_READONLY_EXEC __pgprot(_PAGE_PRESENT | _PAGE_USER | \
_PAGE_ACCESSED)
+#define PAGE_READONLY_NOEXEC PAGE_READONLY
+#define PAGE_SHARED_NOEXEC PAGE_SHARED
+
#define __PAGE_KERNEL_EXEC \
(_PAGE_PRESENT | _PAGE_RW | _PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_GLOBAL)
#define __PAGE_KERNEL (__PAGE_KERNEL_EXEC | _PAGE_NX)
#define __PAGE_KERNEL_RO (__PAGE_KERNEL & ~_PAGE_RW)
#define __PAGE_KERNEL_RX (__PAGE_KERNEL_EXEC & ~_PAGE_RW)
#define __PAGE_KERNEL_NOCACHE (__PAGE_KERNEL | _PAGE_NOCACHE)
-#define __PAGE_KERNEL_VSYSCALL (__PAGE_KERNEL_RX | _PAGE_USER)
+#define __PAGE_KERNEL_VSYSCALL (__PAGE_KERNEL_RO | _PAGE_USER)
#define __PAGE_KERNEL_VVAR (__PAGE_KERNEL_RO | _PAGE_USER)
#define __PAGE_KERNEL_LARGE (__PAGE_KERNEL | _PAGE_PSE)
#define __PAGE_KERNEL_LARGE_EXEC (__PAGE_KERNEL_EXEC | _PAGE_PSE)
#ifdef CONFIG_X86_64
#define __PAGE_KERNEL_IDENT_LARGE_EXEC __PAGE_KERNEL_LARGE_EXEC
#else
-#define PTE_IDENT_ATTR 0x003 /* PRESENT+RW */
+#define PTE_IDENT_ATTR 0x063 /* PRESENT+RW+DIRTY+ACCESSED */
#define PDE_IDENT_ATTR 0x063 /* PRESENT+RW+DIRTY+ACCESSED */
#define PGD_IDENT_ATTR 0x001 /* PRESENT (no other attributes) */
#endif
{
return native_pgd_val(pgd) & PTE_FLAGS_MASK;
}
+#endif
+#if PAGETABLE_LEVELS == 3
+#include <asm-generic/pgtable-nopud.h>
+#endif
+
+#if PAGETABLE_LEVELS == 2
+#include <asm-generic/pgtable-nopmd.h>
+#endif
+
+#ifndef __ASSEMBLY__
#if PAGETABLE_LEVELS > 3
typedef struct { pudval_t pud; } pud_t;
return pud.pud;
}
#else
-#include <asm-generic/pgtable-nopud.h>
-
static inline pudval_t native_pud_val(pud_t pud)
{
return native_pgd_val(pud.pgd);
return pmd.pmd;
}
#else
-#include <asm-generic/pgtable-nopmd.h>
-
static inline pmdval_t native_pmd_val(pmd_t pmd)
{
return native_pgd_val(pmd.pud.pgd);
extern pteval_t __supported_pte_mask;
extern void set_nx(void);
-extern int nx_enabled;
#define pgprot_writecombine pgprot_writecombine
extern pgprot_t pgprot_writecombine(pgprot_t prot);
*/
static __always_inline bool __preempt_count_dec_and_test(void)
{
- GEN_UNARY_RMWcc("decl", __preempt_count, __percpu_arg(0), "e");
+ GEN_UNARY_RMWcc("decl", "incl", __preempt_count, __percpu_arg(0), "e");
}
/*
/* Index into per_cpu list: */
u16 cpu_index;
u32 microcode;
-};
+} __randomize_layout;
#define X86_VENDOR_INTEL 0
#define X86_VENDOR_CYRIX 1
: "memory");
}
+/* invpcid (%rdx),%rax */
+#define __ASM_INVPCID ".byte 0x66,0x0f,0x38,0x82,0x02"
+
+#define INVPCID_SINGLE_ADDRESS 0UL
+#define INVPCID_SINGLE_CONTEXT 1UL
+#define INVPCID_ALL_GLOBAL 2UL
+#define INVPCID_ALL_NONGLOBAL 3UL
+
+#define PCID_KERNEL 0UL
+#define PCID_USER 1UL
+#define PCID_NOFLUSH (1UL << 63)
+
static inline void load_cr3(pgd_t *pgdir)
{
- write_cr3(__pa(pgdir));
+ write_cr3(__pa(pgdir) | PCID_KERNEL);
}
#ifdef CONFIG_X86_32
} ____cacheline_aligned;
-DECLARE_PER_CPU_SHARED_ALIGNED(struct tss_struct, init_tss);
+extern struct tss_struct init_tss[NR_CPUS];
/*
* Save the original ist values for checking stack pointers during debugging
unsigned short ds;
unsigned short fsindex;
unsigned short gsindex;
+ unsigned short ss;
#endif
#ifdef CONFIG_X86_32
unsigned long ip;
extern unsigned long mmu_cr4_features;
extern u32 *trampoline_cr4_features;
-static inline void set_in_cr4(unsigned long mask)
-{
- unsigned long cr4;
-
- mmu_cr4_features |= mask;
- if (trampoline_cr4_features)
- *trampoline_cr4_features = mmu_cr4_features;
- cr4 = read_cr4();
- cr4 |= mask;
- write_cr4(cr4);
-}
-
-static inline void clear_in_cr4(unsigned long mask)
-{
- unsigned long cr4;
-
- mmu_cr4_features &= ~mask;
- if (trampoline_cr4_features)
- *trampoline_cr4_features = mmu_cr4_features;
- cr4 = read_cr4();
- cr4 &= ~mask;
- write_cr4(cr4);
-}
+extern void set_in_cr4(unsigned long mask);
+extern void clear_in_cr4(unsigned long mask);
typedef struct {
unsigned long seg;
*/
#define TASK_SIZE PAGE_OFFSET
#define TASK_SIZE_MAX TASK_SIZE
+
+#ifdef CONFIG_PAX_SEGMEXEC
+#define SEGMEXEC_TASK_SIZE (TASK_SIZE / 2)
+#define STACK_TOP ((current->mm->pax_flags & MF_PAX_SEGMEXEC)?SEGMEXEC_TASK_SIZE:TASK_SIZE)
+#else
#define STACK_TOP TASK_SIZE
-#define STACK_TOP_MAX STACK_TOP
+#endif
+
+#define STACK_TOP_MAX TASK_SIZE
#define INIT_THREAD { \
- .sp0 = sizeof(init_stack) + (long)&init_stack, \
+ .sp0 = sizeof(init_stack) + (long)&init_stack - 8, \
.vm86_info = NULL, \
.sysenter_cs = __KERNEL_CS, \
.io_bitmap_ptr = NULL, \
*/
#define INIT_TSS { \
.x86_tss = { \
- .sp0 = sizeof(init_stack) + (long)&init_stack, \
+ .sp0 = sizeof(init_stack) + (long)&init_stack - 8, \
.ss0 = __KERNEL_DS, \
.ss1 = __KERNEL_CS, \
.io_bitmap_base = INVALID_IO_BITMAP_OFFSET, \
extern unsigned long thread_saved_pc(struct task_struct *tsk);
#define THREAD_SIZE_LONGS (THREAD_SIZE/sizeof(unsigned long))
-#define KSTK_TOP(info) \
-({ \
- unsigned long *__ptr = (unsigned long *)(info); \
- (unsigned long)(&__ptr[THREAD_SIZE_LONGS]); \
-})
+#define KSTK_TOP(info) ((container_of(info, struct task_struct, tinfo))->thread.sp0)
/*
* The below -8 is to reserve 8 bytes on top of the ring0 stack.
#define task_pt_regs(task) \
({ \
struct pt_regs *__regs__; \
- __regs__ = (struct pt_regs *)(KSTK_TOP(task_stack_page(task))-8); \
+ __regs__ = (struct pt_regs *)((task)->thread.sp0); \
__regs__ - 1; \
})
* particular problem by preventing anything from being mapped
* at the maximum canonical address.
*/
-#define TASK_SIZE_MAX ((1UL << 47) - PAGE_SIZE)
+#define TASK_SIZE_MAX ((1UL << TASK_SIZE_MAX_SHIFT) - PAGE_SIZE)
/* This decides where the kernel will search for a free chunk of vm
* space during mmap's.
*/
#define IA32_PAGE_OFFSET ((current->personality & ADDR_LIMIT_3GB) ? \
- 0xc0000000 : 0xFFFFe000)
+ 0xc0000000 : 0xFFFFf000)
#define TASK_SIZE (test_thread_flag(TIF_ADDR32) ? \
IA32_PAGE_OFFSET : TASK_SIZE_MAX)
#define STACK_TOP_MAX TASK_SIZE_MAX
#define INIT_THREAD { \
- .sp0 = (unsigned long)&init_stack + sizeof(init_stack) \
+ .sp0 = (unsigned long)&init_stack + sizeof(init_stack) - 16 \
}
#define INIT_TSS { \
- .x86_tss.sp0 = (unsigned long)&init_stack + sizeof(init_stack) \
+ .x86_tss.sp0 = (unsigned long)&init_stack + sizeof(init_stack) - 16 \
}
/*
*/
#define TASK_UNMAPPED_BASE (PAGE_ALIGN(TASK_SIZE / 3))
+#ifdef CONFIG_PAX_SEGMEXEC
+#define SEGMEXEC_TASK_UNMAPPED_BASE (PAGE_ALIGN(SEGMEXEC_TASK_SIZE / 3))
+#endif
+
#define KSTK_EIP(task) (task_pt_regs(task)->ip)
/* Get/set a process' ability to use the timestamp counter instruction */
return 0;
}
-extern unsigned long arch_align_stack(unsigned long sp);
+#define arch_align_stack(x) ((x) & ~0xfUL)
extern void free_init_pages(char *what, unsigned long begin, unsigned long end);
void default_idle(void);
#define xen_set_default_idle 0
#endif
-void stop_this_cpu(void *dummy);
+void stop_this_cpu(void *dummy) __noreturn;
void df_debug(struct pt_regs *regs, long error_code);
#endif /* _ASM_X86_PROCESSOR_H */
}
/*
- * user_mode_vm(regs) determines whether a register set came from user mode.
+ * user_mode(regs) determines whether a register set came from user mode.
* This is true if V8086 mode was enabled OR if the register set was from
* protected mode with RPL-3 CS value. This tricky test checks that with
* one comparison. Many places in the kernel can bypass this full check
- * if they have already ruled out V8086 mode, so user_mode(regs) can be used.
+ * if they have already ruled out V8086 mode, so user_mode_novm(regs) can
+ * be used.
*/
-static inline int user_mode(struct pt_regs *regs)
+static inline int user_mode_novm(struct pt_regs *regs)
{
#ifdef CONFIG_X86_32
return (regs->cs & SEGMENT_RPL_MASK) == USER_RPL;
#else
- return !!(regs->cs & 3);
+ return !!(regs->cs & SEGMENT_RPL_MASK);
#endif
}
-static inline int user_mode_vm(struct pt_regs *regs)
+static inline int user_mode(struct pt_regs *regs)
{
#ifdef CONFIG_X86_32
return ((regs->cs & SEGMENT_RPL_MASK) | (regs->flags & X86_VM_MASK)) >=
USER_RPL;
#else
- return user_mode(regs);
+ return user_mode_novm(regs);
#endif
}
#ifdef CONFIG_X86_64
static inline bool user_64bit_mode(struct pt_regs *regs)
{
+ unsigned long cs = regs->cs & 0xffff;
#ifndef CONFIG_PARAVIRT
/*
* On non-paravirt systems, this is the only long mode CPL 3
* selector. We do not allow long mode selectors in the LDT.
*/
- return regs->cs == __USER_CS;
+ return cs == __USER_CS;
#else
/* Headers are too twisted for this to go in paravirt.h. */
- return regs->cs == __USER_CS || regs->cs == pv_info.extra_user_64bit_cs;
+ return cs == __USER_CS || cs == pv_info.extra_user_64bit_cs;
#endif
}
* Traps from the kernel do not save sp and ss.
* Use the helper function to retrieve sp.
*/
- if (offset == offsetof(struct pt_regs, sp) &&
- regs->cs == __KERNEL_CS)
- return kernel_stack_pointer(regs);
+ if (offset == offsetof(struct pt_regs, sp)) {
+ unsigned long cs = regs->cs & 0xffff;
+ if (cs == __KERNEL_CS || cs == __KERNEXEC_KERNEL_CS)
+ return kernel_stack_pointer(regs);
+ }
#endif
return *(unsigned long *)((unsigned long)regs + offset);
}
#define queue_write_unlock queue_write_unlock
static inline void queue_write_unlock(struct qrwlock *lock)
{
- barrier();
- ACCESS_ONCE(*(u8 *)&lock->cnts) = 0;
+ barrier();
+ ACCESS_ONCE_RW(*(u8 *)&lock->cnts) = 0;
}
#endif
#endif
/* APM/BIOS reboot */
u32 machine_real_restart_asm;
-#ifdef CONFIG_X86_64
u32 machine_real_restart_seg;
-#endif
};
/* This must match data at trampoline_32/64.S */
struct trampoline_header {
#ifdef CONFIG_X86_32
u32 start;
- u16 gdt_pad;
+ u16 boot_cs;
u16 gdt_limit;
u32 gdt_base;
#else
struct pt_regs;
struct machine_ops {
- void (*restart)(char *cmd);
- void (*halt)(void);
- void (*power_off)(void);
+ void (* __noreturn restart)(char *cmd);
+ void (* __noreturn halt)(void);
+ void (* __noreturn power_off)(void);
void (*shutdown)(void);
void (*crash_shutdown)(struct pt_regs *);
- void (*emergency_restart)(void);
-};
+ void (* __noreturn emergency_restart)(void);
+} __no_const;
extern struct machine_ops machine_ops;
#ifdef CC_HAVE_ASM_GOTO
-#define __GEN_RMWcc(fullop, var, cc, ...) \
+#ifdef CONFIG_PAX_REFCOUNT
+#define __GEN_RMWcc(fullop, fullantiop, var, cc, ...) \
+do { \
+ asm_volatile_goto (fullop \
+ ";jno 0f\n" \
+ fullantiop \
+ ";int $4\n0:\n" \
+ _ASM_EXTABLE(0b, 0b) \
+ ";j" cc " %l[cc_label]" \
+ : : "m" (var), ## __VA_ARGS__ \
+ : "memory" : cc_label); \
+ return 0; \
+cc_label: \
+ return 1; \
+} while (0)
+#else
+#define __GEN_RMWcc(fullop, fullantiop, var, cc, ...) \
+do { \
+ asm_volatile_goto (fullop ";j" cc " %l[cc_label]" \
+ : : "m" (var), ## __VA_ARGS__ \
+ : "memory" : cc_label); \
+ return 0; \
+cc_label: \
+ return 1; \
+} while (0)
+#endif
+
+#define __GEN_RMWcc_unchecked(fullop, var, cc, ...) \
do { \
asm_volatile_goto (fullop "; j" cc " %l[cc_label]" \
: : "m" (var), ## __VA_ARGS__ \
return 1; \
} while (0)
-#define GEN_UNARY_RMWcc(op, var, arg0, cc) \
- __GEN_RMWcc(op " " arg0, var, cc)
+#define GEN_UNARY_RMWcc(op, antiop, var, arg0, cc) \
+ __GEN_RMWcc(op " " arg0, antiop " " arg0, var, cc)
+
+#define GEN_UNARY_RMWcc_unchecked(op, var, arg0, cc) \
+ __GEN_RMWcc_unchecked(op " " arg0, var, cc)
+
+#define GEN_BINARY_RMWcc(op, antiop, var, vcon, val, arg0, cc) \
+ __GEN_RMWcc(op " %1, " arg0, antiop " %1, " arg0, var, cc, vcon (val))
-#define GEN_BINARY_RMWcc(op, var, vcon, val, arg0, cc) \
- __GEN_RMWcc(op " %1, " arg0, var, cc, vcon (val))
+#define GEN_BINARY_RMWcc_unchecked(op, var, vcon, val, arg0, cc) \
+ __GEN_RMWcc_unchecked(op " %1, " arg0, var, cc, vcon (val))
#else /* !CC_HAVE_ASM_GOTO */
-#define __GEN_RMWcc(fullop, var, cc, ...) \
+#ifdef CONFIG_PAX_REFCOUNT
+#define __GEN_RMWcc(fullop, fullantiop, var, cc, ...) \
+do { \
+ char c; \
+ asm volatile (fullop \
+ ";jno 0f\n" \
+ fullantiop \
+ ";int $4\n0:\n" \
+ _ASM_EXTABLE(0b, 0b) \
+ "; set" cc " %1" \
+ : "+m" (var), "=qm" (c) \
+ : __VA_ARGS__ : "memory"); \
+ return c != 0; \
+} while (0)
+#else
+#define __GEN_RMWcc(fullop, fullantiop, var, cc, ...) \
+do { \
+ char c; \
+ asm volatile (fullop "; set" cc " %1" \
+ : "+m" (var), "=qm" (c) \
+ : __VA_ARGS__ : "memory"); \
+ return c != 0; \
+} while (0)
+#endif
+
+#define __GEN_RMWcc_unchecked(fullop, var, cc, ...) \
do { \
char c; \
asm volatile (fullop "; set" cc " %1" \
return c != 0; \
} while (0)
-#define GEN_UNARY_RMWcc(op, var, arg0, cc) \
- __GEN_RMWcc(op " " arg0, var, cc)
+#define GEN_UNARY_RMWcc(op, antiop, var, arg0, cc) \
+ __GEN_RMWcc(op " " arg0, antiop " " arg0, var, cc)
+
+#define GEN_UNARY_RMWcc_unchecked(op, var, arg0, cc) \
+ __GEN_RMWcc_unchecked(op " " arg0, var, cc)
+
+#define GEN_BINARY_RMWcc(op, antiop, var, vcon, val, arg0, cc) \
+ __GEN_RMWcc(op " %2, " arg0, antiop " %2, " arg0, var, cc, vcon (val))
-#define GEN_BINARY_RMWcc(op, var, vcon, val, arg0, cc) \
- __GEN_RMWcc(op " %2, " arg0, var, cc, vcon (val))
+#define GEN_BINARY_RMWcc_unchecked(op, var, vcon, val, arg0, cc) \
+ __GEN_RMWcc_unchecked(op " %2, " arg0, var, cc, vcon (val))
#endif /* CC_HAVE_ASM_GOTO */
{
asm volatile("# beginning down_read\n\t"
LOCK_PREFIX _ASM_INC "(%1)\n\t"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ LOCK_PREFIX _ASM_DEC "(%1)\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
/* adds 0x00000001 */
" jns 1f\n"
" call call_rwsem_down_read_failed\n"
"1:\n\t"
" mov %1,%2\n\t"
" add %3,%2\n\t"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ "sub %3,%2\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
" jle 2f\n\t"
LOCK_PREFIX " cmpxchg %2,%0\n\t"
" jnz 1b\n\t"
long tmp;
asm volatile("# beginning down_write\n\t"
LOCK_PREFIX " xadd %1,(%2)\n\t"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ "mov %1,(%2)\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
/* adds 0xffff0001, returns the old value */
" test " __ASM_SEL(%w1,%k1) "," __ASM_SEL(%w1,%k1) "\n\t"
/* was the active mask 0 before? */
long tmp;
asm volatile("# beginning __up_read\n\t"
LOCK_PREFIX " xadd %1,(%2)\n\t"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ "mov %1,(%2)\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
/* subtracts 1, returns the old value */
" jns 1f\n\t"
" call call_rwsem_wake\n" /* expects old value in %edx */
long tmp;
asm volatile("# beginning __up_write\n\t"
LOCK_PREFIX " xadd %1,(%2)\n\t"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ "mov %1,(%2)\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
/* subtracts 0xffff0001, returns the old value */
" jns 1f\n\t"
" call call_rwsem_wake\n" /* expects old value in %edx */
{
asm volatile("# beginning __downgrade_write\n\t"
LOCK_PREFIX _ASM_ADD "%2,(%1)\n\t"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ LOCK_PREFIX _ASM_SUB "%2,(%1)\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
/*
* transitions 0xZZZZ0001 -> 0xYYYY0001 (i386)
* 0xZZZZZZZZ00000001 -> 0xYYYYYYYY00000001 (x86_64)
*/
static inline void rwsem_atomic_add(long delta, struct rw_semaphore *sem)
{
- asm volatile(LOCK_PREFIX _ASM_ADD "%1,%0"
+ asm volatile(LOCK_PREFIX _ASM_ADD "%1,%0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ LOCK_PREFIX _ASM_SUB "%1,%0\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
: "+m" (sem->count)
: "er" (delta));
}
*/
static inline long rwsem_atomic_update(long delta, struct rw_semaphore *sem)
{
- return delta + xadd(&sem->count, delta);
+ return delta + xadd_check_overflow(&sem->count, delta);
}
#endif /* __KERNEL__ */
* 26 - ESPFIX small SS
* 27 - per-cpu [ offset to per-cpu data area ]
* 28 - stack_canary-20 [ for stack protector ]
- * 29 - unused
- * 30 - unused
+ * 29 - PCI BIOS CS
+ * 30 - PCI BIOS DS
* 31 - TSS for double fault handler
*/
+#define GDT_ENTRY_KERNEXEC_EFI_CS (1)
+#define GDT_ENTRY_KERNEXEC_EFI_DS (2)
+#define __KERNEXEC_EFI_CS (GDT_ENTRY_KERNEXEC_EFI_CS*8)
+#define __KERNEXEC_EFI_DS (GDT_ENTRY_KERNEXEC_EFI_DS*8)
+
#define GDT_ENTRY_TLS_MIN 6
#define GDT_ENTRY_TLS_MAX (GDT_ENTRY_TLS_MIN + GDT_ENTRY_TLS_ENTRIES - 1)
#define GDT_ENTRY_KERNEL_CS (GDT_ENTRY_KERNEL_BASE+0)
+#define GDT_ENTRY_KERNEXEC_KERNEL_CS (4)
+
#define GDT_ENTRY_KERNEL_DS (GDT_ENTRY_KERNEL_BASE+1)
#define GDT_ENTRY_TSS (GDT_ENTRY_KERNEL_BASE+4)
#define __KERNEL_STACK_CANARY 0
#endif
+#define GDT_ENTRY_PCIBIOS_CS (GDT_ENTRY_KERNEL_BASE+17)
+#define __PCIBIOS_CS (GDT_ENTRY_PCIBIOS_CS * 8)
+
+#define GDT_ENTRY_PCIBIOS_DS (GDT_ENTRY_KERNEL_BASE+18)
+#define __PCIBIOS_DS (GDT_ENTRY_PCIBIOS_DS * 8)
+
#define GDT_ENTRY_DOUBLEFAULT_TSS 31
/*
*/
/* Matches PNP_CS32 and PNP_CS16 (they must be consecutive) */
-#define SEGMENT_IS_PNP_CODE(x) (((x) & 0xf4) == GDT_ENTRY_PNPBIOS_BASE * 8)
+#define SEGMENT_IS_PNP_CODE(x) (((x) & 0xFFFCU) == PNP_CS32 || ((x) & 0xFFFCU) == PNP_CS16)
#else
#define __USER32_CS (GDT_ENTRY_DEFAULT_USER32_CS*8+3)
#define __USER32_DS __USER_DS
+#define GDT_ENTRY_KERNEXEC_KERNEL_CS 7
+
#define GDT_ENTRY_TSS 8 /* needs two entries */
#define GDT_ENTRY_LDT 10 /* needs two entries */
#define GDT_ENTRY_TLS_MIN 12
#define GDT_ENTRY_PER_CPU 15 /* Abused to load per CPU data from limit */
#define __PER_CPU_SEG (GDT_ENTRY_PER_CPU * 8 + 3)
+#define GDT_ENTRY_UDEREF_KERNEL_DS 16
+
/* TLS indexes for 64bit - hardcoded in arch_prctl */
#define FS_TLS 0
#define GS_TLS 1
#define GS_TLS_SEL ((GDT_ENTRY_TLS_MIN+GS_TLS)*8 + 3)
#define FS_TLS_SEL ((GDT_ENTRY_TLS_MIN+FS_TLS)*8 + 3)
-#define GDT_ENTRIES 16
+#define GDT_ENTRIES 17
#endif
#define __KERNEL_CS (GDT_ENTRY_KERNEL_CS*8)
+#define __KERNEXEC_KERNEL_CS (GDT_ENTRY_KERNEXEC_KERNEL_CS*8)
#define __KERNEL_DS (GDT_ENTRY_KERNEL_DS*8)
+#define __UDEREF_KERNEL_DS (GDT_ENTRY_UDEREF_KERNEL_DS*8)
#define __USER_DS (GDT_ENTRY_DEFAULT_USER_DS*8+3)
#define __USER_CS (GDT_ENTRY_DEFAULT_USER_CS*8+3)
#ifndef CONFIG_PARAVIRT
{
unsigned long __limit;
asm("lsll %1,%0" : "=r" (__limit) : "r" (segment));
- return __limit + 1;
+ return __limit;
}
#endif /* !__ASSEMBLY__ */
#include <asm/alternative-asm.h>
+#if defined(CONFIG_X86_64) && defined(CONFIG_PAX_MEMORY_UDEREF)
+#define ASM_PAX_OPEN_USERLAND \
+ 661: jmp 663f; \
+ .pushsection .altinstr_replacement, "a" ; \
+ 662: pushq %rax; nop; \
+ .popsection ; \
+ .pushsection .altinstructions, "a" ; \
+ altinstruction_entry 661b, 662b, X86_FEATURE_STRONGUDEREF, 2, 2;\
+ .popsection ; \
+ call __pax_open_userland; \
+ popq %rax; \
+ 663:
+
+#define ASM_PAX_CLOSE_USERLAND \
+ 661: jmp 663f; \
+ .pushsection .altinstr_replacement, "a" ; \
+ 662: pushq %rax; nop; \
+ .popsection; \
+ .pushsection .altinstructions, "a" ; \
+ altinstruction_entry 661b, 662b, X86_FEATURE_STRONGUDEREF, 2, 2;\
+ .popsection; \
+ call __pax_close_userland; \
+ popq %rax; \
+ 663:
+#else
+#define ASM_PAX_OPEN_USERLAND
+#define ASM_PAX_CLOSE_USERLAND
+#endif
+
#ifdef CONFIG_X86_SMAP
#define ASM_CLAC \
661: ASM_NOP3 ; \
- .pushsection .altinstr_replacement, "ax" ; \
+ .pushsection .altinstr_replacement, "a" ; \
662: __ASM_CLAC ; \
.popsection ; \
.pushsection .altinstructions, "a" ; \
#define ASM_STAC \
661: ASM_NOP3 ; \
- .pushsection .altinstr_replacement, "ax" ; \
+ .pushsection .altinstr_replacement, "a" ; \
662: __ASM_STAC ; \
.popsection ; \
.pushsection .altinstructions, "a" ; \
#include <asm/alternative.h>
+#define __HAVE_ARCH_PAX_OPEN_USERLAND
+#define __HAVE_ARCH_PAX_CLOSE_USERLAND
+
+extern void __pax_open_userland(void);
+static __always_inline unsigned long pax_open_userland(void)
+{
+
+#if defined(CONFIG_X86_64) && defined(CONFIG_PAX_MEMORY_UDEREF)
+ asm volatile(ALTERNATIVE(ASM_NOP5, "call %P[open]", X86_FEATURE_STRONGUDEREF)
+ :
+ : [open] "i" (__pax_open_userland)
+ : "memory", "rax");
+#endif
+
+ return 0;
+}
+
+extern void __pax_close_userland(void);
+static __always_inline unsigned long pax_close_userland(void)
+{
+
+#if defined(CONFIG_X86_64) && defined(CONFIG_PAX_MEMORY_UDEREF)
+ asm volatile(ALTERNATIVE(ASM_NOP5, "call %P[close]", X86_FEATURE_STRONGUDEREF)
+ :
+ : [close] "i" (__pax_close_userland)
+ : "memory", "rax");
+#endif
+
+ return 0;
+}
+
#ifdef CONFIG_X86_SMAP
static __always_inline void clac(void)
/* cpus sharing the last level cache: */
DECLARE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_llc_shared_map);
DECLARE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id);
-DECLARE_PER_CPU_READ_MOSTLY(int, cpu_number);
+DECLARE_PER_CPU_READ_MOSTLY(unsigned int, cpu_number);
static inline struct cpumask *cpu_sibling_mask(int cpu)
{
void (*send_call_func_ipi)(const struct cpumask *mask);
void (*send_call_func_single_ipi)(int cpu);
-};
+} __no_const;
/* Globals due to paravirt */
extern void set_cpu_sibling_map(int cpu);
extern int safe_smp_processor_id(void);
#elif defined(CONFIG_X86_64_SMP)
-#define raw_smp_processor_id() (this_cpu_read(cpu_number))
-
-#define stack_smp_processor_id() \
-({ \
- struct thread_info *ti; \
- __asm__("andq %%rsp,%0; ":"=r" (ti) : "0" (CURRENT_MASK)); \
- ti->cpu; \
-})
+#define raw_smp_processor_id() (this_cpu_read(cpu_number))
+#define stack_smp_processor_id() raw_smp_processor_id()
#define safe_smp_processor_id() smp_processor_id()
#endif
* head_32 for boot CPU and setup_per_cpu_areas() for others.
*/
#define GDT_STACK_CANARY_INIT \
- [GDT_ENTRY_STACK_CANARY] = GDT_ENTRY_INIT(0x4090, 0, 0x18),
+ [GDT_ENTRY_STACK_CANARY] = GDT_ENTRY_INIT(0x4090, 0, 0x17),
/*
* Initialize the stackprotector canary value.
static inline void load_stack_canary_segment(void)
{
-#ifdef CONFIG_X86_32
+#if defined(CONFIG_X86_32) && !defined(CONFIG_PAX_MEMORY_UDEREF)
asm volatile ("mov %0, %%gs" : : "r" (0));
#endif
}
extern int kstack_depth_to_print;
-struct thread_info;
+struct task_struct;
struct stacktrace_ops;
-typedef unsigned long (*walk_stack_t)(struct thread_info *tinfo,
- unsigned long *stack,
- unsigned long bp,
- const struct stacktrace_ops *ops,
- void *data,
- unsigned long *end,
- int *graph);
-
-extern unsigned long
-print_context_stack(struct thread_info *tinfo,
- unsigned long *stack, unsigned long bp,
- const struct stacktrace_ops *ops, void *data,
- unsigned long *end, int *graph);
-
-extern unsigned long
-print_context_stack_bp(struct thread_info *tinfo,
- unsigned long *stack, unsigned long bp,
- const struct stacktrace_ops *ops, void *data,
- unsigned long *end, int *graph);
+typedef unsigned long walk_stack_t(struct task_struct *task,
+ void *stack_start,
+ unsigned long *stack,
+ unsigned long bp,
+ const struct stacktrace_ops *ops,
+ void *data,
+ unsigned long *end,
+ int *graph);
+
+extern walk_stack_t print_context_stack;
+extern walk_stack_t print_context_stack_bp;
/* Generic stack tracer with callbacks */
void (*address)(void *data, unsigned long address, int reliable);
/* On negative return stop dumping */
int (*stack)(void *data, char *name);
- walk_stack_t walk_stack;
+ walk_stack_t *walk_stack;
};
void dump_trace(struct task_struct *tsk, struct pt_regs *regs,
"call __switch_to\n\t" \
"movq "__percpu_arg([current_task])",%%rsi\n\t" \
__switch_canary \
- "movq %P[thread_info](%%rsi),%%r8\n\t" \
+ "movq "__percpu_arg([thread_info])",%%r8\n\t" \
"movq %%rax,%%rdi\n\t" \
"testl %[_tif_fork],%P[ti_flags](%%r8)\n\t" \
"jnz ret_from_fork\n\t" \
[threadrsp] "i" (offsetof(struct task_struct, thread.sp)), \
[ti_flags] "i" (offsetof(struct thread_info, flags)), \
[_tif_fork] "i" (_TIF_FORK), \
- [thread_info] "i" (offsetof(struct task_struct, stack)), \
+ [thread_info] "m" (current_tinfo), \
[current_task] "m" (current_task) \
__switch_canary_iparam \
: "memory", "cc" __EXTRA_CLOBBER)
#include <linux/atomic.h>
struct thread_info {
- struct task_struct *task; /* main task structure */
struct exec_domain *exec_domain; /* execution domain */
__u32 flags; /* low level flags */
__u32 status; /* thread synchronous flags */
mm_segment_t addr_limit;
struct restart_block restart_block;
void __user *sysenter_return;
+ unsigned long lowest_stack;
unsigned int sig_on_uaccess_error:1;
unsigned int uaccess_err:1; /* uaccess failed */
};
-#define INIT_THREAD_INFO(tsk) \
+#define INIT_THREAD_INFO \
{ \
- .task = &tsk, \
.exec_domain = &default_exec_domain, \
.flags = 0, \
.cpu = 0, \
}, \
}
-#define init_thread_info (init_thread_union.thread_info)
+#define init_thread_info (init_thread_union.stack)
#define init_stack (init_thread_union.stack)
#else /* !__ASSEMBLY__ */
#define TIF_SYSCALL_TRACEPOINT 28 /* syscall tracepoint instrumentation */
#define TIF_ADDR32 29 /* 32-bit address space on 64 bits */
#define TIF_X32 30 /* 32-bit native x86-64 binary */
+#define TIF_GRSEC_SETXID 31 /* update credentials on syscall entry/exit */
#define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE)
#define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME)
#define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT)
#define _TIF_ADDR32 (1 << TIF_ADDR32)
#define _TIF_X32 (1 << TIF_X32)
+#define _TIF_GRSEC_SETXID (1 << TIF_GRSEC_SETXID)
/* work to do in syscall_trace_enter() */
#define _TIF_WORK_SYSCALL_ENTRY \
(_TIF_SYSCALL_TRACE | _TIF_SYSCALL_EMU | _TIF_SYSCALL_AUDIT | \
_TIF_SECCOMP | _TIF_SINGLESTEP | _TIF_SYSCALL_TRACEPOINT | \
- _TIF_NOHZ)
+ _TIF_NOHZ | _TIF_GRSEC_SETXID)
/* work to do in syscall_trace_leave() */
#define _TIF_WORK_SYSCALL_EXIT \
(_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SINGLESTEP | \
- _TIF_SYSCALL_TRACEPOINT | _TIF_NOHZ)
+ _TIF_SYSCALL_TRACEPOINT | _TIF_NOHZ | _TIF_GRSEC_SETXID)
/* work to do on interrupt/exception return */
#define _TIF_WORK_MASK \
/* work to do on any return to user space */
#define _TIF_ALLWORK_MASK \
((0x0000FFFF & ~_TIF_SECCOMP) | _TIF_SYSCALL_TRACEPOINT | \
- _TIF_NOHZ)
+ _TIF_NOHZ | _TIF_GRSEC_SETXID)
/* Only used for 64 bit */
#define _TIF_DO_NOTIFY_MASK \
#define _TIF_WORK_CTXSW_NEXT (_TIF_WORK_CTXSW)
#define STACK_WARN (THREAD_SIZE/8)
-#define KERNEL_STACK_OFFSET (5*(BITS_PER_LONG/8))
/*
* macros/functions for gaining access to the thread information structure
DECLARE_PER_CPU(unsigned long, kernel_stack);
+DECLARE_PER_CPU(struct thread_info *, current_tinfo);
+
static inline struct thread_info *current_thread_info(void)
{
- struct thread_info *ti;
- ti = (void *)(this_cpu_read_stable(kernel_stack) +
- KERNEL_STACK_OFFSET - THREAD_SIZE);
- return ti;
+ return this_cpu_read_stable(current_tinfo);
}
#else /* !__ASSEMBLY__ */
/* how to get the thread information struct from ASM */
#define GET_THREAD_INFO(reg) \
- _ASM_MOV PER_CPU_VAR(kernel_stack),reg ; \
- _ASM_SUB $(THREAD_SIZE-KERNEL_STACK_OFFSET),reg ;
-
-/*
- * Same if PER_CPU_VAR(kernel_stack) is, perhaps with some offset, already in
- * a certain register (to be used in assembler memory operands).
- */
-#define THREAD_INFO(reg, off) KERNEL_STACK_OFFSET+(off)-THREAD_SIZE(reg)
+ _ASM_MOV PER_CPU_VAR(current_tinfo),reg ;
#endif
extern void arch_task_cache_init(void);
extern int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src);
extern void arch_release_task_struct(struct task_struct *tsk);
+
+#define __HAVE_THREAD_FUNCTIONS
+#define task_thread_info(task) (&(task)->tinfo)
+#define task_stack_page(task) ((task)->stack)
+#define setup_thread_stack(p, org) do {} while (0)
+#define end_of_stack(p) ((unsigned long *)task_stack_page(p) + 1)
+
#endif
#endif /* _ASM_X86_THREAD_INFO_H */
static inline void __native_flush_tlb(void)
{
+ if (static_cpu_has(X86_FEATURE_INVPCID)) {
+ u64 descriptor[2];
+
+ descriptor[0] = PCID_KERNEL;
+ asm volatile(__ASM_INVPCID : : "d"(&descriptor), "a"(INVPCID_ALL_NONGLOBAL) : "memory");
+ return;
+ }
+
+#if defined(CONFIG_X86_64) && defined(CONFIG_PAX_MEMORY_UDEREF)
+ if (static_cpu_has(X86_FEATURE_PCID)) {
+ unsigned int cpu = raw_get_cpu();
+
+ native_write_cr3(__pa(get_cpu_pgd(cpu, user)) | PCID_USER);
+ native_write_cr3(__pa(get_cpu_pgd(cpu, kernel)) | PCID_KERNEL);
+ raw_put_cpu_no_resched();
+ return;
+ }
+#endif
+
native_write_cr3(native_read_cr3());
}
static inline void __native_flush_tlb_global_irq_disabled(void)
{
- unsigned long cr4;
-
- cr4 = native_read_cr4();
- /* clear PGE */
- native_write_cr4(cr4 & ~X86_CR4_PGE);
- /* write old PGE again and flush TLBs */
- native_write_cr4(cr4);
+ if (static_cpu_has(X86_FEATURE_INVPCID)) {
+ u64 descriptor[2];
+
+ descriptor[0] = PCID_KERNEL;
+ asm volatile(__ASM_INVPCID : : "d"(&descriptor), "a"(INVPCID_ALL_GLOBAL) : "memory");
+ } else {
+ unsigned long cr4;
+
+ cr4 = native_read_cr4();
+ /* clear PGE */
+ native_write_cr4(cr4 & ~X86_CR4_PGE);
+ /* write old PGE again and flush TLBs */
+ native_write_cr4(cr4);
+ }
}
static inline void __native_flush_tlb_global(void)
static inline void __native_flush_tlb_single(unsigned long addr)
{
+ if (static_cpu_has(X86_FEATURE_INVPCID)) {
+ u64 descriptor[2];
+
+ descriptor[0] = PCID_KERNEL;
+ descriptor[1] = addr;
+
+#if defined(CONFIG_X86_64) && defined(CONFIG_PAX_MEMORY_UDEREF)
+ if (!static_cpu_has(X86_FEATURE_STRONGUDEREF) || addr >= TASK_SIZE_MAX) {
+ if (addr < TASK_SIZE_MAX)
+ descriptor[1] += pax_user_shadow_base;
+ asm volatile(__ASM_INVPCID : : "d"(&descriptor), "a"(INVPCID_SINGLE_ADDRESS) : "memory");
+ }
+
+ descriptor[0] = PCID_USER;
+ descriptor[1] = addr;
+#endif
+
+ asm volatile(__ASM_INVPCID : : "d"(&descriptor), "a"(INVPCID_SINGLE_ADDRESS) : "memory");
+ return;
+ }
+
+#if defined(CONFIG_X86_64) && defined(CONFIG_PAX_MEMORY_UDEREF)
+ if (static_cpu_has(X86_FEATURE_PCID)) {
+ unsigned int cpu = raw_get_cpu();
+
+ native_write_cr3(__pa(get_cpu_pgd(cpu, user)) | PCID_USER | PCID_NOFLUSH);
+ asm volatile("invlpg (%0)" ::"r" (addr) : "memory");
+ native_write_cr3(__pa(get_cpu_pgd(cpu, kernel)) | PCID_KERNEL | PCID_NOFLUSH);
+ raw_put_cpu_no_resched();
+
+ if (!static_cpu_has(X86_FEATURE_STRONGUDEREF) && addr < TASK_SIZE_MAX)
+ addr += pax_user_shadow_base;
+ }
+#endif
+
asm volatile("invlpg (%0)" ::"r" (addr) : "memory");
}
#include <linux/compiler.h>
#include <linux/thread_info.h>
#include <linux/string.h>
+#include <linux/spinlock.h>
#include <asm/asm.h>
#include <asm/page.h>
#include <asm/smap.h>
#define get_ds() (KERNEL_DS)
#define get_fs() (current_thread_info()->addr_limit)
+#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_MEMORY_UDEREF)
+void __set_fs(mm_segment_t x);
+void set_fs(mm_segment_t x);
+#else
#define set_fs(x) (current_thread_info()->addr_limit = (x))
+#endif
#define segment_eq(a, b) ((a).seg == (b).seg)
* checks that the pointer is in the user space range - after calling
* this function, memory access functions may still return -EFAULT.
*/
-#define access_ok(type, addr, size) \
- likely(!__range_not_ok(addr, size, user_addr_max()))
+extern int _cond_resched(void);
+#define access_ok_noprefault(type, addr, size) (likely(!__range_not_ok(addr, size, user_addr_max())))
+#define access_ok(type, addr, size) \
+({ \
+ unsigned long __size = size; \
+ unsigned long __addr = (unsigned long)addr; \
+ bool __ret_ao = __range_not_ok(__addr, __size, user_addr_max()) == 0;\
+ if (__ret_ao && __size) { \
+ unsigned long __addr_ao = __addr & PAGE_MASK; \
+ unsigned long __end_ao = __addr + __size - 1; \
+ if (unlikely((__end_ao ^ __addr_ao) & PAGE_MASK)) { \
+ while (__addr_ao <= __end_ao) { \
+ char __c_ao; \
+ __addr_ao += PAGE_SIZE; \
+ if (__size > PAGE_SIZE) \
+ _cond_resched(); \
+ if (__get_user(__c_ao, (char __user *)__addr)) \
+ break; \
+ if (type != VERIFY_WRITE) { \
+ __addr = __addr_ao; \
+ continue; \
+ } \
+ if (__put_user(__c_ao, (char __user *)__addr)) \
+ break; \
+ __addr = __addr_ao; \
+ } \
+ } \
+ } \
+ __ret_ao; \
+})
/*
* The exception table consists of pairs of addresses relative to the
register __inttype(*(ptr)) __val_gu asm("%"_ASM_DX); \
__chk_user_ptr(ptr); \
might_fault(); \
+ pax_open_userland(); \
asm volatile("call __get_user_%P3" \
: "=a" (__ret_gu), "=r" (__val_gu) \
: "0" (ptr), "i" (sizeof(*(ptr)))); \
(x) = (__typeof__(*(ptr))) __val_gu; \
+ pax_close_userland(); \
__ret_gu; \
})
asm volatile("call __put_user_" #size : "=a" (__ret_pu) \
: "0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
-
+#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_MEMORY_UDEREF)
+#define __copyuser_seg "gs;"
+#define __COPYUSER_SET_ES "pushl %%gs; popl %%es\n"
+#define __COPYUSER_RESTORE_ES "pushl %%ss; popl %%es\n"
+#else
+#define __copyuser_seg
+#define __COPYUSER_SET_ES
+#define __COPYUSER_RESTORE_ES
+#endif
#ifdef CONFIG_X86_32
#define __put_user_asm_u64(x, addr, err, errret) \
asm volatile(ASM_STAC "\n" \
- "1: movl %%eax,0(%2)\n" \
- "2: movl %%edx,4(%2)\n" \
+ "1: "__copyuser_seg"movl %%eax,0(%2)\n" \
+ "2: "__copyuser_seg"movl %%edx,4(%2)\n" \
"3: " ASM_CLAC "\n" \
".section .fixup,\"ax\"\n" \
"4: movl %3,%0\n" \
#define __put_user_asm_ex_u64(x, addr) \
asm volatile(ASM_STAC "\n" \
- "1: movl %%eax,0(%1)\n" \
- "2: movl %%edx,4(%1)\n" \
+ "1: "__copyuser_seg"movl %%eax,0(%1)\n" \
+ "2: "__copyuser_seg"movl %%edx,4(%1)\n" \
"3: " ASM_CLAC "\n" \
_ASM_EXTABLE_EX(1b, 2b) \
_ASM_EXTABLE_EX(2b, 3b) \
__typeof__(*(ptr)) __pu_val; \
__chk_user_ptr(ptr); \
might_fault(); \
- __pu_val = x; \
+ __pu_val = (x); \
+ pax_open_userland(); \
switch (sizeof(*(ptr))) { \
case 1: \
__put_user_x(1, __pu_val, ptr, __ret_pu); \
__put_user_x(X, __pu_val, ptr, __ret_pu); \
break; \
} \
+ pax_close_userland(); \
__ret_pu; \
})
} while (0)
#define __get_user_asm(x, addr, err, itype, rtype, ltype, errret) \
+do { \
+ pax_open_userland(); \
asm volatile(ASM_STAC "\n" \
- "1: mov"itype" %2,%"rtype"1\n" \
+ "1: "__copyuser_seg"mov"itype" %2,%"rtype"1\n"\
"2: " ASM_CLAC "\n" \
".section .fixup,\"ax\"\n" \
"3: mov %3,%0\n" \
" jmp 2b\n" \
".previous\n" \
_ASM_EXTABLE(1b, 3b) \
- : "=r" (err), ltype(x) \
- : "m" (__m(addr)), "i" (errret), "0" (err))
+ : "=r" (err), ltype (x) \
+ : "m" (__m(addr)), "i" (errret), "0" (err)); \
+ pax_close_userland(); \
+} while (0)
#define __get_user_size_ex(x, ptr, size) \
do { \
} while (0)
#define __get_user_asm_ex(x, addr, itype, rtype, ltype) \
- asm volatile("1: mov"itype" %1,%"rtype"0\n" \
+ asm volatile("1: "__copyuser_seg"mov"itype" %1,%"rtype"0\n"\
"2:\n" \
_ASM_EXTABLE_EX(1b, 2b) \
: ltype(x) : "m" (__m(addr)))
int __gu_err; \
unsigned long __gu_val; \
__get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT); \
- (x) = (__force __typeof__(*(ptr)))__gu_val; \
+ (x) = (__typeof__(*(ptr)))__gu_val; \
__gu_err; \
})
/* FIXME: this hack is definitely wrong -AK */
struct __large_struct { unsigned long buf[100]; };
-#define __m(x) (*(struct __large_struct __user *)(x))
+#if defined(CONFIG_X86_64) && defined(CONFIG_PAX_MEMORY_UDEREF)
+#define ____m(x) \
+({ \
+ unsigned long ____x = (unsigned long)(x); \
+ if (____x < pax_user_shadow_base) \
+ ____x += pax_user_shadow_base; \
+ (typeof(x))____x; \
+})
+#else
+#define ____m(x) (x)
+#endif
+#define __m(x) (*(struct __large_struct __user *)____m(x))
/*
* Tell gcc we read from memory instead of writing: this is because
* aliasing issues.
*/
#define __put_user_asm(x, addr, err, itype, rtype, ltype, errret) \
+do { \
+ pax_open_userland(); \
asm volatile(ASM_STAC "\n" \
- "1: mov"itype" %"rtype"1,%2\n" \
+ "1: "__copyuser_seg"mov"itype" %"rtype"1,%2\n"\
"2: " ASM_CLAC "\n" \
".section .fixup,\"ax\"\n" \
"3: mov %3,%0\n" \
".previous\n" \
_ASM_EXTABLE(1b, 3b) \
: "=r"(err) \
- : ltype(x), "m" (__m(addr)), "i" (errret), "0" (err))
+ : ltype (x), "m" (__m(addr)), "i" (errret), "0" (err));\
+ pax_close_userland(); \
+} while (0)
#define __put_user_asm_ex(x, addr, itype, rtype, ltype) \
- asm volatile("1: mov"itype" %"rtype"0,%1\n" \
+ asm volatile("1: "__copyuser_seg"mov"itype" %"rtype"0,%1\n"\
"2:\n" \
_ASM_EXTABLE_EX(1b, 2b) \
: : ltype(x), "m" (__m(addr)))
*/
#define uaccess_try do { \
current_thread_info()->uaccess_err = 0; \
+ pax_open_userland(); \
stac(); \
barrier();
#define uaccess_catch(err) \
clac(); \
+ pax_close_userland(); \
(err) |= (current_thread_info()->uaccess_err ? -EFAULT : 0); \
} while (0)
* On error, the variable @x is set to zero.
*/
+#if defined(CONFIG_X86_64) && defined(CONFIG_PAX_MEMORY_UDEREF)
+#define __get_user(x, ptr) get_user((x), (ptr))
+#else
#define __get_user(x, ptr) \
__get_user_nocheck((x), (ptr), sizeof(*(ptr)))
+#endif
/**
* __put_user: - Write a simple value into user space, with less checking.
* Returns zero on success, or -EFAULT on error.
*/
+#if defined(CONFIG_X86_64) && defined(CONFIG_PAX_MEMORY_UDEREF)
+#define __put_user(x, ptr) put_user((x), (ptr))
+#else
#define __put_user(x, ptr) \
__put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
+#endif
#define __get_user_unaligned __get_user
#define __put_user_unaligned __put_user
#define get_user_ex(x, ptr) do { \
unsigned long __gue_val; \
__get_user_size_ex((__gue_val), (ptr), (sizeof(*(ptr)))); \
- (x) = (__force __typeof__(*(ptr)))__gue_val; \
+ (x) = (__typeof__(*(ptr)))__gue_val; \
} while (0)
#define put_user_try uaccess_try
extern __must_check long strnlen_user(const char __user *str, long n);
unsigned long __must_check clear_user(void __user *mem, unsigned long len);
-unsigned long __must_check __clear_user(void __user *mem, unsigned long len);
+unsigned long __must_check __clear_user(void __user *mem, unsigned long len) __size_overflow(2);
extern void __cmpxchg_wrong_size(void)
__compiletime_error("Bad argument size for cmpxchg");
__typeof__(ptr) __uval = (uval); \
__typeof__(*(ptr)) __old = (old); \
__typeof__(*(ptr)) __new = (new); \
+ pax_open_userland(); \
switch (size) { \
case 1: \
{ \
asm volatile("\t" ASM_STAC "\n" \
- "1:\t" LOCK_PREFIX "cmpxchgb %4, %2\n" \
+ "1:\t" LOCK_PREFIX __copyuser_seg"cmpxchgb %4, %2\n"\
"2:\t" ASM_CLAC "\n" \
"\t.section .fixup, \"ax\"\n" \
"3:\tmov %3, %0\n" \
"\tjmp 2b\n" \
"\t.previous\n" \
_ASM_EXTABLE(1b, 3b) \
- : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \
+ : "+r" (__ret), "=a" (__old), "+m" (*____m(ptr))\
: "i" (-EFAULT), "q" (__new), "1" (__old) \
: "memory" \
); \
case 2: \
{ \
asm volatile("\t" ASM_STAC "\n" \
- "1:\t" LOCK_PREFIX "cmpxchgw %4, %2\n" \
+ "1:\t" LOCK_PREFIX __copyuser_seg"cmpxchgw %4, %2\n"\
"2:\t" ASM_CLAC "\n" \
"\t.section .fixup, \"ax\"\n" \
"3:\tmov %3, %0\n" \
"\tjmp 2b\n" \
"\t.previous\n" \
_ASM_EXTABLE(1b, 3b) \
- : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \
+ : "+r" (__ret), "=a" (__old), "+m" (*____m(ptr))\
: "i" (-EFAULT), "r" (__new), "1" (__old) \
: "memory" \
); \
case 4: \
{ \
asm volatile("\t" ASM_STAC "\n" \
- "1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n" \
+ "1:\t" LOCK_PREFIX __copyuser_seg"cmpxchgl %4, %2\n"\
"2:\t" ASM_CLAC "\n" \
"\t.section .fixup, \"ax\"\n" \
"3:\tmov %3, %0\n" \
"\tjmp 2b\n" \
"\t.previous\n" \
_ASM_EXTABLE(1b, 3b) \
- : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \
+ : "+r" (__ret), "=a" (__old), "+m" (*____m(ptr))\
: "i" (-EFAULT), "r" (__new), "1" (__old) \
: "memory" \
); \
__cmpxchg_wrong_size(); \
\
asm volatile("\t" ASM_STAC "\n" \
- "1:\t" LOCK_PREFIX "cmpxchgq %4, %2\n" \
+ "1:\t" LOCK_PREFIX __copyuser_seg"cmpxchgq %4, %2\n"\
"2:\t" ASM_CLAC "\n" \
"\t.section .fixup, \"ax\"\n" \
"3:\tmov %3, %0\n" \
"\tjmp 2b\n" \
"\t.previous\n" \
_ASM_EXTABLE(1b, 3b) \
- : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \
+ : "+r" (__ret), "=a" (__old), "+m" (*____m(ptr))\
: "i" (-EFAULT), "r" (__new), "1" (__old) \
: "memory" \
); \
default: \
__cmpxchg_wrong_size(); \
} \
+ pax_close_userland(); \
*__uval = __old; \
__ret; \
})
#define ARCH_HAS_NOCACHE_UACCESS 1
-#ifdef CONFIG_X86_32
-# include <asm/uaccess_32.h>
-#else
-# include <asm/uaccess_64.h>
-#endif
-
-unsigned long __must_check _copy_from_user(void *to, const void __user *from,
- unsigned n);
-unsigned long __must_check _copy_to_user(void __user *to, const void *from,
- unsigned n);
-
#ifdef CONFIG_DEBUG_STRICT_USER_COPY_CHECKS
# define copy_user_diag __compiletime_error
#else
extern void copy_user_diag("copy_from_user() buffer size is too small")
copy_from_user_overflow(void);
extern void copy_user_diag("copy_to_user() buffer size is too small")
-copy_to_user_overflow(void) __asm__("copy_from_user_overflow");
+copy_to_user_overflow(void);
#undef copy_user_diag
extern void
__compiletime_warning("copy_to_user() buffer size is not provably correct")
-__copy_to_user_overflow(void) __asm__("copy_from_user_overflow");
+__copy_to_user_overflow(void) __asm__("copy_to_user_overflow");
#define __copy_to_user_overflow(size, count) __copy_to_user_overflow()
#else
#endif
+#ifdef CONFIG_X86_32
+# include <asm/uaccess_32.h>
+#else
+# include <asm/uaccess_64.h>
+#endif
+
static inline unsigned long __must_check
copy_from_user(void *to, const void __user *from, unsigned long n)
{
- int sz = __compiletime_object_size(to);
+ size_t sz = __compiletime_object_size(to);
might_fault();
* case, and do only runtime checking for non-constant sizes.
*/
- if (likely(sz < 0 || sz >= n))
- n = _copy_from_user(to, from, n);
- else if(__builtin_constant_p(n))
- copy_from_user_overflow();
- else
- __copy_from_user_overflow(sz, n);
+ if (likely(sz != (size_t)-1 && sz < n)) {
+ if(__builtin_constant_p(n))
+ copy_from_user_overflow();
+ else
+ __copy_from_user_overflow(sz, n);
+ } else if (access_ok(VERIFY_READ, from, n))
+ n = __copy_from_user(to, from, n);
+ else if ((long)n > 0)
+ memset(to, 0, n);
return n;
}
static inline unsigned long __must_check
copy_to_user(void __user *to, const void *from, unsigned long n)
{
- int sz = __compiletime_object_size(from);
+ size_t sz = __compiletime_object_size(from);
might_fault();
/* See the comment in copy_from_user() above. */
- if (likely(sz < 0 || sz >= n))
- n = _copy_to_user(to, from, n);
- else if(__builtin_constant_p(n))
- copy_to_user_overflow();
- else
- __copy_to_user_overflow(sz, n);
+ if (likely(sz != (size_t)-1 && sz < n)) {
+ if(__builtin_constant_p(n))
+ copy_to_user_overflow();
+ else
+ __copy_to_user_overflow(sz, n);
+ } else if (access_ok(VERIFY_WRITE, to, n))
+ n = __copy_to_user(to, from, n);
return n;
}
* anything, so this is accurate.
*/
-static __always_inline unsigned long __must_check
+static __always_inline __size_overflow(3) unsigned long __must_check
__copy_to_user_inatomic(void __user *to, const void *from, unsigned long n)
{
+ if ((long)n < 0)
+ return n;
+
+ check_object_size(from, n, true);
+
if (__builtin_constant_p(n)) {
unsigned long ret;
__copy_to_user(void __user *to, const void *from, unsigned long n)
{
might_fault();
+
return __copy_to_user_inatomic(to, from, n);
}
-static __always_inline unsigned long
+static __always_inline __size_overflow(3) unsigned long
__copy_from_user_inatomic(void *to, const void __user *from, unsigned long n)
{
+ if ((long)n < 0)
+ return n;
+
/* Avoid zeroing the tail if the copy fails..
* If 'n' is constant and 1, 2, or 4, we do still zero on a failure,
* but as the zeroing behaviour is only significant when n is not
__copy_from_user(void *to, const void __user *from, unsigned long n)
{
might_fault();
+
+ if ((long)n < 0)
+ return n;
+
+ check_object_size(to, n, false);
+
if (__builtin_constant_p(n)) {
unsigned long ret;
const void __user *from, unsigned long n)
{
might_fault();
+
+ if ((long)n < 0)
+ return n;
+
if (__builtin_constant_p(n)) {
unsigned long ret;
__copy_from_user_inatomic_nocache(void *to, const void __user *from,
unsigned long n)
{
- return __copy_from_user_ll_nocache_nozero(to, from, n);
+ if ((long)n < 0)
+ return n;
+
+ return __copy_from_user_ll_nocache_nozero(to, from, n);
}
#endif /* _ASM_X86_UACCESS_32_H */
#include <asm/alternative.h>
#include <asm/cpufeature.h>
#include <asm/page.h>
+#include <asm/pgtable.h>
+
+#define set_fs(x) (current_thread_info()->addr_limit = (x))
/*
* Copy To/From Userspace
__must_check unsigned long
copy_user_generic_unrolled(void *to, const void *from, unsigned len);
-static __always_inline __must_check unsigned long
-copy_user_generic(void *to, const void *from, unsigned len)
+static __always_inline __must_check __size_overflow(3) unsigned long
+copy_user_generic(void *to, const void *from, unsigned long len)
{
unsigned ret;
}
__must_check unsigned long
-copy_in_user(void __user *to, const void __user *from, unsigned len);
+copy_in_user(void __user *to, const void __user *from, unsigned long len);
static __always_inline __must_check
-int __copy_from_user_nocheck(void *dst, const void __user *src, unsigned size)
+unsigned long __copy_from_user_nocheck(void *dst, const void __user *src, unsigned long size)
{
- int ret = 0;
+ size_t sz = __compiletime_object_size(dst);
+ unsigned ret = 0;
+
+ if (size > INT_MAX)
+ return size;
+
+ check_object_size(dst, size, false);
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ if (!access_ok_noprefault(VERIFY_READ, src, size))
+ return size;
+#endif
+
+ if (unlikely(sz != (size_t)-1 && sz < size)) {
+ if(__builtin_constant_p(size))
+ copy_from_user_overflow();
+ else
+ __copy_from_user_overflow(sz, size);
+ return size;
+ }
if (!__builtin_constant_p(size))
- return copy_user_generic(dst, (__force void *)src, size);
+ return copy_user_generic(dst, (__force_kernel const void *)____m(src), size);
switch (size) {
- case 1:__get_user_asm(*(u8 *)dst, (u8 __user *)src,
+ case 1:__get_user_asm(*(u8 *)dst, (const u8 __user *)src,
ret, "b", "b", "=q", 1);
return ret;
- case 2:__get_user_asm(*(u16 *)dst, (u16 __user *)src,
+ case 2:__get_user_asm(*(u16 *)dst, (const u16 __user *)src,
ret, "w", "w", "=r", 2);
return ret;
- case 4:__get_user_asm(*(u32 *)dst, (u32 __user *)src,
+ case 4:__get_user_asm(*(u32 *)dst, (const u32 __user *)src,
ret, "l", "k", "=r", 4);
return ret;
- case 8:__get_user_asm(*(u64 *)dst, (u64 __user *)src,
+ case 8:__get_user_asm(*(u64 *)dst, (const u64 __user *)src,
ret, "q", "", "=r", 8);
return ret;
case 10:
- __get_user_asm(*(u64 *)dst, (u64 __user *)src,
+ __get_user_asm(*(u64 *)dst, (const u64 __user *)src,
ret, "q", "", "=r", 10);
if (unlikely(ret))
return ret;
__get_user_asm(*(u16 *)(8 + (char *)dst),
- (u16 __user *)(8 + (char __user *)src),
+ (const u16 __user *)(8 + (const char __user *)src),
ret, "w", "w", "=r", 2);
return ret;
case 16:
- __get_user_asm(*(u64 *)dst, (u64 __user *)src,
+ __get_user_asm(*(u64 *)dst, (const u64 __user *)src,
ret, "q", "", "=r", 16);
if (unlikely(ret))
return ret;
__get_user_asm(*(u64 *)(8 + (char *)dst),
- (u64 __user *)(8 + (char __user *)src),
+ (const u64 __user *)(8 + (const char __user *)src),
ret, "q", "", "=r", 8);
return ret;
default:
- return copy_user_generic(dst, (__force void *)src, size);
+ return copy_user_generic(dst, (__force_kernel const void *)____m(src), size);
}
}
static __always_inline __must_check
-int __copy_from_user(void *dst, const void __user *src, unsigned size)
+unsigned long __copy_from_user(void *dst, const void __user *src, unsigned long size)
{
might_fault();
return __copy_from_user_nocheck(dst, src, size);
}
static __always_inline __must_check
-int __copy_to_user_nocheck(void __user *dst, const void *src, unsigned size)
+unsigned long __copy_to_user_nocheck(void __user *dst, const void *src, unsigned long size)
{
- int ret = 0;
+ size_t sz = __compiletime_object_size(src);
+ unsigned ret = 0;
+
+ if (size > INT_MAX)
+ return size;
+
+ check_object_size(src, size, true);
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ if (!access_ok_noprefault(VERIFY_WRITE, dst, size))
+ return size;
+#endif
+
+ if (unlikely(sz != (size_t)-1 && sz < size)) {
+ if(__builtin_constant_p(size))
+ copy_to_user_overflow();
+ else
+ __copy_to_user_overflow(sz, size);
+ return size;
+ }
if (!__builtin_constant_p(size))
- return copy_user_generic((__force void *)dst, src, size);
+ return copy_user_generic((__force_kernel void *)____m(dst), src, size);
switch (size) {
- case 1:__put_user_asm(*(u8 *)src, (u8 __user *)dst,
+ case 1:__put_user_asm(*(const u8 *)src, (u8 __user *)dst,
ret, "b", "b", "iq", 1);
return ret;
- case 2:__put_user_asm(*(u16 *)src, (u16 __user *)dst,
+ case 2:__put_user_asm(*(const u16 *)src, (u16 __user *)dst,
ret, "w", "w", "ir", 2);
return ret;
- case 4:__put_user_asm(*(u32 *)src, (u32 __user *)dst,
+ case 4:__put_user_asm(*(const u32 *)src, (u32 __user *)dst,
ret, "l", "k", "ir", 4);
return ret;
- case 8:__put_user_asm(*(u64 *)src, (u64 __user *)dst,
+ case 8:__put_user_asm(*(const u64 *)src, (u64 __user *)dst,
ret, "q", "", "er", 8);
return ret;
case 10:
- __put_user_asm(*(u64 *)src, (u64 __user *)dst,
+ __put_user_asm(*(const u64 *)src, (u64 __user *)dst,
ret, "q", "", "er", 10);
if (unlikely(ret))
return ret;
asm("":::"memory");
- __put_user_asm(4[(u16 *)src], 4 + (u16 __user *)dst,
+ __put_user_asm(4[(const u16 *)src], 4 + (u16 __user *)dst,
ret, "w", "w", "ir", 2);
return ret;
case 16:
- __put_user_asm(*(u64 *)src, (u64 __user *)dst,
+ __put_user_asm(*(const u64 *)src, (u64 __user *)dst,
ret, "q", "", "er", 16);
if (unlikely(ret))
return ret;
asm("":::"memory");
- __put_user_asm(1[(u64 *)src], 1 + (u64 __user *)dst,
+ __put_user_asm(1[(const u64 *)src], 1 + (u64 __user *)dst,
ret, "q", "", "er", 8);
return ret;
default:
- return copy_user_generic((__force void *)dst, src, size);
+ return copy_user_generic((__force_kernel void *)____m(dst), src, size);
}
}
static __always_inline __must_check
-int __copy_to_user(void __user *dst, const void *src, unsigned size)
+unsigned long __copy_to_user(void __user *dst, const void *src, unsigned long size)
{
might_fault();
return __copy_to_user_nocheck(dst, src, size);
}
static __always_inline __must_check
-int __copy_in_user(void __user *dst, const void __user *src, unsigned size)
+unsigned long __copy_in_user(void __user *dst, const void __user *src, unsigned size)
{
- int ret = 0;
+ unsigned ret = 0;
might_fault();
+
+ if (size > INT_MAX)
+ return size;
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ if (!access_ok_noprefault(VERIFY_READ, src, size))
+ return size;
+ if (!access_ok_noprefault(VERIFY_WRITE, dst, size))
+ return size;
+#endif
+
if (!__builtin_constant_p(size))
- return copy_user_generic((__force void *)dst,
- (__force void *)src, size);
+ return copy_user_generic((__force_kernel void *)____m(dst),
+ (__force_kernel const void *)____m(src), size);
switch (size) {
case 1: {
u8 tmp;
- __get_user_asm(tmp, (u8 __user *)src,
+ __get_user_asm(tmp, (const u8 __user *)src,
ret, "b", "b", "=q", 1);
if (likely(!ret))
__put_user_asm(tmp, (u8 __user *)dst,
}
case 2: {
u16 tmp;
- __get_user_asm(tmp, (u16 __user *)src,
+ __get_user_asm(tmp, (const u16 __user *)src,
ret, "w", "w", "=r", 2);
if (likely(!ret))
__put_user_asm(tmp, (u16 __user *)dst,
case 4: {
u32 tmp;
- __get_user_asm(tmp, (u32 __user *)src,
+ __get_user_asm(tmp, (const u32 __user *)src,
ret, "l", "k", "=r", 4);
if (likely(!ret))
__put_user_asm(tmp, (u32 __user *)dst,
}
case 8: {
u64 tmp;
- __get_user_asm(tmp, (u64 __user *)src,
+ __get_user_asm(tmp, (const u64 __user *)src,
ret, "q", "", "=r", 8);
if (likely(!ret))
__put_user_asm(tmp, (u64 __user *)dst,
return ret;
}
default:
- return copy_user_generic((__force void *)dst,
- (__force void *)src, size);
+ return copy_user_generic((__force_kernel void *)____m(dst),
+ (__force_kernel const void *)____m(src), size);
}
}
-static __must_check __always_inline int
-__copy_from_user_inatomic(void *dst, const void __user *src, unsigned size)
+static __must_check __always_inline unsigned long
+__copy_from_user_inatomic(void *dst, const void __user *src, unsigned long size)
{
return __copy_from_user_nocheck(dst, src, size);
}
-static __must_check __always_inline int
-__copy_to_user_inatomic(void __user *dst, const void *src, unsigned size)
+static __must_check __always_inline unsigned long
+__copy_to_user_inatomic(void __user *dst, const void *src, unsigned long size)
{
return __copy_to_user_nocheck(dst, src, size);
}
-extern long __copy_user_nocache(void *dst, const void __user *src,
- unsigned size, int zerorest);
+extern unsigned long __copy_user_nocache(void *dst, const void __user *src,
+ unsigned long size, int zerorest);
-static inline int
-__copy_from_user_nocache(void *dst, const void __user *src, unsigned size)
+static inline unsigned long
+__copy_from_user_nocache(void *dst, const void __user *src, unsigned long size)
{
might_fault();
+
+ if (size > INT_MAX)
+ return size;
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ if (!access_ok_noprefault(VERIFY_READ, src, size))
+ return size;
+#endif
+
return __copy_user_nocache(dst, src, size, 1);
}
-static inline int
+static inline unsigned long
__copy_from_user_inatomic_nocache(void *dst, const void __user *src,
- unsigned size)
+ unsigned long size)
{
+ if (size > INT_MAX)
+ return size;
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ if (!access_ok_noprefault(VERIFY_READ, src, size))
+ return size;
+#endif
+
return __copy_user_nocache(dst, src, size, 0);
}
unsigned long
-copy_user_handle_tail(char *to, char *from, unsigned len, unsigned zerorest);
+copy_user_handle_tail(char __user *to, char __user *from, unsigned long len, unsigned zerorest) __size_overflow(3);
#endif /* _ASM_X86_UACCESS_64_H */
* and shift, for example.
*/
struct word_at_a_time {
- const unsigned long one_bits, high_bits;
+ unsigned long one_bits, high_bits;
};
#define WORD_AT_A_TIME_CONSTANTS { REPEAT_BYTE(0x01), REPEAT_BYTE(0x80) }
struct x86_init_timers timers;
struct x86_init_iommu iommu;
struct x86_init_pci pci;
-};
+} __no_const;
/**
* struct x86_cpuinit_ops - platform specific cpu hotplug setups
void (*setup_percpu_clockev)(void);
void (*early_percpu_clock_init)(void);
void (*fixup_cpu_id)(struct cpuinfo_x86 *c, int node);
-};
+} __no_const;
struct timespec;
void (*save_sched_clock_state)(void);
void (*restore_sched_clock_state)(void);
void (*apic_post_init)(void);
-};
+} __no_const;
struct pci_dev;
struct msi_msg;
void (*teardown_msi_irqs)(struct pci_dev *dev);
void (*restore_msi_irqs)(struct pci_dev *dev);
int (*setup_hpet_msi)(unsigned int irq, unsigned int id);
-};
+} __no_const;
struct IO_APIC_route_entry;
struct io_apic_irq_attr;
unsigned int destination, int vector,
struct io_apic_irq_attr *attr);
void (*eoi_ioapic_pin)(int apic, int pin, int vector);
-};
+} __no_const;
extern struct x86_init_ops x86_init;
extern struct x86_cpuinit_ops x86_cpuinit;
* - get_phys_to_machine() is to be called by __pfn_to_mfn() only in special
* cases needing an extended handling.
*/
-static inline unsigned long __pfn_to_mfn(unsigned long pfn)
+static inline unsigned long __intentional_overflow(-1) __pfn_to_mfn(unsigned long pfn)
{
unsigned long mfn;
if (unlikely(err))
return -EFAULT;
+ pax_open_userland();
__asm__ __volatile__(ASM_STAC "\n"
- "1:"XSAVE"\n"
+ "1:"
+ __copyuser_seg
+ XSAVE"\n"
"2: " ASM_CLAC "\n"
xstate_fault
: "D" (buf), "a" (-1), "d" (-1), "0" (0)
: "memory");
+ pax_close_userland();
return err;
}
static inline int xrestore_user(struct xsave_struct __user *buf, u64 mask)
{
int err = 0;
- struct xsave_struct *xstate = ((__force struct xsave_struct *)buf);
+ struct xsave_struct *xstate = ((__force_kernel struct xsave_struct *)buf);
u32 lmask = mask;
u32 hmask = mask >> 32;
+ pax_open_userland();
__asm__ __volatile__(ASM_STAC "\n"
- "1:"XRSTOR"\n"
+ "1:"
+ __copyuser_seg
+ XRSTOR"\n"
"2: " ASM_CLAC "\n"
xstate_fault
: "D" (xstate), "a" (lmask), "d" (hmask), "0" (0)
: "memory"); /* memory required? */
+ pax_close_userland();
return err;
}
#define ISA_START_ADDRESS 0xa0000
#define ISA_END_ADDRESS 0x100000
-#define BIOS_BEGIN 0x000a0000
+#define BIOS_BEGIN 0x000c0000
#define BIOS_END 0x00100000
#define BIOS_ROM_BASE 0xffe00000
#define EFLAGS 144
#define RSP 152
#define SS 160
-#define ARGOFFSET R11
#endif /* __ASSEMBLY__ */
/* top of stack page */
obj-y += setup.o x86_init.o i8259.o irqinit.o jump_label.o
obj-$(CONFIG_IRQ_WORK) += irq_work.o
obj-y += probe_roms.o
-obj-$(CONFIG_X86_32) += i386_ksyms_32.o
+obj-$(CONFIG_X86_32) += sys_i386_32.o i386_ksyms_32.o
obj-$(CONFIG_X86_64) += sys_x86_64.o x8664_ksyms_64.o
obj-$(CONFIG_X86_64) += mcount_64.o
obj-y += syscall_$(BITS).o vsyscall_gtod.o
* If your system is blacklisted here, but you find that acpi=force
* works for you, please contact linux-acpi@vger.kernel.org
*/
-static struct dmi_system_id __initdata acpi_dmi_table[] = {
+static const struct dmi_system_id __initconst acpi_dmi_table[] = {
/*
* Boxes that need ACPI disabled
*/
};
/* second table for DMI checks that should run after early-quirks */
-static struct dmi_system_id __initdata acpi_dmi_table_late[] = {
+static const struct dmi_system_id __initconst acpi_dmi_table_late[] = {
/*
* HP laptops which use a DSDT reporting as HP/SB400/10000,
* which includes some code which overrides all temperature
#else /* CONFIG_64BIT */
#ifdef CONFIG_SMP
stack_start = (unsigned long)temp_stack + sizeof(temp_stack);
+
+ pax_open_kernel();
early_gdt_descr.address =
(unsigned long)get_cpu_gdt_table(smp_processor_id());
+ pax_close_kernel();
+
initial_gs = per_cpu_offset(smp_processor_id());
#endif
initial_code = (unsigned long)wakeup_long64;
# and restore the stack ... but you need gdt for this to work
movl saved_context_esp, %esp
- movl %cs:saved_magic, %eax
- cmpl $0x12345678, %eax
+ cmpl $0x12345678, saved_magic
jne bogus_magic
# jump to place where we left off
- movl saved_eip, %eax
- jmp *%eax
+ jmp *(saved_eip)
bogus_magic:
jmp bogus_magic
*/
for (a = start; a < end; a++) {
instr = (u8 *)&a->instr_offset + a->instr_offset;
+
+#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_KERNEXEC)
+ instr += ____LOAD_PHYSICAL_ADDR - LOAD_PHYSICAL_ADDR;
+ if (instr < (u8 *)_text || (u8 *)_einittext <= instr)
+ instr -= ____LOAD_PHYSICAL_ADDR - LOAD_PHYSICAL_ADDR;
+#endif
+
replacement = (u8 *)&a->repl_offset + a->repl_offset;
BUG_ON(a->replacementlen > a->instrlen);
BUG_ON(a->instrlen > sizeof(insnbuf));
add_nops(insnbuf + a->replacementlen,
a->instrlen - a->replacementlen);
+#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_KERNEXEC)
+ if (instr < (u8 *)_text || (u8 *)_einittext <= instr)
+ instr = ktva_ktla(instr);
+#endif
+
text_poke_early(instr, insnbuf, a->instrlen);
}
}
for (poff = start; poff < end; poff++) {
u8 *ptr = (u8 *)poff + *poff;
+#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_KERNEXEC)
+ ptr += ____LOAD_PHYSICAL_ADDR - LOAD_PHYSICAL_ADDR;
+ if (ptr < (u8 *)_text || (u8 *)_einittext <= ptr)
+ ptr -= ____LOAD_PHYSICAL_ADDR - LOAD_PHYSICAL_ADDR;
+#endif
+
if (!*poff || ptr < text || ptr >= text_end)
continue;
/* turn DS segment override prefix into lock prefix */
- if (*ptr == 0x3e)
+ if (*ktla_ktva(ptr) == 0x3e)
text_poke(ptr, ((unsigned char []){0xf0}), 1);
}
mutex_unlock(&text_mutex);
for (poff = start; poff < end; poff++) {
u8 *ptr = (u8 *)poff + *poff;
+#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_KERNEXEC)
+ ptr += ____LOAD_PHYSICAL_ADDR - LOAD_PHYSICAL_ADDR;
+ if (ptr < (u8 *)_text || (u8 *)_einittext <= ptr)
+ ptr -= ____LOAD_PHYSICAL_ADDR - LOAD_PHYSICAL_ADDR;
+#endif
+
if (!*poff || ptr < text || ptr >= text_end)
continue;
/* turn lock prefix into DS segment override prefix */
- if (*ptr == 0xf0)
+ if (*ktla_ktva(ptr) == 0xf0)
text_poke(ptr, ((unsigned char []){0x3E}), 1);
}
mutex_unlock(&text_mutex);
BUG_ON(p->len > MAX_PATCH_LEN);
/* prep the buffer with the original instructions */
- memcpy(insnbuf, p->instr, p->len);
+ memcpy(insnbuf, ktla_ktva(p->instr), p->len);
used = pv_init_ops.patch(p->instrtype, p->clobbers, insnbuf,
(unsigned long)p->instr, p->len);
if (!uniproc_patched || num_possible_cpus() == 1)
free_init_pages("SMP alternatives",
(unsigned long)__smp_locks,
- (unsigned long)__smp_locks_end);
+ PAGE_ALIGN((unsigned long)__smp_locks_end));
#endif
apply_paravirt(__parainstructions, __parainstructions_end);
* instructions. And on the local CPU you need to be protected again NMI or MCE
* handlers seeing an inconsistent instruction while you patch.
*/
-void *__init_or_module text_poke_early(void *addr, const void *opcode,
+void *__kprobes text_poke_early(void *addr, const void *opcode,
size_t len)
{
unsigned long flags;
local_irq_save(flags);
- memcpy(addr, opcode, len);
+
+ pax_open_kernel();
+ memcpy(ktla_ktva(addr), opcode, len);
sync_core();
+ pax_close_kernel();
+
local_irq_restore(flags);
/* Could also do a CLFLUSH here to speed up CPU recovery; but
that causes hangs on some VIA CPUs. */
*/
void *text_poke(void *addr, const void *opcode, size_t len)
{
- unsigned long flags;
- char *vaddr;
+ unsigned char *vaddr = ktla_ktva(addr);
struct page *pages[2];
- int i;
+ size_t i;
if (!core_kernel_text((unsigned long)addr)) {
- pages[0] = vmalloc_to_page(addr);
- pages[1] = vmalloc_to_page(addr + PAGE_SIZE);
+ pages[0] = vmalloc_to_page(vaddr);
+ pages[1] = vmalloc_to_page(vaddr + PAGE_SIZE);
} else {
- pages[0] = virt_to_page(addr);
+ pages[0] = virt_to_page(vaddr);
WARN_ON(!PageReserved(pages[0]));
- pages[1] = virt_to_page(addr + PAGE_SIZE);
+ pages[1] = virt_to_page(vaddr + PAGE_SIZE);
}
BUG_ON(!pages[0]);
- local_irq_save(flags);
- set_fixmap(FIX_TEXT_POKE0, page_to_phys(pages[0]));
- if (pages[1])
- set_fixmap(FIX_TEXT_POKE1, page_to_phys(pages[1]));
- vaddr = (char *)fix_to_virt(FIX_TEXT_POKE0);
- memcpy(&vaddr[(unsigned long)addr & ~PAGE_MASK], opcode, len);
- clear_fixmap(FIX_TEXT_POKE0);
- if (pages[1])
- clear_fixmap(FIX_TEXT_POKE1);
- local_flush_tlb();
- sync_core();
- /* Could also do a CLFLUSH here to speed up CPU recovery; but
- that causes hangs on some VIA CPUs. */
+ text_poke_early(addr, opcode, len);
for (i = 0; i < len; i++)
- BUG_ON(((char *)addr)[i] != ((char *)opcode)[i]);
- local_irq_restore(flags);
+ BUG_ON((vaddr)[i] != ((const unsigned char *)opcode)[i]);
return addr;
}
if (likely(!bp_patching_in_progress))
return 0;
- if (user_mode_vm(regs) || regs->ip != (unsigned long)bp_int3_addr)
+ if (user_mode(regs) || regs->ip != (unsigned long)bp_int3_addr)
return 0;
/* set up the specified breakpoint handler */
*/
void *text_poke_bp(void *addr, const void *opcode, size_t len, void *handler)
{
- unsigned char int3 = 0xcc;
+ const unsigned char int3 = 0xcc;
bp_int3_handler = handler;
bp_int3_addr = (u8 *)addr + sizeof(int3);
/*
* Debug level, exported for io_apic.c
*/
-unsigned int apic_verbosity;
+int apic_verbosity;
int pic_mode;
apic_write(APIC_ESR, 0);
v = apic_read(APIC_ESR);
ack_APIC_irq();
- atomic_inc(&irq_err_count);
+ atomic_inc_unchecked(&irq_err_count);
apic_printk(APIC_DEBUG, KERN_DEBUG "APIC error on CPU%d: %02x",
smp_processor_id(), v);
return 1;
}
-static struct apic apic_flat = {
+static struct apic apic_flat __read_only = {
.name = "flat",
.probe = flat_probe,
.acpi_madt_oem_check = flat_acpi_madt_oem_check,
return 0;
}
-static struct apic apic_physflat = {
+static struct apic apic_physflat __read_only = {
.name = "physical flat",
.probe = physflat_probe,
WARN_ON_ONCE(cpu_has_apic && !disable_apic);
}
-struct apic apic_noop = {
+struct apic apic_noop __read_only = {
.name = "noop",
.probe = noop_probe,
.acpi_madt_oem_check = NULL,
return dmi_bigsmp;
}
-static struct apic apic_bigsmp = {
+static struct apic apic_bigsmp __read_only = {
.name = "bigsmp",
.probe = probe_bigsmp,
return ret;
}
-atomic_t irq_mis_count;
+atomic_unchecked_t irq_mis_count;
#ifdef CONFIG_GENERIC_PENDING_IRQ
static bool io_apic_level_ack_pending(struct irq_cfg *cfg)
* at the cpu.
*/
if (!(v & (1 << (i & 0x1f)))) {
- atomic_inc(&irq_mis_count);
+ atomic_inc_unchecked(&irq_mis_count);
eoi_ioapic_irq(irq, cfg);
}
return 1;
}
-static struct apic apic_default = {
+static struct apic apic_default __read_only = {
.name = "default",
.probe = probe_default,
static DEFINE_RAW_SPINLOCK(vector_lock);
-void lock_vector_lock(void)
+void lock_vector_lock(void) __acquires(vector_lock)
{
/* Used to the online set of cpus does not change
* during assign_irq_vector.
raw_spin_lock(&vector_lock);
}
-void unlock_vector_lock(void)
+void unlock_vector_lock(void) __releases(vector_lock)
{
raw_spin_unlock(&vector_lock);
}
return notifier_from_errno(err);
}
-static struct notifier_block __refdata x2apic_cpu_notifier = {
+static struct notifier_block x2apic_cpu_notifier = {
.notifier_call = update_clusterinfo,
};
cpumask_and(retmask, mask, per_cpu(cpus_in_cluster, cpu));
}
-static struct apic apic_x2apic_cluster = {
+static struct apic apic_x2apic_cluster __read_only = {
.name = "cluster x2apic",
.probe = x2apic_cluster_probe,
return apic == &apic_x2apic_phys;
}
-static struct apic apic_x2apic_phys = {
+static struct apic apic_x2apic_phys __read_only = {
.name = "physical x2apic",
.probe = x2apic_phys_probe,
return apic == &apic_x2apic_uv_x;
}
-static struct apic __refdata apic_x2apic_uv_x = {
+static struct apic apic_x2apic_uv_x __read_only = {
.name = "UV large system",
.probe = uv_probe,
* This is for buggy BIOS's that refer to (real mode) segment 0x40
* even though they are called in protected mode.
*/
-static struct desc_struct bad_bios_desc = GDT_ENTRY_INIT(0x4092,
+static const struct desc_struct bad_bios_desc = GDT_ENTRY_INIT(0x4093,
(unsigned long)__va(0x400UL), PAGE_SIZE - 0x400 - 1);
static const char driver_version[] = "1.16ac"; /* no spaces */
BUG_ON(cpu != 0);
gdt = get_cpu_gdt_table(cpu);
save_desc_40 = gdt[0x40 / 8];
+
+ pax_open_kernel();
gdt[0x40 / 8] = bad_bios_desc;
+ pax_close_kernel();
apm_irq_save(flags);
APM_DO_SAVE_SEGS;
&call->esi);
APM_DO_RESTORE_SEGS;
apm_irq_restore(flags);
+
+ pax_open_kernel();
gdt[0x40 / 8] = save_desc_40;
+ pax_close_kernel();
+
put_cpu();
return call->eax & 0xff;
BUG_ON(cpu != 0);
gdt = get_cpu_gdt_table(cpu);
save_desc_40 = gdt[0x40 / 8];
+
+ pax_open_kernel();
gdt[0x40 / 8] = bad_bios_desc;
+ pax_close_kernel();
apm_irq_save(flags);
APM_DO_SAVE_SEGS;
&call->eax);
APM_DO_RESTORE_SEGS;
apm_irq_restore(flags);
+
+ pax_open_kernel();
gdt[0x40 / 8] = save_desc_40;
+ pax_close_kernel();
+
put_cpu();
return error;
}
* code to that CPU.
*/
gdt = get_cpu_gdt_table(0);
+
+ pax_open_kernel();
set_desc_base(&gdt[APM_CS >> 3],
(unsigned long)__va((unsigned long)apm_info.bios.cseg << 4));
set_desc_base(&gdt[APM_CS_16 >> 3],
(unsigned long)__va((unsigned long)apm_info.bios.cseg_16 << 4));
set_desc_base(&gdt[APM_DS >> 3],
(unsigned long)__va((unsigned long)apm_info.bios.dseg << 4));
+ pax_close_kernel();
proc_create("apm", 0, NULL, &apm_file_ops);
OFFSET(TI_flags, thread_info, flags);
OFFSET(TI_status, thread_info, status);
OFFSET(TI_addr_limit, thread_info, addr_limit);
+ OFFSET(TI_lowest_stack, thread_info, lowest_stack);
+ DEFINE(TI_task_thread_sp0, offsetof(struct task_struct, thread.sp0) - offsetof(struct task_struct, tinfo));
BLANK();
OFFSET(crypto_tfm_ctx_offset, crypto_tfm, __crt_ctx);
OFFSET(PV_CPU_irq_enable_sysexit, pv_cpu_ops, irq_enable_sysexit);
OFFSET(PV_CPU_read_cr0, pv_cpu_ops, read_cr0);
OFFSET(PV_MMU_read_cr2, pv_mmu_ops, read_cr2);
+
+#ifdef CONFIG_PAX_KERNEXEC
+ OFFSET(PV_CPU_write_cr0, pv_cpu_ops, write_cr0);
+#endif
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ OFFSET(PV_MMU_read_cr3, pv_mmu_ops, read_cr3);
+ OFFSET(PV_MMU_write_cr3, pv_mmu_ops, write_cr3);
+#ifdef CONFIG_X86_64
+ OFFSET(PV_MMU_set_pgd_batched, pv_mmu_ops, set_pgd_batched);
+#endif
#endif
+#endif
+
+ BLANK();
+ DEFINE(PAGE_SIZE_asm, PAGE_SIZE);
+ DEFINE(PAGE_SHIFT_asm, PAGE_SHIFT);
+ DEFINE(THREAD_SIZE_asm, THREAD_SIZE);
+
#ifdef CONFIG_XEN
BLANK();
OFFSET(XEN_vcpu_info_mask, vcpu_info, evtchn_upcall_mask);
BLANK();
#undef ENTRY
+ DEFINE(TSS_size, sizeof(struct tss_struct));
OFFSET(TSS_ist, tss_struct, x86_tss.ist);
BLANK();
CFLAGS_REMOVE_perf_event.o = -pg
endif
-# Make sure load_percpu_segment has no stackprotector
-nostackp := $(call cc-option, -fno-stack-protector)
-CFLAGS_common.o := $(nostackp)
-
obj-y := intel_cacheinfo.o scattered.o topology.o
obj-y += common.o
obj-y += rdrand.o
static unsigned int amd_size_cache(struct cpuinfo_x86 *c, unsigned int size)
{
/* AMD errata T13 (order #21922) */
- if ((c->x86 == 6)) {
+ if (c->x86 == 6) {
/* Duron Rev A0 */
if (c->x86_model == 3 && c->x86_mask == 0)
size = 64;
static const struct cpu_dev *this_cpu = &default_cpu;
-DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
-#ifdef CONFIG_X86_64
- /*
- * We need valid kernel segments for data and code in long mode too
- * IRET will check the segment types kkeil 2000/10/28
- * Also sysret mandates a special GDT layout
- *
- * TLS descriptors are currently at a different place compared to i386.
- * Hopefully nobody expects them at a fixed place (Wine?)
- */
- [GDT_ENTRY_KERNEL32_CS] = GDT_ENTRY_INIT(0xc09b, 0, 0xfffff),
- [GDT_ENTRY_KERNEL_CS] = GDT_ENTRY_INIT(0xa09b, 0, 0xfffff),
- [GDT_ENTRY_KERNEL_DS] = GDT_ENTRY_INIT(0xc093, 0, 0xfffff),
- [GDT_ENTRY_DEFAULT_USER32_CS] = GDT_ENTRY_INIT(0xc0fb, 0, 0xfffff),
- [GDT_ENTRY_DEFAULT_USER_DS] = GDT_ENTRY_INIT(0xc0f3, 0, 0xfffff),
- [GDT_ENTRY_DEFAULT_USER_CS] = GDT_ENTRY_INIT(0xa0fb, 0, 0xfffff),
-#else
- [GDT_ENTRY_KERNEL_CS] = GDT_ENTRY_INIT(0xc09a, 0, 0xfffff),
- [GDT_ENTRY_KERNEL_DS] = GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
- [GDT_ENTRY_DEFAULT_USER_CS] = GDT_ENTRY_INIT(0xc0fa, 0, 0xfffff),
- [GDT_ENTRY_DEFAULT_USER_DS] = GDT_ENTRY_INIT(0xc0f2, 0, 0xfffff),
- /*
- * Segments used for calling PnP BIOS have byte granularity.
- * They code segments and data segments have fixed 64k limits,
- * the transfer segment sizes are set at run time.
- */
- /* 32-bit code */
- [GDT_ENTRY_PNPBIOS_CS32] = GDT_ENTRY_INIT(0x409a, 0, 0xffff),
- /* 16-bit code */
- [GDT_ENTRY_PNPBIOS_CS16] = GDT_ENTRY_INIT(0x009a, 0, 0xffff),
- /* 16-bit data */
- [GDT_ENTRY_PNPBIOS_DS] = GDT_ENTRY_INIT(0x0092, 0, 0xffff),
- /* 16-bit data */
- [GDT_ENTRY_PNPBIOS_TS1] = GDT_ENTRY_INIT(0x0092, 0, 0),
- /* 16-bit data */
- [GDT_ENTRY_PNPBIOS_TS2] = GDT_ENTRY_INIT(0x0092, 0, 0),
- /*
- * The APM segments have byte granularity and their bases
- * are set at run time. All have 64k limits.
- */
- /* 32-bit code */
- [GDT_ENTRY_APMBIOS_BASE] = GDT_ENTRY_INIT(0x409a, 0, 0xffff),
- /* 16-bit code */
- [GDT_ENTRY_APMBIOS_BASE+1] = GDT_ENTRY_INIT(0x009a, 0, 0xffff),
- /* data */
- [GDT_ENTRY_APMBIOS_BASE+2] = GDT_ENTRY_INIT(0x4092, 0, 0xffff),
-
- [GDT_ENTRY_ESPFIX_SS] = GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
- [GDT_ENTRY_PERCPU] = GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
- GDT_STACK_CANARY_INIT
-#endif
-} };
-EXPORT_PER_CPU_SYMBOL_GPL(gdt_page);
-
static int __init x86_xsave_setup(char *s)
{
if (strlen(s))
}
}
+#ifdef CONFIG_X86_64
+static __init int setup_disable_pcid(char *arg)
+{
+ setup_clear_cpu_cap(X86_FEATURE_PCID);
+ setup_clear_cpu_cap(X86_FEATURE_INVPCID);
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ if (clone_pgd_mask != ~(pgdval_t)0UL)
+ pax_user_shadow_base = 1UL << TASK_SIZE_MAX_SHIFT;
+#endif
+
+ return 1;
+}
+__setup("nopcid", setup_disable_pcid);
+
+static void setup_pcid(struct cpuinfo_x86 *c)
+{
+ if (!cpu_has(c, X86_FEATURE_PCID)) {
+ clear_cpu_cap(c, X86_FEATURE_INVPCID);
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ if (clone_pgd_mask != ~(pgdval_t)0UL) {
+ pax_open_kernel();
+ pax_user_shadow_base = 1UL << TASK_SIZE_MAX_SHIFT;
+ pax_close_kernel();
+ printk("PAX: slow and weak UDEREF enabled\n");
+ } else
+ printk("PAX: UDEREF disabled\n");
+#endif
+
+ return;
+ }
+
+ printk("PAX: PCID detected\n");
+ set_in_cr4(X86_CR4_PCIDE);
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ pax_open_kernel();
+ clone_pgd_mask = ~(pgdval_t)0UL;
+ pax_close_kernel();
+ if (pax_user_shadow_base)
+ printk("PAX: weak UDEREF enabled\n");
+ else {
+ set_cpu_cap(c, X86_FEATURE_STRONGUDEREF);
+ printk("PAX: strong UDEREF enabled\n");
+ }
+#endif
+
+ if (cpu_has(c, X86_FEATURE_INVPCID))
+ printk("PAX: INVPCID detected\n");
+}
+#endif
+
/*
* Some CPU features depend on higher CPUID levels, which may not always
* be available due to CPUID level capping or broken virtualization
{
struct desc_ptr gdt_descr;
- gdt_descr.address = (long)get_cpu_gdt_table(cpu);
+ gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
gdt_descr.size = GDT_SIZE - 1;
load_gdt(&gdt_descr);
/* Reload the per-cpu base */
setup_smep(c);
setup_smap(c);
+#ifdef CONFIG_X86_64
+ setup_pcid(c);
+#endif
+
/*
* The vendor-specific functions might have changed features.
* Now we do "generic changes."
/* Filter out anything that depends on CPUID levels we don't have */
filter_cpuid_features(c, true);
+#if defined(CONFIG_X86_32) && (defined(CONFIG_PAX_SEGMEXEC) || defined(CONFIG_PAX_KERNEXEC) || defined(CONFIG_PAX_MEMORY_UDEREF))
+ setup_clear_cpu_cap(X86_FEATURE_SEP);
+#endif
+
/* If the model name is still unset, do table lookup. */
if (!c->x86_model_id[0]) {
const char *p;
void enable_sep_cpu(void)
{
int cpu = get_cpu();
- struct tss_struct *tss = &per_cpu(init_tss, cpu);
+ struct tss_struct *tss = init_tss + cpu;
if (!boot_cpu_has(X86_FEATURE_SEP)) {
put_cpu();
}
__setup("clearcpuid=", setup_disablecpuid);
+DEFINE_PER_CPU(struct thread_info *, current_tinfo) = &init_task.tinfo;
+EXPORT_PER_CPU_SYMBOL(current_tinfo);
+
DEFINE_PER_CPU(unsigned long, kernel_stack) =
- (unsigned long)&init_thread_union - KERNEL_STACK_OFFSET + THREAD_SIZE;
+ (unsigned long)&init_thread_union - 16 + THREAD_SIZE;
EXPORT_PER_CPU_SYMBOL(kernel_stack);
#ifdef CONFIG_X86_64
-struct desc_ptr idt_descr = { NR_VECTORS * 16 - 1, (unsigned long) idt_table };
-struct desc_ptr debug_idt_descr = { NR_VECTORS * 16 - 1,
- (unsigned long) debug_idt_table };
+struct desc_ptr idt_descr __read_only = { NR_VECTORS * 16 - 1, (unsigned long) idt_table };
+const struct desc_ptr debug_idt_descr = { NR_VECTORS * 16 - 1, (unsigned long) debug_idt_table };
DEFINE_PER_CPU_FIRST(union irq_stack_union,
irq_stack_union) __aligned(PAGE_SIZE) __visible;
*/
load_ucode_ap();
- t = &per_cpu(init_tss, cpu);
+ t = init_tss + cpu;
oist = &per_cpu(orig_ist, cpu);
#ifdef CONFIG_NUMA
wrmsrl(MSR_KERNEL_GS_BASE, 0);
barrier();
- x86_configure_nx();
enable_x2apic();
/*
{
int cpu = smp_processor_id();
struct task_struct *curr = current;
- struct tss_struct *t = &per_cpu(init_tss, cpu);
+ struct tss_struct *t = init_tss + cpu;
struct thread_struct *thread = &curr->thread;
wait_for_master_cpu(cpu);
};
#ifdef CONFIG_AMD_NB
+static struct attribute *default_attrs_amd_nb[] = {
+ &type.attr,
+ &level.attr,
+ &coherency_line_size.attr,
+ &physical_line_partition.attr,
+ &ways_of_associativity.attr,
+ &number_of_sets.attr,
+ &size.attr,
+ &shared_cpu_map.attr,
+ &shared_cpu_list.attr,
+ NULL,
+ NULL,
+ NULL,
+ NULL
+};
+
static struct attribute **amd_l3_attrs(void)
{
static struct attribute **attrs;
n = ARRAY_SIZE(default_attrs);
- if (amd_nb_has_feature(AMD_NB_L3_INDEX_DISABLE))
- n += 2;
-
- if (amd_nb_has_feature(AMD_NB_L3_PARTITIONING))
- n += 1;
-
- attrs = kzalloc(n * sizeof (struct attribute *), GFP_KERNEL);
- if (attrs == NULL)
- return attrs = default_attrs;
-
- for (n = 0; default_attrs[n]; n++)
- attrs[n] = default_attrs[n];
+ attrs = default_attrs_amd_nb;
if (amd_nb_has_feature(AMD_NB_L3_INDEX_DISABLE)) {
attrs[n++] = &cache_disable_0.attr;
.default_attrs = default_attrs,
};
+#ifdef CONFIG_AMD_NB
+static struct kobj_type ktype_cache_amd_nb = {
+ .sysfs_ops = &sysfs_ops,
+ .default_attrs = default_attrs_amd_nb,
+};
+#endif
+
static struct kobj_type ktype_percpu_entry = {
.sysfs_ops = &sysfs_ops,
};
return retval;
}
+#ifdef CONFIG_AMD_NB
+ amd_l3_attrs();
+#endif
+
for (i = 0; i < num_cache_leaves; i++) {
+ struct kobj_type *ktype;
+
this_object = INDEX_KOBJECT_PTR(cpu, i);
this_object->cpu = cpu;
this_object->index = i;
this_leaf = CPUID4_INFO_IDX(cpu, i);
- ktype_cache.default_attrs = default_attrs;
+ ktype = &ktype_cache;
#ifdef CONFIG_AMD_NB
if (this_leaf->base.nb)
- ktype_cache.default_attrs = amd_l3_attrs();
+ ktype = &ktype_cache_amd_nb;
#endif
retval = kobject_init_and_add(&(this_object->kobj),
- &ktype_cache,
+ ktype,
per_cpu(ici_cache_kobject, cpu),
"index%1lu", i);
if (unlikely(retval)) {
#include <asm/processor.h>
#include <asm/mce.h>
#include <asm/msr.h>
+#include <asm/local.h>
#include "mce-internal.h"
!(m->mcgstatus & MCG_STATUS_EIPV) ? " !INEXACT!" : "",
m->cs, m->ip);
- if (m->cs == __KERNEL_CS)
+ if (m->cs == __KERNEL_CS || m->cs == __KERNEXEC_KERNEL_CS)
print_symbol("{%s}", m->ip);
pr_cont("\n");
}
#define PANIC_TIMEOUT 5 /* 5 seconds */
-static atomic_t mce_panicked;
+static atomic_unchecked_t mce_panicked;
static int fake_panic;
-static atomic_t mce_fake_panicked;
+static atomic_unchecked_t mce_fake_panicked;
/* Panic in progress. Enable interrupts and wait for final IPI */
static void wait_for_panic(void)
/*
* Make sure only one CPU runs in machine check panic
*/
- if (atomic_inc_return(&mce_panicked) > 1)
+ if (atomic_inc_return_unchecked(&mce_panicked) > 1)
wait_for_panic();
barrier();
console_verbose();
} else {
/* Don't log too much for fake panic */
- if (atomic_inc_return(&mce_fake_panicked) > 1)
+ if (atomic_inc_return_unchecked(&mce_fake_panicked) > 1)
return;
}
/* First print corrected ones that are still unlogged */
if (!fake_panic) {
if (panic_timeout == 0)
panic_timeout = mca_cfg.panic_timeout;
- panic(msg);
+ panic("%s", msg);
} else
pr_emerg(HW_ERR "Fake kernel panic: %s\n", msg);
}
* might have been modified by someone else.
*/
rmb();
- if (atomic_read(&mce_panicked))
+ if (atomic_read_unchecked(&mce_panicked))
wait_for_panic();
if (!mca_cfg.monarch_timeout)
goto out;
}
/* Call the installed machine check handler for this CPU setup. */
-void (*machine_check_vector)(struct pt_regs *, long error_code) =
+void (*machine_check_vector)(struct pt_regs *, long error_code) __read_only =
unexpected_machine_check;
/*
return;
}
+ pax_open_kernel();
machine_check_vector = do_machine_check;
+ pax_close_kernel();
__mcheck_cpu_init_generic();
__mcheck_cpu_init_vendor(c);
*/
static DEFINE_SPINLOCK(mce_chrdev_state_lock);
-static int mce_chrdev_open_count; /* #times opened */
+static local_t mce_chrdev_open_count; /* #times opened */
static int mce_chrdev_open_exclu; /* already open exclusive? */
static int mce_chrdev_open(struct inode *inode, struct file *file)
spin_lock(&mce_chrdev_state_lock);
if (mce_chrdev_open_exclu ||
- (mce_chrdev_open_count && (file->f_flags & O_EXCL))) {
+ (local_read(&mce_chrdev_open_count) && (file->f_flags & O_EXCL))) {
spin_unlock(&mce_chrdev_state_lock);
return -EBUSY;
if (file->f_flags & O_EXCL)
mce_chrdev_open_exclu = 1;
- mce_chrdev_open_count++;
+ local_inc(&mce_chrdev_open_count);
spin_unlock(&mce_chrdev_state_lock);
{
spin_lock(&mce_chrdev_state_lock);
- mce_chrdev_open_count--;
+ local_dec(&mce_chrdev_open_count);
mce_chrdev_open_exclu = 0;
spin_unlock(&mce_chrdev_state_lock);
for (i = 0; i < mca_cfg.banks; i++) {
struct mce_bank *b = &mce_banks[i];
- struct device_attribute *a = &b->attr;
+ device_attribute_no_const *a = &b->attr;
sysfs_attr_init(&a->attr);
a->attr.name = b->attrname;
static void mce_reset(void)
{
cpu_missing = 0;
- atomic_set(&mce_fake_panicked, 0);
+ atomic_set_unchecked(&mce_fake_panicked, 0);
atomic_set(&mce_executing, 0);
atomic_set(&mce_callin, 0);
atomic_set(&global_nwo, 0);
#include <asm/processor.h>
#include <asm/mce.h>
#include <asm/msr.h>
+#include <asm/pgtable.h>
/* By default disabled */
int mce_p5_enabled __read_mostly;
if (!cpu_has(c, X86_FEATURE_MCE))
return;
+ pax_open_kernel();
machine_check_vector = pentium_machine_check;
+ pax_close_kernel();
/* Make sure the vector pointer is visible before we enable MCEs: */
wmb();
#include <asm/processor.h>
#include <asm/mce.h>
#include <asm/msr.h>
+#include <asm/pgtable.h>
/* Machine check handler for WinChip C6: */
static void winchip_machine_check(struct pt_regs *regs, long error_code)
{
u32 lo, hi;
+ pax_open_kernel();
machine_check_vector = winchip_machine_check;
+ pax_close_kernel();
/* Make sure the vector pointer is visible before we enable MCEs: */
wmb();
return NOTIFY_OK;
}
-static struct notifier_block __refdata mc_cpu_notifier = {
+static struct notifier_block mc_cpu_notifier = {
.notifier_call = mc_cpu_callback,
};
struct microcode_header_intel mc_header;
unsigned int mc_size;
+ if (leftover < sizeof(mc_header)) {
+ pr_err("error! Truncated header in microcode data file\n");
+ break;
+ }
+
if (get_ucode_data(&mc_header, ucode_ptr, sizeof(mc_header)))
break;
static int get_ucode_user(void *to, const void *from, size_t n)
{
- return copy_from_user(to, from, n);
+ return copy_from_user(to, (const void __force_user *)from, n);
}
static enum ucode_state
request_microcode_user(int cpu, const void __user *buf, size_t size)
{
- return generic_load_microcode(cpu, (void *)buf, size, &get_ucode_user);
+ return generic_load_microcode(cpu, (__force_kernel void *)buf, size, &get_ucode_user);
}
static void microcode_fini_cpu(int cpu)
unsigned int mc_saved_count = mc_saved_data->mc_saved_count;
int i;
- while (leftover) {
+ while (leftover && mc_saved_count < ARRAY_SIZE(mc_saved_tmp)) {
+
+ if (leftover < sizeof(mc_header))
+ break;
+
mc_header = (struct microcode_header_intel *)ucode_ptr;
mc_size = get_totalsize(mc_header);
u64 size_or_mask, size_and_mask;
static bool mtrr_aps_delayed_init;
-static const struct mtrr_ops *mtrr_ops[X86_VENDOR_NUM];
+static const struct mtrr_ops *mtrr_ops[X86_VENDOR_NUM] __read_only;
const struct mtrr_ops *mtrr_if;
int (*validate_add_page)(unsigned long base, unsigned long size,
unsigned int type);
int (*have_wrcomb)(void);
-};
+} __do_const;
extern int generic_get_free_region(unsigned long base, unsigned long size,
int replace_reg);
}
-static struct attribute_group x86_pmu_format_group = {
+static attribute_group_no_const x86_pmu_format_group = {
.name = "format",
.attrs = NULL,
};
NULL,
};
-static struct attribute_group x86_pmu_events_group = {
+static attribute_group_no_const x86_pmu_events_group = {
.name = "events",
.attrs = events_attr,
};
if (idx > GDT_ENTRIES)
return 0;
- desc = raw_cpu_ptr(gdt_page.gdt);
+ desc = get_cpu_gdt_table(smp_processor_id());
}
return get_desc_base(desc + idx);
break;
perf_callchain_store(entry, frame.return_address);
- fp = frame.next_frame;
+ fp = (const void __force_user *)frame.next_frame;
}
}
static __init int _init_events_attrs(struct perf_amd_iommu *perf_iommu)
{
struct attribute **attrs;
- struct attribute_group *attr_group;
+ attribute_group_no_const *attr_group;
int i = 0, j;
while (amd_iommu_v2_event_descs[i].attr.attr.name)
x86_pmu.num_counters_fixed = max((int)edx.split.num_counters_fixed, 3);
if (boot_cpu_has(X86_FEATURE_PDCM)) {
- u64 capabilities;
+ u64 capabilities = x86_pmu.intel_cap.capabilities;
- rdmsrl(MSR_IA32_PERF_CAPABILITIES, capabilities);
- x86_pmu.intel_cap.capabilities = capabilities;
+ if (rdmsrl_safe(MSR_IA32_PERF_CAPABILITIES, &x86_pmu.intel_cap.capabilities))
+ x86_pmu.intel_cap.capabilities = capabilities;
}
intel_ds_init();
NULL,
};
-static struct attribute_group rapl_pmu_events_group = {
+static attribute_group_no_const rapl_pmu_events_group __read_only = {
.name = "events",
.attrs = NULL, /* patched at runtime */
};
static int __init uncore_type_init(struct intel_uncore_type *type)
{
struct intel_uncore_pmu *pmus;
- struct attribute_group *attr_group;
+ attribute_group_no_const *attr_group;
struct attribute **attrs;
int i, j;
struct uncore_event_desc {
struct kobj_attribute attr;
const char *config;
-};
+} __do_const;
ssize_t uncore_event_show(struct kobject *kobj,
struct kobj_attribute *attr, char *buf);
return notifier_from_errno(err);
}
-static struct notifier_block __refdata cpuid_class_cpu_notifier =
+static struct notifier_block cpuid_class_cpu_notifier =
{
.notifier_call = cpuid_class_cpu_callback,
};
#ifdef CONFIG_X86_32
struct pt_regs fixed_regs;
- if (!user_mode_vm(regs)) {
+ if (!user_mode(regs)) {
crash_fixup_ss_esp(&fixed_regs, regs);
regs = &fixed_regs;
}
return -ENOMEM;
if (userbuf) {
- if (copy_to_user(buf, vaddr + offset, csize)) {
+ if (copy_to_user((char __force_user *)buf, vaddr + offset, csize)) {
iounmap(vaddr);
return -EFAULT;
}
#define DOUBLEFAULT_STACKSIZE (1024)
static unsigned long doublefault_stack[DOUBLEFAULT_STACKSIZE];
-#define STACK_START (unsigned long)(doublefault_stack+DOUBLEFAULT_STACKSIZE)
+#define STACK_START (unsigned long)(doublefault_stack+DOUBLEFAULT_STACKSIZE-2)
#define ptr_ok(x) ((x) > PAGE_OFFSET && (x) < PAGE_OFFSET + MAXMEM)
unsigned long gdt, tss;
native_store_gdt(&gdt_desc);
- gdt = gdt_desc.address;
+ gdt = (unsigned long)gdt_desc.address;
printk(KERN_EMERG "PANIC: double fault, gdt at %08lx [%d bytes]\n", gdt, gdt_desc.size);
/* 0x2 bit is always set */
.flags = X86_EFLAGS_SF | 0x2,
.sp = STACK_START,
- .es = __USER_DS,
+ .es = __KERNEL_DS,
.cs = __KERNEL_CS,
.ss = __KERNEL_DS,
- .ds = __USER_DS,
+ .ds = __KERNEL_DS,
.fs = __KERNEL_PERCPU,
.__cr3 = __pa_nodebug(swapper_pg_dir),
* Copyright (C) 1991, 1992 Linus Torvalds
* Copyright (C) 2000, 2001, 2002 Andi Kleen, SuSE Labs
*/
+#ifdef CONFIG_GRKERNSEC_HIDESYM
+#define __INCLUDED_BY_HIDESYM 1
+#endif
#include <linux/kallsyms.h>
#include <linux/kprobes.h>
#include <linux/uaccess.h>
void printk_address(unsigned long address)
{
- pr_cont(" [<%p>] %pS\n", (void *)address, (void *)address);
+ pr_cont(" [<%p>] %pA\n", (void *)address, (void *)address);
}
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
static void
print_ftrace_graph_addr(unsigned long addr, void *data,
const struct stacktrace_ops *ops,
- struct thread_info *tinfo, int *graph)
+ struct task_struct *task, int *graph)
{
- struct task_struct *task;
unsigned long ret_addr;
int index;
if (addr != (unsigned long)return_to_handler)
return;
- task = tinfo->task;
index = task->curr_ret_stack;
if (!task->ret_stack || index < *graph)
static inline void
print_ftrace_graph_addr(unsigned long addr, void *data,
const struct stacktrace_ops *ops,
- struct thread_info *tinfo, int *graph)
+ struct task_struct *task, int *graph)
{ }
#endif
* severe exception (double fault, nmi, stack fault, debug, mce) hardware stack
*/
-static inline int valid_stack_ptr(struct thread_info *tinfo,
- void *p, unsigned int size, void *end)
+static inline int valid_stack_ptr(void *t, void *p, unsigned int size, void *end)
{
- void *t = tinfo;
if (end) {
if (p < end && p >= (end-THREAD_SIZE))
return 1;
}
unsigned long
-print_context_stack(struct thread_info *tinfo,
+print_context_stack(struct task_struct *task, void *stack_start,
unsigned long *stack, unsigned long bp,
const struct stacktrace_ops *ops, void *data,
unsigned long *end, int *graph)
{
struct stack_frame *frame = (struct stack_frame *)bp;
- while (valid_stack_ptr(tinfo, stack, sizeof(*stack), end)) {
+ while (valid_stack_ptr(stack_start, stack, sizeof(*stack), end)) {
unsigned long addr;
addr = *stack;
} else {
ops->address(data, addr, 0);
}
- print_ftrace_graph_addr(addr, data, ops, tinfo, graph);
+ print_ftrace_graph_addr(addr, data, ops, task, graph);
}
stack++;
}
EXPORT_SYMBOL_GPL(print_context_stack);
unsigned long
-print_context_stack_bp(struct thread_info *tinfo,
+print_context_stack_bp(struct task_struct *task, void *stack_start,
unsigned long *stack, unsigned long bp,
const struct stacktrace_ops *ops, void *data,
unsigned long *end, int *graph)
struct stack_frame *frame = (struct stack_frame *)bp;
unsigned long *ret_addr = &frame->return_address;
- while (valid_stack_ptr(tinfo, ret_addr, sizeof(*ret_addr), end)) {
+ while (valid_stack_ptr(stack_start, ret_addr, sizeof(*ret_addr), end)) {
unsigned long addr = *ret_addr;
if (!__kernel_text_address(addr))
ops->address(data, addr, 1);
frame = frame->next_frame;
ret_addr = &frame->return_address;
- print_ftrace_graph_addr(addr, data, ops, tinfo, graph);
+ print_ftrace_graph_addr(addr, data, ops, task, graph);
}
return (unsigned long)frame;
static void print_trace_address(void *data, unsigned long addr, int reliable)
{
touch_nmi_watchdog();
- printk(data);
+ printk("%s", (char *)data);
printk_stack_address(addr, reliable);
}
EXPORT_SYMBOL_GPL(oops_begin);
NOKPROBE_SYMBOL(oops_begin);
+extern void gr_handle_kernel_exploit(void);
+
void oops_end(unsigned long flags, struct pt_regs *regs, int signr)
{
if (regs && kexec_should_crash(current))
panic("Fatal exception in interrupt");
if (panic_on_oops)
panic("Fatal exception");
- do_exit(signr);
+
+ gr_handle_kernel_exploit();
+
+ do_group_exit(signr);
}
NOKPROBE_SYMBOL(oops_end);
print_modules();
show_regs(regs);
#ifdef CONFIG_X86_32
- if (user_mode_vm(regs)) {
+ if (user_mode(regs)) {
sp = regs->sp;
ss = regs->ss & 0xffff;
} else {
unsigned long flags = oops_begin();
int sig = SIGSEGV;
- if (!user_mode_vm(regs))
+ if (!user_mode(regs))
report_bug(regs->ip, regs);
if (__die(str, regs, err))
bp = stack_frame(task, regs);
for (;;) {
- struct thread_info *context;
+ void *stack_start = (void *)((unsigned long)stack & ~(THREAD_SIZE-1));
void *end_stack;
end_stack = is_hardirq_stack(stack, cpu);
if (!end_stack)
end_stack = is_softirq_stack(stack, cpu);
- context = task_thread_info(task);
- bp = ops->walk_stack(context, stack, bp, ops, data,
+ bp = ops->walk_stack(task, stack_start, stack, bp, ops, data,
end_stack, &graph);
/* Stop if not on irq stack */
int i;
show_regs_print_info(KERN_EMERG);
- __show_regs(regs, !user_mode_vm(regs));
+ __show_regs(regs, !user_mode(regs));
/*
* When in-kernel, we also print out the stack and code at the
* time of the fault..
*/
- if (!user_mode_vm(regs)) {
+ if (!user_mode(regs)) {
unsigned int code_prologue = code_bytes * 43 / 64;
unsigned int code_len = code_bytes;
unsigned char c;
u8 *ip;
+ unsigned long cs_base = get_desc_base(&get_cpu_gdt_table(0)[(0xffff & regs->cs) >> 3]);
pr_emerg("Stack:\n");
show_stack_log_lvl(NULL, regs, ®s->sp, 0, KERN_EMERG);
pr_emerg("Code:");
- ip = (u8 *)regs->ip - code_prologue;
+ ip = (u8 *)regs->ip - code_prologue + cs_base;
if (ip < (u8 *)PAGE_OFFSET || probe_kernel_address(ip, c)) {
/* try starting at IP */
- ip = (u8 *)regs->ip;
+ ip = (u8 *)regs->ip + cs_base;
code_len = code_len - code_prologue + 1;
}
for (i = 0; i < code_len; i++, ip++) {
pr_cont(" Bad EIP value.");
break;
}
- if (ip == (u8 *)regs->ip)
+ if (ip == (u8 *)regs->ip + cs_base)
pr_cont(" <%02x>", c);
else
pr_cont(" %02x", c);
{
unsigned short ud2;
+ ip = ktla_ktva(ip);
if (ip < PAGE_OFFSET)
return 0;
if (probe_kernel_address((unsigned short *)ip, ud2))
return ud2 == 0x0b0f;
}
+
+#if defined(CONFIG_PAX_MEMORY_STACKLEAK) || defined(CONFIG_PAX_USERCOPY)
+void pax_check_alloca(unsigned long size)
+{
+ unsigned long sp = (unsigned long)&sp, stack_left;
+
+ /* all kernel stacks are of the same size */
+ stack_left = sp & (THREAD_SIZE - 1);
+ BUG_ON(stack_left < 256 || size >= stack_left - 256);
+}
+EXPORT_SYMBOL(pax_check_alloca);
+#endif
const struct stacktrace_ops *ops, void *data)
{
const unsigned cpu = get_cpu();
- struct thread_info *tinfo;
unsigned long *irq_stack = (unsigned long *)per_cpu(irq_stack_ptr, cpu);
unsigned long dummy;
unsigned used = 0;
int graph = 0;
int done = 0;
+ void *stack_start;
if (!task)
task = current;
* current stack address. If the stacks consist of nested
* exceptions
*/
- tinfo = task_thread_info(task);
while (!done) {
unsigned long *stack_end;
enum stack_type stype;
if (ops->stack(data, id) < 0)
break;
- bp = ops->walk_stack(tinfo, stack, bp, ops,
+ bp = ops->walk_stack(task, stack_end - EXCEPTION_STKSZ, stack, bp, ops,
data, stack_end, &graph);
ops->stack(data, "<EOE>");
/*
* second-to-last pointer (index -2 to end) in the
* exception stack:
*/
+ if ((u16)stack_end[-1] != __KERNEL_DS)
+ goto out;
stack = (unsigned long *) stack_end[-2];
done = 0;
break;
if (ops->stack(data, "IRQ") < 0)
break;
- bp = ops->walk_stack(tinfo, stack, bp,
+ bp = ops->walk_stack(task, irq_stack, stack, bp,
ops, data, stack_end, &graph);
/*
* We link to the next stack (which would be
/*
* This handles the process stack:
*/
- bp = ops->walk_stack(tinfo, stack, bp, ops, data, NULL, &graph);
+ stack_start = (void *)((unsigned long)stack & ~(THREAD_SIZE-1));
+ bp = ops->walk_stack(task, stack_start, stack, bp, ops, data, NULL, &graph);
+out:
put_cpu();
}
EXPORT_SYMBOL(dump_trace);
{
unsigned short ud2;
- if (__copy_from_user(&ud2, (const void __user *) ip, sizeof(ud2)))
+ if (probe_kernel_address((unsigned short *)ip, ud2))
return 0;
return ud2 == 0x0b0f;
}
+
+#if defined(CONFIG_PAX_MEMORY_STACKLEAK) || defined(CONFIG_PAX_USERCOPY)
+void pax_check_alloca(unsigned long size)
+{
+ unsigned long sp = (unsigned long)&sp, stack_start, stack_end;
+ unsigned cpu, used;
+ char *id;
+
+ /* check the process stack first */
+ stack_start = (unsigned long)task_stack_page(current);
+ stack_end = stack_start + THREAD_SIZE;
+ if (likely(stack_start <= sp && sp < stack_end)) {
+ unsigned long stack_left = sp & (THREAD_SIZE - 1);
+ BUG_ON(stack_left < 256 || size >= stack_left - 256);
+ return;
+ }
+
+ cpu = get_cpu();
+
+ /* check the irq stacks */
+ stack_end = (unsigned long)per_cpu(irq_stack_ptr, cpu);
+ stack_start = stack_end - IRQ_STACK_SIZE;
+ if (stack_start <= sp && sp < stack_end) {
+ unsigned long stack_left = sp & (IRQ_STACK_SIZE - 1);
+ put_cpu();
+ BUG_ON(stack_left < 256 || size >= stack_left - 256);
+ return;
+ }
+
+ /* check the exception stacks */
+ used = 0;
+ stack_end = (unsigned long)in_exception_stack(cpu, sp, &used, &id);
+ stack_start = stack_end - EXCEPTION_STKSZ;
+ if (stack_end && stack_start <= sp && sp < stack_end) {
+ unsigned long stack_left = sp & (EXCEPTION_STKSZ - 1);
+ put_cpu();
+ BUG_ON(stack_left < 256 || size >= stack_left - 256);
+ return;
+ }
+
+ put_cpu();
+
+ /* unknown stack */
+ BUG();
+}
+EXPORT_SYMBOL(pax_check_alloca);
+#endif
static void early_panic(char *msg)
{
- early_printk(msg);
- panic(msg);
+ early_printk("%s", msg);
+ panic("%s", msg);
}
static int userdef __initdata;
#include <linux/pci_regs.h>
#include <linux/pci_ids.h>
#include <linux/errno.h>
+#include <linux/sched.h>
#include <asm/io.h>
#include <asm/processor.h>
#include <asm/fcntl.h>
/*CFI_REL_OFFSET gs, PT_GS*/
.endm
.macro SET_KERNEL_GS reg
+
+#ifdef CONFIG_CC_STACKPROTECTOR
movl $(__KERNEL_STACK_CANARY), \reg
+#elif defined(CONFIG_PAX_MEMORY_UDEREF)
+ movl $(__USER_DS), \reg
+#else
+ xorl \reg, \reg
+#endif
+
movl \reg, %gs
.endm
#endif /* CONFIG_X86_32_LAZY_GS */
-.macro SAVE_ALL
+.macro pax_enter_kernel
+#ifdef CONFIG_PAX_KERNEXEC
+ call pax_enter_kernel
+#endif
+.endm
+
+.macro pax_exit_kernel
+#ifdef CONFIG_PAX_KERNEXEC
+ call pax_exit_kernel
+#endif
+.endm
+
+#ifdef CONFIG_PAX_KERNEXEC
+ENTRY(pax_enter_kernel)
+#ifdef CONFIG_PARAVIRT
+ pushl %eax
+ pushl %ecx
+ call PARA_INDIRECT(pv_cpu_ops+PV_CPU_read_cr0)
+ mov %eax, %esi
+#else
+ mov %cr0, %esi
+#endif
+ bts $16, %esi
+ jnc 1f
+ mov %cs, %esi
+ cmp $__KERNEL_CS, %esi
+ jz 3f
+ ljmp $__KERNEL_CS, $3f
+1: ljmp $__KERNEXEC_KERNEL_CS, $2f
+2:
+#ifdef CONFIG_PARAVIRT
+ mov %esi, %eax
+ call PARA_INDIRECT(pv_cpu_ops+PV_CPU_write_cr0)
+#else
+ mov %esi, %cr0
+#endif
+3:
+#ifdef CONFIG_PARAVIRT
+ popl %ecx
+ popl %eax
+#endif
+ ret
+ENDPROC(pax_enter_kernel)
+
+ENTRY(pax_exit_kernel)
+#ifdef CONFIG_PARAVIRT
+ pushl %eax
+ pushl %ecx
+#endif
+ mov %cs, %esi
+ cmp $__KERNEXEC_KERNEL_CS, %esi
+ jnz 2f
+#ifdef CONFIG_PARAVIRT
+ call PARA_INDIRECT(pv_cpu_ops+PV_CPU_read_cr0);
+ mov %eax, %esi
+#else
+ mov %cr0, %esi
+#endif
+ btr $16, %esi
+ ljmp $__KERNEL_CS, $1f
+1:
+#ifdef CONFIG_PARAVIRT
+ mov %esi, %eax
+ call PARA_INDIRECT(pv_cpu_ops+PV_CPU_write_cr0);
+#else
+ mov %esi, %cr0
+#endif
+2:
+#ifdef CONFIG_PARAVIRT
+ popl %ecx
+ popl %eax
+#endif
+ ret
+ENDPROC(pax_exit_kernel)
+#endif
+
+ .macro pax_erase_kstack
+#ifdef CONFIG_PAX_MEMORY_STACKLEAK
+ call pax_erase_kstack
+#endif
+ .endm
+
+#ifdef CONFIG_PAX_MEMORY_STACKLEAK
+/*
+ * ebp: thread_info
+ */
+ENTRY(pax_erase_kstack)
+ pushl %edi
+ pushl %ecx
+ pushl %eax
+
+ mov TI_lowest_stack(%ebp), %edi
+ mov $-0xBEEF, %eax
+ std
+
+1: mov %edi, %ecx
+ and $THREAD_SIZE_asm - 1, %ecx
+ shr $2, %ecx
+ repne scasl
+ jecxz 2f
+
+ cmp $2*16, %ecx
+ jc 2f
+
+ mov $2*16, %ecx
+ repe scasl
+ jecxz 2f
+ jne 1b
+
+2: cld
+ or $2*4, %edi
+ mov %esp, %ecx
+ sub %edi, %ecx
+
+ cmp $THREAD_SIZE_asm, %ecx
+ jb 3f
+ ud2
+3:
+
+ shr $2, %ecx
+ rep stosl
+
+ mov TI_task_thread_sp0(%ebp), %edi
+ sub $128, %edi
+ mov %edi, TI_lowest_stack(%ebp)
+
+ popl %eax
+ popl %ecx
+ popl %edi
+ ret
+ENDPROC(pax_erase_kstack)
+#endif
+
+.macro __SAVE_ALL _DS
cld
PUSH_GS
pushl_cfi %fs
CFI_REL_OFFSET ecx, 0
pushl_cfi %ebx
CFI_REL_OFFSET ebx, 0
- movl $(__USER_DS), %edx
+ movl $\_DS, %edx
movl %edx, %ds
movl %edx, %es
movl $(__KERNEL_PERCPU), %edx
SET_KERNEL_GS %edx
.endm
+.macro SAVE_ALL
+#if defined(CONFIG_PAX_KERNEXEC) || defined(CONFIG_PAX_PAGEEXEC) || defined(CONFIG_PAX_SEGMEXEC) || defined(CONFIG_PAX_MEMORY_UDEREF)
+ __SAVE_ALL __KERNEL_DS
+ pax_enter_kernel
+#else
+ __SAVE_ALL __USER_DS
+#endif
+.endm
+
.macro RESTORE_INT_REGS
popl_cfi %ebx
CFI_RESTORE ebx
popfl_cfi
jmp syscall_exit
CFI_ENDPROC
-END(ret_from_fork)
+ENDPROC(ret_from_fork)
ENTRY(ret_from_kernel_thread)
CFI_STARTPROC
andl $SEGMENT_RPL_MASK, %eax
#endif
cmpl $USER_RPL, %eax
+
+#ifdef CONFIG_PAX_KERNEXEC
+ jae resume_userspace
+
+ pax_exit_kernel
+ jmp resume_kernel
+#else
jb resume_kernel # not returning to v8086 or userspace
+#endif
ENTRY(resume_userspace)
LOCKDEP_SYS_EXIT
andl $_TIF_WORK_MASK, %ecx # is there any work to be done on
# int/exception return?
jne work_pending
- jmp restore_all
-END(ret_from_exception)
+ jmp restore_all_pax
+ENDPROC(ret_from_exception)
#ifdef CONFIG_PREEMPT
ENTRY(resume_kernel)
jz restore_all
call preempt_schedule_irq
jmp need_resched
-END(resume_kernel)
+ENDPROC(resume_kernel)
#endif
CFI_ENDPROC
/*CFI_REL_OFFSET cs, 0*/
/*
* Push current_thread_info()->sysenter_return to the stack.
- * A tiny bit of offset fixup is necessary - 4*4 means the 4 words
- * pushed above; +8 corresponds to copy_thread's esp0 setting.
*/
- pushl_cfi ((TI_sysenter_return)-THREAD_SIZE+8+4*4)(%esp)
+ pushl_cfi $0
CFI_REL_OFFSET eip, 0
pushl_cfi %eax
SAVE_ALL
+ GET_THREAD_INFO(%ebp)
+ movl TI_sysenter_return(%ebp),%ebp
+ movl %ebp,PT_EIP(%esp)
ENABLE_INTERRUPTS(CLBR_NONE)
/*
* Load the potential sixth argument from user stack.
* Careful about security.
*/
+ movl PT_OLDESP(%esp),%ebp
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ mov PT_OLDSS(%esp),%ds
+1: movl %ds:(%ebp),%ebp
+ push %ss
+ pop %ds
+#else
cmpl $__PAGE_OFFSET-3,%ebp
jae syscall_fault
ASM_STAC
1: movl (%ebp),%ebp
ASM_CLAC
+#endif
+
movl %ebp,PT_EBP(%esp)
_ASM_EXTABLE(1b,syscall_fault)
GET_THREAD_INFO(%ebp)
+#ifdef CONFIG_PAX_RANDKSTACK
+ pax_erase_kstack
+#endif
+
testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp)
jnz sysenter_audit
sysenter_do_call:
testl $_TIF_ALLWORK_MASK, %ecx
jne sysexit_audit
sysenter_exit:
+
+#ifdef CONFIG_PAX_RANDKSTACK
+ pushl_cfi %eax
+ movl %esp, %eax
+ call pax_randomize_kstack
+ popl_cfi %eax
+#endif
+
+ pax_erase_kstack
+
/* if something modifies registers it must also disable sysexit */
movl PT_EIP(%esp), %edx
movl PT_OLDESP(%esp), %ecx
xorl %ebp,%ebp
TRACE_IRQS_ON
1: mov PT_FS(%esp), %fs
+2: mov PT_DS(%esp), %ds
+3: mov PT_ES(%esp), %es
PTGS_TO_GS
ENABLE_INTERRUPTS_SYSEXIT
pushl_cfi PT_ESI(%esp) /* a3: 5th arg */
pushl_cfi PT_EDX+4(%esp) /* a2: 4th arg */
call __audit_syscall_entry
+
+ pax_erase_kstack
+
popl_cfi %ecx /* get that remapped edx off the stack */
popl_cfi %ecx /* get that remapped esi off the stack */
movl PT_EAX(%esp),%eax /* reload syscall number */
CFI_ENDPROC
.pushsection .fixup,"ax"
-2: movl $0,PT_FS(%esp)
+4: movl $0,PT_FS(%esp)
+ jmp 1b
+5: movl $0,PT_DS(%esp)
+ jmp 1b
+6: movl $0,PT_ES(%esp)
jmp 1b
.popsection
- _ASM_EXTABLE(1b,2b)
+ _ASM_EXTABLE(1b,4b)
+ _ASM_EXTABLE(2b,5b)
+ _ASM_EXTABLE(3b,6b)
PTGS_TO_GS_EX
ENDPROC(ia32_sysenter_target)
pushl_cfi %eax # save orig_eax
SAVE_ALL
GET_THREAD_INFO(%ebp)
+
+#ifdef CONFIG_PAX_RANDKSTACK
+ pax_erase_kstack
+#endif
+
# system call tracing in operation / emulation
testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp)
jnz syscall_trace_entry
testl $_TIF_ALLWORK_MASK, %ecx # current->work
jne syscall_exit_work
+restore_all_pax:
+
+#ifdef CONFIG_PAX_RANDKSTACK
+ movl %esp, %eax
+ call pax_randomize_kstack
+#endif
+
+ pax_erase_kstack
+
restore_all:
TRACE_IRQS_IRET
restore_all_notrace:
* compensating for the offset by changing to the ESPFIX segment with
* a base address that matches for the difference.
*/
-#define GDT_ESPFIX_SS PER_CPU_VAR(gdt_page) + (GDT_ENTRY_ESPFIX_SS * 8)
+#define GDT_ESPFIX_SS (GDT_ENTRY_ESPFIX_SS * 8)(%ebx)
mov %esp, %edx /* load kernel esp */
mov PT_OLDESP(%esp), %eax /* load userspace esp */
mov %dx, %ax /* eax: new kernel esp */
sub %eax, %edx /* offset (low word is 0) */
+#ifdef CONFIG_SMP
+ movl PER_CPU_VAR(cpu_number), %ebx
+ shll $PAGE_SHIFT_asm, %ebx
+ addl $cpu_gdt_table, %ebx
+#else
+ movl $cpu_gdt_table, %ebx
+#endif
shr $16, %edx
- mov %dl, GDT_ESPFIX_SS + 4 /* bits 16..23 */
- mov %dh, GDT_ESPFIX_SS + 7 /* bits 24..31 */
+
+#ifdef CONFIG_PAX_KERNEXEC
+ mov %cr0, %esi
+ btr $16, %esi
+ mov %esi, %cr0
+#endif
+
+ mov %dl, 4 + GDT_ESPFIX_SS /* bits 16..23 */
+ mov %dh, 7 + GDT_ESPFIX_SS /* bits 24..31 */
+
+#ifdef CONFIG_PAX_KERNEXEC
+ bts $16, %esi
+ mov %esi, %cr0
+#endif
+
pushl_cfi $__ESPFIX_SS
pushl_cfi %eax /* new kernel esp */
/* Disable interrupts, but do not irqtrace this section: we
movl TI_flags(%ebp), %ecx
andl $_TIF_WORK_MASK, %ecx # is there any work to be done other
# than syscall tracing?
- jz restore_all
+ jz restore_all_pax
testb $_TIF_NEED_RESCHED, %cl
jnz work_resched
work_notifysig: # deal with pending signals and
# notify-resume requests
+ movl %esp, %eax
#ifdef CONFIG_VM86
testl $X86_EFLAGS_VM, PT_EFLAGS(%esp)
- movl %esp, %eax
jne work_notifysig_v86 # returning to kernel-space or
# vm86-space
1:
-#else
- movl %esp, %eax
#endif
TRACE_IRQS_ON
ENABLE_INTERRUPTS(CLBR_NONE)
movl %eax, %esp
jmp 1b
#endif
-END(work_pending)
+ENDPROC(work_pending)
# perform syscall exit tracing
ALIGN
movl $-ENOSYS,PT_EAX(%esp)
movl %esp, %eax
call syscall_trace_enter
+
+ pax_erase_kstack
+
/* What it returned is what we'll actually use. */
cmpl $(NR_syscalls), %eax
jnae syscall_call
jmp syscall_exit
-END(syscall_trace_entry)
+ENDPROC(syscall_trace_entry)
# perform syscall exit tracing
ALIGN
movl %esp, %eax
call syscall_trace_leave
jmp resume_userspace
-END(syscall_exit_work)
+ENDPROC(syscall_exit_work)
CFI_ENDPROC
RING0_INT_FRAME # can't unwind into user space anyway
syscall_fault:
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ push %ss
+ pop %ds
+#endif
ASM_CLAC
GET_THREAD_INFO(%ebp)
movl $-EFAULT,PT_EAX(%esp)
jmp resume_userspace
-END(syscall_fault)
+ENDPROC(syscall_fault)
syscall_badsys:
movl $-ENOSYS,%eax
jmp syscall_after_call
-END(syscall_badsys)
+ENDPROC(syscall_badsys)
sysenter_badsys:
movl $-ENOSYS,%eax
jmp sysenter_after_call
-END(sysenter_badsys)
+ENDPROC(sysenter_badsys)
CFI_ENDPROC
.macro FIXUP_ESPFIX_STACK
*/
#ifdef CONFIG_X86_ESPFIX32
/* fixup the stack */
- mov GDT_ESPFIX_SS + 4, %al /* bits 16..23 */
- mov GDT_ESPFIX_SS + 7, %ah /* bits 24..31 */
+#ifdef CONFIG_SMP
+ movl PER_CPU_VAR(cpu_number), %ebx
+ shll $PAGE_SHIFT_asm, %ebx
+ addl $cpu_gdt_table, %ebx
+#else
+ movl $cpu_gdt_table, %ebx
+#endif
+ mov 4 + GDT_ESPFIX_SS, %al /* bits 16..23 */
+ mov 7 + GDT_ESPFIX_SS, %ah /* bits 24..31 */
shl $16, %eax
addl %esp, %eax /* the adjusted stack pointer */
pushl_cfi $__KERNEL_DS
.endr
2: jmp common_interrupt
.endr
-END(irq_entries_start)
+ENDPROC(irq_entries_start)
.previous
END(interrupt)
pushl_cfi $do_coprocessor_error
jmp error_code
CFI_ENDPROC
-END(coprocessor_error)
+ENDPROC(coprocessor_error)
ENTRY(simd_coprocessor_error)
RING0_INT_FRAME
.section .altinstructions,"a"
altinstruction_entry 661b, 663f, X86_FEATURE_XMM, 662b-661b, 664f-663f
.previous
-.section .altinstr_replacement,"ax"
+.section .altinstr_replacement,"a"
663: pushl $do_simd_coprocessor_error
664:
.previous
#endif
jmp error_code
CFI_ENDPROC
-END(simd_coprocessor_error)
+ENDPROC(simd_coprocessor_error)
ENTRY(device_not_available)
RING0_INT_FRAME
pushl_cfi $do_device_not_available
jmp error_code
CFI_ENDPROC
-END(device_not_available)
+ENDPROC(device_not_available)
#ifdef CONFIG_PARAVIRT
ENTRY(native_iret)
iret
_ASM_EXTABLE(native_iret, iret_exc)
-END(native_iret)
+ENDPROC(native_iret)
ENTRY(native_irq_enable_sysexit)
sti
sysexit
-END(native_irq_enable_sysexit)
+ENDPROC(native_irq_enable_sysexit)
#endif
ENTRY(overflow)
pushl_cfi $do_overflow
jmp error_code
CFI_ENDPROC
-END(overflow)
+ENDPROC(overflow)
ENTRY(bounds)
RING0_INT_FRAME
pushl_cfi $do_bounds
jmp error_code
CFI_ENDPROC
-END(bounds)
+ENDPROC(bounds)
ENTRY(invalid_op)
RING0_INT_FRAME
pushl_cfi $do_invalid_op
jmp error_code
CFI_ENDPROC
-END(invalid_op)
+ENDPROC(invalid_op)
ENTRY(coprocessor_segment_overrun)
RING0_INT_FRAME
pushl_cfi $do_coprocessor_segment_overrun
jmp error_code
CFI_ENDPROC
-END(coprocessor_segment_overrun)
+ENDPROC(coprocessor_segment_overrun)
ENTRY(invalid_TSS)
RING0_EC_FRAME
pushl_cfi $do_invalid_TSS
jmp error_code
CFI_ENDPROC
-END(invalid_TSS)
+ENDPROC(invalid_TSS)
ENTRY(segment_not_present)
RING0_EC_FRAME
pushl_cfi $do_segment_not_present
jmp error_code
CFI_ENDPROC
-END(segment_not_present)
+ENDPROC(segment_not_present)
ENTRY(stack_segment)
RING0_EC_FRAME
pushl_cfi $do_stack_segment
jmp error_code
CFI_ENDPROC
-END(stack_segment)
+ENDPROC(stack_segment)
ENTRY(alignment_check)
RING0_EC_FRAME
pushl_cfi $do_alignment_check
jmp error_code
CFI_ENDPROC
-END(alignment_check)
+ENDPROC(alignment_check)
ENTRY(divide_error)
RING0_INT_FRAME
pushl_cfi $do_divide_error
jmp error_code
CFI_ENDPROC
-END(divide_error)
+ENDPROC(divide_error)
#ifdef CONFIG_X86_MCE
ENTRY(machine_check)
pushl_cfi machine_check_vector
jmp error_code
CFI_ENDPROC
-END(machine_check)
+ENDPROC(machine_check)
#endif
ENTRY(spurious_interrupt_bug)
pushl_cfi $do_spurious_interrupt_bug
jmp error_code
CFI_ENDPROC
-END(spurious_interrupt_bug)
+ENDPROC(spurious_interrupt_bug)
#ifdef CONFIG_XEN
/* Xen doesn't set %esp to be precisely what the normal sysenter
ENTRY(mcount)
ret
-END(mcount)
+ENDPROC(mcount)
ENTRY(ftrace_caller)
pushl %eax
.globl ftrace_stub
ftrace_stub:
ret
-END(ftrace_caller)
+ENDPROC(ftrace_caller)
ENTRY(ftrace_regs_caller)
pushf /* push flags before compare (in cs location) */
popl %ecx
popl %eax
jmp ftrace_stub
-END(mcount)
+ENDPROC(mcount)
#endif /* CONFIG_DYNAMIC_FTRACE */
#endif /* CONFIG_FUNCTION_TRACER */
popl %ecx
popl %eax
ret
-END(ftrace_graph_caller)
+ENDPROC(ftrace_graph_caller)
.globl return_to_handler
return_to_handler:
movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart
REG_TO_PTGS %ecx
SET_KERNEL_GS %ecx
- movl $(__USER_DS), %ecx
+ movl $(__KERNEL_DS), %ecx
movl %ecx, %ds
movl %ecx, %es
+
+ pax_enter_kernel
+
TRACE_IRQS_OFF
movl %esp,%eax # pt_regs pointer
call *%edi
jmp ret_from_exception
CFI_ENDPROC
-END(page_fault)
+ENDPROC(page_fault)
/*
* Debug traps and NMI can happen at the one SYSENTER instruction
call do_debug
jmp ret_from_exception
CFI_ENDPROC
-END(debug)
+ENDPROC(debug)
/*
* NMI is doubly nasty. It can happen _while_ we're handling
xorl %edx,%edx # zero error code
movl %esp,%eax # pt_regs pointer
call do_nmi
+
+ pax_exit_kernel
+
jmp restore_all_notrace
CFI_ENDPROC
FIXUP_ESPFIX_STACK # %eax == %esp
xorl %edx,%edx # zero error code
call do_nmi
+
+ pax_exit_kernel
+
RESTORE_REGS
lss 12+4(%esp), %esp # back to espfix stack
CFI_ADJUST_CFA_OFFSET -24
jmp irq_return
#endif
CFI_ENDPROC
-END(nmi)
+ENDPROC(nmi)
ENTRY(int3)
RING0_INT_FRAME
call do_int3
jmp ret_from_exception
CFI_ENDPROC
-END(int3)
+ENDPROC(int3)
ENTRY(general_protection)
RING0_EC_FRAME
pushl_cfi $do_general_protection
jmp error_code
CFI_ENDPROC
-END(general_protection)
+ENDPROC(general_protection)
#ifdef CONFIG_KVM_GUEST
ENTRY(async_page_fault)
pushl_cfi $do_async_page_fault
jmp error_code
CFI_ENDPROC
-END(async_page_fault)
+ENDPROC(async_page_fault)
#endif
#include <asm/smap.h>
#include <asm/pgtable_types.h>
#include <linux/err.h>
+#include <asm/pgtable.h>
+#include <asm/alternative-asm.h>
/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */
#include <linux/elf-em.h>
ENDPROC(native_usergs_sysret64)
#endif /* CONFIG_PARAVIRT */
+ .macro ljmpq sel, off
+#if defined(CONFIG_MPSC) || defined(CONFIG_MCORE2) || defined (CONFIG_MATOM)
+ .byte 0x48; ljmp *1234f(%rip)
+ .pushsection .rodata
+ .align 16
+ 1234: .quad \off; .word \sel
+ .popsection
+#else
+ pushq $\sel
+ pushq $\off
+ lretq
+#endif
+ .endm
+
+ .macro pax_enter_kernel
+ pax_set_fptr_mask
+#if defined(CONFIG_PAX_KERNEXEC) || defined(CONFIG_PAX_MEMORY_UDEREF)
+ call pax_enter_kernel
+#endif
+ .endm
+
+ .macro pax_exit_kernel
+#if defined(CONFIG_PAX_KERNEXEC) || defined(CONFIG_PAX_MEMORY_UDEREF)
+ call pax_exit_kernel
+#endif
+
+ .endm
+
+#if defined(CONFIG_PAX_KERNEXEC) || defined(CONFIG_PAX_MEMORY_UDEREF)
+ENTRY(pax_enter_kernel)
+ pushq %rdi
+
+#ifdef CONFIG_PARAVIRT
+ PV_SAVE_REGS(CLBR_RDI)
+#endif
+
+#ifdef CONFIG_PAX_KERNEXEC
+ GET_CR0_INTO_RDI
+ bts $16,%rdi
+ jnc 3f
+ mov %cs,%edi
+ cmp $__KERNEL_CS,%edi
+ jnz 2f
+1:
+#endif
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ 661: jmp 111f
+ .pushsection .altinstr_replacement, "a"
+ 662: ASM_NOP2
+ .popsection
+ .pushsection .altinstructions, "a"
+ altinstruction_entry 661b, 662b, X86_FEATURE_PCID, 2, 2
+ .popsection
+ GET_CR3_INTO_RDI
+ cmp $0,%dil
+ jnz 112f
+ mov $__KERNEL_DS,%edi
+ mov %edi,%ss
+ jmp 111f
+112: cmp $1,%dil
+ jz 113f
+ ud2
+113: sub $4097,%rdi
+ bts $63,%rdi
+ SET_RDI_INTO_CR3
+ mov $__UDEREF_KERNEL_DS,%edi
+ mov %edi,%ss
+111:
+#endif
+
+#ifdef CONFIG_PARAVIRT
+ PV_RESTORE_REGS(CLBR_RDI)
+#endif
+
+ popq %rdi
+ pax_force_retaddr
+ retq
+
+#ifdef CONFIG_PAX_KERNEXEC
+2: ljmpq __KERNEL_CS,1b
+3: ljmpq __KERNEXEC_KERNEL_CS,4f
+4: SET_RDI_INTO_CR0
+ jmp 1b
+#endif
+ENDPROC(pax_enter_kernel)
+
+ENTRY(pax_exit_kernel)
+ pushq %rdi
+
+#ifdef CONFIG_PARAVIRT
+ PV_SAVE_REGS(CLBR_RDI)
+#endif
+
+#ifdef CONFIG_PAX_KERNEXEC
+ mov %cs,%rdi
+ cmp $__KERNEXEC_KERNEL_CS,%edi
+ jz 2f
+ GET_CR0_INTO_RDI
+ bts $16,%rdi
+ jnc 4f
+1:
+#endif
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ 661: jmp 111f
+ .pushsection .altinstr_replacement, "a"
+ 662: ASM_NOP2
+ .popsection
+ .pushsection .altinstructions, "a"
+ altinstruction_entry 661b, 662b, X86_FEATURE_PCID, 2, 2
+ .popsection
+ mov %ss,%edi
+ cmp $__UDEREF_KERNEL_DS,%edi
+ jnz 111f
+ GET_CR3_INTO_RDI
+ cmp $0,%dil
+ jz 112f
+ ud2
+112: add $4097,%rdi
+ bts $63,%rdi
+ SET_RDI_INTO_CR3
+ mov $__KERNEL_DS,%edi
+ mov %edi,%ss
+111:
+#endif
+
+#ifdef CONFIG_PARAVIRT
+ PV_RESTORE_REGS(CLBR_RDI);
+#endif
+
+ popq %rdi
+ pax_force_retaddr
+ retq
+
+#ifdef CONFIG_PAX_KERNEXEC
+2: GET_CR0_INTO_RDI
+ btr $16,%rdi
+ jnc 4f
+ ljmpq __KERNEL_CS,3f
+3: SET_RDI_INTO_CR0
+ jmp 1b
+4: ud2
+ jmp 4b
+#endif
+ENDPROC(pax_exit_kernel)
+#endif
+
+ .macro pax_enter_kernel_user
+ pax_set_fptr_mask
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ call pax_enter_kernel_user
+#endif
+ .endm
+
+ .macro pax_exit_kernel_user
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ call pax_exit_kernel_user
+#endif
+#ifdef CONFIG_PAX_RANDKSTACK
+ pushq %rax
+ pushq %r11
+ call pax_randomize_kstack
+ popq %r11
+ popq %rax
+#endif
+ .endm
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ENTRY(pax_enter_kernel_user)
+ pushq %rdi
+ pushq %rbx
+
+#ifdef CONFIG_PARAVIRT
+ PV_SAVE_REGS(CLBR_RDI)
+#endif
+
+ 661: jmp 111f
+ .pushsection .altinstr_replacement, "a"
+ 662: ASM_NOP2
+ .popsection
+ .pushsection .altinstructions, "a"
+ altinstruction_entry 661b, 662b, X86_FEATURE_PCID, 2, 2
+ .popsection
+ GET_CR3_INTO_RDI
+ cmp $1,%dil
+ jnz 4f
+ sub $4097,%rdi
+ bts $63,%rdi
+ SET_RDI_INTO_CR3
+ jmp 3f
+111:
+
+ GET_CR3_INTO_RDI
+ mov %rdi,%rbx
+ add $__START_KERNEL_map,%rbx
+ sub phys_base(%rip),%rbx
+
+#ifdef CONFIG_PARAVIRT
+ cmpl $0, pv_info+PARAVIRT_enabled
+ jz 1f
+ pushq %rdi
+ i = 0
+ .rept USER_PGD_PTRS
+ mov i*8(%rbx),%rsi
+ mov $0,%sil
+ lea i*8(%rbx),%rdi
+ call PARA_INDIRECT(pv_mmu_ops+PV_MMU_set_pgd_batched)
+ i = i + 1
+ .endr
+ popq %rdi
+ jmp 2f
+1:
+#endif
+
+ i = 0
+ .rept USER_PGD_PTRS
+ movb $0,i*8(%rbx)
+ i = i + 1
+ .endr
+
+2: SET_RDI_INTO_CR3
+
+#ifdef CONFIG_PAX_KERNEXEC
+ GET_CR0_INTO_RDI
+ bts $16,%rdi
+ SET_RDI_INTO_CR0
+#endif
+
+3:
+
+#ifdef CONFIG_PARAVIRT
+ PV_RESTORE_REGS(CLBR_RDI)
+#endif
+
+ popq %rbx
+ popq %rdi
+ pax_force_retaddr
+ retq
+4: ud2
+ENDPROC(pax_enter_kernel_user)
+
+ENTRY(pax_exit_kernel_user)
+ pushq %rdi
+ pushq %rbx
+
+#ifdef CONFIG_PARAVIRT
+ PV_SAVE_REGS(CLBR_RDI)
+#endif
+
+ GET_CR3_INTO_RDI
+ 661: jmp 1f
+ .pushsection .altinstr_replacement, "a"
+ 662: ASM_NOP2
+ .popsection
+ .pushsection .altinstructions, "a"
+ altinstruction_entry 661b, 662b, X86_FEATURE_PCID, 2, 2
+ .popsection
+ cmp $0,%dil
+ jnz 3f
+ add $4097,%rdi
+ bts $63,%rdi
+ SET_RDI_INTO_CR3
+ jmp 2f
+1:
+
+ mov %rdi,%rbx
+
+#ifdef CONFIG_PAX_KERNEXEC
+ GET_CR0_INTO_RDI
+ btr $16,%rdi
+ jnc 3f
+ SET_RDI_INTO_CR0
+#endif
+
+ add $__START_KERNEL_map,%rbx
+ sub phys_base(%rip),%rbx
+
+#ifdef CONFIG_PARAVIRT
+ cmpl $0, pv_info+PARAVIRT_enabled
+ jz 1f
+ i = 0
+ .rept USER_PGD_PTRS
+ mov i*8(%rbx),%rsi
+ mov $0x67,%sil
+ lea i*8(%rbx),%rdi
+ call PARA_INDIRECT(pv_mmu_ops+PV_MMU_set_pgd_batched)
+ i = i + 1
+ .endr
+ jmp 2f
+1:
+#endif
+
+ i = 0
+ .rept USER_PGD_PTRS
+ movb $0x67,i*8(%rbx)
+ i = i + 1
+ .endr
+2:
+
+#ifdef CONFIG_PARAVIRT
+ PV_RESTORE_REGS(CLBR_RDI)
+#endif
+
+ popq %rbx
+ popq %rdi
+ pax_force_retaddr
+ retq
+3: ud2
+ENDPROC(pax_exit_kernel_user)
+#endif
+
+ .macro pax_enter_kernel_nmi
+ pax_set_fptr_mask
+
+#ifdef CONFIG_PAX_KERNEXEC
+ GET_CR0_INTO_RDI
+ bts $16,%rdi
+ jc 110f
+ SET_RDI_INTO_CR0
+ or $2,%ebx
+110:
+#endif
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ 661: jmp 111f
+ .pushsection .altinstr_replacement, "a"
+ 662: ASM_NOP2
+ .popsection
+ .pushsection .altinstructions, "a"
+ altinstruction_entry 661b, 662b, X86_FEATURE_PCID, 2, 2
+ .popsection
+ GET_CR3_INTO_RDI
+ cmp $0,%dil
+ jz 111f
+ sub $4097,%rdi
+ or $4,%ebx
+ bts $63,%rdi
+ SET_RDI_INTO_CR3
+ mov $__UDEREF_KERNEL_DS,%edi
+ mov %edi,%ss
+111:
+#endif
+ .endm
+
+ .macro pax_exit_kernel_nmi
+#ifdef CONFIG_PAX_KERNEXEC
+ btr $1,%ebx
+ jnc 110f
+ GET_CR0_INTO_RDI
+ btr $16,%rdi
+ SET_RDI_INTO_CR0
+110:
+#endif
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ btr $2,%ebx
+ jnc 111f
+ GET_CR3_INTO_RDI
+ add $4097,%rdi
+ bts $63,%rdi
+ SET_RDI_INTO_CR3
+ mov $__KERNEL_DS,%edi
+ mov %edi,%ss
+111:
+#endif
+ .endm
+
+ .macro pax_erase_kstack
+#ifdef CONFIG_PAX_MEMORY_STACKLEAK
+ call pax_erase_kstack
+#endif
+ .endm
+
+#ifdef CONFIG_PAX_MEMORY_STACKLEAK
+ENTRY(pax_erase_kstack)
+ pushq %rdi
+ pushq %rcx
+ pushq %rax
+ pushq %r11
+
+ GET_THREAD_INFO(%r11)
+ mov TI_lowest_stack(%r11), %rdi
+ mov $-0xBEEF, %rax
+ std
+
+1: mov %edi, %ecx
+ and $THREAD_SIZE_asm - 1, %ecx
+ shr $3, %ecx
+ repne scasq
+ jecxz 2f
+
+ cmp $2*8, %ecx
+ jc 2f
+
+ mov $2*8, %ecx
+ repe scasq
+ jecxz 2f
+ jne 1b
+
+2: cld
+ or $2*8, %rdi
+ mov %esp, %ecx
+ sub %edi, %ecx
+
+ cmp $THREAD_SIZE_asm, %rcx
+ jb 3f
+ ud2
+3:
+
+ shr $3, %ecx
+ rep stosq
+
+ mov TI_task_thread_sp0(%r11), %rdi
+ sub $256, %rdi
+ mov %rdi, TI_lowest_stack(%r11)
+
+ popq %r11
+ popq %rax
+ popq %rcx
+ popq %rdi
+ pax_force_retaddr
+ ret
+ENDPROC(pax_erase_kstack)
+#endif
.macro TRACE_IRQS_IRETQ offset=ARGOFFSET
#ifdef CONFIG_TRACE_IRQFLAGS
.endm
.macro TRACE_IRQS_IRETQ_DEBUG offset=ARGOFFSET
- bt $9,EFLAGS-\offset(%rsp) /* interrupts off? */
+ bt $X86_EFLAGS_IF_BIT,EFLAGS-\offset(%rsp) /* interrupts off? */
jnc 1f
TRACE_IRQS_ON_DEBUG
1:
movq \tmp,R11+\offset(%rsp)
.endm
- .macro FAKE_STACK_FRAME child_rip
- /* push in order ss, rsp, eflags, cs, rip */
- xorl %eax, %eax
- pushq_cfi $__KERNEL_DS /* ss */
- /*CFI_REL_OFFSET ss,0*/
- pushq_cfi %rax /* rsp */
- CFI_REL_OFFSET rsp,0
- pushq_cfi $(X86_EFLAGS_IF|X86_EFLAGS_FIXED) /* eflags - interrupts on */
- /*CFI_REL_OFFSET rflags,0*/
- pushq_cfi $__KERNEL_CS /* cs */
- /*CFI_REL_OFFSET cs,0*/
- pushq_cfi \child_rip /* rip */
- CFI_REL_OFFSET rip,0
- pushq_cfi %rax /* orig rax */
- .endm
-
- .macro UNFAKE_STACK_FRAME
- addq $8*6, %rsp
- CFI_ADJUST_CFA_OFFSET -(6*8)
- .endm
-
/*
* initial frame state for interrupts (and exceptions without error code)
*/
/* save partial stack frame */
.macro SAVE_ARGS_IRQ
cld
- /* start from rbp in pt_regs and jump over */
- movq_cfi rdi, (RDI-RBP)
- movq_cfi rsi, (RSI-RBP)
- movq_cfi rdx, (RDX-RBP)
- movq_cfi rcx, (RCX-RBP)
- movq_cfi rax, (RAX-RBP)
- movq_cfi r8, (R8-RBP)
- movq_cfi r9, (R9-RBP)
- movq_cfi r10, (R10-RBP)
- movq_cfi r11, (R11-RBP)
+ /* start from r15 in pt_regs and jump over */
+ movq_cfi rdi, RDI
+ movq_cfi rsi, RSI
+ movq_cfi rdx, RDX
+ movq_cfi rcx, RCX
+ movq_cfi rax, RAX
+ movq_cfi r8, R8
+ movq_cfi r9, R9
+ movq_cfi r10, R10
+ movq_cfi r11, R11
+ movq_cfi r12, R12
/* Save rbp so that we can unwind from get_irq_regs() */
- movq_cfi rbp, 0
+ movq_cfi rbp, RBP
/* Save previous stack value */
movq %rsp, %rsi
- leaq -RBP(%rsp),%rdi /* arg1 for handler */
- testl $3, CS-RBP(%rsi)
+ movq %rsp,%rdi /* arg1 for handler */
+ testb $3, CS(%rsi)
je 1f
SWAPGS
/*
0x06 /* DW_OP_deref */, \
0x08 /* DW_OP_const1u */, SS+8-RBP, \
0x22 /* DW_OP_plus */
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ testb $3, CS(%rdi)
+ jnz 1f
+ pax_enter_kernel
+ jmp 2f
+1: pax_enter_kernel_user
+2:
+#else
+ pax_enter_kernel
+#endif
+
/* We entered an interrupt context - irqs are off: */
TRACE_IRQS_OFF
.endm
js 1f /* negative -> in kernel */
SWAPGS
xorl %ebx,%ebx
-1: ret
+1:
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ testb $3, CS+8(%rsp)
+ jnz 1f
+ pax_enter_kernel
+ jmp 2f
+1: pax_enter_kernel_user
+2:
+#else
+ pax_enter_kernel
+#endif
+ pax_force_retaddr
+ ret
CFI_ENDPROC
-END(save_paranoid)
+ENDPROC(save_paranoid)
+
+ENTRY(save_paranoid_nmi)
+ XCPT_FRAME 1 RDI+8
+ cld
+ movq_cfi rdi, RDI+8
+ movq_cfi rsi, RSI+8
+ movq_cfi rdx, RDX+8
+ movq_cfi rcx, RCX+8
+ movq_cfi rax, RAX+8
+ movq_cfi r8, R8+8
+ movq_cfi r9, R9+8
+ movq_cfi r10, R10+8
+ movq_cfi r11, R11+8
+ movq_cfi rbx, RBX+8
+ movq_cfi rbp, RBP+8
+ movq_cfi r12, R12+8
+ movq_cfi r13, R13+8
+ movq_cfi r14, R14+8
+ movq_cfi r15, R15+8
+ movl $1,%ebx
+ movl $MSR_GS_BASE,%ecx
+ rdmsr
+ testl %edx,%edx
+ js 1f /* negative -> in kernel */
+ SWAPGS
+ xorl %ebx,%ebx
+1: pax_enter_kernel_nmi
+ pax_force_retaddr
+ ret
+ CFI_ENDPROC
+ENDPROC(save_paranoid_nmi)
/*
* A newly forked process directly context switches into this address.
RESTORE_REST
- testl $3, CS-ARGOFFSET(%rsp) # from kernel_thread?
+ testb $3, CS-ARGOFFSET(%rsp) # from kernel_thread?
jz 1f
/*
jmp int_ret_from_sys_call
1:
- subq $REST_SKIP, %rsp # leave space for volatiles
- CFI_ADJUST_CFA_OFFSET REST_SKIP
movq %rbp, %rdi
call *%rbx
movl $0, RAX(%rsp)
RESTORE_REST
jmp int_ret_from_sys_call
CFI_ENDPROC
-END(ret_from_fork)
+ENDPROC(ret_from_fork)
/*
* System call entry. Up to 6 arguments in registers are supported.
ENTRY(system_call)
CFI_STARTPROC simple
CFI_SIGNAL_FRAME
- CFI_DEF_CFA rsp,KERNEL_STACK_OFFSET
+ CFI_DEF_CFA rsp,0
CFI_REGISTER rip,rcx
/*CFI_REGISTER rflags,r11*/
SWAPGS_UNSAFE_STACK
movq %rsp,PER_CPU_VAR(old_rsp)
movq PER_CPU_VAR(kernel_stack),%rsp
+ SAVE_ARGS 8*6, 0, rax_enosys=1
+ pax_enter_kernel_user
+
+#ifdef CONFIG_PAX_RANDKSTACK
+ pax_erase_kstack
+#endif
+
/*
* No need to follow this irqs off/on section - it's straight
* and short:
*/
ENABLE_INTERRUPTS(CLBR_NONE)
- SAVE_ARGS 8, 0, rax_enosys=1
movq_cfi rax,(ORIG_RAX-ARGOFFSET)
movq %rcx,RIP-ARGOFFSET(%rsp)
CFI_REL_OFFSET rip,RIP-ARGOFFSET
- testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
+ GET_THREAD_INFO(%rcx)
+ testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%rcx)
jnz tracesys
system_call_fastpath:
#if __SYSCALL_MASK == ~0
LOCKDEP_SYS_EXIT
DISABLE_INTERRUPTS(CLBR_NONE)
TRACE_IRQS_OFF
- movl TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET),%edx
+ GET_THREAD_INFO(%rcx)
+ movl TI_flags(%rcx),%edx
andl %edi,%edx
jnz sysret_careful
CFI_REMEMBER_STATE
+ pax_exit_kernel_user
+ pax_erase_kstack
/*
* sysretq will re-enable interrupts:
*/
/* Do syscall tracing */
tracesys:
- leaq -REST_SKIP(%rsp), %rdi
+ movq %rsp, %rdi
movq $AUDIT_ARCH_X86_64, %rsi
call syscall_trace_enter_phase1
test %rax, %rax
jnz tracesys_phase2 /* if needed, run the slow path */
- LOAD_ARGS 0 /* else restore clobbered regs */
+
+ pax_erase_kstack
+
+ LOAD_ARGS /* else restore clobbered regs */
jmp system_call_fastpath /* and return to the fast path */
tracesys_phase2:
movq %rax,%rdx
call syscall_trace_enter_phase2
+ pax_erase_kstack
+
/*
* Reload arg registers from stack in case ptrace changed them.
* We don't reload %rax because syscall_trace_entry_phase2() returned
* the value it wants us to use in the table lookup.
*/
- LOAD_ARGS ARGOFFSET, 1
+ LOAD_ARGS 1
RESTORE_REST
#if __SYSCALL_MASK == ~0
cmpq $__NR_syscall_max,%rax
andl %edi,%edx
jnz int_careful
andl $~TS_COMPAT,TI_status(%rcx)
- jmp retint_swapgs
+ pax_exit_kernel_user
+ pax_erase_kstack
+ jmp retint_swapgs_pax
/* Either reschedule or signal or syscall exit tracking needed. */
/* First do a reschedule test. */
TRACE_IRQS_OFF
jmp int_with_check
CFI_ENDPROC
-END(system_call)
+ENDPROC(system_call)
.macro FORK_LIKE func
ENTRY(stub_\func)
DEFAULT_FRAME 0 8 /* offset 8: return address */
call sys_\func
RESTORE_TOP_OF_STACK %r11, 8
- ret $REST_SKIP /* pop extended registers */
+ pax_force_retaddr
+ ret
CFI_ENDPROC
-END(stub_\func)
+ENDPROC(stub_\func)
.endm
.macro FIXED_FRAME label,func
FIXUP_TOP_OF_STACK %r11, 8-ARGOFFSET
call \func
RESTORE_TOP_OF_STACK %r11, 8-ARGOFFSET
+ pax_force_retaddr
ret
CFI_ENDPROC
-END(\label)
+ENDPROC(\label)
.endm
FORK_LIKE clone
FORK_LIKE vfork
FIXED_FRAME stub_iopl, sys_iopl
-ENTRY(ptregscall_common)
- DEFAULT_FRAME 1 8 /* offset 8: return address */
- RESTORE_TOP_OF_STACK %r11, 8
- movq_cfi_restore R15+8, r15
- movq_cfi_restore R14+8, r14
- movq_cfi_restore R13+8, r13
- movq_cfi_restore R12+8, r12
- movq_cfi_restore RBP+8, rbp
- movq_cfi_restore RBX+8, rbx
- ret $REST_SKIP /* pop extended registers */
- CFI_ENDPROC
-END(ptregscall_common)
-
ENTRY(stub_execve)
CFI_STARTPROC
addq $8, %rsp
RESTORE_REST
jmp int_ret_from_sys_call
CFI_ENDPROC
-END(stub_execve)
+ENDPROC(stub_execve)
ENTRY(stub_execveat)
CFI_STARTPROC
RESTORE_REST
jmp int_ret_from_sys_call
CFI_ENDPROC
-END(stub_execveat)
+ENDPROC(stub_execveat)
/*
* sigreturn is special because it needs to restore all registers on return.
RESTORE_REST
jmp int_ret_from_sys_call
CFI_ENDPROC
-END(stub_rt_sigreturn)
+ENDPROC(stub_rt_sigreturn)
#ifdef CONFIG_X86_X32_ABI
ENTRY(stub_x32_rt_sigreturn)
RESTORE_REST
jmp int_ret_from_sys_call
CFI_ENDPROC
-END(stub_x32_rt_sigreturn)
+ENDPROC(stub_x32_rt_sigreturn)
ENTRY(stub_x32_execve)
CFI_STARTPROC
2: jmp common_interrupt
.endr
CFI_ENDPROC
-END(irq_entries_start)
+ENDPROC(irq_entries_start)
.previous
END(interrupt)
/* 0(%rsp): ~(interrupt number) */
.macro interrupt func
/* reserve pt_regs for scratch regs and rbp */
- subq $ORIG_RAX-RBP, %rsp
- CFI_ADJUST_CFA_OFFSET ORIG_RAX-RBP
+ subq $ORIG_RAX, %rsp
+ CFI_ADJUST_CFA_OFFSET ORIG_RAX
SAVE_ARGS_IRQ
call \func
.endm
/* Restore saved previous stack */
popq %rsi
- CFI_DEF_CFA rsi,SS+8-RBP /* reg/off reset after def_cfa_expr */
- leaq ARGOFFSET-RBP(%rsi), %rsp
+ CFI_DEF_CFA rsi,SS+8 /* reg/off reset after def_cfa_expr */
+ movq %rsi, %rsp
CFI_DEF_CFA_REGISTER rsp
- CFI_ADJUST_CFA_OFFSET RBP-ARGOFFSET
+ CFI_ADJUST_CFA_OFFSET -ARGOFFSET
exit_intr:
GET_THREAD_INFO(%rcx)
- testl $3,CS-ARGOFFSET(%rsp)
+ testb $3,CS-ARGOFFSET(%rsp)
je retint_kernel
/* Interrupt came from user space */
* The iretq could re-enable interrupts:
*/
DISABLE_INTERRUPTS(CLBR_ANY)
+ pax_exit_kernel_user
+retint_swapgs_pax:
TRACE_IRQS_IRETQ
SWAPGS
jmp restore_args
retint_restore_args: /* return to kernel space */
DISABLE_INTERRUPTS(CLBR_ANY)
+ pax_exit_kernel
+
+#if defined(CONFIG_EFI) && defined(CONFIG_PAX_KERNEXEC)
+ /* This is a quirk to allow IRQs/NMIs/MCEs during early EFI setup,
+ * namely calling EFI runtime services with a phys mapping. We're
+ * starting off with NOPs and patch in the real instrumentation
+ * (BTS/OR) before starting any userland process; even before starting
+ * up the APs.
+ */
+ .pushsection .altinstr_replacement, "a"
+ 601: pax_force_retaddr (RIP-ARGOFFSET)
+ 602:
+ .popsection
+ 603: .fill 602b-601b, 1, 0x90
+ .pushsection .altinstructions, "a"
+ altinstruction_entry 603b, 601b, X86_FEATURE_ALWAYS, 602b-601b, 602b-601b
+ .popsection
+#else
+ pax_force_retaddr (RIP-ARGOFFSET)
+#endif
+
/*
* The iretq could re-enable interrupts:
*/
SWAPGS
movq PER_CPU_VAR(espfix_waddr),%rdi
movq %rax,(0*8)(%rdi) /* RAX */
- movq (2*8)(%rsp),%rax /* RIP */
+ movq (2*8 + RIP-RIP)(%rsp),%rax /* RIP */
movq %rax,(1*8)(%rdi)
- movq (3*8)(%rsp),%rax /* CS */
+ movq (2*8 + CS-RIP)(%rsp),%rax /* CS */
movq %rax,(2*8)(%rdi)
- movq (4*8)(%rsp),%rax /* RFLAGS */
+ movq (2*8 + EFLAGS-RIP)(%rsp),%rax /* RFLAGS */
movq %rax,(3*8)(%rdi)
- movq (6*8)(%rsp),%rax /* SS */
+ movq (2*8 + SS-RIP)(%rsp),%rax /* SS */
movq %rax,(5*8)(%rdi)
- movq (5*8)(%rsp),%rax /* RSP */
+ movq (2*8 + RSP-RIP)(%rsp),%rax /* RSP */
movq %rax,(4*8)(%rdi)
andl $0xffff0000,%eax
popq_cfi %rdi
jmp exit_intr
#endif
CFI_ENDPROC
-END(common_interrupt)
+ENDPROC(common_interrupt)
/*
* APIC interrupts.
interrupt \do_sym
jmp ret_from_intr
CFI_ENDPROC
-END(\sym)
+ENDPROC(\sym)
.endm
#ifdef CONFIG_TRACING
/*
* Exception entry points.
*/
-#define INIT_TSS_IST(x) PER_CPU_VAR(init_tss) + (TSS_ist + ((x) - 1) * 8)
+#define INIT_TSS_IST(x) (TSS_ist + ((x) - 1) * 8)(%r13)
.macro idtentry sym do_sym has_error_code:req paranoid=0 shift_ist=-1
ENTRY(\sym)
.endif
.if \shift_ist != -1
+#ifdef CONFIG_SMP
+ imul $TSS_size, PER_CPU_VAR(cpu_number), %r13d
+ lea init_tss(%r13), %r13
+#else
+ lea init_tss(%rip), %r13
+#endif
subq $EXCEPTION_STKSZ, INIT_TSS_IST(\shift_ist)
.endif
.endif
CFI_ENDPROC
-END(\sym)
+ENDPROC(\sym)
.endm
#ifdef CONFIG_TRACING
2: mfence /* workaround */
SWAPGS
popfq_cfi
+ pax_force_retaddr
ret
CFI_ENDPROC
-END(native_load_gs_index)
+ENDPROC(native_load_gs_index)
_ASM_EXTABLE(gs_change,bad_gs)
.section .fixup,"ax"
CFI_DEF_CFA_REGISTER rsp
CFI_ADJUST_CFA_OFFSET -8
decl PER_CPU_VAR(irq_count)
+ pax_force_retaddr
ret
CFI_ENDPROC
-END(do_softirq_own_stack)
+ENDPROC(do_softirq_own_stack)
#ifdef CONFIG_XEN
idtentry xen_hypervisor_callback xen_do_hypervisor_callback has_error_code=0
decl PER_CPU_VAR(irq_count)
jmp error_exit
CFI_ENDPROC
-END(xen_do_hypervisor_callback)
+ENDPROC(xen_do_hypervisor_callback)
/*
* Hypervisor uses this for application faults while it executes.
SAVE_ALL
jmp error_exit
CFI_ENDPROC
-END(xen_failsafe_callback)
+ENDPROC(xen_failsafe_callback)
apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
xen_hvm_callback_vector xen_evtchn_do_upcall
DEFAULT_FRAME
DISABLE_INTERRUPTS(CLBR_NONE)
TRACE_IRQS_OFF_DEBUG
- testl %ebx,%ebx /* swapgs needed? */
+ testl $1,%ebx /* swapgs needed? */
jnz paranoid_restore
- testl $3,CS(%rsp)
+ testb $3,CS(%rsp)
jnz paranoid_userspace
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ pax_exit_kernel
+ TRACE_IRQS_IRETQ 0
+ SWAPGS_UNSAFE_STACK
+ RESTORE_ALL 8
+ pax_force_retaddr_bts
+ jmp irq_return
+#endif
paranoid_swapgs:
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ pax_exit_kernel_user
+#else
+ pax_exit_kernel
+#endif
TRACE_IRQS_IRETQ 0
SWAPGS_UNSAFE_STACK
RESTORE_ALL 8
jmp irq_return
paranoid_restore:
+ pax_exit_kernel
TRACE_IRQS_IRETQ_DEBUG 0
RESTORE_ALL 8
+ pax_force_retaddr_bts
jmp irq_return
paranoid_userspace:
GET_THREAD_INFO(%rcx)
TRACE_IRQS_OFF
jmp paranoid_userspace
CFI_ENDPROC
-END(paranoid_exit)
+ENDPROC(paranoid_exit)
/*
* Exception entry point. This expects an error code/orig_rax on the stack.
movq %r14, R14+8(%rsp)
movq %r15, R15+8(%rsp)
xorl %ebx,%ebx
- testl $3,CS+8(%rsp)
+ testb $3,CS+8(%rsp)
je error_kernelspace
error_swapgs:
SWAPGS
error_sti:
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ testb $3, CS+8(%rsp)
+ jnz 1f
+ pax_enter_kernel
+ jmp 2f
+1: pax_enter_kernel_user
+2:
+#else
+ pax_enter_kernel
+#endif
TRACE_IRQS_OFF
+ pax_force_retaddr
ret
/*
decl %ebx /* Return to usergs */
jmp error_sti
CFI_ENDPROC
-END(error_entry)
+ENDPROC(error_entry)
/* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
DISABLE_INTERRUPTS(CLBR_NONE)
TRACE_IRQS_OFF
GET_THREAD_INFO(%rcx)
- testl %eax,%eax
+ testl $1,%eax
jne retint_kernel
LOCKDEP_SYS_EXIT_IRQ
movl TI_flags(%rcx),%edx
jnz retint_careful
jmp retint_swapgs
CFI_ENDPROC
-END(error_exit)
+ENDPROC(error_exit)
/*
* Test if a given stack is an NMI stack or not.
* If %cs was not the kernel segment, then the NMI triggered in user
* space, which means it is definitely not nested.
*/
+ cmpl $__KERNEXEC_KERNEL_CS, 16(%rsp)
+ je 1f
cmpl $__KERNEL_CS, 16(%rsp)
jne first_nmi
-
+1:
/*
* Check the special variable on the stack to see if NMIs are
* executing.
1:
/* Set up the interrupted NMIs stack to jump to repeat_nmi */
- leaq -1*8(%rsp), %rdx
- movq %rdx, %rsp
+ subq $8, %rsp
CFI_ADJUST_CFA_OFFSET 1*8
leaq -10*8(%rsp), %rdx
pushq_cfi $__KERNEL_DS
CFI_RESTORE rdx
/* No need to check faults here */
+# pax_force_retaddr_bts
INTERRUPT_RETURN
CFI_RESTORE_STATE
subq $ORIG_RAX-R15, %rsp
CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
/*
- * Use save_paranoid to handle SWAPGS, but no need to use paranoid_exit
+ * Use save_paranoid_nmi to handle SWAPGS, but no need to use paranoid_exit
* as we should not be calling schedule in NMI context.
* Even with normal interrupts enabled. An NMI should not be
* setting NEED_RESCHED or anything that normal interrupts and
* exceptions might do.
*/
- call save_paranoid
+ call save_paranoid_nmi
DEFAULT_FRAME 0
/*
* NMI itself takes a page fault, the page fault that was preempted
* will read the information from the NMI page fault and not the
* origin fault. Save it off and restore it if it changes.
- * Use the r12 callee-saved register.
+ * Use the r13 callee-saved register.
*/
- movq %cr2, %r12
+ movq %cr2, %r13
/* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */
movq %rsp,%rdi
/* Did the NMI take a page fault? Restore cr2 if it did */
movq %cr2, %rcx
- cmpq %rcx, %r12
+ cmpq %rcx, %r13
je 1f
- movq %r12, %cr2
+ movq %r13, %cr2
1:
- testl %ebx,%ebx /* swapgs needed? */
+ testl $1,%ebx /* swapgs needed? */
jnz nmi_restore
nmi_swapgs:
SWAPGS_UNSAFE_STACK
nmi_restore:
+ pax_exit_kernel_nmi
/* Pop the extra iret frame at once */
RESTORE_ALL 6*8
+ testb $3, 8(%rsp)
+ jnz 1f
+ pax_force_retaddr_bts
+1:
/* Clear the NMI executing stack variable */
movq $0, 5*8(%rsp)
jmp irq_return
CFI_ENDPROC
-END(nmi)
+ENDPROC(nmi)
ENTRY(ignore_sysret)
CFI_STARTPROC
mov $-ENOSYS,%eax
sysret
CFI_ENDPROC
-END(ignore_sysret)
+ENDPROC(ignore_sysret)
#define ESPFIX_MAX_PAGES DIV_ROUND_UP(CONFIG_NR_CPUS, ESPFIX_STACKS_PER_PAGE)
static void *espfix_pages[ESPFIX_MAX_PAGES];
-static __page_aligned_bss pud_t espfix_pud_page[PTRS_PER_PUD]
- __aligned(PAGE_SIZE);
+static pud_t espfix_pud_page[PTRS_PER_PUD] __page_aligned_rodata;
static unsigned int page_random, slot_random;
void __init init_espfix_bsp(void)
{
pgd_t *pgd_p;
+ unsigned long index = pgd_index(ESPFIX_BASE_ADDR);
/* Install the espfix pud into the kernel page directory */
- pgd_p = &init_level4_pgt[pgd_index(ESPFIX_BASE_ADDR)];
+ pgd_p = &init_level4_pgt[index];
pgd_populate(&init_mm, pgd_p, (pud_t *)espfix_pud_page);
+#ifdef CONFIG_PAX_PER_CPU_PGD
+ clone_pgd_range(get_cpu_pgd(0, kernel) + index, swapper_pg_dir + index, 1);
+ clone_pgd_range(get_cpu_pgd(0, user) + index, swapper_pg_dir + index, 1);
+#endif
+
/* Randomize the locations */
init_espfix_random();
set_pte(&pte_p[n*PTE_STRIDE], pte);
/* Job is done for this CPU and any CPU which shares this page */
- ACCESS_ONCE(espfix_pages[page]) = stack_page;
+ ACCESS_ONCE_RW(espfix_pages[page]) = stack_page;
unlock_done:
mutex_unlock(&espfix_init_mutex);
* kernel identity mapping to modify code.
*/
if (within(ip, (unsigned long)_text, (unsigned long)_etext))
- ip = (unsigned long)__va(__pa_symbol(ip));
+ ip = (unsigned long)__va(__pa_symbol(ktla_ktva(ip)));
return ip;
}
{
unsigned char replaced[MCOUNT_INSN_SIZE];
+ ip = ktla_ktva(ip);
+
/*
* Note: Due to modules and __init, code can
* disappear and change, we need to protect against faulting
unsigned char old[MCOUNT_INSN_SIZE];
int ret;
- memcpy(old, (void *)ip, MCOUNT_INSN_SIZE);
+ memcpy(old, (void *)ktla_ktva(ip), MCOUNT_INSN_SIZE);
ftrace_update_func = ip;
/* Make sure the breakpoints see the ftrace_update_func update */
unsigned char replaced[MCOUNT_INSN_SIZE];
unsigned char brk = BREAKPOINT_INSTRUCTION;
- if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
+ if (probe_kernel_read(replaced, (void *)ktla_ktva(ip), MCOUNT_INSN_SIZE))
return -EFAULT;
/* Make sure it is what we expect it to be */
pgd = *pgd_p;
/*
- * The use of __START_KERNEL_map rather than __PAGE_OFFSET here is
- * critical -- __PAGE_OFFSET would point us back into the dynamic
+ * The use of __early_va rather than __va here is critical:
+ * __va would point us back into the dynamic
* range and we might end up looping forever...
*/
if (pgd)
- pud_p = (pudval_t *)((pgd & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
+ pud_p = (pudval_t *)(__early_va(pgd & PTE_PFN_MASK));
else {
if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
reset_early_page_tables();
pud_p = (pudval_t *)early_dynamic_pgts[next_early_pgt++];
for (i = 0; i < PTRS_PER_PUD; i++)
pud_p[i] = 0;
- *pgd_p = (pgdval_t)pud_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
+ *pgd_p = (pgdval_t)__pa(pud_p) + _KERNPG_TABLE;
}
pud_p += pud_index(address);
pud = *pud_p;
if (pud)
- pmd_p = (pmdval_t *)((pud & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
+ pmd_p = (pmdval_t *)(__early_va(pud & PTE_PFN_MASK));
else {
if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
reset_early_page_tables();
pmd_p = (pmdval_t *)early_dynamic_pgts[next_early_pgt++];
for (i = 0; i < PTRS_PER_PMD; i++)
pmd_p[i] = 0;
- *pud_p = (pudval_t)pmd_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
+ *pud_p = (pudval_t)__pa(pmd_p) + _KERNPG_TABLE;
}
pmd = (physaddr & PMD_MASK) + early_pmd_flags;
pmd_p[pmd_index(address)] = pmd;
if (console_loglevel >= CONSOLE_LOGLEVEL_DEBUG)
early_printk("Kernel alive\n");
- clear_page(init_level4_pgt);
/* set init_level4_pgt kernel high mapping*/
init_level4_pgt[511] = early_level4_pgt[511];
/* Physical address */
#define pa(X) ((X) - __PAGE_OFFSET)
+#ifdef CONFIG_PAX_KERNEXEC
+#define ta(X) (X)
+#else
+#define ta(X) ((X) - __PAGE_OFFSET)
+#endif
+
/*
* References to members of the new_cpu_data structure.
*/
* and small than max_low_pfn, otherwise will waste some page table entries
*/
-#if PTRS_PER_PMD > 1
-#define PAGE_TABLE_SIZE(pages) (((pages) / PTRS_PER_PMD) + PTRS_PER_PGD)
-#else
-#define PAGE_TABLE_SIZE(pages) ((pages) / PTRS_PER_PGD)
-#endif
+#define PAGE_TABLE_SIZE(pages) ((pages) / PTRS_PER_PTE)
/* Number of possible pages in the lowmem region */
LOWMEM_PAGES = (((1<<32) - __PAGE_OFFSET) >> PAGE_SHIFT)
INIT_MAP_SIZE = PAGE_TABLE_SIZE(KERNEL_PAGES) * PAGE_SIZE
RESERVE_BRK(pagetables, INIT_MAP_SIZE)
+/*
+ * Real beginning of normal "text" segment
+ */
+ENTRY(stext)
+ENTRY(_stext)
+
/*
* 32-bit kernel entrypoint; only used by the boot CPU. On entry,
* %esi points to the real-mode code as a 32-bit pointer.
* can.
*/
__HEAD
+
+#ifdef CONFIG_PAX_KERNEXEC
+ jmp startup_32
+/* PaX: fill first page in .text with int3 to catch NULL derefs in kernel mode */
+.fill PAGE_SIZE-5,1,0xcc
+#endif
+
ENTRY(startup_32)
movl pa(stack_start),%ecx
2:
leal -__PAGE_OFFSET(%ecx),%esp
+#ifdef CONFIG_SMP
+ movl $pa(cpu_gdt_table),%edi
+ movl $__per_cpu_load,%eax
+ movw %ax,GDT_ENTRY_PERCPU * 8 + 2(%edi)
+ rorl $16,%eax
+ movb %al,GDT_ENTRY_PERCPU * 8 + 4(%edi)
+ movb %ah,GDT_ENTRY_PERCPU * 8 + 7(%edi)
+ movl $__per_cpu_end - 1,%eax
+ subl $__per_cpu_start,%eax
+ movw %ax,GDT_ENTRY_PERCPU * 8 + 0(%edi)
+#endif
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ movl $NR_CPUS,%ecx
+ movl $pa(cpu_gdt_table),%edi
+1:
+ movl $((((__PAGE_OFFSET-1) & 0xf0000000) >> 12) | 0x00c09700),GDT_ENTRY_KERNEL_DS * 8 + 4(%edi)
+ movl $((((__PAGE_OFFSET-1) & 0xf0000000) >> 12) | 0x00c0fb00),GDT_ENTRY_DEFAULT_USER_CS * 8 + 4(%edi)
+ movl $((((__PAGE_OFFSET-1) & 0xf0000000) >> 12) | 0x00c0f300),GDT_ENTRY_DEFAULT_USER_DS * 8 + 4(%edi)
+ addl $PAGE_SIZE_asm,%edi
+ loop 1b
+#endif
+
+#ifdef CONFIG_PAX_KERNEXEC
+ movl $pa(boot_gdt),%edi
+ movl $__LOAD_PHYSICAL_ADDR,%eax
+ movw %ax,GDT_ENTRY_BOOT_CS * 8 + 2(%edi)
+ rorl $16,%eax
+ movb %al,GDT_ENTRY_BOOT_CS * 8 + 4(%edi)
+ movb %ah,GDT_ENTRY_BOOT_CS * 8 + 7(%edi)
+ rorl $16,%eax
+
+ ljmp $(__BOOT_CS),$1f
+1:
+
+ movl $NR_CPUS,%ecx
+ movl $pa(cpu_gdt_table),%edi
+ addl $__PAGE_OFFSET,%eax
+1:
+ movb $0xc0,GDT_ENTRY_KERNEL_CS * 8 + 6(%edi)
+ movb $0xc0,GDT_ENTRY_KERNEXEC_KERNEL_CS * 8 + 6(%edi)
+ movw %ax,GDT_ENTRY_KERNEL_CS * 8 + 2(%edi)
+ movw %ax,GDT_ENTRY_KERNEXEC_KERNEL_CS * 8 + 2(%edi)
+ rorl $16,%eax
+ movb %al,GDT_ENTRY_KERNEL_CS * 8 + 4(%edi)
+ movb %al,GDT_ENTRY_KERNEXEC_KERNEL_CS * 8 + 4(%edi)
+ movb %ah,GDT_ENTRY_KERNEL_CS * 8 + 7(%edi)
+ movb %ah,GDT_ENTRY_KERNEXEC_KERNEL_CS * 8 + 7(%edi)
+ rorl $16,%eax
+ addl $PAGE_SIZE_asm,%edi
+ loop 1b
+#endif
+
/*
* Clear BSS first so that there are no surprises...
*/
movl %eax, pa(max_pfn_mapped)
/* Do early initialization of the fixmap area */
- movl $pa(initial_pg_fixmap)+PDE_IDENT_ATTR,%eax
- movl %eax,pa(initial_pg_pmd+0x1000*KPMDS-8)
+#ifdef CONFIG_COMPAT_VDSO
+ movl $pa(initial_pg_fixmap)+PDE_IDENT_ATTR+_PAGE_USER,pa(initial_pg_pmd+0x1000*KPMDS-8)
+#else
+ movl $pa(initial_pg_fixmap)+PDE_IDENT_ATTR,pa(initial_pg_pmd+0x1000*KPMDS-8)
+#endif
#else /* Not PAE */
page_pde_offset = (__PAGE_OFFSET >> 20);
movl %eax, pa(max_pfn_mapped)
/* Do early initialization of the fixmap area */
- movl $pa(initial_pg_fixmap)+PDE_IDENT_ATTR,%eax
- movl %eax,pa(initial_page_table+0xffc)
+#ifdef CONFIG_COMPAT_VDSO
+ movl $pa(initial_pg_fixmap)+PDE_IDENT_ATTR+_PAGE_USER,pa(initial_page_table+0xffc)
+#else
+ movl $pa(initial_pg_fixmap)+PDE_IDENT_ATTR,pa(initial_page_table+0xffc)
+#endif
#endif
#ifdef CONFIG_PARAVIRT
cmpl $num_subarch_entries, %eax
jae bad_subarch
- movl pa(subarch_entries)(,%eax,4), %eax
- subl $__PAGE_OFFSET, %eax
- jmp *%eax
+ jmp *pa(subarch_entries)(,%eax,4)
bad_subarch:
WEAK(lguest_entry)
__INITDATA
subarch_entries:
- .long default_entry /* normal x86/PC */
- .long lguest_entry /* lguest hypervisor */
- .long xen_entry /* Xen hypervisor */
- .long default_entry /* Moorestown MID */
+ .long ta(default_entry) /* normal x86/PC */
+ .long ta(lguest_entry) /* lguest hypervisor */
+ .long ta(xen_entry) /* Xen hypervisor */
+ .long ta(default_entry) /* Moorestown MID */
num_subarch_entries = (. - subarch_entries) / 4
.previous
#else
movl pa(mmu_cr4_features),%eax
movl %eax,%cr4
+#ifdef CONFIG_X86_PAE
testb $X86_CR4_PAE, %al # check if PAE is enabled
jz enable_paging
/* Make changes effective */
wrmsr
+ btsl $_PAGE_BIT_NX-32,pa(__supported_pte_mask+4)
+#endif
+
enable_paging:
/*
1: movl $(__KERNEL_DS),%eax # reload all the segment registers
movl %eax,%ss # after changing gdt.
- movl $(__USER_DS),%eax # DS/ES contains default USER segment
+# movl $(__KERNEL_DS),%eax # DS/ES contains default KERNEL segment
movl %eax,%ds
movl %eax,%es
movl $(__KERNEL_PERCPU), %eax
movl %eax,%fs # set this cpu's percpu
+#ifdef CONFIG_CC_STACKPROTECTOR
movl $(__KERNEL_STACK_CANARY),%eax
+#elif defined(CONFIG_PAX_MEMORY_UDEREF)
+ movl $(__USER_DS),%eax
+#else
+ xorl %eax,%eax
+#endif
movl %eax,%gs
xorl %eax,%eax # Clear LDT
* relocation. Manually set base address in stack canary
* segment descriptor.
*/
- movl $gdt_page,%eax
+ movl $cpu_gdt_table,%eax
movl $stack_canary,%ecx
+#ifdef CONFIG_SMP
+ addl $__per_cpu_load,%ecx
+#endif
movw %cx, 8 * GDT_ENTRY_STACK_CANARY + 2(%eax)
shrl $16, %ecx
movb %cl, 8 * GDT_ENTRY_STACK_CANARY + 4(%eax)
cmpl $2,(%esp) # X86_TRAP_NMI
je is_nmi # Ignore NMI
- cmpl $2,%ss:early_recursion_flag
+ cmpl $1,%ss:early_recursion_flag
je hlt_loop
incl %ss:early_recursion_flag
pushl (20+6*4)(%esp) /* trapno */
pushl $fault_msg
call printk
-#endif
call dump_stack
+#endif
hlt_loop:
hlt
jmp hlt_loop
/* This is the default interrupt "handler" :-) */
ALIGN
ignore_int:
- cld
#ifdef CONFIG_PRINTK
+ cmpl $2,%ss:early_recursion_flag
+ je hlt_loop
+ incl %ss:early_recursion_flag
+ cld
pushl %eax
pushl %ecx
pushl %edx
movl $(__KERNEL_DS),%eax
movl %eax,%ds
movl %eax,%es
- cmpl $2,early_recursion_flag
- je hlt_loop
- incl early_recursion_flag
pushl 16(%esp)
pushl 24(%esp)
pushl 32(%esp)
/*
* BSS section
*/
-__PAGE_ALIGNED_BSS
- .align PAGE_SIZE
#ifdef CONFIG_X86_PAE
+.section .initial_pg_pmd,"a",@progbits
initial_pg_pmd:
.fill 1024*KPMDS,4,0
#else
+.section .initial_page_table,"a",@progbits
ENTRY(initial_page_table)
.fill 1024,4,0
#endif
+.section .initial_pg_fixmap,"a",@progbits
initial_pg_fixmap:
.fill 1024,4,0
+.section .empty_zero_page,"a",@progbits
ENTRY(empty_zero_page)
.fill 4096,1,0
+.section .swapper_pg_dir,"a",@progbits
ENTRY(swapper_pg_dir)
+#ifdef CONFIG_X86_PAE
+ .fill 4,8,0
+#else
.fill 1024,4,0
+#endif
/*
* This starts the data section.
*/
#ifdef CONFIG_X86_PAE
-__PAGE_ALIGNED_DATA
- /* Page-aligned for the benefit of paravirt? */
- .align PAGE_SIZE
+.section .initial_page_table,"a",@progbits
ENTRY(initial_page_table)
.long pa(initial_pg_pmd+PGD_IDENT_ATTR),0 /* low identity map */
# if KPMDS == 3
# error "Kernel PMDs should be 1, 2 or 3"
# endif
.align PAGE_SIZE /* needs to be page-sized too */
+
+#ifdef CONFIG_PAX_PER_CPU_PGD
+ENTRY(cpu_pgd)
+ .rept 2*NR_CPUS
+ .fill 4,8,0
+ .endr
+#endif
+
#endif
.data
.balign 4
ENTRY(stack_start)
- .long init_thread_union+THREAD_SIZE
+ .long init_thread_union+THREAD_SIZE-8
__INITRODATA
int_msg:
* segment size, and 32-bit linear address value:
*/
- .data
+.section .rodata,"a",@progbits
.globl boot_gdt_descr
.globl idt_descr
.word 0 # 32 bit align gdt_desc.address
boot_gdt_descr:
.word __BOOT_DS+7
- .long boot_gdt - __PAGE_OFFSET
+ .long pa(boot_gdt)
.word 0 # 32-bit align idt_desc.address
idt_descr:
.word 0 # 32 bit align gdt_desc.address
ENTRY(early_gdt_descr)
.word GDT_ENTRIES*8-1
- .long gdt_page /* Overwritten for secondary CPUs */
+ .long cpu_gdt_table /* Overwritten for secondary CPUs */
/*
* The boot_gdt must mirror the equivalent in setup.S and is
.align L1_CACHE_BYTES
ENTRY(boot_gdt)
.fill GDT_ENTRY_BOOT_CS,8,0
- .quad 0x00cf9a000000ffff /* kernel 4GB code at 0x00000000 */
- .quad 0x00cf92000000ffff /* kernel 4GB data at 0x00000000 */
+ .quad 0x00cf9b000000ffff /* kernel 4GB code at 0x00000000 */
+ .quad 0x00cf93000000ffff /* kernel 4GB data at 0x00000000 */
+
+ .align PAGE_SIZE_asm
+ENTRY(cpu_gdt_table)
+ .rept NR_CPUS
+ .quad 0x0000000000000000 /* NULL descriptor */
+ .quad 0x0000000000000000 /* 0x0b reserved */
+ .quad 0x0000000000000000 /* 0x13 reserved */
+ .quad 0x0000000000000000 /* 0x1b reserved */
+
+#ifdef CONFIG_PAX_KERNEXEC
+ .quad 0x00cf9b000000ffff /* 0x20 alternate kernel 4GB code at 0x00000000 */
+#else
+ .quad 0x0000000000000000 /* 0x20 unused */
+#endif
+
+ .quad 0x0000000000000000 /* 0x28 unused */
+ .quad 0x0000000000000000 /* 0x33 TLS entry 1 */
+ .quad 0x0000000000000000 /* 0x3b TLS entry 2 */
+ .quad 0x0000000000000000 /* 0x43 TLS entry 3 */
+ .quad 0x0000000000000000 /* 0x4b reserved */
+ .quad 0x0000000000000000 /* 0x53 reserved */
+ .quad 0x0000000000000000 /* 0x5b reserved */
+
+ .quad 0x00cf9b000000ffff /* 0x60 kernel 4GB code at 0x00000000 */
+ .quad 0x00cf93000000ffff /* 0x68 kernel 4GB data at 0x00000000 */
+ .quad 0x00cffb000000ffff /* 0x73 user 4GB code at 0x00000000 */
+ .quad 0x00cff3000000ffff /* 0x7b user 4GB data at 0x00000000 */
+
+ .quad 0x0000000000000000 /* 0x80 TSS descriptor */
+ .quad 0x0000000000000000 /* 0x88 LDT descriptor */
+
+ /*
+ * Segments used for calling PnP BIOS have byte granularity.
+ * The code segments and data segments have fixed 64k limits,
+ * the transfer segment sizes are set at run time.
+ */
+ .quad 0x00409b000000ffff /* 0x90 32-bit code */
+ .quad 0x00009b000000ffff /* 0x98 16-bit code */
+ .quad 0x000093000000ffff /* 0xa0 16-bit data */
+ .quad 0x0000930000000000 /* 0xa8 16-bit data */
+ .quad 0x0000930000000000 /* 0xb0 16-bit data */
+
+ /*
+ * The APM segments have byte granularity and their bases
+ * are set at run time. All have 64k limits.
+ */
+ .quad 0x00409b000000ffff /* 0xb8 APM CS code */
+ .quad 0x00009b000000ffff /* 0xc0 APM CS 16 code (16 bit) */
+ .quad 0x004093000000ffff /* 0xc8 APM DS data */
+
+ .quad 0x00c093000000ffff /* 0xd0 - ESPFIX SS */
+ .quad 0x0040930000000000 /* 0xd8 - PERCPU */
+ .quad 0x0040910000000017 /* 0xe0 - STACK_CANARY */
+ .quad 0x0000000000000000 /* 0xe8 - PCIBIOS_CS */
+ .quad 0x0000000000000000 /* 0xf0 - PCIBIOS_DS */
+ .quad 0x0000000000000000 /* 0xf8 - GDT entry 31: double-fault TSS */
+
+ /* Be sure this is zeroed to avoid false validations in Xen */
+ .fill PAGE_SIZE_asm - GDT_SIZE,1,0
+ .endr
#include <asm/processor-flags.h>
#include <asm/percpu.h>
#include <asm/nops.h>
+#include <asm/cpufeature.h>
+#include <asm/alternative-asm.h>
#ifdef CONFIG_PARAVIRT
#include <asm/asm-offsets.h>
L3_PAGE_OFFSET = pud_index(__PAGE_OFFSET)
L4_START_KERNEL = pgd_index(__START_KERNEL_map)
L3_START_KERNEL = pud_index(__START_KERNEL_map)
+L4_VMALLOC_START = pgd_index(VMALLOC_START)
+L3_VMALLOC_START = pud_index(VMALLOC_START)
+L4_VMALLOC_END = pgd_index(VMALLOC_END)
+L3_VMALLOC_END = pud_index(VMALLOC_END)
+L4_VMEMMAP_START = pgd_index(VMEMMAP_START)
+L3_VMEMMAP_START = pud_index(VMEMMAP_START)
.text
__HEAD
* Fixup the physical addresses in the page table
*/
addq %rbp, early_level4_pgt + (L4_START_KERNEL*8)(%rip)
+ addq %rbp, init_level4_pgt + (L4_PAGE_OFFSET*8)(%rip)
+ addq %rbp, init_level4_pgt + (L4_VMALLOC_START*8)(%rip)
+ addq %rbp, init_level4_pgt + (L4_VMALLOC_END*8)(%rip)
+ addq %rbp, init_level4_pgt + (L4_VMEMMAP_START*8)(%rip)
+ addq %rbp, init_level4_pgt + (L4_START_KERNEL*8)(%rip)
- addq %rbp, level3_kernel_pgt + (510*8)(%rip)
- addq %rbp, level3_kernel_pgt + (511*8)(%rip)
+ addq %rbp, level3_ident_pgt + (0*8)(%rip)
+#ifndef CONFIG_XEN
+ addq %rbp, level3_ident_pgt + (1*8)(%rip)
+#endif
+
+ addq %rbp, level3_vmemmap_pgt + (L3_VMEMMAP_START*8)(%rip)
+
+ addq %rbp, level3_kernel_pgt + (L3_START_KERNEL*8)(%rip)
+ addq %rbp, level3_kernel_pgt + ((L3_START_KERNEL+1)*8)(%rip)
addq %rbp, level2_fixmap_pgt + (506*8)(%rip)
+ addq %rbp, level2_fixmap_pgt + (507*8)(%rip)
/*
* Set up the identity mapping for the switchover. These
* after the boot processor executes this code.
*/
+ orq $-1, %rbp
movq $(init_level4_pgt - __START_KERNEL_map), %rax
1:
- /* Enable PAE mode and PGE */
- movl $(X86_CR4_PAE | X86_CR4_PGE), %ecx
+ /* Enable PAE mode and PSE/PGE */
+ movl $(X86_CR4_PSE | X86_CR4_PAE | X86_CR4_PGE), %ecx
movq %rcx, %cr4
/* Setup early boot stage 4 level pagetables. */
movl $MSR_EFER, %ecx
rdmsr
btsl $_EFER_SCE, %eax /* Enable System Call */
- btl $20,%edi /* No Execute supported? */
+ btl $(X86_FEATURE_NX & 31),%edi /* No Execute supported? */
jnc 1f
btsl $_EFER_NX, %eax
+ cmpq $-1, %rbp
+ je 1f
btsq $_PAGE_BIT_NX,early_pmd_flags(%rip)
+ btsq $_PAGE_BIT_NX, init_level4_pgt + 8*L4_PAGE_OFFSET(%rip)
+ btsq $_PAGE_BIT_NX, init_level4_pgt + 8*L4_VMALLOC_START(%rip)
+ btsq $_PAGE_BIT_NX, init_level4_pgt + 8*L4_VMALLOC_END(%rip)
+ btsq $_PAGE_BIT_NX, init_level4_pgt + 8*L4_VMEMMAP_START(%rip)
+ btsq $_PAGE_BIT_NX, level2_fixmap_pgt + 8*506(%rip)
+ btsq $_PAGE_BIT_NX, level2_fixmap_pgt + 8*507(%rip)
+ btsq $_PAGE_BIT_NX, __supported_pte_mask(%rip)
1: wrmsr /* Make changes effective */
/* Setup cr0 */
* REX.W + FF /5 JMP m16:64 Jump far, absolute indirect,
* address given in m16:64.
*/
+ pax_set_fptr_mask
movq initial_code(%rip),%rax
pushq $0 # fake return address to stop unwinder
pushq $__KERNEL_CS # set correct cs
.quad INIT_PER_CPU_VAR(irq_stack_union)
GLOBAL(stack_start)
- .quad init_thread_union+THREAD_SIZE-8
+ .quad init_thread_union+THREAD_SIZE-16
.word 0
__FINITDATA
call dump_stack
#ifdef CONFIG_KALLSYMS
leaq early_idt_ripmsg(%rip),%rdi
- movq 40(%rsp),%rsi # %rip again
+ movq 88(%rsp),%rsi # %rip again
call __print_symbol
#endif
#endif /* EARLY_PRINTK */
early_recursion_flag:
.long 0
+ .section .rodata,"a",@progbits
#ifdef CONFIG_EARLY_PRINTK
early_idt_msg:
.asciz "PANIC: early exception %02lx rip %lx:%lx error %lx cr2 %lx\n"
NEXT_PAGE(early_dynamic_pgts)
.fill 512*EARLY_DYNAMIC_PAGE_TABLES,8,0
- .data
+ .section .rodata,"a",@progbits
-#ifndef CONFIG_XEN
NEXT_PAGE(init_level4_pgt)
- .fill 512,8,0
-#else
-NEXT_PAGE(init_level4_pgt)
- .quad level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE
.org init_level4_pgt + L4_PAGE_OFFSET*8, 0
.quad level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE
+ .org init_level4_pgt + L4_VMALLOC_START*8, 0
+ .quad level3_vmalloc_start_pgt - __START_KERNEL_map + _KERNPG_TABLE
+ .org init_level4_pgt + L4_VMALLOC_END*8, 0
+ .quad level3_vmalloc_end_pgt - __START_KERNEL_map + _KERNPG_TABLE
+ .org init_level4_pgt + L4_VMEMMAP_START*8, 0
+ .quad level3_vmemmap_pgt - __START_KERNEL_map + _KERNPG_TABLE
.org init_level4_pgt + L4_START_KERNEL*8, 0
/* (2^48-(2*1024*1024*1024))/(2^39) = 511 */
.quad level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE
+#ifdef CONFIG_PAX_PER_CPU_PGD
+NEXT_PAGE(cpu_pgd)
+ .rept 2*NR_CPUS
+ .fill 512,8,0
+ .endr
+#endif
+
NEXT_PAGE(level3_ident_pgt)
.quad level2_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE
+#ifdef CONFIG_XEN
.fill 511, 8, 0
+#else
+ .quad level2_ident_pgt + PAGE_SIZE - __START_KERNEL_map + _KERNPG_TABLE
+ .fill 510,8,0
+#endif
+
+NEXT_PAGE(level3_vmalloc_start_pgt)
+ .fill 512,8,0
+
+NEXT_PAGE(level3_vmalloc_end_pgt)
+ .fill 512,8,0
+
+NEXT_PAGE(level3_vmemmap_pgt)
+ .fill L3_VMEMMAP_START,8,0
+ .quad level2_vmemmap_pgt - __START_KERNEL_map + _KERNPG_TABLE
+
NEXT_PAGE(level2_ident_pgt)
- /* Since I easily can, map the first 1G.
+ /* Since I easily can, map the first 2G.
* Don't set NX because code runs from these pages.
*/
- PMDS(0, __PAGE_KERNEL_IDENT_LARGE_EXEC, PTRS_PER_PMD)
-#endif
+ PMDS(0, __PAGE_KERNEL_IDENT_LARGE_EXEC, 2*PTRS_PER_PMD)
NEXT_PAGE(level3_kernel_pgt)
.fill L3_START_KERNEL,8,0
.quad level2_kernel_pgt - __START_KERNEL_map + _KERNPG_TABLE
.quad level2_fixmap_pgt - __START_KERNEL_map + _PAGE_TABLE
+NEXT_PAGE(level2_vmemmap_pgt)
+ .fill 512,8,0
+
NEXT_PAGE(level2_kernel_pgt)
/*
* 512 MB kernel mapping. We spend a full page on this pagetable
NEXT_PAGE(level2_fixmap_pgt)
.fill 506,8,0
.quad level1_fixmap_pgt - __START_KERNEL_map + _PAGE_TABLE
- /* 8MB reserved for vsyscalls + a 2MB hole = 4 + 1 entries */
- .fill 5,8,0
+ .quad level1_vsyscall_pgt - __START_KERNEL_map + _PAGE_TABLE
+ /* 6MB reserved for vsyscalls + a 2MB hole = 3 + 1 entries */
+ .fill 4,8,0
NEXT_PAGE(level1_fixmap_pgt)
.fill 512,8,0
+NEXT_PAGE(level1_vsyscall_pgt)
+ .fill 512,8,0
+
#undef PMDS
- .data
+ .align PAGE_SIZE
+ENTRY(cpu_gdt_table)
+ .rept NR_CPUS
+ .quad 0x0000000000000000 /* NULL descriptor */
+ .quad 0x00cf9b000000ffff /* __KERNEL32_CS */
+ .quad 0x00af9b000000ffff /* __KERNEL_CS */
+ .quad 0x00cf93000000ffff /* __KERNEL_DS */
+ .quad 0x00cffb000000ffff /* __USER32_CS */
+ .quad 0x00cff3000000ffff /* __USER_DS, __USER32_DS */
+ .quad 0x00affb000000ffff /* __USER_CS */
+
+#ifdef CONFIG_PAX_KERNEXEC
+ .quad 0x00af9b000000ffff /* __KERNEXEC_KERNEL_CS */
+#else
+ .quad 0x0 /* unused */
+#endif
+
+ .quad 0,0 /* TSS */
+ .quad 0,0 /* LDT */
+ .quad 0,0,0 /* three TLS descriptors */
+ .quad 0x0000f40000000000 /* node/CPU stored in limit */
+ /* asm/segment.h:GDT_ENTRIES must match this */
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ .quad 0x00cf93000000ffff /* __UDEREF_KERNEL_DS */
+#else
+ .quad 0x0 /* unused */
+#endif
+
+ /* zero the remaining page */
+ .fill PAGE_SIZE / 8 - GDT_ENTRIES,8,0
+ .endr
+
.align 16
.globl early_gdt_descr
early_gdt_descr:
.word GDT_ENTRIES*8-1
early_gdt_descr_base:
- .quad INIT_PER_CPU_VAR(gdt_page)
+ .quad cpu_gdt_table
ENTRY(phys_base)
/* This must match the first entry in level2_kernel_pgt */
.quad 0x0000000000000000
#include "../../x86/xen/xen-head.S"
-
- __PAGE_ALIGNED_BSS
+
+ .section .rodata,"a",@progbits
NEXT_PAGE(empty_zero_page)
.skip PAGE_SIZE
EXPORT_SYMBOL(cmpxchg8b_emu);
#endif
+EXPORT_SYMBOL_GPL(cpu_gdt_table);
+
/* Networking helper routines. */
EXPORT_SYMBOL(csum_partial_copy_generic);
+EXPORT_SYMBOL(csum_partial_copy_generic_to_user);
+EXPORT_SYMBOL(csum_partial_copy_generic_from_user);
EXPORT_SYMBOL(__get_user_1);
EXPORT_SYMBOL(__get_user_2);
EXPORT_SYMBOL(___preempt_schedule_context);
#endif
#endif
+
+#ifdef CONFIG_PAX_KERNEXEC
+EXPORT_SYMBOL(__LOAD_PHYSICAL_ADDR);
+#endif
+
+#ifdef CONFIG_PAX_PER_CPU_PGD
+EXPORT_SYMBOL(cpu_pgd);
+#endif
static inline bool interrupted_user_mode(void)
{
struct pt_regs *regs = get_irq_regs();
- return regs && user_mode_vm(regs);
+ return regs && user_mode(regs);
}
/*
static void make_8259A_irq(unsigned int irq)
{
disable_irq_nosync(irq);
- io_apic_irqs &= ~(1<<irq);
+ io_apic_irqs &= ~(1UL<<irq);
irq_set_chip_and_handler(irq, &i8259A_chip, handle_level_irq);
enable_irq(irq);
}
"spurious 8259A interrupt: IRQ%d.\n", irq);
spurious_irq_mask |= irqmask;
}
- atomic_inc(&irq_err_count);
+ atomic_inc_unchecked(&irq_err_count);
/*
* Theoretically we do not have to handle this IRQ,
* but in Linux this does not cause problems and is
/* (slave's support for AEOI in flat mode is to be investigated) */
outb_pic(SLAVE_ICW4_DEFAULT, PIC_SLAVE_IMR);
+ pax_open_kernel();
if (auto_eoi)
/*
* In AEOI mode we just have to mask the interrupt
* when acking.
*/
- i8259A_chip.irq_mask_ack = disable_8259A_irq;
+ *(void **)&i8259A_chip.irq_mask_ack = disable_8259A_irq;
else
- i8259A_chip.irq_mask_ack = mask_and_ack_8259A;
+ *(void **)&i8259A_chip.irq_mask_ack = mask_and_ack_8259A;
+ pax_close_kernel();
udelay(100); /* wait for 8259A to initialize */
* Quirk table for systems that misbehave (lock up, etc.) if port
* 0x80 is used:
*/
-static struct dmi_system_id __initdata io_delay_0xed_port_dmi_table[] = {
+static const struct dmi_system_id __initconst io_delay_0xed_port_dmi_table[] = {
{
.callback = dmi_io_delay_0xed_port,
.ident = "Compaq Presario V6000",
#include <linux/sched.h>
#include <linux/kernel.h>
#include <linux/capability.h>
+#include <linux/security.h>
#include <linux/errno.h>
#include <linux/types.h>
#include <linux/ioport.h>
return -EINVAL;
if (turn_on && !capable(CAP_SYS_RAWIO))
return -EPERM;
+#ifdef CONFIG_GRKERNSEC_IO
+ if (turn_on && grsec_disable_privio) {
+ gr_handle_ioperm();
+ return -ENODEV;
+ }
+#endif
/*
* If it's the first ioperm() call in this thread's lifetime, set the
* because the ->io_bitmap_max value must match the bitmap
* contents:
*/
- tss = &per_cpu(init_tss, get_cpu());
+ tss = init_tss + get_cpu();
if (turn_on)
bitmap_clear(t->io_bitmap_ptr, from, num);
if (level > old) {
if (!capable(CAP_SYS_RAWIO))
return -EPERM;
+#ifdef CONFIG_GRKERNSEC_IO
+ if (grsec_disable_privio) {
+ gr_handle_iopl();
+ return -ENODEV;
+ }
+#endif
}
regs->flags = (regs->flags & ~X86_EFLAGS_IOPL) | (level << 12);
t->iopl = level << 12;
#define CREATE_TRACE_POINTS
#include <asm/trace/irq_vectors.h>
-atomic_t irq_err_count;
+atomic_unchecked_t irq_err_count;
/* Function pointer for generic interrupt vector handling */
void (*x86_platform_ipi_callback)(void) = NULL;
seq_printf(p, "%10u ", irq_stats(j)->irq_hv_callback_count);
seq_puts(p, " Hypervisor callback interrupts\n");
#endif
- seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read(&irq_err_count));
+ seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read_unchecked(&irq_err_count));
#if defined(CONFIG_X86_IO_APIC)
- seq_printf(p, "%*s: %10u\n", prec, "MIS", atomic_read(&irq_mis_count));
+ seq_printf(p, "%*s: %10u\n", prec, "MIS", atomic_read_unchecked(&irq_mis_count));
#endif
return 0;
}
u64 arch_irq_stat(void)
{
- u64 sum = atomic_read(&irq_err_count);
+ u64 sum = atomic_read_unchecked(&irq_err_count);
return sum;
}
#ifdef CONFIG_DEBUG_STACKOVERFLOW
+extern void gr_handle_kernel_exploit(void);
+
int sysctl_panic_on_stackoverflow __read_mostly;
/* Debugging check for stack overflow: is there less than 1KB free? */
__asm__ __volatile__("andl %%esp,%0" :
"=r" (sp) : "0" (THREAD_SIZE - 1));
- return sp < (sizeof(struct thread_info) + STACK_WARN);
+ return sp < STACK_WARN;
}
static void print_stack_overflow(void)
{
printk(KERN_WARNING "low stack detected by irq handler\n");
dump_stack();
+ gr_handle_kernel_exploit();
if (sysctl_panic_on_stackoverflow)
panic("low stack detected by irq handler - check messages\n");
}
static inline int
execute_on_irq_stack(int overflow, struct irq_desc *desc, int irq)
{
- struct irq_stack *curstk, *irqstk;
+ struct irq_stack *irqstk;
u32 *isp, *prev_esp, arg1, arg2;
- curstk = (struct irq_stack *) current_stack();
irqstk = __this_cpu_read(hardirq_stack);
/*
* handler) we can't do that and just have to keep using the
* current stack (which is the irq stack already after all)
*/
- if (unlikely(curstk == irqstk))
+ if (unlikely((void *)current_stack_pointer - (void *)irqstk < THREAD_SIZE))
return 0;
- isp = (u32 *) ((char *)irqstk + sizeof(*irqstk));
+ isp = (u32 *) ((char *)irqstk + sizeof(*irqstk) - 8);
/* Save the next esp at the bottom of the stack */
prev_esp = (u32 *)irqstk;
*prev_esp = current_stack_pointer;
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ __set_fs(MAKE_MM_SEG(0));
+#endif
+
if (unlikely(overflow))
call_on_stack(print_stack_overflow, isp);
: "0" (irq), "1" (desc), "2" (isp),
"D" (desc->handle_irq)
: "memory", "cc", "ecx");
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ __set_fs(current_thread_info()->addr_limit);
+#endif
+
return 1;
}
*/
void irq_ctx_init(int cpu)
{
- struct irq_stack *irqstk;
-
if (per_cpu(hardirq_stack, cpu))
return;
- irqstk = page_address(alloc_pages_node(cpu_to_node(cpu),
- THREADINFO_GFP,
- THREAD_SIZE_ORDER));
- per_cpu(hardirq_stack, cpu) = irqstk;
-
- irqstk = page_address(alloc_pages_node(cpu_to_node(cpu),
- THREADINFO_GFP,
- THREAD_SIZE_ORDER));
- per_cpu(softirq_stack, cpu) = irqstk;
-
- printk(KERN_DEBUG "CPU %u irqstacks, hard=%p soft=%p\n",
- cpu, per_cpu(hardirq_stack, cpu), per_cpu(softirq_stack, cpu));
+ per_cpu(hardirq_stack, cpu) = page_address(alloc_pages_node(cpu_to_node(cpu), THREADINFO_GFP, THREAD_SIZE_ORDER));
+ per_cpu(softirq_stack, cpu) = page_address(alloc_pages_node(cpu_to_node(cpu), THREADINFO_GFP, THREAD_SIZE_ORDER));
}
void do_softirq_own_stack(void)
{
- struct thread_info *curstk;
struct irq_stack *irqstk;
u32 *isp, *prev_esp;
- curstk = current_stack();
irqstk = __this_cpu_read(softirq_stack);
/* build the stack frame on the softirq stack */
prev_esp = (u32 *)irqstk;
*prev_esp = current_stack_pointer;
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ __set_fs(MAKE_MM_SEG(0));
+#endif
+
call_on_stack(__do_softirq, isp);
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ __set_fs(current_thread_info()->addr_limit);
+#endif
+
}
bool handle_irq(unsigned irq, struct pt_regs *regs)
if (unlikely(!desc))
return false;
- if (user_mode_vm(regs) || !execute_on_irq_stack(overflow, desc, irq)) {
+ if (user_mode(regs) || !execute_on_irq_stack(overflow, desc, irq)) {
if (unlikely(overflow))
print_stack_overflow();
desc->handle_irq(irq, desc);
DEFINE_PER_CPU(struct pt_regs *, irq_regs);
EXPORT_PER_CPU_SYMBOL(irq_regs);
+extern void gr_handle_kernel_exploit(void);
+
int sysctl_panic_on_stackoverflow;
/*
u64 estack_top, estack_bottom;
u64 curbase = (u64)task_stack_page(current);
- if (user_mode_vm(regs))
+ if (user_mode(regs))
return;
if (regs->sp >= curbase + sizeof(struct thread_info) +
irq_stack_top, irq_stack_bottom,
estack_top, estack_bottom);
+ gr_handle_kernel_exploit();
+
if (sysctl_panic_on_stackoverflow)
panic("low stack detected by irq handler - check messages\n");
#endif
* Jump label is enabled for the first time.
* So we expect a default_nop...
*/
- if (unlikely(memcmp((void *)entry->code, default_nop, 5)
+ if (unlikely(memcmp((void *)ktla_ktva(entry->code), default_nop, 5)
!= 0))
bug_at((void *)entry->code, __LINE__);
} else {
* ...otherwise expect an ideal_nop. Otherwise
* something went horribly wrong.
*/
- if (unlikely(memcmp((void *)entry->code, ideal_nop, 5)
+ if (unlikely(memcmp((void *)ktla_ktva(entry->code), ideal_nop, 5)
!= 0))
bug_at((void *)entry->code, __LINE__);
}
* are converting the default nop to the ideal nop.
*/
if (init) {
- if (unlikely(memcmp((void *)entry->code, default_nop, 5) != 0))
+ if (unlikely(memcmp((void *)ktla_ktva(entry->code), default_nop, 5) != 0))
bug_at((void *)entry->code, __LINE__);
} else {
code.jump = 0xe9;
code.offset = entry->target -
(entry->code + JUMP_LABEL_NOP_SIZE);
- if (unlikely(memcmp((void *)entry->code, &code, 5) != 0))
+ if (unlikely(memcmp((void *)ktla_ktva(entry->code), &code, 5) != 0))
bug_at((void *)entry->code, __LINE__);
}
memcpy(&code, ideal_nops[NOP_ATOMIC5], JUMP_LABEL_NOP_SIZE);
#ifdef CONFIG_X86_32
switch (regno) {
case GDB_SS:
- if (!user_mode_vm(regs))
+ if (!user_mode(regs))
*(unsigned long *)mem = __KERNEL_DS;
break;
case GDB_SP:
- if (!user_mode_vm(regs))
+ if (!user_mode(regs))
*(unsigned long *)mem = kernel_stack_pointer(regs);
break;
case GDB_GS:
bp->attr.bp_addr = breakinfo[breakno].addr;
bp->attr.bp_len = breakinfo[breakno].len;
bp->attr.bp_type = breakinfo[breakno].type;
- info->address = breakinfo[breakno].addr;
+ if (breakinfo[breakno].type == X86_BREAKPOINT_EXECUTE)
+ info->address = ktla_ktva(breakinfo[breakno].addr);
+ else
+ info->address = breakinfo[breakno].addr;
info->len = breakinfo[breakno].len;
info->type = breakinfo[breakno].type;
val = arch_install_hw_breakpoint(bp);
case 'k':
/* clear the trace bit */
linux_regs->flags &= ~X86_EFLAGS_TF;
- atomic_set(&kgdb_cpu_doing_single_step, -1);
+ atomic_set_unchecked(&kgdb_cpu_doing_single_step, -1);
/* set the trace bit if we're stepping */
if (remcomInBuffer[0] == 's') {
linux_regs->flags |= X86_EFLAGS_TF;
- atomic_set(&kgdb_cpu_doing_single_step,
+ atomic_set_unchecked(&kgdb_cpu_doing_single_step,
raw_smp_processor_id());
}
switch (cmd) {
case DIE_DEBUG:
- if (atomic_read(&kgdb_cpu_doing_single_step) != -1) {
+ if (atomic_read_unchecked(&kgdb_cpu_doing_single_step) != -1) {
if (user_mode(regs))
return single_step_cont(regs, args);
break;
#endif /* CONFIG_DEBUG_RODATA */
bpt->type = BP_BREAKPOINT;
- err = probe_kernel_read(bpt->saved_instr, (char *)bpt->bpt_addr,
+ err = probe_kernel_read(bpt->saved_instr, ktla_ktva((char *)bpt->bpt_addr),
BREAK_INSTR_SIZE);
if (err)
return err;
- err = probe_kernel_write((char *)bpt->bpt_addr,
+ err = probe_kernel_write(ktla_ktva((char *)bpt->bpt_addr),
arch_kgdb_ops.gdb_bpt_instr, BREAK_INSTR_SIZE);
#ifdef CONFIG_DEBUG_RODATA
if (!err)
return -EBUSY;
text_poke((void *)bpt->bpt_addr, arch_kgdb_ops.gdb_bpt_instr,
BREAK_INSTR_SIZE);
- err = probe_kernel_read(opc, (char *)bpt->bpt_addr, BREAK_INSTR_SIZE);
+ err = probe_kernel_read(opc, ktla_ktva((char *)bpt->bpt_addr), BREAK_INSTR_SIZE);
if (err)
return err;
if (memcmp(opc, arch_kgdb_ops.gdb_bpt_instr, BREAK_INSTR_SIZE))
if (mutex_is_locked(&text_mutex))
goto knl_write;
text_poke((void *)bpt->bpt_addr, bpt->saved_instr, BREAK_INSTR_SIZE);
- err = probe_kernel_read(opc, (char *)bpt->bpt_addr, BREAK_INSTR_SIZE);
+ err = probe_kernel_read(opc, ktla_ktva((char *)bpt->bpt_addr), BREAK_INSTR_SIZE);
if (err || memcmp(opc, bpt->saved_instr, BREAK_INSTR_SIZE))
goto knl_write;
return err;
knl_write:
#endif /* CONFIG_DEBUG_RODATA */
- return probe_kernel_write((char *)bpt->bpt_addr,
+ return probe_kernel_write(ktla_ktva((char *)bpt->bpt_addr),
(char *)bpt->saved_instr, BREAK_INSTR_SIZE);
}
s32 raddr;
} __packed *insn;
- insn = (struct __arch_relative_insn *)from;
+ insn = (struct __arch_relative_insn *)ktla_ktva(from);
+
+ pax_open_kernel();
insn->raddr = (s32)((long)(to) - ((long)(from) + 5));
insn->op = op;
+ pax_close_kernel();
}
/* Insert a jump instruction at address 'from', which jumps to address 'to'.*/
kprobe_opcode_t opcode;
kprobe_opcode_t *orig_opcodes = opcodes;
- if (search_exception_tables((unsigned long)opcodes))
+ if (search_exception_tables(ktva_ktla((unsigned long)opcodes)))
return 0; /* Page fault may occur on this address. */
retry:
* for the first byte, we can recover the original instruction
* from it and kp->opcode.
*/
- memcpy(buf, kp->addr, MAX_INSN_SIZE * sizeof(kprobe_opcode_t));
+ memcpy(buf, ktla_ktva(kp->addr), MAX_INSN_SIZE * sizeof(kprobe_opcode_t));
buf[0] = kp->opcode;
- return (unsigned long)buf;
+ return ktva_ktla((unsigned long)buf);
}
/*
/* Another subsystem puts a breakpoint, failed to recover */
if (insn.opcode.bytes[0] == BREAKPOINT_INSTRUCTION)
return 0;
+ pax_open_kernel();
memcpy(dest, insn.kaddr, insn.length);
+ pax_close_kernel();
#ifdef CONFIG_X86_64
if (insn_rip_relative(&insn)) {
return 0;
}
disp = (u8 *) dest + insn_offset_displacement(&insn);
+ pax_open_kernel();
*(s32 *) disp = (s32) newdisp;
+ pax_close_kernel();
}
#endif
return insn.length;
* nor set current_kprobe, because it doesn't use single
* stepping.
*/
- regs->ip = (unsigned long)p->ainsn.insn;
+ regs->ip = ktva_ktla((unsigned long)p->ainsn.insn);
preempt_enable_no_resched();
return;
}
regs->flags &= ~X86_EFLAGS_IF;
/* single step inline if the instruction is an int3 */
if (p->opcode == BREAKPOINT_INSTRUCTION)
- regs->ip = (unsigned long)p->addr;
+ regs->ip = ktla_ktva((unsigned long)p->addr);
else
- regs->ip = (unsigned long)p->ainsn.insn;
+ regs->ip = ktva_ktla((unsigned long)p->ainsn.insn);
}
NOKPROBE_SYMBOL(setup_singlestep);
struct kprobe *p;
struct kprobe_ctlblk *kcb;
- if (user_mode_vm(regs))
+ if (user_mode(regs))
return 0;
addr = (kprobe_opcode_t *)(regs->ip - sizeof(kprobe_opcode_t));
setup_singlestep(p, regs, kcb, 0);
return 1;
}
- } else if (*addr != BREAKPOINT_INSTRUCTION) {
+ } else if (*(kprobe_opcode_t *)ktla_ktva((unsigned long)addr) != BREAKPOINT_INSTRUCTION) {
/*
* The breakpoint instruction was removed right
* after we hit it. Another cpu has removed
" movq %rax, 152(%rsp)\n"
RESTORE_REGS_STRING
" popfq\n"
+#ifdef KERNEXEC_PLUGIN
+ " btsq $63,(%rsp)\n"
+#endif
#else
" pushf\n"
SAVE_REGS_STRING
struct kprobe_ctlblk *kcb)
{
unsigned long *tos = stack_addr(regs);
- unsigned long copy_ip = (unsigned long)p->ainsn.insn;
+ unsigned long copy_ip = ktva_ktla((unsigned long)p->ainsn.insn);
unsigned long orig_ip = (unsigned long)p->addr;
kprobe_opcode_t *insn = p->ainsn.insn;
struct die_args *args = data;
int ret = NOTIFY_DONE;
- if (args->regs && user_mode_vm(args->regs))
+ if (args->regs && user_mode(args->regs))
return ret;
if (val == DIE_GPF) {
/* Insert a move instruction which sets a pointer to eax/rdi (1st arg). */
static void synthesize_set_arg1(kprobe_opcode_t *addr, unsigned long val)
{
+ pax_open_kernel();
#ifdef CONFIG_X86_64
*addr++ = 0x48;
*addr++ = 0xbf;
*addr++ = 0xb8;
#endif
*(unsigned long *)addr = val;
+ pax_close_kernel();
}
asm (
* Verify if the address gap is in 2GB range, because this uses
* a relative jump.
*/
- rel = (long)op->optinsn.insn - (long)op->kp.addr + RELATIVEJUMP_SIZE;
+ rel = (long)op->optinsn.insn - ktla_ktva((long)op->kp.addr) + RELATIVEJUMP_SIZE;
if (abs(rel) > 0x7fffffff) {
__arch_remove_optimized_kprobe(op, 0);
return -ERANGE;
op->optinsn.size = ret;
/* Copy arch-dep-instance from template */
- memcpy(buf, &optprobe_template_entry, TMPL_END_IDX);
+ pax_open_kernel();
+ memcpy(buf, ktla_ktva(&optprobe_template_entry), TMPL_END_IDX);
+ pax_close_kernel();
/* Set probe information */
synthesize_set_arg1(buf + TMPL_MOVE_IDX, (unsigned long)op);
/* Set probe function call */
- synthesize_relcall(buf + TMPL_CALL_IDX, optimized_callback);
+ synthesize_relcall(ktva_ktla(buf) + TMPL_CALL_IDX, optimized_callback);
/* Set returning jmp instruction at the tail of out-of-line buffer */
- synthesize_reljump(buf + TMPL_END_IDX + op->optinsn.size,
+ synthesize_reljump(ktva_ktla(buf) + TMPL_END_IDX + op->optinsn.size,
(u8 *)op->kp.addr + op->optinsn.size);
flush_icache_range((unsigned long) buf,
WARN_ON(kprobe_disabled(&op->kp));
/* Backup instructions which will be replaced by jump address */
- memcpy(op->optinsn.copied_insn, op->kp.addr + INT3_SIZE,
+ memcpy(op->optinsn.copied_insn, ktla_ktva(op->kp.addr) + INT3_SIZE,
RELATIVE_ADDR_SIZE);
insn_buf[0] = RELATIVEJUMP_OPCODE;
/* This kprobe is really able to run optimized path. */
op = container_of(p, struct optimized_kprobe, kp);
/* Detour through copied instructions */
- regs->ip = (unsigned long)op->optinsn.insn + TMPL_END_IDX;
+ regs->ip = ktva_ktla((unsigned long)op->optinsn.insn) + TMPL_END_IDX;
if (!reenter)
reset_current_kprobe();
preempt_enable_no_resched();
static struct kobj_attribute type_attr = __ATTR_RO(type);
-static struct bin_attribute data_attr = {
+static bin_attribute_no_const data_attr __read_only = {
.attr = {
.name = "data",
.mode = S_IRUGO,
if (reload) {
#ifdef CONFIG_SMP
preempt_disable();
- load_LDT(pc);
+ load_LDT_nolock(pc);
if (!cpumask_equal(mm_cpumask(current->mm),
cpumask_of(smp_processor_id())))
smp_call_function(flush_ldt, current->mm, 1);
preempt_enable();
#else
- load_LDT(pc);
+ load_LDT_nolock(pc);
#endif
}
if (oldsize) {
return err;
for (i = 0; i < old->size; i++)
- write_ldt_entry(new->ldt, i, old->ldt + i * LDT_ENTRY_SIZE);
+ write_ldt_entry(new->ldt, i, old->ldt + i);
return 0;
}
retval = copy_ldt(&mm->context, &old_mm->context);
mutex_unlock(&old_mm->context.lock);
}
+
+ if (tsk == current) {
+ mm->context.vdso = 0;
+
+#ifdef CONFIG_X86_32
+#if defined(CONFIG_PAX_PAGEEXEC) || defined(CONFIG_PAX_SEGMEXEC)
+ mm->context.user_cs_base = 0UL;
+ mm->context.user_cs_limit = ~0UL;
+
+#if defined(CONFIG_PAX_PAGEEXEC) && defined(CONFIG_SMP)
+ cpus_clear(mm->context.cpu_user_cs_mask);
+#endif
+
+#endif
+#endif
+
+ }
+
return retval;
}
}
}
+#ifdef CONFIG_PAX_SEGMEXEC
+ if ((mm->pax_flags & MF_PAX_SEGMEXEC) && (ldt_info.contents & MODIFY_LDT_CONTENTS_CODE)) {
+ error = -EINVAL;
+ goto out_unlock;
+ }
+#endif
+
if (!IS_ENABLED(CONFIG_X86_16BIT) && !ldt_info.seg_32bit) {
error = -EINVAL;
goto out_unlock;
#include <asm/cacheflush.h>
#include <asm/debugreg.h>
-static void set_idt(void *newidt, __u16 limit)
+static void set_idt(struct desc_struct *newidt, __u16 limit)
{
struct desc_ptr curidt;
}
-static void set_gdt(void *newgdt, __u16 limit)
+static void set_gdt(struct desc_struct *newgdt, __u16 limit)
{
struct desc_ptr curgdt;
}
control_page = page_address(image->control_code_page);
- memcpy(control_page, relocate_kernel, KEXEC_CONTROL_CODE_MAX_SIZE);
+ memcpy(control_page, (void *)ktla_ktva((unsigned long)relocate_kernel), KEXEC_CONTROL_CODE_MAX_SIZE);
relocate_kernel_ptr = control_page;
page_list[PA_CONTROL_PAGE] = __pa(control_page);
#include <linux/linkage.h>
#include <asm/ptrace.h>
#include <asm/ftrace.h>
-
+#include <asm/alternative-asm.h>
.code64
.section .entry.text, "ax"
#ifdef CONFIG_DYNAMIC_FTRACE
ENTRY(function_hook)
+ pax_force_retaddr
retq
-END(function_hook)
+ENDPROC(function_hook)
ENTRY(ftrace_caller)
/* save_mcount_regs fills in first two parameters */
#endif
GLOBAL(ftrace_stub)
+ pax_force_retaddr
retq
-END(ftrace_caller)
+ENDPROC(ftrace_caller)
ENTRY(ftrace_regs_caller)
/* Save the current flags before any operations that can change them */
jmp ftrace_return
-END(ftrace_regs_caller)
+ENDPROC(ftrace_regs_caller)
#else /* ! CONFIG_DYNAMIC_FTRACE */
#endif
GLOBAL(ftrace_stub)
+ pax_force_retaddr
retq
trace:
/* save_mcount_regs fills in first two parameters */
save_mcount_regs
+ pax_force_fptr ftrace_trace_function
call *ftrace_trace_function
restore_mcount_regs
jmp fgraph_trace
-END(function_hook)
+ENDPROC(function_hook)
#endif /* CONFIG_DYNAMIC_FTRACE */
#endif /* CONFIG_FUNCTION_TRACER */
restore_mcount_regs
+ pax_force_retaddr
retq
-END(ftrace_graph_caller)
+ENDPROC(ftrace_graph_caller)
GLOBAL(return_to_handler)
subq $24, %rsp
movq 8(%rsp), %rdx
movq (%rsp), %rax
addq $24, %rsp
+ pax_force_fptr %rdi
jmp *%rdi
+ENDPROC(return_to_handler)
#endif
}
#endif
-void *module_alloc(unsigned long size)
+static inline void *__module_alloc(unsigned long size, pgprot_t prot)
{
- if (PAGE_ALIGN(size) > MODULES_LEN)
+ if (!size || PAGE_ALIGN(size) > MODULES_LEN)
return NULL;
return __vmalloc_node_range(size, 1,
MODULES_VADDR + get_module_load_offset(),
- MODULES_END, GFP_KERNEL | __GFP_HIGHMEM,
- PAGE_KERNEL_EXEC, NUMA_NO_NODE,
+ MODULES_END, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO,
+ prot, NUMA_NO_NODE,
__builtin_return_address(0));
}
+void *module_alloc(unsigned long size)
+{
+
+#ifdef CONFIG_PAX_KERNEXEC
+ return __module_alloc(size, PAGE_KERNEL);
+#else
+ return __module_alloc(size, PAGE_KERNEL_EXEC);
+#endif
+
+}
+
+#ifdef CONFIG_PAX_KERNEXEC
+#ifdef CONFIG_X86_32
+void *module_alloc_exec(unsigned long size)
+{
+ struct vm_struct *area;
+
+ if (size == 0)
+ return NULL;
+
+ area = __get_vm_area(size, VM_ALLOC, (unsigned long)&MODULES_EXEC_VADDR, (unsigned long)&MODULES_EXEC_END);
+return area ? area->addr : NULL;
+}
+EXPORT_SYMBOL(module_alloc_exec);
+
+void module_memfree_exec(void *module_region)
+{
+ vunmap(module_region);
+}
+EXPORT_SYMBOL(module_memfree_exec);
+#else
+void module_memfree_exec(void *module_region)
+{
+ module_memfree(module_region);
+}
+EXPORT_SYMBOL(module_memfree_exec);
+
+void *module_alloc_exec(unsigned long size)
+{
+ return __module_alloc(size, PAGE_KERNEL_RX);
+}
+EXPORT_SYMBOL(module_alloc_exec);
+#endif
+#endif
+
#ifdef CONFIG_X86_32
int apply_relocate(Elf32_Shdr *sechdrs,
const char *strtab,
unsigned int i;
Elf32_Rel *rel = (void *)sechdrs[relsec].sh_addr;
Elf32_Sym *sym;
- uint32_t *location;
+ uint32_t *plocation, location;
DEBUGP("Applying relocate section %u to %u\n",
relsec, sechdrs[relsec].sh_info);
for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) {
/* This is where to make the change */
- location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr
- + rel[i].r_offset;
+ plocation = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr + rel[i].r_offset;
+ location = (uint32_t)plocation;
+ if (sechdrs[sechdrs[relsec].sh_info].sh_flags & SHF_EXECINSTR)
+ plocation = ktla_ktva((void *)plocation);
/* This is the symbol it is referring to. Note that all
undefined symbols have been resolved. */
sym = (Elf32_Sym *)sechdrs[symindex].sh_addr
switch (ELF32_R_TYPE(rel[i].r_info)) {
case R_386_32:
/* We add the value into the location given */
- *location += sym->st_value;
+ pax_open_kernel();
+ *plocation += sym->st_value;
+ pax_close_kernel();
break;
case R_386_PC32:
/* Add the value, subtract its position */
- *location += sym->st_value - (uint32_t)location;
+ pax_open_kernel();
+ *plocation += sym->st_value - location;
+ pax_close_kernel();
break;
default:
pr_err("%s: Unknown relocation: %u\n",
case R_X86_64_NONE:
break;
case R_X86_64_64:
+ pax_open_kernel();
*(u64 *)loc = val;
+ pax_close_kernel();
break;
case R_X86_64_32:
+ pax_open_kernel();
*(u32 *)loc = val;
+ pax_close_kernel();
if (val != *(u32 *)loc)
goto overflow;
break;
case R_X86_64_32S:
+ pax_open_kernel();
*(s32 *)loc = val;
+ pax_close_kernel();
if ((s64)val != *(s32 *)loc)
goto overflow;
break;
case R_X86_64_PC32:
val -= (u64)loc;
+ pax_open_kernel();
*(u32 *)loc = val;
+ pax_close_kernel();
+
#if 0
if ((s64)val != *(s32 *)loc)
goto overflow;
#include <linux/notifier.h>
#include <linux/uaccess.h>
#include <linux/gfp.h>
+#include <linux/grsecurity.h>
#include <asm/processor.h>
#include <asm/msr.h>
int err = 0;
ssize_t bytes = 0;
+#ifdef CONFIG_GRKERNSEC_KMEM
+ gr_handle_msr_write();
+ return -EPERM;
+#endif
+
if (count % 8)
return -EINVAL; /* Invalid chunk size */
err = -EBADF;
break;
}
+#ifdef CONFIG_GRKERNSEC_KMEM
+ gr_handle_msr_write();
+ return -EPERM;
+#endif
if (copy_from_user(®s, uregs, sizeof regs)) {
err = -EFAULT;
break;
return notifier_from_errno(err);
}
-static struct notifier_block __refdata msr_class_cpu_notifier = {
+static struct notifier_block msr_class_cpu_notifier = {
.notifier_call = msr_class_cpu_callback,
};
static void nmi_max_handler(struct irq_work *w)
{
- struct nmiaction *a = container_of(w, struct nmiaction, irq_work);
+ struct nmiwork *n = container_of(w, struct nmiwork, irq_work);
int remainder_ns, decimal_msecs;
- u64 whole_msecs = ACCESS_ONCE(a->max_duration);
+ u64 whole_msecs = ACCESS_ONCE(n->max_duration);
remainder_ns = do_div(whole_msecs, (1000 * 1000));
decimal_msecs = remainder_ns / 1000;
printk_ratelimited(KERN_INFO
"INFO: NMI handler (%ps) took too long to run: %lld.%03d msecs\n",
- a->handler, whole_msecs, decimal_msecs);
+ n->action->handler, whole_msecs, decimal_msecs);
}
static int nmi_handle(unsigned int type, struct pt_regs *regs, bool b2b)
delta = sched_clock() - delta;
trace_nmi_handler(a->handler, (int)delta, thishandled);
- if (delta < nmi_longest_ns || delta < a->max_duration)
+ if (delta < nmi_longest_ns || delta < a->work->max_duration)
continue;
- a->max_duration = delta;
- irq_work_queue(&a->irq_work);
+ a->work->max_duration = delta;
+ irq_work_queue(&a->work->irq_work);
}
rcu_read_unlock();
}
NOKPROBE_SYMBOL(nmi_handle);
-int __register_nmi_handler(unsigned int type, struct nmiaction *action)
+int __register_nmi_handler(unsigned int type, const struct nmiaction *action)
{
struct nmi_desc *desc = nmi_to_desc(type);
unsigned long flags;
if (!action->handler)
return -EINVAL;
- init_irq_work(&action->irq_work, nmi_max_handler);
+ action->work->action = action;
+ init_irq_work(&action->work->irq_work, nmi_max_handler);
spin_lock_irqsave(&desc->lock, flags);
* event confuses some handlers (kdump uses this flag)
*/
if (action->flags & NMI_FLAG_FIRST)
- list_add_rcu(&action->list, &desc->head);
+ pax_list_add_rcu((struct list_head *)&action->list, &desc->head);
else
- list_add_tail_rcu(&action->list, &desc->head);
+ pax_list_add_tail_rcu((struct list_head *)&action->list, &desc->head);
spin_unlock_irqrestore(&desc->lock, flags);
return 0;
if (!strcmp(n->name, name)) {
WARN(in_nmi(),
"Trying to free NMI (%s) from NMI context!\n", n->name);
- list_del_rcu(&n->list);
+ pax_list_del_rcu((struct list_head *)&n->list);
break;
}
}
dotraplinkage notrace void
do_nmi(struct pt_regs *regs, long error_code)
{
+
+#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_KERNEXEC)
+ if (!user_mode(regs)) {
+ unsigned long cs = regs->cs & 0xFFFF;
+ unsigned long ip = ktva_ktla(regs->ip);
+
+ if ((cs == __KERNEL_CS || cs == __KERNEXEC_KERNEL_CS) && ip <= (unsigned long)_etext)
+ regs->ip = ip;
+ }
+#endif
+
nmi_nesting_preprocess(regs);
nmi_enter();
{
/* trap all the unknown NMIs we may generate */
register_nmi_handler(NMI_UNKNOWN, nmi_unk_cb, 0, "nmi_selftest_unk",
- __initdata);
+ __initconst);
}
static void __init cleanup_nmi_testsuite(void)
unsigned long timeout;
if (register_nmi_handler(NMI_LOCAL, test_nmi_ipi_callback,
- NMI_FLAG_FIRST, "nmi_selftest", __initdata)) {
+ NMI_FLAG_FIRST, "nmi_selftest", __initconst)) {
nmi_fail = FAILURE;
return;
}
#include <asm/paravirt.h>
-struct pv_lock_ops pv_lock_ops = {
+struct pv_lock_ops pv_lock_ops __read_only = {
#ifdef CONFIG_SMP
.lock_spinning = __PV_IS_CALLEE_SAVE(paravirt_nop),
.unlock_kick = paravirt_nop,
{
return x;
}
+#if defined(CONFIG_X86_32) && defined(CONFIG_X86_PAE)
+PV_CALLEE_SAVE_REGS_THUNK(_paravirt_ident_64);
+#endif
void __init default_banner(void)
{
if (opfunc == NULL)
/* If there's no function, patch it with a ud2a (BUG) */
- ret = paravirt_patch_insns(insnbuf, len, ud2a, ud2a+sizeof(ud2a));
- else if (opfunc == _paravirt_nop)
+ ret = paravirt_patch_insns(insnbuf, len, ktva_ktla(ud2a), ud2a+sizeof(ud2a));
+ else if (opfunc == (void *)_paravirt_nop)
/* If the operation is a nop, then nop the callsite */
ret = paravirt_patch_nop();
/* identity functions just return their single argument */
- else if (opfunc == _paravirt_ident_32)
+ else if (opfunc == (void *)_paravirt_ident_32)
ret = paravirt_patch_ident_32(insnbuf, len);
- else if (opfunc == _paravirt_ident_64)
+ else if (opfunc == (void *)_paravirt_ident_64)
+ ret = paravirt_patch_ident_64(insnbuf, len);
+#if defined(CONFIG_X86_32) && defined(CONFIG_X86_PAE)
+ else if (opfunc == (void *)__raw_callee_save__paravirt_ident_64)
ret = paravirt_patch_ident_64(insnbuf, len);
+#endif
else if (type == PARAVIRT_PATCH(pv_cpu_ops.iret) ||
type == PARAVIRT_PATCH(pv_cpu_ops.irq_enable_sysexit) ||
if (insn_len > len || start == NULL)
insn_len = len;
else
- memcpy(insnbuf, start, insn_len);
+ memcpy(insnbuf, ktla_ktva(start), insn_len);
return insn_len;
}
return this_cpu_read(paravirt_lazy_mode);
}
-struct pv_info pv_info = {
+struct pv_info pv_info __read_only = {
.name = "bare hardware",
.paravirt_enabled = 0,
.kernel_rpl = 0,
#endif
};
-struct pv_init_ops pv_init_ops = {
+struct pv_init_ops pv_init_ops __read_only = {
.patch = native_patch,
};
-struct pv_time_ops pv_time_ops = {
+struct pv_time_ops pv_time_ops __read_only = {
.sched_clock = native_sched_clock,
.steal_clock = native_steal_clock,
};
-__visible struct pv_irq_ops pv_irq_ops = {
+__visible struct pv_irq_ops pv_irq_ops __read_only = {
.save_fl = __PV_IS_CALLEE_SAVE(native_save_fl),
.restore_fl = __PV_IS_CALLEE_SAVE(native_restore_fl),
.irq_disable = __PV_IS_CALLEE_SAVE(native_irq_disable),
#endif
};
-__visible struct pv_cpu_ops pv_cpu_ops = {
+__visible struct pv_cpu_ops pv_cpu_ops __read_only = {
.cpuid = native_cpuid,
.get_debugreg = native_get_debugreg,
.set_debugreg = native_set_debugreg,
NOKPROBE_SYMBOL(native_set_debugreg);
NOKPROBE_SYMBOL(native_load_idt);
-struct pv_apic_ops pv_apic_ops = {
+struct pv_apic_ops pv_apic_ops __read_only= {
#ifdef CONFIG_X86_LOCAL_APIC
.startup_ipi_hook = paravirt_nop,
#endif
};
-#if defined(CONFIG_X86_32) && !defined(CONFIG_X86_PAE)
+#ifdef CONFIG_X86_32
+#ifdef CONFIG_X86_PAE
+/* 64-bit pagetable entries */
+#define PTE_IDENT PV_CALLEE_SAVE(_paravirt_ident_64)
+#else
/* 32-bit pagetable entries */
#define PTE_IDENT __PV_IS_CALLEE_SAVE(_paravirt_ident_32)
+#endif
#else
/* 64-bit pagetable entries */
#define PTE_IDENT __PV_IS_CALLEE_SAVE(_paravirt_ident_64)
#endif
-struct pv_mmu_ops pv_mmu_ops = {
+struct pv_mmu_ops pv_mmu_ops __read_only = {
.read_cr2 = native_read_cr2,
.write_cr2 = native_write_cr2,
.make_pud = PTE_IDENT,
.set_pgd = native_set_pgd,
+ .set_pgd_batched = native_set_pgd_batched,
#endif
#endif /* PAGETABLE_LEVELS >= 3 */
},
.set_fixmap = native_set_fixmap,
+
+#ifdef CONFIG_PAX_KERNEXEC
+ .pax_open_kernel = native_pax_open_kernel,
+ .pax_close_kernel = native_pax_close_kernel,
+#endif
+
};
EXPORT_SYMBOL_GPL(pv_time_ops);
DEF_NATIVE(pv_mmu_ops, read_cr2, "movq %cr2, %rax");
DEF_NATIVE(pv_mmu_ops, read_cr3, "movq %cr3, %rax");
DEF_NATIVE(pv_mmu_ops, write_cr3, "movq %rdi, %cr3");
+
+#ifndef CONFIG_PAX_MEMORY_UDEREF
DEF_NATIVE(pv_mmu_ops, flush_tlb_single, "invlpg (%rdi)");
+#endif
+
DEF_NATIVE(pv_cpu_ops, clts, "clts");
DEF_NATIVE(pv_cpu_ops, wbinvd, "wbinvd");
PATCH_SITE(pv_mmu_ops, read_cr3);
PATCH_SITE(pv_mmu_ops, write_cr3);
PATCH_SITE(pv_cpu_ops, clts);
+
+#ifndef CONFIG_PAX_MEMORY_UDEREF
PATCH_SITE(pv_mmu_ops, flush_tlb_single);
+#endif
+
PATCH_SITE(pv_cpu_ops, wbinvd);
patch_site:
tce_space = be64_to_cpu(readq(target));
tce_space = tce_space & TAR_SW_BITS;
- tce_space = tce_space & (~specified_table_size);
+ tce_space = tce_space & (~(unsigned long)specified_table_size);
info->tce_space = (u64 *)__va(tce_space);
}
}
#include <asm/iommu_table.h>
#include <linux/string.h>
#include <linux/kallsyms.h>
-
+#include <linux/sched.h>
#define DEBUG 1
struct dma_attrs *attrs)
{
if (is_swiotlb_buffer(dma_to_phys(dev, dma_addr)))
- swiotlb_free_coherent(dev, size, vaddr, dma_addr);
+ swiotlb_free_coherent(dev, size, vaddr, dma_addr, attrs);
else
dma_generic_free_coherent(dev, size, vaddr, dma_addr, attrs);
}
* section. Since TSS's are completely CPU-local, we want them
* on exact cacheline boundaries, to eliminate cacheline ping-pong.
*/
-__visible DEFINE_PER_CPU_SHARED_ALIGNED(struct tss_struct, init_tss) = INIT_TSS;
+struct tss_struct init_tss[NR_CPUS] __visible ____cacheline_internodealigned_in_smp = { [0 ... NR_CPUS-1] = INIT_TSS };
+EXPORT_SYMBOL(init_tss);
#ifdef CONFIG_X86_64
static DEFINE_PER_CPU(unsigned char, is_idle);
task_xstate_cachep =
kmem_cache_create("task_xstate", xstate_size,
__alignof__(union thread_xstate),
- SLAB_PANIC | SLAB_NOTRACK, NULL);
+ SLAB_PANIC | SLAB_NOTRACK | SLAB_USERCOPY, NULL);
setup_xstate_comp();
}
unsigned long *bp = t->io_bitmap_ptr;
if (bp) {
- struct tss_struct *tss = &per_cpu(init_tss, get_cpu());
+ struct tss_struct *tss = init_tss + get_cpu();
t->io_bitmap_ptr = NULL;
clear_thread_flag(TIF_IO_BITMAP);
{
struct task_struct *tsk = current;
+#if defined(CONFIG_X86_32) && !defined(CONFIG_CC_STACKPROTECTOR) && !defined(CONFIG_PAX_MEMORY_UDEREF)
+ loadsegment(gs, 0);
+#endif
flush_ptrace_hw_breakpoint(tsk);
memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));
drop_init_fpu(tsk);
void exit_idle(void)
{
/* idle loop has pid 0 */
- if (current->pid)
+ if (task_pid_nr(current))
return;
__exit_idle();
}
return ret;
}
#endif
-void stop_this_cpu(void *dummy)
+__noreturn void stop_this_cpu(void *dummy)
{
local_irq_disable();
/*
}
early_param("idle", idle_setup);
-unsigned long arch_align_stack(unsigned long sp)
+#ifdef CONFIG_PAX_RANDKSTACK
+void pax_randomize_kstack(struct pt_regs *regs)
{
- if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
- sp -= get_random_int() % 8192;
- return sp & ~0xf;
-}
+ struct thread_struct *thread = ¤t->thread;
+ unsigned long time;
-unsigned long arch_randomize_brk(struct mm_struct *mm)
-{
- unsigned long range_end = mm->brk + 0x02000000;
- return randomize_range(mm->brk, range_end, 0) ? : mm->brk;
-}
+ if (!randomize_va_space)
+ return;
+
+ if (v8086_mode(regs))
+ return;
+ rdtscl(time);
+
+ /* P4 seems to return a 0 LSB, ignore it */
+#ifdef CONFIG_MPENTIUM4
+ time &= 0x3EUL;
+ time <<= 2;
+#elif defined(CONFIG_X86_64)
+ time &= 0xFUL;
+ time <<= 4;
+#else
+ time &= 0x1FUL;
+ time <<= 3;
+#endif
+
+ thread->sp0 ^= time;
+ load_sp0(init_tss + smp_processor_id(), thread);
+
+#ifdef CONFIG_X86_64
+ this_cpu_write(kernel_stack, thread->sp0);
+#endif
+}
+#endif
unsigned long thread_saved_pc(struct task_struct *tsk)
{
return ((unsigned long *)tsk->thread.sp)[3];
+//XXX return tsk->thread.eip;
}
void __show_regs(struct pt_regs *regs, int all)
unsigned long sp;
unsigned short ss, gs;
- if (user_mode_vm(regs)) {
+ if (user_mode(regs)) {
sp = regs->sp;
ss = regs->ss & 0xffff;
- gs = get_user_gs(regs);
} else {
sp = kernel_stack_pointer(regs);
savesegment(ss, ss);
- savesegment(gs, gs);
}
+ gs = get_user_gs(regs);
printk(KERN_DEFAULT "EIP: %04x:[<%08lx>] EFLAGS: %08lx CPU: %d\n",
(u16)regs->cs, regs->ip, regs->flags,
- smp_processor_id());
+ raw_smp_processor_id());
print_symbol("EIP is at %s\n", regs->ip);
printk(KERN_DEFAULT "EAX: %08lx EBX: %08lx ECX: %08lx EDX: %08lx\n",
int copy_thread(unsigned long clone_flags, unsigned long sp,
unsigned long arg, struct task_struct *p)
{
- struct pt_regs *childregs = task_pt_regs(p);
+ struct pt_regs *childregs = task_stack_page(p) + THREAD_SIZE - sizeof(struct pt_regs) - 8;
struct task_struct *tsk;
int err;
p->thread.sp = (unsigned long) childregs;
p->thread.sp0 = (unsigned long) (childregs+1);
+ p->tinfo.lowest_stack = (unsigned long)task_stack_page(p) + 2 * sizeof(unsigned long);
memset(p->thread.ptrace_bps, 0, sizeof(p->thread.ptrace_bps));
if (unlikely(p->flags & PF_KTHREAD)) {
/* kernel thread */
memset(childregs, 0, sizeof(struct pt_regs));
p->thread.ip = (unsigned long) ret_from_kernel_thread;
- task_user_gs(p) = __KERNEL_STACK_CANARY;
- childregs->ds = __USER_DS;
- childregs->es = __USER_DS;
+ savesegment(gs, childregs->gs);
+ childregs->ds = __KERNEL_DS;
+ childregs->es = __KERNEL_DS;
childregs->fs = __KERNEL_PERCPU;
childregs->bx = sp; /* function */
childregs->bp = arg;
struct thread_struct *prev = &prev_p->thread,
*next = &next_p->thread;
int cpu = smp_processor_id();
- struct tss_struct *tss = &per_cpu(init_tss, cpu);
+ struct tss_struct *tss = init_tss + cpu;
fpu_switch_t fpu;
/* never put a printk in __switch_to... printk() calls wake_up*() indirectly */
*/
lazy_save_gs(prev->gs);
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ __set_fs(task_thread_info(next_p)->addr_limit);
+#endif
+
/*
* Load the per-thread Thread-Local Storage descriptor.
*/
*/
arch_end_context_switch(next_p);
- this_cpu_write(kernel_stack,
- (unsigned long)task_stack_page(next_p) +
- THREAD_SIZE - KERNEL_STACK_OFFSET);
+ this_cpu_write(current_task, next_p);
+ this_cpu_write(current_tinfo, &next_p->tinfo);
+ this_cpu_write(kernel_stack, next->sp0);
/*
* Restore %gs if needed (which is common)
switch_fpu_finish(next_p, fpu);
- this_cpu_write(current_task, next_p);
-
return prev_p;
}
} while (count++ < 16);
return 0;
}
-
struct pt_regs *childregs;
struct task_struct *me = current;
- p->thread.sp0 = (unsigned long)task_stack_page(p) + THREAD_SIZE;
+ p->thread.sp0 = (unsigned long)task_stack_page(p) + THREAD_SIZE - 16;
childregs = task_pt_regs(p);
p->thread.sp = (unsigned long) childregs;
p->thread.usersp = me->thread.usersp;
+ p->tinfo.lowest_stack = (unsigned long)task_stack_page(p) + 2 * sizeof(unsigned long);
set_tsk_thread_flag(p, TIF_FORK);
p->thread.io_bitmap_ptr = NULL;
p->thread.fs = p->thread.fsindex ? 0 : me->thread.fs;
savesegment(es, p->thread.es);
savesegment(ds, p->thread.ds);
+ savesegment(ss, p->thread.ss);
+ BUG_ON(p->thread.ss == __UDEREF_KERNEL_DS);
memset(p->thread.ptrace_bps, 0, sizeof(p->thread.ptrace_bps));
if (unlikely(p->flags & PF_KTHREAD)) {
struct thread_struct *prev = &prev_p->thread;
struct thread_struct *next = &next_p->thread;
int cpu = smp_processor_id();
- struct tss_struct *tss = &per_cpu(init_tss, cpu);
+ struct tss_struct *tss = init_tss + cpu;
unsigned fsindex, gsindex;
fpu_switch_t fpu;
if (unlikely(next->ds | prev->ds))
loadsegment(ds, next->ds);
+ savesegment(ss, prev->ss);
+ if (unlikely(next->ss != prev->ss))
+ loadsegment(ss, next->ss);
+
/*
* Switch FS and GS.
*
prev->usersp = this_cpu_read(old_rsp);
this_cpu_write(old_rsp, next->usersp);
this_cpu_write(current_task, next_p);
+ this_cpu_write(current_tinfo, &next_p->tinfo);
/*
* If it were not for PREEMPT_ACTIVE we could guarantee that the
task_thread_info(prev_p)->saved_preempt_count = this_cpu_read(__preempt_count);
this_cpu_write(__preempt_count, task_thread_info(next_p)->saved_preempt_count);
- this_cpu_write(kernel_stack,
- (unsigned long)task_stack_page(next_p) +
- THREAD_SIZE - KERNEL_STACK_OFFSET);
+ this_cpu_write(kernel_stack, next->sp0);
/*
* Now maybe reload the debug registers and handle I/O bitmaps
if (!p || p == current || p->state == TASK_RUNNING)
return 0;
stack = (unsigned long)task_stack_page(p);
- if (p->thread.sp < stack || p->thread.sp >= stack+THREAD_SIZE)
+ if (p->thread.sp < stack || p->thread.sp > stack+THREAD_SIZE-16-sizeof(u64))
return 0;
fp = *(u64 *)(p->thread.sp);
do {
- if (fp < (unsigned long)stack ||
- fp >= (unsigned long)stack+THREAD_SIZE)
+ if (fp < stack || fp > stack+THREAD_SIZE-16-sizeof(u64))
return 0;
ip = *(u64 *)(fp+8);
if (!in_sched_functions(ip))
unsigned long sp = (unsigned long)®s->sp;
u32 *prev_esp;
- if (context == (sp & ~(THREAD_SIZE - 1)))
+ if (context == ((sp + 8) & ~(THREAD_SIZE - 1)))
return sp;
- prev_esp = (u32 *)(context);
+ prev_esp = *(u32 **)(context);
if (prev_esp)
return (unsigned long)prev_esp;
if (child->thread.gs != value)
return do_arch_prctl(child, ARCH_SET_GS, value);
return 0;
+
+ case offsetof(struct user_regs_struct,ip):
+ /*
+ * Protect against any attempt to set ip to an
+ * impossible address. There are dragons lurking if the
+ * address is noncanonical. (This explicitly allows
+ * setting ip to TASK_SIZE_MAX, because user code can do
+ * that all by itself by running off the end of its
+ * address space.
+ */
+ if (value > TASK_SIZE_MAX)
+ return -EIO;
+ break;
+
#endif
}
static unsigned long ptrace_get_dr7(struct perf_event *bp[])
{
int i;
- int dr7 = 0;
+ unsigned long dr7 = 0;
struct arch_hw_breakpoint *info;
for (i = 0; i < HBP_NUM; i++) {
unsigned long addr, unsigned long data)
{
int ret;
- unsigned long __user *datap = (unsigned long __user *)data;
+ unsigned long __user *datap = (__force unsigned long __user *)data;
switch (request) {
/* read the word at location addr in the USER area. */
if ((int) addr < 0)
return -EIO;
ret = do_get_thread_area(child, addr,
- (struct user_desc __user *)data);
+ (__force struct user_desc __user *) data);
break;
case PTRACE_SET_THREAD_AREA:
if ((int) addr < 0)
return -EIO;
ret = do_set_thread_area(child, addr,
- (struct user_desc __user *)data, 0);
+ (__force struct user_desc __user *) data, 0);
break;
#endif
#ifdef CONFIG_X86_64
-static struct user_regset x86_64_regsets[] __read_mostly = {
+static user_regset_no_const x86_64_regsets[] __read_only = {
[REGSET_GENERAL] = {
.core_note_type = NT_PRSTATUS,
.n = sizeof(struct user_regs_struct) / sizeof(long),
#endif /* CONFIG_X86_64 */
#if defined CONFIG_X86_32 || defined CONFIG_IA32_EMULATION
-static struct user_regset x86_32_regsets[] __read_mostly = {
+static user_regset_no_const x86_32_regsets[] __read_only = {
[REGSET_GENERAL] = {
.core_note_type = NT_PRSTATUS,
.n = sizeof(struct user_regs_struct32) / sizeof(u32),
*/
u64 xstate_fx_sw_bytes[USER_XSTATE_FX_SW_WORDS];
-void update_regset_xstate_info(unsigned int size, u64 xstate_mask)
+void __init update_regset_xstate_info(unsigned int size, u64 xstate_mask)
{
#ifdef CONFIG_X86_64
x86_64_regsets[REGSET_XSTATE].n = size / sizeof(u64);
memset(info, 0, sizeof(*info));
info->si_signo = SIGTRAP;
info->si_code = si_code;
- info->si_addr = user_mode_vm(regs) ? (void __user *)regs->ip : NULL;
+ info->si_addr = user_mode(regs) ? (__force void __user *)regs->ip : NULL;
}
void user_single_step_siginfo(struct task_struct *tsk,
}
}
+#ifdef CONFIG_GRKERNSEC_SETXID
+extern void gr_delayed_cred_worker(void);
+#endif
+
/*
* We can return 0 to resume the syscall or anything else to go to phase
* 2. If we resume the syscall, we need to put something appropriate in
BUG_ON(regs != task_pt_regs(current));
+#ifdef CONFIG_GRKERNSEC_SETXID
+ if (unlikely(test_and_clear_thread_flag(TIF_GRSEC_SETXID)))
+ gr_delayed_cred_worker();
+#endif
+
/*
* If we stepped into a sysenter/syscall insn, it trapped in
* kernel mode; do_debug() cleared TF and set TIF_SINGLESTEP.
*/
user_exit();
+#ifdef CONFIG_GRKERNSEC_SETXID
+ if (unlikely(test_and_clear_thread_flag(TIF_GRSEC_SETXID)))
+ gr_delayed_cred_worker();
+#endif
+
audit_syscall_exit(regs);
if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT)))
reset_hung_task_detector();
}
-static atomic64_t last_value = ATOMIC64_INIT(0);
+static atomic64_unchecked_t last_value = ATOMIC64_INIT(0);
void pvclock_resume(void)
{
- atomic64_set(&last_value, 0);
+ atomic64_set_unchecked(&last_value, 0);
}
u8 pvclock_read_flags(struct pvclock_vcpu_time_info *src)
* updating at the same time, and one of them could be slightly behind,
* making the assumption that last_value always go forward fail to hold.
*/
- last = atomic64_read(&last_value);
+ last = atomic64_read_unchecked(&last_value);
do {
if (ret < last)
return last;
- last = atomic64_cmpxchg(&last_value, last, ret);
+ last = atomic64_cmpxchg_unchecked(&last_value, last, ret);
} while (unlikely(last != ret));
return ret;
void __noreturn machine_real_restart(unsigned int type)
{
+
+#if defined(CONFIG_X86_32) && (defined(CONFIG_PAX_KERNEXEC) || defined(CONFIG_PAX_MEMORY_UDEREF))
+ struct desc_struct *gdt;
+#endif
+
local_irq_disable();
/*
/* Jump to the identity-mapped low memory code */
#ifdef CONFIG_X86_32
- asm volatile("jmpl *%0" : :
+
+#if defined(CONFIG_PAX_KERNEXEC) || defined(CONFIG_PAX_MEMORY_UDEREF)
+ gdt = get_cpu_gdt_table(smp_processor_id());
+ pax_open_kernel();
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ gdt[GDT_ENTRY_KERNEL_DS].type = 3;
+ gdt[GDT_ENTRY_KERNEL_DS].limit = 0xf;
+ loadsegment(ds, __KERNEL_DS);
+ loadsegment(es, __KERNEL_DS);
+ loadsegment(ss, __KERNEL_DS);
+#endif
+#ifdef CONFIG_PAX_KERNEXEC
+ gdt[GDT_ENTRY_KERNEL_CS].base0 = 0;
+ gdt[GDT_ENTRY_KERNEL_CS].base1 = 0;
+ gdt[GDT_ENTRY_KERNEL_CS].base2 = 0;
+ gdt[GDT_ENTRY_KERNEL_CS].limit0 = 0xffff;
+ gdt[GDT_ENTRY_KERNEL_CS].limit = 0xf;
+ gdt[GDT_ENTRY_KERNEL_CS].g = 1;
+#endif
+ pax_close_kernel();
+#endif
+
+ asm volatile("ljmpl *%0" : :
"rm" (real_mode_header->machine_real_restart_asm),
"a" (type));
#else
* This means that this function can never return, it can misbehave
* by not rebooting properly and hanging.
*/
-static void native_machine_emergency_restart(void)
+static void __noreturn native_machine_emergency_restart(void)
{
int i;
int attempt = 0;
#endif
}
-static void __machine_emergency_restart(int emergency)
+static void __noreturn __machine_emergency_restart(int emergency)
{
reboot_emergency = emergency;
machine_ops.emergency_restart();
}
-static void native_machine_restart(char *__unused)
+static void __noreturn native_machine_restart(char *__unused)
{
pr_notice("machine restart\n");
__machine_emergency_restart(0);
}
-static void native_machine_halt(void)
+static void __noreturn native_machine_halt(void)
{
/* Stop other cpus and apics */
machine_shutdown();
stop_this_cpu(NULL);
}
-static void native_machine_power_off(void)
+static void __noreturn native_machine_power_off(void)
{
if (pm_power_off) {
if (!reboot_force)
}
/* A fallback in case there is no PM info available */
tboot_shutdown(TB_SHUTDOWN_HALT);
+ unreachable();
}
-struct machine_ops machine_ops = {
+struct machine_ops machine_ops __read_only = {
.power_off = native_machine_power_off,
.shutdown = native_machine_shutdown,
.emergency_restart = native_machine_emergency_restart,
unsigned int vendor;
unsigned int device;
void (*reboot_fixup)(struct pci_dev *);
-};
+} __do_const;
/*
* PCI ids solely used for fixups_table go here
/* jump to identity mapped page */
addq $(identity_mapped - relocate_kernel), %r8
- pushq %r8
- ret
+ jmp *%r8
identity_mapped:
/* set return address to 0 if not preserving context */
#include <asm/mce.h>
#include <asm/alternative.h>
#include <asm/prom.h>
+#include <asm/boot.h>
/*
* max_low_pfn_mapped: highest direct mapped pfn under 4GB
#endif
-#if !defined(CONFIG_X86_PAE) || defined(CONFIG_X86_64)
-__visible unsigned long mmu_cr4_features;
+#ifdef CONFIG_X86_64
+__visible unsigned long mmu_cr4_features __read_only = X86_CR4_PSE | X86_CR4_PAE | X86_CR4_PGE;
+#elif defined(CONFIG_X86_PAE)
+__visible unsigned long mmu_cr4_features __read_only = X86_CR4_PAE;
#else
-__visible unsigned long mmu_cr4_features = X86_CR4_PAE;
+__visible unsigned long mmu_cr4_features __read_only;
#endif
+void set_in_cr4(unsigned long mask)
+{
+ unsigned long cr4 = read_cr4();
+
+ if ((cr4 & mask) == mask && cr4 == mmu_cr4_features)
+ return;
+
+ pax_open_kernel();
+ mmu_cr4_features |= mask;
+ pax_close_kernel();
+
+ if (trampoline_cr4_features)
+ *trampoline_cr4_features = mmu_cr4_features;
+ cr4 |= mask;
+ write_cr4(cr4);
+}
+EXPORT_SYMBOL(set_in_cr4);
+
+void clear_in_cr4(unsigned long mask)
+{
+ unsigned long cr4 = read_cr4();
+
+ if (!(cr4 & mask) && cr4 == mmu_cr4_features)
+ return;
+
+ pax_open_kernel();
+ mmu_cr4_features &= ~mask;
+ pax_close_kernel();
+
+ if (trampoline_cr4_features)
+ *trampoline_cr4_features = mmu_cr4_features;
+ cr4 &= ~mask;
+ write_cr4(cr4);
+}
+EXPORT_SYMBOL(clear_in_cr4);
+
/* Boot loader ID and version as integers, for the benefit of proc_dointvec */
int bootloader_type, bootloader_version;
* area (640->1Mb) as ram even though it is not.
* take them out.
*/
- e820_remove_range(BIOS_BEGIN, BIOS_END - BIOS_BEGIN, E820_RAM, 1);
+ e820_remove_range(ISA_START_ADDRESS, ISA_END_ADDRESS - ISA_START_ADDRESS, E820_RAM, 1);
sanitize_e820_map(e820.map, ARRAY_SIZE(e820.map), &e820.nr_map);
}
/* called before trim_bios_range() to spare extra sanitize */
static void __init e820_add_kernel_range(void)
{
- u64 start = __pa_symbol(_text);
+ u64 start = __pa_symbol(ktla_ktva(_text));
u64 size = __pa_symbol(_end) - start;
/*
void __init setup_arch(char **cmdline_p)
{
+#ifdef CONFIG_X86_32
+ memblock_reserve(LOAD_PHYSICAL_ADDR, __pa_symbol(__bss_stop) - LOAD_PHYSICAL_ADDR);
+#else
memblock_reserve(__pa_symbol(_text),
(unsigned long)__bss_stop - (unsigned long)_text);
+#endif
early_reserve_initrd();
if (!boot_params.hdr.root_flags)
root_mountflags &= ~MS_RDONLY;
- init_mm.start_code = (unsigned long) _text;
- init_mm.end_code = (unsigned long) _etext;
+ init_mm.start_code = ktla_ktva((unsigned long) _text);
+ init_mm.end_code = ktla_ktva((unsigned long) _etext);
init_mm.end_data = (unsigned long) _edata;
init_mm.brk = _brk_end;
mpx_mm_init(&init_mm);
- code_resource.start = __pa_symbol(_text);
- code_resource.end = __pa_symbol(_etext)-1;
- data_resource.start = __pa_symbol(_etext);
+ code_resource.start = __pa_symbol(ktla_ktva(_text));
+ code_resource.end = __pa_symbol(ktla_ktva(_etext))-1;
+ data_resource.start = __pa_symbol(_sdata);
data_resource.end = __pa_symbol(_edata)-1;
bss_resource.start = __pa_symbol(__bss_start);
bss_resource.end = __pa_symbol(__bss_stop)-1;
#include <asm/cpu.h>
#include <asm/stackprotector.h>
-DEFINE_PER_CPU_READ_MOSTLY(int, cpu_number);
+#ifdef CONFIG_SMP
+DEFINE_PER_CPU_READ_MOSTLY(unsigned int, cpu_number);
EXPORT_PER_CPU_SYMBOL(cpu_number);
+#endif
-#ifdef CONFIG_X86_64
#define BOOT_PERCPU_OFFSET ((unsigned long)__per_cpu_load)
-#else
-#define BOOT_PERCPU_OFFSET 0
-#endif
DEFINE_PER_CPU_READ_MOSTLY(unsigned long, this_cpu_off) = BOOT_PERCPU_OFFSET;
EXPORT_PER_CPU_SYMBOL(this_cpu_off);
-unsigned long __per_cpu_offset[NR_CPUS] __read_mostly = {
+unsigned long __per_cpu_offset[NR_CPUS] __read_only = {
[0 ... NR_CPUS-1] = BOOT_PERCPU_OFFSET,
};
EXPORT_SYMBOL(__per_cpu_offset);
{
#ifdef CONFIG_NEED_MULTIPLE_NODES
pg_data_t *last = NULL;
- unsigned int cpu;
+ int cpu;
for_each_possible_cpu(cpu) {
int node = early_cpu_to_node(cpu);
{
#ifdef CONFIG_X86_32
struct desc_struct gdt;
+ unsigned long base = per_cpu_offset(cpu);
- pack_descriptor(&gdt, per_cpu_offset(cpu), 0xFFFFF,
- 0x2 | DESCTYPE_S, 0x8);
- gdt.s = 1;
+ pack_descriptor(&gdt, base, (VMALLOC_END - base - 1) >> PAGE_SHIFT,
+ 0x83 | DESCTYPE_S, 0xC);
write_gdt_entry(get_cpu_gdt_table(cpu),
GDT_ENTRY_PERCPU, &gdt, DESCTYPE_S);
#endif
/* alrighty, percpu areas up and running */
delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
for_each_possible_cpu(cpu) {
+#ifdef CONFIG_CC_STACKPROTECTOR
+#ifdef CONFIG_X86_32
+ unsigned long canary = per_cpu(stack_canary.canary, cpu);
+#endif
+#endif
per_cpu_offset(cpu) = delta + pcpu_unit_offsets[cpu];
per_cpu(this_cpu_off, cpu) = per_cpu_offset(cpu);
per_cpu(cpu_number, cpu) = cpu;
* So set them all (boot cpu and all APs).
*/
set_cpu_numa_node(cpu, early_cpu_to_node(cpu));
+#endif
+#ifdef CONFIG_CC_STACKPROTECTOR
+#ifdef CONFIG_X86_32
+ if (!cpu)
+ per_cpu(stack_canary.canary, cpu) = canary;
+#endif
#endif
/*
* Up to this point, the boot CPU has been using .init.data
* Align the stack pointer according to the i386 ABI,
* i.e. so that on function entry ((sp + 4) & 15) == 0.
*/
- sp = ((sp + 4) & -16ul) - 4;
+ sp = ((sp - 12) & -16ul) - 4;
#else /* !CONFIG_X86_32 */
sp = round_down(sp, 16) - 8;
#endif
}
if (current->mm->context.vdso)
- restorer = current->mm->context.vdso +
- selected_vdso32->sym___kernel_sigreturn;
+ restorer = (void __force_user *)(current->mm->context.vdso + selected_vdso32->sym___kernel_sigreturn);
else
- restorer = &frame->retcode;
+ restorer = (void __user *)&frame->retcode;
if (ksig->ka.sa.sa_flags & SA_RESTORER)
restorer = ksig->ka.sa.sa_restorer;
* reasons and because gdb uses it as a signature to notice
* signal handler stack frames.
*/
- err |= __put_user(*((u64 *)&retcode), (u64 *)frame->retcode);
+ err |= __put_user(*((u64 *)&retcode), (u64 __user *)frame->retcode);
if (err)
return -EFAULT;
save_altstack_ex(&frame->uc.uc_stack, regs->sp);
/* Set up to return from userspace. */
- restorer = current->mm->context.vdso +
- selected_vdso32->sym___kernel_rt_sigreturn;
+ if (current->mm->context.vdso)
+ restorer = (void __force_user *)(current->mm->context.vdso + selected_vdso32->sym___kernel_rt_sigreturn);
+ else
+ restorer = (void __user *)&frame->retcode;
if (ksig->ka.sa.sa_flags & SA_RESTORER)
restorer = ksig->ka.sa.sa_restorer;
put_user_ex(restorer, &frame->pretcode);
* reasons and because gdb uses it as a signature to notice
* signal handler stack frames.
*/
- put_user_ex(*((u64 *)&rt_retcode), (u64 *)frame->retcode);
+ put_user_ex(*((u64 *)&rt_retcode), (u64 __user *)frame->retcode);
} put_user_catch(err);
err |= copy_siginfo_to_user(&frame->info, &ksig->info);
{
int usig = signr_convert(ksig->sig);
sigset_t *set = sigmask_to_save();
- compat_sigset_t *cset = (compat_sigset_t *) set;
+ sigset_t sigcopy;
+ compat_sigset_t *cset;
+
+ sigcopy = *set;
+
+ cset = (compat_sigset_t *) &sigcopy;
/* Set up the stack frame */
if (is_ia32_frame()) {
} else if (is_x32_frame()) {
return x32_setup_rt_frame(ksig, cset, regs);
} else {
- return __setup_rt_frame(ksig->sig, ksig, set, regs);
+ return __setup_rt_frame(ksig->sig, ksig, &sigcopy, regs);
}
}
__setup("nonmi_ipi", nonmi_ipi_setup);
-struct smp_ops smp_ops = {
+struct smp_ops smp_ops __read_only = {
.smp_prepare_boot_cpu = native_smp_prepare_boot_cpu,
.smp_prepare_cpus = native_smp_prepare_cpus,
.smp_cpus_done = native_smp_cpus_done,
enable_start_cpu0 = 0;
-#ifdef CONFIG_X86_32
+ /* otherwise gcc will move up smp_processor_id before the cpu_init */
+ barrier();
+
/* switch away from the initial page table */
+#ifdef CONFIG_PAX_PER_CPU_PGD
+ load_cr3(get_cpu_pgd(smp_processor_id(), kernel));
+#else
load_cr3(swapper_pg_dir);
- __flush_tlb_all();
#endif
+ __flush_tlb_all();
- /* otherwise gcc will move up smp_processor_id before the cpu_init */
- barrier();
/*
* Check TSC synchronization with the BP:
*/
alternatives_enable_smp();
idle->thread.sp = (unsigned long) (((struct pt_regs *)
- (THREAD_SIZE + task_stack_page(idle))) - 1);
+ (THREAD_SIZE - 16 + task_stack_page(idle))) - 1);
per_cpu(current_task, cpu) = idle;
+ per_cpu(current_tinfo, cpu) = &idle->tinfo;
#ifdef CONFIG_X86_32
/* Stack for startup_32 can be just as for start_secondary onwards */
clear_tsk_thread_flag(idle, TIF_FORK);
initial_gs = per_cpu_offset(cpu);
#endif
- per_cpu(kernel_stack, cpu) =
- (unsigned long)task_stack_page(idle) -
- KERNEL_STACK_OFFSET + THREAD_SIZE;
+ per_cpu(kernel_stack, cpu) = (unsigned long)task_stack_page(idle) - 16 + THREAD_SIZE;
+ pax_open_kernel();
early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
+ pax_close_kernel();
initial_code = (unsigned long)start_secondary;
stack_start = idle->thread.sp;
/* the FPU context is blank, nobody can own it */
__cpu_disable_lazy_restore(cpu);
+#ifdef CONFIG_PAX_PER_CPU_PGD
+ clone_pgd_range(get_cpu_pgd(cpu, kernel) + KERNEL_PGD_BOUNDARY,
+ swapper_pg_dir + KERNEL_PGD_BOUNDARY,
+ KERNEL_PGD_PTRS);
+ clone_pgd_range(get_cpu_pgd(cpu, user) + KERNEL_PGD_BOUNDARY,
+ swapper_pg_dir + KERNEL_PGD_BOUNDARY,
+ KERNEL_PGD_PTRS);
+#endif
+
err = do_boot_cpu(apicid, cpu, tidle);
if (err) {
pr_err("do_boot_cpu failed(%d) to wakeup CPU#%u\n", err, cpu);
struct desc_struct *desc;
unsigned long base;
- seg &= ~7UL;
+ seg >>= 3;
mutex_lock(&child->mm->context.lock);
- if (unlikely((seg >> 3) >= child->mm->context.size))
+ if (unlikely(seg >= child->mm->context.size))
addr = -1L; /* bogus selector, access would fault */
else {
desc = child->mm->context.ldt + seg;
addr += base;
}
mutex_unlock(&child->mm->context.lock);
- }
+ } else if (seg == __KERNEL_CS || seg == __KERNEXEC_KERNEL_CS)
+ addr = ktla_ktva(addr);
return addr;
}
unsigned char opcode[15];
unsigned long addr = convert_ip_to_linear(child, regs);
+ if (addr == -EINVAL)
+ return 0;
+
copied = access_process_vm(child, addr, opcode, sizeof(opcode), 0);
for (i = 0; i < copied; i++) {
switch (opcode[i]) {
return error;
}
-static void find_start_end(unsigned long flags, unsigned long *begin,
- unsigned long *end)
+static void find_start_end(struct mm_struct *mm, unsigned long flags,
+ unsigned long *begin, unsigned long *end)
{
if (!test_thread_flag(TIF_ADDR32) && (flags & MAP_32BIT)) {
unsigned long new_begin;
*begin = new_begin;
}
} else {
- *begin = current->mm->mmap_legacy_base;
+ *begin = mm->mmap_legacy_base;
*end = TASK_SIZE;
}
}
struct vm_area_struct *vma;
struct vm_unmapped_area_info info;
unsigned long begin, end;
+ unsigned long offset = gr_rand_threadstack_offset(mm, filp, flags);
if (flags & MAP_FIXED)
return addr;
- find_start_end(flags, &begin, &end);
+ find_start_end(mm, flags, &begin, &end);
if (len > end)
return -ENOMEM;
+#ifdef CONFIG_PAX_RANDMMAP
+ if (!(mm->pax_flags & MF_PAX_RANDMMAP))
+#endif
+
if (addr) {
addr = PAGE_ALIGN(addr);
vma = find_vma(mm, addr);
- if (end - len >= addr &&
- (!vma || addr + len <= vma->vm_start))
+ if (end - len >= addr && check_heap_stack_gap(vma, addr, len, offset))
return addr;
}
info.high_limit = end;
info.align_mask = filp ? get_align_mask() : 0;
info.align_offset = pgoff << PAGE_SHIFT;
+ info.threadstack_offset = offset;
return vm_unmapped_area(&info);
}
struct mm_struct *mm = current->mm;
unsigned long addr = addr0;
struct vm_unmapped_area_info info;
+ unsigned long offset = gr_rand_threadstack_offset(mm, filp, flags);
/* requested length too big for entire address space */
if (len > TASK_SIZE)
if (!test_thread_flag(TIF_ADDR32) && (flags & MAP_32BIT))
goto bottomup;
+#ifdef CONFIG_PAX_RANDMMAP
+ if (!(mm->pax_flags & MF_PAX_RANDMMAP))
+#endif
+
/* requesting a specific address */
if (addr) {
addr = PAGE_ALIGN(addr);
vma = find_vma(mm, addr);
- if (TASK_SIZE - len >= addr &&
- (!vma || addr + len <= vma->vm_start))
+ if (TASK_SIZE - len >= addr && check_heap_stack_gap(vma, addr, len, offset))
return addr;
}
info.high_limit = mm->mmap_base;
info.align_mask = filp ? get_align_mask() : 0;
info.align_offset = pgoff << PAGE_SHIFT;
+ info.threadstack_offset = offset;
addr = vm_unmapped_area(&info);
if (!(addr & ~PAGE_MASK))
return addr;
void tboot_shutdown(u32 shutdown_type)
{
- void (*shutdown)(void);
+ void (* __noreturn shutdown)(void);
if (!tboot_enabled())
return;
switch_to_tboot_pt();
- shutdown = (void(*)(void))(unsigned long)tboot->shutdown_entry;
+ shutdown = (void *)(unsigned long)tboot->shutdown_entry;
shutdown();
/* should not reach here */
return -ENODEV;
}
-static atomic_t ap_wfs_count;
+static atomic_unchecked_t ap_wfs_count;
static int tboot_wait_for_aps(int num_aps)
{
{
switch (action) {
case CPU_DYING:
- atomic_inc(&ap_wfs_count);
+ atomic_inc_unchecked(&ap_wfs_count);
if (num_online_cpus() == 1)
- if (tboot_wait_for_aps(atomic_read(&ap_wfs_count)))
+ if (tboot_wait_for_aps(atomic_read_unchecked(&ap_wfs_count)))
return NOTIFY_BAD;
break;
}
tboot_create_trampoline();
- atomic_set(&ap_wfs_count, 0);
+ atomic_set_unchecked(&ap_wfs_count, 0);
register_hotcpu_notifier(&tboot_cpu_notifier);
#ifdef CONFIG_DEBUG_FS
{
unsigned long pc = instruction_pointer(regs);
- if (!user_mode_vm(regs) && in_lock_functions(pc)) {
+ if (!user_mode(regs) && in_lock_functions(pc)) {
#ifdef CONFIG_FRAME_POINTER
- return *(unsigned long *)(regs->bp + sizeof(long));
+ return ktla_ktva(*(unsigned long *)(regs->bp + sizeof(long)));
#else
unsigned long *sp =
(unsigned long *)kernel_stack_pointer(regs);
* or above a saved flags. Eflags has bits 22-31 zero,
* kernel addresses don't.
*/
+
+#ifdef CONFIG_PAX_KERNEXEC
+ return ktla_ktva(sp[0]);
+#else
if (sp[0] >> 22)
return sp[0];
if (sp[1] >> 22)
return sp[1];
+#endif
+
#endif
}
return pc;
if (idx < GDT_ENTRY_TLS_MIN || idx > GDT_ENTRY_TLS_MAX)
return -EINVAL;
+#ifdef CONFIG_PAX_SEGMEXEC
+ if ((p->mm->pax_flags & MF_PAX_SEGMEXEC) && (info.contents & MODIFY_LDT_CONTENTS_CODE))
+ return -EINVAL;
+#endif
+
set_tls_desc(p, idx, &info, 1);
return 0;
if (kbuf)
info = kbuf;
- else if (__copy_from_user(infobuf, ubuf, count))
+ else if (count > sizeof infobuf || __copy_from_user(infobuf, ubuf, count))
return -EFAULT;
else
info = infobuf;
#include <linux/atomic.h>
atomic_t trace_idt_ctr = ATOMIC_INIT(0);
-struct desc_ptr trace_idt_descr = { NR_VECTORS * 16 - 1,
+const struct desc_ptr trace_idt_descr = { NR_VECTORS * 16 - 1,
(unsigned long) trace_idt_table };
/* No need to be aligned, but done to keep all IDTs defined the same way. */
-gate_desc trace_idt_table[NR_VECTORS] __page_aligned_bss;
+gate_desc trace_idt_table[NR_VECTORS] __page_aligned_rodata;
static int trace_irq_vector_refcount;
static DEFINE_MUTEX(irq_vector_mutex);
#include <asm/proto.h>
/* No need to be aligned, but done to keep all IDTs defined the same way. */
-gate_desc debug_idt_table[NR_VECTORS] __page_aligned_bss;
+gate_desc debug_idt_table[NR_VECTORS] __page_aligned_rodata;
#else
#include <asm/processor-flags.h>
#include <asm/setup.h>
#endif
/* Must be page-aligned because the real IDT is used in a fixmap. */
-gate_desc idt_table[NR_VECTORS] __page_aligned_bss;
+gate_desc idt_table[NR_VECTORS] __page_aligned_rodata;
DECLARE_BITMAP(used_vectors, NR_VECTORS);
EXPORT_SYMBOL_GPL(used_vectors);
}
static nokprobe_inline int
-do_trap_no_signal(struct task_struct *tsk, int trapnr, char *str,
+do_trap_no_signal(struct task_struct *tsk, int trapnr, const char *str,
struct pt_regs *regs, long error_code)
{
#ifdef CONFIG_X86_32
- if (regs->flags & X86_VM_MASK) {
+ if (v8086_mode(regs)) {
/*
* Traps 0, 1, 3, 4, and 5 should be forwarded to vm86.
* On nmi (interrupt 2), do_trap should not be called.
return -1;
}
#endif
- if (!user_mode(regs)) {
+ if (!user_mode_novm(regs)) {
if (!fixup_exception(regs)) {
tsk->thread.error_code = error_code;
tsk->thread.trap_nr = trapnr;
+
+#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_KERNEXEC)
+ if (trapnr == X86_TRAP_SS && ((regs->cs & 0xFFFF) == __KERNEL_CS || (regs->cs & 0xFFFF) == __KERNEXEC_KERNEL_CS))
+ str = "PAX: suspicious stack segment fault";
+#endif
+
die(str, regs, error_code);
}
+
+#ifdef CONFIG_PAX_REFCOUNT
+ if (trapnr == X86_TRAP_OF)
+ pax_report_refcount_overflow(regs);
+#endif
+
return 0;
}
}
static void
-do_trap(int trapnr, int signr, char *str, struct pt_regs *regs,
+do_trap(int trapnr, int signr, const char *str, struct pt_regs *regs,
long error_code, siginfo_t *info)
{
struct task_struct *tsk = current;
if (show_unhandled_signals && unhandled_signal(tsk, signr) &&
printk_ratelimit()) {
pr_info("%s[%d] trap %s ip:%lx sp:%lx error:%lx",
- tsk->comm, tsk->pid, str,
+ tsk->comm, task_pid_nr(tsk), str,
regs->ip, regs->sp, error_code);
print_vma_addr(" in ", regs->ip);
pr_cont("\n");
tsk->thread.error_code = error_code;
tsk->thread.trap_nr = X86_TRAP_DF;
+#ifdef CONFIG_GRKERNSEC_KSTACKOVERFLOW
+ if ((unsigned long)tsk->stack - regs->sp <= PAGE_SIZE)
+ die("grsec: kernel stack overflow detected", regs, error_code);
+#endif
+
#ifdef CONFIG_DOUBLEFAULT
df_debug(regs, error_code);
#endif
conditional_sti(regs);
#ifdef CONFIG_X86_32
- if (regs->flags & X86_VM_MASK) {
+ if (v8086_mode(regs)) {
local_irq_enable();
handle_vm86_fault((struct kernel_vm86_regs *) regs, error_code);
goto exit;
#endif
tsk = current;
- if (!user_mode(regs)) {
+ if (!user_mode_novm(regs)) {
if (fixup_exception(regs))
goto exit;
tsk->thread.error_code = error_code;
tsk->thread.trap_nr = X86_TRAP_GP;
if (notify_die(DIE_GPF, "general protection fault", regs, error_code,
- X86_TRAP_GP, SIGSEGV) != NOTIFY_STOP)
+ X86_TRAP_GP, SIGSEGV) != NOTIFY_STOP) {
+
+#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_KERNEXEC)
+ if ((regs->cs & 0xFFFF) == __KERNEL_CS || (regs->cs & 0xFFFF) == __KERNEXEC_KERNEL_CS)
+ die("PAX: suspicious general protection fault", regs, error_code);
+ else
+#endif
+
die("general protection fault", regs, error_code);
+ }
goto exit;
}
+#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_PAGEEXEC)
+ if (!(__supported_pte_mask & _PAGE_NX) && tsk->mm && (tsk->mm->pax_flags & MF_PAX_PAGEEXEC)) {
+ struct mm_struct *mm = tsk->mm;
+ unsigned long limit;
+
+ down_write(&mm->mmap_sem);
+ limit = mm->context.user_cs_limit;
+ if (limit < TASK_SIZE) {
+ track_exec_limit(mm, limit, TASK_SIZE, VM_EXEC);
+ up_write(&mm->mmap_sem);
+ return;
+ }
+ up_write(&mm->mmap_sem);
+ }
+#endif
+
tsk->thread.error_code = error_code;
tsk->thread.trap_nr = X86_TRAP_GP;
container_of(task_pt_regs(current),
struct bad_iret_stack, regs);
+ if ((current->thread.sp0 ^ (unsigned long)s) < THREAD_SIZE)
+ new_stack = s;
+
/* Copy the IRET target to the new stack. */
memmove(&new_stack->regs.ip, (void *)s->regs.sp, 5*8);
/* Copy the remainder of the stack from the current stack. */
memmove(new_stack, s, offsetof(struct bad_iret_stack, regs.ip));
- BUG_ON(!user_mode_vm(&new_stack->regs));
+ BUG_ON(!user_mode(&new_stack->regs));
return new_stack;
}
NOKPROBE_SYMBOL(fixup_bad_iret);
/* It's safe to allow irq's after DR6 has been saved */
preempt_conditional_sti(regs);
- if (regs->flags & X86_VM_MASK) {
+ if (v8086_mode(regs)) {
handle_vm86_trap((struct kernel_vm86_regs *) regs, error_code,
X86_TRAP_DB);
preempt_conditional_cli(regs);
* We already checked v86 mode above, so we can check for kernel mode
* by just checking the CPL of CS.
*/
- if ((dr6 & DR_STEP) && !user_mode(regs)) {
+ if ((dr6 & DR_STEP) && !user_mode_novm(regs)) {
tsk->thread.debugreg6 &= ~DR_STEP;
set_tsk_thread_flag(tsk, TIF_SINGLESTEP);
regs->flags &= ~X86_EFLAGS_TF;
return;
conditional_sti(regs);
- if (!user_mode_vm(regs))
+ if (!user_mode(regs))
{
if (!fixup_exception(regs)) {
task->thread.error_code = error_code;
*/
smp_wmb();
- ACCESS_ONCE(c2n->head) = data;
+ ACCESS_ONCE_RW(c2n->head) = data;
}
/*
int ret = NOTIFY_DONE;
/* We are only interested in userspace traps */
- if (regs && !user_mode_vm(regs))
+ if (regs && !user_mode(regs))
return NOTIFY_DONE;
switch (val) {
if (nleft != rasize) {
pr_err("uprobe: return address clobbered: pid=%d, %%sp=%#lx, "
- "%%ip=%#lx\n", current->pid, regs->sp, regs->ip);
+ "%%ip=%#lx\n", task_pid_nr(current), regs->sp, regs->ip);
force_sig_info(SIGSEGV, SEND_SIG_FORCED, current);
}
* arch/x86/boot/compressed/head_64.S: Boot cpu verification
* arch/x86/kernel/trampoline_64.S: secondary processor verification
* arch/x86/kernel/head_32.S: processor startup
+ * arch/x86/kernel/acpi/realmode/wakeup.S: 32bit processor resume
*
* verify_cpu, returns the status of longmode and SSE in register %eax.
* 0: Success 1: Failure
#include <linux/ptrace.h>
#include <linux/audit.h>
#include <linux/stddef.h>
+#include <linux/grsecurity.h>
#include <asm/uaccess.h>
#include <asm/io.h>
do_exit(SIGSEGV);
}
- tss = &per_cpu(init_tss, get_cpu());
+ tss = init_tss + get_cpu();
current->thread.sp0 = current->thread.saved_sp0;
current->thread.sysenter_cs = __KERNEL_CS;
load_sp0(tss, ¤t->thread);
if (tsk->thread.saved_sp0)
return -EPERM;
+
+#ifdef CONFIG_GRKERNSEC_VM86
+ if (!capable(CAP_SYS_RAWIO)) {
+ gr_handle_vm86();
+ return -EPERM;
+ }
+#endif
+
tmp = copy_vm86_regs_from_user(&info.regs, &v86->regs,
offsetof(struct kernel_vm86_struct, vm86plus) -
sizeof(info.regs));
int tmp;
struct vm86plus_struct __user *v86;
+#ifdef CONFIG_GRKERNSEC_VM86
+ if (!capable(CAP_SYS_RAWIO)) {
+ gr_handle_vm86();
+ return -EPERM;
+ }
+#endif
+
tsk = current;
switch (cmd) {
case VM86_REQUEST_IRQ:
tsk->thread.saved_fs = info->regs32->fs;
tsk->thread.saved_gs = get_user_gs(info->regs32);
- tss = &per_cpu(init_tss, get_cpu());
+ tss = init_tss + get_cpu();
tsk->thread.sp0 = (unsigned long) &info->VM86_TSS_ESP0;
if (cpu_has_sep)
tsk->thread.sysenter_cs = 0;
goto cannot_handle;
if (i == 0x21 && is_revectored(AH(regs), &KVM86->int21_revectored))
goto cannot_handle;
- intr_ptr = (unsigned long __user *) (i << 2);
+ intr_ptr = (__force unsigned long __user *) (i << 2);
if (get_user(segoffs, intr_ptr))
goto cannot_handle;
if ((segoffs >> 16) == BIOSSEG)
#include <asm/page_types.h>
#include <asm/cache.h>
#include <asm/boot.h>
+#include <asm/segment.h>
+
+#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_KERNEXEC)
+#define __KERNEL_TEXT_OFFSET (LOAD_OFFSET + ____LOAD_PHYSICAL_ADDR)
+#else
+#define __KERNEL_TEXT_OFFSET 0
+#endif
#undef i386 /* in case the preprocessor is a 32bit one */
PHDRS {
text PT_LOAD FLAGS(5); /* R_E */
+#ifdef CONFIG_X86_32
+ module PT_LOAD FLAGS(5); /* R_E */
+#endif
+#ifdef CONFIG_XEN
+ rodata PT_LOAD FLAGS(5); /* R_E */
+#else
+ rodata PT_LOAD FLAGS(4); /* R__ */
+#endif
data PT_LOAD FLAGS(6); /* RW_ */
-#ifdef CONFIG_X86_64
+ init.begin PT_LOAD FLAGS(6); /* RW_ */
#ifdef CONFIG_SMP
percpu PT_LOAD FLAGS(6); /* RW_ */
#endif
+ text.init PT_LOAD FLAGS(5); /* R_E */
+ text.exit PT_LOAD FLAGS(5); /* R_E */
init PT_LOAD FLAGS(7); /* RWE */
-#endif
note PT_NOTE FLAGS(0); /* ___ */
}
SECTIONS
{
#ifdef CONFIG_X86_32
- . = LOAD_OFFSET + LOAD_PHYSICAL_ADDR;
- phys_startup_32 = startup_32 - LOAD_OFFSET;
+ . = LOAD_OFFSET + ____LOAD_PHYSICAL_ADDR;
#else
- . = __START_KERNEL;
- phys_startup_64 = startup_64 - LOAD_OFFSET;
+ . = __START_KERNEL;
#endif
/* Text and read-only data */
- .text : AT(ADDR(.text) - LOAD_OFFSET) {
- _text = .;
+ .text (. - __KERNEL_TEXT_OFFSET): AT(ADDR(.text) - LOAD_OFFSET + __KERNEL_TEXT_OFFSET) {
/* bootstrapping code */
+#ifdef CONFIG_X86_32
+ phys_startup_32 = startup_32 - LOAD_OFFSET + __KERNEL_TEXT_OFFSET;
+#else
+ phys_startup_64 = startup_64 - LOAD_OFFSET + __KERNEL_TEXT_OFFSET;
+#endif
+ __LOAD_PHYSICAL_ADDR = . - LOAD_OFFSET + __KERNEL_TEXT_OFFSET;
+ _text = .;
HEAD_TEXT
. = ALIGN(8);
_stext = .;
IRQENTRY_TEXT
*(.fixup)
*(.gnu.warning)
- /* End of text section */
- _etext = .;
} :text = 0x9090
- NOTES :text :note
+ . += __KERNEL_TEXT_OFFSET;
+
+#ifdef CONFIG_X86_32
+ . = ALIGN(PAGE_SIZE);
+ .module.text : AT(ADDR(.module.text) - LOAD_OFFSET) {
+
+#ifdef CONFIG_PAX_KERNEXEC
+ MODULES_EXEC_VADDR = .;
+ BYTE(0)
+ . += (CONFIG_PAX_KERNEXEC_MODULE_TEXT * 1024 * 1024);
+ . = ALIGN(HPAGE_SIZE) - 1;
+ MODULES_EXEC_END = .;
+#endif
+
+ } :module
+#endif
- EXCEPTION_TABLE(16) :text = 0x9090
+ .text.end : AT(ADDR(.text.end) - LOAD_OFFSET) {
+ /* End of text section */
+ BYTE(0)
+ _etext = . - __KERNEL_TEXT_OFFSET;
+ }
+
+#ifdef CONFIG_X86_32
+ . = ALIGN(PAGE_SIZE);
+ .rodata.page_aligned : AT(ADDR(.rodata.page_aligned) - LOAD_OFFSET) {
+ . = ALIGN(PAGE_SIZE);
+ *(.empty_zero_page)
+ *(.initial_pg_fixmap)
+ *(.initial_pg_pmd)
+ *(.initial_page_table)
+ *(.swapper_pg_dir)
+ } :rodata
+#endif
+
+ . = ALIGN(PAGE_SIZE);
+ NOTES :rodata :note
+
+ EXCEPTION_TABLE(16) :rodata
#if defined(CONFIG_DEBUG_RODATA)
/* .text should occupy whole number of pages */
/* Data */
.data : AT(ADDR(.data) - LOAD_OFFSET) {
+
+#ifdef CONFIG_PAX_KERNEXEC
+ . = ALIGN(HPAGE_SIZE);
+#else
+ . = ALIGN(PAGE_SIZE);
+#endif
+
/* Start of data section */
_sdata = .;
/* init_task */
INIT_TASK_DATA(THREAD_SIZE)
-#ifdef CONFIG_X86_32
- /* 32 bit has nosave before _edata */
NOSAVE_DATA
-#endif
PAGE_ALIGNED_DATA(PAGE_SIZE)
. = ALIGN(__vvar_page + PAGE_SIZE, PAGE_SIZE);
/* Init code and data - will be freed after init */
- . = ALIGN(PAGE_SIZE);
.init.begin : AT(ADDR(.init.begin) - LOAD_OFFSET) {
+ BYTE(0)
+
+#ifdef CONFIG_PAX_KERNEXEC
+ . = ALIGN(HPAGE_SIZE);
+#else
+ . = ALIGN(PAGE_SIZE);
+#endif
+
__init_begin = .; /* paired with __init_end */
- }
+ } :init.begin
-#if defined(CONFIG_X86_64) && defined(CONFIG_SMP)
+#ifdef CONFIG_SMP
/*
* percpu offsets are zero-based on SMP. PERCPU_VADDR() changes the
* output PHDR, so the next output section - .init.text - should
"per-CPU data too large - increase CONFIG_PHYSICAL_START")
#endif
- INIT_TEXT_SECTION(PAGE_SIZE)
-#ifdef CONFIG_X86_64
- :init
-#endif
+ . = ALIGN(PAGE_SIZE);
+ init_begin = .;
+ .init.text (. - __KERNEL_TEXT_OFFSET): AT(init_begin - LOAD_OFFSET) {
+ VMLINUX_SYMBOL(_sinittext) = .;
+ INIT_TEXT
+ . = ALIGN(PAGE_SIZE);
+ } :text.init
- INIT_DATA_SECTION(16)
+ /*
+ * .exit.text is discard at runtime, not link time, to deal with
+ * references from .altinstructions and .eh_frame
+ */
+ .exit.text : AT(ADDR(.exit.text) - LOAD_OFFSET + __KERNEL_TEXT_OFFSET) {
+ EXIT_TEXT
+ VMLINUX_SYMBOL(_einittext) = .;
+ . = ALIGN(16);
+ } :text.exit
+ . = init_begin + SIZEOF(.init.text) + SIZEOF(.exit.text);
+
+ . = ALIGN(PAGE_SIZE);
+ INIT_DATA_SECTION(16) :init
.x86_cpu_dev.init : AT(ADDR(.x86_cpu_dev.init) - LOAD_OFFSET) {
__x86_cpu_dev_start = .;
}
. = ALIGN(8);
- /*
- * .exit.text is discard at runtime, not link time, to deal with
- * references from .altinstructions and .eh_frame
- */
- .exit.text : AT(ADDR(.exit.text) - LOAD_OFFSET) {
- EXIT_TEXT
- }
.exit.data : AT(ADDR(.exit.data) - LOAD_OFFSET) {
EXIT_DATA
}
-#if !defined(CONFIG_X86_64) || !defined(CONFIG_SMP)
+#ifndef CONFIG_SMP
PERCPU_SECTION(INTERNODE_CACHE_BYTES)
#endif
.smp_locks : AT(ADDR(.smp_locks) - LOAD_OFFSET) {
__smp_locks = .;
*(.smp_locks)
- . = ALIGN(PAGE_SIZE);
__smp_locks_end = .;
+ . = ALIGN(PAGE_SIZE);
}
-#ifdef CONFIG_X86_64
- .data_nosave : AT(ADDR(.data_nosave) - LOAD_OFFSET) {
- NOSAVE_DATA
- }
-#endif
-
/* BSS */
. = ALIGN(PAGE_SIZE);
.bss : AT(ADDR(.bss) - LOAD_OFFSET) {
__brk_base = .;
. += 64 * 1024; /* 64k alignment slop space */
*(.brk_reservation) /* areas brk users have reserved */
+ . = ALIGN(HPAGE_SIZE);
__brk_limit = .;
}
* for the boot processor.
*/
#define INIT_PER_CPU(x) init_per_cpu__##x = x + __per_cpu_load
-INIT_PER_CPU(gdt_page);
INIT_PER_CPU(irq_stack_union);
/*
* Build-time check on the image size:
*/
-. = ASSERT((_end - _text <= KERNEL_IMAGE_SIZE),
+. = ASSERT((_end - _text - __KERNEL_TEXT_OFFSET <= KERNEL_IMAGE_SIZE),
"kernel image bigger than KERNEL_IMAGE_SIZE");
#ifdef CONFIG_SMP
#define CREATE_TRACE_POINTS
#include "vsyscall_trace.h"
-static enum { EMULATE, NATIVE, NONE } vsyscall_mode = EMULATE;
+static enum { EMULATE, NONE } vsyscall_mode = EMULATE;
static int __init vsyscall_setup(char *str)
{
if (str) {
if (!strcmp("emulate", str))
vsyscall_mode = EMULATE;
- else if (!strcmp("native", str))
- vsyscall_mode = NATIVE;
else if (!strcmp("none", str))
vsyscall_mode = NONE;
else
return true;
sigsegv:
- force_sig(SIGSEGV, current);
- return true;
+ do_group_exit(SIGKILL);
}
/*
static struct vm_area_struct gate_vma = {
.vm_start = VSYSCALL_ADDR,
.vm_end = VSYSCALL_ADDR + PAGE_SIZE,
- .vm_page_prot = PAGE_READONLY_EXEC,
- .vm_flags = VM_READ | VM_EXEC,
+ .vm_page_prot = PAGE_READONLY,
+ .vm_flags = VM_READ,
.vm_ops = &gate_vma_ops,
};
unsigned long physaddr_vsyscall = __pa_symbol(&__vsyscall_page);
if (vsyscall_mode != NONE)
- __set_fixmap(VSYSCALL_PAGE, physaddr_vsyscall,
- vsyscall_mode == NATIVE
- ? PAGE_KERNEL_VSYSCALL
- : PAGE_KERNEL_VVAR);
+ __set_fixmap(VSYSCALL_PAGE, physaddr_vsyscall, PAGE_KERNEL_VVAR);
BUILD_BUG_ON((unsigned long)__fix_to_virt(VSYSCALL_PAGE) !=
(unsigned long)VSYSCALL_ADDR);
EXPORT_SYMBOL(copy_user_generic_unrolled);
EXPORT_SYMBOL(copy_user_enhanced_fast_string);
EXPORT_SYMBOL(__copy_user_nocache);
-EXPORT_SYMBOL(_copy_from_user);
-EXPORT_SYMBOL(_copy_to_user);
EXPORT_SYMBOL(copy_page);
EXPORT_SYMBOL(clear_page);
EXPORT_SYMBOL(___preempt_schedule_context);
#endif
#endif
+
+#ifdef CONFIG_PAX_PER_CPU_PGD
+EXPORT_SYMBOL(cpu_pgd);
+#endif
static void default_nmi_init(void) { };
static int default_i8042_detect(void) { return 1; };
-struct x86_platform_ops x86_platform = {
+struct x86_platform_ops x86_platform __read_only = {
.calibrate_tsc = native_calibrate_tsc,
.get_wallclock = mach_get_cmos_time,
.set_wallclock = mach_set_rtc_mmss,
EXPORT_SYMBOL_GPL(x86_platform);
#if defined(CONFIG_PCI_MSI)
-struct x86_msi_ops x86_msi = {
+struct x86_msi_ops x86_msi __read_only = {
.setup_msi_irqs = native_setup_msi_irqs,
.compose_msi_msg = native_compose_msi_msg,
.teardown_msi_irq = native_teardown_msi_irq,
}
#endif
-struct x86_io_apic_ops x86_io_apic_ops = {
+struct x86_io_apic_ops x86_io_apic_ops __read_only = {
.init = native_io_apic_init_mappings,
.read = native_io_apic_read,
.write = native_io_apic_write,
/* Setup the bytes not touched by the [f]xsave and reserved for SW. */
sw_bytes = ia32_frame ? &fx_sw_reserved_ia32 : &fx_sw_reserved;
- err = __copy_to_user(&x->i387.sw_reserved, sw_bytes, sizeof(*sw_bytes));
+ err = __copy_to_user(x->i387.sw_reserved, sw_bytes, sizeof(*sw_bytes));
if (!use_xsave())
return err;
- err |= __put_user(FP_XSTATE_MAGIC2, (__u32 *)(buf + xstate_size));
+ err |= __put_user(FP_XSTATE_MAGIC2, (__u32 __user *)(buf + xstate_size));
/*
* Read the xstate_bv which we copied (directly from the cpu or
* from the state in task struct) to the user buffers.
*/
- err |= __get_user(xstate_bv, (__u32 *)&x->xsave_hdr.xstate_bv);
+ err |= __get_user(xstate_bv, (__u32 __user *)&x->xsave_hdr.xstate_bv);
/*
* For legacy compatible, we always set FP/SSE bits in the bit
*/
xstate_bv |= XSTATE_FPSSE;
- err |= __put_user(xstate_bv, (__u32 *)&x->xsave_hdr.xstate_bv);
+ err |= __put_user(xstate_bv, (__u32 __user *)&x->xsave_hdr.xstate_bv);
return err;
}
{
int err;
+ buf = (struct xsave_struct __user *)____m(buf);
if (use_xsave())
err = xsave_user(buf);
else if (use_fxsr())
*/
static inline int restore_user_xstate(void __user *buf, u64 xbv, int fx_only)
{
+ buf = (void __user *)____m(buf);
if (use_xsave()) {
if ((unsigned long)buf % 64 || fx_only) {
u64 init_bv = pcntxt_mask & ~XSTATE_FPSSE;
struct kvm_cpuid2 *cpuid,
struct kvm_cpuid_entry2 __user *entries)
{
- int r;
+ int r, i;
r = -E2BIG;
if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
goto out;
r = -EFAULT;
- if (copy_from_user(&vcpu->arch.cpuid_entries, entries,
- cpuid->nent * sizeof(struct kvm_cpuid_entry2)))
+ if (!access_ok(VERIFY_READ, entries, cpuid->nent * sizeof(struct kvm_cpuid_entry2)))
goto out;
+ for (i = 0; i < cpuid->nent; ++i) {
+ struct kvm_cpuid_entry2 cpuid_entry;
+ if (__copy_from_user(&cpuid_entry, entries + i, sizeof(cpuid_entry)))
+ goto out;
+ vcpu->arch.cpuid_entries[i] = cpuid_entry;
+ }
vcpu->arch.cpuid_nent = cpuid->nent;
kvm_apic_set_version(vcpu);
kvm_x86_ops->cpuid_update(vcpu);
struct kvm_cpuid2 *cpuid,
struct kvm_cpuid_entry2 __user *entries)
{
- int r;
+ int r, i;
r = -E2BIG;
if (cpuid->nent < vcpu->arch.cpuid_nent)
goto out;
r = -EFAULT;
- if (copy_to_user(entries, &vcpu->arch.cpuid_entries,
- vcpu->arch.cpuid_nent * sizeof(struct kvm_cpuid_entry2)))
+ if (!access_ok(VERIFY_WRITE, entries, vcpu->arch.cpuid_nent * sizeof(struct kvm_cpuid_entry2)))
goto out;
+ for (i = 0; i < vcpu->arch.cpuid_nent; ++i) {
+ struct kvm_cpuid_entry2 cpuid_entry = vcpu->arch.cpuid_entries[i];
+ if (__copy_to_user(entries + i, &cpuid_entry, sizeof(cpuid_entry)))
+ goto out;
+ }
return 0;
out:
int cr = ctxt->modrm_reg;
u64 efer = 0;
- static u64 cr_reserved_bits[] = {
+ static const u64 cr_reserved_bits[] = {
0xffffffff00000000ULL,
0, 0, 0, /* CR3 checked later */
CR4_RESERVED_BITS,
#define APIC_BUS_CYCLE_NS 1
/* #define apic_debug(fmt,arg...) printk(KERN_WARNING fmt,##arg) */
-#define apic_debug(fmt, arg...)
+#define apic_debug(fmt, arg...) do {} while (0)
#define APIC_LVT_NUM 6
/* 14 is the version for Xeon and Pentium 8.4.8*/
if (unlikely(kvm_is_error_hva(host_addr)))
goto error;
- ptep_user = (pt_element_t __user *)((void *)host_addr + offset);
+ ptep_user = (pt_element_t __force_user *)((void *)host_addr + offset);
if (unlikely(__copy_from_user(&pte, ptep_user, sizeof(pte))))
goto error;
walker->ptep_user[walker->level - 1] = ptep_user;
int cpu = raw_smp_processor_id();
struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
+
+ pax_open_kernel();
sd->tss_desc->type = 9; /* available 32/64-bit TSS */
+ pax_close_kernel();
+
load_TR_desc();
}
#endif
#endif
+#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_MEMORY_UDEREF)
+ __set_fs(current_thread_info()->addr_limit);
+#endif
+
reload_tss(vcpu);
local_irq_disable();
#endif
}
-static void vmcs_clear_bits(unsigned long field, u32 mask)
+static void vmcs_clear_bits(unsigned long field, unsigned long mask)
{
vmcs_writel(field, vmcs_readl(field) & ~mask);
}
-static void vmcs_set_bits(unsigned long field, u32 mask)
+static void vmcs_set_bits(unsigned long field, unsigned long mask)
{
vmcs_writel(field, vmcs_readl(field) | mask);
}
struct desc_struct *descs;
descs = (void *)gdt->address;
+
+ pax_open_kernel();
descs[GDT_ENTRY_TSS].type = 9; /* available TSS */
+ pax_close_kernel();
+
load_TR_desc();
}
vmcs_writel(HOST_TR_BASE, kvm_read_tr_base()); /* 22.2.4 */
vmcs_writel(HOST_GDTR_BASE, gdt->address); /* 22.2.4 */
+#ifdef CONFIG_PAX_PER_CPU_PGD
+ vmcs_writel(HOST_CR3, read_cr3()); /* 22.2.3 FIXME: shadow tables */
+#endif
+
rdmsrl(MSR_IA32_SYSENTER_ESP, sysenter_esp);
vmcs_writel(HOST_IA32_SYSENTER_ESP, sysenter_esp); /* 22.2.3 */
vmx->loaded_vmcs->cpu = cpu;
* reads and returns guest's timestamp counter "register"
* guest_tsc = host_tsc + tsc_offset -- 21.3
*/
-static u64 guest_read_tsc(void)
+static u64 __intentional_overflow(-1) guest_read_tsc(void)
{
u64 host_tsc, tsc_offset;
unsigned long cr4;
vmcs_writel(HOST_CR0, read_cr0() & ~X86_CR0_TS); /* 22.2.3 */
+
+#ifndef CONFIG_PAX_PER_CPU_PGD
vmcs_writel(HOST_CR3, read_cr3()); /* 22.2.3 FIXME: shadow tables */
+#endif
/* Save the most likely value for this task's CR4 in the VMCS. */
cr4 = read_cr4();
vmcs_writel(HOST_IDTR_BASE, dt.address); /* 22.2.4 */
vmx->host_idt_base = dt.address;
- vmcs_writel(HOST_RIP, vmx_return); /* 22.2.5 */
+ vmcs_writel(HOST_RIP, ktla_ktva(vmx_return)); /* 22.2.5 */
rdmsr(MSR_IA32_SYSENTER_CS, low32, high32);
vmcs_write32(HOST_IA32_SYSENTER_CS, low32);
* page upon invalidation. No need to do anything if the
* processor does not have the APIC_ACCESS_ADDR VMCS field.
*/
- kvm_x86_ops->set_apic_access_page_addr = NULL;
+ pax_open_kernel();
+ *(void **)&kvm_x86_ops->set_apic_access_page_addr = NULL;
+ pax_close_kernel();
}
- if (!cpu_has_vmx_tpr_shadow())
- kvm_x86_ops->update_cr8_intercept = NULL;
+ if (!cpu_has_vmx_tpr_shadow()) {
+ pax_open_kernel();
+ *(void **)&kvm_x86_ops->update_cr8_intercept = NULL;
+ pax_close_kernel();
+ }
if (enable_ept && !cpu_has_vmx_ept_2m_page())
kvm_disable_largepages();
if (!cpu_has_vmx_apicv())
enable_apicv = 0;
+ pax_open_kernel();
if (enable_apicv)
- kvm_x86_ops->update_cr8_intercept = NULL;
+ *(void **)&kvm_x86_ops->update_cr8_intercept = NULL;
else {
- kvm_x86_ops->hwapic_irr_update = NULL;
- kvm_x86_ops->deliver_posted_interrupt = NULL;
- kvm_x86_ops->sync_pir_to_irr = vmx_sync_pir_to_irr_dummy;
+ *(void **)&kvm_x86_ops->hwapic_irr_update = NULL;
+ *(void **)&kvm_x86_ops->deliver_posted_interrupt = NULL;
+ *(void **)&kvm_x86_ops->sync_pir_to_irr = vmx_sync_pir_to_irr_dummy;
}
+ pax_close_kernel();
if (nested)
nested_vmx_setup_ctls_msrs();
"jmp 2f \n\t"
"1: " __ex(ASM_VMX_VMRESUME) "\n\t"
"2: "
+
+#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_KERNEXEC)
+ "ljmp %[cs],$3f\n\t"
+ "3: "
+#endif
+
/* Save guest registers, load host registers, keep flags */
"mov %0, %c[wordsize](%%" _ASM_SP ") \n\t"
"pop %0 \n\t"
#endif
[cr2]"i"(offsetof(struct vcpu_vmx, vcpu.arch.cr2)),
[wordsize]"i"(sizeof(ulong))
+
+#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_KERNEXEC)
+ ,[cs]"i"(__KERNEL_CS)
+#endif
+
: "cc", "memory"
#ifdef CONFIG_X86_64
, "rax", "rbx", "rdi", "rsi"
if (debugctlmsr)
update_debugctlmsr(debugctlmsr);
-#ifndef CONFIG_X86_64
+#ifdef CONFIG_X86_32
/*
* The sysexit path does not restore ds/es, so we must set them to
* a reasonable value ourselves.
* may be executed in interrupt context, which saves and restore segments
* around it, nullifying its effect.
*/
- loadsegment(ds, __USER_DS);
- loadsegment(es, __USER_DS);
+ loadsegment(ds, __KERNEL_DS);
+ loadsegment(es, __KERNEL_DS);
+ loadsegment(ss, __KERNEL_DS);
+
+#ifdef CONFIG_PAX_KERNEXEC
+ loadsegment(fs, __KERNEL_PERCPU);
+#endif
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ __set_fs(current_thread_info()->addr_limit);
+#endif
+
#endif
vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP)
{
struct kvm *kvm = vcpu->kvm;
int lm = is_long_mode(vcpu);
- u8 *blob_addr = lm ? (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_64
- : (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_32;
+ u8 __user *blob_addr = lm ? (u8 __user *)(long)kvm->arch.xen_hvm_config.blob_addr_64
+ : (u8 __user *)(long)kvm->arch.xen_hvm_config.blob_addr_32;
u8 blob_size = lm ? kvm->arch.xen_hvm_config.blob_size_64
: kvm->arch.xen_hvm_config.blob_size_32;
u32 page_num = data & ~PAGE_MASK;
if (n < msr_list.nmsrs)
goto out;
r = -EFAULT;
+ if (num_msrs_to_save > ARRAY_SIZE(msrs_to_save))
+ goto out;
if (copy_to_user(user_msr_list->indices, &msrs_to_save,
num_msrs_to_save * sizeof(u32)))
goto out;
};
#endif
-int kvm_arch_init(void *opaque)
+int kvm_arch_init(const void *opaque)
{
int r;
struct kvm_x86_ops *ops = opaque;
* Rebooting also tells the Host we're finished, but the RESTART flag tells the
* Launcher to reboot us.
*/
-static void lguest_restart(char *reason)
+static __noreturn void lguest_restart(char *reason)
{
hcall(LHCALL_SHUTDOWN, __pa(reason), LGUEST_SHUTDOWN_RESTART, 0, 0);
+ BUG();
}
/*G:050
movl (v), %eax
movl 4(v), %edx
RET_ENDP
+BEGIN(read_unchecked)
+ movl (v), %eax
+ movl 4(v), %edx
+RET_ENDP
#undef v
#define v %esi
movl %ebx, (v)
movl %ecx, 4(v)
RET_ENDP
+BEGIN(set_unchecked)
+ movl %ebx, (v)
+ movl %ecx, 4(v)
+RET_ENDP
#undef v
#define v %esi
BEGIN(add)
addl %eax, (v)
adcl %edx, 4(v)
+
+#ifdef CONFIG_PAX_REFCOUNT
+ jno 0f
+ subl %eax, (v)
+ sbbl %edx, 4(v)
+ int $4
+0:
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+RET_ENDP
+BEGIN(add_unchecked)
+ addl %eax, (v)
+ adcl %edx, 4(v)
RET_ENDP
#undef v
BEGIN(add_return)
addl (v), %eax
adcl 4(v), %edx
+
+#ifdef CONFIG_PAX_REFCOUNT
+ into
+1234:
+ _ASM_EXTABLE(1234b, 2f)
+#endif
+
+ movl %eax, (v)
+ movl %edx, 4(v)
+
+#ifdef CONFIG_PAX_REFCOUNT
+2:
+#endif
+
+RET_ENDP
+BEGIN(add_return_unchecked)
+ addl (v), %eax
+ adcl 4(v), %edx
movl %eax, (v)
movl %edx, 4(v)
RET_ENDP
BEGIN(sub)
subl %eax, (v)
sbbl %edx, 4(v)
+
+#ifdef CONFIG_PAX_REFCOUNT
+ jno 0f
+ addl %eax, (v)
+ adcl %edx, 4(v)
+ int $4
+0:
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+RET_ENDP
+BEGIN(sub_unchecked)
+ subl %eax, (v)
+ sbbl %edx, 4(v)
RET_ENDP
#undef v
sbbl $0, %edx
addl (v), %eax
adcl 4(v), %edx
+
+#ifdef CONFIG_PAX_REFCOUNT
+ into
+1234:
+ _ASM_EXTABLE(1234b, 2f)
+#endif
+
+ movl %eax, (v)
+ movl %edx, 4(v)
+
+#ifdef CONFIG_PAX_REFCOUNT
+2:
+#endif
+
+RET_ENDP
+BEGIN(sub_return_unchecked)
+ negl %edx
+ negl %eax
+ sbbl $0, %edx
+ addl (v), %eax
+ adcl 4(v), %edx
movl %eax, (v)
movl %edx, 4(v)
RET_ENDP
BEGIN(inc)
addl $1, (v)
adcl $0, 4(v)
+
+#ifdef CONFIG_PAX_REFCOUNT
+ jno 0f
+ subl $1, (v)
+ sbbl $0, 4(v)
+ int $4
+0:
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+RET_ENDP
+BEGIN(inc_unchecked)
+ addl $1, (v)
+ adcl $0, 4(v)
RET_ENDP
#undef v
movl 4(v), %edx
addl $1, %eax
adcl $0, %edx
+
+#ifdef CONFIG_PAX_REFCOUNT
+ into
+1234:
+ _ASM_EXTABLE(1234b, 2f)
+#endif
+
+ movl %eax, (v)
+ movl %edx, 4(v)
+
+#ifdef CONFIG_PAX_REFCOUNT
+2:
+#endif
+
+RET_ENDP
+BEGIN(inc_return_unchecked)
+ movl (v), %eax
+ movl 4(v), %edx
+ addl $1, %eax
+ adcl $0, %edx
movl %eax, (v)
movl %edx, 4(v)
RET_ENDP
BEGIN(dec)
subl $1, (v)
sbbl $0, 4(v)
+
+#ifdef CONFIG_PAX_REFCOUNT
+ jno 0f
+ addl $1, (v)
+ adcl $0, 4(v)
+ int $4
+0:
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+RET_ENDP
+BEGIN(dec_unchecked)
+ subl $1, (v)
+ sbbl $0, 4(v)
RET_ENDP
#undef v
movl 4(v), %edx
subl $1, %eax
sbbl $0, %edx
+
+#ifdef CONFIG_PAX_REFCOUNT
+ into
+1234:
+ _ASM_EXTABLE(1234b, 2f)
+#endif
+
+ movl %eax, (v)
+ movl %edx, 4(v)
+
+#ifdef CONFIG_PAX_REFCOUNT
+2:
+#endif
+
+RET_ENDP
+BEGIN(dec_return_unchecked)
+ movl (v), %eax
+ movl 4(v), %edx
+ subl $1, %eax
+ sbbl $0, %edx
movl %eax, (v)
movl %edx, 4(v)
RET_ENDP
adcl %edx, %edi
addl (v), %eax
adcl 4(v), %edx
+
+#ifdef CONFIG_PAX_REFCOUNT
+ into
+1234:
+ _ASM_EXTABLE(1234b, 2f)
+#endif
+
cmpl %eax, %ecx
je 3f
1:
1:
addl $1, %eax
adcl $0, %edx
+
+#ifdef CONFIG_PAX_REFCOUNT
+ into
+1234:
+ _ASM_EXTABLE(1234b, 2f)
+#endif
+
movl %eax, (v)
movl %edx, 4(v)
movl $1, %eax
movl 4(v), %edx
subl $1, %eax
sbbl $0, %edx
+
+#ifdef CONFIG_PAX_REFCOUNT
+ into
+1234:
+ _ASM_EXTABLE(1234b, 1f)
+#endif
+
js 1f
movl %eax, (v)
movl %edx, 4(v)
CFI_STARTPROC
read64 %ecx
+ pax_force_retaddr
ret
CFI_ENDPROC
ENDPROC(atomic64_read_cx8)
+ENTRY(atomic64_read_unchecked_cx8)
+ CFI_STARTPROC
+
+ read64 %ecx
+ pax_force_retaddr
+ ret
+ CFI_ENDPROC
+ENDPROC(atomic64_read_unchecked_cx8)
+
ENTRY(atomic64_set_cx8)
CFI_STARTPROC
cmpxchg8b (%esi)
jne 1b
+ pax_force_retaddr
ret
CFI_ENDPROC
ENDPROC(atomic64_set_cx8)
+ENTRY(atomic64_set_unchecked_cx8)
+ CFI_STARTPROC
+
+1:
+/* we don't need LOCK_PREFIX since aligned 64-bit writes
+ * are atomic on 586 and newer */
+ cmpxchg8b (%esi)
+ jne 1b
+
+ pax_force_retaddr
+ ret
+ CFI_ENDPROC
+ENDPROC(atomic64_set_unchecked_cx8)
+
ENTRY(atomic64_xchg_cx8)
CFI_STARTPROC
cmpxchg8b (%esi)
jne 1b
+ pax_force_retaddr
ret
CFI_ENDPROC
ENDPROC(atomic64_xchg_cx8)
-.macro addsub_return func ins insc
-ENTRY(atomic64_\func\()_return_cx8)
+.macro addsub_return func ins insc unchecked=""
+ENTRY(atomic64_\func\()_return\unchecked\()_cx8)
CFI_STARTPROC
SAVE ebp
SAVE ebx
movl %edx, %ecx
\ins\()l %esi, %ebx
\insc\()l %edi, %ecx
+
+.ifb \unchecked
+#ifdef CONFIG_PAX_REFCOUNT
+ into
+2:
+ _ASM_EXTABLE(2b, 3f)
+#endif
+.endif
+
LOCK_PREFIX
cmpxchg8b (%ebp)
jne 1b
-
-10:
movl %ebx, %eax
movl %ecx, %edx
+
+.ifb \unchecked
+#ifdef CONFIG_PAX_REFCOUNT
+3:
+#endif
+.endif
+
RESTORE edi
RESTORE esi
RESTORE ebx
RESTORE ebp
+ pax_force_retaddr
ret
CFI_ENDPROC
-ENDPROC(atomic64_\func\()_return_cx8)
+ENDPROC(atomic64_\func\()_return\unchecked\()_cx8)
.endm
addsub_return add add adc
addsub_return sub sub sbb
+addsub_return add add adc _unchecked
+addsub_return sub sub sbb _unchecked
-.macro incdec_return func ins insc
-ENTRY(atomic64_\func\()_return_cx8)
+.macro incdec_return func ins insc unchecked=""
+ENTRY(atomic64_\func\()_return\unchecked\()_cx8)
CFI_STARTPROC
SAVE ebx
movl %edx, %ecx
\ins\()l $1, %ebx
\insc\()l $0, %ecx
+
+.ifb \unchecked
+#ifdef CONFIG_PAX_REFCOUNT
+ into
+2:
+ _ASM_EXTABLE(2b, 3f)
+#endif
+.endif
+
LOCK_PREFIX
cmpxchg8b (%esi)
jne 1b
-10:
movl %ebx, %eax
movl %ecx, %edx
+
+.ifb \unchecked
+#ifdef CONFIG_PAX_REFCOUNT
+3:
+#endif
+.endif
+
RESTORE ebx
+ pax_force_retaddr
ret
CFI_ENDPROC
-ENDPROC(atomic64_\func\()_return_cx8)
+ENDPROC(atomic64_\func\()_return\unchecked\()_cx8)
.endm
incdec_return inc add adc
incdec_return dec sub sbb
+incdec_return inc add adc _unchecked
+incdec_return dec sub sbb _unchecked
ENTRY(atomic64_dec_if_positive_cx8)
CFI_STARTPROC
movl %edx, %ecx
subl $1, %ebx
sbb $0, %ecx
+
+#ifdef CONFIG_PAX_REFCOUNT
+ into
+1234:
+ _ASM_EXTABLE(1234b, 2f)
+#endif
+
js 2f
LOCK_PREFIX
cmpxchg8b (%esi)
movl %ebx, %eax
movl %ecx, %edx
RESTORE ebx
+ pax_force_retaddr
ret
CFI_ENDPROC
ENDPROC(atomic64_dec_if_positive_cx8)
movl %edx, %ecx
addl %ebp, %ebx
adcl %edi, %ecx
+
+#ifdef CONFIG_PAX_REFCOUNT
+ into
+1234:
+ _ASM_EXTABLE(1234b, 3f)
+#endif
+
LOCK_PREFIX
cmpxchg8b (%esi)
jne 1b
CFI_ADJUST_CFA_OFFSET -8
RESTORE ebx
RESTORE ebp
+ pax_force_retaddr
ret
4:
cmpl %edx, 4(%esp)
xorl %ecx, %ecx
addl $1, %ebx
adcl %edx, %ecx
+
+#ifdef CONFIG_PAX_REFCOUNT
+ into
+1234:
+ _ASM_EXTABLE(1234b, 3f)
+#endif
+
LOCK_PREFIX
cmpxchg8b (%esi)
jne 1b
movl $1, %eax
3:
RESTORE ebx
+ pax_force_retaddr
ret
CFI_ENDPROC
ENDPROC(atomic64_inc_not_zero_cx8)
#include <asm/dwarf2.h>
#include <asm/errno.h>
#include <asm/asm.h>
-
+#include <asm/segment.h>
+
/*
* computes a partial checksum, e.g. for TCP/UDP fragments
*/
#define ARGBASE 16
#define FP 12
-
-ENTRY(csum_partial_copy_generic)
+
+ENTRY(csum_partial_copy_generic_to_user)
CFI_STARTPROC
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ pushl_cfi %gs
+ popl_cfi %es
+ jmp csum_partial_copy_generic
+#endif
+
+ENTRY(csum_partial_copy_generic_from_user)
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ pushl_cfi %gs
+ popl_cfi %ds
+#endif
+
+ENTRY(csum_partial_copy_generic)
subl $4,%esp
CFI_ADJUST_CFA_OFFSET 4
pushl_cfi %edi
jmp 4f
SRC(1: movw (%esi), %bx )
addl $2, %esi
-DST( movw %bx, (%edi) )
+DST( movw %bx, %es:(%edi) )
addl $2, %edi
addw %bx, %ax
adcl $0, %eax
SRC(1: movl (%esi), %ebx )
SRC( movl 4(%esi), %edx )
adcl %ebx, %eax
-DST( movl %ebx, (%edi) )
+DST( movl %ebx, %es:(%edi) )
adcl %edx, %eax
-DST( movl %edx, 4(%edi) )
+DST( movl %edx, %es:4(%edi) )
SRC( movl 8(%esi), %ebx )
SRC( movl 12(%esi), %edx )
adcl %ebx, %eax
-DST( movl %ebx, 8(%edi) )
+DST( movl %ebx, %es:8(%edi) )
adcl %edx, %eax
-DST( movl %edx, 12(%edi) )
+DST( movl %edx, %es:12(%edi) )
SRC( movl 16(%esi), %ebx )
SRC( movl 20(%esi), %edx )
adcl %ebx, %eax
-DST( movl %ebx, 16(%edi) )
+DST( movl %ebx, %es:16(%edi) )
adcl %edx, %eax
-DST( movl %edx, 20(%edi) )
+DST( movl %edx, %es:20(%edi) )
SRC( movl 24(%esi), %ebx )
SRC( movl 28(%esi), %edx )
adcl %ebx, %eax
-DST( movl %ebx, 24(%edi) )
+DST( movl %ebx, %es:24(%edi) )
adcl %edx, %eax
-DST( movl %edx, 28(%edi) )
+DST( movl %edx, %es:28(%edi) )
lea 32(%esi), %esi
lea 32(%edi), %edi
shrl $2, %edx # This clears CF
SRC(3: movl (%esi), %ebx )
adcl %ebx, %eax
-DST( movl %ebx, (%edi) )
+DST( movl %ebx, %es:(%edi) )
lea 4(%esi), %esi
lea 4(%edi), %edi
dec %edx
jb 5f
SRC( movw (%esi), %cx )
leal 2(%esi), %esi
-DST( movw %cx, (%edi) )
+DST( movw %cx, %es:(%edi) )
leal 2(%edi), %edi
je 6f
shll $16,%ecx
SRC(5: movb (%esi), %cl )
-DST( movb %cl, (%edi) )
+DST( movb %cl, %es:(%edi) )
6: addl %ecx, %eax
adcl $0, %eax
7:
6001:
movl ARGBASE+20(%esp), %ebx # src_err_ptr
- movl $-EFAULT, (%ebx)
+ movl $-EFAULT, %ss:(%ebx)
# zero the complete destination - computing the rest
# is too much work
6002:
movl ARGBASE+24(%esp), %ebx # dst_err_ptr
- movl $-EFAULT,(%ebx)
+ movl $-EFAULT,%ss:(%ebx)
jmp 5000b
.previous
+ pushl_cfi %ss
+ popl_cfi %ds
+ pushl_cfi %ss
+ popl_cfi %es
popl_cfi %ebx
CFI_RESTORE ebx
popl_cfi %esi
popl_cfi %ecx # equivalent to addl $4,%esp
ret
CFI_ENDPROC
-ENDPROC(csum_partial_copy_generic)
+ENDPROC(csum_partial_copy_generic_to_user)
#else
/* Version for PentiumII/PPro */
#define ROUND1(x) \
+ nop; nop; nop; \
SRC(movl x(%esi), %ebx ) ; \
addl %ebx, %eax ; \
- DST(movl %ebx, x(%edi) ) ;
+ DST(movl %ebx, %es:x(%edi)) ;
#define ROUND(x) \
+ nop; nop; nop; \
SRC(movl x(%esi), %ebx ) ; \
adcl %ebx, %eax ; \
- DST(movl %ebx, x(%edi) ) ;
+ DST(movl %ebx, %es:x(%edi)) ;
#define ARGBASE 12
-
-ENTRY(csum_partial_copy_generic)
+
+ENTRY(csum_partial_copy_generic_to_user)
CFI_STARTPROC
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ pushl_cfi %gs
+ popl_cfi %es
+ jmp csum_partial_copy_generic
+#endif
+
+ENTRY(csum_partial_copy_generic_from_user)
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ pushl_cfi %gs
+ popl_cfi %ds
+#endif
+
+ENTRY(csum_partial_copy_generic)
pushl_cfi %ebx
CFI_REL_OFFSET ebx, 0
pushl_cfi %edi
subl %ebx, %edi
lea -1(%esi),%edx
andl $-32,%edx
- lea 3f(%ebx,%ebx), %ebx
+ lea 3f(%ebx,%ebx,2), %ebx
testl %esi, %esi
jmp *%ebx
1: addl $64,%esi
jb 5f
SRC( movw (%esi), %dx )
leal 2(%esi), %esi
-DST( movw %dx, (%edi) )
+DST( movw %dx, %es:(%edi) )
leal 2(%edi), %edi
je 6f
shll $16,%edx
5:
SRC( movb (%esi), %dl )
-DST( movb %dl, (%edi) )
+DST( movb %dl, %es:(%edi) )
6: addl %edx, %eax
adcl $0, %eax
7:
.section .fixup, "ax"
6001: movl ARGBASE+20(%esp), %ebx # src_err_ptr
- movl $-EFAULT, (%ebx)
+ movl $-EFAULT, %ss:(%ebx)
# zero the complete destination (computing the rest is too much work)
movl ARGBASE+8(%esp),%edi # dst
movl ARGBASE+12(%esp),%ecx # len
rep; stosb
jmp 7b
6002: movl ARGBASE+24(%esp), %ebx # dst_err_ptr
- movl $-EFAULT, (%ebx)
+ movl $-EFAULT, %ss:(%ebx)
jmp 7b
.previous
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ pushl_cfi %ss
+ popl_cfi %ds
+ pushl_cfi %ss
+ popl_cfi %es
+#endif
+
popl_cfi %esi
CFI_RESTORE esi
popl_cfi %edi
CFI_RESTORE ebx
ret
CFI_ENDPROC
-ENDPROC(csum_partial_copy_generic)
+ENDPROC(csum_partial_copy_generic_to_user)
#undef ROUND
#undef ROUND1
movl $4096/8,%ecx
xorl %eax,%eax
rep stosq
+ pax_force_retaddr
ret
CFI_ENDPROC
ENDPROC(clear_page_c)
movl $4096,%ecx
xorl %eax,%eax
rep stosb
+ pax_force_retaddr
ret
CFI_ENDPROC
ENDPROC(clear_page_c_e)
leaq 64(%rdi),%rdi
jnz .Lloop
nop
+ pax_force_retaddr
ret
CFI_ENDPROC
.Lclear_page_end:
#include <asm/cpufeature.h>
- .section .altinstr_replacement,"ax"
+ .section .altinstr_replacement,"a"
1: .byte 0xeb /* jmp <disp8> */
.byte (clear_page_c - clear_page) - (2f - 1b) /* offset */
2: .byte 0xeb /* jmp <disp8> */
#include <linux/linkage.h>
#include <asm/dwarf2.h>
#include <asm/percpu.h>
+#include <asm/alternative-asm.h>
.text
CFI_REMEMBER_STATE
popfq_cfi
mov $1, %al
+ pax_force_retaddr
ret
CFI_RESTORE_STATE
.Lnot_same:
popfq_cfi
xor %al,%al
+ pax_force_retaddr
ret
CFI_ENDPROC
CFI_STARTPROC
movl $4096/8, %ecx
rep movsq
+ pax_force_retaddr
ret
CFI_ENDPROC
ENDPROC(copy_page_rep)
CFI_ADJUST_CFA_OFFSET 2*8
movq %rbx, (%rsp)
CFI_REL_OFFSET rbx, 0
- movq %r12, 1*8(%rsp)
- CFI_REL_OFFSET r12, 1*8
+ movq %r13, 1*8(%rsp)
+ CFI_REL_OFFSET r13, 1*8
movl $(4096/64)-5, %ecx
.p2align 4
movq 0x8*4(%rsi), %r9
movq 0x8*5(%rsi), %r10
movq 0x8*6(%rsi), %r11
- movq 0x8*7(%rsi), %r12
+ movq 0x8*7(%rsi), %r13
prefetcht0 5*64(%rsi)
movq %r9, 0x8*4(%rdi)
movq %r10, 0x8*5(%rdi)
movq %r11, 0x8*6(%rdi)
- movq %r12, 0x8*7(%rdi)
+ movq %r13, 0x8*7(%rdi)
leaq 64 (%rsi), %rsi
leaq 64 (%rdi), %rdi
movq 0x8*4(%rsi), %r9
movq 0x8*5(%rsi), %r10
movq 0x8*6(%rsi), %r11
- movq 0x8*7(%rsi), %r12
+ movq 0x8*7(%rsi), %r13
movq %rax, 0x8*0(%rdi)
movq %rbx, 0x8*1(%rdi)
movq %r9, 0x8*4(%rdi)
movq %r10, 0x8*5(%rdi)
movq %r11, 0x8*6(%rdi)
- movq %r12, 0x8*7(%rdi)
+ movq %r13, 0x8*7(%rdi)
leaq 64(%rdi), %rdi
leaq 64(%rsi), %rsi
movq (%rsp), %rbx
CFI_RESTORE rbx
- movq 1*8(%rsp), %r12
- CFI_RESTORE r12
+ movq 1*8(%rsp), %r13
+ CFI_RESTORE r13
addq $2*8, %rsp
CFI_ADJUST_CFA_OFFSET -2*8
+ pax_force_retaddr
ret
.Lcopy_page_end:
CFI_ENDPROC
#include <asm/cpufeature.h>
- .section .altinstr_replacement,"ax"
+ .section .altinstr_replacement,"a"
1: .byte 0xeb /* jmp <disp8> */
.byte (copy_page_rep - copy_page) - (2f - 1b) /* offset */
2:
#include <asm/alternative-asm.h>
#include <asm/asm.h>
#include <asm/smap.h>
-
-/*
- * By placing feature2 after feature1 in altinstructions section, we logically
- * implement:
- * If CPU has feature2, jmp to alt2 is used
- * else if CPU has feature1, jmp to alt1 is used
- * else jmp to orig is used.
- */
- .macro ALTERNATIVE_JUMP feature1,feature2,orig,alt1,alt2
-0:
- .byte 0xe9 /* 32bit jump */
- .long \orig-1f /* by default jump to orig */
-1:
- .section .altinstr_replacement,"ax"
-2: .byte 0xe9 /* near jump with 32bit immediate */
- .long \alt1-1b /* offset */ /* or alternatively to alt1 */
-3: .byte 0xe9 /* near jump with 32bit immediate */
- .long \alt2-1b /* offset */ /* or alternatively to alt2 */
- .previous
-
- .section .altinstructions,"a"
- altinstruction_entry 0b,2b,\feature1,5,5
- altinstruction_entry 0b,3b,\feature2,5,5
- .previous
- .endm
+#include <asm/pgtable.h>
.macro ALIGN_DESTINATION
#ifdef FIX_ALIGNMENT
#endif
.endm
-/* Standard copy_to_user with segment limit checking */
-ENTRY(_copy_to_user)
- CFI_STARTPROC
- GET_THREAD_INFO(%rax)
- movq %rdi,%rcx
- addq %rdx,%rcx
- jc bad_to_user
- cmpq TI_addr_limit(%rax),%rcx
- ja bad_to_user
- ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,X86_FEATURE_ERMS, \
- copy_user_generic_unrolled,copy_user_generic_string, \
- copy_user_enhanced_fast_string
- CFI_ENDPROC
-ENDPROC(_copy_to_user)
-
-/* Standard copy_from_user with segment limit checking */
-ENTRY(_copy_from_user)
- CFI_STARTPROC
- GET_THREAD_INFO(%rax)
- movq %rsi,%rcx
- addq %rdx,%rcx
- jc bad_from_user
- cmpq TI_addr_limit(%rax),%rcx
- ja bad_from_user
- ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,X86_FEATURE_ERMS, \
- copy_user_generic_unrolled,copy_user_generic_string, \
- copy_user_enhanced_fast_string
- CFI_ENDPROC
-ENDPROC(_copy_from_user)
-
- .section .fixup,"ax"
- /* must zero dest */
-ENTRY(bad_from_user)
-bad_from_user:
- CFI_STARTPROC
- movl %edx,%ecx
- xorl %eax,%eax
- rep
- stosb
-bad_to_user:
- movl %edx,%eax
- ret
- CFI_ENDPROC
-ENDPROC(bad_from_user)
- .previous
-
/*
* copy_user_generic_unrolled - memory copy with exception handling.
* This version is for CPUs like P4 that don't have efficient micro
*/
ENTRY(copy_user_generic_unrolled)
CFI_STARTPROC
+ ASM_PAX_OPEN_USERLAND
ASM_STAC
cmpl $8,%edx
jb 20f /* less then 8 bytes, go to byte copy loop */
jnz 21b
23: xor %eax,%eax
ASM_CLAC
+ ASM_PAX_CLOSE_USERLAND
+ pax_force_retaddr
ret
.section .fixup,"ax"
*/
ENTRY(copy_user_generic_string)
CFI_STARTPROC
+ ASM_PAX_OPEN_USERLAND
ASM_STAC
cmpl $8,%edx
jb 2f /* less than 8 bytes, go to byte copy loop */
movsb
xorl %eax,%eax
ASM_CLAC
+ ASM_PAX_CLOSE_USERLAND
+ pax_force_retaddr
ret
.section .fixup,"ax"
*/
ENTRY(copy_user_enhanced_fast_string)
CFI_STARTPROC
+ ASM_PAX_OPEN_USERLAND
ASM_STAC
movl %edx,%ecx
1: rep
movsb
xorl %eax,%eax
ASM_CLAC
+ ASM_PAX_CLOSE_USERLAND
+ pax_force_retaddr
ret
.section .fixup,"ax"
#include <linux/linkage.h>
#include <asm/dwarf2.h>
+#include <asm/alternative-asm.h>
#define FIX_ALIGNMENT 1
#include <asm/thread_info.h>
#include <asm/asm.h>
#include <asm/smap.h>
+#include <asm/pgtable.h>
.macro ALIGN_DESTINATION
#ifdef FIX_ALIGNMENT
*/
ENTRY(__copy_user_nocache)
CFI_STARTPROC
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ mov pax_user_shadow_base,%rcx
+ cmp %rcx,%rsi
+ jae 1f
+ add %rcx,%rsi
+1:
+#endif
+
+ ASM_PAX_OPEN_USERLAND
ASM_STAC
cmpl $8,%edx
jb 20f /* less then 8 bytes, go to byte copy loop */
jnz 21b
23: xorl %eax,%eax
ASM_CLAC
+ ASM_PAX_CLOSE_USERLAND
sfence
+ pax_force_retaddr
ret
.section .fixup,"ax"
#include <asm/dwarf2.h>
#include <asm/errno.h>
#include <asm/asm.h>
+#include <asm/alternative-asm.h>
/*
* Checksum copy with exception handling.
CFI_ADJUST_CFA_OFFSET 7*8
movq %rbx, 2*8(%rsp)
CFI_REL_OFFSET rbx, 2*8
- movq %r12, 3*8(%rsp)
- CFI_REL_OFFSET r12, 3*8
+ movq %r15, 3*8(%rsp)
+ CFI_REL_OFFSET r15, 3*8
movq %r14, 4*8(%rsp)
CFI_REL_OFFSET r14, 4*8
movq %r13, 5*8(%rsp)
movl %edx, %ecx
xorl %r9d, %r9d
- movq %rcx, %r12
+ movq %rcx, %r15
- shrq $6, %r12
+ shrq $6, %r15
jz .Lhandle_tail /* < 64 */
clc
/* main loop. clear in 64 byte blocks */
/* r9: zero, r8: temp2, rbx: temp1, rax: sum, rcx: saved length */
- /* r11: temp3, rdx: temp4, r12 loopcnt */
+ /* r11: temp3, rdx: temp4, r15 loopcnt */
/* r10: temp5, rbp: temp6, r14 temp7, r13 temp8 */
.p2align 4
.Lloop:
adcq %r14, %rax
adcq %r13, %rax
- decl %r12d
+ decl %r15d
dest
movq %rbx, (%rsi)
.Lende:
movq 2*8(%rsp), %rbx
CFI_RESTORE rbx
- movq 3*8(%rsp), %r12
- CFI_RESTORE r12
+ movq 3*8(%rsp), %r15
+ CFI_RESTORE r15
movq 4*8(%rsp), %r14
CFI_RESTORE r14
movq 5*8(%rsp), %r13
CFI_RESTORE rbp
addq $7*8, %rsp
CFI_ADJUST_CFA_OFFSET -7*8
+ pax_force_retaddr
ret
CFI_RESTORE_STATE
len -= 2;
}
}
+ pax_open_userland();
stac();
- isum = csum_partial_copy_generic((__force const void *)src,
+ isum = csum_partial_copy_generic((const void __force_kernel *)____m(src),
dst, len, isum, errp, NULL);
clac();
+ pax_close_userland();
if (unlikely(*errp))
goto out_err;
}
*errp = 0;
+ pax_open_userland();
stac();
- ret = csum_partial_copy_generic(src, (void __force *)dst,
+ ret = csum_partial_copy_generic(src, (void __force_kernel *)____m(dst),
len, isum, NULL, errp);
clac();
+ pax_close_userland();
return ret;
}
EXPORT_SYMBOL(csum_partial_copy_to_user);
#include <asm/thread_info.h>
#include <asm/asm.h>
#include <asm/smap.h>
+#include <asm/segment.h>
+#include <asm/pgtable.h>
+#include <asm/alternative-asm.h>
+
+#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_MEMORY_UDEREF)
+#define __copyuser_seg gs;
+#else
+#define __copyuser_seg
+#endif
.text
ENTRY(__get_user_1)
CFI_STARTPROC
+
+#if !defined(CONFIG_X86_32) || !defined(CONFIG_PAX_MEMORY_UDEREF)
GET_THREAD_INFO(%_ASM_DX)
cmp TI_addr_limit(%_ASM_DX),%_ASM_AX
jae bad_get_user
ASM_STAC
-1: movzbl (%_ASM_AX),%edx
+
+#if defined(CONFIG_X86_64) && defined(CONFIG_PAX_MEMORY_UDEREF)
+ mov pax_user_shadow_base,%_ASM_DX
+ cmp %_ASM_DX,%_ASM_AX
+ jae 1234f
+ add %_ASM_DX,%_ASM_AX
+1234:
+#endif
+
+#endif
+
+1: __copyuser_seg movzbl (%_ASM_AX),%edx
xor %eax,%eax
ASM_CLAC
+ pax_force_retaddr
ret
CFI_ENDPROC
ENDPROC(__get_user_1)
ENTRY(__get_user_2)
CFI_STARTPROC
add $1,%_ASM_AX
+
+#if !defined(CONFIG_X86_32) || !defined(CONFIG_PAX_MEMORY_UDEREF)
jc bad_get_user
GET_THREAD_INFO(%_ASM_DX)
cmp TI_addr_limit(%_ASM_DX),%_ASM_AX
jae bad_get_user
ASM_STAC
-2: movzwl -1(%_ASM_AX),%edx
+
+#if defined(CONFIG_X86_64) && defined(CONFIG_PAX_MEMORY_UDEREF)
+ mov pax_user_shadow_base,%_ASM_DX
+ cmp %_ASM_DX,%_ASM_AX
+ jae 1234f
+ add %_ASM_DX,%_ASM_AX
+1234:
+#endif
+
+#endif
+
+2: __copyuser_seg movzwl -1(%_ASM_AX),%edx
xor %eax,%eax
ASM_CLAC
+ pax_force_retaddr
ret
CFI_ENDPROC
ENDPROC(__get_user_2)
ENTRY(__get_user_4)
CFI_STARTPROC
add $3,%_ASM_AX
+
+#if !defined(CONFIG_X86_32) || !defined(CONFIG_PAX_MEMORY_UDEREF)
jc bad_get_user
GET_THREAD_INFO(%_ASM_DX)
cmp TI_addr_limit(%_ASM_DX),%_ASM_AX
jae bad_get_user
ASM_STAC
-3: movl -3(%_ASM_AX),%edx
+
+#if defined(CONFIG_X86_64) && defined(CONFIG_PAX_MEMORY_UDEREF)
+ mov pax_user_shadow_base,%_ASM_DX
+ cmp %_ASM_DX,%_ASM_AX
+ jae 1234f
+ add %_ASM_DX,%_ASM_AX
+1234:
+#endif
+
+#endif
+
+3: __copyuser_seg movl -3(%_ASM_AX),%edx
xor %eax,%eax
ASM_CLAC
+ pax_force_retaddr
ret
CFI_ENDPROC
ENDPROC(__get_user_4)
GET_THREAD_INFO(%_ASM_DX)
cmp TI_addr_limit(%_ASM_DX),%_ASM_AX
jae bad_get_user
+
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+ mov pax_user_shadow_base,%_ASM_DX
+ cmp %_ASM_DX,%_ASM_AX
+ jae 1234f
+ add %_ASM_DX,%_ASM_AX
+1234:
+#endif
+
ASM_STAC
4: movq -7(%_ASM_AX),%rdx
xor %eax,%eax
ASM_CLAC
+ pax_force_retaddr
ret
#else
add $7,%_ASM_AX
cmp TI_addr_limit(%_ASM_DX),%_ASM_AX
jae bad_get_user_8
ASM_STAC
-4: movl -7(%_ASM_AX),%edx
-5: movl -3(%_ASM_AX),%ecx
+4: __copyuser_seg movl -7(%_ASM_AX),%edx
+5: __copyuser_seg movl -3(%_ASM_AX),%ecx
xor %eax,%eax
ASM_CLAC
+ pax_force_retaddr
ret
#endif
CFI_ENDPROC
xor %edx,%edx
mov $(-EFAULT),%_ASM_AX
ASM_CLAC
+ pax_force_retaddr
ret
CFI_ENDPROC
END(bad_get_user)
xor %ecx,%ecx
mov $(-EFAULT),%_ASM_AX
ASM_CLAC
+ pax_force_retaddr
ret
CFI_ENDPROC
END(bad_get_user_8)
#ifdef __KERNEL__
#include <linux/string.h>
+#include <asm/pgtable_types.h>
#else
#include <string.h>
+#define ktla_ktva(addr) addr
#endif
#include <asm/inat.h>
#include <asm/insn.h>
void insn_init(struct insn *insn, const void *kaddr, int buf_len, int x86_64)
{
memset(insn, 0, sizeof(*insn));
- insn->kaddr = kaddr;
- insn->end_kaddr = kaddr + buf_len;
- insn->next_byte = kaddr;
+ insn->kaddr = ktla_ktva(kaddr);
+ insn->end_kaddr = insn->kaddr + buf_len;
+ insn->next_byte = insn->kaddr;
insn->x86_64 = x86_64 ? 1 : 0;
insn->opnd_bytes = 4;
if (x86_64)
#include <linux/linkage.h>
#include <asm/dwarf2.h>
+#include <asm/alternative-asm.h>
/*
* override generic version in lib/iomap_copy.c
CFI_STARTPROC
movl %edx,%ecx
rep movsd
+ pax_force_retaddr
ret
CFI_ENDPROC
ENDPROC(__iowrite32_copy)
* This gets patched over the unrolled variant (below) via the
* alternative instructions framework:
*/
- .section .altinstr_replacement, "ax", @progbits
+ .section .altinstr_replacement, "a", @progbits
.Lmemcpy_c:
movq %rdi, %rax
movq %rdx, %rcx
rep movsq
movl %edx, %ecx
rep movsb
+ pax_force_retaddr
ret
.Lmemcpy_e:
.previous
* This gets patched over the unrolled variant (below) via the
* alternative instructions framework:
*/
- .section .altinstr_replacement, "ax", @progbits
+ .section .altinstr_replacement, "a", @progbits
.Lmemcpy_c_e:
movq %rdi, %rax
movq %rdx, %rcx
rep movsb
+ pax_force_retaddr
ret
.Lmemcpy_e_e:
.previous
movq %r9, 1*8(%rdi)
movq %r10, -2*8(%rdi, %rdx)
movq %r11, -1*8(%rdi, %rdx)
+ pax_force_retaddr
retq
.p2align 4
.Lless_16bytes:
movq -1*8(%rsi, %rdx), %r9
movq %r8, 0*8(%rdi)
movq %r9, -1*8(%rdi, %rdx)
+ pax_force_retaddr
retq
.p2align 4
.Lless_8bytes:
movl -4(%rsi, %rdx), %r8d
movl %ecx, (%rdi)
movl %r8d, -4(%rdi, %rdx)
+ pax_force_retaddr
retq
.p2align 4
.Lless_3bytes:
movb %cl, (%rdi)
.Lend:
+ pax_force_retaddr
retq
CFI_ENDPROC
ENDPROC(memcpy)
movb (%rsi), %r11b
movb %r11b, (%rdi)
13:
+ pax_force_retaddr
retq
CFI_ENDPROC
- .section .altinstr_replacement,"ax"
+ .section .altinstr_replacement,"a"
.Lmemmove_begin_forward_efs:
/* Forward moving data. */
movq %rdx, %rcx
rep movsb
+ pax_force_retaddr
retq
.Lmemmove_end_forward_efs:
.previous
*
* rax original destination
*/
- .section .altinstr_replacement, "ax", @progbits
+ .section .altinstr_replacement, "a", @progbits
.Lmemset_c:
movq %rdi,%r9
movq %rdx,%rcx
movl %edx,%ecx
rep stosb
movq %r9,%rax
+ pax_force_retaddr
ret
.Lmemset_e:
.previous
*
* rax original destination
*/
- .section .altinstr_replacement, "ax", @progbits
+ .section .altinstr_replacement, "a", @progbits
.Lmemset_c_e:
movq %rdi,%r9
movb %sil,%al
movq %rdx,%rcx
rep stosb
movq %r9,%rax
+ pax_force_retaddr
ret
.Lmemset_e_e:
.previous
.Lende:
movq %r10,%rax
+ pax_force_retaddr
ret
CFI_RESTORE_STATE
{
void *p;
int i;
+ unsigned long cr0;
if (unlikely(in_interrupt()))
return __memcpy(to, from, len);
kernel_fpu_begin();
__asm__ __volatile__ (
- "1: prefetch (%0)\n" /* This set is 28 bytes */
- " prefetch 64(%0)\n"
- " prefetch 128(%0)\n"
- " prefetch 192(%0)\n"
- " prefetch 256(%0)\n"
+ "1: prefetch (%1)\n" /* This set is 28 bytes */
+ " prefetch 64(%1)\n"
+ " prefetch 128(%1)\n"
+ " prefetch 192(%1)\n"
+ " prefetch 256(%1)\n"
"2: \n"
".section .fixup, \"ax\"\n"
- "3: movw $0x1AEB, 1b\n" /* jmp on 26 bytes */
+ "3: \n"
+
+#ifdef CONFIG_PAX_KERNEXEC
+ " movl %%cr0, %0\n"
+ " movl %0, %%eax\n"
+ " andl $0xFFFEFFFF, %%eax\n"
+ " movl %%eax, %%cr0\n"
+#endif
+
+ " movw $0x1AEB, 1b\n" /* jmp on 26 bytes */
+
+#ifdef CONFIG_PAX_KERNEXEC
+ " movl %0, %%cr0\n"
+#endif
+
" jmp 2b\n"
".previous\n"
_ASM_EXTABLE(1b, 3b)
- : : "r" (from));
+ : "=&r" (cr0) : "r" (from) : "ax");
for ( ; i > 5; i--) {
__asm__ __volatile__ (
- "1: prefetch 320(%0)\n"
- "2: movq (%0), %%mm0\n"
- " movq 8(%0), %%mm1\n"
- " movq 16(%0), %%mm2\n"
- " movq 24(%0), %%mm3\n"
- " movq %%mm0, (%1)\n"
- " movq %%mm1, 8(%1)\n"
- " movq %%mm2, 16(%1)\n"
- " movq %%mm3, 24(%1)\n"
- " movq 32(%0), %%mm0\n"
- " movq 40(%0), %%mm1\n"
- " movq 48(%0), %%mm2\n"
- " movq 56(%0), %%mm3\n"
- " movq %%mm0, 32(%1)\n"
- " movq %%mm1, 40(%1)\n"
- " movq %%mm2, 48(%1)\n"
- " movq %%mm3, 56(%1)\n"
+ "1: prefetch 320(%1)\n"
+ "2: movq (%1), %%mm0\n"
+ " movq 8(%1), %%mm1\n"
+ " movq 16(%1), %%mm2\n"
+ " movq 24(%1), %%mm3\n"
+ " movq %%mm0, (%2)\n"
+ " movq %%mm1, 8(%2)\n"
+ " movq %%mm2, 16(%2)\n"
+ " movq %%mm3, 24(%2)\n"
+ " movq 32(%1), %%mm0\n"
+ " movq 40(%1), %%mm1\n"
+ " movq 48(%1), %%mm2\n"
+ " movq 56(%1), %%mm3\n"
+ " movq %%mm0, 32(%2)\n"
+ " movq %%mm1, 40(%2)\n"
+ " movq %%mm2, 48(%2)\n"
+ " movq %%mm3, 56(%2)\n"
".section .fixup, \"ax\"\n"
- "3: movw $0x05EB, 1b\n" /* jmp on 5 bytes */
+ "3:\n"
+
+#ifdef CONFIG_PAX_KERNEXEC
+ " movl %%cr0, %0\n"
+ " movl %0, %%eax\n"
+ " andl $0xFFFEFFFF, %%eax\n"
+ " movl %%eax, %%cr0\n"
+#endif
+
+ " movw $0x05EB, 1b\n" /* jmp on 5 bytes */
+
+#ifdef CONFIG_PAX_KERNEXEC
+ " movl %0, %%cr0\n"
+#endif
+
" jmp 2b\n"
".previous\n"
_ASM_EXTABLE(1b, 3b)
- : : "r" (from), "r" (to) : "memory");
+ : "=&r" (cr0) : "r" (from), "r" (to) : "memory", "ax");
from += 64;
to += 64;
static void fast_copy_page(void *to, void *from)
{
int i;
+ unsigned long cr0;
kernel_fpu_begin();
* but that is for later. -AV
*/
__asm__ __volatile__(
- "1: prefetch (%0)\n"
- " prefetch 64(%0)\n"
- " prefetch 128(%0)\n"
- " prefetch 192(%0)\n"
- " prefetch 256(%0)\n"
+ "1: prefetch (%1)\n"
+ " prefetch 64(%1)\n"
+ " prefetch 128(%1)\n"
+ " prefetch 192(%1)\n"
+ " prefetch 256(%1)\n"
"2: \n"
".section .fixup, \"ax\"\n"
- "3: movw $0x1AEB, 1b\n" /* jmp on 26 bytes */
+ "3: \n"
+
+#ifdef CONFIG_PAX_KERNEXEC
+ " movl %%cr0, %0\n"
+ " movl %0, %%eax\n"
+ " andl $0xFFFEFFFF, %%eax\n"
+ " movl %%eax, %%cr0\n"
+#endif
+
+ " movw $0x1AEB, 1b\n" /* jmp on 26 bytes */
+
+#ifdef CONFIG_PAX_KERNEXEC
+ " movl %0, %%cr0\n"
+#endif
+
" jmp 2b\n"
".previous\n"
- _ASM_EXTABLE(1b, 3b) : : "r" (from));
+ _ASM_EXTABLE(1b, 3b) : "=&r" (cr0) : "r" (from) : "ax");
for (i = 0; i < (4096-320)/64; i++) {
__asm__ __volatile__ (
- "1: prefetch 320(%0)\n"
- "2: movq (%0), %%mm0\n"
- " movntq %%mm0, (%1)\n"
- " movq 8(%0), %%mm1\n"
- " movntq %%mm1, 8(%1)\n"
- " movq 16(%0), %%mm2\n"
- " movntq %%mm2, 16(%1)\n"
- " movq 24(%0), %%mm3\n"
- " movntq %%mm3, 24(%1)\n"
- " movq 32(%0), %%mm4\n"
- " movntq %%mm4, 32(%1)\n"
- " movq 40(%0), %%mm5\n"
- " movntq %%mm5, 40(%1)\n"
- " movq 48(%0), %%mm6\n"
- " movntq %%mm6, 48(%1)\n"
- " movq 56(%0), %%mm7\n"
- " movntq %%mm7, 56(%1)\n"
+ "1: prefetch 320(%1)\n"
+ "2: movq (%1), %%mm0\n"
+ " movntq %%mm0, (%2)\n"
+ " movq 8(%1), %%mm1\n"
+ " movntq %%mm1, 8(%2)\n"
+ " movq 16(%1), %%mm2\n"
+ " movntq %%mm2, 16(%2)\n"
+ " movq 24(%1), %%mm3\n"
+ " movntq %%mm3, 24(%2)\n"
+ " movq 32(%1), %%mm4\n"
+ " movntq %%mm4, 32(%2)\n"
+ " movq 40(%1), %%mm5\n"
+ " movntq %%mm5, 40(%2)\n"
+ " movq 48(%1), %%mm6\n"
+ " movntq %%mm6, 48(%2)\n"
+ " movq 56(%1), %%mm7\n"
+ " movntq %%mm7, 56(%2)\n"
".section .fixup, \"ax\"\n"
- "3: movw $0x05EB, 1b\n" /* jmp on 5 bytes */
+ "3:\n"
+
+#ifdef CONFIG_PAX_KERNEXEC
+ " movl %%cr0, %0\n"
+ " movl %0, %%eax\n"
+ " andl $0xFFFEFFFF, %%eax\n"
+ " movl %%eax, %%cr0\n"
+#endif
+
+ " movw $0x05EB, 1b\n" /* jmp on 5 bytes */
+
+#ifdef CONFIG_PAX_KERNEXEC
+ " movl %0, %%cr0\n"
+#endif
+
" jmp 2b\n"
".previous\n"
- _ASM_EXTABLE(1b, 3b) : : "r" (from), "r" (to) : "memory");
+ _ASM_EXTABLE(1b, 3b) : "=&r" (cr0) : "r" (from), "r" (to) : "memory", "ax");
from += 64;
to += 64;
static void fast_copy_page(void *to, void *from)
{
int i;
+ unsigned long cr0;
kernel_fpu_begin();
__asm__ __volatile__ (
- "1: prefetch (%0)\n"
- " prefetch 64(%0)\n"
- " prefetch 128(%0)\n"
- " prefetch 192(%0)\n"
- " prefetch 256(%0)\n"
+ "1: prefetch (%1)\n"
+ " prefetch 64(%1)\n"
+ " prefetch 128(%1)\n"
+ " prefetch 192(%1)\n"
+ " prefetch 256(%1)\n"
"2: \n"
".section .fixup, \"ax\"\n"
- "3: movw $0x1AEB, 1b\n" /* jmp on 26 bytes */
+ "3: \n"
+
+#ifdef CONFIG_PAX_KERNEXEC
+ " movl %%cr0, %0\n"
+ " movl %0, %%eax\n"
+ " andl $0xFFFEFFFF, %%eax\n"
+ " movl %%eax, %%cr0\n"
+#endif
+
+ " movw $0x1AEB, 1b\n" /* jmp on 26 bytes */
+
+#ifdef CONFIG_PAX_KERNEXEC
+ " movl %0, %%cr0\n"
+#endif
+
" jmp 2b\n"
".previous\n"
- _ASM_EXTABLE(1b, 3b) : : "r" (from));
+ _ASM_EXTABLE(1b, 3b) : "=&r" (cr0) : "r" (from) : "ax");
for (i = 0; i < 4096/64; i++) {
__asm__ __volatile__ (
- "1: prefetch 320(%0)\n"
- "2: movq (%0), %%mm0\n"
- " movq 8(%0), %%mm1\n"
- " movq 16(%0), %%mm2\n"
- " movq 24(%0), %%mm3\n"
- " movq %%mm0, (%1)\n"
- " movq %%mm1, 8(%1)\n"
- " movq %%mm2, 16(%1)\n"
- " movq %%mm3, 24(%1)\n"
- " movq 32(%0), %%mm0\n"
- " movq 40(%0), %%mm1\n"
- " movq 48(%0), %%mm2\n"
- " movq 56(%0), %%mm3\n"
- " movq %%mm0, 32(%1)\n"
- " movq %%mm1, 40(%1)\n"
- " movq %%mm2, 48(%1)\n"
- " movq %%mm3, 56(%1)\n"
+ "1: prefetch 320(%1)\n"
+ "2: movq (%1), %%mm0\n"
+ " movq 8(%1), %%mm1\n"
+ " movq 16(%1), %%mm2\n"
+ " movq 24(%1), %%mm3\n"
+ " movq %%mm0, (%2)\n"
+ " movq %%mm1, 8(%2)\n"
+ " movq %%mm2, 16(%2)\n"
+ " movq %%mm3, 24(%2)\n"
+ " movq 32(%1), %%mm0\n"
+ " movq 40(%1), %%mm1\n"
+ " movq 48(%1), %%mm2\n"
+ " movq 56(%1), %%mm3\n"
+ " movq %%mm0, 32(%2)\n"
+ " movq %%mm1, 40(%2)\n"
+ " movq %%mm2, 48(%2)\n"
+ " movq %%mm3, 56(%2)\n"
".section .fixup, \"ax\"\n"
- "3: movw $0x05EB, 1b\n" /* jmp on 5 bytes */
+ "3:\n"
+
+#ifdef CONFIG_PAX_KERNEXEC
+ " movl %%cr0, %0\n"
+ " movl %0, %%eax\n"
+ " andl $0xFFFEFFFF, %%eax\n"
+ " movl %%eax, %%cr0\n"
+#endif
+
+ " movw $0x05EB, 1b\n" /* jmp on 5 bytes */
+
+#ifdef CONFIG_PAX_KERNEXEC
+ " movl %0, %%cr0\n"
+#endif
+
" jmp 2b\n"
".previous\n"
_ASM_EXTABLE(1b, 3b)
- : : "r" (from), "r" (to) : "memory");
+ : "=&r" (cr0) : "r" (from), "r" (to) : "memory", "ax");
from += 64;
to += 64;
#include <asm/dwarf2.h>
#include <asm/asm.h>
#include <asm/msr.h>
+#include <asm/alternative-asm.h>
#ifdef CONFIG_X86_64
/*
movl %edi, 28(%r10)
popq_cfi %rbp
popq_cfi %rbx
+ pax_force_retaddr
ret
3:
CFI_RESTORE_STATE
#include <asm/errno.h>
#include <asm/asm.h>
#include <asm/smap.h>
-
+#include <asm/segment.h>
+#include <asm/pgtable.h>
+#include <asm/alternative-asm.h>
/*
* __put_user_X
* as they get called from within inline assembly.
*/
-#define ENTER CFI_STARTPROC ; \
- GET_THREAD_INFO(%_ASM_BX)
-#define EXIT ASM_CLAC ; \
- ret ; \
+#define ENTER CFI_STARTPROC
+#define EXIT ASM_CLAC ; \
+ pax_force_retaddr ; \
+ ret ; \
CFI_ENDPROC
+#if defined(CONFIG_X86_64) && defined(CONFIG_PAX_MEMORY_UDEREF)
+#define _DEST %_ASM_CX,%_ASM_BX
+#else
+#define _DEST %_ASM_CX
+#endif
+
+#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_MEMORY_UDEREF)
+#define __copyuser_seg gs;
+#else
+#define __copyuser_seg
+#endif
+
.text
ENTRY(__put_user_1)
ENTER
+
+#if !defined(CONFIG_X86_32) || !defined(CONFIG_PAX_MEMORY_UDEREF)
+ GET_THREAD_INFO(%_ASM_BX)
cmp TI_addr_limit(%_ASM_BX),%_ASM_CX
jae bad_put_user
ASM_STAC
-1: movb %al,(%_ASM_CX)
+
+#if defined(CONFIG_X86_64) && defined(CONFIG_PAX_MEMORY_UDEREF)
+ mov pax_user_shadow_base,%_ASM_BX
+ cmp %_ASM_BX,%_ASM_CX
+ jb 1234f
+ xor %ebx,%ebx
+1234:
+#endif
+
+#endif
+
+1: __copyuser_seg movb %al,(_DEST)
xor %eax,%eax
EXIT
ENDPROC(__put_user_1)
ENTRY(__put_user_2)
ENTER
+
+#if !defined(CONFIG_X86_32) || !defined(CONFIG_PAX_MEMORY_UDEREF)
+ GET_THREAD_INFO(%_ASM_BX)
mov TI_addr_limit(%_ASM_BX),%_ASM_BX
sub $1,%_ASM_BX
cmp %_ASM_BX,%_ASM_CX
jae bad_put_user
ASM_STAC
-2: movw %ax,(%_ASM_CX)
+
+#if defined(CONFIG_X86_64) && defined(CONFIG_PAX_MEMORY_UDEREF)
+ mov pax_user_shadow_base,%_ASM_BX
+ cmp %_ASM_BX,%_ASM_CX
+ jb 1234f
+ xor %ebx,%ebx
+1234:
+#endif
+
+#endif
+
+2: __copyuser_seg movw %ax,(_DEST)
xor %eax,%eax
EXIT
ENDPROC(__put_user_2)
ENTRY(__put_user_4)
ENTER
+
+#if !defined(CONFIG_X86_32) || !defined(CONFIG_PAX_MEMORY_UDEREF)
+ GET_THREAD_INFO(%_ASM_BX)
mov TI_addr_limit(%_ASM_BX),%_ASM_BX
sub $3,%_ASM_BX
cmp %_ASM_BX,%_ASM_CX
jae bad_put_user
ASM_STAC
-3: movl %eax,(%_ASM_CX)
+
+#if defined(CONFIG_X86_64) && defined(CONFIG_PAX_MEMORY_UDEREF)
+ mov pax_user_shadow_base,%_ASM_BX
+ cmp %_ASM_BX,%_ASM_CX
+ jb 1234f
+ xor %ebx,%ebx
+1234:
+#endif
+
+#endif
+
+3: __copyuser_seg movl %eax,(_DEST)
xor %eax,%eax
EXIT
ENDPROC(__put_user_4)
ENTRY(__put_user_8)
ENTER
+
+#if !defined(CONFIG_X86_32) || !defined(CONFIG_PAX_MEMORY_UDEREF)
+ GET_THREAD_INFO(%_ASM_BX)
mov TI_addr_limit(%_ASM_BX),%_ASM_BX
sub $7,%_ASM_BX
cmp %_ASM_BX,%_ASM_CX
jae bad_put_user
ASM_STAC
-4: mov %_ASM_AX,(%_ASM_CX)
+
+#if defined(CONFIG_X86_64) && defined(CONFIG_PAX_MEMORY_UDEREF)
+ mov pax_user_shadow_base,%_ASM_BX
+ cmp %_ASM_BX,%_ASM_CX
+ jb 1234f
+ xor %ebx,%ebx
+1234:
+#endif
+
+#endif
+
+4: __copyuser_seg mov %_ASM_AX,(_DEST)
#ifdef CONFIG_X86_32
-5: movl %edx,4(%_ASM_CX)
+5: __copyuser_seg movl %edx,4(_DEST)
#endif
xor %eax,%eax
EXIT
__ASM_SIZE(pop,_cfi) %__ASM_REG(dx)
CFI_RESTORE __ASM_REG(dx)
restore_common_regs
+ pax_force_retaddr
ret
CFI_ENDPROC
ENDPROC(call_rwsem_down_read_failed)
movq %rax,%rdi
call rwsem_down_write_failed
restore_common_regs
+ pax_force_retaddr
ret
CFI_ENDPROC
ENDPROC(call_rwsem_down_write_failed)
movq %rax,%rdi
call rwsem_wake
restore_common_regs
-1: ret
+1: pax_force_retaddr
+ ret
CFI_ENDPROC
ENDPROC(call_rwsem_wake)
__ASM_SIZE(pop,_cfi) %__ASM_REG(dx)
CFI_RESTORE __ASM_REG(dx)
restore_common_regs
+ pax_force_retaddr
ret
CFI_ENDPROC
ENDPROC(call_rwsem_downgrade_wake)
#include <asm/dwarf2.h>
#include <asm/calling.h>
#include <asm/asm.h>
+#include <asm/alternative-asm.h>
/* rdi: arg1 ... normal C conventions. rax is saved/restored. */
.macro THUNK name, func, put_ret_addr_in_rdi=0
\name:
CFI_STARTPROC
- /* this one pushes 9 elems, the next one would be %rIP */
- SAVE_ARGS
+ /* this one pushes 15+1 elems, the next one would be %rIP */
+ SAVE_ARGS 8
.if \put_ret_addr_in_rdi
- movq_cfi_restore 9*8, rdi
+ movq_cfi_restore RIP, rdi
.endif
call \func
/* SAVE_ARGS below is used only for the .cfi directives it contains. */
CFI_STARTPROC
- SAVE_ARGS
+ SAVE_ARGS 8
restore:
- RESTORE_ARGS
+ RESTORE_ARGS 1,8
+ pax_force_retaddr
ret
CFI_ENDPROC
_ASM_NOKPROBE(restore)
int __d0; \
might_fault(); \
__asm__ __volatile__( \
+ __COPYUSER_SET_ES \
ASM_STAC "\n" \
"0: rep; stosl\n" \
" movl %2,%0\n" \
"1: rep; stosb\n" \
"2: " ASM_CLAC "\n" \
+ __COPYUSER_RESTORE_ES \
".section .fixup,\"ax\"\n" \
"3: lea 0(%2,%0,4),%0\n" \
" jmp 2b\n" \
#ifdef CONFIG_X86_INTEL_USERCOPY
static unsigned long
-__copy_user_intel(void __user *to, const void *from, unsigned long size)
+__generic_copy_to_user_intel(void __user *to, const void *from, unsigned long size)
{
int d0, d1;
__asm__ __volatile__(
" .align 2,0x90\n"
"3: movl 0(%4), %%eax\n"
"4: movl 4(%4), %%edx\n"
- "5: movl %%eax, 0(%3)\n"
- "6: movl %%edx, 4(%3)\n"
+ "5: "__copyuser_seg" movl %%eax, 0(%3)\n"
+ "6: "__copyuser_seg" movl %%edx, 4(%3)\n"
"7: movl 8(%4), %%eax\n"
"8: movl 12(%4),%%edx\n"
- "9: movl %%eax, 8(%3)\n"
- "10: movl %%edx, 12(%3)\n"
+ "9: "__copyuser_seg" movl %%eax, 8(%3)\n"
+ "10: "__copyuser_seg" movl %%edx, 12(%3)\n"
"11: movl 16(%4), %%eax\n"
"12: movl 20(%4), %%edx\n"
- "13: movl %%eax, 16(%3)\n"
- "14: movl %%edx, 20(%3)\n"
+ "13: "__copyuser_seg" movl %%eax, 16(%3)\n"
+ "14: "__copyuser_seg" movl %%edx, 20(%3)\n"
"15: movl 24(%4), %%eax\n"
"16: movl 28(%4), %%edx\n"
- "17: movl %%eax, 24(%3)\n"
- "18: movl %%edx, 28(%3)\n"
+ "17: "__copyuser_seg" movl %%eax, 24(%3)\n"
+ "18: "__copyuser_seg" movl %%edx, 28(%3)\n"
"19: movl 32(%4), %%eax\n"
"20: movl 36(%4), %%edx\n"
- "21: movl %%eax, 32(%3)\n"
- "22: movl %%edx, 36(%3)\n"
+ "21: "__copyuser_seg" movl %%eax, 32(%3)\n"
+ "22: "__copyuser_seg" movl %%edx, 36(%3)\n"
"23: movl 40(%4), %%eax\n"
"24: movl 44(%4), %%edx\n"
- "25: movl %%eax, 40(%3)\n"
- "26: movl %%edx, 44(%3)\n"
+ "25: "__copyuser_seg" movl %%eax, 40(%3)\n"
+ "26: "__copyuser_seg" movl %%edx, 44(%3)\n"
"27: movl 48(%4), %%eax\n"
"28: movl 52(%4), %%edx\n"
- "29: movl %%eax, 48(%3)\n"
- "30: movl %%edx, 52(%3)\n"
+ "29: "__copyuser_seg" movl %%eax, 48(%3)\n"
+ "30: "__copyuser_seg" movl %%edx, 52(%3)\n"
"31: movl 56(%4), %%eax\n"
"32: movl 60(%4), %%edx\n"
- "33: movl %%eax, 56(%3)\n"
- "34: movl %%edx, 60(%3)\n"
+ "33: "__copyuser_seg" movl %%eax, 56(%3)\n"
+ "34: "__copyuser_seg" movl %%edx, 60(%3)\n"
" addl $-64, %0\n"
" addl $64, %4\n"
" addl $64, %3\n"
" shrl $2, %0\n"
" andl $3, %%eax\n"
" cld\n"
+ __COPYUSER_SET_ES
"99: rep; movsl\n"
"36: movl %%eax, %0\n"
"37: rep; movsb\n"
"100:\n"
+ __COPYUSER_RESTORE_ES
+ ".section .fixup,\"ax\"\n"
+ "101: lea 0(%%eax,%0,4),%0\n"
+ " jmp 100b\n"
+ ".previous\n"
+ _ASM_EXTABLE(1b,100b)
+ _ASM_EXTABLE(2b,100b)
+ _ASM_EXTABLE(3b,100b)
+ _ASM_EXTABLE(4b,100b)
+ _ASM_EXTABLE(5b,100b)
+ _ASM_EXTABLE(6b,100b)
+ _ASM_EXTABLE(7b,100b)
+ _ASM_EXTABLE(8b,100b)
+ _ASM_EXTABLE(9b,100b)
+ _ASM_EXTABLE(10b,100b)
+ _ASM_EXTABLE(11b,100b)
+ _ASM_EXTABLE(12b,100b)
+ _ASM_EXTABLE(13b,100b)
+ _ASM_EXTABLE(14b,100b)
+ _ASM_EXTABLE(15b,100b)
+ _ASM_EXTABLE(16b,100b)
+ _ASM_EXTABLE(17b,100b)
+ _ASM_EXTABLE(18b,100b)
+ _ASM_EXTABLE(19b,100b)
+ _ASM_EXTABLE(20b,100b)
+ _ASM_EXTABLE(21b,100b)
+ _ASM_EXTABLE(22b,100b)
+ _ASM_EXTABLE(23b,100b)
+ _ASM_EXTABLE(24b,100b)
+ _ASM_EXTABLE(25b,100b)
+ _ASM_EXTABLE(26b,100b)
+ _ASM_EXTABLE(27b,100b)
+ _ASM_EXTABLE(28b,100b)
+ _ASM_EXTABLE(29b,100b)
+ _ASM_EXTABLE(30b,100b)
+ _ASM_EXTABLE(31b,100b)
+ _ASM_EXTABLE(32b,100b)
+ _ASM_EXTABLE(33b,100b)
+ _ASM_EXTABLE(34b,100b)
+ _ASM_EXTABLE(35b,100b)
+ _ASM_EXTABLE(36b,100b)
+ _ASM_EXTABLE(37b,100b)
+ _ASM_EXTABLE(99b,101b)
+ : "=&c"(size), "=&D" (d0), "=&S" (d1)
+ : "1"(to), "2"(from), "0"(size)
+ : "eax", "edx", "memory");
+ return size;
+}
+
+static unsigned long
+__generic_copy_from_user_intel(void *to, const void __user *from, unsigned long size)
+{
+ int d0, d1;
+ __asm__ __volatile__(
+ " .align 2,0x90\n"
+ "1: "__copyuser_seg" movl 32(%4), %%eax\n"
+ " cmpl $67, %0\n"
+ " jbe 3f\n"
+ "2: "__copyuser_seg" movl 64(%4), %%eax\n"
+ " .align 2,0x90\n"
+ "3: "__copyuser_seg" movl 0(%4), %%eax\n"
+ "4: "__copyuser_seg" movl 4(%4), %%edx\n"
+ "5: movl %%eax, 0(%3)\n"
+ "6: movl %%edx, 4(%3)\n"
+ "7: "__copyuser_seg" movl 8(%4), %%eax\n"
+ "8: "__copyuser_seg" movl 12(%4),%%edx\n"
+ "9: movl %%eax, 8(%3)\n"
+ "10: movl %%edx, 12(%3)\n"
+ "11: "__copyuser_seg" movl 16(%4), %%eax\n"
+ "12: "__copyuser_seg" movl 20(%4), %%edx\n"
+ "13: movl %%eax, 16(%3)\n"
+ "14: movl %%edx, 20(%3)\n"
+ "15: "__copyuser_seg" movl 24(%4), %%eax\n"
+ "16: "__copyuser_seg" movl 28(%4), %%edx\n"
+ "17: movl %%eax, 24(%3)\n"
+ "18: movl %%edx, 28(%3)\n"
+ "19: "__copyuser_seg" movl 32(%4), %%eax\n"
+ "20: "__copyuser_seg" movl 36(%4), %%edx\n"
+ "21: movl %%eax, 32(%3)\n"
+ "22: movl %%edx, 36(%3)\n"
+ "23: "__copyuser_seg" movl 40(%4), %%eax\n"
+ "24: "__copyuser_seg" movl 44(%4), %%edx\n"
+ "25: movl %%eax, 40(%3)\n"
+ "26: movl %%edx, 44(%3)\n"
+ "27: "__copyuser_seg" movl 48(%4), %%eax\n"
+ "28: "__copyuser_seg" movl 52(%4), %%edx\n"
+ "29: movl %%eax, 48(%3)\n"
+ "30: movl %%edx, 52(%3)\n"
+ "31: "__copyuser_seg" movl 56(%4), %%eax\n"
+ "32: "__copyuser_seg" movl 60(%4), %%edx\n"
+ "33: movl %%eax, 56(%3)\n"
+ "34: movl %%edx, 60(%3)\n"
+ " addl $-64, %0\n"
+ " addl $64, %4\n"
+ " addl $64, %3\n"
+ " cmpl $63, %0\n"
+ " ja 1b\n"
+ "35: movl %0, %%eax\n"
+ " shrl $2, %0\n"
+ " andl $3, %%eax\n"
+ " cld\n"
+ "99: rep; "__copyuser_seg" movsl\n"
+ "36: movl %%eax, %0\n"
+ "37: rep; "__copyuser_seg" movsb\n"
+ "100:\n"
".section .fixup,\"ax\"\n"
"101: lea 0(%%eax,%0,4),%0\n"
" jmp 100b\n"
int d0, d1;
__asm__ __volatile__(
" .align 2,0x90\n"
- "0: movl 32(%4), %%eax\n"
+ "0: "__copyuser_seg" movl 32(%4), %%eax\n"
" cmpl $67, %0\n"
" jbe 2f\n"
- "1: movl 64(%4), %%eax\n"
+ "1: "__copyuser_seg" movl 64(%4), %%eax\n"
" .align 2,0x90\n"
- "2: movl 0(%4), %%eax\n"
- "21: movl 4(%4), %%edx\n"
+ "2: "__copyuser_seg" movl 0(%4), %%eax\n"
+ "21: "__copyuser_seg" movl 4(%4), %%edx\n"
" movl %%eax, 0(%3)\n"
" movl %%edx, 4(%3)\n"
- "3: movl 8(%4), %%eax\n"
- "31: movl 12(%4),%%edx\n"
+ "3: "__copyuser_seg" movl 8(%4), %%eax\n"
+ "31: "__copyuser_seg" movl 12(%4),%%edx\n"
" movl %%eax, 8(%3)\n"
" movl %%edx, 12(%3)\n"
- "4: movl 16(%4), %%eax\n"
- "41: movl 20(%4), %%edx\n"
+ "4: "__copyuser_seg" movl 16(%4), %%eax\n"
+ "41: "__copyuser_seg" movl 20(%4), %%edx\n"
" movl %%eax, 16(%3)\n"
" movl %%edx, 20(%3)\n"
- "10: movl 24(%4), %%eax\n"
- "51: movl 28(%4), %%edx\n"
+ "10: "__copyuser_seg" movl 24(%4), %%eax\n"
+ "51: "__copyuser_seg" movl 28(%4), %%edx\n"
" movl %%eax, 24(%3)\n"
" movl %%edx, 28(%3)\n"
- "11: movl 32(%4), %%eax\n"
- "61: movl 36(%4), %%edx\n"
+ "11: "__copyuser_seg" movl 32(%4), %%eax\n"
+ "61: "__copyuser_seg" movl 36(%4), %%edx\n"
" movl %%eax, 32(%3)\n"
" movl %%edx, 36(%3)\n"
- "12: movl 40(%4), %%eax\n"
- "71: movl 44(%4), %%edx\n"
+ "12: "__copyuser_seg" movl 40(%4), %%eax\n"
+ "71: "__copyuser_seg" movl 44(%4), %%edx\n"
" movl %%eax, 40(%3)\n"
" movl %%edx, 44(%3)\n"
- "13: movl 48(%4), %%eax\n"
- "81: movl 52(%4), %%edx\n"
+ "13: "__copyuser_seg" movl 48(%4), %%eax\n"
+ "81: "__copyuser_seg" movl 52(%4), %%edx\n"
" movl %%eax, 48(%3)\n"
" movl %%edx, 52(%3)\n"
- "14: movl 56(%4), %%eax\n"
- "91: movl 60(%4), %%edx\n"
+ "14: "__copyuser_seg" movl 56(%4), %%eax\n"
+ "91: "__copyuser_seg" movl 60(%4), %%edx\n"
" movl %%eax, 56(%3)\n"
" movl %%edx, 60(%3)\n"
" addl $-64, %0\n"
" shrl $2, %0\n"
" andl $3, %%eax\n"
" cld\n"
- "6: rep; movsl\n"
+ "6: rep; "__copyuser_seg" movsl\n"
" movl %%eax,%0\n"
- "7: rep; movsb\n"
+ "7: rep; "__copyuser_seg" movsb\n"
"8:\n"
".section .fixup,\"ax\"\n"
"9: lea 0(%%eax,%0,4),%0\n"
__asm__ __volatile__(
" .align 2,0x90\n"
- "0: movl 32(%4), %%eax\n"
+ "0: "__copyuser_seg" movl 32(%4), %%eax\n"
" cmpl $67, %0\n"
" jbe 2f\n"
- "1: movl 64(%4), %%eax\n"
+ "1: "__copyuser_seg" movl 64(%4), %%eax\n"
" .align 2,0x90\n"
- "2: movl 0(%4), %%eax\n"
- "21: movl 4(%4), %%edx\n"
+ "2: "__copyuser_seg" movl 0(%4), %%eax\n"
+ "21: "__copyuser_seg" movl 4(%4), %%edx\n"
" movnti %%eax, 0(%3)\n"
" movnti %%edx, 4(%3)\n"
- "3: movl 8(%4), %%eax\n"
- "31: movl 12(%4),%%edx\n"
+ "3: "__copyuser_seg" movl 8(%4), %%eax\n"
+ "31: "__copyuser_seg" movl 12(%4),%%edx\n"
" movnti %%eax, 8(%3)\n"
" movnti %%edx, 12(%3)\n"
- "4: movl 16(%4), %%eax\n"
- "41: movl 20(%4), %%edx\n"
+ "4: "__copyuser_seg" movl 16(%4), %%eax\n"
+ "41: "__copyuser_seg" movl 20(%4), %%edx\n"
" movnti %%eax, 16(%3)\n"
" movnti %%edx, 20(%3)\n"
- "10: movl 24(%4), %%eax\n"
- "51: movl 28(%4), %%edx\n"
+ "10: "__copyuser_seg" movl 24(%4), %%eax\n"
+ "51: "__copyuser_seg" movl 28(%4), %%edx\n"
" movnti %%eax, 24(%3)\n"
" movnti %%edx, 28(%3)\n"
- "11: movl 32(%4), %%eax\n"
- "61: movl 36(%4), %%edx\n"
+ "11: "__copyuser_seg" movl 32(%4), %%eax\n"
+ "61: "__copyuser_seg" movl 36(%4), %%edx\n"
" movnti %%eax, 32(%3)\n"
" movnti %%edx, 36(%3)\n"
- "12: movl 40(%4), %%eax\n"
- "71: movl 44(%4), %%edx\n"
+ "12: "__copyuser_seg" movl 40(%4), %%eax\n"
+ "71: "__copyuser_seg" movl 44(%4), %%edx\n"
" movnti %%eax, 40(%3)\n"
" movnti %%edx, 44(%3)\n"
- "13: movl 48(%4), %%eax\n"
- "81: movl 52(%4), %%edx\n"
+ "13: "__copyuser_seg" movl 48(%4), %%eax\n"
+ "81: "__copyuser_seg" movl 52(%4), %%edx\n"
" movnti %%eax, 48(%3)\n"
" movnti %%edx, 52(%3)\n"
- "14: movl 56(%4), %%eax\n"
- "91: movl 60(%4), %%edx\n"
+ "14: "__copyuser_seg" movl 56(%4), %%eax\n"
+ "91: "__copyuser_seg" movl 60(%4), %%edx\n"
" movnti %%eax, 56(%3)\n"
" movnti %%edx, 60(%3)\n"
" addl $-64, %0\n"
" shrl $2, %0\n"
" andl $3, %%eax\n"
" cld\n"
- "6: rep; movsl\n"
+ "6: rep; "__copyuser_seg" movsl\n"
" movl %%eax,%0\n"
- "7: rep; movsb\n"
+ "7: rep; "__copyuser_seg" movsb\n"
"8:\n"
".section .fixup,\"ax\"\n"
"9: lea 0(%%eax,%0,4),%0\n"
__asm__ __volatile__(
" .align 2,0x90\n"
- "0: movl 32(%4), %%eax\n"
+ "0: "__copyuser_seg" movl 32(%4), %%eax\n"
" cmpl $67, %0\n"
" jbe 2f\n"
- "1: movl 64(%4), %%eax\n"
+ "1: "__copyuser_seg" movl 64(%4), %%eax\n"
" .align 2,0x90\n"
- "2: movl 0(%4), %%eax\n"
- "21: movl 4(%4), %%edx\n"
+ "2: "__copyuser_seg" movl 0(%4), %%eax\n"
+ "21: "__copyuser_seg" movl 4(%4), %%edx\n"
" movnti %%eax, 0(%3)\n"
" movnti %%edx, 4(%3)\n"
- "3: movl 8(%4), %%eax\n"
- "31: movl 12(%4),%%edx\n"
+ "3: "__copyuser_seg" movl 8(%4), %%eax\n"
+ "31: "__copyuser_seg" movl 12(%4),%%edx\n"
" movnti %%eax, 8(%3)\n"
" movnti %%edx, 12(%3)\n"
- "4: movl 16(%4), %%eax\n"
- "41: movl 20(%4), %%edx\n"
+ "4: "__copyuser_seg" movl 16(%4), %%eax\n"
+ "41: "__copyuser_seg" movl 20(%4), %%edx\n"
" movnti %%eax, 16(%3)\n"
" movnti %%edx, 20(%3)\n"
- "10: movl 24(%4), %%eax\n"
- "51: movl 28(%4), %%edx\n"
+ "10: "__copyuser_seg" movl 24(%4), %%eax\n"
+ "51: "__copyuser_seg" movl 28(%4), %%edx\n"
" movnti %%eax, 24(%3)\n"
" movnti %%edx, 28(%3)\n"
- "11: movl 32(%4), %%eax\n"
- "61: movl 36(%4), %%edx\n"
+ "11: "__copyuser_seg" movl 32(%4), %%eax\n"
+ "61: "__copyuser_seg" movl 36(%4), %%edx\n"
" movnti %%eax, 32(%3)\n"
" movnti %%edx, 36(%3)\n"
- "12: movl 40(%4), %%eax\n"
- "71: movl 44(%4), %%edx\n"
+ "12: "__copyuser_seg" movl 40(%4), %%eax\n"
+ "71: "__copyuser_seg" movl 44(%4), %%edx\n"
" movnti %%eax, 40(%3)\n"
" movnti %%edx, 44(%3)\n"
- "13: movl 48(%4), %%eax\n"
- "81: movl 52(%4), %%edx\n"
+ "13: "__copyuser_seg" movl 48(%4), %%eax\n"
+ "81: "__copyuser_seg" movl 52(%4), %%edx\n"
" movnti %%eax, 48(%3)\n"
" movnti %%edx, 52(%3)\n"
- "14: movl 56(%4), %%eax\n"
- "91: movl 60(%4), %%edx\n"
+ "14: "__copyuser_seg" movl 56(%4), %%eax\n"
+ "91: "__copyuser_seg" movl 60(%4), %%edx\n"
" movnti %%eax, 56(%3)\n"
" movnti %%edx, 60(%3)\n"
" addl $-64, %0\n"
" shrl $2, %0\n"
" andl $3, %%eax\n"
" cld\n"
- "6: rep; movsl\n"
+ "6: rep; "__copyuser_seg" movsl\n"
" movl %%eax,%0\n"
- "7: rep; movsb\n"
+ "7: rep; "__copyuser_seg" movsb\n"
"8:\n"
".section .fixup,\"ax\"\n"
"9: lea 0(%%eax,%0,4),%0\n"
*/
unsigned long __copy_user_zeroing_intel(void *to, const void __user *from,
unsigned long size);
-unsigned long __copy_user_intel(void __user *to, const void *from,
+unsigned long __generic_copy_to_user_intel(void __user *to, const void *from,
+ unsigned long size);
+unsigned long __generic_copy_from_user_intel(void *to, const void __user *from,
unsigned long size);
unsigned long __copy_user_zeroing_intel_nocache(void *to,
const void __user *from, unsigned long size);
#endif /* CONFIG_X86_INTEL_USERCOPY */
/* Generic arbitrary sized copy. */
-#define __copy_user(to, from, size) \
+#define __copy_user(to, from, size, prefix, set, restore) \
do { \
int __d0, __d1, __d2; \
__asm__ __volatile__( \
+ set \
" cmp $7,%0\n" \
" jbe 1f\n" \
" movl %1,%0\n" \
" negl %0\n" \
" andl $7,%0\n" \
" subl %0,%3\n" \
- "4: rep; movsb\n" \
+ "4: rep; "prefix"movsb\n" \
" movl %3,%0\n" \
" shrl $2,%0\n" \
" andl $3,%3\n" \
" .align 2,0x90\n" \
- "0: rep; movsl\n" \
+ "0: rep; "prefix"movsl\n" \
" movl %3,%0\n" \
- "1: rep; movsb\n" \
+ "1: rep; "prefix"movsb\n" \
"2:\n" \
+ restore \
".section .fixup,\"ax\"\n" \
"5: addl %3,%0\n" \
" jmp 2b\n" \
" negl %0\n" \
" andl $7,%0\n" \
" subl %0,%3\n" \
- "4: rep; movsb\n" \
+ "4: rep; "__copyuser_seg"movsb\n" \
" movl %3,%0\n" \
" shrl $2,%0\n" \
" andl $3,%3\n" \
" .align 2,0x90\n" \
- "0: rep; movsl\n" \
+ "0: rep; "__copyuser_seg"movsl\n" \
" movl %3,%0\n" \
- "1: rep; movsb\n" \
+ "1: rep; "__copyuser_seg"movsb\n" \
"2:\n" \
".section .fixup,\"ax\"\n" \
"5: addl %3,%0\n" \
{
stac();
if (movsl_is_ok(to, from, n))
- __copy_user(to, from, n);
+ __copy_user(to, from, n, "", __COPYUSER_SET_ES, __COPYUSER_RESTORE_ES);
else
- n = __copy_user_intel(to, from, n);
+ n = __generic_copy_to_user_intel(to, from, n);
clac();
return n;
}
{
stac();
if (movsl_is_ok(to, from, n))
- __copy_user(to, from, n);
+ __copy_user(to, from, n, __copyuser_seg, "", "");
else
- n = __copy_user_intel((void __user *)to,
- (const void *)from, n);
+ n = __generic_copy_from_user_intel(to, from, n);
clac();
return n;
}
if (n > 64 && cpu_has_xmm2)
n = __copy_user_intel_nocache(to, from, n);
else
- __copy_user(to, from, n);
+ __copy_user(to, from, n, __copyuser_seg, "", "");
#else
- __copy_user(to, from, n);
+ __copy_user(to, from, n, __copyuser_seg, "", "");
#endif
clac();
return n;
}
EXPORT_SYMBOL(__copy_from_user_ll_nocache_nozero);
-/**
- * copy_to_user: - Copy a block of data into user space.
- * @to: Destination address, in user space.
- * @from: Source address, in kernel space.
- * @n: Number of bytes to copy.
- *
- * Context: User context only. This function may sleep.
- *
- * Copy data from kernel space to user space.
- *
- * Returns number of bytes that could not be copied.
- * On success, this will be zero.
- */
-unsigned long _copy_to_user(void __user *to, const void *from, unsigned n)
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+void __set_fs(mm_segment_t x)
{
- if (access_ok(VERIFY_WRITE, to, n))
- n = __copy_to_user(to, from, n);
- return n;
+ switch (x.seg) {
+ case 0:
+ loadsegment(gs, 0);
+ break;
+ case TASK_SIZE_MAX:
+ loadsegment(gs, __USER_DS);
+ break;
+ case -1UL:
+ loadsegment(gs, __KERNEL_DS);
+ break;
+ default:
+ BUG();
+ }
}
-EXPORT_SYMBOL(_copy_to_user);
+EXPORT_SYMBOL(__set_fs);
-/**
- * copy_from_user: - Copy a block of data from user space.
- * @to: Destination address, in kernel space.
- * @from: Source address, in user space.
- * @n: Number of bytes to copy.
- *
- * Context: User context only. This function may sleep.
- *
- * Copy data from user space to kernel space.
- *
- * Returns number of bytes that could not be copied.
- * On success, this will be zero.
- *
- * If some data could not be copied, this function will pad the copied
- * data to the requested size using zero bytes.
- */
-unsigned long _copy_from_user(void *to, const void __user *from, unsigned n)
+void set_fs(mm_segment_t x)
{
- if (access_ok(VERIFY_READ, from, n))
- n = __copy_from_user(to, from, n);
- else
- memset(to, 0, n);
- return n;
+ current_thread_info()->addr_limit = x;
+ __set_fs(x);
}
-EXPORT_SYMBOL(_copy_from_user);
+EXPORT_SYMBOL(set_fs);
+#endif
might_fault();
/* no memory constraint because it doesn't change any memory gcc knows
about */
+ pax_open_userland();
stac();
asm volatile(
" testq %[size8],%[size8]\n"
_ASM_EXTABLE(0b,3b)
_ASM_EXTABLE(1b,2b)
: [size8] "=&c"(size), [dst] "=&D" (__d0)
- : [size1] "r"(size & 7), "[size8]" (size / 8), "[dst]"(addr),
+ : [size1] "r"(size & 7), "[size8]" (size / 8), "[dst]"(____m(addr)),
[zero] "r" (0UL), [eight] "r" (8UL));
clac();
+ pax_close_userland();
return size;
}
EXPORT_SYMBOL(__clear_user);
}
EXPORT_SYMBOL(clear_user);
-unsigned long copy_in_user(void __user *to, const void __user *from, unsigned len)
+unsigned long copy_in_user(void __user *to, const void __user *from, unsigned long len)
{
- if (access_ok(VERIFY_WRITE, to, len) && access_ok(VERIFY_READ, from, len)) {
- return copy_user_generic((__force void *)to, (__force void *)from, len);
- }
- return len;
+ if (access_ok(VERIFY_WRITE, to, len) && access_ok(VERIFY_READ, from, len))
+ return copy_user_generic((void __force_kernel *)____m(to), (void __force_kernel *)____m(from), len);
+ return len;
}
EXPORT_SYMBOL(copy_in_user);
* it is not necessary to optimize tail handling.
*/
__visible unsigned long
-copy_user_handle_tail(char *to, char *from, unsigned len, unsigned zerorest)
+copy_user_handle_tail(char __user *to, char __user *from, unsigned long len, unsigned zerorest)
{
char c;
unsigned zero_len;
+ clac();
+ pax_close_userland();
for (; len; --len, to++) {
if (__get_user_nocheck(c, from++, sizeof(char)))
break;
for (c = 0, zero_len = len; zerorest && zero_len; --zero_len)
if (__put_user_nocheck(c, to++, sizeof(char)))
break;
- clac();
return len;
}
obj-$(CONFIG_MEMTEST) += memtest.o
obj-$(CONFIG_X86_INTEL_MPX) += mpx.o
+
+quote:="
+obj-$(CONFIG_X86_64) += uderef_64.o
+CFLAGS_uderef_64.o := $(subst $(quote),,$(CONFIG_ARCH_HWEIGHT_CFLAGS))
static inline unsigned long
ex_insn_addr(const struct exception_table_entry *x)
{
- return (unsigned long)&x->insn + x->insn;
+ unsigned long reloc = 0;
+
+#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_KERNEXEC)
+ reloc = ____LOAD_PHYSICAL_ADDR - LOAD_PHYSICAL_ADDR;
+#endif
+
+ return (unsigned long)&x->insn + x->insn + reloc;
}
static inline unsigned long
ex_fixup_addr(const struct exception_table_entry *x)
{
- return (unsigned long)&x->fixup + x->fixup;
+ unsigned long reloc = 0;
+
+#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_KERNEXEC)
+ reloc = ____LOAD_PHYSICAL_ADDR - LOAD_PHYSICAL_ADDR;
+#endif
+
+ return (unsigned long)&x->fixup + x->fixup + reloc;
}
int fixup_exception(struct pt_regs *regs)
unsigned long new_ip;
#ifdef CONFIG_PNPBIOS
- if (unlikely(SEGMENT_IS_PNP_CODE(regs->cs))) {
+ if (unlikely(!v8086_mode(regs) && SEGMENT_IS_PNP_CODE(regs->cs))) {
extern u32 pnp_bios_fault_eip, pnp_bios_fault_esp;
extern u32 pnp_bios_is_utter_crap;
pnp_bios_is_utter_crap = 1;
i += 4;
p->fixup -= i;
i += 4;
+
+#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_KERNEXEC)
+ BUILD_BUG_ON(!IS_ENABLED(CONFIG_BUILDTIME_EXTABLE_SORT));
+ p->insn -= ____LOAD_PHYSICAL_ADDR - LOAD_PHYSICAL_ADDR;
+ p->fixup -= ____LOAD_PHYSICAL_ADDR - LOAD_PHYSICAL_ADDR;
+#endif
+
}
}
#include <linux/hugetlb.h> /* hstate_index_to_shift */
#include <linux/prefetch.h> /* prefetchw */
#include <linux/context_tracking.h> /* exception_enter(), ... */
+#include <linux/unistd.h>
+#include <linux/compiler.h>
#include <asm/traps.h> /* dotraplinkage, ... */
#include <asm/pgalloc.h> /* pgd_*(), ... */
#include <asm/kmemcheck.h> /* kmemcheck_*(), ... */
#include <asm/fixmap.h> /* VSYSCALL_ADDR */
#include <asm/vsyscall.h> /* emulate_vsyscall */
+#include <asm/tlbflush.h>
+
+#if defined(CONFIG_X86_64) && defined(CONFIG_PAX_MEMORY_UDEREF)
+#include <asm/stacktrace.h>
+#endif
#define CREATE_TRACE_POINTS
#include <asm/trace/exceptions.h>
int ret = 0;
/* kprobe_running() needs smp_processor_id() */
- if (kprobes_built_in() && !user_mode_vm(regs)) {
+ if (kprobes_built_in() && !user_mode(regs)) {
preempt_disable();
if (kprobe_running() && kprobe_fault_handler(regs, 14))
ret = 1;
return !instr_lo || (instr_lo>>1) == 1;
case 0x00:
/* Prefetch instruction is 0x0F0D or 0x0F18 */
- if (probe_kernel_address(instr, opcode))
+ if (user_mode(regs)) {
+ if (__copy_from_user_inatomic(&opcode, (unsigned char __force_user *)(instr), 1))
+ return 0;
+ } else if (probe_kernel_address(instr, opcode))
return 0;
*prefetch = (instr_lo == 0xF) &&
while (instr < max_instr) {
unsigned char opcode;
- if (probe_kernel_address(instr, opcode))
+ if (user_mode(regs)) {
+ if (__copy_from_user_inatomic(&opcode, (unsigned char __force_user *)(instr), 1))
+ break;
+ } else if (probe_kernel_address(instr, opcode))
break;
instr++;
force_sig_info(si_signo, &info, tsk);
}
+#if defined(CONFIG_PAX_PAGEEXEC) || defined(CONFIG_PAX_SEGMEXEC)
+static bool pax_is_fetch_fault(struct pt_regs *regs, unsigned long error_code, unsigned long address);
+#endif
+
+#ifdef CONFIG_PAX_EMUTRAMP
+static int pax_handle_fetch_fault(struct pt_regs *regs);
+#endif
+
+#ifdef CONFIG_PAX_PAGEEXEC
+static inline pmd_t * pax_get_pmd(struct mm_struct *mm, unsigned long address)
+{
+ pgd_t *pgd;
+ pud_t *pud;
+ pmd_t *pmd;
+
+ pgd = pgd_offset(mm, address);
+ if (!pgd_present(*pgd))
+ return NULL;
+ pud = pud_offset(pgd, address);
+ if (!pud_present(*pud))
+ return NULL;
+ pmd = pmd_offset(pud, address);
+ if (!pmd_present(*pmd))
+ return NULL;
+ return pmd;
+}
+#endif
+
DEFINE_SPINLOCK(pgd_lock);
LIST_HEAD(pgd_list);
for (address = VMALLOC_START & PMD_MASK;
address >= TASK_SIZE && address < FIXADDR_TOP;
address += PMD_SIZE) {
+
+#ifdef CONFIG_PAX_PER_CPU_PGD
+ unsigned long cpu;
+#else
struct page *page;
+#endif
spin_lock(&pgd_lock);
+
+#ifdef CONFIG_PAX_PER_CPU_PGD
+ for (cpu = 0; cpu < nr_cpu_ids; ++cpu) {
+ pgd_t *pgd = get_cpu_pgd(cpu, user);
+ pmd_t *ret;
+
+ ret = vmalloc_sync_one(pgd, address);
+ if (!ret)
+ break;
+ pgd = get_cpu_pgd(cpu, kernel);
+#else
list_for_each_entry(page, &pgd_list, lru) {
+ pgd_t *pgd;
spinlock_t *pgt_lock;
pmd_t *ret;
pgt_lock = &pgd_page_get_mm(page)->page_table_lock;
spin_lock(pgt_lock);
- ret = vmalloc_sync_one(page_address(page), address);
+ pgd = page_address(page);
+#endif
+
+ ret = vmalloc_sync_one(pgd, address);
+
+#ifndef CONFIG_PAX_PER_CPU_PGD
spin_unlock(pgt_lock);
+#endif
if (!ret)
break;
* an interrupt in the middle of a task switch..
*/
pgd_paddr = read_cr3();
+
+#ifdef CONFIG_PAX_PER_CPU_PGD
+ BUG_ON(__pa(get_cpu_pgd(smp_processor_id(), kernel)) != (pgd_paddr & __PHYSICAL_MASK));
+ vmalloc_sync_one(__va(pgd_paddr + PAGE_SIZE), address);
+#endif
+
pmd_k = vmalloc_sync_one(__va(pgd_paddr), address);
if (!pmd_k)
return -1;
* happen within a race in page table update. In the later
* case just flush:
*/
- pgd = pgd_offset(current->active_mm, address);
+
pgd_ref = pgd_offset_k(address);
if (pgd_none(*pgd_ref))
return -1;
+#ifdef CONFIG_PAX_PER_CPU_PGD
+ BUG_ON(__pa(get_cpu_pgd(smp_processor_id(), kernel)) != (read_cr3() & __PHYSICAL_MASK));
+ pgd = pgd_offset_cpu(smp_processor_id(), user, address);
+ if (pgd_none(*pgd)) {
+ set_pgd(pgd, *pgd_ref);
+ arch_flush_lazy_mmu_mode();
+ } else {
+ BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref));
+ }
+ pgd = pgd_offset_cpu(smp_processor_id(), kernel, address);
+#else
+ pgd = pgd_offset(current->active_mm, address);
+#endif
+
if (pgd_none(*pgd)) {
set_pgd(pgd, *pgd_ref);
arch_flush_lazy_mmu_mode();
static int is_errata100(struct pt_regs *regs, unsigned long address)
{
#ifdef CONFIG_X86_64
- if ((regs->cs == __USER32_CS || (regs->cs & (1<<2))) && (address >> 32))
+ if ((regs->cs == __USER32_CS || (regs->cs & SEGMENT_LDT)) && (address >> 32))
return 1;
#endif
return 0;
}
static const char nx_warning[] = KERN_CRIT
-"kernel tried to execute NX-protected page - exploit attempt? (uid: %d)\n";
+"kernel tried to execute NX-protected page - exploit attempt? (uid: %d, task: %s, pid: %d)\n";
static const char smep_warning[] = KERN_CRIT
-"unable to execute userspace code (SMEP?) (uid: %d)\n";
+"unable to execute userspace code (SMEP?) (uid: %d, task: %s, pid: %d)\n";
static void
show_fault_oops(struct pt_regs *regs, unsigned long error_code,
if (!oops_may_print())
return;
- if (error_code & PF_INSTR) {
+ if ((__supported_pte_mask & _PAGE_NX) && (error_code & PF_INSTR)) {
unsigned int level;
pgd_t *pgd;
pte_t *pte;
pte = lookup_address_in_pgd(pgd, address, &level);
if (pte && pte_present(*pte) && !pte_exec(*pte))
- printk(nx_warning, from_kuid(&init_user_ns, current_uid()));
+ printk(nx_warning, from_kuid_munged(&init_user_ns, current_uid()), current->comm, task_pid_nr(current));
if (pte && pte_present(*pte) && pte_exec(*pte) &&
(pgd_flags(*pgd) & _PAGE_USER) &&
(read_cr4() & X86_CR4_SMEP))
- printk(smep_warning, from_kuid(&init_user_ns, current_uid()));
+ printk(smep_warning, from_kuid(&init_user_ns, current_uid()), current->comm, task_pid_nr(current));
+ }
+
+#ifdef CONFIG_PAX_KERNEXEC
+ if (init_mm.start_code <= address && address < init_mm.end_code) {
+ if (current->signal->curr_ip)
+ printk(KERN_ERR "PAX: From %pI4: %s:%d, uid/euid: %u/%u, attempted to modify kernel code\n",
+ ¤t->signal->curr_ip, current->comm, task_pid_nr(current),
+ from_kuid_munged(&init_user_ns, current_uid()), from_kuid_munged(&init_user_ns, current_euid()));
+ else
+ printk(KERN_ERR "PAX: %s:%d, uid/euid: %u/%u, attempted to modify kernel code\n", current->comm, task_pid_nr(current),
+ from_kuid_munged(&init_user_ns, current_uid()), from_kuid_munged(&init_user_ns, current_euid()));
}
+#endif
printk(KERN_ALERT "BUG: unable to handle kernel ");
if (address < PAGE_SIZE)
return;
}
#endif
+
+#if defined(CONFIG_PAX_PAGEEXEC) || defined(CONFIG_PAX_SEGMEXEC)
+ if (pax_is_fetch_fault(regs, error_code, address)) {
+
+#ifdef CONFIG_PAX_EMUTRAMP
+ switch (pax_handle_fetch_fault(regs)) {
+ case 2:
+ return;
+ }
+#endif
+
+ pax_report_fault(regs, (void *)regs->ip, (void *)regs->sp);
+ do_group_exit(SIGKILL);
+ }
+#endif
+
/* Kernel addresses are always protection faults: */
if (address >= TASK_SIZE)
error_code |= PF_PROT;
if (fault & (VM_FAULT_HWPOISON|VM_FAULT_HWPOISON_LARGE)) {
printk(KERN_ERR
"MCE: Killing %s:%d due to hardware memory corruption fault at %lx\n",
- tsk->comm, tsk->pid, address);
+ tsk->comm, task_pid_nr(tsk), address);
code = BUS_MCEERR_AR;
}
#endif
return 1;
}
+#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_PAGEEXEC)
+static int pax_handle_pageexec_fault(struct pt_regs *regs, struct mm_struct *mm, unsigned long address, unsigned long error_code)
+{
+ pte_t *pte;
+ pmd_t *pmd;
+ spinlock_t *ptl;
+ unsigned char pte_mask;
+
+ if ((__supported_pte_mask & _PAGE_NX) || (error_code & (PF_PROT|PF_USER)) != (PF_PROT|PF_USER) || v8086_mode(regs) ||
+ !(mm->pax_flags & MF_PAX_PAGEEXEC))
+ return 0;
+
+ /* PaX: it's our fault, let's handle it if we can */
+
+ /* PaX: take a look at read faults before acquiring any locks */
+ if (unlikely(!(error_code & PF_WRITE) && (regs->ip == address))) {
+ /* instruction fetch attempt from a protected page in user mode */
+ up_read(&mm->mmap_sem);
+
+#ifdef CONFIG_PAX_EMUTRAMP
+ switch (pax_handle_fetch_fault(regs)) {
+ case 2:
+ return 1;
+ }
+#endif
+
+ pax_report_fault(regs, (void *)regs->ip, (void *)regs->sp);
+ do_group_exit(SIGKILL);
+ }
+
+ pmd = pax_get_pmd(mm, address);
+ if (unlikely(!pmd))
+ return 0;
+
+ pte = pte_offset_map_lock(mm, pmd, address, &ptl);
+ if (unlikely(!(pte_val(*pte) & _PAGE_PRESENT) || pte_user(*pte))) {
+ pte_unmap_unlock(pte, ptl);
+ return 0;
+ }
+
+ if (unlikely((error_code & PF_WRITE) && !pte_write(*pte))) {
+ /* write attempt to a protected page in user mode */
+ pte_unmap_unlock(pte, ptl);
+ return 0;
+ }
+
+#ifdef CONFIG_SMP
+ if (likely(address > get_limit(regs->cs) && cpu_isset(smp_processor_id(), mm->context.cpu_user_cs_mask)))
+#else
+ if (likely(address > get_limit(regs->cs)))
+#endif
+ {
+ set_pte(pte, pte_mkread(*pte));
+ __flush_tlb_one(address);
+ pte_unmap_unlock(pte, ptl);
+ up_read(&mm->mmap_sem);
+ return 1;
+ }
+
+ pte_mask = _PAGE_ACCESSED | _PAGE_USER | ((error_code & PF_WRITE) << (_PAGE_BIT_DIRTY-1));
+
+ /*
+ * PaX: fill DTLB with user rights and retry
+ */
+ __asm__ __volatile__ (
+ "orb %2,(%1)\n"
+#if defined(CONFIG_M586) || defined(CONFIG_M586TSC)
+/*
+ * PaX: let this uncommented 'invlpg' remind us on the behaviour of Intel's
+ * (and AMD's) TLBs. namely, they do not cache PTEs that would raise *any*
+ * page fault when examined during a TLB load attempt. this is true not only
+ * for PTEs holding a non-present entry but also present entries that will
+ * raise a page fault (such as those set up by PaX, or the copy-on-write
+ * mechanism). in effect it means that we do *not* need to flush the TLBs
+ * for our target pages since their PTEs are simply not in the TLBs at all.
+
+ * the best thing in omitting it is that we gain around 15-20% speed in the
+ * fast path of the page fault handler and can get rid of tracing since we
+ * can no longer flush unintended entries.
+ */
+ "invlpg (%0)\n"
+#endif
+ __copyuser_seg"testb $0,(%0)\n"
+ "xorb %3,(%1)\n"
+ :
+ : "r" (address), "r" (pte), "q" (pte_mask), "i" (_PAGE_USER)
+ : "memory", "cc");
+ pte_unmap_unlock(pte, ptl);
+ up_read(&mm->mmap_sem);
+ return 1;
+}
+#endif
+
/*
* Handle a spurious fault caused by a stale TLB entry.
*
static inline int
access_error(unsigned long error_code, struct vm_area_struct *vma)
{
+ if ((__supported_pte_mask & _PAGE_NX) && (error_code & PF_INSTR) && !(vma->vm_flags & VM_EXEC))
+ return 1;
+
if (error_code & PF_WRITE) {
/* write, present and write, not present: */
if (unlikely(!(vma->vm_flags & VM_WRITE)))
if (error_code & PF_USER)
return false;
- if (!user_mode_vm(regs) && (regs->flags & X86_EFLAGS_AC))
+ if (!user_mode(regs) && (regs->flags & X86_EFLAGS_AC))
return false;
return true;
tsk = current;
mm = tsk->mm;
+#if defined(CONFIG_X86_64) && defined(CONFIG_PAX_MEMORY_UDEREF)
+ if (!user_mode(regs) && address < 2 * pax_user_shadow_base) {
+ if (!search_exception_tables(regs->ip)) {
+ printk(KERN_ERR "PAX: please report this to pageexec@freemail.hu\n");
+ bad_area_nosemaphore(regs, error_code, address);
+ return;
+ }
+ if (address < pax_user_shadow_base) {
+ printk(KERN_ERR "PAX: please report this to pageexec@freemail.hu\n");
+ printk(KERN_ERR "PAX: faulting IP: %pS\n", (void *)regs->ip);
+ show_trace_log_lvl(NULL, NULL, (void *)regs->sp, regs->bp, KERN_ERR);
+ } else
+ address -= pax_user_shadow_base;
+ }
+#endif
+
/*
* Detect and handle instructions that would cause a page fault for
* both a tracked kernel page and a userspace page.
* User-mode registers count as a user access even for any
* potential system fault or CPU buglet:
*/
- if (user_mode_vm(regs)) {
+ if (user_mode(regs)) {
local_irq_enable();
error_code |= PF_USER;
flags |= FAULT_FLAG_USER;
might_sleep();
}
+#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_PAGEEXEC)
+ if (pax_handle_pageexec_fault(regs, mm, address, error_code))
+ return;
+#endif
+
vma = find_vma(mm, address);
if (unlikely(!vma)) {
bad_area(regs, error_code, address);
bad_area(regs, error_code, address);
return;
}
- if (error_code & PF_USER) {
- /*
- * Accessing the stack below %sp is always a bug.
- * The large cushion allows instructions like enter
- * and pusha to work. ("enter $65535, $31" pushes
- * 32 pointers and then decrements %sp by 65535.)
- */
- if (unlikely(address + 65536 + 32 * sizeof(unsigned long) < regs->sp)) {
- bad_area(regs, error_code, address);
- return;
- }
+ /*
+ * Accessing the stack below %sp is always a bug.
+ * The large cushion allows instructions like enter
+ * and pusha to work. ("enter $65535, $31" pushes
+ * 32 pointers and then decrements %sp by 65535.)
+ */
+ if (unlikely(address + 65536 + 32 * sizeof(unsigned long) < task_pt_regs(tsk)->sp)) {
+ bad_area(regs, error_code, address);
+ return;
+ }
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (unlikely((mm->pax_flags & MF_PAX_SEGMEXEC) && vma->vm_end - SEGMEXEC_TASK_SIZE - 1 < address - SEGMEXEC_TASK_SIZE - 1)) {
+ bad_area(regs, error_code, address);
+ return;
}
+#endif
+
if (unlikely(expand_stack(vma, address))) {
bad_area(regs, error_code, address);
return;
}
NOKPROBE_SYMBOL(trace_do_page_fault);
#endif /* CONFIG_TRACING */
+
+#if defined(CONFIG_PAX_PAGEEXEC) || defined(CONFIG_PAX_SEGMEXEC)
+static bool pax_is_fetch_fault(struct pt_regs *regs, unsigned long error_code, unsigned long address)
+{
+ struct mm_struct *mm = current->mm;
+ unsigned long ip = regs->ip;
+
+ if (v8086_mode(regs))
+ ip = ((regs->cs & 0xffff) << 4) + (ip & 0xffff);
+
+#ifdef CONFIG_PAX_PAGEEXEC
+ if (mm->pax_flags & MF_PAX_PAGEEXEC) {
+ if ((__supported_pte_mask & _PAGE_NX) && (error_code & PF_INSTR))
+ return true;
+ if (!(error_code & (PF_PROT | PF_WRITE)) && ip == address)
+ return true;
+ return false;
+ }
+#endif
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (mm->pax_flags & MF_PAX_SEGMEXEC) {
+ if (!(error_code & (PF_PROT | PF_WRITE)) && (ip + SEGMEXEC_TASK_SIZE == address))
+ return true;
+ return false;
+ }
+#endif
+
+ return false;
+}
+#endif
+
+#ifdef CONFIG_PAX_EMUTRAMP
+static int pax_handle_fetch_fault_32(struct pt_regs *regs)
+{
+ int err;
+
+ do { /* PaX: libffi trampoline emulation */
+ unsigned char mov, jmp;
+ unsigned int addr1, addr2;
+
+#ifdef CONFIG_X86_64
+ if ((regs->ip + 9) >> 32)
+ break;
+#endif
+
+ err = get_user(mov, (unsigned char __user *)regs->ip);
+ err |= get_user(addr1, (unsigned int __user *)(regs->ip + 1));
+ err |= get_user(jmp, (unsigned char __user *)(regs->ip + 5));
+ err |= get_user(addr2, (unsigned int __user *)(regs->ip + 6));
+
+ if (err)
+ break;
+
+ if (mov == 0xB8 && jmp == 0xE9) {
+ regs->ax = addr1;
+ regs->ip = (unsigned int)(regs->ip + addr2 + 10);
+ return 2;
+ }
+ } while (0);
+
+ do { /* PaX: gcc trampoline emulation #1 */
+ unsigned char mov1, mov2;
+ unsigned short jmp;
+ unsigned int addr1, addr2;
+
+#ifdef CONFIG_X86_64
+ if ((regs->ip + 11) >> 32)
+ break;
+#endif
+
+ err = get_user(mov1, (unsigned char __user *)regs->ip);
+ err |= get_user(addr1, (unsigned int __user *)(regs->ip + 1));
+ err |= get_user(mov2, (unsigned char __user *)(regs->ip + 5));
+ err |= get_user(addr2, (unsigned int __user *)(regs->ip + 6));
+ err |= get_user(jmp, (unsigned short __user *)(regs->ip + 10));
+
+ if (err)
+ break;
+
+ if (mov1 == 0xB9 && mov2 == 0xB8 && jmp == 0xE0FF) {
+ regs->cx = addr1;
+ regs->ax = addr2;
+ regs->ip = addr2;
+ return 2;
+ }
+ } while (0);
+
+ do { /* PaX: gcc trampoline emulation #2 */
+ unsigned char mov, jmp;
+ unsigned int addr1, addr2;
+
+#ifdef CONFIG_X86_64
+ if ((regs->ip + 9) >> 32)
+ break;
+#endif
+
+ err = get_user(mov, (unsigned char __user *)regs->ip);
+ err |= get_user(addr1, (unsigned int __user *)(regs->ip + 1));
+ err |= get_user(jmp, (unsigned char __user *)(regs->ip + 5));
+ err |= get_user(addr2, (unsigned int __user *)(regs->ip + 6));
+
+ if (err)
+ break;
+
+ if (mov == 0xB9 && jmp == 0xE9) {
+ regs->cx = addr1;
+ regs->ip = (unsigned int)(regs->ip + addr2 + 10);
+ return 2;
+ }
+ } while (0);
+
+ return 1; /* PaX in action */
+}
+
+#ifdef CONFIG_X86_64
+static int pax_handle_fetch_fault_64(struct pt_regs *regs)
+{
+ int err;
+
+ do { /* PaX: libffi trampoline emulation */
+ unsigned short mov1, mov2, jmp1;
+ unsigned char stcclc, jmp2;
+ unsigned long addr1, addr2;
+
+ err = get_user(mov1, (unsigned short __user *)regs->ip);
+ err |= get_user(addr1, (unsigned long __user *)(regs->ip + 2));
+ err |= get_user(mov2, (unsigned short __user *)(regs->ip + 10));
+ err |= get_user(addr2, (unsigned long __user *)(regs->ip + 12));
+ err |= get_user(stcclc, (unsigned char __user *)(regs->ip + 20));
+ err |= get_user(jmp1, (unsigned short __user *)(regs->ip + 21));
+ err |= get_user(jmp2, (unsigned char __user *)(regs->ip + 23));
+
+ if (err)
+ break;
+
+ if (mov1 == 0xBB49 && mov2 == 0xBA49 && (stcclc == 0xF8 || stcclc == 0xF9) && jmp1 == 0xFF49 && jmp2 == 0xE3) {
+ regs->r11 = addr1;
+ regs->r10 = addr2;
+ if (stcclc == 0xF8)
+ regs->flags &= ~X86_EFLAGS_CF;
+ else
+ regs->flags |= X86_EFLAGS_CF;
+ regs->ip = addr1;
+ return 2;
+ }
+ } while (0);
+
+ do { /* PaX: gcc trampoline emulation #1 */
+ unsigned short mov1, mov2, jmp1;
+ unsigned char jmp2;
+ unsigned int addr1;
+ unsigned long addr2;
+
+ err = get_user(mov1, (unsigned short __user *)regs->ip);
+ err |= get_user(addr1, (unsigned int __user *)(regs->ip + 2));
+ err |= get_user(mov2, (unsigned short __user *)(regs->ip + 6));
+ err |= get_user(addr2, (unsigned long __user *)(regs->ip + 8));
+ err |= get_user(jmp1, (unsigned short __user *)(regs->ip + 16));
+ err |= get_user(jmp2, (unsigned char __user *)(regs->ip + 18));
+
+ if (err)
+ break;
+
+ if (mov1 == 0xBB41 && mov2 == 0xBA49 && jmp1 == 0xFF49 && jmp2 == 0xE3) {
+ regs->r11 = addr1;
+ regs->r10 = addr2;
+ regs->ip = addr1;
+ return 2;
+ }
+ } while (0);
+
+ do { /* PaX: gcc trampoline emulation #2 */
+ unsigned short mov1, mov2, jmp1;
+ unsigned char jmp2;
+ unsigned long addr1, addr2;
+
+ err = get_user(mov1, (unsigned short __user *)regs->ip);
+ err |= get_user(addr1, (unsigned long __user *)(regs->ip + 2));
+ err |= get_user(mov2, (unsigned short __user *)(regs->ip + 10));
+ err |= get_user(addr2, (unsigned long __user *)(regs->ip + 12));
+ err |= get_user(jmp1, (unsigned short __user *)(regs->ip + 20));
+ err |= get_user(jmp2, (unsigned char __user *)(regs->ip + 22));
+
+ if (err)
+ break;
+
+ if (mov1 == 0xBB49 && mov2 == 0xBA49 && jmp1 == 0xFF49 && jmp2 == 0xE3) {
+ regs->r11 = addr1;
+ regs->r10 = addr2;
+ regs->ip = addr1;
+ return 2;
+ }
+ } while (0);
+
+ return 1; /* PaX in action */
+}
+#endif
+
+/*
+ * PaX: decide what to do with offenders (regs->ip = fault address)
+ *
+ * returns 1 when task should be killed
+ * 2 when gcc trampoline was detected
+ */
+static int pax_handle_fetch_fault(struct pt_regs *regs)
+{
+ if (v8086_mode(regs))
+ return 1;
+
+ if (!(current->mm->pax_flags & MF_PAX_EMUTRAMP))
+ return 1;
+
+#ifdef CONFIG_X86_32
+ return pax_handle_fetch_fault_32(regs);
+#else
+ if (regs->cs == __USER32_CS || (regs->cs & SEGMENT_LDT))
+ return pax_handle_fetch_fault_32(regs);
+ else
+ return pax_handle_fetch_fault_64(regs);
+#endif
+}
+#endif
+
+#if defined(CONFIG_PAX_PAGEEXEC) || defined(CONFIG_PAX_SEGMEXEC)
+void pax_report_insns(struct pt_regs *regs, void *pc, void *sp)
+{
+ long i;
+
+ printk(KERN_ERR "PAX: bytes at PC: ");
+ for (i = 0; i < 20; i++) {
+ unsigned char c;
+ if (get_user(c, (unsigned char __force_user *)pc+i))
+ printk(KERN_CONT "?? ");
+ else
+ printk(KERN_CONT "%02x ", c);
+ }
+ printk("\n");
+
+ printk(KERN_ERR "PAX: bytes at SP-%lu: ", (unsigned long)sizeof(long));
+ for (i = -1; i < 80 / (long)sizeof(long); i++) {
+ unsigned long c;
+ if (get_user(c, (unsigned long __force_user *)sp+i)) {
+#ifdef CONFIG_X86_32
+ printk(KERN_CONT "???????? ");
+#else
+ if ((regs->cs == __USER32_CS || (regs->cs & SEGMENT_LDT)))
+ printk(KERN_CONT "???????? ???????? ");
+ else
+ printk(KERN_CONT "???????????????? ");
+#endif
+ } else {
+#ifdef CONFIG_X86_64
+ if ((regs->cs == __USER32_CS || (regs->cs & SEGMENT_LDT))) {
+ printk(KERN_CONT "%08x ", (unsigned int)c);
+ printk(KERN_CONT "%08x ", (unsigned int)(c >> 32));
+ } else
+#endif
+ printk(KERN_CONT "%0*lx ", 2 * (int)sizeof(long), c);
+ }
+ }
+ printk("\n");
+}
+#endif
+
+/**
+ * probe_kernel_write(): safely attempt to write to a location
+ * @dst: address to write to
+ * @src: pointer to the data that shall be written
+ * @size: size of the data chunk
+ *
+ * Safely write to address @dst from the buffer at @src. If a kernel fault
+ * happens, handle that and return -EFAULT.
+ */
+long notrace probe_kernel_write(void *dst, const void *src, size_t size)
+{
+ long ret;
+ mm_segment_t old_fs = get_fs();
+
+ set_fs(KERNEL_DS);
+ pagefault_disable();
+ pax_open_kernel();
+ ret = __copy_to_user_inatomic((void __force_user *)dst, src, size);
+ pax_close_kernel();
+ pagefault_enable();
+ set_fs(old_fs);
+
+ return ret ? -EFAULT : 0;
+}
addr = start;
len = (unsigned long) nr_pages << PAGE_SHIFT;
end = start + len;
- if (unlikely(!access_ok(write ? VERIFY_WRITE : VERIFY_READ,
+ if (unlikely(!access_ok_noprefault(write ? VERIFY_WRITE : VERIFY_READ,
(void __user *)start, len)))
return 0;
goto slow_irqon;
#endif
+ if (unlikely(!access_ok_noprefault(write ? VERIFY_WRITE : VERIFY_READ,
+ (void __user *)start, len)))
+ return 0;
+
/*
* XXX: batch / limit 'nr', to avoid large irq off latency
* needs some instrumenting to determine the common sizes used by
idx = type + KM_TYPE_NR*smp_processor_id();
vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
BUG_ON(!pte_none(*(kmap_pte-idx)));
+
+ pax_open_kernel();
set_pte(kmap_pte-idx, mk_pte(page, prot));
+ pax_close_kernel();
+
arch_flush_lazy_mmu_mode();
return (void *)vaddr;
#ifdef CONFIG_HUGETLB_PAGE
static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file,
unsigned long addr, unsigned long len,
- unsigned long pgoff, unsigned long flags)
+ unsigned long pgoff, unsigned long flags, unsigned long offset)
{
struct hstate *h = hstate_file(file);
struct vm_unmapped_area_info info;
-
+
info.flags = 0;
info.length = len;
info.low_limit = current->mm->mmap_legacy_base;
info.high_limit = TASK_SIZE;
info.align_mask = PAGE_MASK & ~huge_page_mask(h);
info.align_offset = 0;
+ info.threadstack_offset = offset;
return vm_unmapped_area(&info);
}
static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file,
unsigned long addr0, unsigned long len,
- unsigned long pgoff, unsigned long flags)
+ unsigned long pgoff, unsigned long flags, unsigned long offset)
{
struct hstate *h = hstate_file(file);
struct vm_unmapped_area_info info;
info.high_limit = current->mm->mmap_base;
info.align_mask = PAGE_MASK & ~huge_page_mask(h);
info.align_offset = 0;
+ info.threadstack_offset = offset;
addr = vm_unmapped_area(&info);
/*
VM_BUG_ON(addr != -ENOMEM);
info.flags = 0;
info.low_limit = TASK_UNMAPPED_BASE;
+
+#ifdef CONFIG_PAX_RANDMMAP
+ if (current->mm->pax_flags & MF_PAX_RANDMMAP)
+ info.low_limit += current->mm->delta_mmap;
+#endif
+
info.high_limit = TASK_SIZE;
addr = vm_unmapped_area(&info);
}
struct hstate *h = hstate_file(file);
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
+ unsigned long pax_task_size = TASK_SIZE;
+ unsigned long offset = gr_rand_threadstack_offset(mm, file, flags);
if (len & ~huge_page_mask(h))
return -EINVAL;
- if (len > TASK_SIZE)
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (mm->pax_flags & MF_PAX_SEGMEXEC)
+ pax_task_size = SEGMEXEC_TASK_SIZE;
+#endif
+
+ pax_task_size -= PAGE_SIZE;
+
+ if (len > pax_task_size)
return -ENOMEM;
if (flags & MAP_FIXED) {
return addr;
}
+#ifdef CONFIG_PAX_RANDMMAP
+ if (!(mm->pax_flags & MF_PAX_RANDMMAP))
+#endif
+
if (addr) {
addr = ALIGN(addr, huge_page_size(h));
vma = find_vma(mm, addr);
- if (TASK_SIZE - len >= addr &&
- (!vma || addr + len <= vma->vm_start))
+ if (pax_task_size - len >= addr && check_heap_stack_gap(vma, addr, len, offset))
return addr;
}
if (mm->get_unmapped_area == arch_get_unmapped_area)
return hugetlb_get_unmapped_area_bottomup(file, addr, len,
- pgoff, flags);
+ pgoff, flags, offset);
else
return hugetlb_get_unmapped_area_topdown(file, addr, len,
- pgoff, flags);
+ pgoff, flags, offset);
}
#endif /* CONFIG_HUGETLB_PAGE */
#include <linux/swap.h>
#include <linux/memblock.h>
#include <linux/bootmem.h> /* for max_low_pfn */
+#include <linux/tboot.h>
#include <asm/cacheflush.h>
#include <asm/e820.h>
#include <asm/proto.h>
#include <asm/dma.h> /* for MAX_DMA_PFN */
#include <asm/microcode.h>
+#include <asm/desc.h>
+#include <asm/bios_ebda.h>
/*
* We need to define the tracepoints somewhere, and tlb.c
early_ioremap_page_table_range_init();
#endif
+#ifdef CONFIG_PAX_PER_CPU_PGD
+ clone_pgd_range(get_cpu_pgd(0, kernel) + KERNEL_PGD_BOUNDARY,
+ swapper_pg_dir + KERNEL_PGD_BOUNDARY,
+ KERNEL_PGD_PTRS);
+ clone_pgd_range(get_cpu_pgd(0, user) + KERNEL_PGD_BOUNDARY,
+ swapper_pg_dir + KERNEL_PGD_BOUNDARY,
+ KERNEL_PGD_PTRS);
+ load_cr3(get_cpu_pgd(0, kernel));
+#else
load_cr3(swapper_pg_dir);
+#endif
+
__flush_tlb_all();
early_memtest(0, max_pfn_mapped << PAGE_SHIFT);
* Access has to be given to non-kernel-ram areas as well, these contain the PCI
* mmio resources as well as potential bios/acpi data regions.
*/
+
+#ifdef CONFIG_GRKERNSEC_KMEM
+static unsigned int ebda_start __read_only;
+static unsigned int ebda_end __read_only;
+#endif
+
int devmem_is_allowed(unsigned long pagenr)
{
- if (pagenr < 256)
+#ifdef CONFIG_GRKERNSEC_KMEM
+ /* allow BDA */
+ if (!pagenr)
+ return 1;
+ /* allow EBDA */
+ if (pagenr >= ebda_start && pagenr < ebda_end)
return 1;
+ /* if tboot is in use, allow access to its hardcoded serial log range */
+ if (tboot_enabled() && ((0x60000 >> PAGE_SHIFT) <= pagenr) && (pagenr < (0x68000 >> PAGE_SHIFT)))
+ return 1;
+#else
+ if (!pagenr)
+ return 1;
+#ifdef CONFIG_VM86
+ if (pagenr < (ISA_START_ADDRESS >> PAGE_SHIFT))
+ return 1;
+#endif
+#endif
+
+ if ((ISA_START_ADDRESS >> PAGE_SHIFT) <= pagenr && pagenr < (ISA_END_ADDRESS >> PAGE_SHIFT))
+ return 1;
+#ifdef CONFIG_GRKERNSEC_KMEM
+ /* throw out everything else below 1MB */
+ if (pagenr <= 256)
+ return 0;
+#endif
if (iomem_is_exclusive(pagenr << PAGE_SHIFT))
return 0;
if (!page_is_ram(pagenr))
#endif
}
+#ifdef CONFIG_GRKERNSEC_KMEM
+static inline void gr_init_ebda(void)
+{
+ unsigned int ebda_addr;
+ unsigned int ebda_size = 0;
+
+ ebda_addr = get_bios_ebda();
+ if (ebda_addr) {
+ ebda_size = *(unsigned char *)phys_to_virt(ebda_addr);
+ ebda_size <<= 10;
+ }
+ if (ebda_addr && ebda_size) {
+ ebda_start = ebda_addr >> PAGE_SHIFT;
+ ebda_end = min((unsigned int)PAGE_ALIGN(ebda_addr + ebda_size), (unsigned int)0xa0000) >> PAGE_SHIFT;
+ } else {
+ ebda_start = 0x9f000 >> PAGE_SHIFT;
+ ebda_end = 0xa0000 >> PAGE_SHIFT;
+ }
+}
+#else
+static inline void gr_init_ebda(void) { }
+#endif
+
void free_initmem(void)
{
+#ifdef CONFIG_PAX_KERNEXEC
+#ifdef CONFIG_X86_32
+ /* PaX: limit KERNEL_CS to actual size */
+ unsigned long addr, limit;
+ struct desc_struct d;
+ int cpu;
+#else
+ pgd_t *pgd;
+ pud_t *pud;
+ pmd_t *pmd;
+ unsigned long addr, end;
+#endif
+#endif
+
+ gr_init_ebda();
+
+#ifdef CONFIG_PAX_KERNEXEC
+#ifdef CONFIG_X86_32
+ limit = paravirt_enabled() ? ktva_ktla(0xffffffff) : (unsigned long)&_etext;
+ limit = (limit - 1UL) >> PAGE_SHIFT;
+
+ memset(__LOAD_PHYSICAL_ADDR + PAGE_OFFSET, POISON_FREE_INITMEM, PAGE_SIZE);
+ for (cpu = 0; cpu < nr_cpu_ids; cpu++) {
+ pack_descriptor(&d, get_desc_base(&get_cpu_gdt_table(cpu)[GDT_ENTRY_KERNEL_CS]), limit, 0x9B, 0xC);
+ write_gdt_entry(get_cpu_gdt_table(cpu), GDT_ENTRY_KERNEL_CS, &d, DESCTYPE_S);
+ write_gdt_entry(get_cpu_gdt_table(cpu), GDT_ENTRY_KERNEXEC_KERNEL_CS, &d, DESCTYPE_S);
+ }
+
+ /* PaX: make KERNEL_CS read-only */
+ addr = PFN_ALIGN(ktla_ktva((unsigned long)&_text));
+ if (!paravirt_enabled())
+ set_memory_ro(addr, (PFN_ALIGN(_sdata) - addr) >> PAGE_SHIFT);
+/*
+ for (addr = ktla_ktva((unsigned long)&_text); addr < (unsigned long)&_sdata; addr += PMD_SIZE) {
+ pgd = pgd_offset_k(addr);
+ pud = pud_offset(pgd, addr);
+ pmd = pmd_offset(pud, addr);
+ set_pmd(pmd, __pmd(pmd_val(*pmd) & ~_PAGE_RW));
+ }
+*/
+#ifdef CONFIG_X86_PAE
+ set_memory_nx(PFN_ALIGN(__init_begin), (PFN_ALIGN(__init_end) - PFN_ALIGN(__init_begin)) >> PAGE_SHIFT);
+/*
+ for (addr = (unsigned long)&__init_begin; addr < (unsigned long)&__init_end; addr += PMD_SIZE) {
+ pgd = pgd_offset_k(addr);
+ pud = pud_offset(pgd, addr);
+ pmd = pmd_offset(pud, addr);
+ set_pmd(pmd, __pmd(pmd_val(*pmd) | (_PAGE_NX & __supported_pte_mask)));
+ }
+*/
+#endif
+
+#ifdef CONFIG_MODULES
+ set_memory_4k((unsigned long)MODULES_EXEC_VADDR, (MODULES_EXEC_END - MODULES_EXEC_VADDR) >> PAGE_SHIFT);
+#endif
+
+#else
+ /* PaX: make kernel code/rodata read-only, rest non-executable */
+ for (addr = __START_KERNEL_map; addr < __START_KERNEL_map + KERNEL_IMAGE_SIZE; addr += PMD_SIZE) {
+ pgd = pgd_offset_k(addr);
+ pud = pud_offset(pgd, addr);
+ pmd = pmd_offset(pud, addr);
+ if (!pmd_present(*pmd))
+ continue;
+ if ((unsigned long)_text <= addr && addr < (unsigned long)_sdata)
+ set_pmd(pmd, __pmd(pmd_val(*pmd) & ~_PAGE_RW));
+ else
+ set_pmd(pmd, __pmd(pmd_val(*pmd) | (_PAGE_NX & __supported_pte_mask)));
+ }
+
+ addr = (unsigned long)__va(__pa(__START_KERNEL_map));
+ end = addr + KERNEL_IMAGE_SIZE;
+ for (; addr < end; addr += PMD_SIZE) {
+ pgd = pgd_offset_k(addr);
+ pud = pud_offset(pgd, addr);
+ pmd = pmd_offset(pud, addr);
+ if (!pmd_present(*pmd))
+ continue;
+ if ((unsigned long)__va(__pa(_text)) <= addr && addr < (unsigned long)__va(__pa(_sdata)))
+ set_pmd(pmd, __pmd(pmd_val(*pmd) & ~_PAGE_RW));
+ }
+#endif
+
+ flush_tlb_all();
+#endif
+
free_init_pages("unused kernel",
(unsigned long)(&__init_begin),
(unsigned long)(&__init_end));
bool __read_mostly __vmalloc_start_set = false;
-/*
- * Creates a middle page table and puts a pointer to it in the
- * given global directory entry. This only returns the gd entry
- * in non-PAE compilation mode, since the middle layer is folded.
- */
-static pmd_t * __init one_md_table_init(pgd_t *pgd)
-{
- pud_t *pud;
- pmd_t *pmd_table;
-
-#ifdef CONFIG_X86_PAE
- if (!(pgd_val(*pgd) & _PAGE_PRESENT)) {
- pmd_table = (pmd_t *)alloc_low_page();
- paravirt_alloc_pmd(&init_mm, __pa(pmd_table) >> PAGE_SHIFT);
- set_pgd(pgd, __pgd(__pa(pmd_table) | _PAGE_PRESENT));
- pud = pud_offset(pgd, 0);
- BUG_ON(pmd_table != pmd_offset(pud, 0));
-
- return pmd_table;
- }
-#endif
- pud = pud_offset(pgd, 0);
- pmd_table = pmd_offset(pud, 0);
-
- return pmd_table;
-}
-
/*
* Create a page table and place a pointer to it in a middle page
* directory entry:
pte_t *page_table = (pte_t *)alloc_low_page();
paravirt_alloc_pte(&init_mm, __pa(page_table) >> PAGE_SHIFT);
+#if defined(CONFIG_PAX_PAGEEXEC) || defined(CONFIG_PAX_SEGMEXEC)
+ set_pmd(pmd, __pmd(__pa(page_table) | _KERNPG_TABLE));
+#else
set_pmd(pmd, __pmd(__pa(page_table) | _PAGE_TABLE));
+#endif
BUG_ON(page_table != pte_offset_kernel(pmd, 0));
}
return pte_offset_kernel(pmd, 0);
}
+static pmd_t * __init one_md_table_init(pgd_t *pgd)
+{
+ pud_t *pud;
+ pmd_t *pmd_table;
+
+ pud = pud_offset(pgd, 0);
+ pmd_table = pmd_offset(pud, 0);
+
+ return pmd_table;
+}
+
pmd_t * __init populate_extra_pmd(unsigned long vaddr)
{
int pgd_idx = pgd_index(vaddr);
int pgd_idx, pmd_idx;
unsigned long vaddr;
pgd_t *pgd;
+ pud_t *pud;
pmd_t *pmd;
pte_t *pte = NULL;
unsigned long count = page_table_range_init_count(start, end);
pgd = pgd_base + pgd_idx;
for ( ; (pgd_idx < PTRS_PER_PGD) && (vaddr != end); pgd++, pgd_idx++) {
- pmd = one_md_table_init(pgd);
- pmd = pmd + pmd_index(vaddr);
+ pud = pud_offset(pgd, vaddr);
+ pmd = pmd_offset(pud, vaddr);
+
+#ifdef CONFIG_X86_PAE
+ paravirt_alloc_pmd(&init_mm, __pa(pmd) >> PAGE_SHIFT);
+#endif
+
for (; (pmd_idx < PTRS_PER_PMD) && (vaddr != end);
pmd++, pmd_idx++) {
pte = page_table_kmap_check(one_page_table_init(pmd),
}
}
-static inline int is_kernel_text(unsigned long addr)
+static inline int is_kernel_text(unsigned long start, unsigned long end)
{
- if (addr >= (unsigned long)_text && addr <= (unsigned long)__init_end)
- return 1;
- return 0;
+ if ((start >= ktla_ktva((unsigned long)_etext) ||
+ end <= ktla_ktva((unsigned long)_stext)) &&
+ (start >= ktla_ktva((unsigned long)_einittext) ||
+ end <= ktla_ktva((unsigned long)_sinittext)) &&
+
+#ifdef CONFIG_ACPI_SLEEP
+ (start >= (unsigned long)__va(acpi_wakeup_address) + 0x4000 || end <= (unsigned long)__va(acpi_wakeup_address)) &&
+#endif
+
+ (start > (unsigned long)__va(0xfffff) || end <= (unsigned long)__va(0xc0000)))
+ return 0;
+ return 1;
}
/*
unsigned long last_map_addr = end;
unsigned long start_pfn, end_pfn;
pgd_t *pgd_base = swapper_pg_dir;
- int pgd_idx, pmd_idx, pte_ofs;
+ unsigned int pgd_idx, pmd_idx, pte_ofs;
unsigned long pfn;
pgd_t *pgd;
+ pud_t *pud;
pmd_t *pmd;
pte_t *pte;
unsigned pages_2m, pages_4k;
pfn = start_pfn;
pgd_idx = pgd_index((pfn<<PAGE_SHIFT) + PAGE_OFFSET);
pgd = pgd_base + pgd_idx;
- for (; pgd_idx < PTRS_PER_PGD; pgd++, pgd_idx++) {
- pmd = one_md_table_init(pgd);
+ for (; pgd_idx < PTRS_PER_PGD && pfn < max_low_pfn; pgd++, pgd_idx++) {
+ pud = pud_offset(pgd, 0);
+ pmd = pmd_offset(pud, 0);
+
+#ifdef CONFIG_X86_PAE
+ paravirt_alloc_pmd(&init_mm, __pa(pmd) >> PAGE_SHIFT);
+#endif
if (pfn >= end_pfn)
continue;
#endif
for (; pmd_idx < PTRS_PER_PMD && pfn < end_pfn;
pmd++, pmd_idx++) {
- unsigned int addr = pfn * PAGE_SIZE + PAGE_OFFSET;
+ unsigned long address = pfn * PAGE_SIZE + PAGE_OFFSET;
/*
* Map with big pages if possible, otherwise
* create normal page tables:
*/
if (use_pse) {
- unsigned int addr2;
pgprot_t prot = PAGE_KERNEL_LARGE;
/*
* first pass will use the same initial
_PAGE_PSE);
pfn &= PMD_MASK >> PAGE_SHIFT;
- addr2 = (pfn + PTRS_PER_PTE-1) * PAGE_SIZE +
- PAGE_OFFSET + PAGE_SIZE-1;
-
- if (is_kernel_text(addr) ||
- is_kernel_text(addr2))
+ if (is_kernel_text(address, address + PMD_SIZE))
prot = PAGE_KERNEL_LARGE_EXEC;
pages_2m++;
pte_ofs = pte_index((pfn<<PAGE_SHIFT) + PAGE_OFFSET);
pte += pte_ofs;
for (; pte_ofs < PTRS_PER_PTE && pfn < end_pfn;
- pte++, pfn++, pte_ofs++, addr += PAGE_SIZE) {
+ pte++, pfn++, pte_ofs++, address += PAGE_SIZE) {
pgprot_t prot = PAGE_KERNEL;
/*
* first pass will use the same initial
*/
pgprot_t init_prot = __pgprot(PTE_IDENT_ATTR);
- if (is_kernel_text(addr))
+ if (is_kernel_text(address, address + PAGE_SIZE))
prot = PAGE_KERNEL_EXEC;
pages_4k++;
pud = pud_offset(pgd, va);
pmd = pmd_offset(pud, va);
- if (!pmd_present(*pmd))
+ if (!pmd_present(*pmd)) // PAX TODO || pmd_large(*pmd))
break;
/* should not be large page here */
static void __init pagetable_init(void)
{
- pgd_t *pgd_base = swapper_pg_dir;
-
- permanent_kmaps_init(pgd_base);
+ permanent_kmaps_init(swapper_pg_dir);
}
-pteval_t __supported_pte_mask __read_mostly = ~(_PAGE_NX | _PAGE_GLOBAL);
+pteval_t __supported_pte_mask __read_only = ~(_PAGE_NX | _PAGE_GLOBAL);
EXPORT_SYMBOL_GPL(__supported_pte_mask);
/* user-defined highmem size */
((unsigned long)&__init_end -
(unsigned long)&__init_begin) >> 10,
- (unsigned long)&_etext, (unsigned long)&_edata,
- ((unsigned long)&_edata - (unsigned long)&_etext) >> 10,
+ (unsigned long)&_sdata, (unsigned long)&_edata,
+ ((unsigned long)&_edata - (unsigned long)&_sdata) >> 10,
- (unsigned long)&_text, (unsigned long)&_etext,
+ ktla_ktva((unsigned long)&_text), ktla_ktva((unsigned long)&_etext),
((unsigned long)&_etext - (unsigned long)&_text) >> 10);
/*
if (!kernel_set_to_readonly)
return;
+ start = ktla_ktva(start);
pr_debug("Set kernel text: %lx - %lx for read write\n",
start, start+size);
if (!kernel_set_to_readonly)
return;
+ start = ktla_ktva(start);
pr_debug("Set kernel text: %lx - %lx for read only\n",
start, start+size);
unsigned long start = PFN_ALIGN(_text);
unsigned long size = PFN_ALIGN(_etext) - start;
+ start = ktla_ktva(start);
set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT);
printk(KERN_INFO "Write protecting the kernel text: %luk\n",
size >> 10);
* around without checking the pgd every time.
*/
-pteval_t __supported_pte_mask __read_mostly = ~0;
+pteval_t __supported_pte_mask __read_only = ~_PAGE_NX;
EXPORT_SYMBOL_GPL(__supported_pte_mask);
int force_personality32;
for (address = start; address <= end; address += PGDIR_SIZE) {
const pgd_t *pgd_ref = pgd_offset_k(address);
+
+#ifdef CONFIG_PAX_PER_CPU_PGD
+ unsigned long cpu;
+#else
struct page *page;
+#endif
/*
* When it is called after memory hot remove, pgd_none()
continue;
spin_lock(&pgd_lock);
+
+#ifdef CONFIG_PAX_PER_CPU_PGD
+ for (cpu = 0; cpu < nr_cpu_ids; ++cpu) {
+ pgd_t *pgd = pgd_offset_cpu(cpu, user, address);
+
+ if (!pgd_none(*pgd_ref) && !pgd_none(*pgd))
+ BUG_ON(pgd_page_vaddr(*pgd)
+ != pgd_page_vaddr(*pgd_ref));
+
+ if (removed) {
+ if (pgd_none(*pgd_ref) && !pgd_none(*pgd))
+ pgd_clear(pgd);
+ } else {
+ if (pgd_none(*pgd))
+ set_pgd(pgd, *pgd_ref);
+ }
+
+ pgd = pgd_offset_cpu(cpu, kernel, address);
+#else
list_for_each_entry(page, &pgd_list, lru) {
pgd_t *pgd;
spinlock_t *pgt_lock;
/* the pgt_lock only for Xen */
pgt_lock = &pgd_page_get_mm(page)->page_table_lock;
spin_lock(pgt_lock);
+#endif
if (!pgd_none(*pgd_ref) && !pgd_none(*pgd))
BUG_ON(pgd_page_vaddr(*pgd)
set_pgd(pgd, *pgd_ref);
}
+#ifndef CONFIG_PAX_PER_CPU_PGD
spin_unlock(pgt_lock);
+#endif
+
}
spin_unlock(&pgd_lock);
}
{
if (pgd_none(*pgd)) {
pud_t *pud = (pud_t *)spp_getpage();
- pgd_populate(&init_mm, pgd, pud);
+ pgd_populate_kernel(&init_mm, pgd, pud);
if (pud != pud_offset(pgd, 0))
printk(KERN_ERR "PAGETABLE BUG #00! %p <-> %p\n",
pud, pud_offset(pgd, 0));
{
if (pud_none(*pud)) {
pmd_t *pmd = (pmd_t *) spp_getpage();
- pud_populate(&init_mm, pud, pmd);
+ pud_populate_kernel(&init_mm, pud, pmd);
if (pmd != pmd_offset(pud, 0))
printk(KERN_ERR "PAGETABLE BUG #01! %p <-> %p\n",
pmd, pmd_offset(pud, 0));
pmd = fill_pmd(pud, vaddr);
pte = fill_pte(pmd, vaddr);
+ pax_open_kernel();
set_pte(pte, new_pte);
+ pax_close_kernel();
/*
* It's enough to flush this one mapping.
pgd = pgd_offset_k((unsigned long)__va(phys));
if (pgd_none(*pgd)) {
pud = (pud_t *) spp_getpage();
- set_pgd(pgd, __pgd(__pa(pud) | _KERNPG_TABLE |
- _PAGE_USER));
+ set_pgd(pgd, __pgd(__pa(pud) | _PAGE_TABLE));
}
pud = pud_offset(pgd, (unsigned long)__va(phys));
if (pud_none(*pud)) {
pmd = (pmd_t *) spp_getpage();
- set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE |
- _PAGE_USER));
+ set_pud(pud, __pud(__pa(pmd) | _PAGE_TABLE));
}
pmd = pmd_offset(pud, phys);
BUG_ON(!pmd_none(*pmd));
prot);
spin_lock(&init_mm.page_table_lock);
- pud_populate(&init_mm, pud, pmd);
+ pud_populate_kernel(&init_mm, pud, pmd);
spin_unlock(&init_mm.page_table_lock);
}
__flush_tlb_all();
page_size_mask);
spin_lock(&init_mm.page_table_lock);
- pgd_populate(&init_mm, pgd, pud);
+ pgd_populate_kernel(&init_mm, pgd, pud);
spin_unlock(&init_mm.page_table_lock);
pgd_changed = true;
}
type = kmap_atomic_idx_push();
idx = type + KM_TYPE_NR * smp_processor_id();
vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
+
+ pax_open_kernel();
set_pte(kmap_pte - idx, pfn_pte(pfn, prot));
+ pax_close_kernel();
+
arch_flush_lazy_mmu_mode();
return (void *)vaddr;
unsigned long i;
for (i = 0; i < nr_pages; ++i)
- if (pfn_valid(start_pfn + i) &&
- !PageReserved(pfn_to_page(start_pfn + i)))
+ if (pfn_valid(start_pfn + i) && (start_pfn + i >= 0x100 ||
+ !PageReserved(pfn_to_page(start_pfn + i))))
return 1;
WARN_ONCE(1, "ioremap on RAM pfn 0x%lx\n", start_pfn);
*
* Caller must ensure there is only one unmapping for the same pointer.
*/
-void iounmap(volatile void __iomem *addr)
+void iounmap(const volatile void __iomem *addr)
{
struct vm_struct *p, *o;
*/
void *xlate_dev_mem_ptr(phys_addr_t phys)
{
- void *addr;
- unsigned long start = phys & PAGE_MASK;
-
/* If page is RAM, we can use __va. Otherwise ioremap and unmap. */
- if (page_is_ram(start >> PAGE_SHIFT))
+ if (page_is_ram(phys >> PAGE_SHIFT))
+#ifdef CONFIG_HIGHMEM
+ if ((phys >> PAGE_SHIFT) < max_low_pfn)
+#endif
return __va(phys);
- addr = (void __force *)ioremap_cache(start, PAGE_SIZE);
- if (addr)
- addr = (void *)((unsigned long)addr | (phys & ~PAGE_MASK));
-
- return addr;
+ return (void __force *)ioremap_cache(phys, PAGE_SIZE);
}
void unxlate_dev_mem_ptr(phys_addr_t phys, void *addr)
{
if (page_is_ram(phys >> PAGE_SHIFT))
+#ifdef CONFIG_HIGHMEM
+ if ((phys >> PAGE_SHIFT) < max_low_pfn)
+#endif
return;
iounmap((void __iomem *)((unsigned long)addr & PAGE_MASK));
return;
}
-static pte_t bm_pte[PAGE_SIZE/sizeof(pte_t)] __page_aligned_bss;
+static pte_t bm_pte[PAGE_SIZE/sizeof(pte_t)] __read_only __aligned(PAGE_SIZE);
static inline pmd_t * __init early_ioremap_pmd(unsigned long addr)
{
early_ioremap_setup();
pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN));
- memset(bm_pte, 0, sizeof(bm_pte));
- pmd_populate_kernel(&init_mm, pmd, bm_pte);
+ pmd_populate_user(&init_mm, pmd, bm_pte);
/*
* The boot-ioremap range spans multiple pmds, for which
* memory (e.g. tracked pages)? For now, we need this to avoid
* invoking kmemcheck for PnP BIOS calls.
*/
- if (regs->flags & X86_VM_MASK)
+ if (v8086_mode(regs))
return false;
- if (regs->cs != __KERNEL_CS)
+ if (regs->cs != __KERNEL_CS && regs->cs != __KERNEXEC_KERNEL_CS)
return false;
pte = kmemcheck_pte_lookup(address);
* Leave an at least ~128 MB hole with possible stack randomization.
*/
#define MIN_GAP (128*1024*1024UL + stack_maxrandom_size())
-#define MAX_GAP (TASK_SIZE/6*5)
+#define MAX_GAP (pax_task_size/6*5)
static int mmap_is_legacy(void)
{
return rnd << PAGE_SHIFT;
}
-static unsigned long mmap_base(void)
+static unsigned long mmap_base(struct mm_struct *mm)
{
unsigned long gap = rlimit(RLIMIT_STACK);
+ unsigned long pax_task_size = TASK_SIZE;
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (mm->pax_flags & MF_PAX_SEGMEXEC)
+ pax_task_size = SEGMEXEC_TASK_SIZE;
+#endif
if (gap < MIN_GAP)
gap = MIN_GAP;
else if (gap > MAX_GAP)
gap = MAX_GAP;
- return PAGE_ALIGN(TASK_SIZE - gap - mmap_rnd());
+ return PAGE_ALIGN(pax_task_size - gap - mmap_rnd());
}
/*
* Bottom-up (legacy) layout on X86_32 did not support randomization, X86_64
* does, but not when emulating X86_32
*/
-static unsigned long mmap_legacy_base(void)
+static unsigned long mmap_legacy_base(struct mm_struct *mm)
{
- if (mmap_is_ia32())
+ if (mmap_is_ia32()) {
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (mm->pax_flags & MF_PAX_SEGMEXEC)
+ return SEGMEXEC_TASK_UNMAPPED_BASE;
+ else
+#endif
+
return TASK_UNMAPPED_BASE;
- else
+ } else
return TASK_UNMAPPED_BASE + mmap_rnd();
}
*/
void arch_pick_mmap_layout(struct mm_struct *mm)
{
- mm->mmap_legacy_base = mmap_legacy_base();
- mm->mmap_base = mmap_base();
+ mm->mmap_legacy_base = mmap_legacy_base(mm);
+ mm->mmap_base = mmap_base(mm);
+
+#ifdef CONFIG_PAX_RANDMMAP
+ if (mm->pax_flags & MF_PAX_RANDMMAP) {
+ mm->mmap_legacy_base += mm->delta_mmap;
+ mm->mmap_base -= mm->delta_mmap + mm->delta_stack;
+ }
+#endif
if (mmap_is_legacy()) {
mm->mmap_base = mm->mmap_legacy_base;
break;
default:
{
- unsigned char *ip = (unsigned char *)instptr;
+ unsigned char *ip = (unsigned char *)ktla_ktva(instptr);
my_trace->opcode = MMIO_UNKNOWN_OP;
my_trace->width = 0;
my_trace->value = (*ip) << 16 | *(ip + 1) << 8 |
static void ioremap_trace_core(resource_size_t offset, unsigned long size,
void __iomem *addr)
{
- static atomic_t next_id;
+ static atomic_unchecked_t next_id;
struct remap_trace *trace = kmalloc(sizeof(*trace), GFP_KERNEL);
/* These are page-unaligned. */
struct mmiotrace_map map = {
.private = trace
},
.phys = offset,
- .id = atomic_inc_return(&next_id)
+ .id = atomic_inc_return_unchecked(&next_id)
};
map.map_id = trace->id;
ioremap_trace_core(offset, size, addr);
}
-static void iounmap_trace_core(volatile void __iomem *addr)
+static void iounmap_trace_core(const volatile void __iomem *addr)
{
struct mmiotrace_map map = {
.phys = 0,
}
}
-void mmiotrace_iounmap(volatile void __iomem *addr)
+void mmiotrace_iounmap(const volatile void __iomem *addr)
{
might_sleep();
if (is_enabled()) /* recheck and proper locking in *_core() */
}
}
-static int __init numa_register_memblks(struct numa_meminfo *mi)
+static int __init __intentional_overflow(-1) numa_register_memblks(struct numa_meminfo *mi)
{
unsigned long uninitialized_var(pfn_align);
int i, nid;
*/
#ifdef CONFIG_PCI_BIOS
if (pcibios_enabled && within(pfn, BIOS_BEGIN >> PAGE_SHIFT, BIOS_END >> PAGE_SHIFT))
- pgprot_val(forbidden) |= _PAGE_NX;
+ pgprot_val(forbidden) |= _PAGE_NX & __supported_pte_mask;
#endif
/*
* Does not cover __inittext since that is gone later on. On
* 64bit we do not enforce !NX on the low mapping
*/
- if (within(address, (unsigned long)_text, (unsigned long)_etext))
- pgprot_val(forbidden) |= _PAGE_NX;
+ if (within(address, ktla_ktva((unsigned long)_text), ktla_ktva((unsigned long)_etext)))
+ pgprot_val(forbidden) |= _PAGE_NX & __supported_pte_mask;
+#ifdef CONFIG_DEBUG_RODATA
/*
* The .rodata section needs to be read-only. Using the pfn
* catches all aliases.
if (within(pfn, __pa_symbol(__start_rodata) >> PAGE_SHIFT,
__pa_symbol(__end_rodata) >> PAGE_SHIFT))
pgprot_val(forbidden) |= _PAGE_RW;
+#endif
#if defined(CONFIG_X86_64) && defined(CONFIG_DEBUG_RODATA)
/*
}
#endif
+#ifdef CONFIG_PAX_KERNEXEC
+ if (within(pfn, __pa(ktla_ktva((unsigned long)&_text)), __pa((unsigned long)&_sdata))) {
+ pgprot_val(forbidden) |= _PAGE_RW;
+ pgprot_val(forbidden) |= _PAGE_NX & __supported_pte_mask;
+ }
+#endif
+
prot = __pgprot(pgprot_val(prot) & ~pgprot_val(forbidden));
return prot;
static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte)
{
/* change init_mm */
+ pax_open_kernel();
set_pte_atomic(kpte, pte);
+
#ifdef CONFIG_X86_32
if (!SHARED_KERNEL_PMD) {
+
+#ifdef CONFIG_PAX_PER_CPU_PGD
+ unsigned long cpu;
+#else
struct page *page;
+#endif
+#ifdef CONFIG_PAX_PER_CPU_PGD
+ for (cpu = 0; cpu < nr_cpu_ids; ++cpu) {
+ pgd_t *pgd = get_cpu_pgd(cpu, kernel);
+#else
list_for_each_entry(page, &pgd_list, lru) {
- pgd_t *pgd;
+ pgd_t *pgd = (pgd_t *)page_address(page);
+#endif
+
pud_t *pud;
pmd_t *pmd;
- pgd = (pgd_t *)page_address(page) + pgd_index(address);
+ pgd += pgd_index(address);
pud = pud_offset(pgd, address);
pmd = pmd_offset(pud, address);
set_pte_atomic((pte_t *)pmd, pte);
}
}
#endif
+ pax_close_kernel();
}
static int
unsigned long pg_flags = pg->flags & _PGMT_MASK;
if (pg_flags == _PGMT_DEFAULT)
- return -1;
+ return _PAGE_CACHE_MODE_NUM;
else if (pg_flags == _PGMT_WC)
return _PAGE_CACHE_MODE_WC;
else if (pg_flags == _PGMT_UC_MINUS)
page = pfn_to_page(pfn);
type = get_page_memtype(page);
- if (type != -1) {
+ if (type != _PAGE_CACHE_MODE_NUM) {
pr_info("reserve_ram_pages_type failed [mem %#010Lx-%#010Lx], track 0x%x, req 0x%x\n",
start, end - 1, type, req_type);
if (new_type)
if (!entry) {
printk(KERN_INFO "%s:%d freeing invalid memtype [mem %#010Lx-%#010Lx]\n",
- current->comm, current->pid, start, end - 1);
+ current->comm, task_pid_nr(current), start, end - 1);
return -EINVAL;
}
page = pfn_to_page(paddr >> PAGE_SHIFT);
rettype = get_page_memtype(page);
/*
- * -1 from get_page_memtype() implies RAM page is in its
+ * _PAGE_CACHE_MODE_NUM from get_page_memtype() implies RAM page is in its
* default state and not reserved, and hence of type WB
*/
- if (rettype == -1)
+ if (rettype == _PAGE_CACHE_MODE_NUM)
rettype = _PAGE_CACHE_MODE_WB;
return rettype;
while (cursor < to) {
if (!devmem_is_allowed(pfn)) {
- printk(KERN_INFO "Program %s tried to access /dev/mem between [mem %#010Lx-%#010Lx]\n",
- current->comm, from, to - 1);
+ printk(KERN_INFO "Program %s tried to access /dev/mem between [mem %#010Lx-%#010Lx] (%#010Lx)\n",
+ current->comm, from, to - 1, cursor);
return 0;
}
cursor += PAGE_SIZE;
if (ioremap_change_attr((unsigned long)__va(base), id_sz, pcm) < 0) {
printk(KERN_INFO "%s:%d ioremap_change_attr failed %s "
"for [mem %#010Lx-%#010Lx]\n",
- current->comm, current->pid,
+ current->comm, task_pid_nr(current),
cattr_name(pcm),
base, (unsigned long long)(base + size-1));
return -EINVAL;
pcm = lookup_memtype(paddr);
if (want_pcm != pcm) {
printk(KERN_WARNING "%s:%d map pfn RAM range req %s for [mem %#010Lx-%#010Lx], got %s\n",
- current->comm, current->pid,
+ current->comm, task_pid_nr(current),
cattr_name(want_pcm),
(unsigned long long)paddr,
(unsigned long long)(paddr + size - 1),
free_memtype(paddr, paddr + size);
printk(KERN_ERR "%s:%d map pfn expected mapping type %s"
" for [mem %#010Lx-%#010Lx], got %s\n",
- current->comm, current->pid,
+ current->comm, task_pid_nr(current),
cattr_name(want_pcm),
(unsigned long long)paddr,
(unsigned long long)(paddr + size - 1),
failure:
printk(KERN_INFO "%s:%d conflicting memory types "
- "%Lx-%Lx %s<->%s\n", current->comm, current->pid, start,
+ "%Lx-%Lx %s<->%s\n", current->comm, task_pid_nr(current), start,
end, cattr_name(found_type), cattr_name(match->type));
return -EBUSY;
}
int i;
enum reason_type rv = OTHERS;
- p = (unsigned char *)ins_addr;
+ p = (unsigned char *)ktla_ktva(ins_addr);
p += skip_prefix(p, &prf);
p += get_opcode(p, &opcode);
struct prefix_bits prf;
int i;
- p = (unsigned char *)ins_addr;
+ p = (unsigned char *)ktla_ktva(ins_addr);
p += skip_prefix(p, &prf);
p += get_opcode(p, &opcode);
struct prefix_bits prf;
int i;
- p = (unsigned char *)ins_addr;
+ p = (unsigned char *)ktla_ktva(ins_addr);
p += skip_prefix(p, &prf);
p += get_opcode(p, &opcode);
struct prefix_bits prf;
int i;
- p = (unsigned char *)ins_addr;
+ p = (unsigned char *)ktla_ktva(ins_addr);
p += skip_prefix(p, &prf);
p += get_opcode(p, &opcode);
for (i = 0; i < ARRAY_SIZE(reg_rop); i++)
struct prefix_bits prf;
int i;
- p = (unsigned char *)ins_addr;
+ p = (unsigned char *)ktla_ktva(ins_addr);
p += skip_prefix(p, &prf);
p += get_opcode(p, &opcode);
for (i = 0; i < ARRAY_SIZE(imm_wop); i++)
list_del(&page->lru);
}
-#define UNSHARED_PTRS_PER_PGD \
- (SHARED_KERNEL_PMD ? KERNEL_PGD_BOUNDARY : PTRS_PER_PGD)
+#if defined(CONFIG_X86_64) && defined(CONFIG_PAX_MEMORY_UDEREF)
+pgdval_t clone_pgd_mask __read_only = ~_PAGE_PRESENT;
+void __shadow_user_pgds(pgd_t *dst, const pgd_t *src)
+{
+ unsigned int count = USER_PGD_PTRS;
+
+ if (!pax_user_shadow_base)
+ return;
+
+ while (count--)
+ *dst++ = __pgd((pgd_val(*src++) | (_PAGE_NX & __supported_pte_mask)) & ~_PAGE_USER);
+}
+#endif
+
+#ifdef CONFIG_PAX_PER_CPU_PGD
+void __clone_user_pgds(pgd_t *dst, const pgd_t *src)
+{
+ unsigned int count = USER_PGD_PTRS;
+
+ while (count--) {
+ pgd_t pgd;
+
+#ifdef CONFIG_X86_64
+ pgd = __pgd(pgd_val(*src++) | _PAGE_USER);
+#else
+ pgd = *src++;
+#endif
+
+#if defined(CONFIG_X86_64) && defined(CONFIG_PAX_MEMORY_UDEREF)
+ pgd = __pgd(pgd_val(pgd) & clone_pgd_mask);
+#endif
+ *dst++ = pgd;
+ }
+
+}
+#endif
+
+#ifdef CONFIG_X86_64
+#define pxd_t pud_t
+#define pyd_t pgd_t
+#define paravirt_release_pxd(pfn) paravirt_release_pud(pfn)
+#define pgtable_pxd_page_ctor(page) true
+#define pgtable_pxd_page_dtor(page)
+#define pxd_free(mm, pud) pud_free((mm), (pud))
+#define pyd_populate(mm, pgd, pud) pgd_populate((mm), (pgd), (pud))
+#define pyd_offset(mm, address) pgd_offset((mm), (address))
+#define PYD_SIZE PGDIR_SIZE
+#else
+#define pxd_t pmd_t
+#define pyd_t pud_t
+#define paravirt_release_pxd(pfn) paravirt_release_pmd(pfn)
+#define pgtable_pxd_page_ctor(page) pgtable_pmd_page_ctor(page)
+#define pgtable_pxd_page_dtor(page) pgtable_pmd_page_dtor(page)
+#define pxd_free(mm, pud) pmd_free((mm), (pud))
+#define pyd_populate(mm, pgd, pud) pud_populate((mm), (pgd), (pud))
+#define pyd_offset(mm, address) pud_offset((mm), (address))
+#define PYD_SIZE PUD_SIZE
+#endif
+
+#ifdef CONFIG_PAX_PER_CPU_PGD
+static inline void pgd_ctor(struct mm_struct *mm, pgd_t *pgd) {}
+static inline void pgd_dtor(pgd_t *pgd) {}
+#else
static void pgd_set_mm(pgd_t *pgd, struct mm_struct *mm)
{
BUILD_BUG_ON(sizeof(virt_to_page(pgd)->index) < sizeof(mm));
pgd_list_del(pgd);
spin_unlock(&pgd_lock);
}
+#endif
/*
* List of all pgd's needed for non-PAE so it can invalidate entries
* -- nyc
*/
-#ifdef CONFIG_X86_PAE
+#if defined(CONFIG_X86_32) && defined(CONFIG_X86_PAE)
/*
* In PAE mode, we need to do a cr3 reload (=tlb flush) when
* updating the top-level pagetable entries to guarantee the
* not shared between pagetables (!SHARED_KERNEL_PMDS), we allocate
* and initialize the kernel pmds here.
*/
-#define PREALLOCATED_PMDS UNSHARED_PTRS_PER_PGD
+#define PREALLOCATED_PXDS (SHARED_KERNEL_PMD ? KERNEL_PGD_BOUNDARY : PTRS_PER_PGD)
void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd)
{
*/
flush_tlb_mm(mm);
}
+#elif defined(CONFIG_X86_64) && defined(CONFIG_PAX_PER_CPU_PGD)
+#define PREALLOCATED_PXDS USER_PGD_PTRS
#else /* !CONFIG_X86_PAE */
/* No need to prepopulate any pagetable entries in non-PAE modes. */
-#define PREALLOCATED_PMDS 0
+#define PREALLOCATED_PXDS 0
#endif /* CONFIG_X86_PAE */
-static void free_pmds(pmd_t *pmds[])
+static void free_pxds(pxd_t *pxds[])
{
int i;
- for(i = 0; i < PREALLOCATED_PMDS; i++)
- if (pmds[i]) {
- pgtable_pmd_page_dtor(virt_to_page(pmds[i]));
- free_page((unsigned long)pmds[i]);
+ for(i = 0; i < PREALLOCATED_PXDS; i++)
+ if (pxds[i]) {
+ pgtable_pxd_page_dtor(virt_to_page(pxds[i]));
+ free_page((unsigned long)pxds[i]);
}
}
-static int preallocate_pmds(pmd_t *pmds[])
+static int preallocate_pxds(pxd_t *pxds[])
{
int i;
bool failed = false;
- for(i = 0; i < PREALLOCATED_PMDS; i++) {
- pmd_t *pmd = (pmd_t *)__get_free_page(PGALLOC_GFP);
- if (!pmd)
+ for(i = 0; i < PREALLOCATED_PXDS; i++) {
+ pxd_t *pxd = (pxd_t *)__get_free_page(PGALLOC_GFP);
+ if (!pxd)
failed = true;
- if (pmd && !pgtable_pmd_page_ctor(virt_to_page(pmd))) {
- free_page((unsigned long)pmd);
- pmd = NULL;
+ if (pxd && !pgtable_pxd_page_ctor(virt_to_page(pxd))) {
+ free_page((unsigned long)pxd);
+ pxd = NULL;
failed = true;
}
- pmds[i] = pmd;
+ pxds[i] = pxd;
}
if (failed) {
- free_pmds(pmds);
+ free_pxds(pxds);
return -ENOMEM;
}
* preallocate which never got a corresponding vma will need to be
* freed manually.
*/
-static void pgd_mop_up_pmds(struct mm_struct *mm, pgd_t *pgdp)
+static void pgd_mop_up_pxds(struct mm_struct *mm, pgd_t *pgdp)
{
int i;
- for(i = 0; i < PREALLOCATED_PMDS; i++) {
+ for(i = 0; i < PREALLOCATED_PXDS; i++) {
pgd_t pgd = pgdp[i];
if (pgd_val(pgd) != 0) {
- pmd_t *pmd = (pmd_t *)pgd_page_vaddr(pgd);
+ pxd_t *pxd = (pxd_t *)pgd_page_vaddr(pgd);
- pgdp[i] = native_make_pgd(0);
+ set_pgd(pgdp + i, native_make_pgd(0));
- paravirt_release_pmd(pgd_val(pgd) >> PAGE_SHIFT);
- pmd_free(mm, pmd);
+ paravirt_release_pxd(pgd_val(pgd) >> PAGE_SHIFT);
+ pxd_free(mm, pxd);
}
}
}
-static void pgd_prepopulate_pmd(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmds[])
+static void pgd_prepopulate_pxd(struct mm_struct *mm, pgd_t *pgd, pxd_t *pxds[])
{
- pud_t *pud;
+ pyd_t *pyd;
int i;
- if (PREALLOCATED_PMDS == 0) /* Work around gcc-3.4.x bug */
+ if (PREALLOCATED_PXDS == 0) /* Work around gcc-3.4.x bug */
return;
- pud = pud_offset(pgd, 0);
-
- for (i = 0; i < PREALLOCATED_PMDS; i++, pud++) {
- pmd_t *pmd = pmds[i];
+#ifdef CONFIG_X86_64
+ pyd = pyd_offset(mm, 0L);
+#else
+ pyd = pyd_offset(pgd, 0L);
+#endif
+ for (i = 0; i < PREALLOCATED_PXDS; i++, pyd++) {
+ pxd_t *pxd = pxds[i];
if (i >= KERNEL_PGD_BOUNDARY)
- memcpy(pmd, (pmd_t *)pgd_page_vaddr(swapper_pg_dir[i]),
- sizeof(pmd_t) * PTRS_PER_PMD);
+ memcpy(pxd, (pxd_t *)pgd_page_vaddr(swapper_pg_dir[i]),
+ sizeof(pxd_t) * PTRS_PER_PMD);
- pud_populate(mm, pud, pmd);
+ pyd_populate(mm, pyd, pxd);
}
}
pgd_t *pgd_alloc(struct mm_struct *mm)
{
pgd_t *pgd;
- pmd_t *pmds[PREALLOCATED_PMDS];
+ pxd_t *pxds[PREALLOCATED_PXDS];
pgd = (pgd_t *)__get_free_page(PGALLOC_GFP);
mm->pgd = pgd;
- if (preallocate_pmds(pmds) != 0)
+ if (preallocate_pxds(pxds) != 0)
goto out_free_pgd;
if (paravirt_pgd_alloc(mm) != 0)
- goto out_free_pmds;
+ goto out_free_pxds;
/*
* Make sure that pre-populating the pmds is atomic with
spin_lock(&pgd_lock);
pgd_ctor(mm, pgd);
- pgd_prepopulate_pmd(mm, pgd, pmds);
+ pgd_prepopulate_pxd(mm, pgd, pxds);
spin_unlock(&pgd_lock);
return pgd;
-out_free_pmds:
- free_pmds(pmds);
+out_free_pxds:
+ free_pxds(pxds);
out_free_pgd:
free_page((unsigned long)pgd);
out:
void pgd_free(struct mm_struct *mm, pgd_t *pgd)
{
- pgd_mop_up_pmds(mm, pgd);
+ pgd_mop_up_pxds(mm, pgd);
pgd_dtor(pgd);
paravirt_pgd_free(mm, pgd);
free_page((unsigned long)pgd);
return;
}
pte = pte_offset_kernel(pmd, vaddr);
+
+ pax_open_kernel();
if (pte_val(pteval))
set_pte_at(&init_mm, vaddr, pte, pteval);
else
pte_clear(&init_mm, vaddr, pte);
+ pax_close_kernel();
/*
* It's enough to flush this one mapping.
#ifdef CONFIG_X86_64
#ifdef CONFIG_DEBUG_VIRTUAL
-unsigned long __phys_addr(unsigned long x)
+unsigned long __intentional_overflow(-1) __phys_addr(unsigned long x)
{
unsigned long y = x - __START_KERNEL_map;
#else
#ifdef CONFIG_DEBUG_VIRTUAL
-unsigned long __phys_addr(unsigned long x)
+unsigned long __intentional_overflow(-1) __phys_addr(unsigned long x)
{
unsigned long phys_addr = x - PAGE_OFFSET;
/* VMALLOC_* aren't constants */
#include <asm/pgtable.h>
#include <asm/proto.h>
+#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
static int disable_nx;
+#ifndef CONFIG_PAX_PAGEEXEC
/*
* noexec = on|off
*
return 0;
}
early_param("noexec", noexec_setup);
+#endif
+
+#endif
void x86_configure_nx(void)
{
+#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
if (cpu_has_nx && !disable_nx)
__supported_pte_mask |= _PAGE_NX;
else
+#endif
__supported_pte_mask &= ~_PAGE_NX;
}
BUG();
if (cpumask_test_cpu(cpu, mm_cpumask(active_mm))) {
cpumask_clear_cpu(cpu, mm_cpumask(active_mm));
+
+#ifndef CONFIG_PAX_PER_CPU_PGD
load_cr3(swapper_pg_dir);
+#endif
+
/*
* This gets called in the idle path where RCU
* functions differently. Tracing normally
*/
#include <linux/linkage.h>
#include <asm/dwarf2.h>
+#include <asm/alternative-asm.h>
/*
* Calling convention :
jle bpf_slow_path_word
mov (SKBDATA,%rsi),%eax
bswap %eax /* ntohl() */
+ pax_force_retaddr
ret
sk_load_half:
jle bpf_slow_path_half
movzwl (SKBDATA,%rsi),%eax
rol $8,%ax # ntohs()
+ pax_force_retaddr
ret
sk_load_byte:
cmp %esi,%r9d /* if (offset >= hlen) goto bpf_slow_path_byte */
jle bpf_slow_path_byte
movzbl (SKBDATA,%rsi),%eax
+ pax_force_retaddr
ret
/* rsi contains offset and can be scratched */
js bpf_error
mov - MAX_BPF_STACK + 32(%rbp),%eax
bswap %eax
+ pax_force_retaddr
ret
bpf_slow_path_half:
mov - MAX_BPF_STACK + 32(%rbp),%ax
rol $8,%ax
movzwl %ax,%eax
+ pax_force_retaddr
ret
bpf_slow_path_byte:
bpf_slow_path_common(1)
js bpf_error
movzbl - MAX_BPF_STACK + 32(%rbp),%eax
+ pax_force_retaddr
ret
#define sk_negative_common(SIZE) \
sk_negative_common(4)
mov (%rax), %eax
bswap %eax
+ pax_force_retaddr
ret
bpf_slow_path_half_neg:
mov (%rax),%ax
rol $8,%ax
movzwl %ax,%eax
+ pax_force_retaddr
ret
bpf_slow_path_byte_neg:
.globl sk_load_byte_negative_offset
sk_negative_common(1)
movzbl (%rax), %eax
+ pax_force_retaddr
ret
bpf_error:
mov - MAX_BPF_STACK + 16(%rbp),%r14
mov - MAX_BPF_STACK + 24(%rbp),%r15
leaveq
+ pax_force_retaddr
ret
#include <linux/if_vlan.h>
#include <asm/cacheflush.h>
+#ifdef CONFIG_GRKERNSEC_BPF_HARDEN
+int bpf_jit_enable __read_only;
+#else
int bpf_jit_enable __read_mostly;
+#endif
/*
* assembly code in arch/x86/net/bpf_jit.S
static void jit_fill_hole(void *area, unsigned int size)
{
/* fill whole space with int3 instructions */
+ pax_open_kernel();
memset(area, 0xcc, size);
+ pax_close_kernel();
}
struct jit_context {
pr_err("bpf_jit_compile fatal error\n");
return -EFAULT;
}
+ pax_open_kernel();
memcpy(image + proglen, temp, ilen);
+ pax_close_kernel();
}
proglen += ilen;
addrs[i] = proglen;
if (image) {
bpf_flush_icache(header, image + proglen);
- set_memory_ro((unsigned long)header, header->pages);
prog->bpf_func = (void *)image;
prog->jited = true;
}
unsigned long addr = (unsigned long)fp->bpf_func & PAGE_MASK;
struct bpf_binary_header *header = (void *)addr;
- if (!fp->jited)
- goto free_filter;
+ if (fp->jited)
+ bpf_jit_binary_free(header);
- set_memory_rw(addr, header->pages);
- bpf_jit_binary_free(header);
-
-free_filter:
bpf_prog_unlock_free(fp);
}
struct stack_frame_ia32 *fp;
unsigned long bytes;
- bytes = copy_from_user_nmi(bufhead, head, sizeof(bufhead));
+ bytes = copy_from_user_nmi(bufhead, (const char __force_user *)head, sizeof(bufhead));
if (bytes != 0)
return NULL;
- fp = (struct stack_frame_ia32 *) compat_ptr(bufhead[0].next_frame);
+ fp = (struct stack_frame_ia32 __force_kernel *) compat_ptr(bufhead[0].next_frame);
oprofile_add_trace(bufhead[0].return_address);
struct stack_frame bufhead[2];
unsigned long bytes;
- bytes = copy_from_user_nmi(bufhead, head, sizeof(bufhead));
+ bytes = copy_from_user_nmi(bufhead, (const char __force_user *)head, sizeof(bufhead));
if (bytes != 0)
return NULL;
{
struct stack_frame *head = (struct stack_frame *)frame_pointer(regs);
- if (!user_mode_vm(regs)) {
+ if (!user_mode(regs)) {
unsigned long stack = kernel_stack_pointer(regs);
if (depth)
dump_trace(NULL, regs, (unsigned long *)stack, 0,
#include <asm/nmi.h>
#include <asm/msr.h>
#include <asm/apic.h>
+#include <asm/pgtable.h>
#include "op_counter.h"
#include "op_x86_model.h"
if (ret)
return ret;
- if (!model->num_virt_counters)
- model->num_virt_counters = model->num_counters;
+ if (!model->num_virt_counters) {
+ pax_open_kernel();
+ *(unsigned int *)&model->num_virt_counters = model->num_counters;
+ pax_close_kernel();
+ }
mux_init(ops);
num_counters = AMD64_NUM_COUNTERS;
}
- op_amd_spec.num_counters = num_counters;
- op_amd_spec.num_controls = num_counters;
- op_amd_spec.num_virt_counters = max(num_counters, NUM_VIRT_COUNTERS);
+ pax_open_kernel();
+ *(unsigned int *)&op_amd_spec.num_counters = num_counters;
+ *(unsigned int *)&op_amd_spec.num_controls = num_counters;
+ *(unsigned int *)&op_amd_spec.num_virt_counters = max(num_counters, NUM_VIRT_COUNTERS);
+ pax_close_kernel();
return 0;
}
#include <asm/msr.h>
#include <asm/apic.h>
#include <asm/nmi.h>
+#include <asm/pgtable.h>
#include "op_x86_model.h"
#include "op_counter.h"
num_counters = min((int)eax.split.num_counters, OP_MAX_COUNTER);
- op_arch_perfmon_spec.num_counters = num_counters;
- op_arch_perfmon_spec.num_controls = num_counters;
+ pax_open_kernel();
+ *(unsigned int *)&op_arch_perfmon_spec.num_counters = num_counters;
+ *(unsigned int *)&op_arch_perfmon_spec.num_controls = num_counters;
+ pax_close_kernel();
}
static int arch_perfmon_init(struct oprofile_operations *ignore)
void (*switch_ctrl)(struct op_x86_model_spec const *model,
struct op_msrs const * const msrs);
#endif
-};
+} __do_const;
struct op_counter_config;
pci_mmcfg_late_init();
pcibios_enable_irq = intel_mid_pci_irq_enable;
pcibios_disable_irq = intel_mid_pci_irq_disable;
- pci_root_ops = intel_mid_pci_ops;
+ memcpy((void *)&pci_root_ops, &intel_mid_pci_ops, sizeof pci_root_ops);
pci_soc_mode = 1;
/* Continue with standard init */
return 1;
struct irq_router_handler {
u16 vendor;
int (*probe)(struct irq_router *r, struct pci_dev *router, u16 device);
-};
+} __do_const;
int (*pcibios_enable_irq)(struct pci_dev *dev) = pirq_enable_irq;
void (*pcibios_disable_irq)(struct pci_dev *dev) = pirq_disable_irq;
return 0;
}
-static __initdata struct irq_router_handler pirq_routers[] = {
+static __initconst const struct irq_router_handler pirq_routers[] = {
{ PCI_VENDOR_ID_INTEL, intel_router_probe },
{ PCI_VENDOR_ID_AL, ali_router_probe },
{ PCI_VENDOR_ID_ITE, ite_router_probe },
static void __init pirq_find_router(struct irq_router *r)
{
struct irq_routing_table *rt = pirq_table;
- struct irq_router_handler *h;
+ const struct irq_router_handler *h;
#ifdef CONFIG_PCI_BIOS
if (!rt->signature) {
return 0;
}
-static struct dmi_system_id __initdata pciirq_dmi_table[] = {
+static const struct dmi_system_id __initconst pciirq_dmi_table[] = {
{
.callback = fix_broken_hp_bios_irq9,
.ident = "HP Pavilion N5400 Series Laptop",
static struct {
unsigned long address;
unsigned short segment;
-} bios32_indirect __initdata = { 0, __KERNEL_CS };
+} bios32_indirect __initconst = { 0, __PCIBIOS_CS };
/*
* Returns the entry point for the given service, NULL on error
unsigned long length; /* %ecx */
unsigned long entry; /* %edx */
unsigned long flags;
+ struct desc_struct d, *gdt;
local_irq_save(flags);
- __asm__("lcall *(%%edi); cld"
+
+ gdt = get_cpu_gdt_table(smp_processor_id());
+
+ pack_descriptor(&d, 0UL, 0xFFFFFUL, 0x9B, 0xC);
+ write_gdt_entry(gdt, GDT_ENTRY_PCIBIOS_CS, &d, DESCTYPE_S);
+ pack_descriptor(&d, 0UL, 0xFFFFFUL, 0x93, 0xC);
+ write_gdt_entry(gdt, GDT_ENTRY_PCIBIOS_DS, &d, DESCTYPE_S);
+
+ __asm__("movw %w7, %%ds; lcall *(%%edi); push %%ss; pop %%ds; cld"
: "=a" (return_code),
"=b" (address),
"=c" (length),
"=d" (entry)
: "0" (service),
"1" (0),
- "D" (&bios32_indirect));
+ "D" (&bios32_indirect),
+ "r"(__PCIBIOS_DS)
+ : "memory");
+
+ pax_open_kernel();
+ gdt[GDT_ENTRY_PCIBIOS_CS].a = 0;
+ gdt[GDT_ENTRY_PCIBIOS_CS].b = 0;
+ gdt[GDT_ENTRY_PCIBIOS_DS].a = 0;
+ gdt[GDT_ENTRY_PCIBIOS_DS].b = 0;
+ pax_close_kernel();
+
local_irq_restore(flags);
switch (return_code) {
- case 0:
- return address + entry;
- case 0x80: /* Not present */
- printk(KERN_WARNING "bios32_service(0x%lx): not present\n", service);
- return 0;
- default: /* Shouldn't happen */
- printk(KERN_WARNING "bios32_service(0x%lx): returned 0x%x -- BIOS bug!\n",
- service, return_code);
+ case 0: {
+ int cpu;
+ unsigned char flags;
+
+ printk(KERN_INFO "bios32_service: base:%08lx length:%08lx entry:%08lx\n", address, length, entry);
+ if (address >= 0xFFFF0 || length > 0x100000 - address || length <= entry) {
+ printk(KERN_WARNING "bios32_service: not valid\n");
return 0;
+ }
+ address = address + PAGE_OFFSET;
+ length += 16UL; /* some BIOSs underreport this... */
+ flags = 4;
+ if (length >= 64*1024*1024) {
+ length >>= PAGE_SHIFT;
+ flags |= 8;
+ }
+
+ for (cpu = 0; cpu < nr_cpu_ids; cpu++) {
+ gdt = get_cpu_gdt_table(cpu);
+ pack_descriptor(&d, address, length, 0x9b, flags);
+ write_gdt_entry(gdt, GDT_ENTRY_PCIBIOS_CS, &d, DESCTYPE_S);
+ pack_descriptor(&d, address, length, 0x93, flags);
+ write_gdt_entry(gdt, GDT_ENTRY_PCIBIOS_DS, &d, DESCTYPE_S);
+ }
+ return entry;
+ }
+ case 0x80: /* Not present */
+ printk(KERN_WARNING "bios32_service(0x%lx): not present\n", service);
+ return 0;
+ default: /* Shouldn't happen */
+ printk(KERN_WARNING "bios32_service(0x%lx): returned 0x%x -- BIOS bug!\n",
+ service, return_code);
+ return 0;
}
}
static struct {
unsigned long address;
unsigned short segment;
-} pci_indirect = { 0, __KERNEL_CS };
+} pci_indirect __read_only = { 0, __PCIBIOS_CS };
-static int pci_bios_present;
+static int pci_bios_present __read_only;
static int __init check_pcibios(void)
{
unsigned long flags, pcibios_entry;
if ((pcibios_entry = bios32_service(PCI_SERVICE))) {
- pci_indirect.address = pcibios_entry + PAGE_OFFSET;
+ pci_indirect.address = pcibios_entry;
local_irq_save(flags);
- __asm__(
- "lcall *(%%edi); cld\n\t"
+ __asm__("movw %w6, %%ds\n\t"
+ "lcall *%%ss:(%%edi); cld\n\t"
+ "push %%ss\n\t"
+ "pop %%ds\n\t"
"jc 1f\n\t"
"xor %%ah, %%ah\n"
"1:"
"=b" (ebx),
"=c" (ecx)
: "1" (PCIBIOS_PCI_BIOS_PRESENT),
- "D" (&pci_indirect)
+ "D" (&pci_indirect),
+ "r" (__PCIBIOS_DS)
: "memory");
local_irq_restore(flags);
switch (len) {
case 1:
- __asm__("lcall *(%%esi); cld\n\t"
+ __asm__("movw %w6, %%ds\n\t"
+ "lcall *%%ss:(%%esi); cld\n\t"
+ "push %%ss\n\t"
+ "pop %%ds\n\t"
"jc 1f\n\t"
"xor %%ah, %%ah\n"
"1:"
: "1" (PCIBIOS_READ_CONFIG_BYTE),
"b" (bx),
"D" ((long)reg),
- "S" (&pci_indirect));
+ "S" (&pci_indirect),
+ "r" (__PCIBIOS_DS));
/*
* Zero-extend the result beyond 8 bits, do not trust the
* BIOS having done it:
*value &= 0xff;
break;
case 2:
- __asm__("lcall *(%%esi); cld\n\t"
+ __asm__("movw %w6, %%ds\n\t"
+ "lcall *%%ss:(%%esi); cld\n\t"
+ "push %%ss\n\t"
+ "pop %%ds\n\t"
"jc 1f\n\t"
"xor %%ah, %%ah\n"
"1:"
: "1" (PCIBIOS_READ_CONFIG_WORD),
"b" (bx),
"D" ((long)reg),
- "S" (&pci_indirect));
+ "S" (&pci_indirect),
+ "r" (__PCIBIOS_DS));
/*
* Zero-extend the result beyond 16 bits, do not trust the
* BIOS having done it:
*value &= 0xffff;
break;
case 4:
- __asm__("lcall *(%%esi); cld\n\t"
+ __asm__("movw %w6, %%ds\n\t"
+ "lcall *%%ss:(%%esi); cld\n\t"
+ "push %%ss\n\t"
+ "pop %%ds\n\t"
"jc 1f\n\t"
"xor %%ah, %%ah\n"
"1:"
: "1" (PCIBIOS_READ_CONFIG_DWORD),
"b" (bx),
"D" ((long)reg),
- "S" (&pci_indirect));
+ "S" (&pci_indirect),
+ "r" (__PCIBIOS_DS));
break;
}
switch (len) {
case 1:
- __asm__("lcall *(%%esi); cld\n\t"
+ __asm__("movw %w6, %%ds\n\t"
+ "lcall *%%ss:(%%esi); cld\n\t"
+ "push %%ss\n\t"
+ "pop %%ds\n\t"
"jc 1f\n\t"
"xor %%ah, %%ah\n"
"1:"
"c" (value),
"b" (bx),
"D" ((long)reg),
- "S" (&pci_indirect));
+ "S" (&pci_indirect),
+ "r" (__PCIBIOS_DS));
break;
case 2:
- __asm__("lcall *(%%esi); cld\n\t"
+ __asm__("movw %w6, %%ds\n\t"
+ "lcall *%%ss:(%%esi); cld\n\t"
+ "push %%ss\n\t"
+ "pop %%ds\n\t"
"jc 1f\n\t"
"xor %%ah, %%ah\n"
"1:"
"c" (value),
"b" (bx),
"D" ((long)reg),
- "S" (&pci_indirect));
+ "S" (&pci_indirect),
+ "r" (__PCIBIOS_DS));
break;
case 4:
- __asm__("lcall *(%%esi); cld\n\t"
+ __asm__("movw %w6, %%ds\n\t"
+ "lcall *%%ss:(%%esi); cld\n\t"
+ "push %%ss\n\t"
+ "pop %%ds\n\t"
"jc 1f\n\t"
"xor %%ah, %%ah\n"
"1:"
"c" (value),
"b" (bx),
"D" ((long)reg),
- "S" (&pci_indirect));
+ "S" (&pci_indirect),
+ "r" (__PCIBIOS_DS));
break;
}
DBG("PCI: Fetching IRQ routing table... ");
__asm__("push %%es\n\t"
+ "movw %w8, %%ds\n\t"
"push %%ds\n\t"
"pop %%es\n\t"
- "lcall *(%%esi); cld\n\t"
+ "lcall *%%ss:(%%esi); cld\n\t"
"pop %%es\n\t"
+ "push %%ss\n\t"
+ "pop %%ds\n"
"jc 1f\n\t"
"xor %%ah, %%ah\n"
"1:"
"1" (0),
"D" ((long) &opt),
"S" (&pci_indirect),
- "m" (opt)
+ "m" (opt),
+ "r" (__PCIBIOS_DS)
: "memory");
DBG("OK ret=%d, size=%d, map=%x\n", ret, opt.size, map);
if (ret & 0xff00)
{
int ret;
- __asm__("lcall *(%%esi); cld\n\t"
+ __asm__("movw %w5, %%ds\n\t"
+ "lcall *%%ss:(%%esi); cld\n\t"
+ "push %%ss\n\t"
+ "pop %%ds\n"
"jc 1f\n\t"
"xor %%ah, %%ah\n"
"1:"
: "0" (PCIBIOS_SET_PCI_HW_INT),
"b" ((dev->bus->number << 8) | dev->devfn),
"c" ((irq << 8) | (pin + 10)),
- "S" (&pci_indirect));
+ "S" (&pci_indirect),
+ "r" (__PCIBIOS_DS));
return !(ret & 0xff00);
}
EXPORT_SYMBOL(pcibios_set_irq_routing);
{
struct desc_ptr gdt_descr;
+#ifdef CONFIG_PAX_KERNEXEC
+ struct desc_struct d;
+#endif
+
local_irq_save(efi_rt_eflags);
load_cr3(initial_page_table);
__flush_tlb_all();
+#ifdef CONFIG_PAX_KERNEXEC
+ pack_descriptor(&d, 0, 0xFFFFF, 0x9B, 0xC);
+ write_gdt_entry(get_cpu_gdt_table(0), GDT_ENTRY_KERNEXEC_EFI_CS, &d, DESCTYPE_S);
+ pack_descriptor(&d, 0, 0xFFFFF, 0x93, 0xC);
+ write_gdt_entry(get_cpu_gdt_table(0), GDT_ENTRY_KERNEXEC_EFI_DS, &d, DESCTYPE_S);
+#endif
+
gdt_descr.address = __pa(get_cpu_gdt_table(0));
gdt_descr.size = GDT_SIZE - 1;
load_gdt(&gdt_descr);
{
struct desc_ptr gdt_descr;
+#ifdef CONFIG_PAX_KERNEXEC
+ struct desc_struct d;
+
+ memset(&d, 0, sizeof d);
+ write_gdt_entry(get_cpu_gdt_table(0), GDT_ENTRY_KERNEXEC_EFI_CS, &d, DESCTYPE_S);
+ write_gdt_entry(get_cpu_gdt_table(0), GDT_ENTRY_KERNEXEC_EFI_DS, &d, DESCTYPE_S);
+#endif
+
gdt_descr.address = (unsigned long)get_cpu_gdt_table(0);
gdt_descr.size = GDT_SIZE - 1;
load_gdt(&gdt_descr);
+#ifdef CONFIG_PAX_PER_CPU_PGD
+ load_cr3(get_cpu_pgd(smp_processor_id(), kernel));
+#else
load_cr3(swapper_pg_dir);
+#endif
+
__flush_tlb_all();
local_irq_restore(efi_rt_eflags);
vaddress = (unsigned long)__va(pgd * PGDIR_SIZE);
set_pgd(pgd_offset_k(pgd * PGDIR_SIZE), *pgd_offset_k(vaddress));
}
+
+#ifdef CONFIG_PAX_PER_CPU_PGD
+ load_cr3(swapper_pg_dir);
+#endif
+
__flush_tlb_all();
}
for (pgd = 0; pgd < n_pgds; pgd++)
set_pgd(pgd_offset_k(pgd * PGDIR_SIZE), save_pgd[pgd]);
kfree(save_pgd);
+
+#ifdef CONFIG_PAX_PER_CPU_PGD
+ load_cr3(get_cpu_pgd(smp_processor_id(), kernel));
+#endif
+
__flush_tlb_all();
local_irq_restore(efi_flags);
early_code_mapping_set_exec(0);
unsigned npages;
pgd_t *pgd;
- if (efi_enabled(EFI_OLD_MEMMAP))
+ if (efi_enabled(EFI_OLD_MEMMAP)) {
+ /* PaX: We need to disable the NX bit in the PGD, otherwise we won't be
+ * able to execute the EFI services.
+ */
+ if (__supported_pte_mask & _PAGE_NX) {
+ unsigned long addr = (unsigned long) __va(0);
+ pgd_t pe = __pgd(pgd_val(*pgd_offset_k(addr)) & ~_PAGE_NX);
+
+ pr_alert("PAX: Disabling NX protection for low memory map. Try booting without \"efi=old_map\"\n");
+#ifdef CONFIG_PAX_PER_CPU_PGD
+ set_pgd(pgd_offset_cpu(0, kernel, addr), pe);
+#endif
+ set_pgd(pgd_offset_k(addr), pe);
+ }
+
return 0;
+ }
efi_scratch.efi_pgt = (pgd_t *)(unsigned long)real_mode_header->trampoline_pgd;
pgd = __va(efi_scratch.efi_pgt);
*/
#include <linux/linkage.h>
+#include <linux/init.h>
#include <asm/page_types.h>
+#include <asm/segment.h>
/*
* efi_call_phys(void *, ...) is a function with variable parameters.
* service functions will comply with gcc calling convention, too.
*/
-.text
+__INIT
ENTRY(efi_call_phys)
/*
* 0. The function can only be called in Linux kernel. So CS has been
* The mapping of lower virtual memory has been created in prolog and
* epilog.
*/
- movl $1f, %edx
- subl $__PAGE_OFFSET, %edx
- jmp *%edx
+#ifdef CONFIG_PAX_KERNEXEC
+ movl $(__KERNEXEC_EFI_DS), %edx
+ mov %edx, %ds
+ mov %edx, %es
+ mov %edx, %ss
+ addl $2f,(1f)
+ ljmp *(1f)
+
+__INITDATA
+1: .long __LOAD_PHYSICAL_ADDR, __KERNEXEC_EFI_CS
+.previous
+
+2:
+ subl $2b,(1b)
+#else
+ jmp 1f-__PAGE_OFFSET
1:
+#endif
/*
* 2. Now on the top of stack is the return
* parameter 2, ..., param n. To make things easy, we save the return
* address of efi_call_phys in a global variable.
*/
- popl %edx
- movl %edx, saved_return_addr
- /* get the function pointer into ECX*/
- popl %ecx
- movl %ecx, efi_rt_function_ptr
- movl $2f, %edx
- subl $__PAGE_OFFSET, %edx
- pushl %edx
+ popl (saved_return_addr)
+ popl (efi_rt_function_ptr)
/*
* 3. Clear PG bit in %CR0.
/*
* 5. Call the physical function.
*/
- jmp *%ecx
+ call *(efi_rt_function_ptr-__PAGE_OFFSET)
-2:
/*
* 6. After EFI runtime service returns, control will return to
* following instruction. We'd better readjust stack pointer first.
movl %cr0, %edx
orl $0x80000000, %edx
movl %edx, %cr0
- jmp 1f
-1:
+
/*
* 8. Now restore the virtual mode from flat mode by
* adding EIP with PAGE_OFFSET.
*/
- movl $1f, %edx
- jmp *%edx
+#ifdef CONFIG_PAX_KERNEXEC
+ movl $(__KERNEL_DS), %edx
+ mov %edx, %ds
+ mov %edx, %es
+ mov %edx, %ss
+ ljmp $(__KERNEL_CS),$1f
+#else
+ jmp 1f+__PAGE_OFFSET
+#endif
1:
/*
* 9. Balance the stack. And because EAX contain the return value,
* we'd better not clobber it.
*/
- leal efi_rt_function_ptr, %edx
- movl (%edx), %ecx
- pushl %ecx
+ pushl (efi_rt_function_ptr)
/*
- * 10. Push the saved return address onto the stack and return.
+ * 10. Return to the saved return address.
*/
- leal saved_return_addr, %edx
- movl (%edx), %ecx
- pushl %ecx
- ret
+ jmpl *(saved_return_addr)
ENDPROC(efi_call_phys)
.previous
-.data
+__INITDATA
saved_return_addr:
.long 0
efi_rt_function_ptr:
#include <asm/msr.h>
#include <asm/processor-flags.h>
#include <asm/page_types.h>
+#include <asm/alternative-asm.h>
#define SAVE_XMM \
mov %rsp, %rax; \
RESTORE_PGT
addq $48, %rsp
RESTORE_XMM
+ pax_force_retaddr 0, 1
ret
ENDPROC(efi_call)
{
};
-static void intel_mid_reboot(void)
+static void __noreturn intel_mid_reboot(void)
{
intel_scu_ipc_simple_command(IPCMSG_COLD_BOOT, 0);
+ BUG();
}
static unsigned long __init intel_mid_calibrate_tsc(void)
/* For every CPU addition a new get_<cpuname>_ops interface needs
* to be added.
*/
-extern void *get_penwell_ops(void);
-extern void *get_cloverview_ops(void);
-extern void *get_tangier_ops(void);
+extern const void *get_penwell_ops(void);
+extern const void *get_cloverview_ops(void);
+extern const void *get_tangier_ops(void);
pm_power_off = mfld_power_off;
}
-void *get_penwell_ops(void)
+const void *get_penwell_ops(void)
{
return &penwell_ops;
}
-void *get_cloverview_ops(void)
+const void *get_cloverview_ops(void)
{
return &penwell_ops;
}
.arch_setup = tangier_arch_setup,
};
-void *get_tangier_ops(void)
+const void *get_tangier_ops(void)
{
return &tangier_ops;
}
return res;
}
-static struct of_pdt_ops prom_olpc_ops __initdata = {
+static struct of_pdt_ops prom_olpc_ops __initconst = {
.nextprop = olpc_dt_nextprop,
.getproplen = olpc_dt_getproplen,
.getproperty = olpc_dt_getproperty,
static void fix_processor_context(void)
{
int cpu = smp_processor_id();
- struct tss_struct *t = &per_cpu(init_tss, cpu);
-#ifdef CONFIG_X86_64
- struct desc_struct *desc = get_cpu_gdt_table(cpu);
- tss_desc tss;
-#endif
+ struct tss_struct *t = init_tss + cpu;
+
set_tss_desc(cpu, t); /*
* This just modifies memory; should not be
* necessary. But... This is necessary, because
*/
#ifdef CONFIG_X86_64
- memcpy(&tss, &desc[GDT_ENTRY_TSS], sizeof(tss_desc));
- tss.type = 0x9; /* The available 64-bit TSS (see AMD vol 2, pg 91 */
- write_gdt_entry(desc, GDT_ENTRY_TSS, &tss, DESC_TSS);
-
syscall_init(); /* This sets MSR_*STAR and related */
#endif
load_TR_desc(); /* This does ltr */
__va(real_mode_header->trampoline_header);
#ifdef CONFIG_X86_32
- trampoline_header->start = __pa_symbol(startup_32_smp);
+ trampoline_header->start = __pa_symbol(ktla_ktva(startup_32_smp));
+
+#ifdef CONFIG_PAX_KERNEXEC
+ trampoline_header->start -= LOAD_PHYSICAL_ADDR;
+#endif
+
+ trampoline_header->boot_cs = __BOOT_CS;
trampoline_header->gdt_limit = __BOOT_DS + 7;
trampoline_header->gdt_base = __pa_symbol(boot_gdt);
#else
*trampoline_cr4_features = read_cr4();
trampoline_pgd = (u64 *) __va(real_mode_header->trampoline_pgd);
- trampoline_pgd[0] = init_level4_pgt[pgd_index(__PAGE_OFFSET)].pgd;
+ trampoline_pgd[0] = init_level4_pgt[pgd_index(__PAGE_OFFSET)].pgd & ~_PAGE_NX;
trampoline_pgd[511] = init_level4_pgt[511].pgd;
#endif
}
KBUILD_CFLAGS := $(LINUXINCLUDE) $(REALMODE_CFLAGS) -D_SETUP -D_WAKEUP \
-I$(srctree)/arch/x86/boot
+ifdef CONSTIFY_PLUGIN
+KBUILD_CFLAGS += -fplugin-arg-constify_plugin-no-constify
+endif
KBUILD_AFLAGS := $(KBUILD_CFLAGS) -D__ASSEMBLY__
GCOV_PROFILE := n
#endif
/* APM/BIOS reboot */
.long pa_machine_real_restart_asm
-#ifdef CONFIG_X86_64
+#ifdef CONFIG_X86_32
+ .long __KERNEL_CS
+#else
.long __KERNEL32_CS
#endif
END(real_mode_header)
#include <asm/page_types.h>
#include "realmode.h"
+#ifdef CONFIG_PAX_KERNEXEC
+#define ta(X) (X)
+#else
+#define ta(X) (pa_ ## X)
+#endif
+
.text
.code16
cli # We should be safe anyway
- movl tr_start, %eax # where we need to go
-
movl $0xA5A5A5A5, trampoline_status
# write marker for master knows we're running
movw $1, %dx # protected mode (PE) bit
lmsw %dx # into protected mode
- ljmpl $__BOOT_CS, $pa_startup_32
+ ljmpl *(trampoline_header)
.section ".text32","ax"
.code32
.balign 8
GLOBAL(trampoline_header)
tr_start: .space 4
- tr_gdt_pad: .space 2
+ tr_boot_cs: .space 2
tr_gdt: .space 6
END(trampoline_header)
movl %edx, %gs
movl pa_tr_cr4, %eax
+ andl $~X86_CR4_PCIDE, %eax
movl %eax, %cr4 # Enable PAE mode
# Setup trampoline 4 level pagetables
wrmsr
# Enable paging and in turn activate Long Mode
- movl $(X86_CR0_PG | X86_CR0_WP | X86_CR0_PE), %eax
+ movl $(X86_CR0_PG | X86_CR0_PE), %eax
movl %eax, %cr0
/*
lgdtl pmode_gdt
/* This really couldn't... */
- movl pmode_entry, %eax
movl pmode_cr0, %ecx
movl %ecx, %cr0
- ljmpl $__KERNEL_CS, $pa_startup_32
- /* -> jmp *%eax in trampoline_32.S */
+
+ ljmpl *pmode_entry
#else
jmp trampoline_start
#endif
$(obj)/insn_sanity.o: $(srctree)/arch/x86/lib/insn.c $(srctree)/arch/x86/lib/inat.c $(srctree)/arch/x86/include/asm/inat_types.h $(srctree)/arch/x86/include/asm/inat.h $(srctree)/arch/x86/include/asm/insn.h $(objtree)/arch/x86/lib/inat-tables.c
-HOST_EXTRACFLAGS += -I$(srctree)/tools/include
+HOST_EXTRACFLAGS += -I$(srctree)/tools/include -ggdb
hostprogs-y += relocs
relocs-objs := relocs_32.o relocs_64.o relocs_common.o
PHONY += relocs
/* This is included from relocs_32/64.c */
+#include "../../../include/generated/autoconf.h"
+
#define ElfW(type) _ElfW(ELF_BITS, type)
#define _ElfW(bits, type) __ElfW(bits, type)
#define __ElfW(bits, type) Elf##bits##_##type
#define Elf_Sym ElfW(Sym)
static Elf_Ehdr ehdr;
+static Elf_Phdr *phdr;
struct relocs {
uint32_t *offset;
}
}
+static void read_phdrs(FILE *fp)
+{
+ unsigned int i;
+
+ phdr = calloc(ehdr.e_phnum, sizeof(Elf_Phdr));
+ if (!phdr) {
+ die("Unable to allocate %d program headers\n",
+ ehdr.e_phnum);
+ }
+ if (fseek(fp, ehdr.e_phoff, SEEK_SET) < 0) {
+ die("Seek to %d failed: %s\n",
+ ehdr.e_phoff, strerror(errno));
+ }
+ if (fread(phdr, sizeof(*phdr), ehdr.e_phnum, fp) != ehdr.e_phnum) {
+ die("Cannot read ELF program headers: %s\n",
+ strerror(errno));
+ }
+ for(i = 0; i < ehdr.e_phnum; i++) {
+ phdr[i].p_type = elf_word_to_cpu(phdr[i].p_type);
+ phdr[i].p_offset = elf_off_to_cpu(phdr[i].p_offset);
+ phdr[i].p_vaddr = elf_addr_to_cpu(phdr[i].p_vaddr);
+ phdr[i].p_paddr = elf_addr_to_cpu(phdr[i].p_paddr);
+ phdr[i].p_filesz = elf_word_to_cpu(phdr[i].p_filesz);
+ phdr[i].p_memsz = elf_word_to_cpu(phdr[i].p_memsz);
+ phdr[i].p_flags = elf_word_to_cpu(phdr[i].p_flags);
+ phdr[i].p_align = elf_word_to_cpu(phdr[i].p_align);
+ }
+
+}
+
static void read_shdrs(FILE *fp)
{
- int i;
+ unsigned int i;
Elf_Shdr shdr;
secs = calloc(ehdr.e_shnum, sizeof(struct section));
static void read_strtabs(FILE *fp)
{
- int i;
+ unsigned int i;
for (i = 0; i < ehdr.e_shnum; i++) {
struct section *sec = &secs[i];
if (sec->shdr.sh_type != SHT_STRTAB) {
static void read_symtabs(FILE *fp)
{
- int i,j;
+ unsigned int i,j;
for (i = 0; i < ehdr.e_shnum; i++) {
struct section *sec = &secs[i];
if (sec->shdr.sh_type != SHT_SYMTAB) {
}
-static void read_relocs(FILE *fp)
+static void read_relocs(FILE *fp, int use_real_mode)
{
- int i,j;
+ unsigned int i,j;
+ uint32_t base;
+
for (i = 0; i < ehdr.e_shnum; i++) {
struct section *sec = &secs[i];
if (sec->shdr.sh_type != SHT_REL_TYPE) {
die("Cannot read symbol table: %s\n",
strerror(errno));
}
+ base = 0;
+
+#ifdef CONFIG_X86_32
+ for (j = 0; !use_real_mode && j < ehdr.e_phnum; j++) {
+ if (phdr[j].p_type != PT_LOAD )
+ continue;
+ if (secs[sec->shdr.sh_info].shdr.sh_offset < phdr[j].p_offset || secs[sec->shdr.sh_info].shdr.sh_offset >= phdr[j].p_offset + phdr[j].p_filesz)
+ continue;
+ base = CONFIG_PAGE_OFFSET + phdr[j].p_paddr - phdr[j].p_vaddr;
+ break;
+ }
+#endif
+
for (j = 0; j < sec->shdr.sh_size/sizeof(Elf_Rel); j++) {
Elf_Rel *rel = &sec->reltab[j];
- rel->r_offset = elf_addr_to_cpu(rel->r_offset);
+ rel->r_offset = elf_addr_to_cpu(rel->r_offset) + base;
rel->r_info = elf_xword_to_cpu(rel->r_info);
#if (SHT_REL_TYPE == SHT_RELA)
rel->r_addend = elf_xword_to_cpu(rel->r_addend);
static void print_absolute_symbols(void)
{
- int i;
+ unsigned int i;
const char *format;
if (ELF_BITS == 64)
for (i = 0; i < ehdr.e_shnum; i++) {
struct section *sec = &secs[i];
char *sym_strtab;
- int j;
+ unsigned int j;
if (sec->shdr.sh_type != SHT_SYMTAB) {
continue;
static void print_absolute_relocs(void)
{
- int i, printed = 0;
+ unsigned int i, printed = 0;
const char *format;
if (ELF_BITS == 64)
struct section *sec_applies, *sec_symtab;
char *sym_strtab;
Elf_Sym *sh_symtab;
- int j;
+ unsigned int j;
if (sec->shdr.sh_type != SHT_REL_TYPE) {
continue;
}
static void walk_relocs(int (*process)(struct section *sec, Elf_Rel *rel,
Elf_Sym *sym, const char *symname))
{
- int i;
+ unsigned int i;
/* Walk through the relocations */
for (i = 0; i < ehdr.e_shnum; i++) {
char *sym_strtab;
Elf_Sym *sh_symtab;
struct section *sec_applies, *sec_symtab;
- int j;
+ unsigned int j;
struct section *sec = &secs[i];
if (sec->shdr.sh_type != SHT_REL_TYPE) {
{
unsigned r_type = ELF32_R_TYPE(rel->r_info);
int shn_abs = (sym->st_shndx == SHN_ABS) && !is_reloc(S_REL, symname);
+ char *sym_strtab = sec->link->link->strtab;
+
+ /* Don't relocate actual per-cpu variables, they are absolute indices, not addresses */
+ if (!strcmp(sec_name(sym->st_shndx), ".data..percpu") && strcmp(sym_name(sym_strtab, sym), "__per_cpu_load"))
+ return 0;
+
+#ifdef CONFIG_PAX_KERNEXEC
+ /* Don't relocate actual code, they are relocated implicitly by the base address of KERNEL_CS */
+ if (!strcmp(sec_name(sym->st_shndx), ".text.end") && !strcmp(sym_name(sym_strtab, sym), "_etext"))
+ return 0;
+ if (!strcmp(sec_name(sym->st_shndx), ".init.text"))
+ return 0;
+ if (!strcmp(sec_name(sym->st_shndx), ".exit.text"))
+ return 0;
+ if (!strcmp(sec_name(sym->st_shndx), ".text") && strcmp(sym_name(sym_strtab, sym), "__LOAD_PHYSICAL_ADDR"))
+ return 0;
+#endif
switch (r_type) {
case R_386_NONE:
static void emit_relocs(int as_text, int use_real_mode)
{
- int i;
+ unsigned int i;
int (*write_reloc)(uint32_t, FILE *) = write32;
int (*do_reloc)(struct section *sec, Elf_Rel *rel, Elf_Sym *sym,
const char *symname);
{
regex_init(use_real_mode);
read_ehdr(fp);
+ read_phdrs(fp);
read_shdrs(fp);
read_strtabs(fp);
read_symtabs(fp);
- read_relocs(fp);
+ read_relocs(fp, use_real_mode);
if (ELF_BITS == 64)
percpu_init();
if (show_absolute_syms) {
gate_vma.vm_start = FIXADDR_USER_START;
gate_vma.vm_end = FIXADDR_USER_END;
gate_vma.vm_flags = VM_READ | VM_MAYREAD | VM_EXEC | VM_MAYEXEC;
- gate_vma.vm_page_prot = __P101;
+ gate_vma.vm_page_prot = vm_get_page_prot(gate_vma.vm_flags);
return 0;
}
if (unlikely(task == current &&
!t->arch.tls_array[idx - GDT_ENTRY_TLS_MIN].flushed)) {
printk(KERN_ERR "get_tls_entry: task with pid %d got here "
- "without flushed TLS.", current->pid);
+ "without flushed TLS.", task_pid_nr(current));
}
return 0;
-Wl,-T,$(filter %.lds,$^) $(filter %.o,$^) && \
sh $(srctree)/$(src)/checkundef.sh '$(NM)' '$@'
-VDSO_LDFLAGS = -fPIC -shared $(call cc-ldoption, -Wl$(comma)--hash-style=sysv) \
+VDSO_LDFLAGS = -fPIC -shared -Wl,--no-undefined $(call cc-ldoption, -Wl$(comma)--hash-style=sysv) \
$(call cc-ldoption, -Wl$(comma)--build-id) -Wl,-Bsymbolic $(LTO_CFLAGS)
GCOV_PROFILE := n
unsigned long load_size = -1; /* Work around bogus warning */
unsigned long mapping_size;
ELF(Ehdr) *hdr = (ELF(Ehdr) *)raw_addr;
- int i;
+ unsigned int i;
unsigned long j;
ELF(Shdr) *symtab_hdr = NULL, *strtab_hdr, *secstrings_hdr,
*alt_sec = NULL;
#include <asm/cpufeature.h>
#include <asm/processor.h>
#include <asm/vdso.h>
+#include <asm/mman.h>
#ifdef CONFIG_COMPAT_VDSO
#define VDSO_DEFAULT 0
#include <asm/page.h>
#include <asm/hpet.h>
#include <asm/desc.h>
-
-#if defined(CONFIG_X86_64)
-unsigned int __read_mostly vdso64_enabled = 1;
-#endif
+#include <asm/mman.h>
void __init init_vdso_image(const struct vdso_image *image)
{
.pages = no_pages,
};
+#ifdef CONFIG_PAX_RANDMMAP
+ if (mm->pax_flags & MF_PAX_RANDMMAP)
+ calculate_addr = false;
+#endif
+
if (calculate_addr) {
addr = vdso_addr(current->mm->start_stack,
image->size - image->sym_vvar_start);
down_write(&mm->mmap_sem);
addr = get_unmapped_area(NULL, addr,
- image->size - image->sym_vvar_start, 0, 0);
+ image->size - image->sym_vvar_start, 0, MAP_EXECUTABLE);
if (IS_ERR_VALUE(addr)) {
ret = addr;
goto up_fail;
}
text_start = addr - image->sym_vvar_start;
- current->mm->context.vdso = (void __user *)text_start;
+ mm->context.vdso = text_start;
/*
* MAYWRITE to allow gdb to COW and set breakpoints
hpet_address >> PAGE_SHIFT,
PAGE_SIZE,
pgprot_noncached(PAGE_READONLY));
-
- if (ret)
- goto up_fail;
}
#endif
up_fail:
if (ret)
- current->mm->context.vdso = NULL;
+ current->mm->context.vdso = 0;
up_write(&mm->mmap_sem);
return ret;
if (selected_vdso32->sym_VDSO32_SYSENTER_RETURN)
current_thread_info()->sysenter_return =
- current->mm->context.vdso +
- selected_vdso32->sym_VDSO32_SYSENTER_RETURN;
+ (void __force_user *)(current->mm->context.vdso +
+ selected_vdso32->sym_VDSO32_SYSENTER_RETURN);
return 0;
}
#ifdef CONFIG_X86_64
int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
{
- if (!vdso64_enabled)
- return 0;
-
return map_vdso(&vdso_image_64, true);
}
int uses_interp)
{
#ifdef CONFIG_X86_X32_ABI
- if (test_thread_flag(TIF_X32)) {
- if (!vdso64_enabled)
- return 0;
-
+ if (test_thread_flag(TIF_X32))
return map_vdso(&vdso_image_x32, true);
- }
#endif
return load_vdso32();
}
#endif
-#ifdef CONFIG_X86_64
-static __init int vdso_setup(char *s)
-{
- vdso64_enabled = simple_strtoul(s, NULL, 0);
- return 0;
-}
-__setup("vdso=", vdso_setup);
-#endif
-
#ifdef CONFIG_X86_64
static void vgetcpu_cpu_init(void *arg)
{
select XEN_HAVE_PVMMU
depends on X86_64 || (X86_32 && X86_PAE)
depends on X86_TSC
+ depends on !GRKERNSEC_CONFIG_AUTO || GRKERNSEC_CONFIG_VIRT_XEN
help
This is the Linux Xen port. Enabling this will allow the
kernel to boot in a paravirtualized environment under the
struct shared_info xen_dummy_shared_info;
-void *xen_initial_gdt;
-
RESERVE_BRK(shared_info_page_brk, PAGE_SIZE);
__read_mostly int xen_have_vector_callback;
EXPORT_SYMBOL_GPL(xen_have_vector_callback);
{
unsigned long va = dtr->address;
unsigned int size = dtr->size + 1;
- unsigned pages = (size + PAGE_SIZE - 1) / PAGE_SIZE;
- unsigned long frames[pages];
+ unsigned long frames[65536 / PAGE_SIZE];
int f;
/*
{
unsigned long va = dtr->address;
unsigned int size = dtr->size + 1;
- unsigned pages = (size + PAGE_SIZE - 1) / PAGE_SIZE;
- unsigned long frames[pages];
+ unsigned long frames[(GDT_SIZE + PAGE_SIZE - 1) / PAGE_SIZE];
int f;
/*
* 8-byte entries, or 16 4k pages..
*/
- BUG_ON(size > 65536);
+ BUG_ON(size > GDT_SIZE);
BUG_ON(va & ~PAGE_MASK);
for (f = 0; va < dtr->address + size; va += PAGE_SIZE, f++) {
return 0;
}
-static void set_xen_basic_apic_ops(void)
+static void __init set_xen_basic_apic_ops(void)
{
apic->read = xen_apic_read;
apic->write = xen_apic_write;
#endif
};
-static void xen_reboot(int reason)
+static __noreturn void xen_reboot(int reason)
{
struct sched_shutdown r = { .reason = reason };
- if (HYPERVISOR_sched_op(SCHEDOP_shutdown, &r))
- BUG();
+ HYPERVISOR_sched_op(SCHEDOP_shutdown, &r);
+ BUG();
}
-static void xen_restart(char *msg)
+static __noreturn void xen_restart(char *msg)
{
xen_reboot(SHUTDOWN_reboot);
}
-static void xen_emergency_restart(void)
+static __noreturn void xen_emergency_restart(void)
{
xen_reboot(SHUTDOWN_reboot);
}
-static void xen_machine_halt(void)
+static __noreturn void xen_machine_halt(void)
{
xen_reboot(SHUTDOWN_poweroff);
}
-static void xen_machine_power_off(void)
+static __noreturn void xen_machine_power_off(void)
{
if (pm_power_off)
pm_power_off();
pv_cpu_ops.write_gdt_entry = xen_write_gdt_entry_boot;
pv_cpu_ops.load_gdt = xen_load_gdt_boot;
- setup_stack_canary_segment(0);
- switch_to_new_gdt(0);
+ setup_stack_canary_segment(cpu);
+#ifdef CONFIG_X86_64
+ load_percpu_segment(cpu);
+#endif
+ switch_to_new_gdt(cpu);
pv_cpu_ops.write_gdt_entry = xen_write_gdt_entry;
pv_cpu_ops.load_gdt = xen_load_gdt;
__userpte_alloc_gfp &= ~__GFP_HIGHMEM;
/* Work out if we support NX */
- x86_configure_nx();
+#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
+ if ((cpuid_eax(0x80000000) & 0xffff0000) == 0x80000000 &&
+ (cpuid_edx(0x80000001) & (1U << (X86_FEATURE_NX & 31)))) {
+ unsigned l, h;
+
+ __supported_pte_mask |= _PAGE_NX;
+ rdmsr(MSR_EFER, l, h);
+ l |= EFER_NX;
+ wrmsr(MSR_EFER, l, h);
+ }
+#endif
/* Get mfn list */
xen_build_dynamic_phys_to_machine();
machine_ops = xen_machine_ops;
- /*
- * The only reliable way to retain the initial address of the
- * percpu gdt_page is to remember it here, so we can go and
- * mark it RW later, when the initial percpu area is freed.
- */
- xen_initial_gdt = &per_cpu(gdt_page, 0);
-
xen_smp_init();
#ifdef CONFIG_ACPI_NUMA
return val;
}
-static pteval_t pte_pfn_to_mfn(pteval_t val)
+static pteval_t __intentional_overflow(-1) pte_pfn_to_mfn(pteval_t val)
{
if (val & _PAGE_PRESENT) {
unsigned long pfn = (val & PTE_PFN_MASK) >> PAGE_SHIFT;
* L3_k[511] -> level2_fixmap_pgt */
convert_pfn_mfn(level3_kernel_pgt);
+ convert_pfn_mfn(level3_vmalloc_start_pgt);
+ convert_pfn_mfn(level3_vmalloc_end_pgt);
+ convert_pfn_mfn(level3_vmemmap_pgt);
/* L3_k[511][506] -> level1_fixmap_pgt */
+ /* L3_k[511][507] -> level1_vsyscall_pgt */
convert_pfn_mfn(level2_fixmap_pgt);
}
/* We get [511][511] and have Xen's version of level2_kernel_pgt */
set_page_prot(init_level4_pgt, PAGE_KERNEL_RO);
set_page_prot(level3_ident_pgt, PAGE_KERNEL_RO);
set_page_prot(level3_kernel_pgt, PAGE_KERNEL_RO);
+ set_page_prot(level3_vmalloc_start_pgt, PAGE_KERNEL_RO);
+ set_page_prot(level3_vmalloc_end_pgt, PAGE_KERNEL_RO);
+ set_page_prot(level3_vmemmap_pgt, PAGE_KERNEL_RO);
set_page_prot(level3_user_vsyscall, PAGE_KERNEL_RO);
set_page_prot(level2_ident_pgt, PAGE_KERNEL_RO);
+ set_page_prot(level2_vmemmap_pgt, PAGE_KERNEL_RO);
set_page_prot(level2_kernel_pgt, PAGE_KERNEL_RO);
set_page_prot(level2_fixmap_pgt, PAGE_KERNEL_RO);
set_page_prot(level1_fixmap_pgt, PAGE_KERNEL_RO);
+ set_page_prot(level1_vsyscall_pgt, PAGE_KERNEL_RO);
/* Pin down new L4 */
pin_pagetable_pfn(MMUEXT_PIN_L4_TABLE,
pv_mmu_ops.set_pud = xen_set_pud;
#if PAGETABLE_LEVELS == 4
pv_mmu_ops.set_pgd = xen_set_pgd;
+ pv_mmu_ops.set_pgd_batched = xen_set_pgd;
#endif
/* This will work as long as patching hasn't happened yet
.pud_val = PV_CALLEE_SAVE(xen_pud_val),
.make_pud = PV_CALLEE_SAVE(xen_make_pud),
.set_pgd = xen_set_pgd_hyper,
+ .set_pgd_batched = xen_set_pgd_hyper,
.alloc_pud = xen_alloc_pmd_init,
.release_pud = xen_release_pmd_init,
if (xen_pv_domain()) {
if (!xen_feature(XENFEAT_writable_page_tables))
- /* We've switched to the "real" per-cpu gdt, so make
- * sure the old memory can be recycled. */
- make_lowmem_page_readwrite(xen_initial_gdt);
-
#ifdef CONFIG_X86_32
/*
* Xen starts us with XEN_FLAT_RING1_DS, but linux code
* expects __USER_DS
*/
- loadsegment(ds, __USER_DS);
- loadsegment(es, __USER_DS);
+ loadsegment(ds, __KERNEL_DS);
+ loadsegment(es, __KERNEL_DS);
#endif
xen_filter_cpu_maps();
#ifdef CONFIG_X86_32
/* Note: PVH is not yet supported on x86_32. */
ctxt->user_regs.fs = __KERNEL_PERCPU;
- ctxt->user_regs.gs = __KERNEL_STACK_CANARY;
+ savesegment(gs, ctxt->user_regs.gs);
#endif
memset(&ctxt->fpu_ctxt, 0, sizeof(ctxt->fpu_ctxt));
ctxt->user_regs.eip = (unsigned long)cpu_bringup_and_idle;
ctxt->flags = VGCF_IN_KERNEL;
ctxt->user_regs.eflags = 0x1000; /* IOPL_RING1 */
- ctxt->user_regs.ds = __USER_DS;
- ctxt->user_regs.es = __USER_DS;
+ ctxt->user_regs.ds = __KERNEL_DS;
+ ctxt->user_regs.es = __KERNEL_DS;
ctxt->user_regs.ss = __KERNEL_DS;
xen_copy_trap_info(ctxt->trap_ctxt);
int rc;
per_cpu(current_task, cpu) = idle;
+ per_cpu(current_tinfo, cpu) = &idle->tinfo;
#ifdef CONFIG_X86_32
irq_ctx_init(cpu);
#else
clear_tsk_thread_flag(idle, TIF_FORK);
#endif
- per_cpu(kernel_stack, cpu) =
- (unsigned long)task_stack_page(idle) -
- KERNEL_STACK_OFFSET + THREAD_SIZE;
+ per_cpu(kernel_stack, cpu) = (unsigned long)task_stack_page(idle) - 16 + THREAD_SIZE;
xen_setup_runstate_info(cpu);
xen_setup_timer(cpu);
void __init xen_smp_init(void)
{
- smp_ops = xen_smp_ops;
+ memcpy((void *)&smp_ops, &xen_smp_ops, sizeof smp_ops);
xen_fill_possible_map();
}
pushw %fs
movl $(__KERNEL_PERCPU), %eax
movl %eax, %fs
- movl %fs:xen_vcpu, %eax
+ mov PER_CPU_VAR(xen_vcpu), %eax
POP_FS
#else
movl %ss:xen_vcpu, %eax
#ifdef CONFIG_X86_32
mov %esi,xen_start_info
mov $init_thread_union+THREAD_SIZE,%esp
+#ifdef CONFIG_SMP
+ movl $cpu_gdt_table,%edi
+ movl $__per_cpu_load,%eax
+ movw %ax,__KERNEL_PERCPU + 2(%edi)
+ rorl $16,%eax
+ movb %al,__KERNEL_PERCPU + 4(%edi)
+ movb %ah,__KERNEL_PERCPU + 7(%edi)
+ movl $__per_cpu_end - 1,%eax
+ subl $__per_cpu_start,%eax
+ movw %ax,__KERNEL_PERCPU + 0(%edi)
+#endif
#else
mov %rsi,xen_start_info
mov $init_thread_union+THREAD_SIZE,%rsp
extern const char xen_hypervisor_callback[];
extern const char xen_failsafe_callback[];
-extern void *xen_initial_gdt;
-
struct trap_info;
void xen_copy_trap_info(struct trap_info *traps);
----------------------------------------------------------------------*/
#define XCHAL_ICACHE_LINESIZE 32 /* I-cache line size in bytes */
-#define XCHAL_DCACHE_LINESIZE 32 /* D-cache line size in bytes */
#define XCHAL_ICACHE_LINEWIDTH 5 /* log2(I line size in bytes) */
#define XCHAL_DCACHE_LINEWIDTH 5 /* log2(D line size in bytes) */
+#define XCHAL_DCACHE_LINESIZE (_AC(1,UL) << XCHAL_DCACHE_LINEWIDTH) /* D-cache line size in bytes */
#define XCHAL_ICACHE_SIZE 16384 /* I-cache size in bytes or 0 */
#define XCHAL_DCACHE_SIZE 16384 /* D-cache size in bytes or 0 */
#ifndef _XTENSA_CORE_H
#define _XTENSA_CORE_H
+#include <linux/const.h>
/****************************************************************************
Parameters Useful for Any Code, USER or PRIVILEGED
----------------------------------------------------------------------*/
#define XCHAL_ICACHE_LINESIZE 16 /* I-cache line size in bytes */
-#define XCHAL_DCACHE_LINESIZE 16 /* D-cache line size in bytes */
#define XCHAL_ICACHE_LINEWIDTH 4 /* log2(I line size in bytes) */
#define XCHAL_DCACHE_LINEWIDTH 4 /* log2(D line size in bytes) */
+#define XCHAL_DCACHE_LINESIZE (_AC(1,UL) << XCHAL_DCACHE_LINEWIDTH) /* D-cache line size in bytes */
#define XCHAL_ICACHE_SIZE 8192 /* I-cache size in bytes or 0 */
#define XCHAL_DCACHE_SIZE 8192 /* D-cache size in bytes or 0 */
/*
* Overflow, abort
*/
- if (end < start)
+ if (end < start || end - start > INT_MAX - nr_pages)
return ERR_PTR(-EINVAL);
nr_pages += end - start;
/*
* Overflow, abort
*/
- if (end < start)
+ if (end < start || end - start > INT_MAX - nr_pages)
return ERR_PTR(-EINVAL);
nr_pages += end - start;
const int read = bio_data_dir(bio) == READ;
struct bio_map_data *bmd = bio->bi_private;
int i;
- char *p = bmd->sgvecs[0].iov_base;
+ char *p = (char __force_kernel *)bmd->sgvecs[0].iov_base;
bio_for_each_segment_all(bvec, bio, i) {
char *addr = page_address(bvec->bv_page);
}
EXPORT_SYMBOL(blk_iopoll_complete);
-static void blk_iopoll_softirq(struct softirq_action *h)
+static __latent_entropy void blk_iopoll_softirq(void)
{
struct list_head *list = this_cpu_ptr(&blk_cpu_iopoll);
int rearm = 0, budget = blk_iopoll_budget;
if (!len || !kbuf)
return -EINVAL;
- do_copy = !blk_rq_aligned(q, addr, len) || object_is_on_stack(kbuf);
+ do_copy = !blk_rq_aligned(q, addr, len) || object_starts_on_stack(kbuf);
if (do_copy)
bio = bio_copy_kern(q, kbuf, len, gfp_mask, reading);
else
* Softirq action handler - move entries to local list and loop over them
* while passing them to the queue registered handler.
*/
-static void blk_done_softirq(struct softirq_action *h)
+static __latent_entropy void blk_done_softirq(void)
{
struct list_head *cpu_list, local_list;
struct sg_io_v4 *hdr, struct bsg_device *bd,
fmode_t has_write_perm)
{
+ unsigned char tmpcmd[sizeof(rq->__cmd)];
+ unsigned char *cmdptr;
+
if (hdr->request_len > BLK_MAX_CDB) {
rq->cmd = kzalloc(hdr->request_len, GFP_KERNEL);
if (!rq->cmd)
return -ENOMEM;
- }
+ cmdptr = rq->cmd;
+ } else
+ cmdptr = tmpcmd;
- if (copy_from_user(rq->cmd, (void __user *)(unsigned long)hdr->request,
+ if (copy_from_user(cmdptr, (void __user *)(unsigned long)hdr->request,
hdr->request_len))
return -EFAULT;
+ if (cmdptr != rq->cmd)
+ memcpy(rq->cmd, cmdptr, hdr->request_len);
+
if (hdr->subprotocol == BSG_SUB_PROTOCOL_SCSI_CMD) {
if (blk_verify_command(rq->cmd, has_write_perm))
return -EPERM;
cgc = compat_alloc_user_space(sizeof(*cgc));
cgc32 = compat_ptr(arg);
- if (copy_in_user(&cgc->cmd, &cgc32->cmd, sizeof(cgc->cmd)) ||
+ if (copy_in_user(cgc->cmd, cgc32->cmd, sizeof(cgc->cmd)) ||
get_user(data, &cgc32->buffer) ||
put_user(compat_ptr(data), &cgc->buffer) ||
copy_in_user(&cgc->buflen, &cgc32->buflen,
err |= __get_user(f->spec1, &uf->spec1);
err |= __get_user(f->fmt_gap, &uf->fmt_gap);
err |= __get_user(name, &uf->name);
- f->name = compat_ptr(name);
+ f->name = (void __force_kernel *)compat_ptr(name);
if (err) {
err = -EFAULT;
goto out;
/*
* Register device numbers dev..(dev+range-1)
- * range must be nonzero
+ * Noop if @range is zero.
* The hash chain is sorted on range, so that subranges can override.
*/
void blk_register_region(dev_t devt, unsigned long range, struct module *module,
struct kobject *(*probe)(dev_t, int *, void *),
int (*lock)(dev_t, void *), void *data)
{
- kobj_map(bdev_map, devt, range, module, probe, lock, data);
+ if (range)
+ kobj_map(bdev_map, devt, range, module, probe, lock, data);
}
EXPORT_SYMBOL(blk_register_region);
+/* undo blk_register_region(), noop if @range is zero */
void blk_unregister_region(dev_t devt, unsigned long range)
{
- kobj_unmap(bdev_map, devt, range);
+ if (range)
+ kobj_unmap(bdev_map, devt, range);
}
EXPORT_SYMBOL(blk_unregister_region);
if (!gpt)
return NULL;
- count = le32_to_cpu(gpt->num_partition_entries) *
- le32_to_cpu(gpt->sizeof_partition_entry);
- if (!count)
+ if (!le32_to_cpu(gpt->num_partition_entries))
return NULL;
- pte = kmalloc(count, GFP_KERNEL);
+ pte = kcalloc(le32_to_cpu(gpt->num_partition_entries), le32_to_cpu(gpt->sizeof_partition_entry), GFP_KERNEL);
if (!pte)
return NULL;
+ count = le32_to_cpu(gpt->num_partition_entries) *
+ le32_to_cpu(gpt->sizeof_partition_entry);
if (read_lba(state, le64_to_cpu(gpt->partition_entry_lba),
(u8 *) pte, count) < count) {
kfree(pte);
return put_user(0, p);
}
-static int sg_get_timeout(struct request_queue *q)
+static int __intentional_overflow(-1) sg_get_timeout(struct request_queue *q)
{
return jiffies_to_clock_t(q->sg_timeout);
}
static int blk_fill_sghdr_rq(struct request_queue *q, struct request *rq,
struct sg_io_hdr *hdr, fmode_t mode)
{
- if (copy_from_user(rq->cmd, hdr->cmdp, hdr->cmd_len))
+ unsigned char tmpcmd[sizeof(rq->__cmd)];
+ unsigned char *cmdptr;
+
+ if (rq->cmd != rq->__cmd)
+ cmdptr = rq->cmd;
+ else
+ cmdptr = tmpcmd;
+
+ if (copy_from_user(cmdptr, hdr->cmdp, hdr->cmd_len))
return -EFAULT;
+
+ if (cmdptr != rq->cmd)
+ memcpy(rq->cmd, cmdptr, hdr->cmd_len);
+
if (blk_verify_command(rq->cmd, mode & FMODE_WRITE))
return -EPERM;
int err;
unsigned int in_len, out_len, bytes, opcode, cmdlen;
char *buffer = NULL, sense[SCSI_SENSE_BUFFERSIZE];
+ unsigned char tmpcmd[sizeof(rq->__cmd)];
+ unsigned char *cmdptr;
if (!sic)
return -EINVAL;
*/
err = -EFAULT;
rq->cmd_len = cmdlen;
- if (copy_from_user(rq->cmd, sic->data, cmdlen))
+
+ if (rq->cmd != rq->__cmd)
+ cmdptr = rq->cmd;
+ else
+ cmdptr = tmpcmd;
+
+ if (copy_from_user(cmdptr, sic->data, cmdlen))
goto error;
+ if (rq->cmd != cmdptr)
+ memcpy(rq->cmd, cmdptr, cmdlen);
+
if (in_len && copy_from_user(buffer, sic->data + cmdlen, in_len))
goto error;
struct cryptd_blkcipher_request_ctx {
crypto_completion_t complete;
-};
+} __no_const;
struct cryptd_hash_ctx {
struct crypto_shash *child;
struct cryptd_aead_request_ctx {
crypto_completion_t complete;
-};
+} __no_const;
static void cryptd_queue_worker(struct work_struct *work);
int ret;
pinst->kobj.kset = pcrypt_kset;
- ret = kobject_add(&pinst->kobj, NULL, name);
+ ret = kobject_add(&pinst->kobj, NULL, "%s", name);
if (!ret)
kobject_uevent(&pinst->kobj, KOBJ_ADD);
/* Legacy functions are optional, based upon ACPI_REDUCED_HARDWARE */
static struct acpi_sleep_functions acpi_sleep_dispatch[] = {
- {ACPI_HW_OPTIONAL_FUNCTION(acpi_hw_legacy_sleep),
- acpi_hw_extended_sleep},
- {ACPI_HW_OPTIONAL_FUNCTION(acpi_hw_legacy_wake_prep),
- acpi_hw_extended_wake_prep},
- {ACPI_HW_OPTIONAL_FUNCTION(acpi_hw_legacy_wake), acpi_hw_extended_wake}
+ {.legacy_function = ACPI_HW_OPTIONAL_FUNCTION(acpi_hw_legacy_sleep),
+ .extended_function = acpi_hw_extended_sleep},
+ {.legacy_function = ACPI_HW_OPTIONAL_FUNCTION(acpi_hw_legacy_wake_prep),
+ .extended_function = acpi_hw_extended_wake_prep},
+ {.legacy_function = ACPI_HW_OPTIONAL_FUNCTION(acpi_hw_legacy_wake),
+ .extended_function = acpi_hw_extended_wake}
};
/*
struct apei_exec_ins_type {
u32 flags;
apei_exec_ins_func_t run;
-};
+} __do_const;
struct apei_exec_context {
u32 ip;
const struct acpi_hest_generic *generic,
const struct acpi_hest_generic_status *estatus)
{
- static atomic_t seqno;
+ static atomic_unchecked_t seqno;
unsigned int curr_seqno;
char pfx_seq[64];
else
pfx = KERN_ERR;
}
- curr_seqno = atomic_inc_return(&seqno);
+ curr_seqno = atomic_inc_return_unchecked(&seqno);
snprintf(pfx_seq, sizeof(pfx_seq), "%s{%u}" HW_ERR, pfx, curr_seqno);
printk("%s""Hardware error from APEI Generic Hardware Error Source: %d\n",
pfx_seq, generic->header.source_id);
if (!bgrt_image)
return -ENODEV;
- bin_attr_image.private = bgrt_image;
- bin_attr_image.size = bgrt_image_size;
+ pax_open_kernel();
+ *(void **)&bin_attr_image.private = bgrt_image;
+ *(size_t *)&bin_attr_image.size = bgrt_image_size;
+ pax_close_kernel();
bgrt_kobj = kobject_create_and_add("bgrt", acpi_kobj);
if (!bgrt_kobj)
u32 is_critical_error;
};
-static struct dmi_system_id acpi_osi_dmi_table[] __initdata;
+static const struct dmi_system_id acpi_osi_dmi_table[] __initconst;
/*
* POLICY: If *anything* doesn't work, put it on the blacklist.
return 0;
}
-static struct dmi_system_id acpi_osi_dmi_table[] __initdata = {
+static const struct dmi_system_id acpi_osi_dmi_table[] __initconst = {
{
.callback = dmi_disable_osi_vista,
.ident = "Fujitsu Siemens",
struct acpi_table_header table;
acpi_status status;
+#ifdef CONFIG_GRKERNSEC_KMEM
+ return -EPERM;
+#endif
+
if (!(*ppos)) {
/* parse the table header to get the table length */
if (count <= sizeof(struct acpi_table_header))
#endif /* CONFIG_PM_SLEEP */
+static void acpi_dev_pm_detach(struct device *dev, bool power_off);
+
static struct dev_pm_domain acpi_general_pm_domain = {
.ops = {
#ifdef CONFIG_PM
#endif
#endif
},
+ .detach = acpi_dev_pm_detach
};
/**
acpi_device_wakeup(adev, ACPI_STATE_S0, false);
}
- dev->pm_domain->detach = acpi_dev_pm_detach;
return 0;
}
EXPORT_SYMBOL_GPL(acpi_dev_pm_attach);
{
int i, count = CPUIDLE_DRIVER_STATE_START;
struct acpi_processor_cx *cx;
- struct cpuidle_state *state;
+ cpuidle_state_no_const *state;
struct cpuidle_driver *drv = &acpi_idle_driver;
if (!pr->flags.power_setup_done)
static struct attribute **all_attrs;
static u32 acpi_gpe_count;
-static struct attribute_group interrupt_stats_attr_group = {
+static attribute_group_no_const interrupt_stats_attr_group = {
.name = "interrupts",
};
-static struct kobj_attribute *counter_attrs;
+static kobj_attribute_no_const *counter_attrs;
static void delete_gpe_attr_array(void)
{
}
EXPORT_SYMBOL_GPL(ahci_kick_engine);
-static int ahci_exec_polled_cmd(struct ata_port *ap, int pmp,
+static int __intentional_overflow(-1) ahci_exec_polled_cmd(struct ata_port *ap, int pmp,
struct ata_taskfile *tf, int is_cmd, u16 flags,
unsigned long timeout_msec)
{
static void ata_dev_xfermask(struct ata_device *dev);
static unsigned long ata_dev_blacklisted(const struct ata_device *dev);
-atomic_t ata_print_id = ATOMIC_INIT(0);
+atomic_unchecked_t ata_print_id = ATOMIC_INIT(0);
struct ata_force_param {
const char *name;
struct ata_port *ap;
unsigned int tag;
- WARN_ON_ONCE(qc == NULL); /* ata_qc_from_tag _might_ return NULL */
+ BUG_ON(qc == NULL); /* ata_qc_from_tag _might_ return NULL */
ap = qc->ap;
qc->flags = 0;
struct ata_port *ap;
struct ata_link *link;
- WARN_ON_ONCE(qc == NULL); /* ata_qc_from_tag _might_ return NULL */
+ BUG_ON(qc == NULL); /* ata_qc_from_tag _might_ return NULL */
WARN_ON_ONCE(!(qc->flags & ATA_QCFLAG_ACTIVE));
ap = qc->ap;
link = qc->dev->link;
return;
spin_lock(&lock);
+ pax_open_kernel();
for (cur = ops->inherits; cur; cur = cur->inherits) {
void **inherit = (void **)cur;
if (IS_ERR(*pp))
*pp = NULL;
- ops->inherits = NULL;
+ *(struct ata_port_operations **)&ops->inherits = NULL;
+ pax_close_kernel();
spin_unlock(&lock);
}
/* give ports names and add SCSI hosts */
for (i = 0; i < host->n_ports; i++) {
- host->ports[i]->print_id = atomic_inc_return(&ata_print_id);
+ host->ports[i]->print_id = atomic_inc_return_unchecked(&ata_print_id);
host->ports[i]->local_port_no = i + 1;
}
if (rc)
return rc;
- ap->print_id = atomic_inc_return(&ata_print_id);
+ ap->print_id = atomic_inc_return_unchecked(&ata_print_id);
return 0;
}
EXPORT_SYMBOL_GPL(ata_sas_port_init);
ATA_DNXFER_QUIET = (1 << 31),
};
-extern atomic_t ata_print_id;
+extern atomic_unchecked_t ata_print_id;
extern int atapi_passthru16;
extern int libata_fua;
extern int libata_noacpi;
/* Handle platform specific quirks */
if (quirk) {
if (quirk & CF_BROKEN_PIO) {
- ap->ops->set_piomode = NULL;
+ pax_open_kernel();
+ *(void **)&ap->ops->set_piomode = NULL;
+ pax_close_kernel();
ap->pio_mask = 0;
}
if (quirk & CF_BROKEN_MWDMA)
vcc->pop(vcc, skb);
else
dev_kfree_skb_any(skb);
- atomic_inc(&vcc->stats->tx);
+ atomic_inc_unchecked(&vcc->stats->tx);
return 0;
}
PRINTD (DBG_FLOW|DBG_TX, "tx_complete %p %p", dev, tx);
// VC layer stats
- atomic_inc(&ATM_SKB(skb)->vcc->stats->tx);
+ atomic_inc_unchecked(&ATM_SKB(skb)->vcc->stats->tx);
// free the descriptor
kfree (tx_descr);
dump_skb ("<<<", vc, skb);
// VC layer stats
- atomic_inc(&atm_vcc->stats->rx);
+ atomic_inc_unchecked(&atm_vcc->stats->rx);
__net_timestamp(skb);
// end of our responsibility
atm_vcc->push (atm_vcc, skb);
} else {
PRINTK (KERN_INFO, "dropped over-size frame");
// should we count this?
- atomic_inc(&atm_vcc->stats->rx_drop);
+ atomic_inc_unchecked(&atm_vcc->stats->rx_drop);
}
} else {
}
if (check_area (skb->data, skb->len)) {
- atomic_inc(&atm_vcc->stats->tx_err);
+ atomic_inc_unchecked(&atm_vcc->stats->tx_err);
return -ENOMEM; // ?
}
if (vcc->pop) vcc->pop(vcc,skb);
else dev_kfree_skb(skb);
if (dev_data) return 0;
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
return -ENOLINK;
}
size = skb->len+sizeof(struct atmtcp_hdr);
if (!new_skb) {
if (vcc->pop) vcc->pop(vcc,skb);
else dev_kfree_skb(skb);
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
return -ENOBUFS;
}
hdr = (void *) skb_put(new_skb,sizeof(struct atmtcp_hdr));
if (vcc->pop) vcc->pop(vcc,skb);
else dev_kfree_skb(skb);
out_vcc->push(out_vcc,new_skb);
- atomic_inc(&vcc->stats->tx);
- atomic_inc(&out_vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->tx);
+ atomic_inc_unchecked(&out_vcc->stats->rx);
return 0;
}
read_unlock(&vcc_sklist_lock);
if (!out_vcc) {
result = -EUNATCH;
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
goto done;
}
skb_pull(skb,sizeof(struct atmtcp_hdr));
__net_timestamp(new_skb);
skb_copy_from_linear_data(skb, skb_put(new_skb, skb->len), skb->len);
out_vcc->push(out_vcc,new_skb);
- atomic_inc(&vcc->stats->tx);
- atomic_inc(&out_vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->tx);
+ atomic_inc_unchecked(&out_vcc->stats->rx);
done:
if (vcc->pop) vcc->pop(vcc,skb);
else dev_kfree_skb(skb);
DPRINTK(DEV_LABEL "(itf %d): trashing empty cell\n",
vcc->dev->number);
length = 0;
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
}
else {
length = ATM_CELL_SIZE-1; /* no HEC */
size);
}
eff = length = 0;
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
}
else {
size = (descr & MID_RED_COUNT)*(ATM_CELL_PAYLOAD >> 2);
"(VCI=%d,length=%ld,size=%ld (descr 0x%lx))\n",
vcc->dev->number,vcc->vci,length,size << 2,descr);
length = eff = 0;
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
}
}
skb = eff ? atm_alloc_charge(vcc,eff << 2,GFP_ATOMIC) : NULL;
vcc->push(vcc,skb);
pushed++;
}
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
}
wake_up(&eni_dev->rx_wait);
}
PCI_DMA_TODEVICE);
if (vcc->pop) vcc->pop(vcc,skb);
else dev_kfree_skb_irq(skb);
- atomic_inc(&vcc->stats->tx);
+ atomic_inc_unchecked(&vcc->stats->tx);
wake_up(&eni_dev->tx_wait);
dma_complete++;
}
}
}
- atomic_inc(&ATM_SKB(skb)->vcc->stats->tx);
+ atomic_inc_unchecked(&ATM_SKB(skb)->vcc->stats->tx);
fs_dprintk (FS_DEBUG_TXMEM, "i");
fs_dprintk (FS_DEBUG_ALLOC, "Free t-skb: %p\n", skb);
#endif
skb_put (skb, qe->p1 & 0xffff);
ATM_SKB(skb)->vcc = atm_vcc;
- atomic_inc(&atm_vcc->stats->rx);
+ atomic_inc_unchecked(&atm_vcc->stats->rx);
__net_timestamp(skb);
fs_dprintk (FS_DEBUG_ALLOC, "Free rec-skb: %p (pushed)\n", skb);
atm_vcc->push (atm_vcc, skb);
kfree (pe);
}
if (atm_vcc)
- atomic_inc(&atm_vcc->stats->rx_drop);
+ atomic_inc_unchecked(&atm_vcc->stats->rx_drop);
break;
case 0x1f: /* Reassembly abort: no buffers. */
/* Silently increment error counter. */
if (atm_vcc)
- atomic_inc(&atm_vcc->stats->rx_drop);
+ atomic_inc_unchecked(&atm_vcc->stats->rx_drop);
break;
default: /* Hmm. Haven't written the code to handle the others yet... -- REW */
printk (KERN_WARNING "Don't know what to do with RX status %x: %s.\n",
#endif
/* check error condition */
if (*entry->status & STATUS_ERROR)
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
else
- atomic_inc(&vcc->stats->tx);
+ atomic_inc_unchecked(&vcc->stats->tx);
}
}
if (skb == NULL) {
DPRINTK(2, "unable to alloc new skb, rx PDU length = %d\n", pdu_len);
- atomic_inc(&vcc->stats->rx_drop);
+ atomic_inc_unchecked(&vcc->stats->rx_drop);
return -ENOMEM;
}
dev_kfree_skb_any(skb);
- atomic_inc(&vcc->stats->rx_drop);
+ atomic_inc_unchecked(&vcc->stats->rx_drop);
return -ENOMEM;
}
ASSERT(atomic_read(&sk_atm(vcc)->sk_wmem_alloc) >= 0);
vcc->push(vcc, skb);
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
ASSERT(atomic_read(&sk_atm(vcc)->sk_wmem_alloc) >= 0);
DPRINTK(2, "damaged PDU on %d.%d.%d\n",
fore200e->atm_dev->number,
entry->rpd->atm_header.vpi, entry->rpd->atm_header.vci);
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
}
}
goto retry_here;
}
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
fore200e->tx_sat++;
DPRINTK(2, "tx queue of device %s is saturated, PDU dropped - heartbeat is %08x\n",
if (RBRQ_HBUF_ERR(he_dev->rbrq_head)) {
hprintk("HBUF_ERR! (cid 0x%x)\n", cid);
- atomic_inc(&vcc->stats->rx_drop);
+ atomic_inc_unchecked(&vcc->stats->rx_drop);
goto return_host_buffers;
}
RBRQ_LEN_ERR(he_dev->rbrq_head)
? "LEN_ERR" : "",
vcc->vpi, vcc->vci);
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
goto return_host_buffers;
}
vcc->push(vcc, skb);
spin_lock(&he_dev->global_lock);
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
return_host_buffers:
++pdus_assembled;
tpd->vcc->pop(tpd->vcc, tpd->skb);
else
dev_kfree_skb_any(tpd->skb);
- atomic_inc(&tpd->vcc->stats->tx_err);
+ atomic_inc_unchecked(&tpd->vcc->stats->tx_err);
}
pci_pool_free(he_dev->tpd_pool, tpd, TPD_ADDR(tpd->status));
return;
vcc->pop(vcc, skb);
else
dev_kfree_skb_any(skb);
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
return -EINVAL;
}
vcc->pop(vcc, skb);
else
dev_kfree_skb_any(skb);
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
return -EINVAL;
}
#endif
vcc->pop(vcc, skb);
else
dev_kfree_skb_any(skb);
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
spin_unlock_irqrestore(&he_dev->global_lock, flags);
return -ENOMEM;
}
vcc->pop(vcc, skb);
else
dev_kfree_skb_any(skb);
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
spin_unlock_irqrestore(&he_dev->global_lock, flags);
return -ENOMEM;
}
__enqueue_tpd(he_dev, tpd, cid);
spin_unlock_irqrestore(&he_dev->global_lock, flags);
- atomic_inc(&vcc->stats->tx);
+ atomic_inc_unchecked(&vcc->stats->tx);
return 0;
}
{
struct atm_vcc * vcc = ATM_SKB(skb)->vcc;
// VC layer stats
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
__net_timestamp(skb);
// end of our responsibility
vcc->push (vcc, skb);
dev->tx_iovec = NULL;
// VC layer stats
- atomic_inc(&ATM_SKB(skb)->vcc->stats->tx);
+ atomic_inc_unchecked(&ATM_SKB(skb)->vcc->stats->tx);
// free the skb
hrz_kfree_skb (skb);
else
dev_kfree_skb(skb);
- atomic_inc(&vcc->stats->tx);
+ atomic_inc_unchecked(&vcc->stats->tx);
}
atomic_dec(&scq->used);
if ((sb = dev_alloc_skb(64)) == NULL) {
printk("%s: Can't allocate buffers for aal0.\n",
card->name);
- atomic_add(i, &vcc->stats->rx_drop);
+ atomic_add_unchecked(i, &vcc->stats->rx_drop);
break;
}
if (!atm_charge(vcc, sb->truesize)) {
RXPRINTK("%s: atm_charge() dropped aal0 packets.\n",
card->name);
- atomic_add(i - 1, &vcc->stats->rx_drop);
+ atomic_add_unchecked(i - 1, &vcc->stats->rx_drop);
dev_kfree_skb(sb);
break;
}
ATM_SKB(sb)->vcc = vcc;
__net_timestamp(sb);
vcc->push(vcc, sb);
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
cell += ATM_CELL_PAYLOAD;
}
"(CDC: %08x)\n",
card->name, len, rpp->len, readl(SAR_REG_CDC));
recycle_rx_pool_skb(card, rpp);
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
return;
}
if (stat & SAR_RSQE_CRC) {
RXPRINTK("%s: AAL5 CRC error.\n", card->name);
recycle_rx_pool_skb(card, rpp);
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
return;
}
if (skb_queue_len(&rpp->queue) > 1) {
RXPRINTK("%s: Can't alloc RX skb.\n",
card->name);
recycle_rx_pool_skb(card, rpp);
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
return;
}
if (!atm_charge(vcc, skb->truesize)) {
__net_timestamp(skb);
vcc->push(vcc, skb);
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
return;
}
__net_timestamp(skb);
vcc->push(vcc, skb);
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
if (skb->truesize > SAR_FB_SIZE_3)
add_rx_skb(card, 3, SAR_FB_SIZE_3, 1);
if (vcc->qos.aal != ATM_AAL0) {
RPRINTK("%s: raw cell for non AAL0 vc %u.%u\n",
card->name, vpi, vci);
- atomic_inc(&vcc->stats->rx_drop);
+ atomic_inc_unchecked(&vcc->stats->rx_drop);
goto drop;
}
if ((sb = dev_alloc_skb(64)) == NULL) {
printk("%s: Can't allocate buffers for AAL0.\n",
card->name);
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
goto drop;
}
ATM_SKB(sb)->vcc = vcc;
__net_timestamp(sb);
vcc->push(vcc, sb);
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
drop:
skb_pull(queue, 64);
if (vc == NULL) {
printk("%s: NULL connection in send().\n", card->name);
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
dev_kfree_skb(skb);
return -EINVAL;
}
if (!test_bit(VCF_TX, &vc->flags)) {
printk("%s: Trying to transmit on a non-tx VC.\n", card->name);
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
dev_kfree_skb(skb);
return -EINVAL;
}
break;
default:
printk("%s: Unsupported AAL: %d\n", card->name, vcc->qos.aal);
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
dev_kfree_skb(skb);
return -EINVAL;
}
if (skb_shinfo(skb)->nr_frags != 0) {
printk("%s: No scatter-gather yet.\n", card->name);
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
dev_kfree_skb(skb);
return -EINVAL;
}
err = queue_skb(card, vc, skb, oam);
if (err) {
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
dev_kfree_skb(skb);
return err;
}
skb = dev_alloc_skb(64);
if (!skb) {
printk("%s: Out of memory in send_oam().\n", card->name);
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
return -ENOMEM;
}
atomic_add(skb->truesize, &sk_atm(vcc)->sk_wmem_alloc);
status = (u_short) (buf_desc_ptr->desc_mode);
if (status & (RX_CER | RX_PTE | RX_OFL))
{
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
IF_ERR(printk("IA: bad packet, dropping it");)
if (status & RX_CER) {
IF_ERR(printk(" cause: packet CRC error\n");)
len = dma_addr - buf_addr;
if (len > iadev->rx_buf_sz) {
printk("Over %d bytes sdu received, dropped!!!\n", iadev->rx_buf_sz);
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
goto out_free_desc;
}
ia_vcc = INPH_IA_VCC(vcc);
if (ia_vcc == NULL)
{
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
atm_return(vcc, skb->truesize);
dev_kfree_skb_any(skb);
goto INCR_DLE;
if ((length > iadev->rx_buf_sz) || (length >
(skb->len - sizeof(struct cpcs_trailer))))
{
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
IF_ERR(printk("rx_dle_intr: Bad AAL5 trailer %d (skb len %d)",
length, skb->len);)
atm_return(vcc, skb->truesize);
IF_RX(printk("rx_dle_intr: skb push");)
vcc->push(vcc,skb);
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
iadev->rx_pkt_cnt++;
}
INCR_DLE:
{
struct k_sonet_stats *stats;
stats = &PRIV(_ia_dev[board])->sonet_stats;
- printk("section_bip: %d\n", atomic_read(&stats->section_bip));
- printk("line_bip : %d\n", atomic_read(&stats->line_bip));
- printk("path_bip : %d\n", atomic_read(&stats->path_bip));
- printk("line_febe : %d\n", atomic_read(&stats->line_febe));
- printk("path_febe : %d\n", atomic_read(&stats->path_febe));
- printk("corr_hcs : %d\n", atomic_read(&stats->corr_hcs));
- printk("uncorr_hcs : %d\n", atomic_read(&stats->uncorr_hcs));
- printk("tx_cells : %d\n", atomic_read(&stats->tx_cells));
- printk("rx_cells : %d\n", atomic_read(&stats->rx_cells));
+ printk("section_bip: %d\n", atomic_read_unchecked(&stats->section_bip));
+ printk("line_bip : %d\n", atomic_read_unchecked(&stats->line_bip));
+ printk("path_bip : %d\n", atomic_read_unchecked(&stats->path_bip));
+ printk("line_febe : %d\n", atomic_read_unchecked(&stats->line_febe));
+ printk("path_febe : %d\n", atomic_read_unchecked(&stats->path_febe));
+ printk("corr_hcs : %d\n", atomic_read_unchecked(&stats->corr_hcs));
+ printk("uncorr_hcs : %d\n", atomic_read_unchecked(&stats->uncorr_hcs));
+ printk("tx_cells : %d\n", atomic_read_unchecked(&stats->tx_cells));
+ printk("rx_cells : %d\n", atomic_read_unchecked(&stats->rx_cells));
}
ia_cmds.status = 0;
break;
if ((desc == 0) || (desc > iadev->num_tx_desc))
{
IF_ERR(printk(DEV_LABEL "invalid desc for send: %d\n", desc);)
- atomic_inc(&vcc->stats->tx);
+ atomic_inc_unchecked(&vcc->stats->tx);
if (vcc->pop)
vcc->pop(vcc, skb);
else
ATM_DESC(skb) = vcc->vci;
skb_queue_tail(&iadev->tx_dma_q, skb);
- atomic_inc(&vcc->stats->tx);
+ atomic_inc_unchecked(&vcc->stats->tx);
iadev->tx_pkt_cnt++;
/* Increment transaction counter */
writel(2, iadev->dma+IPHASE5575_TX_COUNTER);
#if 0
/* add flow control logic */
- if (atomic_read(&vcc->stats->tx) % 20 == 0) {
+ if (atomic_read_unchecked(&vcc->stats->tx) % 20 == 0) {
if (iavcc->vc_desc_cnt > 10) {
vcc->tx_quota = vcc->tx_quota * 3 / 4;
printk("Tx1: vcc->tx_quota = %d \n", (u32)vcc->tx_quota );
vcc_tx_add_aal5_trailer(lvcc, skb->len, 0, 0);
lanai_endtx(lanai, lvcc);
lanai_free_skb(lvcc->tx.atmvcc, skb);
- atomic_inc(&lvcc->tx.atmvcc->stats->tx);
+ atomic_inc_unchecked(&lvcc->tx.atmvcc->stats->tx);
}
/* Try to fill the buffer - don't call unless there is backlog */
ATM_SKB(skb)->vcc = lvcc->rx.atmvcc;
__net_timestamp(skb);
lvcc->rx.atmvcc->push(lvcc->rx.atmvcc, skb);
- atomic_inc(&lvcc->rx.atmvcc->stats->rx);
+ atomic_inc_unchecked(&lvcc->rx.atmvcc->stats->rx);
out:
lvcc->rx.buf.ptr = end;
cardvcc_write(lvcc, endptr, vcc_rxreadptr);
DPRINTK("(itf %d) got RX service entry 0x%X for non-AAL5 "
"vcc %d\n", lanai->number, (unsigned int) s, vci);
lanai->stats.service_rxnotaal5++;
- atomic_inc(&lvcc->rx.atmvcc->stats->rx_err);
+ atomic_inc_unchecked(&lvcc->rx.atmvcc->stats->rx_err);
return 0;
}
if (likely(!(s & (SERVICE_TRASH | SERVICE_STREAM | SERVICE_CRCERR)))) {
int bytes;
read_unlock(&vcc_sklist_lock);
DPRINTK("got trashed rx pdu on vci %d\n", vci);
- atomic_inc(&lvcc->rx.atmvcc->stats->rx_err);
+ atomic_inc_unchecked(&lvcc->rx.atmvcc->stats->rx_err);
lvcc->stats.x.aal5.service_trash++;
bytes = (SERVICE_GET_END(s) * 16) -
(((unsigned long) lvcc->rx.buf.ptr) -
}
if (s & SERVICE_STREAM) {
read_unlock(&vcc_sklist_lock);
- atomic_inc(&lvcc->rx.atmvcc->stats->rx_err);
+ atomic_inc_unchecked(&lvcc->rx.atmvcc->stats->rx_err);
lvcc->stats.x.aal5.service_stream++;
printk(KERN_ERR DEV_LABEL "(itf %d): Got AAL5 stream "
"PDU on VCI %d!\n", lanai->number, vci);
return 0;
}
DPRINTK("got rx crc error on vci %d\n", vci);
- atomic_inc(&lvcc->rx.atmvcc->stats->rx_err);
+ atomic_inc_unchecked(&lvcc->rx.atmvcc->stats->rx_err);
lvcc->stats.x.aal5.service_rxcrc++;
lvcc->rx.buf.ptr = &lvcc->rx.buf.start[SERVICE_GET_END(s) * 4];
cardvcc_write(lvcc, SERVICE_GET_END(s), vcc_rxreadptr);
if ((vc = (vc_map *) vcc->dev_data) == NULL) {
printk("nicstar%d: vcc->dev_data == NULL on ns_send().\n",
card->index);
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
dev_kfree_skb_any(skb);
return -EINVAL;
}
if (!vc->tx) {
printk("nicstar%d: Trying to transmit on a non-tx VC.\n",
card->index);
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
dev_kfree_skb_any(skb);
return -EINVAL;
}
if (vcc->qos.aal != ATM_AAL5 && vcc->qos.aal != ATM_AAL0) {
printk("nicstar%d: Only AAL0 and AAL5 are supported.\n",
card->index);
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
dev_kfree_skb_any(skb);
return -EINVAL;
}
if (skb_shinfo(skb)->nr_frags != 0) {
printk("nicstar%d: No scatter-gather yet.\n", card->index);
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
dev_kfree_skb_any(skb);
return -EINVAL;
}
}
if (push_scqe(card, vc, scq, &scqe, skb) != 0) {
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
dev_kfree_skb_any(skb);
return -EIO;
}
- atomic_inc(&vcc->stats->tx);
+ atomic_inc_unchecked(&vcc->stats->tx);
return 0;
}
printk
("nicstar%d: Can't allocate buffers for aal0.\n",
card->index);
- atomic_add(i, &vcc->stats->rx_drop);
+ atomic_add_unchecked(i, &vcc->stats->rx_drop);
break;
}
if (!atm_charge(vcc, sb->truesize)) {
RXPRINTK
("nicstar%d: atm_charge() dropped aal0 packets.\n",
card->index);
- atomic_add(i - 1, &vcc->stats->rx_drop); /* already increased by 1 */
+ atomic_add_unchecked(i - 1, &vcc->stats->rx_drop); /* already increased by 1 */
dev_kfree_skb_any(sb);
break;
}
ATM_SKB(sb)->vcc = vcc;
__net_timestamp(sb);
vcc->push(vcc, sb);
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
cell += ATM_CELL_PAYLOAD;
}
if (iovb == NULL) {
printk("nicstar%d: Out of iovec buffers.\n",
card->index);
- atomic_inc(&vcc->stats->rx_drop);
+ atomic_inc_unchecked(&vcc->stats->rx_drop);
recycle_rx_buf(card, skb);
return;
}
small or large buffer itself. */
} else if (NS_PRV_IOVCNT(iovb) >= NS_MAX_IOVECS) {
printk("nicstar%d: received too big AAL5 SDU.\n", card->index);
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
recycle_iovec_rx_bufs(card, (struct iovec *)iovb->data,
NS_MAX_IOVECS);
NS_PRV_IOVCNT(iovb) = 0;
("nicstar%d: Expected a small buffer, and this is not one.\n",
card->index);
which_list(card, skb);
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
recycle_rx_buf(card, skb);
vc->rx_iov = NULL;
recycle_iov_buf(card, iovb);
("nicstar%d: Expected a large buffer, and this is not one.\n",
card->index);
which_list(card, skb);
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
recycle_iovec_rx_bufs(card, (struct iovec *)iovb->data,
NS_PRV_IOVCNT(iovb));
vc->rx_iov = NULL;
printk(" - PDU size mismatch.\n");
else
printk(".\n");
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
recycle_iovec_rx_bufs(card, (struct iovec *)iovb->data,
NS_PRV_IOVCNT(iovb));
vc->rx_iov = NULL;
/* skb points to a small buffer */
if (!atm_charge(vcc, skb->truesize)) {
push_rxbufs(card, skb);
- atomic_inc(&vcc->stats->rx_drop);
+ atomic_inc_unchecked(&vcc->stats->rx_drop);
} else {
skb_put(skb, len);
dequeue_sm_buf(card, skb);
ATM_SKB(skb)->vcc = vcc;
__net_timestamp(skb);
vcc->push(vcc, skb);
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
}
} else if (NS_PRV_IOVCNT(iovb) == 2) { /* One small plus one large buffer */
struct sk_buff *sb;
if (len <= NS_SMBUFSIZE) {
if (!atm_charge(vcc, sb->truesize)) {
push_rxbufs(card, sb);
- atomic_inc(&vcc->stats->rx_drop);
+ atomic_inc_unchecked(&vcc->stats->rx_drop);
} else {
skb_put(sb, len);
dequeue_sm_buf(card, sb);
ATM_SKB(sb)->vcc = vcc;
__net_timestamp(sb);
vcc->push(vcc, sb);
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
}
push_rxbufs(card, skb);
if (!atm_charge(vcc, skb->truesize)) {
push_rxbufs(card, skb);
- atomic_inc(&vcc->stats->rx_drop);
+ atomic_inc_unchecked(&vcc->stats->rx_drop);
} else {
dequeue_lg_buf(card, skb);
#ifdef NS_USE_DESTRUCTORS
ATM_SKB(skb)->vcc = vcc;
__net_timestamp(skb);
vcc->push(vcc, skb);
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
}
push_rxbufs(card, sb);
printk
("nicstar%d: Out of huge buffers.\n",
card->index);
- atomic_inc(&vcc->stats->rx_drop);
+ atomic_inc_unchecked(&vcc->stats->rx_drop);
recycle_iovec_rx_bufs(card,
(struct iovec *)
iovb->data,
card->hbpool.count++;
} else
dev_kfree_skb_any(hb);
- atomic_inc(&vcc->stats->rx_drop);
+ atomic_inc_unchecked(&vcc->stats->rx_drop);
} else {
/* Copy the small buffer to the huge buffer */
sb = (struct sk_buff *)iov->iov_base;
#endif /* NS_USE_DESTRUCTORS */
__net_timestamp(hb);
vcc->push(vcc, hb);
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
}
}
}
atm_charge(vcc, skb->truesize);
vcc->push(vcc, skb);
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
break;
case PKT_STATUS:
vcc = SKB_CB(oldskb)->vcc;
if (vcc) {
- atomic_inc(&vcc->stats->tx);
+ atomic_inc_unchecked(&vcc->stats->tx);
solos_pop(vcc, oldskb);
} else {
dev_kfree_skb_irq(oldskb);
#define ADD_LIMITED(s,v) \
- atomic_add((v),&stats->s); \
- if (atomic_read(&stats->s) < 0) atomic_set(&stats->s,INT_MAX);
+ atomic_add_unchecked((v),&stats->s); \
+ if (atomic_read_unchecked(&stats->s) < 0) atomic_set_unchecked(&stats->s,INT_MAX);
static void suni_hz(unsigned long from_timer)
struct sonet_stats tmp;
int error = 0;
- atomic_add(GET(HECCT),&PRIV(dev)->sonet_stats.uncorr_hcs);
+ atomic_add_unchecked(GET(HECCT),&PRIV(dev)->sonet_stats.uncorr_hcs);
sonet_copy_stats(&PRIV(dev)->sonet_stats,&tmp);
if (arg) error = copy_to_user(arg,&tmp,sizeof(tmp));
if (zero && !error) {
#define ADD_LIMITED(s,v) \
- { atomic_add(GET(v),&PRIV(dev)->sonet_stats.s); \
- if (atomic_read(&PRIV(dev)->sonet_stats.s) < 0) \
- atomic_set(&PRIV(dev)->sonet_stats.s,INT_MAX); }
+ { atomic_add_unchecked(GET(v),&PRIV(dev)->sonet_stats.s); \
+ if (atomic_read_unchecked(&PRIV(dev)->sonet_stats.s) < 0) \
+ atomic_set_unchecked(&PRIV(dev)->sonet_stats.s,INT_MAX); }
static void stat_event(struct atm_dev *dev)
if (reason & uPD98402_INT_PFM) stat_event(dev);
if (reason & uPD98402_INT_PCO) {
(void) GET(PCOCR); /* clear interrupt cause */
- atomic_add(GET(HECCT),
+ atomic_add_unchecked(GET(HECCT),
&PRIV(dev)->sonet_stats.uncorr_hcs);
}
if ((reason & uPD98402_INT_RFO) &&
PUT(~(uPD98402_INT_PFM | uPD98402_INT_ALM | uPD98402_INT_RFO |
uPD98402_INT_LOS),PIMR); /* enable them */
(void) fetch_stats(dev,NULL,1); /* clear kernel counters */
- atomic_set(&PRIV(dev)->sonet_stats.corr_hcs,-1);
- atomic_set(&PRIV(dev)->sonet_stats.tx_cells,-1);
- atomic_set(&PRIV(dev)->sonet_stats.rx_cells,-1);
+ atomic_set_unchecked(&PRIV(dev)->sonet_stats.corr_hcs,-1);
+ atomic_set_unchecked(&PRIV(dev)->sonet_stats.tx_cells,-1);
+ atomic_set_unchecked(&PRIV(dev)->sonet_stats.rx_cells,-1);
return 0;
}
}
if (!size) {
dev_kfree_skb_irq(skb);
- if (vcc) atomic_inc(&vcc->stats->rx_err);
+ if (vcc) atomic_inc_unchecked(&vcc->stats->rx_err);
continue;
}
if (!atm_charge(vcc,skb->truesize)) {
skb->len = size;
ATM_SKB(skb)->vcc = vcc;
vcc->push(vcc,skb);
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
}
zout(pos & 0xffff,MTA(mbx));
#if 0 /* probably a stupid idea */
skb_queue_head(&zatm_vcc->backlog,skb);
break;
}
- atomic_inc(&vcc->stats->tx);
+ atomic_inc_unchecked(&vcc->stats->tx);
wake_up(&zatm_vcc->tx_wait);
}
return -EINVAL;
mutex_lock(&subsys->p->mutex);
- list_add_tail(&sif->node, &subsys->p->interfaces);
+ pax_list_add_tail((struct list_head *)&sif->node, &subsys->p->interfaces);
if (sif->add_dev) {
subsys_dev_iter_init(&iter, subsys, NULL, NULL);
while ((dev = subsys_dev_iter_next(&iter)))
subsys = sif->subsys;
mutex_lock(&subsys->p->mutex);
- list_del_init(&sif->node);
+ pax_list_del_init((struct list_head *)&sif->node);
if (sif->remove_dev) {
subsys_dev_iter_init(&iter, subsys, NULL, NULL);
while ((dev = subsys_dev_iter_next(&iter)))
if (!thread)
return 0;
- err = sys_mount("devtmpfs", (char *)mntdir, "devtmpfs", MS_SILENT, NULL);
+ err = sys_mount((char __force_user *)"devtmpfs", (char __force_user *)mntdir, (char __force_user *)"devtmpfs", MS_SILENT, NULL);
if (err)
printk(KERN_INFO "devtmpfs: error mounting %i\n", err);
else
*err = sys_unshare(CLONE_NEWNS);
if (*err)
goto out;
- *err = sys_mount("devtmpfs", "/", "devtmpfs", MS_SILENT, options);
+ *err = sys_mount((char __force_user *)"devtmpfs", (char __force_user *)"/", (char __force_user *)"devtmpfs", MS_SILENT, (char __force_user *)options);
if (*err)
goto out;
- sys_chdir("/.."); /* will traverse into overmounted root */
- sys_chroot(".");
+ sys_chdir((char __force_user *)"/.."); /* will traverse into overmounted root */
+ sys_chroot((char __force_user *)".");
complete(&setup_done);
while (1) {
spin_lock(&req_lock);
struct node_attr {
struct device_attribute attr;
enum node_states state;
-};
+} __do_const;
static ssize_t show_node_state(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct cpuidle_driver *cpuidle_drv;
struct gpd_cpuidle_data *cpuidle_data;
- struct cpuidle_state *idle_state;
+ cpuidle_state_no_const *idle_state;
int ret = 0;
if (IS_ERR_OR_NULL(genpd) || state < 0)
int pm_genpd_detach_cpuidle(struct generic_pm_domain *genpd)
{
struct gpd_cpuidle_data *cpuidle_data;
- struct cpuidle_state *idle_state;
+ cpuidle_state_no_const *idle_state;
int ret = 0;
if (IS_ERR_OR_NULL(genpd))
return ret;
}
- dev->pm_domain->detach = genpd_dev_pm_detach;
+ pax_open_kernel();
+ *(void **)&dev->pm_domain->detach = genpd_dev_pm_detach;
+ pax_close_kernel();
+
pm_genpd_poweron(pd);
return 0;
return -EIO;
}
}
- return sprintf(buf, p);
+ return sprintf(buf, "%s", p);
}
static DEVICE_ATTR(runtime_status, 0444, rtpm_status_show, NULL);
* They need to be modified together atomically, so it's better to use one
* atomic variable to hold them both.
*/
-static atomic_t combined_event_count = ATOMIC_INIT(0);
+static atomic_unchecked_t combined_event_count = ATOMIC_INIT(0);
#define IN_PROGRESS_BITS (sizeof(int) * 4)
#define MAX_IN_PROGRESS ((1 << IN_PROGRESS_BITS) - 1)
static void split_counters(unsigned int *cnt, unsigned int *inpr)
{
- unsigned int comb = atomic_read(&combined_event_count);
+ unsigned int comb = atomic_read_unchecked(&combined_event_count);
*cnt = (comb >> IN_PROGRESS_BITS);
*inpr = comb & MAX_IN_PROGRESS;
ws->start_prevent_time = ws->last_time;
/* Increment the counter of events in progress. */
- cec = atomic_inc_return(&combined_event_count);
+ cec = atomic_inc_return_unchecked(&combined_event_count);
trace_wakeup_source_activate(ws->name, cec);
}
* Increment the counter of registered wakeup events and decrement the
* couter of wakeup events in progress simultaneously.
*/
- cec = atomic_add_return(MAX_IN_PROGRESS, &combined_event_count);
+ cec = atomic_add_return_unchecked(MAX_IN_PROGRESS, &combined_event_count);
trace_wakeup_source_deactivate(ws->name, cec);
split_counters(&cnt, &inpr);
void register_syscore_ops(struct syscore_ops *ops)
{
mutex_lock(&syscore_ops_lock);
- list_add_tail(&ops->node, &syscore_ops_list);
+ pax_list_add_tail((struct list_head *)&ops->node, &syscore_ops_list);
mutex_unlock(&syscore_ops_lock);
}
EXPORT_SYMBOL_GPL(register_syscore_ops);
void unregister_syscore_ops(struct syscore_ops *ops)
{
mutex_lock(&syscore_ops_lock);
- list_del(&ops->node);
+ pax_list_del((struct list_head *)&ops->node);
mutex_unlock(&syscore_ops_lock);
}
EXPORT_SYMBOL_GPL(unregister_syscore_ops);
while (!list_empty(&h->reqQ)) {
c = list_entry(h->reqQ.next, CommandList_struct, list);
/* can't do anything if fifo is full */
- if ((h->access.fifo_full(h))) {
+ if ((h->access->fifo_full(h))) {
dev_warn(&h->pdev->dev, "fifo full\n");
break;
}
h->Qdepth--;
/* Tell the controller execute command */
- h->access.submit_command(h, c);
+ h->access->submit_command(h, c);
/* Put job onto the completed Q */
addQ(&h->cmpQ, c);
static inline unsigned long get_next_completion(ctlr_info_t *h)
{
- return h->access.command_completed(h);
+ return h->access->command_completed(h);
}
static inline int interrupt_pending(ctlr_info_t *h)
{
- return h->access.intr_pending(h);
+ return h->access->intr_pending(h);
}
static inline long interrupt_not_for_us(ctlr_info_t *h)
{
- return ((h->access.intr_pending(h) == 0) ||
+ return ((h->access->intr_pending(h) == 0) ||
(h->interrupts_enabled == 0));
}
u32 a;
if (unlikely(!(h->transMethod & CFGTBL_Trans_Performant)))
- return h->access.command_completed(h);
+ return h->access->command_completed(h);
if ((*(h->reply_pool_head) & 1) == (h->reply_pool_wraparound)) {
a = *(h->reply_pool_head); /* Next cmd in ring buffer */
trans_support & CFGTBL_Trans_use_short_tags);
/* Change the access methods to the performant access methods */
- h->access = SA5_performant_access;
+ h->access = &SA5_performant_access;
h->transMethod = CFGTBL_Trans_Performant;
return;
if (prod_index < 0)
return -ENODEV;
h->product_name = products[prod_index].product_name;
- h->access = *(products[prod_index].access);
+ h->access = products[prod_index].access;
if (cciss_board_disabled(h)) {
dev_warn(&h->pdev->dev, "controller appears to be disabled\n");
}
/* make sure the board interrupts are off */
- h->access.set_intr_mask(h, CCISS_INTR_OFF);
+ h->access->set_intr_mask(h, CCISS_INTR_OFF);
rc = cciss_request_irq(h, do_cciss_msix_intr, do_cciss_intx);
if (rc)
goto clean2;
* fake ones to scoop up any residual completions.
*/
spin_lock_irqsave(&h->lock, flags);
- h->access.set_intr_mask(h, CCISS_INTR_OFF);
+ h->access->set_intr_mask(h, CCISS_INTR_OFF);
spin_unlock_irqrestore(&h->lock, flags);
free_irq(h->intr[h->intr_mode], h);
rc = cciss_request_irq(h, cciss_msix_discard_completions,
dev_info(&h->pdev->dev, "Board READY.\n");
dev_info(&h->pdev->dev,
"Waiting for stale completions to drain.\n");
- h->access.set_intr_mask(h, CCISS_INTR_ON);
+ h->access->set_intr_mask(h, CCISS_INTR_ON);
msleep(10000);
- h->access.set_intr_mask(h, CCISS_INTR_OFF);
+ h->access->set_intr_mask(h, CCISS_INTR_OFF);
rc = controller_reset_failed(h->cfgtable);
if (rc)
cciss_scsi_setup(h);
/* Turn the interrupts on so we can service requests */
- h->access.set_intr_mask(h, CCISS_INTR_ON);
+ h->access->set_intr_mask(h, CCISS_INTR_ON);
/* Get the firmware version */
inq_buff = kzalloc(sizeof(InquiryData_struct), GFP_KERNEL);
kfree(flush_buf);
if (return_code != IO_OK)
dev_warn(&h->pdev->dev, "Error flushing cache\n");
- h->access.set_intr_mask(h, CCISS_INTR_OFF);
+ h->access->set_intr_mask(h, CCISS_INTR_OFF);
free_irq(h->intr[h->intr_mode], h);
}
/* information about each logical volume */
drive_info_struct *drv[CISS_MAX_LUN];
- struct access_method access;
+ struct access_method *access;
/* queue and queue Info */
struct list_head reqQ;
}
static struct access_method SA5_access = {
- SA5_submit_command,
- SA5_intr_mask,
- SA5_fifo_full,
- SA5_intr_pending,
- SA5_completed,
+ .submit_command = SA5_submit_command,
+ .set_intr_mask = SA5_intr_mask,
+ .fifo_full = SA5_fifo_full,
+ .intr_pending = SA5_intr_pending,
+ .command_completed = SA5_completed,
};
static struct access_method SA5B_access = {
- SA5_submit_command,
- SA5B_intr_mask,
- SA5_fifo_full,
- SA5B_intr_pending,
- SA5_completed,
+ .submit_command = SA5_submit_command,
+ .set_intr_mask = SA5B_intr_mask,
+ .fifo_full = SA5_fifo_full,
+ .intr_pending = SA5B_intr_pending,
+ .command_completed = SA5_completed,
};
static struct access_method SA5_performant_access = {
- SA5_submit_command,
- SA5_performant_intr_mask,
- SA5_fifo_full,
- SA5_performant_intr_pending,
- SA5_performant_completed,
+ .submit_command = SA5_submit_command,
+ .set_intr_mask = SA5_performant_intr_mask,
+ .fifo_full = SA5_fifo_full,
+ .intr_pending = SA5_performant_intr_pending,
+ .command_completed = SA5_performant_completed,
};
struct board_type {
if (register_blkdev(COMPAQ_SMART2_MAJOR+i, hba[i]->devname)) {
goto Enomem4;
}
- hba[i]->access.set_intr_mask(hba[i], 0);
+ hba[i]->access->set_intr_mask(hba[i], 0);
if (request_irq(hba[i]->intr, do_ida_intr,
IRQF_DISABLED|IRQF_SHARED, hba[i]->devname, hba[i]))
{
add_timer(&hba[i]->timer);
/* Enable IRQ now that spinlock and rate limit timer are set up */
- hba[i]->access.set_intr_mask(hba[i], FIFO_NOT_EMPTY);
+ hba[i]->access->set_intr_mask(hba[i], FIFO_NOT_EMPTY);
for(j=0; j<NWD; j++) {
struct gendisk *disk = ida_gendisk[i][j];
for(i=0; i<NR_PRODUCTS; i++) {
if (board_id == products[i].board_id) {
c->product_name = products[i].product_name;
- c->access = *(products[i].access);
+ c->access = products[i].access;
break;
}
}
hba[ctlr]->intr = intr;
sprintf(hba[ctlr]->devname, "ida%d", nr_ctlr);
hba[ctlr]->product_name = products[j].product_name;
- hba[ctlr]->access = *(products[j].access);
+ hba[ctlr]->access = products[j].access;
hba[ctlr]->ctlr = ctlr;
hba[ctlr]->board_id = board_id;
hba[ctlr]->pci_dev = NULL; /* not PCI */
while((c = h->reqQ) != NULL) {
/* Can't do anything if we're busy */
- if (h->access.fifo_full(h) == 0)
+ if (h->access->fifo_full(h) == 0)
return;
/* Get the first entry from the request Q */
h->Qdepth--;
/* Tell the controller to do our bidding */
- h->access.submit_command(h, c);
+ h->access->submit_command(h, c);
/* Get onto the completion Q */
addQ(&h->cmpQ, c);
unsigned long flags;
__u32 a,a1;
- istat = h->access.intr_pending(h);
+ istat = h->access->intr_pending(h);
/* Is this interrupt for us? */
if (istat == 0)
return IRQ_NONE;
*/
spin_lock_irqsave(IDA_LOCK(h->ctlr), flags);
if (istat & FIFO_NOT_EMPTY) {
- while((a = h->access.command_completed(h))) {
+ while((a = h->access->command_completed(h))) {
a1 = a; a &= ~3;
if ((c = h->cmpQ) == NULL)
{
/*
* Disable interrupt
*/
- info_p->access.set_intr_mask(info_p, 0);
+ info_p->access->set_intr_mask(info_p, 0);
/* Make sure there is room in the command FIFO */
/* Actually it should be completely empty at this time. */
for (i = 200000; i > 0; i--) {
- temp = info_p->access.fifo_full(info_p);
+ temp = info_p->access->fifo_full(info_p);
if (temp != 0) {
break;
}
/*
* Send the cmd
*/
- info_p->access.submit_command(info_p, c);
+ info_p->access->submit_command(info_p, c);
complete = pollcomplete(ctlr);
pci_unmap_single(info_p->pci_dev, (dma_addr_t) c->req.sg[0].addr,
* we check the new geometry. Then turn interrupts back on when
* we're done.
*/
- host->access.set_intr_mask(host, 0);
+ host->access->set_intr_mask(host, 0);
getgeometry(ctlr);
- host->access.set_intr_mask(host, FIFO_NOT_EMPTY);
+ host->access->set_intr_mask(host, FIFO_NOT_EMPTY);
for(i=0; i<NWD; i++) {
struct gendisk *disk = ida_gendisk[ctlr][i];
/* Wait (up to 2 seconds) for a command to complete */
for (i = 200000; i > 0; i--) {
- done = hba[ctlr]->access.command_completed(hba[ctlr]);
+ done = hba[ctlr]->access->command_completed(hba[ctlr]);
if (done == 0) {
udelay(10); /* a short fixed delay */
} else
drv_info_t drv[NWD];
struct proc_dir_entry *proc;
- struct access_method access;
+ struct access_method *access;
cmdlist_t *reqQ;
cmdlist_t *cmpQ;
submit_bio(rw, bio);
/* this should not count as user activity and cause the
* resync to throttle -- see drbd_rs_should_slow_down(). */
- atomic_add(len >> 9, &device->rs_sect_ev);
+ atomic_add_unchecked(len >> 9, &device->rs_sect_ev);
}
}
struct drbd_connection *connection;
struct list_head list;
unsigned int barrier_nr;
- atomic_t epoch_size; /* increased on every request added. */
+ atomic_unchecked_t epoch_size; /* increased on every request added. */
atomic_t active; /* increased on every req. added, and dec on every finished. */
unsigned long flags;
};
unsigned int al_tr_number;
int al_tr_cycle;
wait_queue_head_t seq_wait;
- atomic_t packet_seq;
+ atomic_unchecked_t packet_seq;
unsigned int peer_seq;
spinlock_t peer_seq_lock;
unsigned long comm_bm_set; /* communicated number of set bits. */
struct mutex own_state_mutex;
struct mutex *state_mutex; /* either own_state_mutex or first_peer_device(device)->connection->cstate_mutex */
char congestion_reason; /* Why we where congested... */
- atomic_t rs_sect_in; /* for incoming resync data rate, SyncTarget */
- atomic_t rs_sect_ev; /* for submitted resync data rate, both */
+ atomic_unchecked_t rs_sect_in; /* for incoming resync data rate, SyncTarget */
+ atomic_unchecked_t rs_sect_ev; /* for submitted resync data rate, both */
int rs_last_sect_ev; /* counter to compare with */
int rs_last_events; /* counter of read or write "events" (unit sectors)
* on the lower level device when we last looked. */
p->sector = sector;
p->block_id = block_id;
p->blksize = blksize;
- p->seq_num = cpu_to_be32(atomic_inc_return(&peer_device->device->packet_seq));
+ p->seq_num = cpu_to_be32(atomic_inc_return_unchecked(&peer_device->device->packet_seq));
return drbd_send_command(peer_device, sock, cmd, sizeof(*p), NULL, 0);
}
return -EIO;
p->sector = cpu_to_be64(req->i.sector);
p->block_id = (unsigned long)req;
- p->seq_num = cpu_to_be32(atomic_inc_return(&device->packet_seq));
+ p->seq_num = cpu_to_be32(atomic_inc_return_unchecked(&device->packet_seq));
dp_flags = bio_flags_to_wire(peer_device->connection, req->master_bio->bi_rw);
if (device->state.conn >= C_SYNC_SOURCE &&
device->state.conn <= C_PAUSED_SYNC_T)
atomic_set(&device->unacked_cnt, 0);
atomic_set(&device->local_cnt, 0);
atomic_set(&device->pp_in_use_by_net, 0);
- atomic_set(&device->rs_sect_in, 0);
- atomic_set(&device->rs_sect_ev, 0);
+ atomic_set_unchecked(&device->rs_sect_in, 0);
+ atomic_set_unchecked(&device->rs_sect_ev, 0);
atomic_set(&device->ap_in_flight, 0);
atomic_set(&device->md_io.in_use, 0);
struct drbd_connection *connection = container_of(kref, struct drbd_connection, kref);
struct drbd_resource *resource = connection->resource;
- if (atomic_read(&connection->current_epoch->epoch_size) != 0)
- drbd_err(connection, "epoch_size:%d\n", atomic_read(&connection->current_epoch->epoch_size));
+ if (atomic_read_unchecked(&connection->current_epoch->epoch_size) != 0)
+ drbd_err(connection, "epoch_size:%d\n", atomic_read_unchecked(&connection->current_epoch->epoch_size));
kfree(connection->current_epoch);
idr_destroy(&connection->peer_devices);
void drbd_bcast_event(struct drbd_device *device, const struct sib_info *sib)
{
- static atomic_t drbd_genl_seq = ATOMIC_INIT(2); /* two. */
+ static atomic_unchecked_t drbd_genl_seq = ATOMIC_INIT(2); /* two. */
struct sk_buff *msg;
struct drbd_genlmsghdr *d_out;
unsigned seq;
int err = -ENOMEM;
- seq = atomic_inc_return(&drbd_genl_seq);
+ seq = atomic_inc_return_unchecked(&drbd_genl_seq);
msg = genlmsg_new(NLMSG_GOODSIZE, GFP_NOIO);
if (!msg)
goto failed;
struct drbd_device *device = peer_device->device;
int err;
- atomic_set(&device->packet_seq, 0);
+ atomic_set_unchecked(&device->packet_seq, 0);
device->peer_seq = 0;
device->state_mutex = peer_device->connection->agreed_pro_version < 100 ?
do {
next_epoch = NULL;
- epoch_size = atomic_read(&epoch->epoch_size);
+ epoch_size = atomic_read_unchecked(&epoch->epoch_size);
switch (ev & ~EV_CLEANUP) {
case EV_PUT:
rv = FE_DESTROYED;
} else {
epoch->flags = 0;
- atomic_set(&epoch->epoch_size, 0);
+ atomic_set_unchecked(&epoch->epoch_size, 0);
/* atomic_set(&epoch->active, 0); is already zero */
if (rv == FE_STILL_LIVE)
rv = FE_RECYCLED;
conn_wait_active_ee_empty(connection);
drbd_flush(connection);
- if (atomic_read(&connection->current_epoch->epoch_size)) {
+ if (atomic_read_unchecked(&connection->current_epoch->epoch_size)) {
epoch = kmalloc(sizeof(struct drbd_epoch), GFP_NOIO);
if (epoch)
break;
}
epoch->flags = 0;
- atomic_set(&epoch->epoch_size, 0);
+ atomic_set_unchecked(&epoch->epoch_size, 0);
atomic_set(&epoch->active, 0);
spin_lock(&connection->epoch_lock);
- if (atomic_read(&connection->current_epoch->epoch_size)) {
+ if (atomic_read_unchecked(&connection->current_epoch->epoch_size)) {
list_add(&epoch->list, &connection->current_epoch->list);
connection->current_epoch = epoch;
connection->epochs++;
list_add_tail(&peer_req->w.list, &device->sync_ee);
spin_unlock_irq(&device->resource->req_lock);
- atomic_add(pi->size >> 9, &device->rs_sect_ev);
+ atomic_add_unchecked(pi->size >> 9, &device->rs_sect_ev);
if (drbd_submit_peer_request(device, peer_req, WRITE, DRBD_FAULT_RS_WR) == 0)
return 0;
drbd_send_ack_dp(peer_device, P_NEG_ACK, p, pi->size);
}
- atomic_add(pi->size >> 9, &device->rs_sect_in);
+ atomic_add_unchecked(pi->size >> 9, &device->rs_sect_in);
return err;
}
err = wait_for_and_update_peer_seq(peer_device, peer_seq);
drbd_send_ack_dp(peer_device, P_NEG_ACK, p, pi->size);
- atomic_inc(&connection->current_epoch->epoch_size);
+ atomic_inc_unchecked(&connection->current_epoch->epoch_size);
err2 = drbd_drain_block(peer_device, pi->size);
if (!err)
err = err2;
spin_lock(&connection->epoch_lock);
peer_req->epoch = connection->current_epoch;
- atomic_inc(&peer_req->epoch->epoch_size);
+ atomic_inc_unchecked(&peer_req->epoch->epoch_size);
atomic_inc(&peer_req->epoch->active);
spin_unlock(&connection->epoch_lock);
curr_events = (int)part_stat_read(&disk->part0, sectors[0]) +
(int)part_stat_read(&disk->part0, sectors[1]) -
- atomic_read(&device->rs_sect_ev);
+ atomic_read_unchecked(&device->rs_sect_ev);
if (atomic_read(&device->ap_actlog_cnt)
|| curr_events - device->rs_last_events > 64) {
device->use_csums = true;
} else if (pi->cmd == P_OV_REPLY) {
/* track progress, we may need to throttle */
- atomic_add(size >> 9, &device->rs_sect_in);
+ atomic_add_unchecked(size >> 9, &device->rs_sect_in);
peer_req->w.cb = w_e_end_ov_reply;
dec_rs_pending(device);
/* drbd_rs_begin_io done when we sent this request,
goto out_free_e;
submit_for_resync:
- atomic_add(size >> 9, &device->rs_sect_ev);
+ atomic_add_unchecked(size >> 9, &device->rs_sect_ev);
submit:
update_receiver_timing_details(connection, drbd_submit_peer_request);
int expect_payload;
size_t pkt_size;
int (*fn)(struct drbd_connection *, struct packet_info *);
-};
+} __do_const;
static struct data_cmd drbd_cmd_handler[] = {
[P_DATA] = { 1, sizeof(struct p_data), receive_Data },
if (!list_empty(&connection->current_epoch->list))
drbd_err(connection, "ASSERTION FAILED: connection->current_epoch->list not empty\n");
/* ok, no more ee's on the fly, it is safe to reset the epoch_size */
- atomic_set(&connection->current_epoch->epoch_size, 0);
+ atomic_set_unchecked(&connection->current_epoch->epoch_size, 0);
connection->send.seen_any_write_yet = false;
drbd_info(connection, "Connection closed\n");
put_ldev(device);
}
dec_rs_pending(device);
- atomic_add(blksize >> 9, &device->rs_sect_in);
+ atomic_add_unchecked(blksize >> 9, &device->rs_sect_in);
return 0;
}
struct asender_cmd {
size_t pkt_size;
int (*fn)(struct drbd_connection *connection, struct packet_info *);
-};
+} __do_const;
static struct asender_cmd asender_tbl[] = {
[P_PING] = { 0, got_Ping },
list_add_tail(&peer_req->w.list, &device->read_ee);
spin_unlock_irq(&device->resource->req_lock);
- atomic_add(size >> 9, &device->rs_sect_ev);
+ atomic_add_unchecked(size >> 9, &device->rs_sect_ev);
if (drbd_submit_peer_request(device, peer_req, READ, DRBD_FAULT_RS_RD) == 0)
return 0;
unsigned int sect_in; /* Number of sectors that came in since the last turn */
int number, mxb;
- sect_in = atomic_xchg(&device->rs_sect_in, 0);
+ sect_in = atomic_xchg_unchecked(&device->rs_sect_in, 0);
device->rs_in_flight -= sect_in;
rcu_read_lock();
struct gendisk *disk = device->ldev->backing_bdev->bd_contains->bd_disk;
struct fifo_buffer *plan;
- atomic_set(&device->rs_sect_in, 0);
- atomic_set(&device->rs_sect_ev, 0);
+ atomic_set_unchecked(&device->rs_sect_in, 0);
+ atomic_set_unchecked(&device->rs_sect_ev, 0);
device->rs_in_flight = 0;
device->rs_last_events =
(int)part_stat_read(&disk->part0, sectors[0]) +
file_start_write(file);
set_fs(get_ds());
- bw = file->f_op->write(file, buf, len, &pos);
+ bw = file->f_op->write(file, (const char __force_user *)buf, len, &pos);
set_fs(old_fs);
file_end_write(file);
if (likely(bw == len))
static struct task_struct *nvme_thread;
static struct workqueue_struct *nvme_workq;
static wait_queue_head_t nvme_kthread_wait;
-static struct notifier_block nvme_nb;
static void nvme_reset_failed_dev(struct work_struct *ws);
static int nvme_process_cq(struct nvme_queue *nvmeq);
static void __exit nvme_exit(void)
{
pci_unregister_driver(&nvme_driver);
- unregister_hotcpu_notifier(&nvme_nb);
unregister_blkdev(nvme_major, "nvme");
destroy_workqueue(nvme_workq);
BUG_ON(nvme_thread && !IS_ERR(nvme_thread));
static sector_t get_zone(sector_t sector, struct pktcdvd_device *pd)
{
- return (sector + pd->offset) & ~(sector_t)(pd->settings.size - 1);
+ return (sector + pd->offset) & ~(sector_t)(pd->settings.size - 1UL);
}
/*
return -EROFS;
}
pd->settings.fp = ti.fp;
- pd->offset = (be32_to_cpu(ti.track_start) << 2) & (pd->settings.size - 1);
+ pd->offset = (be32_to_cpu(ti.track_start) << 2) & (pd->settings.size - 1UL);
if (ti.nwa_v) {
pd->nwa = be32_to_cpu(ti.next_writable);
* If the counter is already at its maximum value returns
* -EINVAL without updating it.
*/
-static int atomic_inc_return_safe(atomic_t *v)
+static int __intentional_overflow(-1) atomic_inc_return_safe(atomic_t *v)
{
unsigned int counter;
}
static struct access_method smart4_access = {
- smart4_submit_command,
- smart4_intr_mask,
- smart4_fifo_full,
- smart4_intr_pending,
- smart4_completed,
+ .submit_command = smart4_submit_command,
+ .set_intr_mask = smart4_intr_mask,
+ .fifo_full = smart4_fifo_full,
+ .intr_pending = smart4_intr_pending,
+ .command_completed = smart4_completed,
};
/*
}
static struct access_method smart2_access = {
- smart2_submit_command,
- smart2_intr_mask,
- smart2_fifo_full,
- smart2_intr_pending,
- smart2_completed,
+ .submit_command = smart2_submit_command,
+ .set_intr_mask = smart2_intr_mask,
+ .fifo_full = smart2_fifo_full,
+ .intr_pending = smart2_intr_pending,
+ .command_completed = smart2_completed,
};
/*
}
static struct access_method smart2e_access = {
- smart2e_submit_command,
- smart2e_intr_mask,
- smart2e_fifo_full,
- smart2e_intr_pending,
- smart2e_completed,
+ .submit_command = smart2e_submit_command,
+ .set_intr_mask = smart2e_intr_mask,
+ .fifo_full = smart2e_fifo_full,
+ .intr_pending = smart2e_intr_pending,
+ .command_completed = smart2e_completed,
};
/*
}
static struct access_method smart1_access = {
- smart1_submit_command,
- smart1_intr_mask,
- smart1_fifo_full,
- smart1_intr_pending,
- smart1_completed,
+ .submit_command = smart1_submit_command,
+ .set_intr_mask = smart1_intr_mask,
+ .fifo_full = smart1_fifo_full,
+ .intr_pending = smart1_intr_pending,
+ .command_completed = smart1_completed,
};
static int bt_ti_probe(struct platform_device *pdev)
{
- static struct ti_st *hst;
+ struct ti_st *hst;
struct hci_dev *hdev;
int err;
ENSURE(reset, CDC_RESET);
ENSURE(generic_packet, CDC_GENERIC_PACKET);
cdi->mc_flags = 0;
- cdo->n_minors = 0;
cdi->options = CDO_USE_FFLAGS;
if (autoclose == 1 && CDROM_CAN(CDC_CLOSE_TRAY))
else
cdi->cdda_method = CDDA_OLD;
- if (!cdo->generic_packet)
- cdo->generic_packet = cdrom_dummy_generic_packet;
+ if (!cdo->generic_packet) {
+ pax_open_kernel();
+ *(void **)&cdo->generic_packet = cdrom_dummy_generic_packet;
+ pax_close_kernel();
+ }
cd_dbg(CD_REG_UNREG, "drive \"/dev/%s\" registered\n", cdi->name);
mutex_lock(&cdrom_mutex);
if (cdi->exit)
cdi->exit(cdi);
- cdi->ops->n_minors--;
cd_dbg(CD_REG_UNREG, "drive \"/dev/%s\" unregistered\n", cdi->name);
}
*/
nr = nframes;
do {
- cgc.buffer = kmalloc(CD_FRAMESIZE_RAW * nr, GFP_KERNEL);
+ cgc.buffer = kzalloc(CD_FRAMESIZE_RAW * nr, GFP_KERNEL);
if (cgc.buffer)
break;
struct cdrom_device_info *cdi;
int ret;
- ret = scnprintf(info + *pos, max_size - *pos, header);
+ ret = scnprintf(info + *pos, max_size - *pos, "%s", header);
if (!ret)
return 1;
.audio_ioctl = gdrom_audio_ioctl,
.capability = CDC_MULTI_SESSION | CDC_MEDIA_CHANGED |
CDC_RESET | CDC_DRIVE_STATUS | CDC_CD_R,
- .n_minors = 1,
};
static int gdrom_bdops_open(struct block_device *bdev, fmode_t mode)
config DEVKMEM
bool "/dev/kmem virtual device support"
- default y
+ default n
+ depends on !GRKERNSEC_KMEM
help
Say Y here if you want to support the /dev/kmem device. The
/dev/kmem device is rarely used, but can be used for certain
bool
depends on !M68K
depends on ISA || PCI
+ depends on !GRKERNSEC_KMEM
default y
source "drivers/s390/char/Kconfig"
return -ENOMEM;
}
- if (copy_from_user(usegment, (void __user *) ureserve.seg_list,
+ if (copy_from_user(usegment, (void __force_user *) ureserve.seg_list,
sizeof(*usegment) * ureserve.seg_count)) {
kfree(usegment);
kfree(ksegment);
if (copy_from_user(&reserve, arg, sizeof(struct agp_region)))
return -EFAULT;
- if ((unsigned) reserve.seg_count >= ~0U/sizeof(struct agp_segment))
+ if ((unsigned) reserve.seg_count >= ~0U/sizeof(struct agp_segment_priv))
return -EFAULT;
client = agp_find_client_by_pid(reserve.pid);
if (segment == NULL)
return -ENOMEM;
- if (copy_from_user(segment, (void __user *) reserve.seg_list,
+ if (copy_from_user(segment, (void __force_user *) reserve.seg_list,
sizeof(struct agp_segment) * reserve.seg_count)) {
kfree(segment);
return -EFAULT;
switch (cmd) {
case RTC_PLL_GET:
+ memset(&pll, 0, sizeof(pll));
if (get_rtc_pll(&pll))
return -EINVAL;
else
}
static int
-hpet_ioctl_common(struct hpet_dev *devp, int cmd, unsigned long arg,
+hpet_ioctl_common(struct hpet_dev *devp, unsigned int cmd, unsigned long arg,
struct hpet_info *info)
{
struct hpet_timer __iomem *timer;
struct proc_dir_entry *proc_dir;
char proc_dir_name[10];
- atomic_t stats[IPMI_NUM_STATS];
+ atomic_unchecked_t stats[IPMI_NUM_STATS];
/*
* run_to_completion duplicate of smb_info, smi_info
static DEFINE_MUTEX(smi_watchers_mutex);
#define ipmi_inc_stat(intf, stat) \
- atomic_inc(&(intf)->stats[IPMI_STAT_ ## stat])
+ atomic_inc_unchecked(&(intf)->stats[IPMI_STAT_ ## stat])
#define ipmi_get_stat(intf, stat) \
- ((unsigned int) atomic_read(&(intf)->stats[IPMI_STAT_ ## stat]))
+ ((unsigned int) atomic_read_unchecked(&(intf)->stats[IPMI_STAT_ ## stat]))
static char *addr_src_to_str[] = { "invalid", "hotmod", "hardcoded", "SPMI",
"ACPI", "SMBIOS", "PCI",
INIT_LIST_HEAD(&intf->cmd_rcvrs);
init_waitqueue_head(&intf->waitq);
for (i = 0; i < IPMI_NUM_STATS; i++)
- atomic_set(&intf->stats[i], 0);
+ atomic_set_unchecked(&intf->stats[i], 0);
intf->proc_dir = NULL;
unsigned char slave_addr;
/* Counters and things for the proc filesystem. */
- atomic_t stats[SI_NUM_STATS];
+ atomic_unchecked_t stats[SI_NUM_STATS];
struct task_struct *thread;
};
#define smi_inc_stat(smi, stat) \
- atomic_inc(&(smi)->stats[SI_STAT_ ## stat])
+ atomic_inc_unchecked(&(smi)->stats[SI_STAT_ ## stat])
#define smi_get_stat(smi, stat) \
- ((unsigned int) atomic_read(&(smi)->stats[SI_STAT_ ## stat]))
+ ((unsigned int) atomic_read_unchecked(&(smi)->stats[SI_STAT_ ## stat]))
#define SI_MAX_PARMS 4
atomic_set(&new_smi->req_events, 0);
new_smi->run_to_completion = false;
for (i = 0; i < SI_NUM_STATS; i++)
- atomic_set(&new_smi->stats[i], 0);
+ atomic_set_unchecked(&new_smi->stats[i], 0);
new_smi->interrupt_disabled = true;
atomic_set(&new_smi->need_watch, 0);
#include <linux/raw.h>
#include <linux/tty.h>
#include <linux/capability.h>
+#include <linux/security.h>
#include <linux/ptrace.h>
#include <linux/device.h>
#include <linux/highmem.h>
#define DEVPORT_MINOR 4
+#if defined(CONFIG_GRKERNSEC) && !defined(CONFIG_GRKERNSEC_NO_RBAC)
+extern const struct file_operations grsec_fops;
+#endif
+
static inline unsigned long size_inside_page(unsigned long start,
unsigned long size)
{
while (cursor < to) {
if (!devmem_is_allowed(pfn)) {
+#ifdef CONFIG_GRKERNSEC_KMEM
+ gr_handle_mem_readwrite(from, to);
+#else
printk(KERN_INFO
"Program %s tried to access /dev/mem between %Lx->%Lx.\n",
current->comm, from, to);
+#endif
return 0;
}
cursor += PAGE_SIZE;
}
return 1;
}
+#elif defined(CONFIG_GRKERNSEC_KMEM)
+static inline int range_is_allowed(unsigned long pfn, unsigned long size)
+{
+ return 0;
+}
#else
static inline int range_is_allowed(unsigned long pfn, unsigned long size)
{
#endif
while (count > 0) {
- unsigned long remaining;
+ unsigned long remaining = 0;
+ char *temp;
sz = size_inside_page(p, count);
if (!ptr)
return -EFAULT;
- remaining = copy_to_user(buf, ptr, sz);
+#ifdef CONFIG_PAX_USERCOPY
+ temp = kmalloc(sz, GFP_KERNEL|GFP_USERCOPY);
+ if (!temp) {
+ unxlate_dev_mem_ptr(p, ptr);
+ return -ENOMEM;
+ }
+ remaining = probe_kernel_read(temp, ptr, sz);
+#else
+ temp = ptr;
+#endif
+
+ if (!remaining)
+ remaining = copy_to_user(buf, temp, sz);
+
+#ifdef CONFIG_PAX_USERCOPY
+ kfree(temp);
+#endif
+
unxlate_dev_mem_ptr(p, ptr);
if (remaining)
return -EFAULT;
size_t count, loff_t *ppos)
{
unsigned long p = *ppos;
- ssize_t low_count, read, sz;
+ ssize_t low_count, read, sz, err = 0;
char *kbuf; /* k-addr because vread() takes vmlist_lock rwlock */
- int err = 0;
read = 0;
if (p < (unsigned long) high_memory) {
}
#endif
while (low_count > 0) {
+ char *temp;
+
sz = size_inside_page(p, low_count);
/*
*/
kbuf = xlate_dev_kmem_ptr((void *)p);
- if (copy_to_user(buf, kbuf, sz))
+#ifdef CONFIG_PAX_USERCOPY
+ temp = kmalloc(sz, GFP_KERNEL|GFP_USERCOPY);
+ if (!temp)
+ return -ENOMEM;
+ err = probe_kernel_read(temp, kbuf, sz);
+#else
+ temp = kbuf;
+#endif
+
+ if (!err)
+ err = copy_to_user(buf, temp, sz);
+
+#ifdef CONFIG_PAX_USERCOPY
+ kfree(temp);
+#endif
+
+ if (err)
return -EFAULT;
buf += sz;
p += sz;
#ifdef CONFIG_PRINTK
[11] = { "kmsg", 0644, &kmsg_fops, NULL },
#endif
+#if defined(CONFIG_GRKERNSEC) && !defined(CONFIG_GRKERNSEC_NO_RBAC)
+ [13] = { "grsec",S_IRUSR | S_IWUGO, &grsec_fops, NULL },
+#endif
};
static int memory_open(struct inode *inode, struct file *filp)
continue;
device_create(mem_class, NULL, MKDEV(MEM_MAJOR, minor),
- NULL, devlist[minor].name);
+ NULL, "%s", devlist[minor].name);
}
return tty_init();
spin_unlock_irq(&rtc_lock);
- if (copy_to_user(buf, contents, tmp - contents))
+ if (tmp - contents > sizeof(contents) || copy_to_user(buf, contents, tmp - contents))
return -EFAULT;
*ppos = i;
if (debug_level >= DEBUG_LEVEL_INFO)
printk("%s(%d):mgslpc_close(%s) entry, count=%d\n",
- __FILE__, __LINE__, info->device_name, port->count);
+ __FILE__, __LINE__, info->device_name, atomic_read(&port->count));
if (tty_port_close_start(port, tty, filp) == 0)
goto cleanup;
cleanup:
if (debug_level >= DEBUG_LEVEL_INFO)
printk("%s(%d):mgslpc_close(%s) exit, count=%d\n", __FILE__, __LINE__,
- tty->driver->name, port->count);
+ tty->driver->name, atomic_read(&port->count));
}
/* Wait until the transmitter is empty.
if (debug_level >= DEBUG_LEVEL_INFO)
printk("%s(%d):mgslpc_open(%s), old ref count = %d\n",
- __FILE__, __LINE__, tty->driver->name, port->count);
+ __FILE__, __LINE__, tty->driver->name, atomic_read(&port->count));
/* If port is closing, signal caller to try again */
if (port->flags & ASYNC_CLOSING){
goto cleanup;
}
spin_lock(&port->lock);
- port->count++;
+ atomic_inc(&port->count);
spin_unlock(&port->lock);
spin_unlock_irqrestore(&info->netlock, flags);
- if (port->count == 1) {
+ if (atomic_read(&port->count) == 1) {
/* 1st open on this device, init hardware */
retval = startup(info, tty);
if (retval < 0)
unsigned short new_crctype;
/* return error if TTY interface open */
- if (info->port.count)
+ if (atomic_read(&info->port.count))
return -EBUSY;
switch (encoding)
/* arbitrate between network and tty opens */
spin_lock_irqsave(&info->netlock, flags);
- if (info->port.count != 0 || info->netcount != 0) {
+ if (atomic_read(&info->port.count) != 0 || info->netcount != 0) {
printk(KERN_WARNING "%s: hdlc_open returning busy\n", dev->name);
spin_unlock_irqrestore(&info->netlock, flags);
return -EBUSY;
printk("%s:hdlcdev_ioctl(%s)\n", __FILE__, dev->name);
/* return error if TTY interface open */
- if (info->port.count)
+ if (atomic_read(&info->port.count))
return -EBUSY;
if (cmd != SIOCWANDEV)
/*
* To allow fractional bits to be tracked, the entropy_count field is
* denominated in units of 1/8th bits.
- *
- * 2*(ENTROPY_SHIFT + log2(poolbits)) must <= 31, or the multiply in
- * credit_entropy_bits() needs to be 64 bits wide.
*/
#define ENTROPY_SHIFT 3
#define ENTROPY_BITS(r) ((r)->entropy_count >> ENTROPY_SHIFT)
};
static void push_to_pool(struct work_struct *work);
-static __u32 input_pool_data[INPUT_POOL_WORDS];
-static __u32 blocking_pool_data[OUTPUT_POOL_WORDS];
-static __u32 nonblocking_pool_data[OUTPUT_POOL_WORDS];
+static __u32 input_pool_data[INPUT_POOL_WORDS] __latent_entropy;
+static __u32 blocking_pool_data[OUTPUT_POOL_WORDS] __latent_entropy;
+static __u32 nonblocking_pool_data[OUTPUT_POOL_WORDS] __latent_entropy;
static struct entropy_store input_pool = {
.poolinfo = &poolinfo_table[0],
/* The +2 corresponds to the /4 in the denominator */
do {
- unsigned int anfrac = min(pnfrac, pool_size/2);
+ u64 anfrac = min(pnfrac, pool_size/2);
unsigned int add =
((pool_size - entropy_count)*anfrac*3) >> s;
extract_buf(r, tmp);
i = min_t(int, nbytes, EXTRACT_SIZE);
- if (copy_to_user(buf, tmp, i)) {
+ if (i > sizeof(tmp) || copy_to_user(buf, tmp, i)) {
ret = -EFAULT;
break;
}
static int proc_do_uuid(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp, loff_t *ppos)
{
- struct ctl_table fake_table;
+ ctl_table_no_const fake_table;
unsigned char buf[64], tmp_uuid[16], *uuid;
uuid = table->data;
static int proc_do_entropy(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp, loff_t *ppos)
{
- struct ctl_table fake_table;
+ ctl_table_no_const fake_table;
int entropy_count;
entropy_count = *(int *)table->data >> ENTROPY_SHIFT;
#include <asm/uaccess.h>
#include <asm/io.h>
+#include <asm/local.h>
#include <linux/sonypi.h>
spinlock_t fifo_lock;
wait_queue_head_t fifo_proc_list;
struct fasync_struct *fifo_async;
- int open_count;
+ local_t open_count;
int model;
struct input_dev *input_jog_dev;
struct input_dev *input_key_dev;
static int sonypi_misc_release(struct inode *inode, struct file *file)
{
mutex_lock(&sonypi_device.lock);
- sonypi_device.open_count--;
+ local_dec(&sonypi_device.open_count);
mutex_unlock(&sonypi_device.lock);
return 0;
}
{
mutex_lock(&sonypi_device.lock);
/* Flush input queue on first open */
- if (!sonypi_device.open_count)
+ if (!local_read(&sonypi_device.open_count))
kfifo_reset(&sonypi_device.fifo);
- sonypi_device.open_count++;
+ local_inc(&sonypi_device.open_count);
mutex_unlock(&sonypi_device.lock);
return 0;
virt = acpi_os_map_iomem(start, len);
if (!virt) {
kfree(log->bios_event_log);
+ log->bios_event_log = NULL;
printk("%s: ERROR - Unable to map memory\n", __func__);
return -EIO;
}
- memcpy_fromio(log->bios_event_log, virt, len);
+ memcpy_fromio(log->bios_event_log, (const char __force_kernel *)virt, len);
acpi_os_unmap_iomem(virt, len);
return 0;
event = addr;
if ((event->event_type == 0 && event->event_size == 0) ||
- ((addr + sizeof(struct tcpa_event) + event->event_size) >= limit))
+ (event->event_size >= limit - addr - sizeof(struct tcpa_event)))
return NULL;
return addr;
return NULL;
if ((event->event_type == 0 && event->event_size == 0) ||
- ((v + sizeof(struct tcpa_event) + event->event_size) >= limit))
+ (event->event_size >= limit - v - sizeof(struct tcpa_event)))
return NULL;
(*pos)++;
int i;
for (i = 0; i < sizeof(struct tcpa_event) + event->event_size; i++)
- seq_putc(m, data[i]);
+ if (!seq_putc(m, data[i]))
+ return -EFAULT;
return 0;
}
if (to_user) {
ssize_t ret;
- ret = copy_to_user(out_buf, buf->buf + buf->offset, out_count);
+ ret = copy_to_user((char __force_user *)out_buf, buf->buf + buf->offset, out_count);
if (ret)
return -EFAULT;
} else {
if (!port_has_data(port) && !port->host_connected)
return 0;
- return fill_readbuf(port, ubuf, count, true);
+ return fill_readbuf(port, (char __force_kernel *)ubuf, count, true);
}
static int wait_port_writable(struct port *port, bool nonblock)
struct clk *clk;
struct clk_init_data init;
struct clk_composite *composite;
- struct clk_ops *clk_composite_ops;
+ clk_ops_no_const *clk_composite_ops;
composite = kzalloc(sizeof(*composite), GFP_KERNEL);
if (!composite) {
#include <linux/mfd/syscon.h>
#include <linux/of.h>
#include <linux/regmap.h>
+#include <asm/pgtable.h>
#include "clk.h"
return 0;
}
-static struct clk_ops gateclk_ops = {
+static clk_ops_no_const gateclk_ops __read_only = {
.prepare = socfpga_clk_prepare,
.recalc_rate = socfpga_clk_recalc_rate,
.get_parent = socfpga_clk_get_parent,
socfpga_clk->hw.reg = clk_mgr_base_addr + clk_gate[0];
socfpga_clk->hw.bit_idx = clk_gate[1];
- gateclk_ops.enable = clk_gate_ops.enable;
- gateclk_ops.disable = clk_gate_ops.disable;
+ pax_open_kernel();
+ *(void **)&gateclk_ops.enable = clk_gate_ops.enable;
+ *(void **)&gateclk_ops.disable = clk_gate_ops.disable;
+ pax_close_kernel();
}
rc = of_property_read_u32(node, "fixed-divider", &fixed_div);
#include <linux/io.h>
#include <linux/of.h>
#include <linux/of_address.h>
+#include <asm/pgtable.h>
#include "clk.h"
CLK_MGR_PLL_CLK_SRC_MASK;
}
-static struct clk_ops clk_pll_ops = {
+static clk_ops_no_const clk_pll_ops __read_only = {
.recalc_rate = clk_pll_recalc_rate,
.get_parent = clk_pll_get_parent,
};
pll_clk->hw.hw.init = &init;
pll_clk->hw.bit_idx = SOCFPGA_PLL_EXT_ENA;
- clk_pll_ops.enable = clk_gate_ops.enable;
- clk_pll_ops.disable = clk_gate_ops.disable;
+ pax_open_kernel();
+ *(void **)&clk_pll_ops.enable = clk_gate_ops.enable;
+ *(void **)&clk_pll_ops.disable = clk_gate_ops.disable;
+ pax_close_kernel();
clk = clk_register(NULL, &pll_clk->hw.hw);
if (WARN_ON(IS_ERR(clk))) {
data->acpi_data = per_cpu_ptr(acpi_perf_data, cpu);
per_cpu(acfreq_data, cpu) = data;
- if (cpu_has(c, X86_FEATURE_CONSTANT_TSC))
- acpi_cpufreq_driver.flags |= CPUFREQ_CONST_LOOPS;
+ if (cpu_has(c, X86_FEATURE_CONSTANT_TSC)) {
+ pax_open_kernel();
+ *(u8 *)&acpi_cpufreq_driver.flags |= CPUFREQ_CONST_LOOPS;
+ pax_close_kernel();
+ }
result = acpi_processor_register_performance(data->acpi_data, cpu);
if (result)
policy->cur = acpi_cpufreq_guess_freq(data, policy->cpu);
break;
case ACPI_ADR_SPACE_FIXED_HARDWARE:
- acpi_cpufreq_driver.get = get_cur_freq_on_cpu;
+ pax_open_kernel();
+ *(void **)&acpi_cpufreq_driver.get = get_cur_freq_on_cpu;
+ pax_close_kernel();
break;
default:
break;
if (!msrs)
return;
- acpi_cpufreq_driver.boost_supported = true;
- acpi_cpufreq_driver.boost_enabled = boost_state(0);
+ pax_open_kernel();
+ *(bool *)&acpi_cpufreq_driver.boost_supported = true;
+ *(bool *)&acpi_cpufreq_driver.boost_enabled = boost_state(0);
+ pax_close_kernel();
cpu_notifier_register_begin();
if (!IS_ERR(cpu_reg))
regulator_put(cpu_reg);
- dt_cpufreq_driver.driver_data = dev_get_platdata(&pdev->dev);
+ pax_open_kernel();
+ *(void **)&dt_cpufreq_driver.driver_data = dev_get_platdata(&pdev->dev);
+ pax_close_kernel();
ret = cpufreq_register_driver(&dt_cpufreq_driver);
if (ret)
}
mutex_lock(&cpufreq_governor_mutex);
- list_del(&governor->governor_list);
+ pax_list_del(&governor->governor_list);
mutex_unlock(&cpufreq_governor_mutex);
return;
}
return NOTIFY_OK;
}
-static struct notifier_block __refdata cpufreq_cpu_notifier = {
+static struct notifier_block cpufreq_cpu_notifier = {
.notifier_call = cpufreq_cpu_callback,
};
return 0;
write_lock_irqsave(&cpufreq_driver_lock, flags);
- cpufreq_driver->boost_enabled = state;
+ pax_open_kernel();
+ *(bool *)&cpufreq_driver->boost_enabled = state;
+ pax_close_kernel();
write_unlock_irqrestore(&cpufreq_driver_lock, flags);
ret = cpufreq_driver->set_boost(state);
if (ret) {
write_lock_irqsave(&cpufreq_driver_lock, flags);
- cpufreq_driver->boost_enabled = !state;
+ pax_open_kernel();
+ *(bool *)&cpufreq_driver->boost_enabled = !state;
+ pax_close_kernel();
write_unlock_irqrestore(&cpufreq_driver_lock, flags);
pr_err("%s: Cannot %s BOOST\n",
pr_debug("trying to register driver %s\n", driver_data->name);
- if (driver_data->setpolicy)
- driver_data->flags |= CPUFREQ_CONST_LOOPS;
+ if (driver_data->setpolicy) {
+ pax_open_kernel();
+ *(u8 *)&driver_data->flags |= CPUFREQ_CONST_LOOPS;
+ pax_close_kernel();
+ }
write_lock_irqsave(&cpufreq_driver_lock, flags);
if (cpufreq_driver) {
* Check if driver provides function to enable boost -
* if not, use cpufreq_boost_set_sw as default
*/
- if (!cpufreq_driver->set_boost)
- cpufreq_driver->set_boost = cpufreq_boost_set_sw;
+ if (!cpufreq_driver->set_boost) {
+ pax_open_kernel();
+ *(void **)&cpufreq_driver->set_boost = cpufreq_boost_set_sw;
+ pax_close_kernel();
+ }
ret = cpufreq_sysfs_create_file(&boost.attr);
if (ret) {
struct dbs_data *dbs_data;
struct od_cpu_dbs_info_s *od_dbs_info = NULL;
struct cs_cpu_dbs_info_s *cs_dbs_info = NULL;
- struct od_ops *od_ops = NULL;
+ const struct od_ops *od_ops = NULL;
struct od_dbs_tuners *od_tuners = NULL;
struct cs_dbs_tuners *cs_tuners = NULL;
struct cpu_dbs_common_info *cpu_cdbs;
if ((cdata->governor == GOV_CONSERVATIVE) &&
(!policy->governor->initialized)) {
- struct cs_ops *cs_ops = dbs_data->cdata->gov_ops;
+ const struct cs_ops *cs_ops = dbs_data->cdata->gov_ops;
cpufreq_register_notifier(cs_ops->notifier_block,
CPUFREQ_TRANSITION_NOTIFIER);
if ((dbs_data->cdata->governor == GOV_CONSERVATIVE) &&
(policy->governor->initialized == 1)) {
- struct cs_ops *cs_ops = dbs_data->cdata->gov_ops;
+ const struct cs_ops *cs_ops = dbs_data->cdata->gov_ops;
cpufreq_unregister_notifier(cs_ops->notifier_block,
CPUFREQ_TRANSITION_NOTIFIER);
void (*exit)(struct dbs_data *dbs_data);
/* Governor specific ops, see below */
- void *gov_ops;
+ const void *gov_ops;
};
/* Governor Per policy data */
unsigned int (*powersave_bias_target)(struct cpufreq_policy *policy,
unsigned int freq_next, unsigned int relation);
void (*freq_increase)(struct cpufreq_policy *policy, unsigned int freq);
-};
+} __no_const;
struct cs_ops {
struct notifier_block *notifier_block;
define_get_cpu_dbs_routines(od_cpu_dbs_info);
-static struct od_ops od_ops = {
+static struct od_ops od_ops __read_only = {
.powersave_bias_init_cpu = ondemand_powersave_bias_init_cpu,
.powersave_bias_target = generic_powersave_bias_target,
.freq_increase = dbs_freq_increase,
(struct cpufreq_policy *, unsigned int, unsigned int),
unsigned int powersave_bias)
{
- od_ops.powersave_bias_target = f;
+ pax_open_kernel();
+ *(void **)&od_ops.powersave_bias_target = f;
+ pax_close_kernel();
od_set_powersave_bias(powersave_bias);
}
EXPORT_SYMBOL_GPL(od_register_powersave_bias_handler);
void od_unregister_powersave_bias_handler(void)
{
- od_ops.powersave_bias_target = generic_powersave_bias_target;
+ pax_open_kernel();
+ *(void **)&od_ops.powersave_bias_target = generic_powersave_bias_target;
+ pax_close_kernel();
od_set_powersave_bias(0);
}
EXPORT_SYMBOL_GPL(od_unregister_powersave_bias_handler);
struct cpu_defaults {
struct pstate_adjust_policy pid_policy;
struct pstate_funcs funcs;
-};
+} __do_const;
static struct pstate_adjust_policy pid_params;
-static struct pstate_funcs pstate_funcs;
+static struct pstate_funcs *pstate_funcs;
static int hwp_active;
struct perf_limits {
cpu->pstate.current_pstate = pstate;
- pstate_funcs.set(cpu, pstate);
+ pstate_funcs->set(cpu, pstate);
}
static void intel_pstate_get_cpu_pstates(struct cpudata *cpu)
{
- cpu->pstate.min_pstate = pstate_funcs.get_min();
- cpu->pstate.max_pstate = pstate_funcs.get_max();
- cpu->pstate.turbo_pstate = pstate_funcs.get_turbo();
- cpu->pstate.scaling = pstate_funcs.get_scaling();
+ cpu->pstate.min_pstate = pstate_funcs->get_min();
+ cpu->pstate.max_pstate = pstate_funcs->get_max();
+ cpu->pstate.turbo_pstate = pstate_funcs->get_turbo();
+ cpu->pstate.scaling = pstate_funcs->get_scaling();
- if (pstate_funcs.get_vid)
- pstate_funcs.get_vid(cpu);
+ if (pstate_funcs->get_vid)
+ pstate_funcs->get_vid(cpu);
intel_pstate_set_pstate(cpu, cpu->pstate.min_pstate);
}
rdmsrl(MSR_IA32_APERF, aperf);
rdmsrl(MSR_IA32_MPERF, mperf);
- if (!pstate_funcs.get_max() ||
- !pstate_funcs.get_min() ||
- !pstate_funcs.get_turbo())
+ if (!pstate_funcs->get_max() ||
+ !pstate_funcs->get_min() ||
+ !pstate_funcs->get_turbo())
return -ENODEV;
rdmsrl(MSR_IA32_APERF, tmp);
return 0;
}
-static void copy_pid_params(struct pstate_adjust_policy *policy)
+static void copy_pid_params(const struct pstate_adjust_policy *policy)
{
pid_params.sample_rate_ms = policy->sample_rate_ms;
pid_params.p_gain_pct = policy->p_gain_pct;
static void copy_cpu_funcs(struct pstate_funcs *funcs)
{
- pstate_funcs.get_max = funcs->get_max;
- pstate_funcs.get_min = funcs->get_min;
- pstate_funcs.get_turbo = funcs->get_turbo;
- pstate_funcs.get_scaling = funcs->get_scaling;
- pstate_funcs.set = funcs->set;
- pstate_funcs.get_vid = funcs->get_vid;
+ pstate_funcs = funcs;
}
#if IS_ENABLED(CONFIG_ACPI)
case 0x0F: /* Core Duo */
case 0x16: /* Celeron Core */
case 0x1C: /* Atom */
- p4clockmod_driver.flags |= CPUFREQ_CONST_LOOPS;
+ pax_open_kernel();
+ *(u8 *)&p4clockmod_driver.flags |= CPUFREQ_CONST_LOOPS;
+ pax_close_kernel();
return speedstep_get_frequency(SPEEDSTEP_CPU_PCORE);
case 0x0D: /* Pentium M (Dothan) */
- p4clockmod_driver.flags |= CPUFREQ_CONST_LOOPS;
+ pax_open_kernel();
+ *(u8 *)&p4clockmod_driver.flags |= CPUFREQ_CONST_LOOPS;
+ pax_close_kernel();
/* fall through */
case 0x09: /* Pentium M (Banias) */
return speedstep_get_frequency(SPEEDSTEP_CPU_PM);
/* on P-4s, the TSC runs with constant frequency independent whether
* throttling is active or not. */
- p4clockmod_driver.flags |= CPUFREQ_CONST_LOOPS;
+ pax_open_kernel();
+ *(u8 *)&p4clockmod_driver.flags |= CPUFREQ_CONST_LOOPS;
+ pax_close_kernel();
if (speedstep_detect_processor() == SPEEDSTEP_CPU_P4M) {
printk(KERN_WARNING PFX "Warning: Pentium 4-M detected. "
#include <asm/head.h>
#include <asm/timer.h>
-static struct cpufreq_driver *cpufreq_us3_driver;
-
struct us3_freq_percpu_info {
struct cpufreq_frequency_table table[4];
};
/* Indexed by cpu number. */
-static struct us3_freq_percpu_info *us3_freq_table;
+static struct us3_freq_percpu_info us3_freq_table[NR_CPUS];
/* UltraSPARC-III has three dividers: 1, 2, and 32. These are controlled
* in the Safari config register.
static int us3_freq_cpu_exit(struct cpufreq_policy *policy)
{
- if (cpufreq_us3_driver)
- us3_freq_target(policy, 0);
+ us3_freq_target(policy, 0);
return 0;
}
+static int __init us3_freq_init(void);
+static void __exit us3_freq_exit(void);
+
+static struct cpufreq_driver cpufreq_us3_driver = {
+ .init = us3_freq_cpu_init,
+ .verify = cpufreq_generic_frequency_table_verify,
+ .target_index = us3_freq_target,
+ .get = us3_freq_get,
+ .exit = us3_freq_cpu_exit,
+ .name = "UltraSPARC-III",
+
+};
+
static int __init us3_freq_init(void)
{
unsigned long manuf, impl, ver;
- int ret;
if (tlb_type != cheetah && tlb_type != cheetah_plus)
return -ENODEV;
(impl == CHEETAH_IMPL ||
impl == CHEETAH_PLUS_IMPL ||
impl == JAGUAR_IMPL ||
- impl == PANTHER_IMPL)) {
- struct cpufreq_driver *driver;
-
- ret = -ENOMEM;
- driver = kzalloc(sizeof(*driver), GFP_KERNEL);
- if (!driver)
- goto err_out;
-
- us3_freq_table = kzalloc((NR_CPUS * sizeof(*us3_freq_table)),
- GFP_KERNEL);
- if (!us3_freq_table)
- goto err_out;
-
- driver->init = us3_freq_cpu_init;
- driver->verify = cpufreq_generic_frequency_table_verify;
- driver->target_index = us3_freq_target;
- driver->get = us3_freq_get;
- driver->exit = us3_freq_cpu_exit;
- strcpy(driver->name, "UltraSPARC-III");
-
- cpufreq_us3_driver = driver;
- ret = cpufreq_register_driver(driver);
- if (ret)
- goto err_out;
-
- return 0;
-
-err_out:
- if (driver) {
- kfree(driver);
- cpufreq_us3_driver = NULL;
- }
- kfree(us3_freq_table);
- us3_freq_table = NULL;
- return ret;
- }
+ impl == PANTHER_IMPL))
+ return cpufreq_register_driver(&cpufreq_us3_driver);
return -ENODEV;
}
static void __exit us3_freq_exit(void)
{
- if (cpufreq_us3_driver) {
- cpufreq_unregister_driver(cpufreq_us3_driver);
- kfree(cpufreq_us3_driver);
- cpufreq_us3_driver = NULL;
- kfree(us3_freq_table);
- us3_freq_table = NULL;
- }
+ cpufreq_unregister_driver(&cpufreq_us3_driver);
}
MODULE_AUTHOR("David S. Miller <davem@redhat.com>");
!cpu_has(cpu, X86_FEATURE_EST))
return -ENODEV;
- if (cpu_has(cpu, X86_FEATURE_CONSTANT_TSC))
- centrino_driver.flags |= CPUFREQ_CONST_LOOPS;
+ if (cpu_has(cpu, X86_FEATURE_CONSTANT_TSC)) {
+ pax_open_kernel();
+ *(u8 *)¢rino_driver.flags |= CPUFREQ_CONST_LOOPS;
+ pax_close_kernel();
+ }
if (policy->cpu != 0)
return -ENODEV;
static void poll_idle_init(struct cpuidle_driver *drv)
{
- struct cpuidle_state *state = &drv->states[0];
+ cpuidle_state_no_const *state = &drv->states[0];
snprintf(state->name, CPUIDLE_NAME_LEN, "POLL");
snprintf(state->desc, CPUIDLE_DESC_LEN, "CPUIDLE CORE POLL IDLE");
mutex_lock(&cpuidle_lock);
if (__cpuidle_find_governor(gov->name) == NULL) {
ret = 0;
- list_add_tail(&gov->governor_list, &cpuidle_governors);
+ pax_list_add_tail((struct list_head *)&gov->governor_list, &cpuidle_governors);
if (!cpuidle_curr_governor ||
cpuidle_curr_governor->rating < gov->rating)
cpuidle_switch_governor(gov);
NULL
};
-static struct attribute_group cpuidle_attr_group = {
+static attribute_group_no_const cpuidle_attr_group = {
.attrs = cpuidle_default_attrs,
.name = "cpuidle",
};
MODULE_PARM_DESC(hifn_pll_ref,
"PLL reference clock (pci[freq] or ext[freq], default ext)");
-static atomic_t hifn_dev_number;
+static atomic_unchecked_t hifn_dev_number;
#define ACRYPTO_OP_DECRYPT 0
#define ACRYPTO_OP_ENCRYPT 1
goto err_out_disable_pci_device;
snprintf(name, sizeof(name), "hifn%d",
- atomic_inc_return(&hifn_dev_number)-1);
+ atomic_inc_return_unchecked(&hifn_dev_number)-1);
err = pci_request_regions(pdev, name);
if (err)
goto err_out;
}
- list_add(&governor->node, &devfreq_governor_list);
+ pax_list_add((struct list_head *)&governor->node, &devfreq_governor_list);
list_for_each_entry(devfreq, &devfreq_list, node) {
int ret = 0;
}
}
- list_del(&governor->node);
+ pax_list_del((struct list_head *)&governor->node);
err_out:
mutex_unlock(&devfreq_list_lock);
schan->slave_id = -EINVAL;
}
- schan->desc = kcalloc(NR_DESCS_PER_CHANNEL,
- sdev->desc_size, GFP_KERNEL);
+ schan->desc = kcalloc(sdev->desc_size,
+ NR_DESCS_PER_CHANNEL, GFP_KERNEL);
if (!schan->desc) {
ret = -ENOMEM;
goto edescalloc;
return ret;
}
-static struct notifier_block sh_dmae_nmi_notifier __read_mostly = {
+static struct notifier_block sh_dmae_nmi_notifier = {
.notifier_call = sh_dmae_nmi_handler,
/* Run before NMI debug handler and KGDB */
*/
int edac_device_alloc_index(void)
{
- static atomic_t device_indexes = ATOMIC_INIT(0);
+ static atomic_unchecked_t device_indexes = ATOMIC_INIT(0);
- return atomic_inc_return(&device_indexes) - 1;
+ return atomic_inc_return_unchecked(&device_indexes) - 1;
}
EXPORT_SYMBOL_GPL(edac_device_alloc_index);
struct dev_ch_attribute {
struct device_attribute attr;
int channel;
-};
+} __do_const;
#define DEVICE_CHANNEL(_name, _mode, _show, _store, _var) \
struct dev_ch_attribute dev_attr_legacy_##_name = \
}
if (mci->set_sdram_scrub_rate || mci->get_sdram_scrub_rate) {
+ pax_open_kernel();
if (mci->get_sdram_scrub_rate) {
- dev_attr_sdram_scrub_rate.attr.mode |= S_IRUGO;
- dev_attr_sdram_scrub_rate.show = &mci_sdram_scrub_rate_show;
+ *(umode_t *)&dev_attr_sdram_scrub_rate.attr.mode |= S_IRUGO;
+ *(void **)&dev_attr_sdram_scrub_rate.show = &mci_sdram_scrub_rate_show;
}
if (mci->set_sdram_scrub_rate) {
- dev_attr_sdram_scrub_rate.attr.mode |= S_IWUSR;
- dev_attr_sdram_scrub_rate.store = &mci_sdram_scrub_rate_store;
+ *(umode_t *)&dev_attr_sdram_scrub_rate.attr.mode |= S_IWUSR;
+ *(void **)&dev_attr_sdram_scrub_rate.store = &mci_sdram_scrub_rate_store;
}
+ pax_close_kernel();
err = device_create_file(&mci->dev,
&dev_attr_sdram_scrub_rate);
if (err) {
static DEFINE_MUTEX(edac_pci_ctls_mutex);
static LIST_HEAD(edac_pci_list);
-static atomic_t pci_indexes = ATOMIC_INIT(0);
+static atomic_unchecked_t pci_indexes = ATOMIC_INIT(0);
/*
* edac_pci_alloc_ctl_info
*/
int edac_pci_alloc_index(void)
{
- return atomic_inc_return(&pci_indexes) - 1;
+ return atomic_inc_return_unchecked(&pci_indexes) - 1;
}
EXPORT_SYMBOL_GPL(edac_pci_alloc_index);
static int edac_pci_log_npe = 1; /* log PCI non-parity error errors */
static int edac_pci_poll_msec = 1000; /* one second workq period */
-static atomic_t pci_parity_count = ATOMIC_INIT(0);
-static atomic_t pci_nonparity_count = ATOMIC_INIT(0);
+static atomic_unchecked_t pci_parity_count = ATOMIC_INIT(0);
+static atomic_unchecked_t pci_nonparity_count = ATOMIC_INIT(0);
static struct kobject *edac_pci_top_main_kobj;
static atomic_t edac_pci_sysfs_refcount = ATOMIC_INIT(0);
void *value;
ssize_t(*show) (void *, char *);
ssize_t(*store) (void *, const char *, size_t);
-};
+} __do_const;
/* Set of show/store abstract level functions for PCI Parity object */
static ssize_t edac_pci_dev_show(struct kobject *kobj, struct attribute *attr,
edac_printk(KERN_CRIT, EDAC_PCI,
"Signaled System Error on %s\n",
pci_name(dev));
- atomic_inc(&pci_nonparity_count);
+ atomic_inc_unchecked(&pci_nonparity_count);
}
if (status & (PCI_STATUS_PARITY)) {
"Master Data Parity Error on %s\n",
pci_name(dev));
- atomic_inc(&pci_parity_count);
+ atomic_inc_unchecked(&pci_parity_count);
}
if (status & (PCI_STATUS_DETECTED_PARITY)) {
"Detected Parity Error on %s\n",
pci_name(dev));
- atomic_inc(&pci_parity_count);
+ atomic_inc_unchecked(&pci_parity_count);
}
}
edac_printk(KERN_CRIT, EDAC_PCI, "Bridge "
"Signaled System Error on %s\n",
pci_name(dev));
- atomic_inc(&pci_nonparity_count);
+ atomic_inc_unchecked(&pci_nonparity_count);
}
if (status & (PCI_STATUS_PARITY)) {
"Master Data Parity Error on "
"%s\n", pci_name(dev));
- atomic_inc(&pci_parity_count);
+ atomic_inc_unchecked(&pci_parity_count);
}
if (status & (PCI_STATUS_DETECTED_PARITY)) {
"Detected Parity Error on %s\n",
pci_name(dev));
- atomic_inc(&pci_parity_count);
+ atomic_inc_unchecked(&pci_parity_count);
}
}
}
if (!check_pci_errors)
return;
- before_count = atomic_read(&pci_parity_count);
+ before_count = atomic_read_unchecked(&pci_parity_count);
/* scan all PCI devices looking for a Parity Error on devices and
* bridges.
/* Only if operator has selected panic on PCI Error */
if (edac_pci_get_panic_on_pe()) {
/* If the count is different 'after' from 'before' */
- if (before_count != atomic_read(&pci_parity_count))
+ if (before_count != atomic_read_unchecked(&pci_parity_count))
panic("EDAC: PCI Parity Error");
}
}
bool (*mc0_mce)(u16, u8);
bool (*mc1_mce)(u16, u8);
bool (*mc2_mce)(u16, u8);
-};
+} __no_const;
void amd_report_gart_errors(bool);
void amd_register_ecc_decoder(void (*f)(int, struct mce *));
const struct fw_card_driver *driver,
struct device *device)
{
- static atomic_t index = ATOMIC_INIT(-1);
+ static atomic_unchecked_t index = ATOMIC_INIT(-1);
- card->index = atomic_inc_return(&index);
+ card->index = atomic_inc_return_unchecked(&index);
card->driver = driver;
card->device = device;
card->current_tlabel = 0;
void fw_core_remove_card(struct fw_card *card)
{
- struct fw_card_driver dummy_driver = dummy_driver_template;
+ fw_card_driver_no_const dummy_driver = dummy_driver_template;
card->driver->update_phy_reg(card, 4,
PHY_LINK_ACTIVE | PHY_CONTENDER, 0);
struct config_rom_attribute {
struct device_attribute attr;
u32 key;
-};
+} __do_const;
static ssize_t show_immediate(struct device *dev,
struct device_attribute *dattr, char *buf)
#include <linux/timer.h>
#include <linux/types.h>
#include <linux/workqueue.h>
+#include <linux/sched.h>
#include <asm/byteorder.h>
int (*stop_iso)(struct fw_iso_context *ctx);
};
+typedef struct fw_card_driver __no_const fw_card_driver_no_const;
void fw_card_initialize(struct fw_card *card,
const struct fw_card_driver *driver, struct device *device);
be32_to_cpu(ohci->next_header));
}
+#ifndef CONFIG_GRKERNSEC
if (param_remote_dma) {
reg_write(ohci, OHCI1394_PhyReqFilterHiSet, ~0);
reg_write(ohci, OHCI1394_PhyReqFilterLoSet, ~0);
}
+#endif
spin_unlock_irq(&ohci->lock);
unsigned long flags;
int n, ret = 0;
+#ifndef CONFIG_GRKERNSEC
if (param_remote_dma)
return 0;
+#endif
/*
* FIXME: Make sure this bitmask is cleared when we clear the busReset
struct dmi_device_attribute{
struct device_attribute dev_attr;
int field;
-};
+} __do_const;
#define to_dmi_dev_attr(_dev_attr) \
container_of(_dev_attr, struct dmi_device_attribute, dev_attr)
if (buf == NULL)
return -1;
- dmi_table(buf, dmi_len, dmi_num, decode, private_data);
+ dmi_table((char __force_kernel *)buf, dmi_len, dmi_num, decode, private_data);
dmi_unmap(buf);
return 0;
*/
u64 cper_next_record_id(void)
{
- static atomic64_t seq;
+ static atomic64_unchecked_t seq;
- if (!atomic64_read(&seq))
- atomic64_set(&seq, ((u64)get_seconds()) << 32);
+ if (!atomic64_read_unchecked(&seq))
+ atomic64_set_unchecked(&seq, ((u64)get_seconds()) << 32);
- return atomic64_inc_return(&seq);
+ return atomic64_inc_return_unchecked(&seq);
}
EXPORT_SYMBOL_GPL(cper_next_record_id);
};
static struct efivars generic_efivars;
-static struct efivar_operations generic_ops;
+static efivar_operations_no_const generic_ops __read_only;
static int generic_ops_register(void)
{
- generic_ops.get_variable = efi.get_variable;
- generic_ops.set_variable = efi.set_variable;
- generic_ops.get_next_variable = efi.get_next_variable;
- generic_ops.query_variable_store = efi_query_variable_store;
+ pax_open_kernel();
+ *(void **)&generic_ops.get_variable = efi.get_variable;
+ *(void **)&generic_ops.set_variable = efi.set_variable;
+ *(void **)&generic_ops.get_next_variable = efi.get_next_variable;
+ *(void **)&generic_ops.query_variable_store = efi_query_variable_store;
+ pax_close_kernel();
return efivars_register(&generic_efivars, &generic_ops, efi_kobj);
}
static int
create_efivars_bin_attributes(void)
{
- struct bin_attribute *attr;
+ bin_attribute_no_const *attr;
int error;
/* new_var */
if (!found_memconsole())
return -ENODEV;
- memconsole_bin_attr.size = memconsole_length;
+ pax_open_kernel();
+ *(size_t *)&memconsole_bin_attr.size = memconsole_length;
+ pax_close_kernel();
+
return sysfs_create_bin_file(firmware_kobj, &memconsole_bin_attr);
}
struct em_gio_priv *p;
struct resource *io[2], *irq[2];
struct gpio_chip *gpio_chip;
- struct irq_chip *irq_chip;
+ irq_chip_no_const *irq_chip;
const char *name = dev_name(&pdev->dev);
int ret;
* this option allows driver caching written output values
*/
bool use_outlvl_cache;
-};
+} __do_const;
static struct {
spinlock_t lock;
const struct omap_gpio_platform_data *pdata;
struct resource *res;
struct gpio_bank *bank;
- struct irq_chip *irqc;
+ irq_chip_no_const *irqc;
int ret;
match = of_match_device(of_match_ptr(omap_gpio_match), dev);
struct gpio_rcar_priv *p;
struct resource *io, *irq;
struct gpio_chip *gpio_chip;
- struct irq_chip *irq_chip;
+ irq_chip_no_const *irq_chip;
struct device *dev = &pdev->dev;
const char *name = dev_name(dev);
int ret;
printk(KERN_ERR "spurious GIU interrupt: %04x(%04x),%04x(%04x)\n",
maskl, pendl, maskh, pendh);
- atomic_inc(&irq_err_count);
+ atomic_inc_unchecked(&irq_err_count);
return -EINVAL;
}
}
if (gpiochip->irqchip) {
- gpiochip->irqchip->irq_request_resources = NULL;
- gpiochip->irqchip->irq_release_resources = NULL;
+ pax_open_kernel();
+ *(void **)&gpiochip->irqchip->irq_request_resources = NULL;
+ *(void **)&gpiochip->irqchip->irq_release_resources = NULL;
+ pax_close_kernel();
gpiochip->irqchip = NULL;
}
}
gpiochip->irqchip = NULL;
return -EINVAL;
}
- irqchip->irq_request_resources = gpiochip_irq_reqres;
- irqchip->irq_release_resources = gpiochip_irq_relres;
+
+ pax_open_kernel();
+ *(void **)&irqchip->irq_request_resources = gpiochip_irq_reqres;
+ *(void **)&irqchip->irq_release_resources = gpiochip_irq_relres;
+ pax_close_kernel();
/*
* Prepare the mapping since the irqchip shall be orthogonal to
goto done;
}
- if (copy_to_user(&enum_ptr[copied].name,
+ if (copy_to_user(enum_ptr[copied].name,
&prop_enum->name, DRM_PROP_NAME_LEN)) {
ret = -EFAULT;
goto done;
drm_device_set_unplugged(dev);
- if (dev->open_count == 0) {
+ if (local_read(&dev->open_count) == 0) {
drm_put_dev(dev);
}
mutex_unlock(&drm_global_mutex);
return PTR_ERR(minor);
dev = minor->dev;
- if (!dev->open_count++)
+ if (local_inc_return(&dev->open_count) == 1)
need_setup = 1;
/* share address_space across all char-devs of a single device */
return 0;
err_undo:
- dev->open_count--;
+ local_dec(&dev->open_count);
drm_minor_release(minor);
return retcode;
}
mutex_lock(&drm_global_mutex);
- DRM_DEBUG("open_count = %d\n", dev->open_count);
+ DRM_DEBUG("open_count = %ld\n", local_read(&dev->open_count));
mutex_lock(&dev->struct_mutex);
list_del(&file_priv->lhead);
* Begin inline drm_release
*/
- DRM_DEBUG("pid = %d, device = 0x%lx, open_count = %d\n",
+ DRM_DEBUG("pid = %d, device = 0x%lx, open_count = %ld\n",
task_pid_nr(current),
(long)old_encode_dev(file_priv->minor->kdev->devt),
- dev->open_count);
+ local_read(&dev->open_count));
/* Release any auth tokens that might point to this file_priv,
(do that under the drm_global_mutex) */
* End inline drm_release
*/
- if (!--dev->open_count) {
+ if (local_dec_and_test(&dev->open_count)) {
retcode = drm_lastclose(dev);
if (drm_device_is_unplugged(dev))
drm_put_dev(dev);
struct drm_global_item {
struct mutex mutex;
void *object;
- int refcount;
+ atomic_t refcount;
};
static struct drm_global_item glob[DRM_GLOBAL_NUM];
struct drm_global_item *item = &glob[i];
mutex_init(&item->mutex);
item->object = NULL;
- item->refcount = 0;
+ atomic_set(&item->refcount, 0);
}
}
for (i = 0; i < DRM_GLOBAL_NUM; ++i) {
struct drm_global_item *item = &glob[i];
BUG_ON(item->object != NULL);
- BUG_ON(item->refcount != 0);
+ BUG_ON(atomic_read(&item->refcount) != 0);
}
}
struct drm_global_item *item = &glob[ref->global_type];
mutex_lock(&item->mutex);
- if (item->refcount == 0) {
+ if (atomic_read(&item->refcount) == 0) {
item->object = kzalloc(ref->size, GFP_KERNEL);
if (unlikely(item->object == NULL)) {
ret = -ENOMEM;
goto out_err;
}
- ++item->refcount;
+ atomic_inc(&item->refcount);
ref->object = item->object;
mutex_unlock(&item->mutex);
return 0;
struct drm_global_item *item = &glob[ref->global_type];
mutex_lock(&item->mutex);
- BUG_ON(item->refcount == 0);
+ BUG_ON(atomic_read(&item->refcount) == 0);
BUG_ON(ref->object != item->object);
- if (--item->refcount == 0) {
+ if (atomic_dec_and_test(&item->refcount)) {
ref->release(ref);
item->object = NULL;
}
struct drm_local_map *map;
struct drm_map_list *r_list;
- /* Hardcoded from _DRM_FRAME_BUFFER,
- _DRM_REGISTERS, _DRM_SHM, _DRM_AGP, and
- _DRM_SCATTER_GATHER and _DRM_CONSISTENT */
- const char *types[] = { "FB", "REG", "SHM", "AGP", "SG", "PCI" };
+ static const char * const types[] = {
+ [_DRM_FRAME_BUFFER] = "FB",
+ [_DRM_REGISTERS] = "REG",
+ [_DRM_SHM] = "SHM",
+ [_DRM_AGP] = "AGP",
+ [_DRM_SCATTER_GATHER] = "SG",
+ [_DRM_CONSISTENT] = "PCI"};
const char *type;
int i;
map = r_list->map;
if (!map)
continue;
- if (map->type < 0 || map->type > 5)
+ if (map->type >= ARRAY_SIZE(types))
type = "??";
else
type = types[map->type];
request = compat_alloc_user_space(nbytes);
if (!access_ok(VERIFY_WRITE, request, nbytes))
return -EFAULT;
- list = (struct drm_buf_desc *) (request + 1);
+ list = (struct drm_buf_desc __user *) (request + 1);
if (__put_user(count, &request->count)
|| __put_user(list, &request->list))
request = compat_alloc_user_space(nbytes);
if (!access_ok(VERIFY_WRITE, request, nbytes))
return -EFAULT;
- list = (struct drm_buf_pub *) (request + 1);
+ list = (struct drm_buf_pub __user *) (request + 1);
if (__put_user(count, &request->count)
|| __put_user(list, &request->list))
return 0;
}
-drm_ioctl_compat_t *drm_compat_ioctls[] = {
+drm_ioctl_compat_t drm_compat_ioctls[] = {
[DRM_IOCTL_NR(DRM_IOCTL_VERSION32)] = compat_drm_version,
[DRM_IOCTL_NR(DRM_IOCTL_GET_UNIQUE32)] = compat_drm_getunique,
[DRM_IOCTL_NR(DRM_IOCTL_GET_MAP32)] = compat_drm_getmap,
long drm_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
{
unsigned int nr = DRM_IOCTL_NR(cmd);
- drm_ioctl_compat_t *fn;
int ret;
/* Assume that ioctls without an explicit compat routine will just
if (nr >= ARRAY_SIZE(drm_compat_ioctls))
return drm_ioctl(filp, cmd, arg);
- fn = drm_compat_ioctls[nr];
-
- if (fn != NULL)
- ret = (*fn) (filp, cmd, arg);
+ if (drm_compat_ioctls[nr] != NULL)
+ ret = (*drm_compat_ioctls[nr]) (filp, cmd, arg);
else
ret = drm_ioctl(filp, cmd, arg);
struct drm_file *file_priv = filp->private_data;
struct drm_device *dev;
const struct drm_ioctl_desc *ioctl = NULL;
- drm_ioctl_t *func;
+ drm_ioctl_no_const_t func;
unsigned int nr = DRM_IOCTL_NR(cmd);
int retcode = -EINVAL;
char stack_kdata[128];
int page_flipping;
wait_queue_head_t irq_queue;
- atomic_t irq_received;
- atomic_t irq_emitted;
+ atomic_unchecked_t irq_received;
+ atomic_unchecked_t irq_emitted;
int front_offset;
} drm_i810_private_t;
* locking inversion with the driver load path. And the access here is
* completely racy anyway. So don't bother with locking for now.
*/
- return dev->open_count == 0;
+ return local_read(&dev->open_count) == 0;
}
static const struct vga_switcheroo_client_ops i915_switcheroo_ops = {
static int
validate_exec_list(struct drm_device *dev,
struct drm_i915_gem_exec_object2 *exec,
- int count)
+ unsigned int count)
{
unsigned relocs_total = 0;
unsigned relocs_max = UINT_MAX / sizeof(struct drm_i915_gem_relocation_entry);
unsigned invalid_flags;
- int i;
+ unsigned int i;
invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS;
if (USES_FULL_PPGTT(dev))
(unsigned long)request);
}
-static drm_ioctl_compat_t *i915_compat_ioctls[] = {
+static drm_ioctl_compat_t i915_compat_ioctls[] = {
[DRM_I915_BATCHBUFFER] = compat_i915_batchbuffer,
[DRM_I915_CMDBUFFER] = compat_i915_cmdbuffer,
[DRM_I915_GETPARAM] = compat_i915_getparam,
long i915_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
{
unsigned int nr = DRM_IOCTL_NR(cmd);
- drm_ioctl_compat_t *fn = NULL;
int ret;
if (nr < DRM_COMMAND_BASE)
return drm_compat_ioctl(filp, cmd, arg);
- if (nr < DRM_COMMAND_BASE + ARRAY_SIZE(i915_compat_ioctls))
- fn = i915_compat_ioctls[nr - DRM_COMMAND_BASE];
-
- if (fn != NULL)
+ if (nr < DRM_COMMAND_BASE + ARRAY_SIZE(i915_compat_ioctls)) {
+ drm_ioctl_compat_t fn = i915_compat_ioctls[nr - DRM_COMMAND_BASE];
ret = (*fn) (filp, cmd, arg);
- else
+ } else
ret = drm_ioctl(filp, cmd, arg);
return ret;
int subsystem_vendor;
int subsystem_device;
void (*hook)(struct drm_device *dev);
-};
+} __do_const;
/* For systems that don't have a meaningful PCI subdevice/subvendor ID */
struct intel_dmi_quirk {
void (*hook)(struct drm_device *dev);
const struct dmi_system_id (*dmi_id_list)[];
-};
+} __do_const;
static int intel_dmi_reverse_brightness(const struct dmi_system_id *id)
{
return 1;
}
-static const struct intel_dmi_quirk intel_dmi_quirks[] = {
+static const struct dmi_system_id intel_dmi_quirks_table[] = {
{
- .dmi_id_list = &(const struct dmi_system_id[]) {
- {
- .callback = intel_dmi_reverse_brightness,
- .ident = "NCR Corporation",
- .matches = {DMI_MATCH(DMI_SYS_VENDOR, "NCR Corporation"),
- DMI_MATCH(DMI_PRODUCT_NAME, ""),
- },
- },
- { } /* terminating entry */
+ .callback = intel_dmi_reverse_brightness,
+ .ident = "NCR Corporation",
+ .matches = {DMI_MATCH(DMI_SYS_VENDOR, "NCR Corporation"),
+ DMI_MATCH(DMI_PRODUCT_NAME, ""),
},
+ },
+ { } /* terminating entry */
+};
+
+static const struct intel_dmi_quirk intel_dmi_quirks[] = {
+ {
+ .dmi_id_list = &intel_dmi_quirks_table,
.hook = quirk_invert_brightness,
},
};
if (imxdrm->pipes >= MAX_CRTC)
return -EINVAL;
- if (imxdrm->drm->open_count)
+ if (local_read(&imxdrm->drm->open_count))
return -EBUSY;
imx_drm_crtc = kzalloc(sizeof(*imx_drm_crtc), GFP_KERNEL);
u32 clear_cmd;
u32 maccess;
- atomic_t vbl_received; /**< Number of vblanks received. */
+ atomic_unchecked_t vbl_received; /**< Number of vblanks received. */
wait_queue_head_t fence_queue;
- atomic_t last_fence_retired;
+ atomic_unchecked_t last_fence_retired;
u32 next_fence_to_post;
unsigned int fb_cpp;
return 0;
}
-drm_ioctl_compat_t *mga_compat_ioctls[] = {
+drm_ioctl_compat_t mga_compat_ioctls[] = {
[DRM_MGA_INIT] = compat_mga_init,
[DRM_MGA_GETPARAM] = compat_mga_getparam,
[DRM_MGA_DMA_BOOTSTRAP] = compat_mga_dma_bootstrap,
long mga_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
{
unsigned int nr = DRM_IOCTL_NR(cmd);
- drm_ioctl_compat_t *fn = NULL;
int ret;
if (nr < DRM_COMMAND_BASE)
return drm_compat_ioctl(filp, cmd, arg);
- if (nr < DRM_COMMAND_BASE + ARRAY_SIZE(mga_compat_ioctls))
- fn = mga_compat_ioctls[nr - DRM_COMMAND_BASE];
-
- if (fn != NULL)
+ if (nr < DRM_COMMAND_BASE + ARRAY_SIZE(mga_compat_ioctls)) {
+ drm_ioctl_compat_t fn = mga_compat_ioctls[nr - DRM_COMMAND_BASE];
ret = (*fn) (filp, cmd, arg);
- else
+ } else
ret = drm_ioctl(filp, cmd, arg);
return ret;
if (crtc != 0)
return 0;
- return atomic_read(&dev_priv->vbl_received);
+ return atomic_read_unchecked(&dev_priv->vbl_received);
}
/* VBLANK interrupt */
if (status & MGA_VLINEPEN) {
MGA_WRITE(MGA_ICLEAR, MGA_VLINEICLR);
- atomic_inc(&dev_priv->vbl_received);
+ atomic_inc_unchecked(&dev_priv->vbl_received);
drm_handle_vblank(dev, 0);
handled = 1;
}
if ((prim_start & ~0x03) != (prim_end & ~0x03))
MGA_WRITE(MGA_PRIMEND, prim_end);
- atomic_inc(&dev_priv->last_fence_retired);
+ atomic_inc_unchecked(&dev_priv->last_fence_retired);
wake_up(&dev_priv->fence_queue);
handled = 1;
}
* using fences.
*/
DRM_WAIT_ON(ret, dev_priv->fence_queue, 3 * HZ,
- (((cur_fence = atomic_read(&dev_priv->last_fence_retired))
+ (((cur_fence = atomic_read_unchecked(&dev_priv->last_fence_retired))
- *sequence) <= (1 << 23)));
*sequence = cur_fence;
struct bit_table {
const char id;
int (* const parse_fn)(struct drm_device *, struct nvbios *, struct bit_entry *);
-};
+} __no_const;
#define BIT_TABLE(id, funcid) ((struct bit_table){ id, parse_bit_##funcid##_tbl_entry })
struct drm_global_reference mem_global_ref;
struct ttm_bo_global_ref bo_global_ref;
struct ttm_bo_device bdev;
- atomic_t validate_sequence;
int (*move)(struct nouveau_channel *,
struct ttm_buffer_object *,
struct ttm_mem_reg *, struct ttm_mem_reg *);
unsigned long arg)
{
unsigned int nr = DRM_IOCTL_NR(cmd);
- drm_ioctl_compat_t *fn = NULL;
+ drm_ioctl_compat_t fn = NULL;
int ret;
if (nr < DRM_COMMAND_BASE)
}
const struct ttm_mem_type_manager_func nouveau_vram_manager = {
- nouveau_vram_manager_init,
- nouveau_vram_manager_fini,
- nouveau_vram_manager_new,
- nouveau_vram_manager_del,
- nouveau_vram_manager_debug
+ .init = nouveau_vram_manager_init,
+ .takedown = nouveau_vram_manager_fini,
+ .get_node = nouveau_vram_manager_new,
+ .put_node = nouveau_vram_manager_del,
+ .debug = nouveau_vram_manager_debug
};
static int
}
const struct ttm_mem_type_manager_func nouveau_gart_manager = {
- nouveau_gart_manager_init,
- nouveau_gart_manager_fini,
- nouveau_gart_manager_new,
- nouveau_gart_manager_del,
- nouveau_gart_manager_debug
+ .init = nouveau_gart_manager_init,
+ .takedown = nouveau_gart_manager_fini,
+ .get_node = nouveau_gart_manager_new,
+ .put_node = nouveau_gart_manager_del,
+ .debug = nouveau_gart_manager_debug
};
/*XXX*/
}
const struct ttm_mem_type_manager_func nv04_gart_manager = {
- nv04_gart_manager_init,
- nv04_gart_manager_fini,
- nv04_gart_manager_new,
- nv04_gart_manager_del,
- nv04_gart_manager_debug
+ .init = nv04_gart_manager_init,
+ .takedown = nv04_gart_manager_fini,
+ .get_node = nv04_gart_manager_new,
+ .put_node = nv04_gart_manager_del,
+ .debug = nv04_gart_manager_debug
};
int
* locking inversion with the driver load path. And the access here is
* completely racy anyway. So don't bother with locking for now.
*/
- return dev->open_count == 0;
+ return local_read(&dev->open_count) == 0;
}
static const struct vga_switcheroo_client_ops
int ret;
mutex_lock(&qdev->async_io_mutex);
- irq_num = atomic_read(&qdev->irq_received_io_cmd);
+ irq_num = atomic_read_unchecked(&qdev->irq_received_io_cmd);
if (qdev->last_sent_io_cmd > irq_num) {
if (intr)
ret = wait_event_interruptible_timeout(qdev->io_cmd_event,
- atomic_read(&qdev->irq_received_io_cmd) > irq_num, 5*HZ);
+ atomic_read_unchecked(&qdev->irq_received_io_cmd) > irq_num, 5*HZ);
else
ret = wait_event_timeout(qdev->io_cmd_event,
- atomic_read(&qdev->irq_received_io_cmd) > irq_num, 5*HZ);
+ atomic_read_unchecked(&qdev->irq_received_io_cmd) > irq_num, 5*HZ);
/* 0 is timeout, just bail the "hw" has gone away */
if (ret <= 0)
goto out;
- irq_num = atomic_read(&qdev->irq_received_io_cmd);
+ irq_num = atomic_read_unchecked(&qdev->irq_received_io_cmd);
}
outb(val, addr);
qdev->last_sent_io_cmd = irq_num + 1;
if (intr)
ret = wait_event_interruptible_timeout(qdev->io_cmd_event,
- atomic_read(&qdev->irq_received_io_cmd) > irq_num, 5*HZ);
+ atomic_read_unchecked(&qdev->irq_received_io_cmd) > irq_num, 5*HZ);
else
ret = wait_event_timeout(qdev->io_cmd_event,
- atomic_read(&qdev->irq_received_io_cmd) > irq_num, 5*HZ);
+ atomic_read_unchecked(&qdev->irq_received_io_cmd) > irq_num, 5*HZ);
out:
if (ret > 0)
ret = 0;
struct drm_info_node *node = (struct drm_info_node *) m->private;
struct qxl_device *qdev = node->minor->dev->dev_private;
- seq_printf(m, "%d\n", atomic_read(&qdev->irq_received));
- seq_printf(m, "%d\n", atomic_read(&qdev->irq_received_display));
- seq_printf(m, "%d\n", atomic_read(&qdev->irq_received_cursor));
- seq_printf(m, "%d\n", atomic_read(&qdev->irq_received_io_cmd));
+ seq_printf(m, "%d\n", atomic_read_unchecked(&qdev->irq_received));
+ seq_printf(m, "%d\n", atomic_read_unchecked(&qdev->irq_received_display));
+ seq_printf(m, "%d\n", atomic_read_unchecked(&qdev->irq_received_cursor));
+ seq_printf(m, "%d\n", atomic_read_unchecked(&qdev->irq_received_io_cmd));
seq_printf(m, "%d\n", qdev->irq_received_error);
return 0;
}
unsigned int last_sent_io_cmd;
/* interrupt handling */
- atomic_t irq_received;
- atomic_t irq_received_display;
- atomic_t irq_received_cursor;
- atomic_t irq_received_io_cmd;
+ atomic_unchecked_t irq_received;
+ atomic_unchecked_t irq_received_display;
+ atomic_unchecked_t irq_received_cursor;
+ atomic_unchecked_t irq_received_io_cmd;
unsigned irq_received_error;
wait_queue_head_t display_event;
wait_queue_head_t cursor_event;
/* TODO copy slow path code from i915 */
fb_cmd = qxl_bo_kmap_atomic_page(qdev, cmd_bo, (release->release_offset & PAGE_SIZE));
- unwritten = __copy_from_user_inatomic_nocache(fb_cmd + sizeof(union qxl_release_info) + (release->release_offset & ~PAGE_SIZE), (void *)(unsigned long)cmd->command, cmd->command_size);
+ unwritten = __copy_from_user_inatomic_nocache(fb_cmd + sizeof(union qxl_release_info) + (release->release_offset & ~PAGE_SIZE), (void __force_user *)(unsigned long)cmd->command, cmd->command_size);
{
struct qxl_drawable *draw = fb_cmd;
struct drm_qxl_reloc reloc;
if (copy_from_user(&reloc,
- &((struct drm_qxl_reloc *)(uintptr_t)cmd->relocs)[i],
+ &((struct drm_qxl_reloc __force_user *)(uintptr_t)cmd->relocs)[i],
sizeof(reloc))) {
ret = -EFAULT;
goto out_free_bos;
for (cmd_num = 0; cmd_num < execbuffer->commands_num; ++cmd_num) {
- struct drm_qxl_command *commands =
- (struct drm_qxl_command *)(uintptr_t)execbuffer->commands;
+ struct drm_qxl_command __user *commands =
+ (struct drm_qxl_command __user *)(uintptr_t)execbuffer->commands;
- if (copy_from_user(&user_cmd, &commands[cmd_num],
+ if (copy_from_user(&user_cmd, (struct drm_qxl_command __force_user *)&commands[cmd_num],
sizeof(user_cmd)))
return -EFAULT;
if (!pending)
return IRQ_NONE;
- atomic_inc(&qdev->irq_received);
+ atomic_inc_unchecked(&qdev->irq_received);
if (pending & QXL_INTERRUPT_DISPLAY) {
- atomic_inc(&qdev->irq_received_display);
+ atomic_inc_unchecked(&qdev->irq_received_display);
wake_up_all(&qdev->display_event);
qxl_queue_garbage_collect(qdev, false);
}
if (pending & QXL_INTERRUPT_CURSOR) {
- atomic_inc(&qdev->irq_received_cursor);
+ atomic_inc_unchecked(&qdev->irq_received_cursor);
wake_up_all(&qdev->cursor_event);
}
if (pending & QXL_INTERRUPT_IO_CMD) {
- atomic_inc(&qdev->irq_received_io_cmd);
+ atomic_inc_unchecked(&qdev->irq_received_io_cmd);
wake_up_all(&qdev->io_cmd_event);
}
if (pending & QXL_INTERRUPT_ERROR) {
init_waitqueue_head(&qdev->io_cmd_event);
INIT_WORK(&qdev->client_monitors_config_work,
qxl_client_monitors_config_work_func);
- atomic_set(&qdev->irq_received, 0);
- atomic_set(&qdev->irq_received_display, 0);
- atomic_set(&qdev->irq_received_cursor, 0);
- atomic_set(&qdev->irq_received_io_cmd, 0);
+ atomic_set_unchecked(&qdev->irq_received, 0);
+ atomic_set_unchecked(&qdev->irq_received_display, 0);
+ atomic_set_unchecked(&qdev->irq_received_cursor, 0);
+ atomic_set_unchecked(&qdev->irq_received_io_cmd, 0);
qdev->irq_received_error = 0;
ret = drm_irq_install(qdev->ddev, qdev->ddev->pdev->irq);
qdev->ram_header->int_mask = QXL_INTERRUPT_MASK;
}
}
-static struct vm_operations_struct qxl_ttm_vm_ops;
+static vm_operations_struct_no_const qxl_ttm_vm_ops __read_only;
static const struct vm_operations_struct *ttm_vm_ops;
static int qxl_ttm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
return r;
if (unlikely(ttm_vm_ops == NULL)) {
ttm_vm_ops = vma->vm_ops;
+ pax_open_kernel();
qxl_ttm_vm_ops = *ttm_vm_ops;
qxl_ttm_vm_ops.fault = &qxl_ttm_fault;
+ pax_close_kernel();
}
vma->vm_ops = &qxl_ttm_vm_ops;
return 0;
static int qxl_ttm_debugfs_init(struct qxl_device *qdev)
{
#if defined(CONFIG_DEBUG_FS)
- static struct drm_info_list qxl_mem_types_list[QXL_DEBUGFS_MEM_TYPES];
- static char qxl_mem_types_names[QXL_DEBUGFS_MEM_TYPES][32];
- unsigned i;
-
- for (i = 0; i < QXL_DEBUGFS_MEM_TYPES; i++) {
- if (i == 0)
- sprintf(qxl_mem_types_names[i], "qxl_mem_mm");
- else
- sprintf(qxl_mem_types_names[i], "qxl_surf_mm");
- qxl_mem_types_list[i].name = qxl_mem_types_names[i];
- qxl_mem_types_list[i].show = &qxl_mm_dump_table;
- qxl_mem_types_list[i].driver_features = 0;
- if (i == 0)
- qxl_mem_types_list[i].data = qdev->mman.bdev.man[TTM_PL_VRAM].priv;
- else
- qxl_mem_types_list[i].data = qdev->mman.bdev.man[TTM_PL_PRIV0].priv;
+ static struct drm_info_list qxl_mem_types_list[QXL_DEBUGFS_MEM_TYPES] = {
+ {
+ .name = "qxl_mem_mm",
+ .show = &qxl_mm_dump_table,
+ },
+ {
+ .name = "qxl_surf_mm",
+ .show = &qxl_mm_dump_table,
+ }
+ };
- }
- return qxl_debugfs_add_files(qdev, qxl_mem_types_list, i);
+ pax_open_kernel();
+ *(void **)&qxl_mem_types_list[0].data = qdev->mman.bdev.man[TTM_PL_VRAM].priv;
+ *(void **)&qxl_mem_types_list[1].data = qdev->mman.bdev.man[TTM_PL_PRIV0].priv;
+ pax_close_kernel();
+
+ return qxl_debugfs_add_files(qdev, qxl_mem_types_list, QXL_DEBUGFS_MEM_TYPES);
#else
return 0;
#endif
/* GH: Simple idle check.
*/
- atomic_set(&dev_priv->idle_count, 0);
+ atomic_set_unchecked(&dev_priv->idle_count, 0);
/* We don't support anything other than bus-mastering ring mode,
* but the ring can be in either AGP or PCI space for the ring
int is_pci;
unsigned long cce_buffers_offset;
- atomic_t idle_count;
+ atomic_unchecked_t idle_count;
int page_flipping;
int current_page;
u32 crtc_offset;
u32 crtc_offset_cntl;
- atomic_t vbl_received;
+ atomic_unchecked_t vbl_received;
u32 color_fmt;
unsigned int front_offset;
return drm_ioctl(file, DRM_IOCTL_R128_GETPARAM, (unsigned long)getparam);
}
-drm_ioctl_compat_t *r128_compat_ioctls[] = {
+drm_ioctl_compat_t r128_compat_ioctls[] = {
[DRM_R128_INIT] = compat_r128_init,
[DRM_R128_DEPTH] = compat_r128_depth,
[DRM_R128_STIPPLE] = compat_r128_stipple,
long r128_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
{
unsigned int nr = DRM_IOCTL_NR(cmd);
- drm_ioctl_compat_t *fn = NULL;
int ret;
if (nr < DRM_COMMAND_BASE)
return drm_compat_ioctl(filp, cmd, arg);
- if (nr < DRM_COMMAND_BASE + ARRAY_SIZE(r128_compat_ioctls))
- fn = r128_compat_ioctls[nr - DRM_COMMAND_BASE];
-
- if (fn != NULL)
+ if (nr < DRM_COMMAND_BASE + ARRAY_SIZE(r128_compat_ioctls)) {
+ drm_ioctl_compat_t fn = r128_compat_ioctls[nr - DRM_COMMAND_BASE];
ret = (*fn) (filp, cmd, arg);
- else
+ } else
ret = drm_ioctl(filp, cmd, arg);
return ret;
if (crtc != 0)
return 0;
- return atomic_read(&dev_priv->vbl_received);
+ return atomic_read_unchecked(&dev_priv->vbl_received);
}
irqreturn_t r128_driver_irq_handler(int irq, void *arg)
/* VBLANK interrupt */
if (status & R128_CRTC_VBLANK_INT) {
R128_WRITE(R128_GEN_INT_STATUS, R128_CRTC_VBLANK_INT_AK);
- atomic_inc(&dev_priv->vbl_received);
+ atomic_inc_unchecked(&dev_priv->vbl_received);
drm_handle_vblank(dev, 0);
return IRQ_HANDLED;
}
static void r128_cce_performance_boxes(drm_r128_private_t *dev_priv)
{
- if (atomic_read(&dev_priv->idle_count) == 0)
+ if (atomic_read_unchecked(&dev_priv->idle_count) == 0)
r128_clear_box(dev_priv, 64, 4, 8, 8, 0, 255, 0);
else
- atomic_set(&dev_priv->idle_count, 0);
+ atomic_set_unchecked(&dev_priv->idle_count, 0);
}
#endif
regex_t mask_rex;
regmatch_t match[4];
char buf[1024];
- size_t end;
+ long end;
int len;
int done = 0;
int r;
unsigned o;
struct offset *offset;
char last_reg_s[10];
- int last_reg;
+ unsigned long last_reg;
if (regcomp
(&mask_rex, "(0x[0-9a-fA-F]*) *([_a-zA-Z0-9]*)", REG_EXTENDED)) {
* locking inversion with the driver load path. And the access here is
* completely racy anyway. So don't bother with locking for now.
*/
- return dev->open_count == 0;
+ return local_read(&dev->open_count) == 0;
}
static const struct vga_switcheroo_client_ops radeon_switcheroo_ops = {
/* SW interrupt */
wait_queue_head_t swi_queue;
- atomic_t swi_emitted;
+ atomic_unchecked_t swi_emitted;
int vblank_crtc;
uint32_t irq_enable_reg;
uint32_t r500_disp_irq_reg;
request = compat_alloc_user_space(sizeof(*request));
if (!access_ok(VERIFY_WRITE, request, sizeof(*request))
|| __put_user(req32.param, &request->param)
- || __put_user((void __user *)(unsigned long)req32.value,
+ || __put_user((unsigned long)req32.value,
&request->value))
return -EFAULT;
#define compat_radeon_cp_setparam NULL
#endif /* X86_64 || IA64 */
-static drm_ioctl_compat_t *radeon_compat_ioctls[] = {
+static drm_ioctl_compat_t radeon_compat_ioctls[] = {
[DRM_RADEON_CP_INIT] = compat_radeon_cp_init,
[DRM_RADEON_CLEAR] = compat_radeon_cp_clear,
[DRM_RADEON_STIPPLE] = compat_radeon_cp_stipple,
long radeon_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
{
unsigned int nr = DRM_IOCTL_NR(cmd);
- drm_ioctl_compat_t *fn = NULL;
int ret;
if (nr < DRM_COMMAND_BASE)
return drm_compat_ioctl(filp, cmd, arg);
- if (nr < DRM_COMMAND_BASE + ARRAY_SIZE(radeon_compat_ioctls))
- fn = radeon_compat_ioctls[nr - DRM_COMMAND_BASE];
-
- if (fn != NULL)
+ if (nr < DRM_COMMAND_BASE + ARRAY_SIZE(radeon_compat_ioctls)) {
+ drm_ioctl_compat_t fn = radeon_compat_ioctls[nr - DRM_COMMAND_BASE];
ret = (*fn) (filp, cmd, arg);
- else
+ } else
ret = drm_ioctl(filp, cmd, arg);
return ret;
unsigned int ret;
RING_LOCALS;
- atomic_inc(&dev_priv->swi_emitted);
- ret = atomic_read(&dev_priv->swi_emitted);
+ atomic_inc_unchecked(&dev_priv->swi_emitted);
+ ret = atomic_read_unchecked(&dev_priv->swi_emitted);
BEGIN_RING(4);
OUT_RING_REG(RADEON_LAST_SWI_REG, ret);
drm_radeon_private_t *dev_priv =
(drm_radeon_private_t *) dev->dev_private;
- atomic_set(&dev_priv->swi_emitted, 0);
+ atomic_set_unchecked(&dev_priv->swi_emitted, 0);
init_waitqueue_head(&dev_priv->swi_queue);
dev->max_vblank_count = 0x001fffff;
if (sarea_priv->nbox > RADEON_NR_SAREA_CLIPRECTS)
sarea_priv->nbox = RADEON_NR_SAREA_CLIPRECTS;
- if (copy_from_user(&depth_boxes, clear->depth_boxes,
+ if (sarea_priv->nbox > RADEON_NR_SAREA_CLIPRECTS || copy_from_user(&depth_boxes, clear->depth_boxes,
sarea_priv->nbox * sizeof(depth_boxes[0])))
return -EFAULT;
{
drm_radeon_private_t *dev_priv = dev->dev_private;
drm_radeon_getparam_t *param = data;
- int value;
+ int value = 0;
DRM_DEBUG("pid=%d\n", DRM_CURRENTPID);
man->size = size >> PAGE_SHIFT;
}
-static struct vm_operations_struct radeon_ttm_vm_ops;
+static vm_operations_struct_no_const radeon_ttm_vm_ops __read_only;
static const struct vm_operations_struct *ttm_vm_ops = NULL;
static int radeon_ttm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
}
if (unlikely(ttm_vm_ops == NULL)) {
ttm_vm_ops = vma->vm_ops;
+ pax_open_kernel();
radeon_ttm_vm_ops = *ttm_vm_ops;
radeon_ttm_vm_ops.fault = &radeon_ttm_fault;
+ pax_close_kernel();
}
vma->vm_ops = &radeon_ttm_vm_ops;
return 0;
}
for (i = 0; i < ARRAY_SIZE(debugfs_files); i++)
- dc->debugfs_files[i].data = dc;
+ *(void **)&dc->debugfs_files[i].data = dc;
err = drm_debugfs_create_files(dc->debugfs_files,
ARRAY_SIZE(debugfs_files),
struct clk *clk_lp;
struct clk *clk;
- struct drm_info_list *debugfs_files;
+ drm_info_list_no_const *debugfs_files;
struct drm_minor *minor;
struct dentry *debugfs;
bool stereo;
bool dvi;
- struct drm_info_list *debugfs_files;
+ drm_info_list_no_const *debugfs_files;
struct drm_minor *minor;
struct dentry *debugfs;
};
}
const struct ttm_mem_type_manager_func ttm_bo_manager_func = {
- ttm_bo_man_init,
- ttm_bo_man_takedown,
- ttm_bo_man_get_node,
- ttm_bo_man_put_node,
- ttm_bo_man_debug
+ .init = ttm_bo_man_init,
+ .takedown = ttm_bo_man_takedown,
+ .get_node = ttm_bo_man_get_node,
+ .put_node = ttm_bo_man_put_node,
+ .debug = ttm_bo_man_debug
};
EXPORT_SYMBOL(ttm_bo_manager_func);
zone->glob = glob;
glob->zone_kernel = zone;
ret = kobject_init_and_add(
- &zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name);
+ &zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, "%s", zone->name);
if (unlikely(ret != 0)) {
kobject_put(&zone->kobj);
return ret;
zone->glob = glob;
glob->zone_dma32 = zone;
ret = kobject_init_and_add(
- &zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name);
+ &zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, "%s", zone->name);
if (unlikely(ret != 0)) {
kobject_put(&zone->kobj);
return ret;
#define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(struct page *))
#define SMALL_ALLOCATION 16
-#define FREE_ALL_PAGES (~0U)
+#define FREE_ALL_PAGES (~0UL)
/* times are in msecs */
#define PAGE_FREE_INTERVAL 1000
* @free_all: If set to true will free all pages in pool
* @use_static: Safe to use static buffer
**/
-static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free,
+static unsigned long ttm_page_pool_free(struct ttm_page_pool *pool, unsigned long nr_free,
bool use_static)
{
static struct page *static_buf[NUM_PAGES_TO_ALLOC];
unsigned long irq_flags;
struct page *p;
struct page **pages_to_free;
- unsigned freed_pages = 0,
- npages_to_free = nr_free;
+ unsigned long freed_pages = 0, npages_to_free = nr_free;
if (NUM_PAGES_TO_ALLOC < nr_free)
npages_to_free = NUM_PAGES_TO_ALLOC;
__list_del(&p->lru, &pool->list);
ttm_pool_update_free_locked(pool, freed_pages);
- nr_free -= freed_pages;
+ if (likely(nr_free != FREE_ALL_PAGES))
+ nr_free -= freed_pages;
}
spin_unlock_irqrestore(&pool->lock, irq_flags);
unsigned i;
unsigned pool_offset;
struct ttm_page_pool *pool;
- int shrink_pages = sc->nr_to_scan;
+ unsigned long shrink_pages = sc->nr_to_scan;
unsigned long freed = 0;
if (!mutex_trylock(&lock))
pool_offset = ++start_pool % NUM_POOLS;
/* select start pool in round robin fashion */
for (i = 0; i < NUM_POOLS; ++i) {
- unsigned nr_free = shrink_pages;
+ unsigned long nr_free = shrink_pages;
if (shrink_pages == 0)
break;
pool = &_manager->pools[(i + pool_offset)%NUM_POOLS];
}
/* Put all pages in pages list to correct pool to wait for reuse */
-static void ttm_put_pages(struct page **pages, unsigned npages, int flags,
+static void ttm_put_pages(struct page **pages, unsigned long npages, int flags,
enum ttm_caching_state cstate)
{
unsigned long irq_flags;
struct list_head plist;
struct page *p = NULL;
gfp_t gfp_flags = GFP_USER;
- unsigned count;
+ unsigned long count;
int r;
/* set zero flag for page allocation if required */
#define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(struct page *))
#define SMALL_ALLOCATION 4
-#define FREE_ALL_PAGES (~0U)
+#define FREE_ALL_PAGES (~0UL)
/* times are in msecs */
#define IS_UNDEFINED (0)
#define IS_WC (1<<1)
* @nr_free: If set to true will free all pages in pool
* @use_static: Safe to use static buffer
**/
-static unsigned ttm_dma_page_pool_free(struct dma_pool *pool, unsigned nr_free,
+static unsigned long ttm_dma_page_pool_free(struct dma_pool *pool, unsigned long nr_free,
bool use_static)
{
static struct page *static_buf[NUM_PAGES_TO_ALLOC];
struct dma_page *dma_p, *tmp;
struct page **pages_to_free;
struct list_head d_pages;
- unsigned freed_pages = 0,
- npages_to_free = nr_free;
+ unsigned long freed_pages = 0, npages_to_free = nr_free;
if (NUM_PAGES_TO_ALLOC < nr_free)
npages_to_free = NUM_PAGES_TO_ALLOC;
/* remove range of pages from the pool */
if (freed_pages) {
ttm_pool_update_free_locked(pool, freed_pages);
- nr_free -= freed_pages;
+ if (likely(nr_free != FREE_ALL_PAGES))
+ nr_free -= freed_pages;
}
spin_unlock_irqrestore(&pool->lock, irq_flags);
struct dma_page *d_page, *next;
enum pool_type type;
bool is_cached = false;
- unsigned count = 0, i, npages = 0;
+ unsigned long count = 0, i, npages = 0;
unsigned long irq_flags;
type = ttm_to_type(ttm->page_flags, ttm->caching_state);
static unsigned start_pool;
unsigned idx = 0;
unsigned pool_offset;
- unsigned shrink_pages = sc->nr_to_scan;
+ unsigned long shrink_pages = sc->nr_to_scan;
struct device_pools *p;
unsigned long freed = 0;
goto out;
pool_offset = ++start_pool % _manager->npools;
list_for_each_entry(p, &_manager->pools, pools) {
- unsigned nr_free;
+ unsigned long nr_free;
if (!p->dev)
continue;
shrink_pages = ttm_dma_page_pool_free(p->pool, nr_free, true);
freed += nr_free - shrink_pages;
- pr_debug("%s: (%s:%d) Asked to shrink %d, have %d more to go\n",
+ pr_debug("%s: (%s:%d) Asked to shrink %lu, have %lu more to go\n",
p->pool->dev_name, p->pool->name, current->pid,
nr_free, shrink_pages);
}
fb_deferred_io_cleanup(info);
kfree(info->fbdefio);
info->fbdefio = NULL;
- info->fbops->fb_mmap = udl_fb_mmap;
}
pr_warn("released /dev/fb%d user=%d count=%d\n",
typedef uint32_t maskarray_t[5];
typedef struct drm_via_irq {
- atomic_t irq_received;
+ atomic_unchecked_t irq_received;
uint32_t pending_mask;
uint32_t enable_mask;
wait_queue_head_t irq_queue;
struct timeval last_vblank;
int last_vblank_valid;
unsigned usec_per_vblank;
- atomic_t vbl_received;
+ atomic_unchecked_t vbl_received;
drm_via_state_t hc_state;
char pci_buf[VIA_PCI_BUF_SIZE];
const uint32_t *fire_offsets[VIA_FIRE_BUF_SIZE];
if (crtc != 0)
return 0;
- return atomic_read(&dev_priv->vbl_received);
+ return atomic_read_unchecked(&dev_priv->vbl_received);
}
irqreturn_t via_driver_irq_handler(int irq, void *arg)
status = VIA_READ(VIA_REG_INTERRUPT);
if (status & VIA_IRQ_VBLANK_PENDING) {
- atomic_inc(&dev_priv->vbl_received);
- if (!(atomic_read(&dev_priv->vbl_received) & 0x0F)) {
+ atomic_inc_unchecked(&dev_priv->vbl_received);
+ if (!(atomic_read_unchecked(&dev_priv->vbl_received) & 0x0F)) {
do_gettimeofday(&cur_vblank);
if (dev_priv->last_vblank_valid) {
dev_priv->usec_per_vblank =
dev_priv->last_vblank = cur_vblank;
dev_priv->last_vblank_valid = 1;
}
- if (!(atomic_read(&dev_priv->vbl_received) & 0xFF)) {
+ if (!(atomic_read_unchecked(&dev_priv->vbl_received) & 0xFF)) {
DRM_DEBUG("US per vblank is: %u\n",
dev_priv->usec_per_vblank);
}
for (i = 0; i < dev_priv->num_irqs; ++i) {
if (status & cur_irq->pending_mask) {
- atomic_inc(&cur_irq->irq_received);
+ atomic_inc_unchecked(&cur_irq->irq_received);
wake_up(&cur_irq->irq_queue);
handled = 1;
if (dev_priv->irq_map[drm_via_irq_dma0_td] == i)
DRM_WAIT_ON(ret, cur_irq->irq_queue, 3 * HZ,
((VIA_READ(masks[irq][2]) & masks[irq][3]) ==
masks[irq][4]));
- cur_irq_sequence = atomic_read(&cur_irq->irq_received);
+ cur_irq_sequence = atomic_read_unchecked(&cur_irq->irq_received);
} else {
DRM_WAIT_ON(ret, cur_irq->irq_queue, 3 * HZ,
(((cur_irq_sequence =
- atomic_read(&cur_irq->irq_received)) -
+ atomic_read_unchecked(&cur_irq->irq_received)) -
*sequence) <= (1 << 23)));
}
*sequence = cur_irq_sequence;
}
for (i = 0; i < dev_priv->num_irqs; ++i) {
- atomic_set(&cur_irq->irq_received, 0);
+ atomic_set_unchecked(&cur_irq->irq_received, 0);
cur_irq->enable_mask = dev_priv->irq_masks[i][0];
cur_irq->pending_mask = dev_priv->irq_masks[i][1];
init_waitqueue_head(&cur_irq->irq_queue);
switch (irqwait->request.type & ~VIA_IRQ_FLAGS_MASK) {
case VIA_IRQ_RELATIVE:
irqwait->request.sequence +=
- atomic_read(&cur_irq->irq_received);
+ atomic_read_unchecked(&cur_irq->irq_received);
irqwait->request.type &= ~_DRM_VBLANK_RELATIVE;
case VIA_IRQ_ABSOLUTE:
break;
* Fencing and IRQs.
*/
- atomic_t marker_seq;
+ atomic_unchecked_t marker_seq;
wait_queue_head_t fence_queue;
wait_queue_head_t fifo_queue;
spinlock_t waiter_lock;
(unsigned int) min,
(unsigned int) fifo->capabilities);
- atomic_set(&dev_priv->marker_seq, dev_priv->last_read_seqno);
+ atomic_set_unchecked(&dev_priv->marker_seq, dev_priv->last_read_seqno);
iowrite32(dev_priv->last_read_seqno, fifo_mem + SVGA_FIFO_FENCE);
vmw_marker_queue_init(&fifo->marker_queue);
return vmw_fifo_send_fence(dev_priv, &dummy);
if (reserveable)
iowrite32(bytes, fifo_mem +
SVGA_FIFO_RESERVED);
- return fifo_mem + (next_cmd >> 2);
+ return (__le32 __force_kernel *)fifo_mem + (next_cmd >> 2);
} else {
need_bounce = true;
}
fm = vmw_fifo_reserve(dev_priv, bytes);
if (unlikely(fm == NULL)) {
- *seqno = atomic_read(&dev_priv->marker_seq);
+ *seqno = atomic_read_unchecked(&dev_priv->marker_seq);
ret = -ENOMEM;
(void)vmw_fallback_wait(dev_priv, false, true, *seqno,
false, 3*HZ);
}
do {
- *seqno = atomic_add_return(1, &dev_priv->marker_seq);
+ *seqno = atomic_add_return_unchecked(1, &dev_priv->marker_seq);
} while (*seqno == 0);
if (!(fifo_state->capabilities & SVGA_FIFO_CAP_FENCE)) {
}
const struct ttm_mem_type_manager_func vmw_gmrid_manager_func = {
- vmw_gmrid_man_init,
- vmw_gmrid_man_takedown,
- vmw_gmrid_man_get_node,
- vmw_gmrid_man_put_node,
- vmw_gmrid_man_debug
+ .init = vmw_gmrid_man_init,
+ .takedown = vmw_gmrid_man_takedown,
+ .get_node = vmw_gmrid_man_get_node,
+ .put_node = vmw_gmrid_man_put_node,
+ .debug = vmw_gmrid_man_debug
};
int ret;
num_clips = arg->num_clips;
- clips_ptr = (struct drm_vmw_rect *)(unsigned long)arg->clips_ptr;
+ clips_ptr = (struct drm_vmw_rect __user *)(unsigned long)arg->clips_ptr;
if (unlikely(num_clips == 0))
return 0;
int ret;
num_clips = arg->num_clips;
- clips_ptr = (struct drm_vmw_rect *)(unsigned long)arg->clips_ptr;
+ clips_ptr = (struct drm_vmw_rect __user *)(unsigned long)arg->clips_ptr;
if (unlikely(num_clips == 0))
return 0;
* emitted. Then the fence is stale and signaled.
*/
- ret = ((atomic_read(&dev_priv->marker_seq) - seqno)
+ ret = ((atomic_read_unchecked(&dev_priv->marker_seq) - seqno)
> VMW_FENCE_WRAP);
return ret;
if (fifo_idle)
down_read(&fifo_state->rwsem);
- signal_seq = atomic_read(&dev_priv->marker_seq);
+ signal_seq = atomic_read_unchecked(&dev_priv->marker_seq);
ret = 0;
for (;;) {
while (!vmw_lag_lt(queue, us)) {
spin_lock(&queue->lock);
if (list_empty(&queue->head))
- seqno = atomic_read(&dev_priv->marker_seq);
+ seqno = atomic_read_unchecked(&dev_priv->marker_seq);
else {
marker = list_first_entry(&queue->head,
struct vmw_marker, head);
/* this version is for the case where the power switch is separate
to the device being powered down. */
-int vga_switcheroo_init_domain_pm_ops(struct device *dev, struct dev_pm_domain *domain)
+int vga_switcheroo_init_domain_pm_ops(struct device *dev, dev_pm_domain_no_const *domain)
{
/* copy over all the bus versions */
if (dev->bus && dev->bus->pm) {
return ret;
}
-int vga_switcheroo_init_domain_pm_optimus_hdmi_audio(struct device *dev, struct dev_pm_domain *domain)
+int vga_switcheroo_init_domain_pm_optimus_hdmi_audio(struct device *dev, dev_pm_domain_no_const *domain)
{
/* copy over all the bus versions */
if (dev->bus && dev->bus->pm) {
int hid_add_device(struct hid_device *hdev)
{
- static atomic_t id = ATOMIC_INIT(0);
+ static atomic_unchecked_t id = ATOMIC_INIT(0);
int ret;
if (WARN_ON(hdev->status & HID_STAT_ADDED))
/* XXX hack, any other cleaner solution after the driver core
* is converted to allow more than 20 bytes as the device name? */
dev_set_name(&hdev->dev, "%04X:%04X:%04X.%04X", hdev->bus,
- hdev->vendor, hdev->product, atomic_inc_return(&id));
+ hdev->vendor, hdev->product, atomic_inc_return_unchecked(&id));
hid_debug_register(hdev, dev_name(&hdev->dev));
ret = device_add(&hdev->dev);
* case we forward it to the correct hid device (via hid_input_report()
* ) and return 1 so hid-core does not anything else with it.
*/
+ if ((dj_report->device_index < DJ_DEVICE_INDEX_MIN) ||
+ (dj_report->device_index > DJ_DEVICE_INDEX_MAX)) {
+ dev_err(&hdev->dev, "%s: invalid device index:%d\n",
+ __func__, dj_report->device_index);
+ return false;
+ }
if ((dj_report->device_index < DJ_DEVICE_INDEX_MIN) ||
(dj_report->device_index > DJ_DEVICE_INDEX_MAX)) {
else if (size == 0)
return -EIO;
- if (copy_to_user(u, buf, size))
+ if (size > sizeof(buf) || copy_to_user(u, buf, size))
return -EFAULT;
*off += size;
unsigned long flags;
int ret = 0;
- next_gpadl_handle = atomic_read(&vmbus_connection.next_gpadl_handle);
- atomic_inc(&vmbus_connection.next_gpadl_handle);
+ next_gpadl_handle = atomic_read_unchecked(&vmbus_connection.next_gpadl_handle);
+ atomic_inc_unchecked(&vmbus_connection.next_gpadl_handle);
ret = create_gpadl_header(kbuffer, size, &msginfo, &msgcount);
if (ret)
u64 output_address = (output) ? virt_to_phys(output) : 0;
u32 output_address_hi = output_address >> 32;
u32 output_address_lo = output_address & 0xFFFFFFFF;
- void *hypercall_page = hv_context.hypercall_page;
+ void *hypercall_page = ktva_ktla(hv_context.hypercall_page);
__asm__ __volatile__ ("call *%8" : "=d"(hv_status_hi),
"=a"(hv_status_lo) : "d" (control_hi),
/* See if the hypercall page is already set */
rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
- virtaddr = __vmalloc(PAGE_SIZE, GFP_KERNEL, PAGE_KERNEL_EXEC);
+ virtaddr = __vmalloc(PAGE_SIZE, GFP_KERNEL, PAGE_KERNEL_RX);
if (!virtaddr)
goto cleanup;
module_param(pressure_report_delay, uint, (S_IRUGO | S_IWUSR));
MODULE_PARM_DESC(pressure_report_delay, "Delay in secs in reporting pressure");
-static atomic_t trans_id = ATOMIC_INIT(0);
+static atomic_unchecked_t trans_id = ATOMIC_INIT(0);
static int dm_ring_size = (5 * PAGE_SIZE);
pr_info("Memory hot add failed\n");
dm->state = DM_INITIALIZED;
- resp.hdr.trans_id = atomic_inc_return(&trans_id);
+ resp.hdr.trans_id = atomic_inc_return_unchecked(&trans_id);
vmbus_sendpacket(dm->dev->channel, &resp,
sizeof(struct dm_hot_add_response),
(unsigned long)NULL,
memset(&status, 0, sizeof(struct dm_status));
status.hdr.type = DM_STATUS_REPORT;
status.hdr.size = sizeof(struct dm_status);
- status.hdr.trans_id = atomic_inc_return(&trans_id);
+ status.hdr.trans_id = atomic_inc_return_unchecked(&trans_id);
/*
* The host expects the guest to report free memory.
* send the status. This can happen if we were interrupted
* after we picked our transaction ID.
*/
- if (status.hdr.trans_id != atomic_read(&trans_id))
+ if (status.hdr.trans_id != atomic_read_unchecked(&trans_id))
return;
/*
*/
do {
- bl_resp->hdr.trans_id = atomic_inc_return(&trans_id);
+ bl_resp->hdr.trans_id = atomic_inc_return_unchecked(&trans_id);
ret = vmbus_sendpacket(dm_device.dev->channel,
bl_resp,
bl_resp->hdr.size,
memset(&resp, 0, sizeof(struct dm_unballoon_response));
resp.hdr.type = DM_UNBALLOON_RESPONSE;
- resp.hdr.trans_id = atomic_inc_return(&trans_id);
+ resp.hdr.trans_id = atomic_inc_return_unchecked(&trans_id);
resp.hdr.size = sizeof(struct dm_unballoon_response);
vmbus_sendpacket(dm_device.dev->channel, &resp,
memset(&version_req, 0, sizeof(struct dm_version_request));
version_req.hdr.type = DM_VERSION_REQUEST;
version_req.hdr.size = sizeof(struct dm_version_request);
- version_req.hdr.trans_id = atomic_inc_return(&trans_id);
+ version_req.hdr.trans_id = atomic_inc_return_unchecked(&trans_id);
version_req.version.version = DYNMEM_PROTOCOL_VERSION_WIN7;
version_req.is_last_attempt = 1;
memset(&version_req, 0, sizeof(struct dm_version_request));
version_req.hdr.type = DM_VERSION_REQUEST;
version_req.hdr.size = sizeof(struct dm_version_request);
- version_req.hdr.trans_id = atomic_inc_return(&trans_id);
+ version_req.hdr.trans_id = atomic_inc_return_unchecked(&trans_id);
version_req.version.version = DYNMEM_PROTOCOL_VERSION_WIN8;
version_req.is_last_attempt = 0;
memset(&cap_msg, 0, sizeof(struct dm_capabilities));
cap_msg.hdr.type = DM_CAPABILITIES_REPORT;
cap_msg.hdr.size = sizeof(struct dm_capabilities);
- cap_msg.hdr.trans_id = atomic_inc_return(&trans_id);
+ cap_msg.hdr.trans_id = atomic_inc_return_unchecked(&trans_id);
cap_msg.caps.cap_bits.balloon = 1;
cap_msg.caps.cap_bits.hot_add = 1;
struct vmbus_connection {
enum vmbus_connect_state conn_state;
- atomic_t next_gpadl_handle;
+ atomic_unchecked_t next_gpadl_handle;
/*
* Represents channel interrupts. Each bit position represents a
{
int ret = 0;
- static atomic_t device_num = ATOMIC_INIT(0);
+ static atomic_unchecked_t device_num = ATOMIC_INIT(0);
dev_set_name(&child_device_obj->device, "vmbus_0_%d",
- atomic_inc_return(&device_num));
+ atomic_inc_return_unchecked(&device_num));
child_device_obj->device.bus = &hv_bus;
child_device_obj->device.parent = &hv_acpi_dev->dev;
struct device_attribute *devattr,
const char *buf, size_t count);
int index;
-};
+} __do_const;
/* Averaging interval */
static int update_avg_interval(struct acpi_power_meter_resource *resource)
struct sensor_template *attrs)
{
struct device *dev = &resource->acpi_dev->dev;
- struct sensor_device_attribute *sensors =
+ sensor_device_attribute_no_const *sensors =
&resource->sensors[resource->num_sensors];
int res = 0;
{
struct applesmc_node_group *grp;
struct applesmc_dev_attr *node;
- struct attribute *attr;
+ attribute_no_const *attr;
int ret, i;
for (grp = groups; grp->format; grp++) {
struct atk_sensor_data {
struct list_head list;
struct atk_data *data;
- struct device_attribute label_attr;
- struct device_attribute input_attr;
- struct device_attribute limit1_attr;
- struct device_attribute limit2_attr;
+ device_attribute_no_const label_attr;
+ device_attribute_no_const input_attr;
+ device_attribute_no_const limit1_attr;
+ device_attribute_no_const limit2_attr;
char label_attr_name[ATTR_NAME_SIZE];
char input_attr_name[ATTR_NAME_SIZE];
char limit1_attr_name[ATTR_NAME_SIZE];
static struct device_attribute atk_name_attr =
__ATTR(name, 0444, atk_name_show, NULL);
-static void atk_init_attribute(struct device_attribute *attr, char *name,
+static void atk_init_attribute(device_attribute_no_const *attr, char *name,
sysfs_show_func show)
{
sysfs_attr_init(&attr->attr);
return NOTIFY_OK;
}
-static struct notifier_block coretemp_cpu_notifier __refdata = {
+static struct notifier_block coretemp_cpu_notifier = {
.notifier_call = coretemp_cpu_callback,
};
struct aem_rw_sensor_template *rw)
{
struct device *dev = &data->pdev->dev;
- struct sensor_device_attribute *sensors = data->sensors;
+ sensor_device_attribute_no_const *sensors = data->sensors;
int err;
/* Set up read-only sensors */
{
struct device *dev = &pdev->dev;
struct iio_hwmon_state *st;
- struct sensor_device_attribute *a;
+ sensor_device_attribute_no_const *a;
int ret, i;
int in_i = 1, temp_i = 1, curr_i = 1, humidity_i = 1;
enum iio_chan_type type;
nct6683_create_attr_group(struct device *dev, struct sensor_template_group *tg,
int repeat)
{
- struct sensor_device_attribute_2 *a2;
- struct sensor_device_attribute *a;
+ sensor_device_attribute_2_no_const *a2;
+ sensor_device_attribute_no_const *a;
struct sensor_device_template **t;
struct sensor_device_attr_u *su;
- struct attribute_group *group;
+ attribute_group_no_const *group;
struct attribute **attrs;
int i, j, count;
nct6775_create_attr_group(struct device *dev, struct sensor_template_group *tg,
int repeat)
{
- struct attribute_group *group;
+ attribute_group_no_const *group;
struct sensor_device_attr_u *su;
- struct sensor_device_attribute *a;
- struct sensor_device_attribute_2 *a2;
+ sensor_device_attribute_no_const *a;
+ sensor_device_attribute_2_no_const *a2;
struct attribute **attrs;
struct sensor_device_template **t;
int i, count;
return 0;
}
-static void pmbus_dev_attr_init(struct device_attribute *dev_attr,
+static void pmbus_dev_attr_init(device_attribute_no_const *dev_attr,
const char *name,
umode_t mode,
ssize_t (*show)(struct device *dev,
dev_attr->store = store;
}
-static void pmbus_attr_init(struct sensor_device_attribute *a,
+static void pmbus_attr_init(sensor_device_attribute_no_const *a,
const char *name,
umode_t mode,
ssize_t (*show)(struct device *dev,
u16 reg, u8 mask)
{
struct pmbus_boolean *boolean;
- struct sensor_device_attribute *a;
+ sensor_device_attribute_no_const *a;
boolean = devm_kzalloc(data->dev, sizeof(*boolean), GFP_KERNEL);
if (!boolean)
bool update, bool readonly)
{
struct pmbus_sensor *sensor;
- struct device_attribute *a;
+ device_attribute_no_const *a;
sensor = devm_kzalloc(data->dev, sizeof(*sensor), GFP_KERNEL);
if (!sensor)
const char *lstring, int index)
{
struct pmbus_label *label;
- struct device_attribute *a;
+ device_attribute_no_const *a;
label = devm_kzalloc(data->dev, sizeof(*label), GFP_KERNEL);
if (!label)
int supply_uv;
bool supply_uv_valid;
struct work_struct update_supply_work;
- atomic_t interrupt_handled;
+ atomic_unchecked_t interrupt_handled;
};
/**
ret = gpio_direction_input(data->pdata->gpio_data);
if (ret)
return ret;
- atomic_set(&data->interrupt_handled, 0);
+ atomic_set_unchecked(&data->interrupt_handled, 0);
enable_irq(gpio_to_irq(data->pdata->gpio_data));
if (gpio_get_value(data->pdata->gpio_data) == 0) {
disable_irq_nosync(gpio_to_irq(data->pdata->gpio_data));
/* Only relevant if the interrupt hasn't occurred. */
- if (!atomic_read(&data->interrupt_handled))
+ if (!atomic_read_unchecked(&data->interrupt_handled))
schedule_work(&data->read_work);
}
ret = wait_event_timeout(data->wait_queue,
/* First disable the interrupt */
disable_irq_nosync(irq);
- atomic_inc(&data->interrupt_handled);
+ atomic_inc_unchecked(&data->interrupt_handled);
/* Then schedule a reading work struct */
if (data->state != SHT15_READING_NOTHING)
schedule_work(&data->read_work);
* If not, then start the interrupt again - care here as could
* have gone low in meantime so verify it hasn't!
*/
- atomic_set(&data->interrupt_handled, 0);
+ atomic_set_unchecked(&data->interrupt_handled, 0);
enable_irq(gpio_to_irq(data->pdata->gpio_data));
/* If still not occurred or another handler was scheduled */
if (gpio_get_value(data->pdata->gpio_data)
- || atomic_read(&data->interrupt_handled))
+ || atomic_read_unchecked(&data->interrupt_handled))
return;
}
return NOTIFY_OK;
}
-static struct notifier_block via_cputemp_cpu_notifier __refdata = {
+static struct notifier_block via_cputemp_cpu_notifier = {
.notifier_call = via_cputemp_cpu_callback,
};
extern struct i2c_adapter amd756_smbus;
static struct i2c_adapter *s4882_adapter;
-static struct i2c_algorithm *s4882_algo;
+static i2c_algorithm_no_const *s4882_algo;
/* Wrapper access functions for multiplexed SMBus */
static DEFINE_MUTEX(amd756_lock);
/* usb layer */
/* Send command to device, and get response. */
-static int diolan_usb_transfer(struct i2c_diolan_u2c *dev)
+static int __intentional_overflow(-1) diolan_usb_transfer(struct i2c_diolan_u2c *dev)
{
int ret = 0;
int actual;
extern struct i2c_adapter *nforce2_smbus;
static struct i2c_adapter *s4985_adapter;
-static struct i2c_algorithm *s4985_algo;
+static i2c_algorithm_no_const *s4985_algo;
/* Wrapper access functions for multiplexed SMBus */
static DEFINE_MUTEX(nforce2_lock);
break;
}
- data_ptrs[i] = (u8 __user *)rdwr_pa[i].buf;
+ data_ptrs[i] = (u8 __force_user *)rdwr_pa[i].buf;
rdwr_pa[i].buf = memdup_user(data_ptrs[i], rdwr_pa[i].len);
if (IS_ERR(rdwr_pa[i].buf)) {
res = PTR_ERR(rdwr_pa[i].buf);
alignment = queue_dma_alignment(q) | q->dma_pad_mask;
if ((unsigned long)buf & alignment
|| blk_rq_bytes(rq) & q->dma_pad_mask
- || object_is_on_stack(buf))
+ || object_starts_on_stack(buf))
drive->dma = 0;
}
}
}
static
-int __iio_device_attr_init(struct device_attribute *dev_attr,
+int __iio_device_attr_init(device_attribute_no_const *dev_attr,
const char *postfix,
struct iio_chan_spec const *chan,
ssize_t (*readfunc)(struct device *dev,
struct cm_counter_group {
struct kobject obj;
- atomic_long_t counter[CM_ATTR_COUNT];
+ atomic_long_unchecked_t counter[CM_ATTR_COUNT];
};
struct cm_counter_attribute {
struct ib_mad_send_buf *msg = NULL;
int ret;
- atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
+ atomic_long_inc_unchecked(&work->port->counter_group[CM_RECV_DUPLICATES].
counter[CM_REQ_COUNTER]);
/* Quick state check to discard duplicate REQs. */
if (!cm_id_priv)
return;
- atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
+ atomic_long_inc_unchecked(&work->port->counter_group[CM_RECV_DUPLICATES].
counter[CM_REP_COUNTER]);
ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg);
if (ret)
if (cm_id_priv->id.state != IB_CM_REP_SENT &&
cm_id_priv->id.state != IB_CM_MRA_REP_RCVD) {
spin_unlock_irq(&cm_id_priv->lock);
- atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
+ atomic_long_inc_unchecked(&work->port->counter_group[CM_RECV_DUPLICATES].
counter[CM_RTU_COUNTER]);
goto out;
}
cm_id_priv = cm_acquire_id(dreq_msg->remote_comm_id,
dreq_msg->local_comm_id);
if (!cm_id_priv) {
- atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
+ atomic_long_inc_unchecked(&work->port->counter_group[CM_RECV_DUPLICATES].
counter[CM_DREQ_COUNTER]);
cm_issue_drep(work->port, work->mad_recv_wc);
return -EINVAL;
case IB_CM_MRA_REP_RCVD:
break;
case IB_CM_TIMEWAIT:
- atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
+ atomic_long_inc_unchecked(&work->port->counter_group[CM_RECV_DUPLICATES].
counter[CM_DREQ_COUNTER]);
if (cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg))
goto unlock;
cm_free_msg(msg);
goto deref;
case IB_CM_DREQ_RCVD:
- atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
+ atomic_long_inc_unchecked(&work->port->counter_group[CM_RECV_DUPLICATES].
counter[CM_DREQ_COUNTER]);
goto unlock;
default:
ib_modify_mad(cm_id_priv->av.port->mad_agent,
cm_id_priv->msg, timeout)) {
if (cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
- atomic_long_inc(&work->port->
+ atomic_long_inc_unchecked(&work->port->
counter_group[CM_RECV_DUPLICATES].
counter[CM_MRA_COUNTER]);
goto out;
break;
case IB_CM_MRA_REQ_RCVD:
case IB_CM_MRA_REP_RCVD:
- atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
+ atomic_long_inc_unchecked(&work->port->counter_group[CM_RECV_DUPLICATES].
counter[CM_MRA_COUNTER]);
/* fall through */
default:
case IB_CM_LAP_IDLE:
break;
case IB_CM_MRA_LAP_SENT:
- atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
+ atomic_long_inc_unchecked(&work->port->counter_group[CM_RECV_DUPLICATES].
counter[CM_LAP_COUNTER]);
if (cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg))
goto unlock;
cm_free_msg(msg);
goto deref;
case IB_CM_LAP_RCVD:
- atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
+ atomic_long_inc_unchecked(&work->port->counter_group[CM_RECV_DUPLICATES].
counter[CM_LAP_COUNTER]);
goto unlock;
default:
cur_cm_id_priv = cm_insert_remote_sidr(cm_id_priv);
if (cur_cm_id_priv) {
spin_unlock_irq(&cm.lock);
- atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
+ atomic_long_inc_unchecked(&work->port->counter_group[CM_RECV_DUPLICATES].
counter[CM_SIDR_REQ_COUNTER]);
goto out; /* Duplicate message. */
}
if (!msg->context[0] && (attr_index != CM_REJ_COUNTER))
msg->retries = 1;
- atomic_long_add(1 + msg->retries,
+ atomic_long_add_unchecked(1 + msg->retries,
&port->counter_group[CM_XMIT].counter[attr_index]);
if (msg->retries)
- atomic_long_add(msg->retries,
+ atomic_long_add_unchecked(msg->retries,
&port->counter_group[CM_XMIT_RETRIES].
counter[attr_index]);
}
attr_id = be16_to_cpu(mad_recv_wc->recv_buf.mad->mad_hdr.attr_id);
- atomic_long_inc(&port->counter_group[CM_RECV].
+ atomic_long_inc_unchecked(&port->counter_group[CM_RECV].
counter[attr_id - CM_ATTR_ID_OFFSET]);
work = kmalloc(sizeof *work + sizeof(struct ib_sa_path_rec) * paths,
cm_attr = container_of(attr, struct cm_counter_attribute, attr);
return sprintf(buf, "%ld\n",
- atomic_long_read(&group->counter[cm_attr->index]));
+ atomic_long_read_unchecked(&group->counter[cm_attr->index]));
}
static const struct sysfs_ops cm_counter_ops = {
struct task_struct *thread;
- atomic_t req_ser;
- atomic_t flush_ser;
+ atomic_unchecked_t req_ser;
+ atomic_unchecked_t flush_ser;
wait_queue_head_t force_wait;
};
struct ib_fmr_pool *pool = pool_ptr;
do {
- if (atomic_read(&pool->flush_ser) - atomic_read(&pool->req_ser) < 0) {
+ if (atomic_read_unchecked(&pool->flush_ser) - atomic_read_unchecked(&pool->req_ser) < 0) {
ib_fmr_batch_release(pool);
- atomic_inc(&pool->flush_ser);
+ atomic_inc_unchecked(&pool->flush_ser);
wake_up_interruptible(&pool->force_wait);
if (pool->flush_function)
}
set_current_state(TASK_INTERRUPTIBLE);
- if (atomic_read(&pool->flush_ser) - atomic_read(&pool->req_ser) >= 0 &&
+ if (atomic_read_unchecked(&pool->flush_ser) - atomic_read_unchecked(&pool->req_ser) >= 0 &&
!kthread_should_stop())
schedule();
__set_current_state(TASK_RUNNING);
pool->dirty_watermark = params->dirty_watermark;
pool->dirty_len = 0;
spin_lock_init(&pool->pool_lock);
- atomic_set(&pool->req_ser, 0);
- atomic_set(&pool->flush_ser, 0);
+ atomic_set_unchecked(&pool->req_ser, 0);
+ atomic_set_unchecked(&pool->flush_ser, 0);
init_waitqueue_head(&pool->force_wait);
pool->thread = kthread_run(ib_fmr_cleanup_thread,
}
spin_unlock_irq(&pool->pool_lock);
- serial = atomic_inc_return(&pool->req_ser);
+ serial = atomic_inc_return_unchecked(&pool->req_ser);
wake_up_process(pool->thread);
if (wait_event_interruptible(pool->force_wait,
- atomic_read(&pool->flush_ser) - serial >= 0))
+ atomic_read_unchecked(&pool->flush_ser) - serial >= 0))
return -EINTR;
return 0;
} else {
list_add_tail(&fmr->list, &pool->dirty_list);
if (++pool->dirty_len >= pool->dirty_watermark) {
- atomic_inc(&pool->req_ser);
+ atomic_inc_unchecked(&pool->req_ser);
wake_up_process(pool->thread);
}
}
if (dmasync)
dma_set_attr(DMA_ATTR_WRITE_BARRIER, &attrs);
+ /*
+ * If the combination of the addr and size requested for this memory
+ * region causes an integer overflow, return error.
+ */
+ if ((PAGE_ALIGN(addr + size) <= size) ||
+ (PAGE_ALIGN(addr + size) <= addr))
+ return ERR_PTR(-EINVAL);
+
if (!can_do_mlock())
return ERR_PTR(-EPERM);
int err;
struct fw_ri_tpte tpt;
u32 stag_idx;
- static atomic_t key;
+ static atomic_unchecked_t key;
if (c4iw_fatal_error(rdev))
return -EIO;
if (rdev->stats.stag.cur > rdev->stats.stag.max)
rdev->stats.stag.max = rdev->stats.stag.cur;
mutex_unlock(&rdev->stats.lock);
- *stag = (stag_idx << 8) | (atomic_inc_return(&key) & 0xff);
+ *stag = (stag_idx << 8) | (atomic_inc_return_unchecked(&key) & 0xff);
}
PDBG("%s stag_state 0x%0x type 0x%0x pdid 0x%0x, stag_idx 0x%x\n",
__func__, stag_state, type, pdid, stag_idx);
struct ib_atomic_eth *ateth;
struct ipath_ack_entry *e;
u64 vaddr;
- atomic64_t *maddr;
+ atomic64_unchecked_t *maddr;
u64 sdata;
u32 rkey;
u8 next;
IB_ACCESS_REMOTE_ATOMIC)))
goto nack_acc_unlck;
/* Perform atomic OP and save result. */
- maddr = (atomic64_t *) qp->r_sge.sge.vaddr;
+ maddr = (atomic64_unchecked_t *) qp->r_sge.sge.vaddr;
sdata = be64_to_cpu(ateth->swap_data);
e = &qp->s_ack_queue[qp->r_head_ack_queue];
e->atomic_data = (opcode == OP(FETCH_ADD)) ?
- (u64) atomic64_add_return(sdata, maddr) - sdata :
+ (u64) atomic64_add_return_unchecked(sdata, maddr) - sdata :
(u64) cmpxchg((u64 *) qp->r_sge.sge.vaddr,
be64_to_cpu(ateth->compare_data),
sdata);
unsigned long flags;
struct ib_wc wc;
u64 sdata;
- atomic64_t *maddr;
+ atomic64_unchecked_t *maddr;
enum ib_wc_status send_status;
/*
IB_ACCESS_REMOTE_ATOMIC)))
goto acc_err;
/* Perform atomic OP and save result. */
- maddr = (atomic64_t *) qp->r_sge.sge.vaddr;
+ maddr = (atomic64_unchecked_t *) qp->r_sge.sge.vaddr;
sdata = wqe->wr.wr.atomic.compare_add;
*(u64 *) sqp->s_sge.sge.vaddr =
(wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) ?
- (u64) atomic64_add_return(sdata, maddr) - sdata :
+ (u64) atomic64_add_return_unchecked(sdata, maddr) - sdata :
(u64) cmpxchg((u64 *) qp->r_sge.sge.vaddr,
sdata, wqe->wr.wr.atomic.swap);
goto send_comp;
__be64 mlx4_ib_get_new_demux_tid(struct mlx4_ib_demux_ctx *ctx)
{
- return cpu_to_be64(atomic_inc_return(&ctx->tid)) |
+ return cpu_to_be64(atomic_inc_return_unchecked(&ctx->tid)) |
cpu_to_be64(0xff00000000000000LL);
}
{
char name[20];
- atomic_set(&ctx->tid, 0);
+ atomic_set_unchecked(&ctx->tid, 0);
sprintf(name, "mlx4_ib_mcg%d", ctx->port);
ctx->mcg_wq = create_singlethread_workqueue(name);
if (!ctx->mcg_wq)
struct list_head mcg_mgid0_list;
struct workqueue_struct *mcg_wq;
struct mlx4_ib_demux_pv_ctx **tun;
- atomic_t tid;
+ atomic_unchecked_t tid;
int flushing; /* flushing the work queue */
};
mthca_dbg(dev, "Mapped doorbell page for posting FW commands\n");
}
-int mthca_QUERY_FW(struct mthca_dev *dev)
+int __intentional_overflow(-1) mthca_QUERY_FW(struct mthca_dev *dev)
{
struct mthca_mailbox *mailbox;
u32 *outbox;
CMD_TIME_CLASS_B);
}
-int mthca_WRITE_MTT(struct mthca_dev *dev, struct mthca_mailbox *mailbox,
+int __intentional_overflow(-1) mthca_WRITE_MTT(struct mthca_dev *dev, struct mthca_mailbox *mailbox,
int num_mtt)
{
return mthca_cmd(dev, mailbox->dma, num_mtt, 0, CMD_WRITE_MTT,
0, CMD_MAP_EQ, CMD_TIME_CLASS_B);
}
-int mthca_SW2HW_EQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox,
+int __intentional_overflow(-1) mthca_SW2HW_EQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox,
int eq_num)
{
return mthca_cmd(dev, mailbox->dma, eq_num, 0, CMD_SW2HW_EQ,
CMD_TIME_CLASS_B);
}
-int mthca_MAD_IFC(struct mthca_dev *dev, int ignore_mkey, int ignore_bkey,
+int __intentional_overflow(-1) mthca_MAD_IFC(struct mthca_dev *dev, int ignore_mkey, int ignore_bkey,
int port, struct ib_wc *in_wc, struct ib_grh *in_grh,
void *in_mad, void *response_mad)
{
return err;
}
-static int mthca_setup_hca(struct mthca_dev *dev)
+static int __intentional_overflow(-1) mthca_setup_hca(struct mthca_dev *dev)
{
int err;
* through the bitmaps)
*/
-static u32 mthca_buddy_alloc(struct mthca_buddy *buddy, int order)
+static u32 __intentional_overflow(-1) mthca_buddy_alloc(struct mthca_buddy *buddy, int order)
{
int o;
int m;
return key;
}
-int mthca_mr_alloc(struct mthca_dev *dev, u32 pd, int buffer_size_shift,
+int __intentional_overflow(-1) mthca_mr_alloc(struct mthca_dev *dev, u32 pd, int buffer_size_shift,
u64 iova, u64 total_size, u32 access, struct mthca_mr *mr)
{
struct mthca_mailbox *mailbox;
return mthca_mr_alloc(dev, pd, 12, 0, ~0ULL, access, mr);
}
-int mthca_mr_alloc_phys(struct mthca_dev *dev, u32 pd,
+int __intentional_overflow(-1) mthca_mr_alloc_phys(struct mthca_dev *dev, u32 pd,
u64 *buffer_list, int buffer_size_shift,
int list_len, u64 iova, u64 total_size,
u32 access, struct mthca_mr *mr)
return 0;
}
-static int mthca_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata)
+static int __intentional_overflow(-1) mthca_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata)
{
struct mthca_dev *dev = to_mdev(ibcq->device);
struct mthca_cq *cq = to_mcq(ibcq);
LIST_HEAD(nes_adapter_list);
static LIST_HEAD(nes_dev_list);
-atomic_t qps_destroyed;
+atomic_unchecked_t qps_destroyed;
static unsigned int ee_flsh_adapter;
static unsigned int sysfs_nonidx_addr;
struct nes_qp *nesqp = cqp_request->cqp_callback_pointer;
struct nes_adapter *nesadapter = nesdev->nesadapter;
- atomic_inc(&qps_destroyed);
+ atomic_inc_unchecked(&qps_destroyed);
/* Free the control structures */
extern unsigned int wqm_quanta;
extern struct list_head nes_adapter_list;
-extern atomic_t cm_connects;
-extern atomic_t cm_accepts;
-extern atomic_t cm_disconnects;
-extern atomic_t cm_closes;
-extern atomic_t cm_connecteds;
-extern atomic_t cm_connect_reqs;
-extern atomic_t cm_rejects;
-extern atomic_t mod_qp_timouts;
-extern atomic_t qps_created;
-extern atomic_t qps_destroyed;
-extern atomic_t sw_qps_destroyed;
+extern atomic_unchecked_t cm_connects;
+extern atomic_unchecked_t cm_accepts;
+extern atomic_unchecked_t cm_disconnects;
+extern atomic_unchecked_t cm_closes;
+extern atomic_unchecked_t cm_connecteds;
+extern atomic_unchecked_t cm_connect_reqs;
+extern atomic_unchecked_t cm_rejects;
+extern atomic_unchecked_t mod_qp_timouts;
+extern atomic_unchecked_t qps_created;
+extern atomic_unchecked_t qps_destroyed;
+extern atomic_unchecked_t sw_qps_destroyed;
extern u32 mh_detected;
extern u32 mh_pauses_sent;
extern u32 cm_packets_sent;
extern u32 cm_packets_received;
extern u32 cm_packets_dropped;
extern u32 cm_packets_retrans;
-extern atomic_t cm_listens_created;
-extern atomic_t cm_listens_destroyed;
+extern atomic_unchecked_t cm_listens_created;
+extern atomic_unchecked_t cm_listens_destroyed;
extern u32 cm_backlog_drops;
-extern atomic_t cm_loopbacks;
-extern atomic_t cm_nodes_created;
-extern atomic_t cm_nodes_destroyed;
-extern atomic_t cm_accel_dropped_pkts;
-extern atomic_t cm_resets_recvd;
-extern atomic_t pau_qps_created;
-extern atomic_t pau_qps_destroyed;
+extern atomic_unchecked_t cm_loopbacks;
+extern atomic_unchecked_t cm_nodes_created;
+extern atomic_unchecked_t cm_nodes_destroyed;
+extern atomic_unchecked_t cm_accel_dropped_pkts;
+extern atomic_unchecked_t cm_resets_recvd;
+extern atomic_unchecked_t pau_qps_created;
+extern atomic_unchecked_t pau_qps_destroyed;
extern u32 int_mod_timer_init;
extern u32 int_mod_cq_depth_256;
u32 cm_packets_retrans;
u32 cm_packets_created;
u32 cm_packets_received;
-atomic_t cm_listens_created;
-atomic_t cm_listens_destroyed;
+atomic_unchecked_t cm_listens_created;
+atomic_unchecked_t cm_listens_destroyed;
u32 cm_backlog_drops;
-atomic_t cm_loopbacks;
-atomic_t cm_nodes_created;
-atomic_t cm_nodes_destroyed;
-atomic_t cm_accel_dropped_pkts;
-atomic_t cm_resets_recvd;
+atomic_unchecked_t cm_loopbacks;
+atomic_unchecked_t cm_nodes_created;
+atomic_unchecked_t cm_nodes_destroyed;
+atomic_unchecked_t cm_accel_dropped_pkts;
+atomic_unchecked_t cm_resets_recvd;
static inline int mini_cm_accelerated(struct nes_cm_core *, struct nes_cm_node *);
static struct nes_cm_listener *mini_cm_listen(struct nes_cm_core *, struct nes_vnic *, struct nes_cm_info *);
/* instance of function pointers for client API */
/* set address of this instance to cm_core->cm_ops at cm_core alloc */
static struct nes_cm_ops nes_cm_api = {
- mini_cm_accelerated,
- mini_cm_listen,
- mini_cm_del_listen,
- mini_cm_connect,
- mini_cm_close,
- mini_cm_accept,
- mini_cm_reject,
- mini_cm_recv_pkt,
- mini_cm_dealloc_core,
- mini_cm_get,
- mini_cm_set
+ .accelerated = mini_cm_accelerated,
+ .listen = mini_cm_listen,
+ .stop_listener = mini_cm_del_listen,
+ .connect = mini_cm_connect,
+ .close = mini_cm_close,
+ .accept = mini_cm_accept,
+ .reject = mini_cm_reject,
+ .recv_pkt = mini_cm_recv_pkt,
+ .destroy_cm_core = mini_cm_dealloc_core,
+ .get = mini_cm_get,
+ .set = mini_cm_set
};
static struct nes_cm_core *g_cm_core;
-atomic_t cm_connects;
-atomic_t cm_accepts;
-atomic_t cm_disconnects;
-atomic_t cm_closes;
-atomic_t cm_connecteds;
-atomic_t cm_connect_reqs;
-atomic_t cm_rejects;
+atomic_unchecked_t cm_connects;
+atomic_unchecked_t cm_accepts;
+atomic_unchecked_t cm_disconnects;
+atomic_unchecked_t cm_closes;
+atomic_unchecked_t cm_connecteds;
+atomic_unchecked_t cm_connect_reqs;
+atomic_unchecked_t cm_rejects;
int nes_add_ref_cm_node(struct nes_cm_node *cm_node)
{
kfree(listener);
listener = NULL;
ret = 0;
- atomic_inc(&cm_listens_destroyed);
+ atomic_inc_unchecked(&cm_listens_destroyed);
} else {
spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
}
cm_node->rem_mac);
add_hte_node(cm_core, cm_node);
- atomic_inc(&cm_nodes_created);
+ atomic_inc_unchecked(&cm_nodes_created);
return cm_node;
}
}
atomic_dec(&cm_core->node_cnt);
- atomic_inc(&cm_nodes_destroyed);
+ atomic_inc_unchecked(&cm_nodes_destroyed);
nesqp = cm_node->nesqp;
if (nesqp) {
nesqp->cm_node = NULL;
static void drop_packet(struct sk_buff *skb)
{
- atomic_inc(&cm_accel_dropped_pkts);
+ atomic_inc_unchecked(&cm_accel_dropped_pkts);
dev_kfree_skb_any(skb);
}
{
int reset = 0; /* whether to send reset in case of err.. */
- atomic_inc(&cm_resets_recvd);
+ atomic_inc_unchecked(&cm_resets_recvd);
nes_debug(NES_DBG_CM, "Received Reset, cm_node = %p, state = %u."
" refcnt=%d\n", cm_node, cm_node->state,
atomic_read(&cm_node->ref_count));
rem_ref_cm_node(cm_node->cm_core, cm_node);
return NULL;
}
- atomic_inc(&cm_loopbacks);
+ atomic_inc_unchecked(&cm_loopbacks);
loopbackremotenode->loopbackpartner = cm_node;
loopbackremotenode->tcp_cntxt.rcv_wscale =
NES_CM_DEFAULT_RCV_WND_SCALE;
nes_queue_mgt_skbs(skb, nesvnic, cm_node->nesqp);
else {
rem_ref_cm_node(cm_core, cm_node);
- atomic_inc(&cm_accel_dropped_pkts);
+ atomic_inc_unchecked(&cm_accel_dropped_pkts);
dev_kfree_skb_any(skb);
}
break;
if ((cm_id) && (cm_id->event_handler)) {
if (issue_disconn) {
- atomic_inc(&cm_disconnects);
+ atomic_inc_unchecked(&cm_disconnects);
cm_event.event = IW_CM_EVENT_DISCONNECT;
cm_event.status = disconn_status;
cm_event.local_addr = cm_id->local_addr;
}
if (issue_close) {
- atomic_inc(&cm_closes);
+ atomic_inc_unchecked(&cm_closes);
nes_disconnect(nesqp, 1);
cm_id->provider_data = nesqp;
nes_debug(NES_DBG_CM, "QP%u, cm_node=%p, jiffies = %lu listener = %p\n",
nesqp->hwqp.qp_id, cm_node, jiffies, cm_node->listener);
- atomic_inc(&cm_accepts);
+ atomic_inc_unchecked(&cm_accepts);
nes_debug(NES_DBG_CM, "netdev refcnt = %u.\n",
netdev_refcnt_read(nesvnic->netdev));
struct nes_cm_core *cm_core;
u8 *start_buff;
- atomic_inc(&cm_rejects);
+ atomic_inc_unchecked(&cm_rejects);
cm_node = (struct nes_cm_node *)cm_id->provider_data;
loopback = cm_node->loopbackpartner;
cm_core = cm_node->cm_core;
ntohs(raddr->sin_port), ntohl(laddr->sin_addr.s_addr),
ntohs(laddr->sin_port));
- atomic_inc(&cm_connects);
+ atomic_inc_unchecked(&cm_connects);
nesqp->active_conn = 1;
/* cache the cm_id in the qp */
g_cm_core->api->stop_listener(g_cm_core, (void *)cm_node);
return err;
}
- atomic_inc(&cm_listens_created);
+ atomic_inc_unchecked(&cm_listens_created);
}
cm_id->add_ref(cm_id);
if (nesqp->destroyed)
return;
- atomic_inc(&cm_connecteds);
+ atomic_inc_unchecked(&cm_connecteds);
nes_debug(NES_DBG_CM, "QP%u attempting to connect to 0x%08X:0x%04X on"
" local port 0x%04X. jiffies = %lu.\n",
nesqp->hwqp.qp_id, ntohl(raddr->sin_addr.s_addr),
cm_id->add_ref(cm_id);
ret = cm_id->event_handler(cm_id, &cm_event);
- atomic_inc(&cm_closes);
+ atomic_inc_unchecked(&cm_closes);
cm_event.event = IW_CM_EVENT_CLOSE;
cm_event.status = 0;
cm_event.provider_data = cm_id->provider_data;
return;
cm_id = cm_node->cm_id;
- atomic_inc(&cm_connect_reqs);
+ atomic_inc_unchecked(&cm_connect_reqs);
nes_debug(NES_DBG_CM, "cm_node = %p - cm_id = %p, jiffies = %lu\n",
cm_node, cm_id, jiffies);
return;
cm_id = cm_node->cm_id;
- atomic_inc(&cm_connect_reqs);
+ atomic_inc_unchecked(&cm_connect_reqs);
nes_debug(NES_DBG_CM, "cm_node = %p - cm_id = %p, jiffies = %lu\n",
cm_node, cm_id, jiffies);
#include "nes.h"
#include "nes_mgt.h"
-atomic_t pau_qps_created;
-atomic_t pau_qps_destroyed;
+atomic_unchecked_t pau_qps_created;
+atomic_unchecked_t pau_qps_destroyed;
static void nes_replenish_mgt_rq(struct nes_vnic_mgt *mgtvnic)
{
{
struct sk_buff *skb;
unsigned long flags;
- atomic_inc(&pau_qps_destroyed);
+ atomic_inc_unchecked(&pau_qps_destroyed);
/* Free packets that have not yet been forwarded */
/* Lock is acquired by skb_dequeue when removing the skb */
cq->cq_vbase[head].cqe_words[NES_NIC_CQE_HASH_RCVNXT]);
skb_queue_head_init(&nesqp->pau_list);
spin_lock_init(&nesqp->pau_lock);
- atomic_inc(&pau_qps_created);
+ atomic_inc_unchecked(&pau_qps_created);
nes_change_quad_hash(nesdev, mgtvnic->nesvnic, nesqp);
}
target_stat_values[++index] = mh_detected;
target_stat_values[++index] = mh_pauses_sent;
target_stat_values[++index] = nesvnic->endnode_ipv4_tcp_retransmits;
- target_stat_values[++index] = atomic_read(&cm_connects);
- target_stat_values[++index] = atomic_read(&cm_accepts);
- target_stat_values[++index] = atomic_read(&cm_disconnects);
- target_stat_values[++index] = atomic_read(&cm_connecteds);
- target_stat_values[++index] = atomic_read(&cm_connect_reqs);
- target_stat_values[++index] = atomic_read(&cm_rejects);
- target_stat_values[++index] = atomic_read(&mod_qp_timouts);
- target_stat_values[++index] = atomic_read(&qps_created);
- target_stat_values[++index] = atomic_read(&sw_qps_destroyed);
- target_stat_values[++index] = atomic_read(&qps_destroyed);
- target_stat_values[++index] = atomic_read(&cm_closes);
+ target_stat_values[++index] = atomic_read_unchecked(&cm_connects);
+ target_stat_values[++index] = atomic_read_unchecked(&cm_accepts);
+ target_stat_values[++index] = atomic_read_unchecked(&cm_disconnects);
+ target_stat_values[++index] = atomic_read_unchecked(&cm_connecteds);
+ target_stat_values[++index] = atomic_read_unchecked(&cm_connect_reqs);
+ target_stat_values[++index] = atomic_read_unchecked(&cm_rejects);
+ target_stat_values[++index] = atomic_read_unchecked(&mod_qp_timouts);
+ target_stat_values[++index] = atomic_read_unchecked(&qps_created);
+ target_stat_values[++index] = atomic_read_unchecked(&sw_qps_destroyed);
+ target_stat_values[++index] = atomic_read_unchecked(&qps_destroyed);
+ target_stat_values[++index] = atomic_read_unchecked(&cm_closes);
target_stat_values[++index] = cm_packets_sent;
target_stat_values[++index] = cm_packets_bounced;
target_stat_values[++index] = cm_packets_created;
target_stat_values[++index] = cm_packets_received;
target_stat_values[++index] = cm_packets_dropped;
target_stat_values[++index] = cm_packets_retrans;
- target_stat_values[++index] = atomic_read(&cm_listens_created);
- target_stat_values[++index] = atomic_read(&cm_listens_destroyed);
+ target_stat_values[++index] = atomic_read_unchecked(&cm_listens_created);
+ target_stat_values[++index] = atomic_read_unchecked(&cm_listens_destroyed);
target_stat_values[++index] = cm_backlog_drops;
- target_stat_values[++index] = atomic_read(&cm_loopbacks);
- target_stat_values[++index] = atomic_read(&cm_nodes_created);
- target_stat_values[++index] = atomic_read(&cm_nodes_destroyed);
- target_stat_values[++index] = atomic_read(&cm_accel_dropped_pkts);
- target_stat_values[++index] = atomic_read(&cm_resets_recvd);
+ target_stat_values[++index] = atomic_read_unchecked(&cm_loopbacks);
+ target_stat_values[++index] = atomic_read_unchecked(&cm_nodes_created);
+ target_stat_values[++index] = atomic_read_unchecked(&cm_nodes_destroyed);
+ target_stat_values[++index] = atomic_read_unchecked(&cm_accel_dropped_pkts);
+ target_stat_values[++index] = atomic_read_unchecked(&cm_resets_recvd);
target_stat_values[++index] = nesadapter->free_4kpbl;
target_stat_values[++index] = nesadapter->free_256pbl;
target_stat_values[++index] = int_mod_timer_init;
target_stat_values[++index] = nesvnic->lro_mgr.stats.aggregated;
target_stat_values[++index] = nesvnic->lro_mgr.stats.flushed;
target_stat_values[++index] = nesvnic->lro_mgr.stats.no_desc;
- target_stat_values[++index] = atomic_read(&pau_qps_created);
- target_stat_values[++index] = atomic_read(&pau_qps_destroyed);
+ target_stat_values[++index] = atomic_read_unchecked(&pau_qps_created);
+ target_stat_values[++index] = atomic_read_unchecked(&pau_qps_destroyed);
}
/**
#include <rdma/ib_umem.h>
-atomic_t mod_qp_timouts;
-atomic_t qps_created;
-atomic_t sw_qps_destroyed;
+atomic_unchecked_t mod_qp_timouts;
+atomic_unchecked_t qps_created;
+atomic_unchecked_t sw_qps_destroyed;
static void nes_unregister_ofa_device(struct nes_ib_device *nesibdev);
if (init_attr->create_flags)
return ERR_PTR(-EINVAL);
- atomic_inc(&qps_created);
+ atomic_inc_unchecked(&qps_created);
switch (init_attr->qp_type) {
case IB_QPT_RC:
if (nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) {
struct iw_cm_event cm_event;
int ret = 0;
- atomic_inc(&sw_qps_destroyed);
+ atomic_inc_unchecked(&sw_qps_destroyed);
nesqp->destroyed = 1;
/* Blow away the connection if it exists. */
#include <linux/kref.h>
#include <linux/sched.h>
#include <linux/kthread.h>
+#include <linux/slab.h>
#include "qib_common.h"
#include "qib_verbs.h"
nla_total_size(2); /* IFLA_IPOIB_UMCAST */
}
-static struct rtnl_link_ops ipoib_link_ops __read_mostly = {
+static struct rtnl_link_ops ipoib_link_ops = {
.kind = "ipoib",
.maxtype = IFLA_IPOIB_MAX,
.policy = ipoib_policy,
*/
static void gameport_init_port(struct gameport *gameport)
{
- static atomic_t gameport_no = ATOMIC_INIT(-1);
+ static atomic_unchecked_t gameport_no = ATOMIC_INIT(-1);
__module_get(THIS_MODULE);
mutex_init(&gameport->drv_mutex);
device_initialize(&gameport->dev);
dev_set_name(&gameport->dev, "gameport%lu",
- (unsigned long)atomic_inc_return(&gameport_no));
+ (unsigned long)atomic_inc_return_unchecked(&gameport_no));
gameport->dev.bus = &gameport_bus;
gameport->dev.release = gameport_release_port;
if (gameport->parent)
*/
struct input_dev *input_allocate_device(void)
{
- static atomic_t input_no = ATOMIC_INIT(-1);
+ static atomic_unchecked_t input_no = ATOMIC_INIT(-1);
struct input_dev *dev;
dev = kzalloc(sizeof(struct input_dev), GFP_KERNEL);
INIT_LIST_HEAD(&dev->node);
dev_set_name(&dev->dev, "input%lu",
- (unsigned long)atomic_inc_return(&input_no));
+ (unsigned long)atomic_inc_return_unchecked(&input_no));
__module_get(THIS_MODULE);
}
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/slab.h>
+#include <linux/sched.h>
#include <linux/input.h>
#include <linux/gameport.h>
#include <linux/jiffies.h>
static int xpad_led_probe(struct usb_xpad *xpad)
{
- static atomic_t led_seq = ATOMIC_INIT(-1);
+ static atomic_unchecked_t led_seq = ATOMIC_INIT(-1);
unsigned long led_no;
struct xpad_led *led;
struct led_classdev *led_cdev;
if (!led)
return -ENOMEM;
- led_no = atomic_inc_return(&led_seq);
+ led_no = atomic_inc_return_unchecked(&led_seq);
snprintf(led->name, sizeof(led->name), "xpad%lu", led_no);
led->xpad = xpad;
static int ims_pcu_init_application_mode(struct ims_pcu *pcu)
{
- static atomic_t device_no = ATOMIC_INIT(-1);
+ static atomic_unchecked_t device_no = ATOMIC_INIT(-1);
const struct ims_pcu_device_info *info;
int error;
}
/* Device appears to be operable, complete initialization */
- pcu->device_no = atomic_inc_return(&device_no);
+ pcu->device_no = atomic_inc_return_unchecked(&device_no);
/*
* PCU-B devices, both GEN_1 and GEN_2 do not have OFN sensor
ssize_t (*set)(struct psmouse *psmouse, void *data,
const char *buf, size_t count);
bool protect;
-};
+} __do_const;
#define to_psmouse_attr(a) container_of((a), struct psmouse_attribute, dattr)
ssize_t psmouse_attr_show_helper(struct device *dev, struct device_attribute *attr,
spin_unlock_irq(&client->packet_lock);
- if (copy_to_user(buffer, data, count))
+ if (count > sizeof(data) || copy_to_user(buffer, data, count))
return -EFAULT;
return count;
*/
static void serio_init_port(struct serio *serio)
{
- static atomic_t serio_no = ATOMIC_INIT(-1);
+ static atomic_unchecked_t serio_no = ATOMIC_INIT(-1);
__module_get(THIS_MODULE);
mutex_init(&serio->drv_mutex);
device_initialize(&serio->dev);
dev_set_name(&serio->dev, "serio%lu",
- (unsigned long)atomic_inc_return(&serio_no));
+ (unsigned long)atomic_inc_return_unchecked(&serio_no));
serio->dev.bus = &serio_bus;
serio->dev.release = serio_release_port;
serio->dev.groups = serio_device_attr_groups;
static int serio_raw_connect(struct serio *serio, struct serio_driver *drv)
{
- static atomic_t serio_raw_no = ATOMIC_INIT(-1);
+ static atomic_unchecked_t serio_raw_no = ATOMIC_INIT(-1);
struct serio_raw *serio_raw;
int err;
}
snprintf(serio_raw->name, sizeof(serio_raw->name),
- "serio_raw%ld", (long)atomic_inc_return(&serio_raw_no));
+ "serio_raw%ld", (long)atomic_inc_return_unchecked(&serio_raw_no));
kref_init(&serio_raw->kref);
INIT_LIST_HEAD(&serio_raw->client_list);
init_waitqueue_head(&serio_raw->wait);
static void build_completion_wait(struct iommu_cmd *cmd, u64 address)
{
+ phys_addr_t physaddr;
WARN_ON(address & 0x7ULL);
memset(cmd, 0, sizeof(*cmd));
- cmd->data[0] = lower_32_bits(__pa(address)) | CMD_COMPL_WAIT_STORE_MASK;
- cmd->data[1] = upper_32_bits(__pa(address));
+
+#ifdef CONFIG_GRKERNSEC_KSTACKOVERFLOW
+ if (object_starts_on_stack((void *)address)) {
+ void *adjbuf = (void *)address - current->stack + current->lowmem_stack;
+ physaddr = __pa((u64)adjbuf);
+ } else
+#endif
+ physaddr = __pa(address);
+
+ cmd->data[0] = lower_32_bits(physaddr) | CMD_COMPL_WAIT_STORE_MASK;
+ cmd->data[1] = upper_32_bits(physaddr);
cmd->data[2] = 1;
CMD_SET_TYPE(cmd, CMD_COMPL_WAIT);
}
cfg->irptndx = cfg->cbndx;
}
- ACCESS_ONCE(smmu_domain->smmu) = smmu;
+ ACCESS_ONCE_RW(smmu_domain->smmu) = smmu;
arm_smmu_init_context_bank(smmu_domain);
spin_unlock_irqrestore(&smmu_domain->lock, flags);
static int iommu_bus_init(struct bus_type *bus, const struct iommu_ops *ops)
{
int err;
- struct notifier_block *nb;
+ notifier_block_no_const *nb;
struct iommu_callback_data cb = {
.ops = ops,
};
void panic_if_irq_remap(const char *msg)
{
if (irq_remapping_enabled)
- panic(msg);
+ panic("%s", msg);
}
static void ir_ack_apic_edge(struct irq_data *data)
void irq_remap_modify_chip_defaults(struct irq_chip *chip)
{
- chip->irq_print_chip = ir_print_prefix;
- chip->irq_ack = ir_ack_apic_edge;
- chip->irq_eoi = ir_ack_apic_level;
- chip->irq_set_affinity = x86_io_apic_ops.set_affinity;
+ pax_open_kernel();
+ *(void **)&chip->irq_print_chip = ir_print_prefix;
+ *(void **)&chip->irq_ack = ir_ack_apic_edge;
+ *(void **)&chip->irq_eoi = ir_ack_apic_level;
+ *(void **)&chip->irq_set_affinity = x86_io_apic_ops.set_affinity;
+ pax_close_kernel();
}
bool setup_remapped_irq(int irq, struct irq_cfg *cfg, struct irq_chip *chip)
* Supported arch specific GIC irq extension.
* Default make them NULL.
*/
-struct irq_chip gic_arch_extn = {
+irq_chip_no_const gic_arch_extn = {
.irq_eoi = NULL,
.irq_mask = NULL,
.irq_unmask = NULL,
chained_irq_exit(chip, desc);
}
-static struct irq_chip gic_chip = {
+static irq_chip_no_const gic_chip __read_only = {
.name = "GIC",
.irq_mask = gic_mask_irq,
.irq_unmask = gic_unmask_irq,
struct intc_irqpin_iomem *i;
struct resource *io[INTC_IRQPIN_REG_NR];
struct resource *irq;
- struct irq_chip *irq_chip;
+ irq_chip_no_const *irq_chip;
void (*enable_fn)(struct irq_data *d);
void (*disable_fn)(struct irq_data *d);
const char *name = dev_name(dev);
struct irqc_priv *p;
struct resource *io;
struct resource *irq;
- struct irq_chip *irq_chip;
+ irq_chip_no_const *irq_chip;
const char *name = dev_name(&pdev->dev);
int ret;
int k;
struct capi20_appl *ap;
u32 ncci;
- atomic_t datahandle;
- atomic_t msgid;
+ atomic_unchecked_t datahandle;
+ atomic_unchecked_t msgid;
struct tty_port port;
int ttyinstop;
capimsg_setu16(s, 2, mp->ap->applid);
capimsg_setu8 (s, 4, CAPI_DATA_B3);
capimsg_setu8 (s, 5, CAPI_RESP);
- capimsg_setu16(s, 6, atomic_inc_return(&mp->msgid));
+ capimsg_setu16(s, 6, atomic_inc_return_unchecked(&mp->msgid));
capimsg_setu32(s, 8, mp->ncci);
capimsg_setu16(s, 12, datahandle);
}
mp->outbytes -= len;
spin_unlock_bh(&mp->outlock);
- datahandle = atomic_inc_return(&mp->datahandle);
+ datahandle = atomic_inc_return_unchecked(&mp->datahandle);
skb_push(skb, CAPI_DATA_B3_REQ_LEN);
memset(skb->data, 0, CAPI_DATA_B3_REQ_LEN);
capimsg_setu16(skb->data, 0, CAPI_DATA_B3_REQ_LEN);
capimsg_setu16(skb->data, 2, mp->ap->applid);
capimsg_setu8 (skb->data, 4, CAPI_DATA_B3);
capimsg_setu8 (skb->data, 5, CAPI_REQ);
- capimsg_setu16(skb->data, 6, atomic_inc_return(&mp->msgid));
+ capimsg_setu16(skb->data, 6, atomic_inc_return_unchecked(&mp->msgid));
capimsg_setu32(skb->data, 8, mp->ncci); /* NCCI */
capimsg_setu32(skb->data, 12, (u32)(long)skb->data);/* Data32 */
capimsg_setu16(skb->data, 16, len); /* Data length */
static const struct gigaset_ops gigops = {
- gigaset_write_cmd,
- gigaset_write_room,
- gigaset_chars_in_buffer,
- gigaset_brkchars,
- gigaset_init_bchannel,
- gigaset_close_bchannel,
- gigaset_initbcshw,
- gigaset_freebcshw,
- gigaset_reinitbcshw,
- gigaset_initcshw,
- gigaset_freecshw,
- gigaset_set_modem_ctrl,
- gigaset_baud_rate,
- gigaset_set_line_ctrl,
- gigaset_isoc_send_skb,
- gigaset_isoc_input,
+ .write_cmd = gigaset_write_cmd,
+ .write_room = gigaset_write_room,
+ .chars_in_buffer = gigaset_chars_in_buffer,
+ .brkchars = gigaset_brkchars,
+ .init_bchannel = gigaset_init_bchannel,
+ .close_bchannel = gigaset_close_bchannel,
+ .initbcshw = gigaset_initbcshw,
+ .freebcshw = gigaset_freebcshw,
+ .reinitbcshw = gigaset_reinitbcshw,
+ .initcshw = gigaset_initcshw,
+ .freecshw = gigaset_freecshw,
+ .set_modem_ctrl = gigaset_set_modem_ctrl,
+ .baud_rate = gigaset_baud_rate,
+ .set_line_ctrl = gigaset_set_line_ctrl,
+ .send_skb = gigaset_isoc_send_skb,
+ .handle_input = gigaset_isoc_input,
};
/* bas_gigaset_init
}
tty->driver_data = cs;
- ++cs->port.count;
+ atomic_inc(&cs->port.count);
- if (cs->port.count == 1) {
+ if (atomic_read(&cs->port.count) == 1) {
tty_port_tty_set(&cs->port, tty);
cs->port.low_latency = 1;
}
if (!cs->connected)
gig_dbg(DEBUG_IF, "not connected"); /* nothing to do */
- else if (!cs->port.count)
+ else if (!atomic_read(&cs->port.count))
dev_warn(cs->dev, "%s: device not opened\n", __func__);
- else if (!--cs->port.count)
+ else if (!atomic_dec_return(&cs->port.count))
tty_port_tty_set(&cs->port, NULL);
mutex_unlock(&cs->mutex);
}
static const struct gigaset_ops ops = {
- gigaset_write_cmd,
- gigaset_write_room,
- gigaset_chars_in_buffer,
- gigaset_brkchars,
- gigaset_init_bchannel,
- gigaset_close_bchannel,
- gigaset_initbcshw,
- gigaset_freebcshw,
- gigaset_reinitbcshw,
- gigaset_initcshw,
- gigaset_freecshw,
- gigaset_set_modem_ctrl,
- gigaset_baud_rate,
- gigaset_set_line_ctrl,
- gigaset_m10x_send_skb, /* asyncdata.c */
- gigaset_m10x_input, /* asyncdata.c */
+ .write_cmd = gigaset_write_cmd,
+ .write_room = gigaset_write_room,
+ .chars_in_buffer = gigaset_chars_in_buffer,
+ .brkchars = gigaset_brkchars,
+ .init_bchannel = gigaset_init_bchannel,
+ .close_bchannel = gigaset_close_bchannel,
+ .initbcshw = gigaset_initbcshw,
+ .freebcshw = gigaset_freebcshw,
+ .reinitbcshw = gigaset_reinitbcshw,
+ .initcshw = gigaset_initcshw,
+ .freecshw = gigaset_freecshw,
+ .set_modem_ctrl = gigaset_set_modem_ctrl,
+ .baud_rate = gigaset_baud_rate,
+ .set_line_ctrl = gigaset_set_line_ctrl,
+ .send_skb = gigaset_m10x_send_skb, /* asyncdata.c */
+ .handle_input = gigaset_m10x_input, /* asyncdata.c */
};
gigaset_dbg_buffer(DEBUG_USBREQ, "brkchars", 6, buf);
memcpy(cs->hw.usb->bchars, buf, 6);
return usb_control_msg(udev, usb_sndctrlpipe(udev, 0), 0x19, 0x41,
- 0, 0, &buf, 6, 2000);
+ 0, 0, buf, 6, 2000);
}
static void gigaset_freebcshw(struct bc_state *bcs)
}
static const struct gigaset_ops ops = {
- gigaset_write_cmd,
- gigaset_write_room,
- gigaset_chars_in_buffer,
- gigaset_brkchars,
- gigaset_init_bchannel,
- gigaset_close_bchannel,
- gigaset_initbcshw,
- gigaset_freebcshw,
- gigaset_reinitbcshw,
- gigaset_initcshw,
- gigaset_freecshw,
- gigaset_set_modem_ctrl,
- gigaset_baud_rate,
- gigaset_set_line_ctrl,
- gigaset_m10x_send_skb,
- gigaset_m10x_input,
+ .write_cmd = gigaset_write_cmd,
+ .write_room = gigaset_write_room,
+ .chars_in_buffer = gigaset_chars_in_buffer,
+ .brkchars = gigaset_brkchars,
+ .init_bchannel = gigaset_init_bchannel,
+ .close_bchannel = gigaset_close_bchannel,
+ .initbcshw = gigaset_initbcshw,
+ .freebcshw = gigaset_freebcshw,
+ .reinitbcshw = gigaset_reinitbcshw,
+ .initcshw = gigaset_initcshw,
+ .freecshw = gigaset_freecshw,
+ .set_modem_ctrl = gigaset_set_modem_ctrl,
+ .baud_rate = gigaset_baud_rate,
+ .set_line_ctrl = gigaset_set_line_ctrl,
+ .send_skb = gigaset_m10x_send_skb,
+ .handle_input = gigaset_m10x_input,
};
/*
}
if (left) {
if (t4file->user) {
- if (copy_from_user(buf, dp, left))
+ if (left > sizeof buf || copy_from_user(buf, dp, left))
return -EFAULT;
} else {
memcpy(buf, dp, left);
}
if (left) {
if (config->user) {
- if (copy_from_user(buf, dp, left))
+ if (left > sizeof buf || copy_from_user(buf, dp, left))
return -EFAULT;
} else {
memcpy(buf, dp, left);
} else
return -EINVAL;
case IIOCDBGVAR:
+ if (!capable(CAP_SYS_RAWIO))
+ return -EPERM;
if (arg) {
if (copy_to_user(argp, &dev, sizeof(ulong)))
return -EFAULT;
}
struct concap_device_ops isdn_concap_reliable_dl_dops = {
- &isdn_concap_dl_data_req,
- &isdn_concap_dl_connect_req,
- &isdn_concap_dl_disconn_req
+ .data_req = &isdn_concap_dl_data_req,
+ .connect_req = &isdn_concap_dl_connect_req,
+ .disconn_req = &isdn_concap_dl_disconn_req
};
/* The following should better go into a dedicated source file such that
#ifdef ISDN_DEBUG_MODEM_OPEN
printk(KERN_DEBUG "isdn_tty_open %s, count = %d\n", tty->name,
- port->count);
+ atomic_read(&port->count));
#endif
- port->count++;
+ atomic_inc(&port->count);
port->tty = tty;
/*
* Start up serial port
#endif
return;
}
- if ((tty->count == 1) && (port->count != 1)) {
+ if ((tty->count == 1) && (atomic_read(&port->count) != 1)) {
/*
* Uh, oh. tty->count is 1, which means that the tty
* structure will be freed. Info->count should always
* serial port won't be shutdown.
*/
printk(KERN_ERR "isdn_tty_close: bad port count; tty->count is 1, "
- "info->count is %d\n", port->count);
- port->count = 1;
+ "info->count is %d\n", atomic_read(&port->count));
+ atomic_set(&port->count, 1);
}
- if (--port->count < 0) {
+ if (atomic_dec_return(&port->count) < 0) {
printk(KERN_ERR "isdn_tty_close: bad port count for ttyi%d: %d\n",
- info->line, port->count);
- port->count = 0;
+ info->line, atomic_read(&port->count));
+ atomic_set(&port->count, 0);
}
- if (port->count) {
+ if (atomic_read(&port->count)) {
#ifdef ISDN_DEBUG_MODEM_OPEN
printk(KERN_DEBUG "isdn_tty_close after info->count != 0\n");
#endif
if (isdn_tty_paranoia_check(info, tty->name, "isdn_tty_hangup"))
return;
isdn_tty_shutdown(info);
- port->count = 0;
+ atomic_set(&port->count, 0);
port->flags &= ~ASYNC_NORMAL_ACTIVE;
port->tty = NULL;
wake_up_interruptible(&port->open_wait);
for (i = 0; i < ISDN_MAX_CHANNELS; i++) {
modem_info *info = &dev->mdm.info[i];
- if (info->port.count == 0)
+ if (atomic_read(&info->port.count) == 0)
continue;
if ((info->emu.mdmreg[REG_SI1] & si2bit[si1]) && /* SI1 is matching */
(info->emu.mdmreg[REG_SI2] == si2)) { /* SI2 is matching */
static struct concap_proto_ops ix25_pops = {
- &isdn_x25iface_proto_new,
- &isdn_x25iface_proto_del,
- &isdn_x25iface_proto_restart,
- &isdn_x25iface_proto_close,
- &isdn_x25iface_xmit,
- &isdn_x25iface_receive,
- &isdn_x25iface_connect_ind,
- &isdn_x25iface_disconn_ind
+ .proto_new = &isdn_x25iface_proto_new,
+ .proto_del = &isdn_x25iface_proto_del,
+ .restart = &isdn_x25iface_proto_restart,
+ .close = &isdn_x25iface_proto_close,
+ .encap_and_xmit = &isdn_x25iface_xmit,
+ .data_ind = &isdn_x25iface_receive,
+ .connect_ind = &isdn_x25iface_connect_ind,
+ .disconn_ind = &isdn_x25iface_disconn_ind
};
/* error message helper function */
if (count > len)
count = len;
if (user) {
- if (copy_from_user(msg, buf, count))
+ if (count > sizeof msg || copy_from_user(msg, buf, count))
return -EFAULT;
} else
memcpy(msg, buf, count);
if (ints[0] > 1)
membase = (unsigned long)ints[2];
if (str && *str) {
- strcpy(sid, str);
+ strlcpy(sid, str, sizeof(sid));
icn_id = sid;
if ((p = strchr(sid, ','))) {
*p++ = 0;
static u16 dsp_count; /* last sample count */
static int dsp_count_valid; /* if we have last sample count */
-void
+void __intentional_overflow(-1)
dsp_cmx_send(void *arg)
{
struct dsp_conf *conf;
* detected as working, but in reality it is not) as low as
* possible.
*/
-static struct dmi_system_id clevo_mail_led_dmi_table[] __initdata = {
+static struct dmi_system_id clevo_mail_led_dmi_table[] __initconst = {
{
.callback = clevo_mail_led_dmi_callback,
.ident = "Clevo D410J",
* detected as working, but in reality it is not) as low as
* possible.
*/
-static struct dmi_system_id nas_led_whitelist[] __initdata = {
+static struct dmi_system_id nas_led_whitelist[] __initconst = {
{
.callback = ss4200_led_dmi_callback,
.ident = "Intel SS4200-E",
* The end address needs +1 because __get_vm_area allocates an
* extra guard page, so we need space for that.
*/
+
+#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_KERNEXEC)
+ switcher_vma = __get_vm_area(TOTAL_SWITCHER_PAGES * PAGE_SIZE,
+ VM_ALLOC | VM_KERNEXEC, switcher_addr, switcher_addr
+ + (TOTAL_SWITCHER_PAGES+1) * PAGE_SIZE);
+#else
switcher_vma = __get_vm_area(TOTAL_SWITCHER_PAGES * PAGE_SIZE,
VM_ALLOC, switcher_addr, switcher_addr
+ (TOTAL_SWITCHER_PAGES+1) * PAGE_SIZE);
+#endif
+
if (!switcher_vma) {
err = -ENOMEM;
printk("lguest: could not map switcher pages high\n");
* Now the Switcher is mapped at the right address, we can't fail!
* Copy in the compiled-in Switcher code (from x86/switcher_32.S).
*/
- memcpy(switcher_vma->addr, start_switcher_text,
+ memcpy(switcher_vma->addr, ktla_ktva(start_switcher_text),
end_switcher_text - start_switcher_text);
printk(KERN_INFO "lguest: mapped switcher at %p\n",
/*:*/
#ifdef CONFIG_X86_PAE
-static void release_pmd(pmd_t *spmd)
+static void __intentional_overflow(-1) release_pmd(pmd_t *spmd)
{
/* If the entry's not present, there's nothing to release. */
if (pmd_flags(*spmd) & _PAGE_PRESENT) {
/* Offset from where switcher.S was compiled to where we've copied it */
static unsigned long switcher_offset(void)
{
- return switcher_addr - (unsigned long)start_switcher_text;
+ return switcher_addr - (unsigned long)ktla_ktva(start_switcher_text);
}
/* This cpu's struct lguest_pages (after the Switcher text page) */
* These copies are pretty cheap, so we do them unconditionally: */
/* Save the current Host top-level page directory.
*/
+
+#ifdef CONFIG_PAX_PER_CPU_PGD
+ pages->state.host_cr3 = read_cr3();
+#else
pages->state.host_cr3 = __pa(current->mm->pgd);
+#endif
+
/*
* Set up the Guest's page tables to see this CPU's pages (and no
* other CPU's pages).
* compiled-in switcher code and the high-mapped copy we just made.
*/
for (i = 0; i < IDT_ENTRIES; i++)
- default_idt_entries[i] += switcher_offset();
+ default_idt_entries[i] = ktla_ktva(default_idt_entries[i]) + switcher_offset();
/*
* Set up the Switcher's per-cpu areas.
* it will be undisturbed when we switch. To change %cs and jump we
* need this structure to feed to Intel's "lcall" instruction.
*/
- lguest_entry.offset = (long)switch_to_guest + switcher_offset();
+ lguest_entry.offset = (long)ktla_ktva(switch_to_guest) + switcher_offset();
lguest_entry.segment = LGUEST_CS;
/*
#include <asm/page.h>
#include <asm/segment.h>
#include <asm/lguest.h>
+#include <asm/processor-flags.h>
// We mark the start of the code to copy
// It's placed in .text tho it's never run here
// Changes type when we load it: damn Intel!
// For after we switch over our page tables
// That entry will be read-only: we'd crash.
+
+#ifdef CONFIG_PAX_KERNEXEC
+ mov %cr0, %edx
+ xor $X86_CR0_WP, %edx
+ mov %edx, %cr0
+#endif
+
movl $(GDT_ENTRY_TSS*8), %edx
ltr %dx
// Let's clear it again for our return.
// The GDT descriptor of the Host
// Points to the table after two "size" bytes
- movl (LGUEST_PAGES_host_gdt_desc+2)(%eax), %edx
+ movl (LGUEST_PAGES_host_gdt_desc+2)(%eax), %eax
// Clear "used" from type field (byte 5, bit 2)
- andb $0xFD, (GDT_ENTRY_TSS*8 + 5)(%edx)
+ andb $0xFD, (GDT_ENTRY_TSS*8 + 5)(%eax)
+
+#ifdef CONFIG_PAX_KERNEXEC
+ mov %cr0, %eax
+ xor $X86_CR0_WP, %eax
+ mov %eax, %cr0
+#endif
// Once our page table's switched, the Guest is live!
// The Host fades as we run this final step.
// I consulted gcc, and it gave
// These instructions, which I gladly credit:
leal (%edx,%ebx,8), %eax
- movzwl (%eax),%edx
- movl 4(%eax), %eax
- xorw %ax, %ax
- orl %eax, %edx
+ movl 4(%eax), %edx
+ movw (%eax), %dx
// Now the address of the handler's in %edx
// We call it now: its "iret" drops us home.
- jmp *%edx
+ ljmp $__KERNEL_CS, $1f
+1: jmp *%edx
// Every interrupt can come to us here
// But we must truly tell each apart.
static inline void set_closure_fn(struct closure *cl, closure_fn *fn,
struct workqueue_struct *wq)
{
- BUG_ON(object_is_on_stack(cl));
+ BUG_ON(object_starts_on_stack(cl));
closure_set_ip(cl);
cl->fn = fn;
cl->wq = wq;
chunk_kb ? "KB" : "B");
if (bitmap->storage.file) {
seq_printf(seq, ", file: ");
- seq_path(seq, &bitmap->storage.file->f_path, " \t\n");
+ seq_path(seq, &bitmap->storage.file->f_path, " \t\n\\");
}
seq_printf(seq, "\n");
cmd == DM_LIST_VERSIONS_CMD)
return 0;
- if ((cmd == DM_DEV_CREATE_CMD)) {
+ if (cmd == DM_DEV_CREATE_CMD) {
if (!*param->name) {
DMWARN("name not supplied when creating device");
return -EINVAL;
struct mirror {
struct mirror_set *ms;
- atomic_t error_count;
+ atomic_unchecked_t error_count;
unsigned long error_type;
struct dm_dev *dev;
sector_t offset;
struct mirror *m;
for (m = ms->mirror; m < ms->mirror + ms->nr_mirrors; m++)
- if (!atomic_read(&m->error_count))
+ if (!atomic_read_unchecked(&m->error_count))
return m;
return NULL;
* simple way to tell if a device has encountered
* errors.
*/
- atomic_inc(&m->error_count);
+ atomic_inc_unchecked(&m->error_count);
if (test_and_set_bit(error_type, &m->error_type))
return;
struct mirror *m = get_default_mirror(ms);
do {
- if (likely(!atomic_read(&m->error_count)))
+ if (likely(!atomic_read_unchecked(&m->error_count)))
return m;
if (m-- == ms->mirror)
{
struct mirror *default_mirror = get_default_mirror(m->ms);
- return !atomic_read(&default_mirror->error_count);
+ return !atomic_read_unchecked(&default_mirror->error_count);
}
static int mirror_available(struct mirror_set *ms, struct bio *bio)
*/
if (likely(region_in_sync(ms, region, 1)))
m = choose_mirror(ms, bio->bi_iter.bi_sector);
- else if (m && atomic_read(&m->error_count))
+ else if (m && atomic_read_unchecked(&m->error_count))
m = NULL;
if (likely(m))
}
ms->mirror[mirror].ms = ms;
- atomic_set(&(ms->mirror[mirror].error_count), 0);
+ atomic_set_unchecked(&(ms->mirror[mirror].error_count), 0);
ms->mirror[mirror].error_type = 0;
ms->mirror[mirror].offset = offset;
*/
static char device_status_char(struct mirror *m)
{
- if (!atomic_read(&(m->error_count)))
+ if (!atomic_read_unchecked(&(m->error_count)))
return 'A';
return (test_bit(DM_RAID1_FLUSH_ERROR, &(m->error_type))) ? 'F' :
synchronize_rcu_expedited();
dm_stat_free(&s->rcu_head);
} else {
- ACCESS_ONCE(dm_stat_need_rcu_barrier) = 1;
+ ACCESS_ONCE_RW(dm_stat_need_rcu_barrier) = 1;
call_rcu(&s->rcu_head, dm_stat_free);
}
return 0;
((bi_rw & (REQ_WRITE | REQ_DISCARD)) ==
(ACCESS_ONCE(last->last_rw) & (REQ_WRITE | REQ_DISCARD)))
));
- ACCESS_ONCE(last->last_sector) = end_sector;
- ACCESS_ONCE(last->last_rw) = bi_rw;
+ ACCESS_ONCE_RW(last->last_sector) = end_sector;
+ ACCESS_ONCE_RW(last->last_rw) = bi_rw;
}
rcu_read_lock();
struct dm_dev *dev;
sector_t physical_start;
- atomic_t error_count;
+ atomic_unchecked_t error_count;
};
struct stripe_c {
kfree(sc);
return r;
}
- atomic_set(&(sc->stripe[i].error_count), 0);
+ atomic_set_unchecked(&(sc->stripe[i].error_count), 0);
}
ti->private = sc;
DMEMIT("%d ", sc->stripes);
for (i = 0; i < sc->stripes; i++) {
DMEMIT("%s ", sc->stripe[i].dev->name);
- buffer[i] = atomic_read(&(sc->stripe[i].error_count)) ?
+ buffer[i] = atomic_read_unchecked(&(sc->stripe[i].error_count)) ?
'D' : 'A';
}
buffer[i] = '\0';
*/
for (i = 0; i < sc->stripes; i++)
if (!strcmp(sc->stripe[i].dev->name, major_minor)) {
- atomic_inc(&(sc->stripe[i].error_count));
- if (atomic_read(&(sc->stripe[i].error_count)) <
+ atomic_inc_unchecked(&(sc->stripe[i].error_count));
+ if (atomic_read_unchecked(&(sc->stripe[i].error_count)) <
DM_IO_ERROR_THRESHOLD)
schedule_work(&sc->trigger_event);
}
if (!dev_size)
return 0;
- if ((start >= dev_size) || (start + len > dev_size)) {
+ if ((start >= dev_size) || (len > dev_size - start)) {
DMWARN("%s: %s too small for target: "
"start=%llu, len=%llu, dev_size=%llu",
dm_device_name(ti->table->md), bdevname(bdev, b),
{
pmd->info.tm = pmd->tm;
pmd->info.levels = 2;
- pmd->info.value_type.context = pmd->data_sm;
+ pmd->info.value_type.context = (dm_space_map_no_const *)pmd->data_sm;
pmd->info.value_type.size = sizeof(__le64);
pmd->info.value_type.inc = data_block_inc;
pmd->info.value_type.dec = data_block_dec;
pmd->bl_info.tm = pmd->tm;
pmd->bl_info.levels = 1;
- pmd->bl_info.value_type.context = pmd->data_sm;
+ pmd->bl_info.value_type.context = (dm_space_map_no_const *)pmd->data_sm;
pmd->bl_info.value_type.size = sizeof(__le64);
pmd->bl_info.value_type.inc = data_block_inc;
pmd->bl_info.value_type.dec = data_block_dec;
/*
* Event handling.
*/
- atomic_t event_nr;
+ atomic_unchecked_t event_nr;
wait_queue_head_t eventq;
- atomic_t uevent_seq;
+ atomic_unchecked_t uevent_seq;
struct list_head uevent_list;
spinlock_t uevent_lock; /* Protect access to uevent_list */
spin_lock_init(&md->deferred_lock);
atomic_set(&md->holders, 1);
atomic_set(&md->open_count, 0);
- atomic_set(&md->event_nr, 0);
- atomic_set(&md->uevent_seq, 0);
+ atomic_set_unchecked(&md->event_nr, 0);
+ atomic_set_unchecked(&md->uevent_seq, 0);
INIT_LIST_HEAD(&md->uevent_list);
INIT_LIST_HEAD(&md->table_devices);
spin_lock_init(&md->uevent_lock);
dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
- atomic_inc(&md->event_nr);
+ atomic_inc_unchecked(&md->event_nr);
wake_up(&md->eventq);
}
uint32_t dm_next_uevent_seq(struct mapped_device *md)
{
- return atomic_add_return(1, &md->uevent_seq);
+ return atomic_add_return_unchecked(1, &md->uevent_seq);
}
uint32_t dm_get_event_nr(struct mapped_device *md)
{
- return atomic_read(&md->event_nr);
+ return atomic_read_unchecked(&md->event_nr);
}
int dm_wait_event(struct mapped_device *md, int event_nr)
{
return wait_event_interruptible(md->eventq,
- (event_nr != atomic_read(&md->event_nr)));
+ (event_nr != atomic_read_unchecked(&md->event_nr)));
}
void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
* start build, activate spare
*/
static DECLARE_WAIT_QUEUE_HEAD(md_event_waiters);
-static atomic_t md_event_count;
+static atomic_unchecked_t md_event_count;
void md_new_event(struct mddev *mddev)
{
- atomic_inc(&md_event_count);
+ atomic_inc_unchecked(&md_event_count);
wake_up(&md_event_waiters);
}
EXPORT_SYMBOL_GPL(md_new_event);
*/
static void md_new_event_inintr(struct mddev *mddev)
{
- atomic_inc(&md_event_count);
+ atomic_inc_unchecked(&md_event_count);
wake_up(&md_event_waiters);
}
if ((le32_to_cpu(sb->feature_map) & MD_FEATURE_RESHAPE_ACTIVE) &&
(le32_to_cpu(sb->feature_map) & MD_FEATURE_NEW_OFFSET))
rdev->new_data_offset += (s32)le32_to_cpu(sb->new_offset);
- atomic_set(&rdev->corrected_errors, le32_to_cpu(sb->cnt_corrected_read));
+ atomic_set_unchecked(&rdev->corrected_errors, le32_to_cpu(sb->cnt_corrected_read));
rdev->sb_size = le32_to_cpu(sb->max_dev) * 2 + 256;
bmask = queue_logical_block_size(rdev->bdev->bd_disk->queue)-1;
else
sb->resync_offset = cpu_to_le64(0);
- sb->cnt_corrected_read = cpu_to_le32(atomic_read(&rdev->corrected_errors));
+ sb->cnt_corrected_read = cpu_to_le32(atomic_read_unchecked(&rdev->corrected_errors));
sb->raid_disks = cpu_to_le32(mddev->raid_disks);
sb->size = cpu_to_le64(mddev->dev_sectors);
static ssize_t
errors_show(struct md_rdev *rdev, char *page)
{
- return sprintf(page, "%d\n", atomic_read(&rdev->corrected_errors));
+ return sprintf(page, "%d\n", atomic_read_unchecked(&rdev->corrected_errors));
}
static ssize_t
char *e;
unsigned long n = simple_strtoul(buf, &e, 10);
if (*buf && (*e == 0 || *e == '\n')) {
- atomic_set(&rdev->corrected_errors, n);
+ atomic_set_unchecked(&rdev->corrected_errors, n);
return len;
}
return -EINVAL;
rdev->sb_loaded = 0;
rdev->bb_page = NULL;
atomic_set(&rdev->nr_pending, 0);
- atomic_set(&rdev->read_errors, 0);
- atomic_set(&rdev->corrected_errors, 0);
+ atomic_set_unchecked(&rdev->read_errors, 0);
+ atomic_set_unchecked(&rdev->corrected_errors, 0);
INIT_LIST_HEAD(&rdev->same_set);
init_waitqueue_head(&rdev->blocked_wait);
spin_unlock(&pers_lock);
seq_printf(seq, "\n");
- seq->poll_event = atomic_read(&md_event_count);
+ seq->poll_event = atomic_read_unchecked(&md_event_count);
return 0;
}
if (v == (void*)2) {
return error;
seq = file->private_data;
- seq->poll_event = atomic_read(&md_event_count);
+ seq->poll_event = atomic_read_unchecked(&md_event_count);
return error;
}
/* always allow read */
mask = POLLIN | POLLRDNORM;
- if (seq->poll_event != atomic_read(&md_event_count))
+ if (seq->poll_event != atomic_read_unchecked(&md_event_count))
mask |= POLLERR | POLLPRI;
return mask;
}
struct gendisk *disk = rdev->bdev->bd_contains->bd_disk;
curr_events = (int)part_stat_read(&disk->part0, sectors[0]) +
(int)part_stat_read(&disk->part0, sectors[1]) -
- atomic_read(&disk->sync_io);
+ atomic_read_unchecked(&disk->sync_io);
/* sync IO will cause sync_io to increase before the disk_stats
* as sync_io is counted when a request starts, and
* disk_stats is counted when it completes.
* only maintained for arrays that
* support hot removal
*/
- atomic_t read_errors; /* number of consecutive read errors that
+ atomic_unchecked_t read_errors; /* number of consecutive read errors that
* we have tried to ignore.
*/
struct timespec last_read_error; /* monotonic time since our
* last read error
*/
- atomic_t corrected_errors; /* number of corrected read errors,
+ atomic_unchecked_t corrected_errors; /* number of corrected read errors,
* for reporting to userspace and storing
* in superblock.
*/
static inline void md_sync_acct(struct block_device *bdev, unsigned long nr_sectors)
{
- atomic_add(nr_sectors, &bdev->bd_contains->bd_disk->sync_io);
+ atomic_add_unchecked(nr_sectors, &bdev->bd_contains->bd_disk->sync_io);
}
struct md_personality
* Flick into a mode where all blocks get allocated in the new area.
*/
smm->begin = old_len;
- memcpy(sm, &bootstrap_ops, sizeof(*sm));
+ memcpy((void *)sm, &bootstrap_ops, sizeof(*sm));
/*
* Extend.
/*
* Switch back to normal behaviour.
*/
- memcpy(sm, &ops, sizeof(*sm));
+ memcpy((void *)sm, &ops, sizeof(*sm));
return r;
}
dm_sm_threshold_fn fn,
void *context);
};
+typedef struct dm_space_map __no_const dm_space_map_no_const;
/*----------------------------------------------------------------*/
if (r1_sync_page_io(rdev, sect, s,
bio->bi_io_vec[idx].bv_page,
READ) != 0)
- atomic_add(s, &rdev->corrected_errors);
+ atomic_add_unchecked(s, &rdev->corrected_errors);
}
sectors -= s;
sect += s;
!test_bit(Faulty, &rdev->flags)) {
if (r1_sync_page_io(rdev, sect, s,
conf->tmppage, READ)) {
- atomic_add(s, &rdev->corrected_errors);
+ atomic_add_unchecked(s, &rdev->corrected_errors);
printk(KERN_INFO
"md/raid1:%s: read error corrected "
"(%d sectors at %llu on %s)\n",
/* The write handler will notice the lack of
* R10BIO_Uptodate and record any errors etc
*/
- atomic_add(r10_bio->sectors,
+ atomic_add_unchecked(r10_bio->sectors,
&conf->mirrors[d].rdev->corrected_errors);
/* for reconstruct, we always reschedule after a read.
{
struct timespec cur_time_mon;
unsigned long hours_since_last;
- unsigned int read_errors = atomic_read(&rdev->read_errors);
+ unsigned int read_errors = atomic_read_unchecked(&rdev->read_errors);
ktime_get_ts(&cur_time_mon);
* overflowing the shift of read_errors by hours_since_last.
*/
if (hours_since_last >= 8 * sizeof(read_errors))
- atomic_set(&rdev->read_errors, 0);
+ atomic_set_unchecked(&rdev->read_errors, 0);
else
- atomic_set(&rdev->read_errors, read_errors >> hours_since_last);
+ atomic_set_unchecked(&rdev->read_errors, read_errors >> hours_since_last);
}
static int r10_sync_page_io(struct md_rdev *rdev, sector_t sector,
return;
check_decay_read_errors(mddev, rdev);
- atomic_inc(&rdev->read_errors);
- if (atomic_read(&rdev->read_errors) > max_read_errors) {
+ atomic_inc_unchecked(&rdev->read_errors);
+ if (atomic_read_unchecked(&rdev->read_errors) > max_read_errors) {
char b[BDEVNAME_SIZE];
bdevname(rdev->bdev, b);
"md/raid10:%s: %s: Raid device exceeded "
"read_error threshold [cur %d:max %d]\n",
mdname(mddev), b,
- atomic_read(&rdev->read_errors), max_read_errors);
+ atomic_read_unchecked(&rdev->read_errors), max_read_errors);
printk(KERN_NOTICE
"md/raid10:%s: %s: Failing raid device\n",
mdname(mddev), b);
sect +
choose_data_offset(r10_bio, rdev)),
bdevname(rdev->bdev, b));
- atomic_add(s, &rdev->corrected_errors);
+ atomic_add_unchecked(s, &rdev->corrected_errors);
}
rdev_dec_pending(rdev, mddev);
return 1;
}
+#ifdef CONFIG_GRKERNSEC_HIDESYM
+static atomic_unchecked_t raid5_cache_id = ATOMIC_INIT(0);
+#endif
+
static int grow_stripes(struct r5conf *conf, int num)
{
struct kmem_cache *sc;
"raid%d-%s", conf->level, mdname(conf->mddev));
else
sprintf(conf->cache_name[0],
+#ifdef CONFIG_GRKERNSEC_HIDESYM
+ "raid%d-%08lx", conf->level, atomic_inc_return_unchecked(&raid5_cache_id));
+#else
"raid%d-%p", conf->level, conf->mddev);
+#endif
sprintf(conf->cache_name[1], "%s-alt", conf->cache_name[0]);
conf->active_name = 0;
mdname(conf->mddev), STRIPE_SECTORS,
(unsigned long long)s,
bdevname(rdev->bdev, b));
- atomic_add(STRIPE_SECTORS, &rdev->corrected_errors);
+ atomic_add_unchecked(STRIPE_SECTORS, &rdev->corrected_errors);
clear_bit(R5_ReadError, &sh->dev[i].flags);
clear_bit(R5_ReWrite, &sh->dev[i].flags);
} else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);
- if (atomic_read(&rdev->read_errors))
- atomic_set(&rdev->read_errors, 0);
+ if (atomic_read_unchecked(&rdev->read_errors))
+ atomic_set_unchecked(&rdev->read_errors, 0);
} else {
const char *bdn = bdevname(rdev->bdev, b);
int retry = 0;
int set_bad = 0;
clear_bit(R5_UPTODATE, &sh->dev[i].flags);
- atomic_inc(&rdev->read_errors);
+ atomic_inc_unchecked(&rdev->read_errors);
if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
printk_ratelimited(
KERN_WARNING
mdname(conf->mddev),
(unsigned long long)s,
bdn);
- } else if (atomic_read(&rdev->read_errors)
+ } else if (atomic_read_unchecked(&rdev->read_errors)
> conf->max_nr_stripes)
printk(KERN_WARNING
"md/raid:%s: Too many read errors, failing device %s.\n",
const struct dvb_device *template, void *priv, int type)
{
struct dvb_device *dvbdev;
- struct file_operations *dvbdevfops;
+ file_operations_no_const *dvbdevfops;
struct device *clsdev;
int minor;
int id;
int (*pid_filter_ctrl)(struct dvb_frontend *fe, int onoff);
int (*pid_filter)(struct dvb_frontend *fe, int index, u16 pid,
int onoff);
-};
+} __no_const;
#endif /* AF9033_H */
int (*fifo_ctrl)(struct dvb_frontend *fe, int onoff);
int (*pid_ctrl)(struct dvb_frontend *fe, int index, int pid, int onoff);
int (*tuner_pass_ctrl)(struct dvb_frontend *fe, int onoff, u8 pll_ctrl);
-};
+} __no_const;
#if IS_ENABLED(CONFIG_DVB_DIB3000MB)
extern struct dvb_frontend* dib3000mb_attach(const struct dib3000_config* config,
int (*get_adc_power)(struct dvb_frontend *fe);
int (*slave_reset)(struct dvb_frontend *fe);
struct dvb_frontend *(*init)(struct i2c_adapter *i2c_adap, u8 i2c_addr, struct dib7000p_config *cfg);
-};
+} __no_const;
#if IS_ENABLED(CONFIG_DVB_DIB7000P)
void *dib7000p_attach(struct dib7000p_ops *ops);
int (*pid_filter_ctrl)(struct dvb_frontend *fe, u8 onoff);
int (*pid_filter)(struct dvb_frontend *fe, u8 id, u16 pid, u8 onoff);
struct dvb_frontend *(*init)(struct i2c_adapter *i2c_adap, u8 i2c_addr, struct dib8000_config *cfg);
-};
+} __no_const;
#if IS_ENABLED(CONFIG_DVB_DIB8000)
void *dib8000_attach(struct dib8000_ops *ops);
/* ------------------------------------------------------------------ */
-static unsigned int video_nr[] = {[0 ... (CX88_MAXBOARDS - 1)] = UNSET };
-static unsigned int vbi_nr[] = {[0 ... (CX88_MAXBOARDS - 1)] = UNSET };
-static unsigned int radio_nr[] = {[0 ... (CX88_MAXBOARDS - 1)] = UNSET };
+static int video_nr[] = {[0 ... (CX88_MAXBOARDS - 1)] = UNSET };
+static int vbi_nr[] = {[0 ... (CX88_MAXBOARDS - 1)] = UNSET };
+static int radio_nr[] = {[0 ... (CX88_MAXBOARDS - 1)] = UNSET };
module_param_array(video_nr, int, NULL, 0444);
module_param_array(vbi_nr, int, NULL, 0444);
MODULE_DEVICE_TABLE(pci,ivtv_pci_tbl);
/* ivtv instance counter */
-static atomic_t ivtv_instance = ATOMIC_INIT(0);
+static atomic_unchecked_t ivtv_instance = ATOMIC_INIT(0);
/* Parameter declarations */
static int cardtype[IVTV_MAX_CARDS];
static int solo_sysfs_init(struct solo_dev *solo_dev)
{
- struct bin_attribute *sdram_attr = &solo_dev->sdram_attr;
+ bin_attribute_no_const *sdram_attr = &solo_dev->sdram_attr;
struct device *dev = &solo_dev->dev;
const char *driver;
int i;
int solo_g723_init(struct solo_dev *solo_dev)
{
- static struct snd_device_ops ops = { NULL };
+ static struct snd_device_ops ops = { };
struct snd_card *card;
struct snd_kcontrol_new kctl;
char name[32];
/* Get next ID. According to Softlogic, 6110 has problems on !=0 P2M */
if (solo_dev->type != SOLO_DEV_6110 && multi_p2m) {
- p2m_id = atomic_inc_return(&solo_dev->p2m_count) % SOLO_NR_P2M;
+ p2m_id = atomic_inc_return_unchecked(&solo_dev->p2m_count) % SOLO_NR_P2M;
if (p2m_id < 0)
p2m_id = -p2m_id;
}
/* P2M DMA Engine */
struct solo_p2m_dev p2m_dev[SOLO_NR_P2M];
- atomic_t p2m_count;
+ atomic_unchecked_t p2m_count;
int p2m_jiffies;
unsigned int p2m_timeouts;
module_param_array(card, int, NULL, 0444);
MODULE_PARM_DESC(card, "card type");
-static atomic_t tw68_instance = ATOMIC_INIT(0);
+static atomic_unchecked_t tw68_instance = ATOMIC_INIT(0);
/* ------------------------------------------------------------------ */
OMAP_VIDEO2,
};
-static struct videobuf_queue_ops video_vbq_ops;
/* Variables configurable through module params*/
static u32 video1_numbuffers = 3;
static u32 video2_numbuffers = 3;
{
struct videobuf_queue *q;
struct omap_vout_device *vout = NULL;
+ static struct videobuf_queue_ops video_vbq_ops = {
+ .buf_setup = omap_vout_buffer_setup,
+ .buf_prepare = omap_vout_buffer_prepare,
+ .buf_release = omap_vout_buffer_release,
+ .buf_queue = omap_vout_buffer_queue,
+ };
vout = video_drvdata(file);
v4l2_dbg(1, debug, &vout->vid_dev->v4l2_dev, "Entering %s\n", __func__);
vout->type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
q = &vout->vbq;
- video_vbq_ops.buf_setup = omap_vout_buffer_setup;
- video_vbq_ops.buf_prepare = omap_vout_buffer_prepare;
- video_vbq_ops.buf_release = omap_vout_buffer_release;
- video_vbq_ops.buf_queue = omap_vout_buffer_queue;
spin_lock_init(&vout->vbq_lock);
videobuf_queue_dma_contig_init(q, &video_vbq_ops, q->dev,
/** layer index (unique identifier) */
int idx;
/** callbacks for layer methods */
- struct mxr_layer_ops ops;
+ struct mxr_layer_ops *ops;
/** format array */
const struct mxr_format **fmt_array;
/** size of format array */
{
struct mxr_layer *layer;
int ret;
- struct mxr_layer_ops ops = {
+ static struct mxr_layer_ops ops = {
.release = mxr_graph_layer_release,
.buffer_set = mxr_graph_buffer_set,
.stream_set = mxr_graph_stream_set,
layer->update_buf = next;
}
- layer->ops.buffer_set(layer, layer->update_buf);
+ layer->ops->buffer_set(layer, layer->update_buf);
if (done && done != layer->shadow_buf)
vb2_buffer_done(&done->vb, VB2_BUF_STATE_DONE);
layer->geo.src.height = layer->geo.src.full_height;
mxr_geometry_dump(mdev, &layer->geo);
- layer->ops.fix_geometry(layer, MXR_GEOMETRY_SINK, 0);
+ layer->ops->fix_geometry(layer, MXR_GEOMETRY_SINK, 0);
mxr_geometry_dump(mdev, &layer->geo);
}
layer->geo.dst.full_width = mbus_fmt.width;
layer->geo.dst.full_height = mbus_fmt.height;
layer->geo.dst.field = mbus_fmt.field;
- layer->ops.fix_geometry(layer, MXR_GEOMETRY_SINK, 0);
+ layer->ops->fix_geometry(layer, MXR_GEOMETRY_SINK, 0);
mxr_geometry_dump(mdev, &layer->geo);
}
/* set source size to highest accepted value */
geo->src.full_width = max(geo->dst.full_width, pix->width);
geo->src.full_height = max(geo->dst.full_height, pix->height);
- layer->ops.fix_geometry(layer, MXR_GEOMETRY_SOURCE, 0);
+ layer->ops->fix_geometry(layer, MXR_GEOMETRY_SOURCE, 0);
mxr_geometry_dump(mdev, &layer->geo);
/* set cropping to total visible screen */
geo->src.width = pix->width;
geo->src.x_offset = 0;
geo->src.y_offset = 0;
/* assure consistency of geometry */
- layer->ops.fix_geometry(layer, MXR_GEOMETRY_CROP, MXR_NO_OFFSET);
+ layer->ops->fix_geometry(layer, MXR_GEOMETRY_CROP, MXR_NO_OFFSET);
mxr_geometry_dump(mdev, &layer->geo);
/* set full size to lowest possible value */
geo->src.full_width = 0;
geo->src.full_height = 0;
- layer->ops.fix_geometry(layer, MXR_GEOMETRY_SOURCE, 0);
+ layer->ops->fix_geometry(layer, MXR_GEOMETRY_SOURCE, 0);
mxr_geometry_dump(mdev, &layer->geo);
/* returning results */
target->width = s->r.width;
target->height = s->r.height;
- layer->ops.fix_geometry(layer, stage, s->flags);
+ layer->ops->fix_geometry(layer, stage, s->flags);
/* retrieve update selection rectangle */
res.left = target->x_offset;
mxr_output_get(mdev);
mxr_layer_update_output(layer);
- layer->ops.format_set(layer);
+ layer->ops->format_set(layer);
/* enabling layer in hardware */
spin_lock_irqsave(&layer->enq_slock, flags);
layer->state = MXR_LAYER_STREAMING;
spin_unlock_irqrestore(&layer->enq_slock, flags);
- layer->ops.stream_set(layer, MXR_ENABLE);
+ layer->ops->stream_set(layer, MXR_ENABLE);
mxr_streamer_get(mdev);
return 0;
spin_unlock_irqrestore(&layer->enq_slock, flags);
/* disabling layer in hardware */
- layer->ops.stream_set(layer, MXR_DISABLE);
+ layer->ops->stream_set(layer, MXR_DISABLE);
/* remove one streamer */
mxr_streamer_put(mdev);
/* allow changes in output configuration */
void mxr_layer_release(struct mxr_layer *layer)
{
- if (layer->ops.release)
- layer->ops.release(layer);
+ if (layer->ops->release)
+ layer->ops->release(layer);
}
void mxr_base_layer_release(struct mxr_layer *layer)
layer->mdev = mdev;
layer->idx = idx;
- layer->ops = *ops;
+ layer->ops = ops;
spin_lock_init(&layer->enq_slock);
INIT_LIST_HEAD(&layer->enq_list);
{
struct mxr_layer *layer;
int ret;
- struct mxr_layer_ops ops = {
+ static struct mxr_layer_ops ops = {
.release = mxr_vp_layer_release,
.buffer_set = mxr_vp_buffer_set,
.stream_set = mxr_vp_stream_set,
unsigned char readbuf[RDS_BUFFER];
int i = 0;
+ if (count > RDS_BUFFER)
+ return -EFAULT;
mutex_lock(&dev->lock);
if (dev->rdsstat == 0)
cadet_start_rds(dev);
readbuf[i++] = dev->rdsbuf[dev->rdsout++];
mutex_unlock(&dev->lock);
- if (i && copy_to_user(data, readbuf, i))
- return -EFAULT;
+ if (i > sizeof(readbuf) || (i && copy_to_user(data, readbuf, i)))
+ i = -EFAULT;
+
return i;
}
/* TEA5757 pin mappings */
static const int clk = 1, data = 2, wren = 4, mo_st = 8, power = 16;
-static atomic_t maxiradio_instance = ATOMIC_INIT(0);
+static atomic_unchecked_t maxiradio_instance = ATOMIC_INIT(0);
#define PCI_VENDOR_ID_GUILLEMOT 0x5046
#define PCI_DEVICE_ID_GUILLEMOT_MAXIRADIO 0x1001
u32 last_val;
};
-static atomic_t shark_instance = ATOMIC_INIT(0);
+static atomic_unchecked_t shark_instance = ATOMIC_INIT(0);
static void shark_write_val(struct snd_tea575x *tea, u32 val)
{
u8 *transfer_buffer;
};
-static atomic_t shark_instance = ATOMIC_INIT(0);
+static atomic_unchecked_t shark_instance = ATOMIC_INIT(0);
static int shark_write_reg(struct radio_tea5777 *tea, u64 reg)
{
struct si476x_radio *radio;
struct v4l2_ctrl *ctrl;
- static atomic_t instance = ATOMIC_INIT(0);
+ static atomic_unchecked_t instance = ATOMIC_INIT(0);
radio = devm_kzalloc(&pdev->dev, sizeof(*radio), GFP_KERNEL);
if (!radio)
MODULE_PARM_DESC(rds_buf, "RDS buffer entries");
/* Radio Nr */
-static u32 radio_nr = -1;
+static int radio_nr = -1;
module_param(radio_nr, int, 0444);
MODULE_PARM_DESC(radio_nr, "Radio Nr");
static int cinergyt2_streaming_ctrl(struct dvb_usb_adapter *adap, int enable)
{
- char buf[] = { CINERGYT2_EP1_CONTROL_STREAM_TRANSFER, enable ? 1 : 0 };
- char result[64];
- return dvb_usb_generic_rw(adap->dev, buf, sizeof(buf), result,
- sizeof(result), 0);
+ char *buf;
+ char *result;
+ int retval;
+
+ buf = kmalloc(2, GFP_KERNEL);
+ if (buf == NULL)
+ return -ENOMEM;
+ result = kmalloc(64, GFP_KERNEL);
+ if (result == NULL) {
+ kfree(buf);
+ return -ENOMEM;
+ }
+
+ buf[0] = CINERGYT2_EP1_CONTROL_STREAM_TRANSFER;
+ buf[1] = enable ? 1 : 0;
+
+ retval = dvb_usb_generic_rw(adap->dev, buf, 2, result, 64, 0);
+
+ kfree(buf);
+ kfree(result);
+ return retval;
}
static int cinergyt2_power_ctrl(struct dvb_usb_device *d, int enable)
{
- char buf[] = { CINERGYT2_EP1_SLEEP_MODE, enable ? 0 : 1 };
- char state[3];
- return dvb_usb_generic_rw(d, buf, sizeof(buf), state, sizeof(state), 0);
+ char *buf;
+ char *state;
+ int retval;
+
+ buf = kmalloc(2, GFP_KERNEL);
+ if (buf == NULL)
+ return -ENOMEM;
+ state = kmalloc(3, GFP_KERNEL);
+ if (state == NULL) {
+ kfree(buf);
+ return -ENOMEM;
+ }
+
+ buf[0] = CINERGYT2_EP1_SLEEP_MODE;
+ buf[1] = enable ? 1 : 0;
+
+ retval = dvb_usb_generic_rw(d, buf, 2, state, 3, 0);
+
+ kfree(buf);
+ kfree(state);
+ return retval;
}
static int cinergyt2_frontend_attach(struct dvb_usb_adapter *adap)
{
- char query[] = { CINERGYT2_EP1_GET_FIRMWARE_VERSION };
- char state[3];
+ char *query;
+ char *state;
int ret;
+ query = kmalloc(1, GFP_KERNEL);
+ if (query == NULL)
+ return -ENOMEM;
+ state = kmalloc(3, GFP_KERNEL);
+ if (state == NULL) {
+ kfree(query);
+ return -ENOMEM;
+ }
+
+ query[0] = CINERGYT2_EP1_GET_FIRMWARE_VERSION;
adap->fe_adap[0].fe = cinergyt2_fe_attach(adap->dev);
- ret = dvb_usb_generic_rw(adap->dev, query, sizeof(query), state,
- sizeof(state), 0);
+ ret = dvb_usb_generic_rw(adap->dev, query, 1, state, 3, 0);
if (ret < 0) {
deb_rc("cinergyt2_power_ctrl() Failed to retrieve sleep "
"state info\n");
/* Copy this pointer as we are gonna need it in the release phase */
cinergyt2_usb_device = adap->dev;
-
+ kfree(query);
+ kfree(state);
return 0;
}
static int cinergyt2_rc_query(struct dvb_usb_device *d, u32 *event, int *state)
{
struct cinergyt2_state *st = d->priv;
- u8 key[5] = {0, 0, 0, 0, 0}, cmd = CINERGYT2_EP1_GET_RC_EVENTS;
+ u8 *key, *cmd;
int i;
+ cmd = kmalloc(1, GFP_KERNEL);
+ if (cmd == NULL)
+ return -EINVAL;
+ key = kzalloc(5, GFP_KERNEL);
+ if (key == NULL) {
+ kfree(cmd);
+ return -EINVAL;
+ }
+
+ cmd[0] = CINERGYT2_EP1_GET_RC_EVENTS;
+
*state = REMOTE_NO_KEY_PRESSED;
- dvb_usb_generic_rw(d, &cmd, 1, key, sizeof(key), 0);
+ dvb_usb_generic_rw(d, cmd, 1, key, 5, 0);
if (key[4] == 0xff) {
/* key repeat */
st->rc_counter++;
*event = d->last_event;
deb_rc("repeat key, event %x\n",
*event);
- return 0;
+ goto out;
}
}
deb_rc("repeated key (non repeatable)\n");
}
- return 0;
+ goto out;
}
/* hack to pass checksum on the custom field */
deb_rc("key: %*ph\n", 5, key);
}
+out:
+ kfree(cmd);
+ kfree(key);
return 0;
}
fe_status_t *status)
{
struct cinergyt2_fe_state *state = fe->demodulator_priv;
- struct dvbt_get_status_msg result;
- u8 cmd[] = { CINERGYT2_EP1_GET_TUNER_STATUS };
+ struct dvbt_get_status_msg *result;
+ u8 *cmd;
int ret;
- ret = dvb_usb_generic_rw(state->d, cmd, sizeof(cmd), (u8 *)&result,
- sizeof(result), 0);
+ cmd = kmalloc(1, GFP_KERNEL);
+ if (cmd == NULL)
+ return -ENOMEM;
+ result = kmalloc(sizeof(*result), GFP_KERNEL);
+ if (result == NULL) {
+ kfree(cmd);
+ return -ENOMEM;
+ }
+
+ cmd[0] = CINERGYT2_EP1_GET_TUNER_STATUS;
+
+ ret = dvb_usb_generic_rw(state->d, cmd, 1, (u8 *)result,
+ sizeof(*result), 0);
if (ret < 0)
- return ret;
+ goto out;
*status = 0;
- if (0xffff - le16_to_cpu(result.gain) > 30)
+ if (0xffff - le16_to_cpu(result->gain) > 30)
*status |= FE_HAS_SIGNAL;
- if (result.lock_bits & (1 << 6))
+ if (result->lock_bits & (1 << 6))
*status |= FE_HAS_LOCK;
- if (result.lock_bits & (1 << 5))
+ if (result->lock_bits & (1 << 5))
*status |= FE_HAS_SYNC;
- if (result.lock_bits & (1 << 4))
+ if (result->lock_bits & (1 << 4))
*status |= FE_HAS_CARRIER;
- if (result.lock_bits & (1 << 1))
+ if (result->lock_bits & (1 << 1))
*status |= FE_HAS_VITERBI;
if ((*status & (FE_HAS_CARRIER | FE_HAS_VITERBI | FE_HAS_SYNC)) !=
(FE_HAS_CARRIER | FE_HAS_VITERBI | FE_HAS_SYNC))
*status &= ~FE_HAS_LOCK;
- return 0;
+out:
+ kfree(cmd);
+ kfree(result);
+ return ret;
}
static int cinergyt2_fe_read_ber(struct dvb_frontend *fe, u32 *ber)
{
struct cinergyt2_fe_state *state = fe->demodulator_priv;
- struct dvbt_get_status_msg status;
- char cmd[] = { CINERGYT2_EP1_GET_TUNER_STATUS };
+ struct dvbt_get_status_msg *status;
+ char *cmd;
int ret;
- ret = dvb_usb_generic_rw(state->d, cmd, sizeof(cmd), (char *)&status,
- sizeof(status), 0);
+ cmd = kmalloc(1, GFP_KERNEL);
+ if (cmd == NULL)
+ return -ENOMEM;
+ status = kmalloc(sizeof(*status), GFP_KERNEL);
+ if (status == NULL) {
+ kfree(cmd);
+ return -ENOMEM;
+ }
+
+ cmd[0] = CINERGYT2_EP1_GET_TUNER_STATUS;
+
+ ret = dvb_usb_generic_rw(state->d, cmd, 1, (char *)status,
+ sizeof(*status), 0);
if (ret < 0)
- return ret;
+ goto out;
- *ber = le32_to_cpu(status.viterbi_error_rate);
+ *ber = le32_to_cpu(status->viterbi_error_rate);
+out:
+ kfree(cmd);
+ kfree(status);
return 0;
}
static int cinergyt2_fe_read_unc_blocks(struct dvb_frontend *fe, u32 *unc)
{
struct cinergyt2_fe_state *state = fe->demodulator_priv;
- struct dvbt_get_status_msg status;
- u8 cmd[] = { CINERGYT2_EP1_GET_TUNER_STATUS };
+ struct dvbt_get_status_msg *status;
+ u8 *cmd;
int ret;
- ret = dvb_usb_generic_rw(state->d, cmd, sizeof(cmd), (u8 *)&status,
- sizeof(status), 0);
+ cmd = kmalloc(1, GFP_KERNEL);
+ if (cmd == NULL)
+ return -ENOMEM;
+ status = kmalloc(sizeof(*status), GFP_KERNEL);
+ if (status == NULL) {
+ kfree(cmd);
+ return -ENOMEM;
+ }
+
+ cmd[0] = CINERGYT2_EP1_GET_TUNER_STATUS;
+
+ ret = dvb_usb_generic_rw(state->d, cmd, 1, (u8 *)status,
+ sizeof(*status), 0);
if (ret < 0) {
err("cinergyt2_fe_read_unc_blocks() Failed! (Error=%d)\n",
ret);
- return ret;
+ goto out;
}
- *unc = le32_to_cpu(status.uncorrected_block_count);
- return 0;
+ *unc = le32_to_cpu(status->uncorrected_block_count);
+
+out:
+ kfree(cmd);
+ kfree(status);
+ return ret;
}
static int cinergyt2_fe_read_signal_strength(struct dvb_frontend *fe,
u16 *strength)
{
struct cinergyt2_fe_state *state = fe->demodulator_priv;
- struct dvbt_get_status_msg status;
- char cmd[] = { CINERGYT2_EP1_GET_TUNER_STATUS };
+ struct dvbt_get_status_msg *status;
+ char *cmd;
int ret;
- ret = dvb_usb_generic_rw(state->d, cmd, sizeof(cmd), (char *)&status,
- sizeof(status), 0);
+ cmd = kmalloc(1, GFP_KERNEL);
+ if (cmd == NULL)
+ return -ENOMEM;
+ status = kmalloc(sizeof(*status), GFP_KERNEL);
+ if (status == NULL) {
+ kfree(cmd);
+ return -ENOMEM;
+ }
+
+ cmd[0] = CINERGYT2_EP1_GET_TUNER_STATUS;
+
+ ret = dvb_usb_generic_rw(state->d, cmd, 1, (char *)status,
+ sizeof(*status), 0);
if (ret < 0) {
err("cinergyt2_fe_read_signal_strength() Failed!"
" (Error=%d)\n", ret);
- return ret;
+ goto out;
}
- *strength = (0xffff - le16_to_cpu(status.gain));
+ *strength = (0xffff - le16_to_cpu(status->gain));
+
+out:
+ kfree(cmd);
+ kfree(status);
return 0;
}
static int cinergyt2_fe_read_snr(struct dvb_frontend *fe, u16 *snr)
{
struct cinergyt2_fe_state *state = fe->demodulator_priv;
- struct dvbt_get_status_msg status;
- char cmd[] = { CINERGYT2_EP1_GET_TUNER_STATUS };
+ struct dvbt_get_status_msg *status;
+ char *cmd;
int ret;
- ret = dvb_usb_generic_rw(state->d, cmd, sizeof(cmd), (char *)&status,
- sizeof(status), 0);
+ cmd = kmalloc(1, GFP_KERNEL);
+ if (cmd == NULL)
+ return -ENOMEM;
+ status = kmalloc(sizeof(*status), GFP_KERNEL);
+ if (status == NULL) {
+ kfree(cmd);
+ return -ENOMEM;
+ }
+
+ cmd[0] = CINERGYT2_EP1_GET_TUNER_STATUS;
+
+ ret = dvb_usb_generic_rw(state->d, cmd, 1, (char *)status,
+ sizeof(*status), 0);
if (ret < 0) {
err("cinergyt2_fe_read_snr() Failed! (Error=%d)\n", ret);
- return ret;
+ goto out;
}
- *snr = (status.snr << 8) | status.snr;
- return 0;
+ *snr = (status->snr << 8) | status->snr;
+
+out:
+ kfree(cmd);
+ kfree(status);
+ return ret;
}
static int cinergyt2_fe_init(struct dvb_frontend *fe)
{
struct dtv_frontend_properties *fep = &fe->dtv_property_cache;
struct cinergyt2_fe_state *state = fe->demodulator_priv;
- struct dvbt_set_parameters_msg param;
- char result[2];
+ struct dvbt_set_parameters_msg *param;
+ char *result;
int err;
- param.cmd = CINERGYT2_EP1_SET_TUNER_PARAMETERS;
- param.tps = cpu_to_le16(compute_tps(fep));
- param.freq = cpu_to_le32(fep->frequency / 1000);
- param.flags = 0;
+ result = kmalloc(2, GFP_KERNEL);
+ if (result == NULL)
+ return -ENOMEM;
+ param = kmalloc(sizeof(*param), GFP_KERNEL);
+ if (param == NULL) {
+ kfree(result);
+ return -ENOMEM;
+ }
+
+ param->cmd = CINERGYT2_EP1_SET_TUNER_PARAMETERS;
+ param->tps = cpu_to_le16(compute_tps(fep));
+ param->freq = cpu_to_le32(fep->frequency / 1000);
+ param->flags = 0;
switch (fep->bandwidth_hz) {
default:
case 8000000:
- param.bandwidth = 8;
+ param->bandwidth = 8;
break;
case 7000000:
- param.bandwidth = 7;
+ param->bandwidth = 7;
break;
case 6000000:
- param.bandwidth = 6;
+ param->bandwidth = 6;
break;
}
err = dvb_usb_generic_rw(state->d,
- (char *)¶m, sizeof(param),
- result, sizeof(result), 0);
+ (char *)param, sizeof(*param),
+ result, 2, 0);
if (err < 0)
err("cinergyt2_fe_set_frontend() Failed! err=%d\n", err);
- return (err < 0) ? err : 0;
+ kfree(result);
+ kfree(param);
+ return err;
}
static void cinergyt2_fe_release(struct dvb_frontend *fe)
int usb_cypress_load_firmware(struct usb_device *udev, const struct firmware *fw, int type)
{
- struct hexline hx;
- u8 reset;
+ struct hexline *hx;
+ u8 *reset;
int ret,pos=0;
+ reset = kmalloc(1, GFP_KERNEL);
+ if (reset == NULL)
+ return -ENOMEM;
+
+ hx = kmalloc(sizeof(struct hexline), GFP_KERNEL);
+ if (hx == NULL) {
+ kfree(reset);
+ return -ENOMEM;
+ }
+
/* stop the CPU */
- reset = 1;
- if ((ret = usb_cypress_writemem(udev,cypress[type].cpu_cs_register,&reset,1)) != 1)
+ reset[0] = 1;
+ if ((ret = usb_cypress_writemem(udev,cypress[type].cpu_cs_register,reset,1)) != 1)
err("could not stop the USB controller CPU.");
- while ((ret = dvb_usb_get_hexline(fw,&hx,&pos)) > 0) {
- deb_fw("writing to address 0x%04x (buffer: 0x%02x %02x)\n",hx.addr,hx.len,hx.chk);
- ret = usb_cypress_writemem(udev,hx.addr,hx.data,hx.len);
+ while ((ret = dvb_usb_get_hexline(fw,hx,&pos)) > 0) {
+ deb_fw("writing to address 0x%04x (buffer: 0x%02x %02x)\n",hx->addr,hx->len,hx->chk);
+ ret = usb_cypress_writemem(udev,hx->addr,hx->data,hx->len);
- if (ret != hx.len) {
+ if (ret != hx->len) {
err("error while transferring firmware "
"(transferred size: %d, block size: %d)",
- ret,hx.len);
+ ret,hx->len);
ret = -EINVAL;
break;
}
}
if (ret < 0) {
err("firmware download failed at %d with %d",pos,ret);
+ kfree(reset);
+ kfree(hx);
return ret;
}
if (ret == 0) {
/* restart the CPU */
- reset = 0;
- if (ret || usb_cypress_writemem(udev,cypress[type].cpu_cs_register,&reset,1) != 1) {
+ reset[0] = 0;
+ if (ret || usb_cypress_writemem(udev,cypress[type].cpu_cs_register,reset,1) != 1) {
err("could not restart the USB controller CPU.");
ret = -EINVAL;
}
} else
ret = -EIO;
+ kfree(reset);
+ kfree(hx);
+
return ret;
}
EXPORT_SYMBOL(usb_cypress_load_firmware);
struct s6x0_state {
int (*old_set_voltage)(struct dvb_frontend *f, fe_sec_voltage_t v);
-};
+} __no_const;
/* debug */
static int dvb_usb_dw2102_debug;
static int technisat_usb2_i2c_access(struct usb_device *udev,
u8 device_addr, u8 *tx, u8 txlen, u8 *rx, u8 rxlen)
{
- u8 b[64];
- int ret, actual_length;
+ u8 *b = kmalloc(64, GFP_KERNEL);
+ int ret, actual_length, error = 0;
+
+ if (b == NULL)
+ return -ENOMEM;
deb_i2c("i2c-access: %02x, tx: ", device_addr);
debug_dump(tx, txlen, deb_i2c);
if (ret < 0) {
err("i2c-error: out failed %02x = %d", device_addr, ret);
- return -ENODEV;
+ error = -ENODEV;
+ goto out;
}
ret = usb_bulk_msg(udev,
b, 64, &actual_length, 1000);
if (ret < 0) {
err("i2c-error: in failed %02x = %d", device_addr, ret);
- return -ENODEV;
+ error = -ENODEV;
+ goto out;
}
if (b[0] != I2C_STATUS_OK) {
/* handle tuner-i2c-nak */
if (!(b[0] == I2C_STATUS_NAK &&
device_addr == 0x60
- /* && device_is_technisat_usb2 */))
- return -ENODEV;
+ /* && device_is_technisat_usb2 */)) {
+ error = -ENODEV;
+ goto out;
+ }
}
deb_i2c("status: %d, ", b[0]);
deb_i2c("\n");
- return 0;
+out:
+ kfree(b);
+ return error;
}
static int technisat_usb2_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg *msg,
{
int ret;
- u8 led[8] = {
- red ? SET_RED_LED_VENDOR_REQUEST : SET_GREEN_LED_VENDOR_REQUEST,
- 0
- };
+ u8 *led = kzalloc(8, GFP_KERNEL);
+
+ if (led == NULL)
+ return -ENOMEM;
if (disable_led_control && state != TECH_LED_OFF)
return 0;
+ led[0] = red ? SET_RED_LED_VENDOR_REQUEST : SET_GREEN_LED_VENDOR_REQUEST;
+
switch (state) {
case TECH_LED_ON:
led[1] = 0x82;
red ? SET_RED_LED_VENDOR_REQUEST : SET_GREEN_LED_VENDOR_REQUEST,
USB_TYPE_VENDOR | USB_DIR_OUT,
0, 0,
- led, sizeof(led), 500);
+ led, 8, 500);
mutex_unlock(&d->i2c_mutex);
+
+ kfree(led);
+
return ret;
}
static int technisat_usb2_set_led_timer(struct dvb_usb_device *d, u8 red, u8 green)
{
int ret;
- u8 b = 0;
+ u8 *b = kzalloc(1, GFP_KERNEL);
+
+ if (b == NULL)
+ return -ENOMEM;
if (mutex_lock_interruptible(&d->i2c_mutex) < 0)
return -EAGAIN;
SET_LED_TIMER_DIVIDER_VENDOR_REQUEST,
USB_TYPE_VENDOR | USB_DIR_OUT,
(red << 8) | green, 0,
- &b, 1, 500);
+ b, 1, 500);
mutex_unlock(&d->i2c_mutex);
+ kfree(b);
+
return ret;
}
struct dvb_usb_device_description **desc, int *cold)
{
int ret;
- u8 version[3];
+ u8 *version = kmalloc(3, GFP_KERNEL);
/* first select the interface */
if (usb_set_interface(udev, 0, 1) != 0)
*cold = 0; /* by default do not download a firmware - just in case something is wrong */
+ if (version == NULL)
+ return 0;
+
ret = usb_control_msg(udev, usb_rcvctrlpipe(udev, 0),
GET_VERSION_INFO_VENDOR_REQUEST,
USB_TYPE_VENDOR | USB_DIR_IN,
0, 0,
- version, sizeof(version), 500);
+ version, 3, 500);
if (ret < 0)
*cold = 1;
*cold = 0;
}
+ kfree(version);
+
return 0;
}
static int technisat_usb2_get_ir(struct dvb_usb_device *d)
{
- u8 buf[62], *b;
+ u8 *buf, *b;
int ret;
struct ir_raw_event ev;
+ buf = kmalloc(62, GFP_KERNEL);
+
+ if (buf == NULL)
+ return -ENOMEM;
+
buf[0] = GET_IR_DATA_VENDOR_REQUEST;
buf[1] = 0x08;
buf[2] = 0x8f;
GET_IR_DATA_VENDOR_REQUEST,
USB_TYPE_VENDOR | USB_DIR_IN,
0x8080, 0,
- buf, sizeof(buf), 500);
+ buf, 62, 500);
unlock:
mutex_unlock(&d->i2c_mutex);
- if (ret < 0)
+ if (ret < 0) {
+ kfree(buf);
return ret;
+ }
- if (ret == 1)
+ if (ret == 1) {
+ kfree(buf);
return 0; /* no key pressed */
+ }
/* decoding */
b = buf+1;
ir_raw_event_handle(d->rc_dev);
+ kfree(buf);
+
return 1;
}
* by passing a very big num_planes value */
uplane = compat_alloc_user_space(num_planes *
sizeof(struct v4l2_plane));
- kp->m.planes = (__force struct v4l2_plane *)uplane;
+ kp->m.planes = (__force_kernel struct v4l2_plane *)uplane;
while (--num_planes >= 0) {
ret = get_v4l2_plane32(uplane, uplane32, kp->memory);
if (num_planes == 0)
return 0;
- uplane = (__force struct v4l2_plane __user *)kp->m.planes;
+ uplane = (struct v4l2_plane __force_user *)kp->m.planes;
if (get_user(p, &up->m.planes))
return -EFAULT;
uplane32 = compat_ptr(p);
get_user(kp->flags, &up->flags) ||
copy_from_user(&kp->fmt, &up->fmt, sizeof(up->fmt)))
return -EFAULT;
- kp->base = (__force void *)compat_ptr(tmp);
+ kp->base = (__force_kernel void *)compat_ptr(tmp);
return 0;
}
n * sizeof(struct v4l2_ext_control32)))
return -EFAULT;
kcontrols = compat_alloc_user_space(n * sizeof(struct v4l2_ext_control));
- kp->controls = (__force struct v4l2_ext_control *)kcontrols;
+ kp->controls = (__force_kernel struct v4l2_ext_control *)kcontrols;
while (--n >= 0) {
u32 id;
{
struct v4l2_ext_control32 __user *ucontrols;
struct v4l2_ext_control __user *kcontrols =
- (__force struct v4l2_ext_control __user *)kp->controls;
+ (struct v4l2_ext_control __force_user *)kp->controls;
int n = kp->count;
compat_caddr_t p;
get_user(tmp, &up->edid) ||
copy_from_user(kp->reserved, up->reserved, sizeof(kp->reserved)))
return -EFAULT;
- kp->edid = (__force u8 *)compat_ptr(tmp);
+ kp->edid = (__force_kernel u8 *)compat_ptr(tmp);
return 0;
}
EXPORT_SYMBOL_GPL(v4l2_device_put);
int v4l2_device_set_name(struct v4l2_device *v4l2_dev, const char *basename,
- atomic_t *instance)
+ atomic_unchecked_t *instance)
{
- int num = atomic_inc_return(instance) - 1;
+ int num = atomic_inc_return_unchecked(instance) - 1;
int len = strlen(basename);
if (basename[len - 1] >= '0' && basename[len - 1] <= '9')
struct file *file, void *fh, void *p);
} u;
void (*debug)(const void *arg, bool write_only);
-};
+} __do_const;
+typedef struct v4l2_ioctl_info __no_const v4l2_ioctl_info_no_const;
/* This control needs a priority check */
#define INFO_FL_PRIO (1 << 0)
struct video_device *vfd = video_devdata(file);
const struct v4l2_ioctl_ops *ops = vfd->ioctl_ops;
bool write_only = false;
- struct v4l2_ioctl_info default_info;
+ v4l2_ioctl_info_no_const default_info;
const struct v4l2_ioctl_info *info;
void *fh = file->private_data;
struct v4l2_fh *vfh = NULL;
ret = -EINVAL;
break;
}
- *user_ptr = (void __user *)buf->m.planes;
+ *user_ptr = (void __force_user *)buf->m.planes;
*kernel_ptr = (void **)&buf->m.planes;
*array_size = sizeof(struct v4l2_plane) * buf->length;
ret = 1;
ret = -EINVAL;
break;
}
- *user_ptr = (void __user *)edid->edid;
+ *user_ptr = (void __force_user *)edid->edid;
*kernel_ptr = (void **)&edid->edid;
*array_size = edid->blocks * 128;
ret = 1;
ret = -EINVAL;
break;
}
- *user_ptr = (void __user *)ctrls->controls;
+ *user_ptr = (void __force_user *)ctrls->controls;
*kernel_ptr = (void **)&ctrls->controls;
*array_size = sizeof(struct v4l2_ext_control)
* ctrls->count;
}
if (has_array_args) {
- *kernel_ptr = (void __force *)user_ptr;
+ *kernel_ptr = (void __force_kernel *)user_ptr;
if (copy_to_user(user_ptr, mbuf, array_size))
err = -EFAULT;
goto out_array_args;
};
static struct gpmc_client_irq gpmc_client_irq[GPMC_NR_IRQ];
-static struct irq_chip gpmc_irq_chip;
static int gpmc_irq_start;
static struct resource gpmc_mem_root;
static unsigned int gpmc_irq_noop_ret(struct irq_data *data) { return 0; }
+static struct irq_chip gpmc_irq_chip = {
+ .name = "gpmc",
+ .irq_startup = gpmc_irq_noop_ret,
+ .irq_enable = gpmc_irq_enable,
+ .irq_disable = gpmc_irq_disable,
+ .irq_shutdown = gpmc_irq_noop,
+ .irq_ack = gpmc_irq_noop,
+ .irq_mask = gpmc_irq_noop,
+ .irq_unmask = gpmc_irq_noop,
+};
+
static int gpmc_setup_irq(void)
{
int i;
return gpmc_irq_start;
}
- gpmc_irq_chip.name = "gpmc";
- gpmc_irq_chip.irq_startup = gpmc_irq_noop_ret;
- gpmc_irq_chip.irq_enable = gpmc_irq_enable;
- gpmc_irq_chip.irq_disable = gpmc_irq_disable;
- gpmc_irq_chip.irq_shutdown = gpmc_irq_noop;
- gpmc_irq_chip.irq_ack = gpmc_irq_noop;
- gpmc_irq_chip.irq_mask = gpmc_irq_noop;
- gpmc_irq_chip.irq_unmask = gpmc_irq_noop;
-
gpmc_client_irq[0].bitmask = GPMC_IRQ_FIFOEVENTENABLE;
gpmc_client_irq[1].bitmask = GPMC_IRQ_COUNT_EVENT;
seq_printf(m, " MaxChainDepth = 0x%02x frames\n", ioc->facts.MaxChainDepth);
seq_printf(m, " MinBlockSize = 0x%02x bytes\n", 4*ioc->facts.BlockSize);
+#ifdef CONFIG_GRKERNSEC_HIDESYM
+ seq_printf(m, " RequestFrames @ 0x%p (Dma @ 0x%p)\n", NULL, NULL);
+#else
seq_printf(m, " RequestFrames @ 0x%p (Dma @ 0x%p)\n",
(void *)ioc->req_frames, (void *)(ulong)ioc->req_frames_dma);
+#endif
+
/*
* Rounding UP to nearest 4-kB boundary here...
*/
ioc->facts.GlobalCredits);
seq_printf(m, " Frames @ 0x%p (Dma @ 0x%p)\n",
+#ifdef CONFIG_GRKERNSEC_HIDESYM
+ NULL, NULL);
+#else
(void *)ioc->alloc, (void *)(ulong)ioc->alloc_dma);
+#endif
sz = (ioc->reply_sz * ioc->reply_depth) + 128;
seq_printf(m, " {CurRepSz=%d} x {CurRepDepth=%d} = %d bytes ^= 0x%x\n",
ioc->reply_sz, ioc->reply_depth, ioc->reply_sz*ioc->reply_depth, sz);
return 0;
}
+static inline void
+mptsas_set_rphy(MPT_ADAPTER *ioc, struct mptsas_phyinfo *phy_info, struct sas_rphy *rphy)
+{
+ if (phy_info->port_details) {
+ phy_info->port_details->rphy = rphy;
+ dsaswideprintk(ioc, printk(MYIOC_s_DEBUG_FMT "sas_rphy_add: rphy=%p\n",
+ ioc->name, rphy));
+ }
+
+ if (rphy) {
+ dsaswideprintk(ioc, dev_printk(KERN_DEBUG,
+ &rphy->dev, MYIOC_s_FMT "add:", ioc->name));
+ dsaswideprintk(ioc, printk(MYIOC_s_DEBUG_FMT "rphy=%p release=%p\n",
+ ioc->name, rphy, rphy->dev.release));
+ }
+}
+
/* no mutex */
static void
mptsas_port_delete(MPT_ADAPTER *ioc, struct mptsas_portinfo_details * port_details)
return NULL;
}
-static inline void
-mptsas_set_rphy(MPT_ADAPTER *ioc, struct mptsas_phyinfo *phy_info, struct sas_rphy *rphy)
-{
- if (phy_info->port_details) {
- phy_info->port_details->rphy = rphy;
- dsaswideprintk(ioc, printk(MYIOC_s_DEBUG_FMT "sas_rphy_add: rphy=%p\n",
- ioc->name, rphy));
- }
-
- if (rphy) {
- dsaswideprintk(ioc, dev_printk(KERN_DEBUG,
- &rphy->dev, MYIOC_s_FMT "add:", ioc->name));
- dsaswideprintk(ioc, printk(MYIOC_s_DEBUG_FMT "rphy=%p release=%p\n",
- ioc->name, rphy, rphy->dev.release));
- }
-}
-
static inline struct sas_port *
mptsas_get_port(struct mptsas_phyinfo *phy_info)
{
"Array Controller Device"
};
-static char *chtostr(char *tmp, u8 *chars, int n)
-{
- tmp[0] = 0;
- return strncat(tmp, (char *)chars, n);
-}
-
static int i2o_report_query_status(struct seq_file *seq, int block_status,
char *group)
{
static int i2o_seq_show_hw(struct seq_file *seq, void *v)
{
struct i2o_controller *c = (struct i2o_controller *)seq->private;
- static u32 work32[5];
- static u8 *work8 = (u8 *) work32;
- static u16 *work16 = (u16 *) work32;
+ u32 work32[5];
+ u8 *work8 = (u8 *) work32;
+ u16 *work16 = (u16 *) work32;
int token;
u32 hwcap;
} *result;
i2o_exec_execute_ddm_table ddm_table;
- char tmp[28 + 1];
result = kmalloc(sizeof(*result), GFP_KERNEL);
if (!result)
seq_printf(seq, "%-#7x", ddm_table.i2o_vendor_id);
seq_printf(seq, "%-#8x", ddm_table.module_id);
- seq_printf(seq, "%-29s",
- chtostr(tmp, ddm_table.module_name_version, 28));
+ seq_printf(seq, "%-.28s", ddm_table.module_name_version);
seq_printf(seq, "%9d ", ddm_table.data_size);
seq_printf(seq, "%8d", ddm_table.code_size);
i2o_driver_result_table *result;
i2o_driver_store_table *dst;
- char tmp[28 + 1];
result = kmalloc(sizeof(i2o_driver_result_table), GFP_KERNEL);
if (result == NULL)
seq_printf(seq, "%-#7x", dst->i2o_vendor_id);
seq_printf(seq, "%-#8x", dst->module_id);
- seq_printf(seq, "%-29s",
- chtostr(tmp, dst->module_name_version, 28));
- seq_printf(seq, "%-9s", chtostr(tmp, dst->date, 8));
+ seq_printf(seq, "%-.28s", dst->module_name_version);
+ seq_printf(seq, "%-.8s", dst->date);
seq_printf(seq, "%8d ", dst->module_size);
seq_printf(seq, "%8d ", dst->mpb_size);
seq_printf(seq, "0x%04x", dst->module_flags);
static int i2o_seq_show_dev_identity(struct seq_file *seq, void *v)
{
struct i2o_device *d = (struct i2o_device *)seq->private;
- static u32 work32[128]; // allow for "stuff" + up to 256 byte (max) serial number
+ u32 work32[128]; // allow for "stuff" + up to 256 byte (max) serial number
// == (allow) 512d bytes (max)
- static u16 *work16 = (u16 *) work32;
+ u16 *work16 = (u16 *) work32;
int token;
- char tmp[16 + 1];
token = i2o_parm_field_get(d, 0xF100, -1, &work32, sizeof(work32));
seq_printf(seq, "Device Class : %s\n", i2o_get_class_name(work16[0]));
seq_printf(seq, "Owner TID : %0#5x\n", work16[2]);
seq_printf(seq, "Parent TID : %0#5x\n", work16[3]);
- seq_printf(seq, "Vendor info : %s\n",
- chtostr(tmp, (u8 *) (work32 + 2), 16));
- seq_printf(seq, "Product info : %s\n",
- chtostr(tmp, (u8 *) (work32 + 6), 16));
- seq_printf(seq, "Description : %s\n",
- chtostr(tmp, (u8 *) (work32 + 10), 16));
- seq_printf(seq, "Product rev. : %s\n",
- chtostr(tmp, (u8 *) (work32 + 14), 8));
+ seq_printf(seq, "Vendor info : %.16s\n", (u8 *) (work32 + 2));
+ seq_printf(seq, "Product info : %.16s\n", (u8 *) (work32 + 6));
+ seq_printf(seq, "Description : %.16s\n", (u8 *) (work32 + 10));
+ seq_printf(seq, "Product rev. : %.8s\n", (u8 *) (work32 + 14));
seq_printf(seq, "Serial number : ");
print_serial_number(seq, (u8 *) (work32 + 16),
u8 pad[256]; // allow up to 256 byte (max) serial number
} result;
- char tmp[24 + 1];
-
token = i2o_parm_field_get(d, 0xF101, -1, &result, sizeof(result));
if (token < 0) {
}
seq_printf(seq, "Registering DDM TID : 0x%03x\n", result.ddm_tid);
- seq_printf(seq, "Module name : %s\n",
- chtostr(tmp, result.module_name, 24));
- seq_printf(seq, "Module revision : %s\n",
- chtostr(tmp, result.module_rev, 8));
+ seq_printf(seq, "Module name : %.24s\n", result.module_name);
+ seq_printf(seq, "Module revision : %.8s\n", result.module_rev);
seq_printf(seq, "Serial number : ");
print_serial_number(seq, result.serial_number, sizeof(result) - 36);
u8 instance_number[4];
} result;
- char tmp[64 + 1];
-
token = i2o_parm_field_get(d, 0xF102, -1, &result, sizeof(result));
if (token < 0) {
return 0;
}
- seq_printf(seq, "Device name : %s\n",
- chtostr(tmp, result.device_name, 64));
- seq_printf(seq, "Service name : %s\n",
- chtostr(tmp, result.service_name, 64));
- seq_printf(seq, "Physical name : %s\n",
- chtostr(tmp, result.physical_location, 64));
- seq_printf(seq, "Instance number : %s\n",
- chtostr(tmp, result.instance_number, 4));
+ seq_printf(seq, "Device name : %.64s\n", result.device_name);
+ seq_printf(seq, "Service name : %.64s\n", result.service_name);
+ seq_printf(seq, "Physical name : %.64s\n", result.physical_location);
+ seq_printf(seq, "Instance number : %.4s\n", result.instance_number);
return 0;
}
static int i2o_seq_show_sgl_limits(struct seq_file *seq, void *v)
{
struct i2o_device *d = (struct i2o_device *)seq->private;
- static u32 work32[12];
- static u16 *work16 = (u16 *) work32;
- static u8 *work8 = (u8 *) work32;
+ u32 work32[12];
+ u16 *work16 = (u16 *) work32;
+ u8 *work8 = (u8 *) work32;
int token;
token = i2o_parm_field_get(d, 0xF103, -1, &work32, sizeof(work32));
spin_lock_irqsave(&c->context_list_lock, flags);
- if (unlikely(atomic_inc_and_test(&c->context_list_counter)))
- atomic_inc(&c->context_list_counter);
+ if (unlikely(atomic_inc_and_test_unchecked(&c->context_list_counter)))
+ atomic_inc_unchecked(&c->context_list_counter);
- entry->context = atomic_read(&c->context_list_counter);
+ entry->context = atomic_read_unchecked(&c->context_list_counter);
list_add(&entry->list, &c->context_list);
#if BITS_PER_LONG == 64
spin_lock_init(&c->context_list_lock);
- atomic_set(&c->context_list_counter, 0);
+ atomic_set_unchecked(&c->context_list_counter, 0);
INIT_LIST_HEAD(&c->context_list);
#endif
static u32 *irq_count;
static int num_irqs;
-static struct device_attribute **dev_attr;
+static device_attribute_no_const **dev_attr;
static char **event_name;
static u8 avg_sample = SAMPLE_16;
const struct i2c_device_id *id)
{
struct max8925_platform_data *pdata = dev_get_platdata(&client->dev);
- static struct max8925_chip *chip;
+ struct max8925_chip *chip;
struct device_node *node = client->dev.of_node;
if (node && !pdata) {
struct tps65910_platform_data *pdata)
{
int ret = 0;
- static struct regmap_irq_chip *tps6591x_irqs_chip;
+ struct regmap_irq_chip *tps6591x_irqs_chip;
if (!irq) {
dev_warn(tps65910->dev, "No interrupt support, no core IRQ\n");
#include <linux/of.h>
#include <linux/irqdomain.h>
#include <linux/i2c/twl.h>
+#include <asm/pgtable.h>
#include "twl-core.h"
* Install an irq handler for each of the SIH modules;
* clone dummy irq_chip since PIH can't *do* anything
*/
- twl4030_irq_chip = dummy_irq_chip;
- twl4030_irq_chip.name = "twl4030";
+ pax_open_kernel();
+ memcpy((void *)&twl4030_irq_chip, &dummy_irq_chip, sizeof twl4030_irq_chip);
+ *(const char **)&twl4030_irq_chip.name = "twl4030";
- twl4030_sih_irq_chip.irq_ack = dummy_irq_chip.irq_ack;
+ *(void **)&twl4030_sih_irq_chip.irq_ack = dummy_irq_chip.irq_ack;
+ pax_close_kernel();
for (i = irq_base; i < irq_end; i++) {
irq_set_chip_and_handler(i, &twl4030_irq_chip,
goto error_idr_alloc;
c2dev->id = ret;
- bin_attr_flash_data.size = ops->blocks_num * ops->block_size;
+ pax_open_kernel();
+ *(size_t *)&bin_attr_flash_data.size = ops->blocks_num * ops->block_size;
+ pax_close_kernel();
c2dev->dev = device_create(c2port_class, NULL, 0, c2dev,
"c2port%d", c2dev->id);
platform_set_drvdata(pdev, sid_data);
- sid_bin_attr.size = sid_data->keysize;
+ pax_open_kernel();
+ *(size_t *)&sid_bin_attr.size = sid_data->keysize;
+ pax_close_kernel();
if (device_create_bin_file(&pdev->dev, &sid_bin_attr))
return -ENODEV;
char before[BREAK_INSTR_SIZE];
char after[BREAK_INSTR_SIZE];
- probe_kernel_read(before, (char *)kgdbts_break_test,
+ probe_kernel_read(before, ktla_ktva((char *)kgdbts_break_test),
BREAK_INSTR_SIZE);
init_simple_test();
ts.tst = plant_and_detach_test;
/* Activate test with initial breakpoint */
if (!is_early)
kgdb_breakpoint();
- probe_kernel_read(after, (char *)kgdbts_break_test,
+ probe_kernel_read(after, ktla_ktva((char *)kgdbts_break_test),
BREAK_INSTR_SIZE);
if (memcmp(before, after, BREAK_INSTR_SIZE)) {
printk(KERN_CRIT "kgdbts: ERROR kgdb corrupted memory\n");
* the lid is closed. This leads to interrupts as soon as a little move
* is done.
*/
- atomic_inc(&lis3->count);
+ atomic_inc_unchecked(&lis3->count);
wake_up_interruptible(&lis3->misc_wait);
kill_fasync(&lis3->async_queue, SIGIO, POLL_IN);
if (lis3->pm_dev)
pm_runtime_get_sync(lis3->pm_dev);
- atomic_set(&lis3->count, 0);
+ atomic_set_unchecked(&lis3->count, 0);
return 0;
}
add_wait_queue(&lis3->misc_wait, &wait);
while (true) {
set_current_state(TASK_INTERRUPTIBLE);
- data = atomic_xchg(&lis3->count, 0);
+ data = atomic_xchg_unchecked(&lis3->count, 0);
if (data)
break;
struct lis3lv02d, miscdev);
poll_wait(file, &lis3->misc_wait, wait);
- if (atomic_read(&lis3->count))
+ if (atomic_read_unchecked(&lis3->count))
return POLLIN | POLLRDNORM;
return 0;
}
struct input_polled_dev *idev; /* input device */
struct platform_device *pdev; /* platform device */
struct regulator_bulk_data regulators[2];
- atomic_t count; /* interrupt count after last read */
+ atomic_unchecked_t count; /* interrupt count after last read */
union axis_conversion ac; /* hw -> logical axis */
int mapped_btns[3];
unsigned long nsec;
nsec = CLKS2NSEC(clks);
- atomic_long_inc(&mcs_op_statistics[op].count);
- atomic_long_add(nsec, &mcs_op_statistics[op].total);
+ atomic_long_inc_unchecked(&mcs_op_statistics[op].count);
+ atomic_long_add_unchecked(nsec, &mcs_op_statistics[op].total);
if (mcs_op_statistics[op].max < nsec)
mcs_op_statistics[op].max = nsec;
}
#define printstat(s, f) printstat_val(s, &gru_stats.f, #f)
-static void printstat_val(struct seq_file *s, atomic_long_t *v, char *id)
+static void printstat_val(struct seq_file *s, atomic_long_unchecked_t *v, char *id)
{
- unsigned long val = atomic_long_read(v);
+ unsigned long val = atomic_long_read_unchecked(v);
seq_printf(s, "%16lu %s\n", val, id);
}
seq_printf(s, "%-20s%12s%12s%12s\n", "#id", "count", "aver-clks", "max-clks");
for (op = 0; op < mcsop_last; op++) {
- count = atomic_long_read(&mcs_op_statistics[op].count);
- total = atomic_long_read(&mcs_op_statistics[op].total);
+ count = atomic_long_read_unchecked(&mcs_op_statistics[op].count);
+ total = atomic_long_read_unchecked(&mcs_op_statistics[op].total);
max = mcs_op_statistics[op].max;
seq_printf(s, "%-20s%12ld%12ld%12ld\n", id[op], count,
count ? total / count : 0, max);
* GRU statistics.
*/
struct gru_stats_s {
- atomic_long_t vdata_alloc;
- atomic_long_t vdata_free;
- atomic_long_t gts_alloc;
- atomic_long_t gts_free;
- atomic_long_t gms_alloc;
- atomic_long_t gms_free;
- atomic_long_t gts_double_allocate;
- atomic_long_t assign_context;
- atomic_long_t assign_context_failed;
- atomic_long_t free_context;
- atomic_long_t load_user_context;
- atomic_long_t load_kernel_context;
- atomic_long_t lock_kernel_context;
- atomic_long_t unlock_kernel_context;
- atomic_long_t steal_user_context;
- atomic_long_t steal_kernel_context;
- atomic_long_t steal_context_failed;
- atomic_long_t nopfn;
- atomic_long_t asid_new;
- atomic_long_t asid_next;
- atomic_long_t asid_wrap;
- atomic_long_t asid_reuse;
- atomic_long_t intr;
- atomic_long_t intr_cbr;
- atomic_long_t intr_tfh;
- atomic_long_t intr_spurious;
- atomic_long_t intr_mm_lock_failed;
- atomic_long_t call_os;
- atomic_long_t call_os_wait_queue;
- atomic_long_t user_flush_tlb;
- atomic_long_t user_unload_context;
- atomic_long_t user_exception;
- atomic_long_t set_context_option;
- atomic_long_t check_context_retarget_intr;
- atomic_long_t check_context_unload;
- atomic_long_t tlb_dropin;
- atomic_long_t tlb_preload_page;
- atomic_long_t tlb_dropin_fail_no_asid;
- atomic_long_t tlb_dropin_fail_upm;
- atomic_long_t tlb_dropin_fail_invalid;
- atomic_long_t tlb_dropin_fail_range_active;
- atomic_long_t tlb_dropin_fail_idle;
- atomic_long_t tlb_dropin_fail_fmm;
- atomic_long_t tlb_dropin_fail_no_exception;
- atomic_long_t tfh_stale_on_fault;
- atomic_long_t mmu_invalidate_range;
- atomic_long_t mmu_invalidate_page;
- atomic_long_t flush_tlb;
- atomic_long_t flush_tlb_gru;
- atomic_long_t flush_tlb_gru_tgh;
- atomic_long_t flush_tlb_gru_zero_asid;
-
- atomic_long_t copy_gpa;
- atomic_long_t read_gpa;
-
- atomic_long_t mesq_receive;
- atomic_long_t mesq_receive_none;
- atomic_long_t mesq_send;
- atomic_long_t mesq_send_failed;
- atomic_long_t mesq_noop;
- atomic_long_t mesq_send_unexpected_error;
- atomic_long_t mesq_send_lb_overflow;
- atomic_long_t mesq_send_qlimit_reached;
- atomic_long_t mesq_send_amo_nacked;
- atomic_long_t mesq_send_put_nacked;
- atomic_long_t mesq_page_overflow;
- atomic_long_t mesq_qf_locked;
- atomic_long_t mesq_qf_noop_not_full;
- atomic_long_t mesq_qf_switch_head_failed;
- atomic_long_t mesq_qf_unexpected_error;
- atomic_long_t mesq_noop_unexpected_error;
- atomic_long_t mesq_noop_lb_overflow;
- atomic_long_t mesq_noop_qlimit_reached;
- atomic_long_t mesq_noop_amo_nacked;
- atomic_long_t mesq_noop_put_nacked;
- atomic_long_t mesq_noop_page_overflow;
+ atomic_long_unchecked_t vdata_alloc;
+ atomic_long_unchecked_t vdata_free;
+ atomic_long_unchecked_t gts_alloc;
+ atomic_long_unchecked_t gts_free;
+ atomic_long_unchecked_t gms_alloc;
+ atomic_long_unchecked_t gms_free;
+ atomic_long_unchecked_t gts_double_allocate;
+ atomic_long_unchecked_t assign_context;
+ atomic_long_unchecked_t assign_context_failed;
+ atomic_long_unchecked_t free_context;
+ atomic_long_unchecked_t load_user_context;
+ atomic_long_unchecked_t load_kernel_context;
+ atomic_long_unchecked_t lock_kernel_context;
+ atomic_long_unchecked_t unlock_kernel_context;
+ atomic_long_unchecked_t steal_user_context;
+ atomic_long_unchecked_t steal_kernel_context;
+ atomic_long_unchecked_t steal_context_failed;
+ atomic_long_unchecked_t nopfn;
+ atomic_long_unchecked_t asid_new;
+ atomic_long_unchecked_t asid_next;
+ atomic_long_unchecked_t asid_wrap;
+ atomic_long_unchecked_t asid_reuse;
+ atomic_long_unchecked_t intr;
+ atomic_long_unchecked_t intr_cbr;
+ atomic_long_unchecked_t intr_tfh;
+ atomic_long_unchecked_t intr_spurious;
+ atomic_long_unchecked_t intr_mm_lock_failed;
+ atomic_long_unchecked_t call_os;
+ atomic_long_unchecked_t call_os_wait_queue;
+ atomic_long_unchecked_t user_flush_tlb;
+ atomic_long_unchecked_t user_unload_context;
+ atomic_long_unchecked_t user_exception;
+ atomic_long_unchecked_t set_context_option;
+ atomic_long_unchecked_t check_context_retarget_intr;
+ atomic_long_unchecked_t check_context_unload;
+ atomic_long_unchecked_t tlb_dropin;
+ atomic_long_unchecked_t tlb_preload_page;
+ atomic_long_unchecked_t tlb_dropin_fail_no_asid;
+ atomic_long_unchecked_t tlb_dropin_fail_upm;
+ atomic_long_unchecked_t tlb_dropin_fail_invalid;
+ atomic_long_unchecked_t tlb_dropin_fail_range_active;
+ atomic_long_unchecked_t tlb_dropin_fail_idle;
+ atomic_long_unchecked_t tlb_dropin_fail_fmm;
+ atomic_long_unchecked_t tlb_dropin_fail_no_exception;
+ atomic_long_unchecked_t tfh_stale_on_fault;
+ atomic_long_unchecked_t mmu_invalidate_range;
+ atomic_long_unchecked_t mmu_invalidate_page;
+ atomic_long_unchecked_t flush_tlb;
+ atomic_long_unchecked_t flush_tlb_gru;
+ atomic_long_unchecked_t flush_tlb_gru_tgh;
+ atomic_long_unchecked_t flush_tlb_gru_zero_asid;
+
+ atomic_long_unchecked_t copy_gpa;
+ atomic_long_unchecked_t read_gpa;
+
+ atomic_long_unchecked_t mesq_receive;
+ atomic_long_unchecked_t mesq_receive_none;
+ atomic_long_unchecked_t mesq_send;
+ atomic_long_unchecked_t mesq_send_failed;
+ atomic_long_unchecked_t mesq_noop;
+ atomic_long_unchecked_t mesq_send_unexpected_error;
+ atomic_long_unchecked_t mesq_send_lb_overflow;
+ atomic_long_unchecked_t mesq_send_qlimit_reached;
+ atomic_long_unchecked_t mesq_send_amo_nacked;
+ atomic_long_unchecked_t mesq_send_put_nacked;
+ atomic_long_unchecked_t mesq_page_overflow;
+ atomic_long_unchecked_t mesq_qf_locked;
+ atomic_long_unchecked_t mesq_qf_noop_not_full;
+ atomic_long_unchecked_t mesq_qf_switch_head_failed;
+ atomic_long_unchecked_t mesq_qf_unexpected_error;
+ atomic_long_unchecked_t mesq_noop_unexpected_error;
+ atomic_long_unchecked_t mesq_noop_lb_overflow;
+ atomic_long_unchecked_t mesq_noop_qlimit_reached;
+ atomic_long_unchecked_t mesq_noop_amo_nacked;
+ atomic_long_unchecked_t mesq_noop_put_nacked;
+ atomic_long_unchecked_t mesq_noop_page_overflow;
};
tghop_invalidate, mcsop_last};
struct mcs_op_statistic {
- atomic_long_t count;
- atomic_long_t total;
+ atomic_long_unchecked_t count;
+ atomic_long_unchecked_t total;
unsigned long max;
};
#define STAT(id) do { \
if (gru_options & OPT_STATS) \
- atomic_long_inc(&gru_stats.id); \
+ atomic_long_inc_unchecked(&gru_stats.id); \
} while (0)
#ifdef CONFIG_SGI_GRU_DEBUG
xpc_notify_func, void *);
void (*received) (short, int, void *);
enum xp_retval (*partid_to_nasids) (short, void *);
-};
+} __no_const;
extern struct xpc_interface xpc_interface;
}
struct xpc_interface xpc_interface = {
- (void (*)(int))xpc_notloaded,
- (void (*)(int))xpc_notloaded,
- (enum xp_retval(*)(short, int, u32, void *, u16))xpc_notloaded,
- (enum xp_retval(*)(short, int, u32, void *, u16, xpc_notify_func,
+ .connect = (void (*)(int))xpc_notloaded,
+ .disconnect = (void (*)(int))xpc_notloaded,
+ .send = (enum xp_retval(*)(short, int, u32, void *, u16))xpc_notloaded,
+ .send_notify = (enum xp_retval(*)(short, int, u32, void *, u16, xpc_notify_func,
void *))xpc_notloaded,
- (void (*)(short, int, void *))xpc_notloaded,
- (enum xp_retval(*)(short, void *))xpc_notloaded
+ .received = (void (*)(short, int, void *))xpc_notloaded,
+ .partid_to_nasids = (enum xp_retval(*)(short, void *))xpc_notloaded
};
EXPORT_SYMBOL_GPL(xpc_interface);
void (*received_payload) (struct xpc_channel *, void *);
void (*notify_senders_of_disconnect) (struct xpc_channel *);
};
+typedef struct xpc_arch_operations __no_const xpc_arch_operations_no_const;
/* struct xpc_partition act_state values (for XPC HB) */
/* found in xpc_main.c */
extern struct device *xpc_part;
extern struct device *xpc_chan;
-extern struct xpc_arch_operations xpc_arch_ops;
+extern xpc_arch_operations_no_const xpc_arch_ops;
extern int xpc_disengage_timelimit;
extern int xpc_disengage_timedout;
extern int xpc_activate_IRQ_rcvd;
.notifier_call = xpc_system_die,
};
-struct xpc_arch_operations xpc_arch_ops;
+xpc_arch_operations_no_const xpc_arch_ops;
/*
* Timer function to enforce the timelimit on the partition disengage.
if (((die_args->trapnr == X86_TRAP_MF) ||
(die_args->trapnr == X86_TRAP_XF)) &&
- !user_mode_vm(die_args->regs))
+ !user_mode(die_args->regs))
xpc_die_deactivate();
break;
if (idata->ic.postsleep_min_us)
usleep_range(idata->ic.postsleep_min_us, idata->ic.postsleep_max_us);
- if (copy_to_user(&(ic_ptr->response), cmd.resp, sizeof(cmd.resp))) {
+ if (copy_to_user(ic_ptr->response, cmd.resp, sizeof(cmd.resp))) {
err = -EFAULT;
goto cmd_rel_host;
}
int (*parse_dt)(struct dw_mci *host);
int (*execute_tuning)(struct dw_mci_slot *slot, u32 opcode,
struct dw_mci_tuning_data *tuning_data);
-};
+} __do_const;
#endif /* _DW_MMC_H_ */
mmc->caps |= MMC_CAP_CMD23;
if (variant->busy_detect) {
- mmci_ops.card_busy = mmci_card_busy;
+ pax_open_kernel();
+ *(void **)&mmci_ops.card_busy = mmci_card_busy;
+ pax_close_kernel();
mmci_write_datactrlreg(host, MCI_ST_DPSM_BUSYMODE);
mmc->caps |= MMC_CAP_WAIT_WHILE_BUSY;
mmc->max_busy_timeout = 0;
if (host->pdata->controller_flags & OMAP_HSMMC_BROKEN_MULTIBLOCK_READ) {
dev_info(&pdev->dev, "multiblock reads disabled due to 35xx erratum 2.1.1.128; MMC read performance may suffer\n");
- omap_hsmmc_ops.multi_io_quirk = omap_hsmmc_multi_io_quirk;
+ pax_open_kernel();
+ *(void **)&omap_hsmmc_ops.multi_io_quirk = omap_hsmmc_multi_io_quirk;
+ pax_close_kernel();
}
pm_runtime_enable(host->dev);
host->mmc->caps |= MMC_CAP_1_8V_DDR;
}
- if (imx_data->socdata->flags & ESDHC_FLAG_MAN_TUNING)
- sdhci_esdhc_ops.platform_execute_tuning =
+ if (imx_data->socdata->flags & ESDHC_FLAG_MAN_TUNING) {
+ pax_open_kernel();
+ *(void **)&sdhci_esdhc_ops.platform_execute_tuning =
esdhc_executing_tuning;
+ pax_close_kernel();
+ }
if (imx_data->socdata->flags & ESDHC_FLAG_STD_TUNING)
writel(readl(host->ioaddr + ESDHC_TUNING_CTRL) |
* we can use overriding functions instead of default.
*/
if (sc->no_divider) {
- sdhci_s3c_ops.set_clock = sdhci_cmu_set_clock;
- sdhci_s3c_ops.get_min_clock = sdhci_cmu_get_min_clock;
- sdhci_s3c_ops.get_max_clock = sdhci_cmu_get_max_clock;
+ pax_open_kernel();
+ *(void **)&sdhci_s3c_ops.set_clock = sdhci_cmu_set_clock;
+ *(void **)&sdhci_s3c_ops.get_min_clock = sdhci_cmu_get_min_clock;
+ *(void **)&sdhci_s3c_ops.get_max_clock = sdhci_cmu_get_max_clock;
+ pax_close_kernel();
}
/* It supports additional host capabilities if needed */
size_t totlen = 0, thislen;
int ret = 0;
size_t buflen = 0;
- static char *buffer;
+ char *buffer;
if (!ECCBUF_SIZE) {
/* We should fall back to a general writev implementation.
#include <linux/slab.h>
#include <linux/mtd/mtd.h>
#include <linux/module.h>
+#include <linux/slab.h>
#include "denali.h"
/* first try to map the upper buffer directly */
if (virt_addr_valid(this->upper_buf) &&
- !object_is_on_stack(this->upper_buf)) {
+ !object_starts_on_stack(this->upper_buf)) {
sg_init_one(sgl, this->upper_buf, this->upper_len);
ret = dma_map_sg(this->dev, sgl, 1, dr);
if (ret == 0)
#include <asm/errno.h>
#include <linux/delay.h>
#include <linux/slab.h>
+#include <linux/sched.h>
#include <linux/mtd/mtd.h>
#include <linux/mtd/nand.h>
#include <linux/mtd/nftl.h>
#define SM_CIS_VENDOR_OFFSET 0x59
static struct attribute_group *sm_create_sysfs_attributes(struct sm_ftl *ftl)
{
- struct attribute_group *attr_group;
+ attribute_group_no_const *attr_group;
struct attribute **attributes;
struct sm_sysfs_attribute *vendor_attribute;
char *vendor;
return -EMSGSIZE;
}
-struct rtnl_link_ops bond_link_ops __read_mostly = {
+struct rtnl_link_ops bond_link_ops = {
.kind = "bond",
.priv_size = sizeof(struct bonding),
.setup = bond_setup,
return -ENODEV;
}
-static struct rtnl_link_ops caif_hsi_link_ops __read_mostly = {
+static struct rtnl_link_ops caif_hsi_link_ops = {
.kind = "cfhsi",
.priv_size = sizeof(struct cfhsi),
.setup = cfhsi_setup,
config CAN_FLEXCAN
tristate "Support for Freescale FLEXCAN based chips"
- depends on ARM || PPC
+ depends on (ARM && CPU_LITTLE_ENDIAN) || PPC
---help---
Say Y here if you want to support for Freescale FlexCAN.
skb->pkt_type = PACKET_BROADCAST;
skb->ip_summed = CHECKSUM_UNNECESSARY;
+ skb_reset_mac_header(skb);
+ skb_reset_network_header(skb);
+ skb_reset_transport_header(skb);
+
can_skb_reserve(skb);
can_skb_prv(skb)->ifindex = dev->ifindex;
skb->pkt_type = PACKET_BROADCAST;
skb->ip_summed = CHECKSUM_UNNECESSARY;
+ skb_reset_mac_header(skb);
+ skb_reset_network_header(skb);
+ skb_reset_transport_header(skb);
+
can_skb_reserve(skb);
can_skb_prv(skb)->ifindex = dev->ifindex;
return -EOPNOTSUPP;
}
-static struct rtnl_link_ops can_link_ops __read_mostly = {
+static struct rtnl_link_ops can_link_ops = {
.kind = "can",
.maxtype = IFLA_CAN_MAX,
.policy = can_policy,
dev->destructor = free_netdev;
}
-static struct rtnl_link_ops vcan_link_ops __read_mostly = {
+static struct rtnl_link_ops vcan_link_ops = {
.kind = "vcan",
.setup = vcan_setup,
};
return 0;
}
-static struct rtnl_link_ops dummy_link_ops __read_mostly = {
+static struct rtnl_link_ops dummy_link_ops = {
.kind = DRV_NAME,
.setup = dummy_setup,
.validate = dummy_validate,
if (ax->plat->reg_offsets)
ei_local->reg_offset = ax->plat->reg_offsets;
else {
+ resource_size_t _mem_size = mem_size;
+ do_div(_mem_size, 0x18);
ei_local->reg_offset = ax->reg_offsets;
for (ret = 0; ret < 0x18; ret++)
- ax->reg_offsets[ret] = (mem_size / 0x18) * ret;
+ ax->reg_offsets[ret] = _mem_size * ret;
}
if (!request_mem_region(mem->start, mem_size, pdev->name)) {
return 0;
}
-static struct net_device_ops altera_tse_netdev_ops = {
+static net_device_ops_no_const altera_tse_netdev_ops __read_only = {
.ndo_open = tse_open,
.ndo_stop = tse_shutdown,
.ndo_start_xmit = tse_start_xmit,
ndev->netdev_ops = &altera_tse_netdev_ops;
altera_tse_set_ethtool_ops(ndev);
+ pax_open_kernel();
altera_tse_netdev_ops.ndo_set_rx_mode = tse_set_rx_mode;
if (priv->hash_filter)
altera_tse_netdev_ops.ndo_set_rx_mode =
tse_set_rx_mode_hashfilter;
+ pax_close_kernel();
/* Scatter/gather IO is not supported,
* so it is turned off
* operations, everything works on mask values.
*/
#define XMDIO_READ(_pdata, _mmd, _reg) \
- ((_pdata)->hw_if.read_mmd_regs((_pdata), 0, \
+ ((_pdata)->hw_if->read_mmd_regs((_pdata), 0, \
MII_ADDR_C45 | (_mmd << 16) | ((_reg) & 0xffff)))
#define XMDIO_READ_BITS(_pdata, _mmd, _reg, _mask) \
(XMDIO_READ((_pdata), _mmd, _reg) & _mask)
#define XMDIO_WRITE(_pdata, _mmd, _reg, _val) \
- ((_pdata)->hw_if.write_mmd_regs((_pdata), 0, \
+ ((_pdata)->hw_if->write_mmd_regs((_pdata), 0, \
MII_ADDR_C45 | (_mmd << 16) | ((_reg) & 0xffff), (_val)))
#define XMDIO_WRITE_BITS(_pdata, _mmd, _reg, _mask, _val) \
memcpy(pdata->ets, ets, sizeof(*pdata->ets));
- pdata->hw_if.config_dcb_tc(pdata);
+ pdata->hw_if->config_dcb_tc(pdata);
return 0;
}
memcpy(pdata->pfc, pfc, sizeof(*pdata->pfc));
- pdata->hw_if.config_dcb_pfc(pdata);
+ pdata->hw_if->config_dcb_pfc(pdata);
return 0;
}
static void xgbe_wrapper_tx_descriptor_init(struct xgbe_prv_data *pdata)
{
- struct xgbe_hw_if *hw_if = &pdata->hw_if;
+ struct xgbe_hw_if *hw_if = pdata->hw_if;
struct xgbe_channel *channel;
struct xgbe_ring *ring;
struct xgbe_ring_data *rdata;
static void xgbe_wrapper_rx_descriptor_init(struct xgbe_prv_data *pdata)
{
- struct xgbe_hw_if *hw_if = &pdata->hw_if;
+ struct xgbe_hw_if *hw_if = pdata->hw_if;
struct xgbe_channel *channel;
struct xgbe_ring *ring;
struct xgbe_ring_desc *rdesc;
static void xgbe_realloc_rx_buffer(struct xgbe_channel *channel)
{
struct xgbe_prv_data *pdata = channel->pdata;
- struct xgbe_hw_if *hw_if = &pdata->hw_if;
+ struct xgbe_hw_if *hw_if = pdata->hw_if;
struct xgbe_ring *ring = channel->rx_ring;
struct xgbe_ring_data *rdata;
int i;
DBGPR("<--xgbe_realloc_rx_buffer\n");
}
-void xgbe_init_function_ptrs_desc(struct xgbe_desc_if *desc_if)
-{
- DBGPR("-->xgbe_init_function_ptrs_desc\n");
-
- desc_if->alloc_ring_resources = xgbe_alloc_ring_resources;
- desc_if->free_ring_resources = xgbe_free_ring_resources;
- desc_if->map_tx_skb = xgbe_map_tx_skb;
- desc_if->realloc_rx_buffer = xgbe_realloc_rx_buffer;
- desc_if->unmap_rdata = xgbe_unmap_rdata;
- desc_if->wrapper_tx_desc_init = xgbe_wrapper_tx_descriptor_init;
- desc_if->wrapper_rx_desc_init = xgbe_wrapper_rx_descriptor_init;
-
- DBGPR("<--xgbe_init_function_ptrs_desc\n");
-}
+const struct xgbe_desc_if default_xgbe_desc_if = {
+ .alloc_ring_resources = xgbe_alloc_ring_resources,
+ .free_ring_resources = xgbe_free_ring_resources,
+ .map_tx_skb = xgbe_map_tx_skb,
+ .realloc_rx_buffer = xgbe_realloc_rx_buffer,
+ .unmap_rdata = xgbe_unmap_rdata,
+ .wrapper_tx_desc_init = xgbe_wrapper_tx_descriptor_init,
+ .wrapper_rx_desc_init = xgbe_wrapper_rx_descriptor_init,
+};
static int xgbe_init(struct xgbe_prv_data *pdata)
{
- struct xgbe_desc_if *desc_if = &pdata->desc_if;
+ struct xgbe_desc_if *desc_if = pdata->desc_if;
int ret;
DBGPR("-->xgbe_init\n");
return 0;
}
-void xgbe_init_function_ptrs_dev(struct xgbe_hw_if *hw_if)
-{
- DBGPR("-->xgbe_init_function_ptrs\n");
-
- hw_if->tx_complete = xgbe_tx_complete;
+const struct xgbe_hw_if default_xgbe_hw_if = {
+ .tx_complete = xgbe_tx_complete,
- hw_if->set_promiscuous_mode = xgbe_set_promiscuous_mode;
- hw_if->set_all_multicast_mode = xgbe_set_all_multicast_mode;
- hw_if->add_mac_addresses = xgbe_add_mac_addresses;
- hw_if->set_mac_address = xgbe_set_mac_address;
+ .set_promiscuous_mode = xgbe_set_promiscuous_mode,
+ .set_all_multicast_mode = xgbe_set_all_multicast_mode,
+ .add_mac_addresses = xgbe_add_mac_addresses,
+ .set_mac_address = xgbe_set_mac_address,
- hw_if->enable_rx_csum = xgbe_enable_rx_csum;
- hw_if->disable_rx_csum = xgbe_disable_rx_csum;
+ .enable_rx_csum = xgbe_enable_rx_csum,
+ .disable_rx_csum = xgbe_disable_rx_csum,
- hw_if->enable_rx_vlan_stripping = xgbe_enable_rx_vlan_stripping;
- hw_if->disable_rx_vlan_stripping = xgbe_disable_rx_vlan_stripping;
- hw_if->enable_rx_vlan_filtering = xgbe_enable_rx_vlan_filtering;
- hw_if->disable_rx_vlan_filtering = xgbe_disable_rx_vlan_filtering;
- hw_if->update_vlan_hash_table = xgbe_update_vlan_hash_table;
+ .enable_rx_vlan_stripping = xgbe_enable_rx_vlan_stripping,
+ .disable_rx_vlan_stripping = xgbe_disable_rx_vlan_stripping,
+ .enable_rx_vlan_filtering = xgbe_enable_rx_vlan_filtering,
+ .disable_rx_vlan_filtering = xgbe_disable_rx_vlan_filtering,
+ .update_vlan_hash_table = xgbe_update_vlan_hash_table,
- hw_if->read_mmd_regs = xgbe_read_mmd_regs;
- hw_if->write_mmd_regs = xgbe_write_mmd_regs;
+ .read_mmd_regs = xgbe_read_mmd_regs,
+ .write_mmd_regs = xgbe_write_mmd_regs,
- hw_if->set_gmii_speed = xgbe_set_gmii_speed;
- hw_if->set_gmii_2500_speed = xgbe_set_gmii_2500_speed;
- hw_if->set_xgmii_speed = xgbe_set_xgmii_speed;
+ .set_gmii_speed = xgbe_set_gmii_speed,
+ .set_gmii_2500_speed = xgbe_set_gmii_2500_speed,
+ .set_xgmii_speed = xgbe_set_xgmii_speed,
- hw_if->enable_tx = xgbe_enable_tx;
- hw_if->disable_tx = xgbe_disable_tx;
- hw_if->enable_rx = xgbe_enable_rx;
- hw_if->disable_rx = xgbe_disable_rx;
+ .enable_tx = xgbe_enable_tx,
+ .disable_tx = xgbe_disable_tx,
+ .enable_rx = xgbe_enable_rx,
+ .disable_rx = xgbe_disable_rx,
- hw_if->powerup_tx = xgbe_powerup_tx;
- hw_if->powerdown_tx = xgbe_powerdown_tx;
- hw_if->powerup_rx = xgbe_powerup_rx;
- hw_if->powerdown_rx = xgbe_powerdown_rx;
+ .powerup_tx = xgbe_powerup_tx,
+ .powerdown_tx = xgbe_powerdown_tx,
+ .powerup_rx = xgbe_powerup_rx,
+ .powerdown_rx = xgbe_powerdown_rx,
- hw_if->dev_xmit = xgbe_dev_xmit;
- hw_if->dev_read = xgbe_dev_read;
- hw_if->enable_int = xgbe_enable_int;
- hw_if->disable_int = xgbe_disable_int;
- hw_if->init = xgbe_init;
- hw_if->exit = xgbe_exit;
+ .dev_xmit = xgbe_dev_xmit,
+ .dev_read = xgbe_dev_read,
+ .enable_int = xgbe_enable_int,
+ .disable_int = xgbe_disable_int,
+ .init = xgbe_init,
+ .exit = xgbe_exit,
/* Descriptor related Sequences have to be initialized here */
- hw_if->tx_desc_init = xgbe_tx_desc_init;
- hw_if->rx_desc_init = xgbe_rx_desc_init;
- hw_if->tx_desc_reset = xgbe_tx_desc_reset;
- hw_if->rx_desc_reset = xgbe_rx_desc_reset;
- hw_if->is_last_desc = xgbe_is_last_desc;
- hw_if->is_context_desc = xgbe_is_context_desc;
- hw_if->tx_start_xmit = xgbe_tx_start_xmit;
+ .tx_desc_init = xgbe_tx_desc_init,
+ .rx_desc_init = xgbe_rx_desc_init,
+ .tx_desc_reset = xgbe_tx_desc_reset,
+ .rx_desc_reset = xgbe_rx_desc_reset,
+ .is_last_desc = xgbe_is_last_desc,
+ .is_context_desc = xgbe_is_context_desc,
+ .tx_start_xmit = xgbe_tx_start_xmit,
/* For FLOW ctrl */
- hw_if->config_tx_flow_control = xgbe_config_tx_flow_control;
- hw_if->config_rx_flow_control = xgbe_config_rx_flow_control;
+ .config_tx_flow_control = xgbe_config_tx_flow_control,
+ .config_rx_flow_control = xgbe_config_rx_flow_control,
/* For RX coalescing */
- hw_if->config_rx_coalesce = xgbe_config_rx_coalesce;
- hw_if->config_tx_coalesce = xgbe_config_tx_coalesce;
- hw_if->usec_to_riwt = xgbe_usec_to_riwt;
- hw_if->riwt_to_usec = xgbe_riwt_to_usec;
+ .config_rx_coalesce = xgbe_config_rx_coalesce,
+ .config_tx_coalesce = xgbe_config_tx_coalesce,
+ .usec_to_riwt = xgbe_usec_to_riwt,
+ .riwt_to_usec = xgbe_riwt_to_usec,
/* For RX and TX threshold config */
- hw_if->config_rx_threshold = xgbe_config_rx_threshold;
- hw_if->config_tx_threshold = xgbe_config_tx_threshold;
+ .config_rx_threshold = xgbe_config_rx_threshold,
+ .config_tx_threshold = xgbe_config_tx_threshold,
/* For RX and TX Store and Forward Mode config */
- hw_if->config_rsf_mode = xgbe_config_rsf_mode;
- hw_if->config_tsf_mode = xgbe_config_tsf_mode;
+ .config_rsf_mode = xgbe_config_rsf_mode,
+ .config_tsf_mode = xgbe_config_tsf_mode,
/* For TX DMA Operating on Second Frame config */
- hw_if->config_osp_mode = xgbe_config_osp_mode;
+ .config_osp_mode = xgbe_config_osp_mode,
/* For RX and TX PBL config */
- hw_if->config_rx_pbl_val = xgbe_config_rx_pbl_val;
- hw_if->get_rx_pbl_val = xgbe_get_rx_pbl_val;
- hw_if->config_tx_pbl_val = xgbe_config_tx_pbl_val;
- hw_if->get_tx_pbl_val = xgbe_get_tx_pbl_val;
- hw_if->config_pblx8 = xgbe_config_pblx8;
+ .config_rx_pbl_val = xgbe_config_rx_pbl_val,
+ .get_rx_pbl_val = xgbe_get_rx_pbl_val,
+ .config_tx_pbl_val = xgbe_config_tx_pbl_val,
+ .get_tx_pbl_val = xgbe_get_tx_pbl_val,
+ .config_pblx8 = xgbe_config_pblx8,
/* For MMC statistics support */
- hw_if->tx_mmc_int = xgbe_tx_mmc_int;
- hw_if->rx_mmc_int = xgbe_rx_mmc_int;
- hw_if->read_mmc_stats = xgbe_read_mmc_stats;
+ .tx_mmc_int = xgbe_tx_mmc_int,
+ .rx_mmc_int = xgbe_rx_mmc_int,
+ .read_mmc_stats = xgbe_read_mmc_stats,
/* For PTP config */
- hw_if->config_tstamp = xgbe_config_tstamp;
- hw_if->update_tstamp_addend = xgbe_update_tstamp_addend;
- hw_if->set_tstamp_time = xgbe_set_tstamp_time;
- hw_if->get_tstamp_time = xgbe_get_tstamp_time;
- hw_if->get_tx_tstamp = xgbe_get_tx_tstamp;
+ .config_tstamp = xgbe_config_tstamp,
+ .update_tstamp_addend = xgbe_update_tstamp_addend,
+ .set_tstamp_time = xgbe_set_tstamp_time,
+ .get_tstamp_time = xgbe_get_tstamp_time,
+ .get_tx_tstamp = xgbe_get_tx_tstamp,
/* For Data Center Bridging config */
- hw_if->config_dcb_tc = xgbe_config_dcb_tc;
- hw_if->config_dcb_pfc = xgbe_config_dcb_pfc;
+ .config_dcb_tc = xgbe_config_dcb_tc,
+ .config_dcb_pfc = xgbe_config_dcb_pfc,
/* For Receive Side Scaling */
- hw_if->enable_rss = xgbe_enable_rss;
- hw_if->disable_rss = xgbe_disable_rss;
- hw_if->set_rss_hash_key = xgbe_set_rss_hash_key;
- hw_if->set_rss_lookup_table = xgbe_set_rss_lookup_table;
-
- DBGPR("<--xgbe_init_function_ptrs\n");
-}
+ .enable_rss = xgbe_enable_rss,
+ .disable_rss = xgbe_disable_rss,
+ .set_rss_hash_key = xgbe_set_rss_hash_key,
+ .set_rss_lookup_table = xgbe_set_rss_lookup_table,
+};
* support, tell it now
*/
if (ring->tx.xmit_more)
- pdata->hw_if.tx_start_xmit(channel, ring);
+ pdata->hw_if->tx_start_xmit(channel, ring);
return NETDEV_TX_BUSY;
}
static void xgbe_enable_rx_tx_ints(struct xgbe_prv_data *pdata)
{
- struct xgbe_hw_if *hw_if = &pdata->hw_if;
+ struct xgbe_hw_if *hw_if = pdata->hw_if;
struct xgbe_channel *channel;
enum xgbe_int int_id;
unsigned int i;
static void xgbe_disable_rx_tx_ints(struct xgbe_prv_data *pdata)
{
- struct xgbe_hw_if *hw_if = &pdata->hw_if;
+ struct xgbe_hw_if *hw_if = pdata->hw_if;
struct xgbe_channel *channel;
enum xgbe_int int_id;
unsigned int i;
static irqreturn_t xgbe_isr(int irq, void *data)
{
struct xgbe_prv_data *pdata = data;
- struct xgbe_hw_if *hw_if = &pdata->hw_if;
+ struct xgbe_hw_if *hw_if = pdata->hw_if;
struct xgbe_channel *channel;
unsigned int dma_isr, dma_ch_isr;
unsigned int mac_isr, mac_tssr;
void xgbe_init_tx_coalesce(struct xgbe_prv_data *pdata)
{
- struct xgbe_hw_if *hw_if = &pdata->hw_if;
+ struct xgbe_hw_if *hw_if = pdata->hw_if;
DBGPR("-->xgbe_init_tx_coalesce\n");
void xgbe_init_rx_coalesce(struct xgbe_prv_data *pdata)
{
- struct xgbe_hw_if *hw_if = &pdata->hw_if;
+ struct xgbe_hw_if *hw_if = pdata->hw_if;
DBGPR("-->xgbe_init_rx_coalesce\n");
static void xgbe_free_tx_data(struct xgbe_prv_data *pdata)
{
- struct xgbe_desc_if *desc_if = &pdata->desc_if;
+ struct xgbe_desc_if *desc_if = pdata->desc_if;
struct xgbe_channel *channel;
struct xgbe_ring *ring;
struct xgbe_ring_data *rdata;
static void xgbe_free_rx_data(struct xgbe_prv_data *pdata)
{
- struct xgbe_desc_if *desc_if = &pdata->desc_if;
+ struct xgbe_desc_if *desc_if = pdata->desc_if;
struct xgbe_channel *channel;
struct xgbe_ring *ring;
struct xgbe_ring_data *rdata;
static void xgbe_adjust_link(struct net_device *netdev)
{
struct xgbe_prv_data *pdata = netdev_priv(netdev);
- struct xgbe_hw_if *hw_if = &pdata->hw_if;
+ struct xgbe_hw_if *hw_if = pdata->hw_if;
struct phy_device *phydev = pdata->phydev;
int new_state = 0;
int xgbe_powerdown(struct net_device *netdev, unsigned int caller)
{
struct xgbe_prv_data *pdata = netdev_priv(netdev);
- struct xgbe_hw_if *hw_if = &pdata->hw_if;
+ struct xgbe_hw_if *hw_if = pdata->hw_if;
unsigned long flags;
DBGPR("-->xgbe_powerdown\n");
int xgbe_powerup(struct net_device *netdev, unsigned int caller)
{
struct xgbe_prv_data *pdata = netdev_priv(netdev);
- struct xgbe_hw_if *hw_if = &pdata->hw_if;
+ struct xgbe_hw_if *hw_if = pdata->hw_if;
unsigned long flags;
DBGPR("-->xgbe_powerup\n");
static int xgbe_start(struct xgbe_prv_data *pdata)
{
- struct xgbe_hw_if *hw_if = &pdata->hw_if;
+ struct xgbe_hw_if *hw_if = pdata->hw_if;
struct net_device *netdev = pdata->netdev;
DBGPR("-->xgbe_start\n");
static void xgbe_stop(struct xgbe_prv_data *pdata)
{
- struct xgbe_hw_if *hw_if = &pdata->hw_if;
+ struct xgbe_hw_if *hw_if = pdata->hw_if;
struct xgbe_channel *channel;
struct net_device *netdev = pdata->netdev;
struct netdev_queue *txq;
static void xgbe_restart_dev(struct xgbe_prv_data *pdata, unsigned int reset)
{
struct xgbe_channel *channel;
- struct xgbe_hw_if *hw_if = &pdata->hw_if;
+ struct xgbe_hw_if *hw_if = pdata->hw_if;
unsigned int i;
DBGPR("-->xgbe_restart_dev\n");
return -ERANGE;
}
- pdata->hw_if.config_tstamp(pdata, mac_tscr);
+ pdata->hw_if->config_tstamp(pdata, mac_tscr);
memcpy(&pdata->tstamp_config, &config, sizeof(config));
static int xgbe_open(struct net_device *netdev)
{
struct xgbe_prv_data *pdata = netdev_priv(netdev);
- struct xgbe_hw_if *hw_if = &pdata->hw_if;
- struct xgbe_desc_if *desc_if = &pdata->desc_if;
+ struct xgbe_hw_if *hw_if = pdata->hw_if;
+ struct xgbe_desc_if *desc_if = pdata->desc_if;
struct xgbe_channel *channel = NULL;
unsigned int i = 0;
int ret;
static int xgbe_close(struct net_device *netdev)
{
struct xgbe_prv_data *pdata = netdev_priv(netdev);
- struct xgbe_hw_if *hw_if = &pdata->hw_if;
- struct xgbe_desc_if *desc_if = &pdata->desc_if;
+ struct xgbe_hw_if *hw_if = pdata->hw_if;
+ struct xgbe_desc_if *desc_if = pdata->desc_if;
struct xgbe_channel *channel;
unsigned int i;
static int xgbe_xmit(struct sk_buff *skb, struct net_device *netdev)
{
struct xgbe_prv_data *pdata = netdev_priv(netdev);
- struct xgbe_hw_if *hw_if = &pdata->hw_if;
- struct xgbe_desc_if *desc_if = &pdata->desc_if;
+ struct xgbe_hw_if *hw_if = pdata->hw_if;
+ struct xgbe_desc_if *desc_if = pdata->desc_if;
struct xgbe_channel *channel;
struct xgbe_ring *ring;
struct xgbe_packet_data *packet;
static void xgbe_set_rx_mode(struct net_device *netdev)
{
struct xgbe_prv_data *pdata = netdev_priv(netdev);
- struct xgbe_hw_if *hw_if = &pdata->hw_if;
+ struct xgbe_hw_if *hw_if = pdata->hw_if;
unsigned int pr_mode, am_mode;
DBGPR("-->xgbe_set_rx_mode\n");
static int xgbe_set_mac_address(struct net_device *netdev, void *addr)
{
struct xgbe_prv_data *pdata = netdev_priv(netdev);
- struct xgbe_hw_if *hw_if = &pdata->hw_if;
+ struct xgbe_hw_if *hw_if = pdata->hw_if;
struct sockaddr *saddr = addr;
DBGPR("-->xgbe_set_mac_address\n");
DBGPR("-->%s\n", __func__);
- pdata->hw_if.read_mmc_stats(pdata);
+ pdata->hw_if->read_mmc_stats(pdata);
s->rx_packets = pstats->rxframecount_gb;
s->rx_bytes = pstats->rxoctetcount_gb;
u16 vid)
{
struct xgbe_prv_data *pdata = netdev_priv(netdev);
- struct xgbe_hw_if *hw_if = &pdata->hw_if;
+ struct xgbe_hw_if *hw_if = pdata->hw_if;
DBGPR("-->%s\n", __func__);
u16 vid)
{
struct xgbe_prv_data *pdata = netdev_priv(netdev);
- struct xgbe_hw_if *hw_if = &pdata->hw_if;
+ struct xgbe_hw_if *hw_if = pdata->hw_if;
DBGPR("-->%s\n", __func__);
netdev_features_t features)
{
struct xgbe_prv_data *pdata = netdev_priv(netdev);
- struct xgbe_hw_if *hw_if = &pdata->hw_if;
+ struct xgbe_hw_if *hw_if = pdata->hw_if;
netdev_features_t rxhash, rxcsum, rxvlan, rxvlan_filter;
int ret = 0;
static void xgbe_rx_refresh(struct xgbe_channel *channel)
{
struct xgbe_prv_data *pdata = channel->pdata;
- struct xgbe_desc_if *desc_if = &pdata->desc_if;
+ struct xgbe_desc_if *desc_if = pdata->desc_if;
struct xgbe_ring *ring = channel->rx_ring;
struct xgbe_ring_data *rdata;
static int xgbe_tx_poll(struct xgbe_channel *channel)
{
struct xgbe_prv_data *pdata = channel->pdata;
- struct xgbe_hw_if *hw_if = &pdata->hw_if;
- struct xgbe_desc_if *desc_if = &pdata->desc_if;
+ struct xgbe_hw_if *hw_if = pdata->hw_if;
+ struct xgbe_desc_if *desc_if = pdata->desc_if;
struct xgbe_ring *ring = channel->tx_ring;
struct xgbe_ring_data *rdata;
struct xgbe_ring_desc *rdesc;
static int xgbe_rx_poll(struct xgbe_channel *channel, int budget)
{
struct xgbe_prv_data *pdata = channel->pdata;
- struct xgbe_hw_if *hw_if = &pdata->hw_if;
+ struct xgbe_hw_if *hw_if = pdata->hw_if;
struct xgbe_ring *ring = channel->rx_ring;
struct xgbe_ring_data *rdata;
struct xgbe_packet_data *packet;
DBGPR("-->%s\n", __func__);
- pdata->hw_if.read_mmc_stats(pdata);
+ pdata->hw_if->read_mmc_stats(pdata);
for (i = 0; i < XGBE_STATS_COUNT; i++) {
stat = (u8 *)pdata + xgbe_gstring_stats[i].stat_offset;
*data++ = *(u64 *)stat;
struct ethtool_coalesce *ec)
{
struct xgbe_prv_data *pdata = netdev_priv(netdev);
- struct xgbe_hw_if *hw_if = &pdata->hw_if;
+ struct xgbe_hw_if *hw_if = pdata->hw_if;
unsigned int riwt;
DBGPR("-->xgbe_get_coalesce\n");
struct ethtool_coalesce *ec)
{
struct xgbe_prv_data *pdata = netdev_priv(netdev);
- struct xgbe_hw_if *hw_if = &pdata->hw_if;
+ struct xgbe_hw_if *hw_if = pdata->hw_if;
unsigned int rx_frames, rx_riwt, rx_usecs;
unsigned int tx_frames, tx_usecs;
DBGPR("<--xgbe_default_config\n");
}
-static void xgbe_init_all_fptrs(struct xgbe_prv_data *pdata)
-{
- xgbe_init_function_ptrs_dev(&pdata->hw_if);
- xgbe_init_function_ptrs_desc(&pdata->desc_if);
-}
-
static int xgbe_probe(struct platform_device *pdev)
{
struct xgbe_prv_data *pdata;
netdev->base_addr = (unsigned long)pdata->xgmac_regs;
/* Set all the function pointers */
- xgbe_init_all_fptrs(pdata);
- hw_if = &pdata->hw_if;
- desc_if = &pdata->desc_if;
+ hw_if = pdata->hw_if = &default_xgbe_hw_if;
+ desc_if = pdata->desc_if = &default_xgbe_desc_if;
/* Issue software reset to device */
hw_if->exit(pdata);
static int xgbe_mdio_read(struct mii_bus *mii, int prtad, int mmd_reg)
{
struct xgbe_prv_data *pdata = mii->priv;
- struct xgbe_hw_if *hw_if = &pdata->hw_if;
+ struct xgbe_hw_if *hw_if = pdata->hw_if;
int mmd_data;
DBGPR_MDIO("-->xgbe_mdio_read: prtad=%#x mmd_reg=%#x\n",
u16 mmd_val)
{
struct xgbe_prv_data *pdata = mii->priv;
- struct xgbe_hw_if *hw_if = &pdata->hw_if;
+ struct xgbe_hw_if *hw_if = pdata->hw_if;
int mmd_data = mmd_val;
DBGPR_MDIO("-->xgbe_mdio_write: prtad=%#x mmd_reg=%#x mmd_data=%#x\n",
tstamp_cc);
u64 nsec;
- nsec = pdata->hw_if.get_tstamp_time(pdata);
+ nsec = pdata->hw_if->get_tstamp_time(pdata);
return nsec;
}
spin_lock_irqsave(&pdata->tstamp_lock, flags);
- pdata->hw_if.update_tstamp_addend(pdata, addend);
+ pdata->hw_if->update_tstamp_addend(pdata, addend);
spin_unlock_irqrestore(&pdata->tstamp_lock, flags);
int dev_irq;
unsigned int per_channel_irq;
- struct xgbe_hw_if hw_if;
- struct xgbe_desc_if desc_if;
+ const struct xgbe_hw_if *hw_if;
+ const struct xgbe_desc_if *desc_if;
/* AXI DMA settings */
unsigned int axdomain;
#endif
};
+extern const struct xgbe_hw_if default_xgbe_hw_if;
+extern const struct xgbe_desc_if default_xgbe_desc_if;
+
/* Function prototypes*/
void xgbe_init_function_ptrs_dev(struct xgbe_hw_if *);
static inline void bnx2x_init_bp_objs(struct bnx2x *bp)
{
/* RX_MODE controlling object */
- bnx2x_init_rx_mode_obj(bp, &bp->rx_mode_obj);
+ bnx2x_init_rx_mode_obj(bp);
/* multicast configuration controlling object */
bnx2x_init_mcast_obj(bp, &bp->mcast_obj, bp->fp->cl_id, bp->fp->cid,
return rc;
}
-void bnx2x_init_rx_mode_obj(struct bnx2x *bp,
- struct bnx2x_rx_mode_obj *o)
+void bnx2x_init_rx_mode_obj(struct bnx2x *bp)
{
if (CHIP_IS_E1x(bp)) {
- o->wait_comp = bnx2x_empty_rx_mode_wait;
- o->config_rx_mode = bnx2x_set_rx_mode_e1x;
+ bp->rx_mode_obj.wait_comp = bnx2x_empty_rx_mode_wait;
+ bp->rx_mode_obj.config_rx_mode = bnx2x_set_rx_mode_e1x;
} else {
- o->wait_comp = bnx2x_wait_rx_mode_comp_e2;
- o->config_rx_mode = bnx2x_set_rx_mode_e2;
+ bp->rx_mode_obj.wait_comp = bnx2x_wait_rx_mode_comp_e2;
+ bp->rx_mode_obj.config_rx_mode = bnx2x_set_rx_mode_e2;
}
}
/********************* RX MODE ****************/
-void bnx2x_init_rx_mode_obj(struct bnx2x *bp,
- struct bnx2x_rx_mode_obj *o);
+void bnx2x_init_rx_mode_obj(struct bnx2x *bp);
/**
* bnx2x_config_rx_mode - Send and RX_MODE ramrod according to the provided parameters.
#define CHIPREV_ID_5750_A0 0x4000
#define CHIPREV_ID_5750_A1 0x4001
#define CHIPREV_ID_5750_A3 0x4003
+#define CHIPREV_ID_5750_C1 0x4201
#define CHIPREV_ID_5750_C2 0x4202
#define CHIPREV_ID_5752_A0_HW 0x5000
#define CHIPREV_ID_5752_A0 0x6000
}
static struct bfa_ioc_cbfn bna_ioceth_cbfn = {
- bna_cb_ioceth_enable,
- bna_cb_ioceth_disable,
- bna_cb_ioceth_hbfail,
- bna_cb_ioceth_reset
+ .enable_cbfn = bna_cb_ioceth_enable,
+ .disable_cbfn = bna_cb_ioceth_disable,
+ .hbfail_cbfn = bna_cb_ioceth_hbfail,
+ .reset_cbfn = bna_cb_ioceth_reset
};
static void bna_attr_init(struct bna_ioceth *ioceth)
*/
struct l2t_skb_cb {
arp_failure_handler_func arp_failure_handler;
-};
+} __no_const;
#define L2T_SKB_CB(skb) ((struct l2t_skb_cb *)(skb)->cb)
int i;
struct adapter *ap = netdev2adap(dev);
- static const unsigned int *reg_ranges;
+ const unsigned int *reg_ranges;
int arr_size = 0, buf_size = 0;
if (is_t4(ap->params.chip)) {
for (i=0; i<ETH_ALEN; i++) {
tmp.addr[i] = dev->dev_addr[i];
}
- if (copy_to_user(ioc->data, tmp.addr, ioc->len)) return -EFAULT;
+ if (ioc->len > sizeof tmp.addr || copy_to_user(ioc->data, tmp.addr, ioc->len)) return -EFAULT;
break;
case DE4X5_SET_HWADDR: /* Set the hardware address */
spin_lock_irqsave(&lp->lock, flags);
memcpy(&statbuf, &lp->pktStats, ioc->len);
spin_unlock_irqrestore(&lp->lock, flags);
- if (copy_to_user(ioc->data, &statbuf, ioc->len))
+ if (ioc->len > sizeof statbuf || copy_to_user(ioc->data, &statbuf, ioc->len))
return -EFAULT;
break;
}
if (wrapped)
newacc += 65536;
- ACCESS_ONCE(*acc) = newacc;
+ ACCESS_ONCE_RW(*acc) = newacc;
}
static void populate_erx_stats(struct be_adapter *adapter,
#include <linux/netdevice.h>
#include <linux/phy.h>
#include <linux/platform_device.h>
+#include <linux/interrupt.h>
+#include <linux/irqreturn.h>
#include <net/ip.h>
#include "ftgmac100.h"
#include <linux/module.h>
#include <linux/netdevice.h>
#include <linux/platform_device.h>
+#include <linux/interrupt.h>
+#include <linux/irqreturn.h>
#include "ftmac100.h"
wr32(hw, I40E_PRTTSYN_INC_H, incval >> 32);
/* Update the base adjustement value. */
- ACCESS_ONCE(pf->ptp_base_adj) = incval;
+ ACCESS_ONCE_RW(pf->ptp_base_adj) = incval;
smp_mb(); /* Force the above update. */
}
}
/* update the base incval used to calculate frequency adjustment */
- ACCESS_ONCE(adapter->base_incval) = incval;
+ ACCESS_ONCE_RW(adapter->base_incval) = incval;
smp_mb();
/* need lock to prevent incorrect read while modifying cyclecounter */
wmb();
/* we want to dirty this cache line once */
- ACCESS_ONCE(ring->last_nr_txbb) = last_nr_txbb;
- ACCESS_ONCE(ring->cons) = ring_cons + txbbs_skipped;
+ ACCESS_ONCE_RW(ring->last_nr_txbb) = last_nr_txbb;
+ ACCESS_ONCE_RW(ring->cons) = ring_cons + txbbs_skipped;
netdev_tx_completed_queue(ring->tx_queue, packets, bytes);
struct __vxge_hw_fifo *fifo;
struct vxge_hw_fifo_config *config;
u32 txdl_size, txdl_per_memblock;
- struct vxge_hw_mempool_cbs fifo_mp_callback;
+ static struct vxge_hw_mempool_cbs fifo_mp_callback = {
+ .item_func_alloc = __vxge_hw_fifo_mempool_item_alloc,
+ };
+
struct __vxge_hw_virtualpath *vpath;
if ((vp == NULL) || (attr == NULL)) {
goto exit;
}
- fifo_mp_callback.item_func_alloc = __vxge_hw_fifo_mempool_item_alloc;
-
fifo->mempool =
__vxge_hw_mempool_create(vpath->hldev,
fifo->config->memblock_size,
max_tx_rings = QLCNIC_MAX_VNIC_TX_RINGS;
} else if (ret == QLC_83XX_DEFAULT_OPMODE) {
ahw->nic_mode = QLCNIC_DEFAULT_MODE;
- adapter->nic_ops->init_driver = qlcnic_83xx_init_default_driver;
+ pax_open_kernel();
+ *(void **)&adapter->nic_ops->init_driver = qlcnic_83xx_init_default_driver;
+ pax_close_kernel();
ahw->idc.state_entry = qlcnic_83xx_idc_ready_state_entry;
max_sds_rings = QLCNIC_MAX_SDS_RINGS;
max_tx_rings = QLCNIC_MAX_TX_RINGS;
case QLCNIC_NON_PRIV_FUNC:
ahw->op_mode = QLCNIC_NON_PRIV_FUNC;
ahw->idc.state_entry = qlcnic_83xx_idc_ready_state_entry;
- nic_ops->init_driver = qlcnic_83xx_init_non_privileged_vnic;
+ pax_open_kernel();
+ *(void **)&nic_ops->init_driver = qlcnic_83xx_init_non_privileged_vnic;
+ pax_close_kernel();
break;
case QLCNIC_PRIV_FUNC:
ahw->op_mode = QLCNIC_PRIV_FUNC;
ahw->idc.state_entry = qlcnic_83xx_idc_vnic_pf_entry;
- nic_ops->init_driver = qlcnic_83xx_init_privileged_vnic;
+ pax_open_kernel();
+ *(void **)&nic_ops->init_driver = qlcnic_83xx_init_privileged_vnic;
+ pax_close_kernel();
break;
case QLCNIC_MGMT_FUNC:
ahw->op_mode = QLCNIC_MGMT_FUNC;
ahw->idc.state_entry = qlcnic_83xx_idc_ready_state_entry;
- nic_ops->init_driver = qlcnic_83xx_init_mgmt_vnic;
+ pax_open_kernel();
+ *(void **)&nic_ops->init_driver = qlcnic_83xx_init_mgmt_vnic;
+ pax_close_kernel();
break;
default:
dev_err(&adapter->pdev->dev, "Invalid Virtual NIC opmode\n");
int qlcnic_dump_fw(struct qlcnic_adapter *adapter)
{
struct qlcnic_fw_dump *fw_dump = &adapter->ahw->fw_dump;
- static const struct qlcnic_dump_operations *fw_dump_ops;
+ const struct qlcnic_dump_operations *fw_dump_ops;
struct qlcnic_83xx_dump_template_hdr *hdr_83xx;
u32 entry_offset, dump, no_entries, buf_offset = 0;
int i, k, ops_cnt, ops_index, dump_size = 0;
struct mdio_ops {
void (*write)(struct rtl8169_private *, int, int);
int (*read)(struct rtl8169_private *, int);
- } mdio_ops;
+ } __no_const mdio_ops;
struct pll_power_ops {
void (*down)(struct rtl8169_private *);
void (*up)(struct rtl8169_private *);
- } pll_power_ops;
+ } __no_const pll_power_ops;
struct jumbo_ops {
void (*enable)(struct rtl8169_private *);
void (*disable)(struct rtl8169_private *);
- } jumbo_ops;
+ } __no_const jumbo_ops;
struct csi_ops {
void (*write)(struct rtl8169_private *, int, int);
u32 (*read)(struct rtl8169_private *, int);
- } csi_ops;
+ } __no_const csi_ops;
int (*set_speed)(struct net_device *, u8 aneg, u16 sp, u8 dpx, u32 adv);
int (*get_settings)(struct net_device *, struct ethtool_cmd *);
ptp->start.dma_addr);
/* Clear flag that signals MC ready */
- ACCESS_ONCE(*start) = 0;
+ ACCESS_ONCE_RW(*start) = 0;
rc = efx_mcdi_rpc_start(efx, MC_CMD_PTP, synch_buf,
MC_CMD_PTP_IN_SYNCHRONIZE_LEN);
EFX_BUG_ON_PARANOID(rc);
writel(value, ioaddr + MMC_CNTRL);
- pr_debug("stmmac: MMC ctrl register (offset 0x%x): 0x%08x\n",
- MMC_CNTRL, value);
+// pr_debug("stmmac: MMC ctrl register (offset 0x%x): 0x%08x\n",
+// MMC_CNTRL, value);
}
/* To mask all all interrupts.*/
enum rndis_device_state state;
bool link_state;
bool link_change;
- atomic_t new_req_id;
+ atomic_unchecked_t new_req_id;
spinlock_t request_lock;
struct list_head req_list;
* template
*/
set = &rndis_msg->msg.set_req;
- set->req_id = atomic_inc_return(&dev->new_req_id);
+ set->req_id = atomic_inc_return_unchecked(&dev->new_req_id);
/* Add to the request list */
spin_lock_irqsave(&dev->request_lock, flags);
/* Setup the rndis set */
halt = &request->request_msg.msg.halt_req;
- halt->req_id = atomic_inc_return(&dev->new_req_id);
+ halt->req_id = atomic_inc_return_unchecked(&dev->new_req_id);
/* Ignore return since this msg is optional. */
rndis_filter_send_request(dev, request);
return 0;
}
-static struct rtnl_link_ops ifb_link_ops __read_mostly = {
+static struct rtnl_link_ops ifb_link_ops = {
.kind = "ifb",
.priv_size = sizeof(struct ifb_private),
.setup = ifb_setup,
free_nskb:
kfree_skb(nskb);
err:
- atomic_long_inc(&skb->dev->rx_dropped);
+ atomic_long_inc_unchecked(&skb->dev->rx_dropped);
}
static void macvlan_flush_sources(struct macvlan_port *port,
int macvlan_link_register(struct rtnl_link_ops *ops)
{
/* common fields */
- ops->priv_size = sizeof(struct macvlan_dev);
- ops->validate = macvlan_validate;
- ops->maxtype = IFLA_MACVLAN_MAX;
- ops->policy = macvlan_policy;
- ops->changelink = macvlan_changelink;
- ops->get_size = macvlan_get_size;
- ops->fill_info = macvlan_fill_info;
+ pax_open_kernel();
+ *(size_t *)&ops->priv_size = sizeof(struct macvlan_dev);
+ *(void **)&ops->validate = macvlan_validate;
+ *(int *)&ops->maxtype = IFLA_MACVLAN_MAX;
+ *(const void **)&ops->policy = macvlan_policy;
+ *(void **)&ops->changelink = macvlan_changelink;
+ *(void **)&ops->get_size = macvlan_get_size;
+ *(void **)&ops->fill_info = macvlan_fill_info;
+ pax_close_kernel();
return rtnl_link_register(ops);
};
return NOTIFY_DONE;
}
-static struct notifier_block macvlan_notifier_block __read_mostly = {
+static struct notifier_block macvlan_notifier_block = {
.notifier_call = macvlan_device_event,
};
dev->tx_queue_len = TUN_READQ_SIZE;
}
-static struct rtnl_link_ops macvtap_link_ops __read_mostly = {
+static struct rtnl_link_ops macvtap_link_ops = {
.kind = "macvtap",
.setup = macvtap_setup,
.newlink = macvtap_newlink,
ret = 0;
u = q->flags;
- if (copy_to_user(&ifr->ifr_name, vlan->dev->name, IFNAMSIZ) ||
+ if (copy_to_user(ifr->ifr_name, vlan->dev->name, IFNAMSIZ) ||
put_user(u, &ifr->ifr_flags))
ret = -EFAULT;
macvtap_put_vlan(vlan);
return NOTIFY_DONE;
}
-static struct notifier_block macvtap_notifier_block __read_mostly = {
+static struct notifier_block macvtap_notifier_block = {
.notifier_call = macvtap_device_event,
};
return 0;
}
-static struct rtnl_link_ops nlmon_link_ops __read_mostly = {
+static struct rtnl_link_ops nlmon_link_ops = {
.kind = "nlmon",
.priv_size = sizeof(struct nlmon),
.setup = nlmon_setup,
* zero on success.
*
*/
-static int get_phy_c45_ids(struct mii_bus *bus, int addr, u32 *phy_id,
+static int get_phy_c45_ids(struct mii_bus *bus, int addr, int *phy_id,
struct phy_c45_device_ids *c45_ids) {
int phy_reg;
int i, reg_addr;
* its return value is in turn returned.
*
*/
-static int get_phy_id(struct mii_bus *bus, int addr, u32 *phy_id,
+static int get_phy_id(struct mii_bus *bus, int addr, int *phy_id,
bool is_c45, struct phy_c45_device_ids *c45_ids)
{
int phy_reg;
struct phy_device *get_phy_device(struct mii_bus *bus, int addr, bool is_c45)
{
struct phy_c45_device_ids c45_ids = {0};
- u32 phy_id = 0;
+ int phy_id = 0;
int r;
r = get_phy_id(bus, addr, &phy_id, is_c45, &c45_ids);
void __user *addr = (void __user *) ifr->ifr_ifru.ifru_data;
struct ppp_stats stats;
struct ppp_comp_stats cstats;
- char *vers;
switch (cmd) {
case SIOCGPPPSTATS:
break;
case SIOCGPPPVER:
- vers = PPP_VERSION;
- if (copy_to_user(addr, vers, strlen(vers) + 1))
+ if (copy_to_user(addr, PPP_VERSION, sizeof(PPP_VERSION)))
break;
err = 0;
break;
register struct tcphdr *thp;
register struct iphdr *ip;
register struct cstate *cs;
- int len, hdrlen;
+ long len, hdrlen;
unsigned char *cp = icp;
/* We've got a compressed packet; read the change byte */
return TEAM_DEFAULT_NUM_RX_QUEUES;
}
-static struct rtnl_link_ops team_link_ops __read_mostly = {
+static struct rtnl_link_ops team_link_ops = {
.kind = DRV_NAME,
.priv_size = sizeof(struct team),
.setup = team_setup,
return NOTIFY_DONE;
}
-static struct notifier_block team_notifier_block __read_mostly = {
+static struct notifier_block team_notifier_block = {
.notifier_call = team_device_event,
};
return -EINVAL;
}
-static struct rtnl_link_ops tun_link_ops __read_mostly = {
+static struct rtnl_link_ops tun_link_ops = {
.kind = DRV_NAME,
.priv_size = sizeof(struct tun_struct),
.setup = tun_setup,
}
static long __tun_chr_ioctl(struct file *file, unsigned int cmd,
- unsigned long arg, int ifreq_len)
+ unsigned long arg, size_t ifreq_len)
{
struct tun_file *tfile = file->private_data;
struct tun_struct *tun;
int le;
int ret;
+ if (ifreq_len > sizeof ifr)
+ return -EFAULT;
+
if (cmd == TUNSETIFF || cmd == TUNSETQUEUE || _IOC_TYPE(cmd) == 0x89) {
if (copy_from_user(&ifr, argp, ifreq_len))
return -EFAULT;
#include <asm/byteorder.h>
#include <linux/serial_core.h>
#include <linux/serial.h>
-
+#include <asm/local.h>
#define MOD_AUTHOR "Option Wireless"
#define MOD_DESCRIPTION "USB High Speed Option driver"
struct urb *urb;
urb = serial->rx_urb[0];
- if (serial->port.count > 0) {
+ if (atomic_read(&serial->port.count) > 0) {
count = put_rxbuf_data(urb, serial);
if (count == -1)
return;
DUMP1(urb->transfer_buffer, urb->actual_length);
/* Anyone listening? */
- if (serial->port.count == 0)
+ if (atomic_read(&serial->port.count) == 0)
return;
if (serial->parent->port_spec & HSO_INFO_CRC_BUG)
tty_port_tty_set(&serial->port, tty);
/* check for port already opened, if not set the termios */
- serial->port.count++;
- if (serial->port.count == 1) {
+ if (atomic_inc_return(&serial->port.count) == 1) {
serial->rx_state = RX_IDLE;
/* Force default termio settings */
_hso_serial_set_termios(tty, NULL);
result = hso_start_serial_device(serial->parent, GFP_KERNEL);
if (result) {
hso_stop_serial_device(serial->parent);
- serial->port.count--;
+ atomic_dec(&serial->port.count);
kref_put(&serial->parent->ref, hso_serial_ref_free);
}
} else {
/* reset the rts and dtr */
/* do the actual close */
- serial->port.count--;
+ atomic_dec(&serial->port.count);
- if (serial->port.count <= 0) {
- serial->port.count = 0;
+ if (atomic_read(&serial->port.count) <= 0) {
+ atomic_set(&serial->port.count, 0);
tty_port_tty_set(&serial->port, NULL);
if (!usb_gone)
hso_stop_serial_device(serial->parent);
/* the actual setup */
spin_lock_irqsave(&serial->serial_lock, flags);
- if (serial->port.count)
+ if (atomic_read(&serial->port.count))
_hso_serial_set_termios(tty, old);
else
tty->termios = *old;
D1("Pending read interrupt on port %d\n", i);
spin_lock(&serial->serial_lock);
if (serial->rx_state == RX_IDLE &&
- serial->port.count > 0) {
+ atomic_read(&serial->port.count) > 0) {
/* Setup and send a ctrl req read on
* port i */
if (!serial->rx_urb_filled[0]) {
/* Start all serial ports */
for (i = 0; i < HSO_SERIAL_TTY_MINORS; i++) {
if (serial_table[i] && (serial_table[i]->interface == iface)) {
- if (dev2ser(serial_table[i])->port.count) {
+ if (atomic_read(&dev2ser(serial_table[i])->port.count)) {
result =
hso_start_serial_device(serial_table[i], GFP_NOIO);
hso_kick_transmit(dev2ser(serial_table[i]));
void (*unload)(struct r8152 *);
int (*eee_get)(struct r8152 *, struct ethtool_eee *);
int (*eee_set)(struct r8152 *, struct ethtool_eee *);
- } rtl_ops;
+ } __no_const rtl_ops;
int intr_interval;
u32 saved_wolopts;
/* atomic counter partially included in MAC address to make sure 2 devices
* do not end up with the same MAC - concept breaks in case of > 255 ifaces
*/
-static atomic_t iface_counter = ATOMIC_INIT(0);
+static atomic_unchecked_t iface_counter = ATOMIC_INIT(0);
/*
* SYNC Timer Delay definition used to set the expiry time
dev->net->netdev_ops = &sierra_net_device_ops;
/* change MAC addr to include, ifacenum, and to be unique */
- dev->net->dev_addr[ETH_ALEN-2] = atomic_inc_return(&iface_counter);
+ dev->net->dev_addr[ETH_ALEN-2] = atomic_inc_return_unchecked(&iface_counter);
dev->net->dev_addr[ETH_ALEN-1] = ifacenum;
/* we will have to manufacture ethernet headers, prepare template */
#define RECEIVE_AVG_WEIGHT 64
/* Minimum alignment for mergeable packet buffers. */
-#define MERGEABLE_BUFFER_ALIGN max(L1_CACHE_BYTES, 256)
+#define MERGEABLE_BUFFER_ALIGN max(L1_CACHE_BYTES, 256UL)
#define VIRTNET_DRIVER_VERSION "1.0.0"
return -EMSGSIZE;
}
-static struct rtnl_link_ops vxlan_link_ops __read_mostly = {
+static struct rtnl_link_ops vxlan_link_ops = {
.kind = "vxlan",
.maxtype = IFLA_VXLAN_MAX,
.policy = vxlan_policy,
return NOTIFY_DONE;
}
-static struct notifier_block vxlan_notifier_block __read_mostly = {
+static struct notifier_block vxlan_notifier_block = {
.notifier_call = vxlan_lowerdev_event,
};
static void write_av9110(lmc_softc_t *, u32, u32, u32, u32, u32);
lmc_media_t lmc_ds3_media = {
- lmc_ds3_init, /* special media init stuff */
- lmc_ds3_default, /* reset to default state */
- lmc_ds3_set_status, /* reset status to state provided */
- lmc_dummy_set_1, /* set clock source */
- lmc_dummy_set2_1, /* set line speed */
- lmc_ds3_set_100ft, /* set cable length */
- lmc_ds3_set_scram, /* set scrambler */
- lmc_ds3_get_link_status, /* get link status */
- lmc_dummy_set_1, /* set link status */
- lmc_ds3_set_crc_length, /* set CRC length */
- lmc_dummy_set_1, /* set T1 or E1 circuit type */
- lmc_ds3_watchdog
+ .init = lmc_ds3_init, /* special media init stuff */
+ .defaults = lmc_ds3_default, /* reset to default state */
+ .set_status = lmc_ds3_set_status, /* reset status to state provided */
+ .set_clock_source = lmc_dummy_set_1, /* set clock source */
+ .set_speed = lmc_dummy_set2_1, /* set line speed */
+ .set_cable_length = lmc_ds3_set_100ft, /* set cable length */
+ .set_scrambler = lmc_ds3_set_scram, /* set scrambler */
+ .get_link_status = lmc_ds3_get_link_status, /* get link status */
+ .set_link_status = lmc_dummy_set_1, /* set link status */
+ .set_crc_length = lmc_ds3_set_crc_length, /* set CRC length */
+ .set_circuit_type = lmc_dummy_set_1, /* set T1 or E1 circuit type */
+ .watchdog = lmc_ds3_watchdog
};
lmc_media_t lmc_hssi_media = {
- lmc_hssi_init, /* special media init stuff */
- lmc_hssi_default, /* reset to default state */
- lmc_hssi_set_status, /* reset status to state provided */
- lmc_hssi_set_clock, /* set clock source */
- lmc_dummy_set2_1, /* set line speed */
- lmc_dummy_set_1, /* set cable length */
- lmc_dummy_set_1, /* set scrambler */
- lmc_hssi_get_link_status, /* get link status */
- lmc_hssi_set_link_status, /* set link status */
- lmc_hssi_set_crc_length, /* set CRC length */
- lmc_dummy_set_1, /* set T1 or E1 circuit type */
- lmc_hssi_watchdog
+ .init = lmc_hssi_init, /* special media init stuff */
+ .defaults = lmc_hssi_default, /* reset to default state */
+ .set_status = lmc_hssi_set_status, /* reset status to state provided */
+ .set_clock_source = lmc_hssi_set_clock, /* set clock source */
+ .set_speed = lmc_dummy_set2_1, /* set line speed */
+ .set_cable_length = lmc_dummy_set_1, /* set cable length */
+ .set_scrambler = lmc_dummy_set_1, /* set scrambler */
+ .get_link_status = lmc_hssi_get_link_status, /* get link status */
+ .set_link_status = lmc_hssi_set_link_status, /* set link status */
+ .set_crc_length = lmc_hssi_set_crc_length, /* set CRC length */
+ .set_circuit_type = lmc_dummy_set_1, /* set T1 or E1 circuit type */
+ .watchdog = lmc_hssi_watchdog
};
-lmc_media_t lmc_ssi_media = { lmc_ssi_init, /* special media init stuff */
- lmc_ssi_default, /* reset to default state */
- lmc_ssi_set_status, /* reset status to state provided */
- lmc_ssi_set_clock, /* set clock source */
- lmc_ssi_set_speed, /* set line speed */
- lmc_dummy_set_1, /* set cable length */
- lmc_dummy_set_1, /* set scrambler */
- lmc_ssi_get_link_status, /* get link status */
- lmc_ssi_set_link_status, /* set link status */
- lmc_ssi_set_crc_length, /* set CRC length */
- lmc_dummy_set_1, /* set T1 or E1 circuit type */
- lmc_ssi_watchdog
+lmc_media_t lmc_ssi_media = {
+ .init = lmc_ssi_init, /* special media init stuff */
+ .defaults = lmc_ssi_default, /* reset to default state */
+ .set_status = lmc_ssi_set_status, /* reset status to state provided */
+ .set_clock_source = lmc_ssi_set_clock, /* set clock source */
+ .set_speed = lmc_ssi_set_speed, /* set line speed */
+ .set_cable_length = lmc_dummy_set_1, /* set cable length */
+ .set_scrambler = lmc_dummy_set_1, /* set scrambler */
+ .get_link_status = lmc_ssi_get_link_status, /* get link status */
+ .set_link_status = lmc_ssi_set_link_status, /* set link status */
+ .set_crc_length = lmc_ssi_set_crc_length, /* set CRC length */
+ .set_circuit_type = lmc_dummy_set_1, /* set T1 or E1 circuit type */
+ .watchdog = lmc_ssi_watchdog
};
lmc_media_t lmc_t1_media = {
- lmc_t1_init, /* special media init stuff */
- lmc_t1_default, /* reset to default state */
- lmc_t1_set_status, /* reset status to state provided */
- lmc_t1_set_clock, /* set clock source */
- lmc_dummy_set2_1, /* set line speed */
- lmc_dummy_set_1, /* set cable length */
- lmc_dummy_set_1, /* set scrambler */
- lmc_t1_get_link_status, /* get link status */
- lmc_dummy_set_1, /* set link status */
- lmc_t1_set_crc_length, /* set CRC length */
- lmc_t1_set_circuit_type, /* set T1 or E1 circuit type */
- lmc_t1_watchdog
+ .init = lmc_t1_init, /* special media init stuff */
+ .defaults = lmc_t1_default, /* reset to default state */
+ .set_status = lmc_t1_set_status, /* reset status to state provided */
+ .set_clock_source = lmc_t1_set_clock, /* set clock source */
+ .set_speed = lmc_dummy_set2_1, /* set line speed */
+ .set_cable_length = lmc_dummy_set_1, /* set cable length */
+ .set_scrambler = lmc_dummy_set_1, /* set scrambler */
+ .get_link_status = lmc_t1_get_link_status, /* get link status */
+ .set_link_status = lmc_dummy_set_1, /* set link status */
+ .set_crc_length = lmc_t1_set_crc_length, /* set CRC length */
+ .set_circuit_type = lmc_t1_set_circuit_type, /* set T1 or E1 circuit type */
+ .watchdog = lmc_t1_watchdog
};
static void
struct z8530_irqhandler z8530_sync =
{
- z8530_rx,
- z8530_tx,
- z8530_status
+ .rx = z8530_rx,
+ .tx = z8530_tx,
+ .status = z8530_status
};
EXPORT_SYMBOL(z8530_sync);
}
static struct z8530_irqhandler z8530_dma_sync = {
- z8530_dma_rx,
- z8530_dma_tx,
- z8530_dma_status
+ .rx = z8530_dma_rx,
+ .tx = z8530_dma_tx,
+ .status = z8530_dma_status
};
static struct z8530_irqhandler z8530_txdma_sync = {
- z8530_rx,
- z8530_dma_tx,
- z8530_dma_status
+ .rx = z8530_rx,
+ .tx = z8530_dma_tx,
+ .status = z8530_dma_status
};
/**
struct z8530_irqhandler z8530_nop=
{
- z8530_rx_clear,
- z8530_tx_clear,
- z8530_status_clear
+ .rx = z8530_rx_clear,
+ .tx = z8530_tx_clear,
+ .status = z8530_status_clear
};
if (i2400m->rx_roq == NULL)
goto error_roq_alloc;
- rd = kcalloc(I2400M_RO_CIN + 1, sizeof(*i2400m->rx_roq[0].log),
+ rd = kcalloc(sizeof(*i2400m->rx_roq[0].log), I2400M_RO_CIN + 1,
GFP_KERNEL);
if (rd == NULL) {
result = -ENOMEM;
struct airo_info *ai = dev->ml_priv;
int ridcode;
int enabled;
- static int (* writer)(struct airo_info *, u16 rid, const void *, int, int);
+ int (* writer)(struct airo_info *, u16 rid, const void *, int, int);
unsigned char *iobuf;
/* Only super-user can write RIDs */
}
/* Convert timeout from the DFU status to jiffies */
-static inline unsigned long at76_get_timeout(struct dfu_status *s)
+static inline unsigned long __intentional_overflow(-1) at76_get_timeout(struct dfu_status *s)
{
return msecs_to_jiffies((s->poll_timeout[2] << 16)
| (s->poll_timeout[1] << 8)
/* registered target arrival callback from the HIF layer */
int ath10k_htc_init(struct ath10k *ar)
{
- struct ath10k_hif_cb htc_callbacks;
+ static struct ath10k_hif_cb htc_callbacks = {
+ .rx_completion = ath10k_htc_rx_completion_handler,
+ .tx_completion = ath10k_htc_tx_completion_handler,
+ };
struct ath10k_htc_ep *ep = NULL;
struct ath10k_htc *htc = &ar->htc;
ath10k_htc_reset_endpoint_states(htc);
/* setup HIF layer callbacks */
- htc_callbacks.rx_completion = ath10k_htc_rx_completion_handler;
- htc_callbacks.tx_completion = ath10k_htc_tx_completion_handler;
htc->ar = ar;
/* Get HIF default pipe for HTC message exchange */
struct ath10k_htc_ops {
void (*target_send_suspend_complete)(struct ath10k *ar);
-};
+} __no_const;
struct ath10k_htc_ep_ops {
void (*ep_tx_complete)(struct ath10k *, struct sk_buff *);
void (*ep_rx_complete)(struct ath10k *, struct sk_buff *);
void (*ep_tx_credits)(struct ath10k *);
-};
+} __no_const;
/* service connection information */
struct ath10k_htc_svc_conn_req {
ads->ds_txstatus6 = ads->ds_txstatus7 = 0;
ads->ds_txstatus8 = ads->ds_txstatus9 = 0;
- ACCESS_ONCE(ads->ds_link) = i->link;
- ACCESS_ONCE(ads->ds_data) = i->buf_addr[0];
+ ACCESS_ONCE_RW(ads->ds_link) = i->link;
+ ACCESS_ONCE_RW(ads->ds_data) = i->buf_addr[0];
ctl1 = i->buf_len[0] | (i->is_last ? 0 : AR_TxMore);
ctl6 = SM(i->keytype, AR_EncrType);
if ((i->is_first || i->is_last) &&
i->aggr != AGGR_BUF_MIDDLE && i->aggr != AGGR_BUF_LAST) {
- ACCESS_ONCE(ads->ds_ctl2) = set11nTries(i->rates, 0)
+ ACCESS_ONCE_RW(ads->ds_ctl2) = set11nTries(i->rates, 0)
| set11nTries(i->rates, 1)
| set11nTries(i->rates, 2)
| set11nTries(i->rates, 3)
| (i->dur_update ? AR_DurUpdateEna : 0)
| SM(0, AR_BurstDur);
- ACCESS_ONCE(ads->ds_ctl3) = set11nRate(i->rates, 0)
+ ACCESS_ONCE_RW(ads->ds_ctl3) = set11nRate(i->rates, 0)
| set11nRate(i->rates, 1)
| set11nRate(i->rates, 2)
| set11nRate(i->rates, 3);
} else {
- ACCESS_ONCE(ads->ds_ctl2) = 0;
- ACCESS_ONCE(ads->ds_ctl3) = 0;
+ ACCESS_ONCE_RW(ads->ds_ctl2) = 0;
+ ACCESS_ONCE_RW(ads->ds_ctl3) = 0;
}
if (!i->is_first) {
- ACCESS_ONCE(ads->ds_ctl0) = 0;
- ACCESS_ONCE(ads->ds_ctl1) = ctl1;
- ACCESS_ONCE(ads->ds_ctl6) = ctl6;
+ ACCESS_ONCE_RW(ads->ds_ctl0) = 0;
+ ACCESS_ONCE_RW(ads->ds_ctl1) = ctl1;
+ ACCESS_ONCE_RW(ads->ds_ctl6) = ctl6;
return;
}
break;
}
- ACCESS_ONCE(ads->ds_ctl0) = (i->pkt_len & AR_FrameLen)
+ ACCESS_ONCE_RW(ads->ds_ctl0) = (i->pkt_len & AR_FrameLen)
| (i->flags & ATH9K_TXDESC_VMF ? AR_VirtMoreFrag : 0)
| SM(i->txpower[0], AR_XmitPower0)
| (i->flags & ATH9K_TXDESC_VEOL ? AR_VEOL : 0)
| (i->flags & ATH9K_TXDESC_RTSENA ? AR_RTSEnable :
(i->flags & ATH9K_TXDESC_CTSENA ? AR_CTSEnable : 0));
- ACCESS_ONCE(ads->ds_ctl1) = ctl1;
- ACCESS_ONCE(ads->ds_ctl6) = ctl6;
+ ACCESS_ONCE_RW(ads->ds_ctl1) = ctl1;
+ ACCESS_ONCE_RW(ads->ds_ctl6) = ctl6;
if (i->aggr == AGGR_BUF_MIDDLE || i->aggr == AGGR_BUF_LAST)
return;
- ACCESS_ONCE(ads->ds_ctl4) = set11nPktDurRTSCTS(i->rates, 0)
+ ACCESS_ONCE_RW(ads->ds_ctl4) = set11nPktDurRTSCTS(i->rates, 0)
| set11nPktDurRTSCTS(i->rates, 1);
- ACCESS_ONCE(ads->ds_ctl5) = set11nPktDurRTSCTS(i->rates, 2)
+ ACCESS_ONCE_RW(ads->ds_ctl5) = set11nPktDurRTSCTS(i->rates, 2)
| set11nPktDurRTSCTS(i->rates, 3);
- ACCESS_ONCE(ads->ds_ctl7) = set11nRateFlags(i->rates, 0)
+ ACCESS_ONCE_RW(ads->ds_ctl7) = set11nRateFlags(i->rates, 0)
| set11nRateFlags(i->rates, 1)
| set11nRateFlags(i->rates, 2)
| set11nRateFlags(i->rates, 3)
| SM(i->rtscts_rate, AR_RTSCTSRate);
- ACCESS_ONCE(ads->ds_ctl9) = SM(i->txpower[1], AR_XmitPower1);
- ACCESS_ONCE(ads->ds_ctl10) = SM(i->txpower[2], AR_XmitPower2);
- ACCESS_ONCE(ads->ds_ctl11) = SM(i->txpower[3], AR_XmitPower3);
+ ACCESS_ONCE_RW(ads->ds_ctl9) = SM(i->txpower[1], AR_XmitPower1);
+ ACCESS_ONCE_RW(ads->ds_ctl10) = SM(i->txpower[2], AR_XmitPower2);
+ ACCESS_ONCE_RW(ads->ds_ctl11) = SM(i->txpower[3], AR_XmitPower3);
}
static int ar9002_hw_proc_txdesc(struct ath_hw *ah, void *ds,
(i->qcu << AR_TxQcuNum_S) | desc_len;
checksum += val;
- ACCESS_ONCE(ads->info) = val;
+ ACCESS_ONCE_RW(ads->info) = val;
checksum += i->link;
- ACCESS_ONCE(ads->link) = i->link;
+ ACCESS_ONCE_RW(ads->link) = i->link;
checksum += i->buf_addr[0];
- ACCESS_ONCE(ads->data0) = i->buf_addr[0];
+ ACCESS_ONCE_RW(ads->data0) = i->buf_addr[0];
checksum += i->buf_addr[1];
- ACCESS_ONCE(ads->data1) = i->buf_addr[1];
+ ACCESS_ONCE_RW(ads->data1) = i->buf_addr[1];
checksum += i->buf_addr[2];
- ACCESS_ONCE(ads->data2) = i->buf_addr[2];
+ ACCESS_ONCE_RW(ads->data2) = i->buf_addr[2];
checksum += i->buf_addr[3];
- ACCESS_ONCE(ads->data3) = i->buf_addr[3];
+ ACCESS_ONCE_RW(ads->data3) = i->buf_addr[3];
checksum += (val = (i->buf_len[0] << AR_BufLen_S) & AR_BufLen);
- ACCESS_ONCE(ads->ctl3) = val;
+ ACCESS_ONCE_RW(ads->ctl3) = val;
checksum += (val = (i->buf_len[1] << AR_BufLen_S) & AR_BufLen);
- ACCESS_ONCE(ads->ctl5) = val;
+ ACCESS_ONCE_RW(ads->ctl5) = val;
checksum += (val = (i->buf_len[2] << AR_BufLen_S) & AR_BufLen);
- ACCESS_ONCE(ads->ctl7) = val;
+ ACCESS_ONCE_RW(ads->ctl7) = val;
checksum += (val = (i->buf_len[3] << AR_BufLen_S) & AR_BufLen);
- ACCESS_ONCE(ads->ctl9) = val;
+ ACCESS_ONCE_RW(ads->ctl9) = val;
checksum = (u16) (((checksum & 0xffff) + (checksum >> 16)) & 0xffff);
- ACCESS_ONCE(ads->ctl10) = checksum;
+ ACCESS_ONCE_RW(ads->ctl10) = checksum;
if (i->is_first || i->is_last) {
- ACCESS_ONCE(ads->ctl13) = set11nTries(i->rates, 0)
+ ACCESS_ONCE_RW(ads->ctl13) = set11nTries(i->rates, 0)
| set11nTries(i->rates, 1)
| set11nTries(i->rates, 2)
| set11nTries(i->rates, 3)
| (i->dur_update ? AR_DurUpdateEna : 0)
| SM(0, AR_BurstDur);
- ACCESS_ONCE(ads->ctl14) = set11nRate(i->rates, 0)
+ ACCESS_ONCE_RW(ads->ctl14) = set11nRate(i->rates, 0)
| set11nRate(i->rates, 1)
| set11nRate(i->rates, 2)
| set11nRate(i->rates, 3);
} else {
- ACCESS_ONCE(ads->ctl13) = 0;
- ACCESS_ONCE(ads->ctl14) = 0;
+ ACCESS_ONCE_RW(ads->ctl13) = 0;
+ ACCESS_ONCE_RW(ads->ctl14) = 0;
}
ads->ctl20 = 0;
ctl17 = SM(i->keytype, AR_EncrType);
if (!i->is_first) {
- ACCESS_ONCE(ads->ctl11) = 0;
- ACCESS_ONCE(ads->ctl12) = i->is_last ? 0 : AR_TxMore;
- ACCESS_ONCE(ads->ctl15) = 0;
- ACCESS_ONCE(ads->ctl16) = 0;
- ACCESS_ONCE(ads->ctl17) = ctl17;
- ACCESS_ONCE(ads->ctl18) = 0;
- ACCESS_ONCE(ads->ctl19) = 0;
+ ACCESS_ONCE_RW(ads->ctl11) = 0;
+ ACCESS_ONCE_RW(ads->ctl12) = i->is_last ? 0 : AR_TxMore;
+ ACCESS_ONCE_RW(ads->ctl15) = 0;
+ ACCESS_ONCE_RW(ads->ctl16) = 0;
+ ACCESS_ONCE_RW(ads->ctl17) = ctl17;
+ ACCESS_ONCE_RW(ads->ctl18) = 0;
+ ACCESS_ONCE_RW(ads->ctl19) = 0;
return;
}
- ACCESS_ONCE(ads->ctl11) = (i->pkt_len & AR_FrameLen)
+ ACCESS_ONCE_RW(ads->ctl11) = (i->pkt_len & AR_FrameLen)
| (i->flags & ATH9K_TXDESC_VMF ? AR_VirtMoreFrag : 0)
| SM(i->txpower[0], AR_XmitPower0)
| (i->flags & ATH9K_TXDESC_VEOL ? AR_VEOL : 0)
val = (i->flags & ATH9K_TXDESC_PAPRD) >> ATH9K_TXDESC_PAPRD_S;
ctl12 |= SM(val, AR_PAPRDChainMask);
- ACCESS_ONCE(ads->ctl12) = ctl12;
- ACCESS_ONCE(ads->ctl17) = ctl17;
+ ACCESS_ONCE_RW(ads->ctl12) = ctl12;
+ ACCESS_ONCE_RW(ads->ctl17) = ctl17;
- ACCESS_ONCE(ads->ctl15) = set11nPktDurRTSCTS(i->rates, 0)
+ ACCESS_ONCE_RW(ads->ctl15) = set11nPktDurRTSCTS(i->rates, 0)
| set11nPktDurRTSCTS(i->rates, 1);
- ACCESS_ONCE(ads->ctl16) = set11nPktDurRTSCTS(i->rates, 2)
+ ACCESS_ONCE_RW(ads->ctl16) = set11nPktDurRTSCTS(i->rates, 2)
| set11nPktDurRTSCTS(i->rates, 3);
- ACCESS_ONCE(ads->ctl18) = set11nRateFlags(i->rates, 0)
+ ACCESS_ONCE_RW(ads->ctl18) = set11nRateFlags(i->rates, 0)
| set11nRateFlags(i->rates, 1)
| set11nRateFlags(i->rates, 2)
| set11nRateFlags(i->rates, 3)
| SM(i->rtscts_rate, AR_RTSCTSRate);
- ACCESS_ONCE(ads->ctl19) = AR_Not_Sounding;
+ ACCESS_ONCE_RW(ads->ctl19) = AR_Not_Sounding;
- ACCESS_ONCE(ads->ctl20) = SM(i->txpower[1], AR_XmitPower1);
- ACCESS_ONCE(ads->ctl21) = SM(i->txpower[2], AR_XmitPower2);
- ACCESS_ONCE(ads->ctl22) = SM(i->txpower[3], AR_XmitPower3);
+ ACCESS_ONCE_RW(ads->ctl20) = SM(i->txpower[1], AR_XmitPower1);
+ ACCESS_ONCE_RW(ads->ctl21) = SM(i->txpower[2], AR_XmitPower2);
+ ACCESS_ONCE_RW(ads->ctl22) = SM(i->txpower[3], AR_XmitPower3);
}
static u16 ar9003_calc_ptr_chksum(struct ar9003_txc *ads)
/* ANI */
void (*ani_cache_ini_regs)(struct ath_hw *ah);
-};
+} __no_const;
/**
* struct ath_spec_scan - parameters for Atheros spectral scan
#ifdef CONFIG_ATH9K_BTCOEX_SUPPORT
void (*set_bt_ant_diversity)(struct ath_hw *hw, bool enable);
#endif
-};
+} __no_const;
struct ath_nf_limits {
s16 max;
if (!ath9k_is_chanctx_enabled())
return;
- ath9k_ops.hw_scan = ath9k_hw_scan;
- ath9k_ops.cancel_hw_scan = ath9k_cancel_hw_scan;
- ath9k_ops.remain_on_channel = ath9k_remain_on_channel;
- ath9k_ops.cancel_remain_on_channel = ath9k_cancel_remain_on_channel;
- ath9k_ops.add_chanctx = ath9k_add_chanctx;
- ath9k_ops.remove_chanctx = ath9k_remove_chanctx;
- ath9k_ops.change_chanctx = ath9k_change_chanctx;
- ath9k_ops.assign_vif_chanctx = ath9k_assign_vif_chanctx;
- ath9k_ops.unassign_vif_chanctx = ath9k_unassign_vif_chanctx;
- ath9k_ops.mgd_prepare_tx = ath9k_mgd_prepare_tx;
+ pax_open_kernel();
+ *(void **)&ath9k_ops.hw_scan = ath9k_hw_scan;
+ *(void **)&ath9k_ops.cancel_hw_scan = ath9k_cancel_hw_scan;
+ *(void **)&ath9k_ops.remain_on_channel = ath9k_remain_on_channel;
+ *(void **)&ath9k_ops.cancel_remain_on_channel = ath9k_cancel_remain_on_channel;
+ *(void **)&ath9k_ops.add_chanctx = ath9k_add_chanctx;
+ *(void **)&ath9k_ops.remove_chanctx = ath9k_remove_chanctx;
+ *(void **)&ath9k_ops.change_chanctx = ath9k_change_chanctx;
+ *(void **)&ath9k_ops.assign_vif_chanctx = ath9k_assign_vif_chanctx;
+ *(void **)&ath9k_ops.unassign_vif_chanctx = ath9k_unassign_vif_chanctx;
+ *(void **)&ath9k_ops.mgd_prepare_tx = ath9k_mgd_prepare_tx;
+ pax_close_kernel();
}
#endif
{
struct ssb_bus *bus = dev->dev->sdev->bus;
- static const struct b206x_channel *chandata = NULL;
+ const struct b206x_channel *chandata = NULL;
u32 crystal_freq = bus->chipco.pmu.crystalfreq * 1000;
u32 freqref, vco_freq, val1, val2, val3, timeout, timeoutref, count;
u16 old_comm15, scale;
*/
if (il3945_mod_params.disable_hw_scan) {
D_INFO("Disabling hw_scan\n");
- il3945_mac_ops.hw_scan = NULL;
+ pax_open_kernel();
+ *(void **)&il3945_mac_ops.hw_scan = NULL;
+ pax_close_kernel();
}
D_INFO("*** LOAD DRIVER ***\n");
{
struct iwl_priv *priv = file->private_data;
char buf[64];
- int buf_size;
+ size_t buf_size;
u32 offset, len;
memset(buf, 0, sizeof(buf));
struct iwl_priv *priv = file->private_data;
char buf[8];
- int buf_size;
+ size_t buf_size;
u32 reset_flag;
memset(buf, 0, sizeof(buf));
{
struct iwl_priv *priv = file->private_data;
char buf[8];
- int buf_size;
+ size_t buf_size;
int ht40;
memset(buf, 0, sizeof(buf));
{
struct iwl_priv *priv = file->private_data;
char buf[8];
- int buf_size;
+ size_t buf_size;
int value;
memset(buf, 0, sizeof(buf));
DEBUGFS_READ_WRITE_FILE_OPS(sleep_level_override);
DEBUGFS_READ_FILE_OPS(current_sleep_command);
-static const char *fmt_value = " %-30s %10u\n";
-static const char *fmt_hex = " %-30s 0x%02X\n";
-static const char *fmt_table = " %-30s %10u %10u %10u %10u\n";
-static const char *fmt_header =
+static const char fmt_value[] = " %-30s %10u\n";
+static const char fmt_hex[] = " %-30s 0x%02X\n";
+static const char fmt_table[] = " %-30s %10u %10u %10u %10u\n";
+static const char fmt_header[] =
"%-32s current cumulative delta max\n";
static int iwl_statistics_flag(struct iwl_priv *priv, char *buf, int bufsz)
{
struct iwl_priv *priv = file->private_data;
char buf[8];
- int buf_size;
+ size_t buf_size;
int clear;
memset(buf, 0, sizeof(buf));
{
struct iwl_priv *priv = file->private_data;
char buf[8];
- int buf_size;
+ size_t buf_size;
int trace;
memset(buf, 0, sizeof(buf));
{
struct iwl_priv *priv = file->private_data;
char buf[8];
- int buf_size;
+ size_t buf_size;
int missed;
memset(buf, 0, sizeof(buf));
struct iwl_priv *priv = file->private_data;
char buf[8];
- int buf_size;
+ size_t buf_size;
int plcp;
memset(buf, 0, sizeof(buf));
struct iwl_priv *priv = file->private_data;
char buf[8];
- int buf_size;
+ size_t buf_size;
int flush;
memset(buf, 0, sizeof(buf));
struct iwl_priv *priv = file->private_data;
char buf[8];
- int buf_size;
+ size_t buf_size;
int rts;
if (!priv->cfg->ht_params)
{
struct iwl_priv *priv = file->private_data;
char buf[8];
- int buf_size;
+ size_t buf_size;
memset(buf, 0, sizeof(buf));
buf_size = min(count, sizeof(buf) - 1);
struct iwl_priv *priv = file->private_data;
u32 event_log_flag;
char buf[8];
- int buf_size;
+ size_t buf_size;
/* check that the interface is up */
if (!iwl_is_ready(priv))
struct iwl_priv *priv = file->private_data;
char buf[8];
u32 calib_disabled;
- int buf_size;
+ size_t buf_size;
memset(buf, 0, sizeof(buf));
buf_size = min(count, sizeof(buf) - 1);
struct isr_statistics *isr_stats = &trans_pcie->isr_stats;
char buf[8];
- int buf_size;
+ size_t buf_size;
u32 reset_flag;
memset(buf, 0, sizeof(buf));
{
struct iwl_trans *trans = file->private_data;
char buf[8];
- int buf_size;
+ size_t buf_size;
int csr;
memset(buf, 0, sizeof(buf));
if (channels < 1)
return -EINVAL;
- mac80211_hwsim_mchan_ops = mac80211_hwsim_ops;
- mac80211_hwsim_mchan_ops.hw_scan = mac80211_hwsim_hw_scan;
- mac80211_hwsim_mchan_ops.cancel_hw_scan = mac80211_hwsim_cancel_hw_scan;
- mac80211_hwsim_mchan_ops.sw_scan_start = NULL;
- mac80211_hwsim_mchan_ops.sw_scan_complete = NULL;
- mac80211_hwsim_mchan_ops.remain_on_channel = mac80211_hwsim_roc;
- mac80211_hwsim_mchan_ops.cancel_remain_on_channel = mac80211_hwsim_croc;
- mac80211_hwsim_mchan_ops.add_chanctx = mac80211_hwsim_add_chanctx;
- mac80211_hwsim_mchan_ops.remove_chanctx = mac80211_hwsim_remove_chanctx;
- mac80211_hwsim_mchan_ops.change_chanctx = mac80211_hwsim_change_chanctx;
- mac80211_hwsim_mchan_ops.assign_vif_chanctx =
- mac80211_hwsim_assign_vif_chanctx;
- mac80211_hwsim_mchan_ops.unassign_vif_chanctx =
- mac80211_hwsim_unassign_vif_chanctx;
+ pax_open_kernel();
+ memcpy((void *)&mac80211_hwsim_mchan_ops, &mac80211_hwsim_ops, sizeof mac80211_hwsim_mchan_ops);
+ *(void **)&mac80211_hwsim_mchan_ops.hw_scan = mac80211_hwsim_hw_scan;
+ *(void **)&mac80211_hwsim_mchan_ops.cancel_hw_scan = mac80211_hwsim_cancel_hw_scan;
+ *(void **)&mac80211_hwsim_mchan_ops.sw_scan_start = NULL;
+ *(void **)&mac80211_hwsim_mchan_ops.sw_scan_complete = NULL;
+ *(void **)&mac80211_hwsim_mchan_ops.remain_on_channel = mac80211_hwsim_roc;
+ *(void **)&mac80211_hwsim_mchan_ops.cancel_remain_on_channel = mac80211_hwsim_croc;
+ *(void **)&mac80211_hwsim_mchan_ops.add_chanctx = mac80211_hwsim_add_chanctx;
+ *(void **)&mac80211_hwsim_mchan_ops.remove_chanctx = mac80211_hwsim_remove_chanctx;
+ *(void **)&mac80211_hwsim_mchan_ops.change_chanctx = mac80211_hwsim_change_chanctx;
+ *(void **)&mac80211_hwsim_mchan_ops.assign_vif_chanctx = mac80211_hwsim_assign_vif_chanctx;
+ *(void **)&mac80211_hwsim_mchan_ops.unassign_vif_chanctx = mac80211_hwsim_unassign_vif_chanctx;
+ pax_close_kernel();
spin_lock_init(&hwsim_radio_lock);
INIT_LIST_HEAD(&hwsim_radios);
netdev_dbg(usbdev->net, "%s(): %i\n", __func__, rts_threshold);
- if (rts_threshold < 0 || rts_threshold > 2347)
+ if (rts_threshold > 2347)
rts_threshold = 2347;
tmp = cpu_to_le32(rts_threshold);
* for hardware which doesn't support hardware
* sequence counting.
*/
- atomic_t seqno;
+ atomic_unchecked_t seqno;
};
static inline struct rt2x00_intf* vif_to_intf(struct ieee80211_vif *vif)
* sequence counter given by mac80211.
*/
if (test_bit(ENTRY_TXD_FIRST_FRAGMENT, &txdesc->flags))
- seqno = atomic_add_return(0x10, &intf->seqno);
+ seqno = atomic_add_return_unchecked(0x10, &intf->seqno);
else
- seqno = atomic_read(&intf->seqno);
+ seqno = atomic_read_unchecked(&intf->seqno);
hdr->seq_ctrl &= cpu_to_le16(IEEE80211_SCTL_FRAG);
hdr->seq_ctrl |= cpu_to_le16(seqno);
irq_set_irq_type(wl->irq, IRQ_TYPE_EDGE_RISING);
- wl1251_sdio_ops.enable_irq = wl1251_enable_line_irq;
- wl1251_sdio_ops.disable_irq = wl1251_disable_line_irq;
+ pax_open_kernel();
+ *(void **)&wl1251_sdio_ops.enable_irq = wl1251_enable_line_irq;
+ *(void **)&wl1251_sdio_ops.disable_irq = wl1251_disable_line_irq;
+ pax_close_kernel();
wl1251_info("using dedicated interrupt line");
} else {
- wl1251_sdio_ops.enable_irq = wl1251_sdio_enable_irq;
- wl1251_sdio_ops.disable_irq = wl1251_sdio_disable_irq;
+ pax_open_kernel();
+ *(void **)&wl1251_sdio_ops.enable_irq = wl1251_sdio_enable_irq;
+ *(void **)&wl1251_sdio_ops.disable_irq = wl1251_sdio_disable_irq;
+ pax_close_kernel();
wl1251_info("using SDIO interrupt");
}
sizeof(wl->conf.mem));
/* read data preparation is only needed by wl127x */
- wl->ops->prepare_read = wl127x_prepare_read;
+ pax_open_kernel();
+ *(void **)&wl->ops->prepare_read = wl127x_prepare_read;
+ pax_close_kernel();
wlcore_set_min_fw_ver(wl, WL127X_CHIP_VER,
WL127X_IFTYPE_SR_VER, WL127X_MAJOR_SR_VER,
sizeof(wl->conf.mem));
/* read data preparation is only needed by wl127x */
- wl->ops->prepare_read = wl127x_prepare_read;
+ pax_open_kernel();
+ *(void **)&wl->ops->prepare_read = wl127x_prepare_read;
+ pax_close_kernel();
wlcore_set_min_fw_ver(wl, WL127X_CHIP_VER,
WL127X_IFTYPE_SR_VER, WL127X_MAJOR_SR_VER,
}
if (!checksum_param) {
- wl18xx_ops.set_rx_csum = NULL;
- wl18xx_ops.init_vif = NULL;
+ pax_open_kernel();
+ *(void **)&wl18xx_ops.set_rx_csum = NULL;
+ *(void **)&wl18xx_ops.init_vif = NULL;
+ pax_close_kernel();
}
/* Enable 11a Band only if we have 5G antennas */
{
struct zd_usb *usb = urb->context;
struct zd_usb_interrupt *intr = &usb->intr;
- int len;
+ unsigned int len;
u16 int_num;
ZD_ASSERT(in_interrupt());
static int nfcwilink_probe(struct platform_device *pdev)
{
- static struct nfcwilink *drv;
+ struct nfcwilink *drv;
int rc;
__u32 protocols;
goto exit;
}
- gate = uid_skb->data;
+ memcpy(gate, uid_skb->data, uid_skb->len);
*len = uid_skb->len;
exit:
kfree_skb(uid_skb);
pr_warn("fdt: not creating '/sys/firmware/fdt': CRC check failed\n");
return 0;
}
- of_fdt_raw_attr.size = fdt_totalsize(initial_boot_params);
+ pax_open_kernel();
+ *(size_t *)&of_fdt_raw_attr.size = fdt_totalsize(initial_boot_params);
+ pax_close_kernel();
return sysfs_create_bin_file(firmware_kobj, &of_fdt_raw_attr);
}
late_initcall(of_fdt_raw_init);
if (cookie == NO_COOKIE)
offset = pc;
if (cookie == INVALID_COOKIE) {
- atomic_inc(&oprofile_stats.sample_lost_no_mapping);
+ atomic_inc_unchecked(&oprofile_stats.sample_lost_no_mapping);
offset = pc;
}
if (cookie != last_cookie) {
/* add userspace sample */
if (!mm) {
- atomic_inc(&oprofile_stats.sample_lost_no_mm);
+ atomic_inc_unchecked(&oprofile_stats.sample_lost_no_mm);
return 0;
}
cookie = lookup_dcookie(mm, s->eip, &offset);
if (cookie == INVALID_COOKIE) {
- atomic_inc(&oprofile_stats.sample_lost_no_mapping);
+ atomic_inc_unchecked(&oprofile_stats.sample_lost_no_mapping);
return 0;
}
/* ignore backtraces if failed to add a sample */
if (state == sb_bt_start) {
state = sb_bt_ignore;
- atomic_inc(&oprofile_stats.bt_lost_no_mapping);
+ atomic_inc_unchecked(&oprofile_stats.bt_lost_no_mapping);
}
}
release_mm(mm);
}
if (buffer_pos == buffer_size) {
- atomic_inc(&oprofile_stats.event_lost_overflow);
+ atomic_inc_unchecked(&oprofile_stats.event_lost_overflow);
return;
}
if (oprofile_ops.switch_events())
return;
- atomic_inc(&oprofile_stats.multiplex_counter);
+ atomic_inc_unchecked(&oprofile_stats.multiplex_counter);
start_switch_worker();
}
#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
-static ssize_t timeout_read(struct file *file, char __user *buf,
+static ssize_t __intentional_overflow(-1) timeout_read(struct file *file, char __user *buf,
size_t count, loff_t *offset)
{
return oprofilefs_ulong_to_user(jiffies_to_msecs(oprofile_time_slice),
cpu_buf->sample_invalid_eip = 0;
}
- atomic_set(&oprofile_stats.sample_lost_no_mm, 0);
- atomic_set(&oprofile_stats.sample_lost_no_mapping, 0);
- atomic_set(&oprofile_stats.event_lost_overflow, 0);
- atomic_set(&oprofile_stats.bt_lost_no_mapping, 0);
- atomic_set(&oprofile_stats.multiplex_counter, 0);
+ atomic_set_unchecked(&oprofile_stats.sample_lost_no_mm, 0);
+ atomic_set_unchecked(&oprofile_stats.sample_lost_no_mapping, 0);
+ atomic_set_unchecked(&oprofile_stats.event_lost_overflow, 0);
+ atomic_set_unchecked(&oprofile_stats.bt_lost_no_mapping, 0);
+ atomic_set_unchecked(&oprofile_stats.multiplex_counter, 0);
}
#include <linux/atomic.h>
struct oprofile_stat_struct {
- atomic_t sample_lost_no_mm;
- atomic_t sample_lost_no_mapping;
- atomic_t bt_lost_no_mapping;
- atomic_t event_lost_overflow;
- atomic_t multiplex_counter;
+ atomic_unchecked_t sample_lost_no_mm;
+ atomic_unchecked_t sample_lost_no_mapping;
+ atomic_unchecked_t bt_lost_no_mapping;
+ atomic_unchecked_t event_lost_overflow;
+ atomic_unchecked_t multiplex_counter;
};
extern struct oprofile_stat_struct oprofile_stats;
static ssize_t atomic_read_file(struct file *file, char __user *buf, size_t count, loff_t *offset)
{
- atomic_t *val = file->private_data;
- return oprofilefs_ulong_to_user(atomic_read(val), buf, count, offset);
+ atomic_unchecked_t *val = file->private_data;
+ return oprofilefs_ulong_to_user(atomic_read_unchecked(val), buf, count, offset);
}
int oprofilefs_create_ro_atomic(struct dentry *root,
- char const *name, atomic_t *val)
+ char const *name, atomic_unchecked_t *val)
{
return __oprofilefs_create_file(root, name,
&atomic_ro_fops, 0444, val);
return NOTIFY_OK;
}
-static struct notifier_block __refdata oprofile_cpu_notifier = {
+static struct notifier_block oprofile_cpu_notifier = {
.notifier_call = oprofile_cpu_notify,
};
*ppos += len;
- return copy_to_user(result, buffer, len) ? -EFAULT : 0;
+ return (len > sizeof buffer || copy_to_user(result, buffer, len)) ? -EFAULT : 0;
}
#ifdef CONFIG_PARPORT_1284
*ppos += len;
- return copy_to_user (result, buffer, len) ? -EFAULT : 0;
+ return (len > sizeof buffer || copy_to_user (result, buffer, len)) ? -EFAULT : 0;
}
#endif /* IEEE1284.3 support. */
goto init_cleanup;
}
- ibm_apci_table_attr.size = ibm_get_table_from_acpi(NULL);
+ pax_open_kernel();
+ *(size_t *)&ibm_apci_table_attr.size = ibm_get_table_from_acpi(NULL);
+ pax_close_kernel();
retval = sysfs_create_bin_file(sysdir, &ibm_apci_table_attr);
return retval;
static unsigned int enum_bit;
static u8 enum_mask;
-static struct cpci_hp_controller_ops generic_hpc_ops;
static struct cpci_hp_controller generic_hpc;
static int __init validate_parameters(void)
return ((value & enum_mask) == enum_mask);
}
+static struct cpci_hp_controller_ops generic_hpc_ops = {
+ .query_enum = query_enum,
+};
+
static int __init cpcihp_generic_init(void)
{
int status;
pci_dev_put(dev);
memset(&generic_hpc, 0, sizeof (struct cpci_hp_controller));
- generic_hpc_ops.query_enum = query_enum;
generic_hpc.ops = &generic_hpc_ops;
status = cpci_hp_register_controller(&generic_hpc);
/* local variables */
static bool debug;
static bool poll;
-static struct cpci_hp_controller_ops zt5550_hpc_ops;
static struct cpci_hp_controller zt5550_hpc;
/* Primary cPCI bus bridge device */
return 0;
}
+static struct cpci_hp_controller_ops zt5550_hpc_ops = {
+ .query_enum = zt5550_hc_query_enum,
+};
+
static int zt5550_hc_init_one (struct pci_dev *pdev, const struct pci_device_id *ent)
{
int status;
dbg("returned from zt5550_hc_config");
memset(&zt5550_hpc, 0, sizeof (struct cpci_hp_controller));
- zt5550_hpc_ops.query_enum = zt5550_hc_query_enum;
zt5550_hpc.ops = &zt5550_hpc_ops;
if (!poll) {
zt5550_hpc.irq = hc_dev->irq;
zt5550_hpc.irq_flags = IRQF_SHARED;
zt5550_hpc.dev_id = hc_dev;
- zt5550_hpc_ops.enable_irq = zt5550_hc_enable_irq;
- zt5550_hpc_ops.disable_irq = zt5550_hc_disable_irq;
- zt5550_hpc_ops.check_irq = zt5550_hc_check_irq;
+ pax_open_kernel();
+ *(void **)&zt5550_hpc_ops.enable_irq = zt5550_hc_enable_irq;
+ *(void **)&zt5550_hpc_ops.disable_irq = zt5550_hc_disable_irq;
+ *(void **)&zt5550_hpc_ops.check_irq = zt5550_hc_check_irq;
+ pax_open_kernel();
} else {
info("using ENUM# polling mode");
}
void compaq_nvram_init (void __iomem *rom_start)
{
+#ifndef CONFIG_PAX_KERNEXEC
if (rom_start)
compaq_int15_entry_point = (rom_start + ROM_INT15_PHY_ADDR - ROM_PHY_ADDR);
+#endif
dbg("int15 entry = %p\n", compaq_int15_entry_point);
return -EINVAL;
}
- slot->ops->owner = owner;
- slot->ops->mod_name = mod_name;
+ pax_open_kernel();
+ *(struct module **)&slot->ops->owner = owner;
+ *(const char **)&slot->ops->mod_name = mod_name;
+ pax_close_kernel();
mutex_lock(&pci_hp_mutex);
/*
struct slot *slot = ctrl->slot;
struct hotplug_slot *hotplug = NULL;
struct hotplug_slot_info *info = NULL;
- struct hotplug_slot_ops *ops = NULL;
+ hotplug_slot_ops_no_const *ops = NULL;
char name[SLOT_NAME_SIZE];
int retval = -ENOMEM;
{
struct attribute **msi_attrs;
struct attribute *msi_attr;
- struct device_attribute *msi_dev_attr;
- struct attribute_group *msi_irq_group;
+ device_attribute_no_const *msi_dev_attr;
+ attribute_group_no_const *msi_irq_group;
const struct attribute_group **msi_irq_groups;
struct msi_desc *entry;
int ret = -ENOMEM;
count = 0;
msi_attr = msi_attrs[count];
while (msi_attr) {
- msi_dev_attr = container_of(msi_attr, struct device_attribute, attr);
+ msi_dev_attr = container_of(msi_attr, device_attribute_no_const, attr);
kfree(msi_attr->name);
kfree(msi_dev_attr);
++count;
{
/* allocate attribute structure, piggyback attribute name */
int name_len = write_combine ? 13 : 10;
- struct bin_attribute *res_attr;
+ bin_attribute_no_const *res_attr;
int retval;
res_attr = kzalloc(sizeof(*res_attr) + name_len, GFP_ATOMIC);
static int pci_create_capabilities_sysfs(struct pci_dev *dev)
{
int retval;
- struct bin_attribute *attr;
+ bin_attribute_no_const *attr;
/* If the device has VPD, try to expose it in sysfs. */
if (dev->vpd) {
{
int retval;
int rom_size = 0;
- struct bin_attribute *attr;
+ bin_attribute_no_const *attr;
if (!sysfs_initialized)
return -EACCES;
struct pci_vpd {
unsigned int len;
const struct pci_vpd_ops *ops;
- struct bin_attribute *attr; /* descriptor for sysfs VPD entry */
+ bin_attribute_no_const *attr; /* descriptor for sysfs VPD entry */
};
int pci_vpd_pci22_init(struct pci_dev *dev);
#define MODULE_PARAM_PREFIX "pcie_aspm."
/* Note: those are not register definitions */
-#define ASPM_STATE_L0S_UP (1) /* Upstream direction L0s state */
-#define ASPM_STATE_L0S_DW (2) /* Downstream direction L0s state */
-#define ASPM_STATE_L1 (4) /* L1 state */
+#define ASPM_STATE_L0S_UP (1U) /* Upstream direction L0s state */
+#define ASPM_STATE_L0S_DW (2U) /* Downstream direction L0s state */
+#define ASPM_STATE_L1 (4U) /* L1 state */
#define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW)
#define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1)
u16 orig_cmd;
struct pci_bus_region region, inverted_region;
- mask = type ? PCI_ROM_ADDRESS_MASK : ~0;
+ mask = type ? (u32)PCI_ROM_ADDRESS_MASK : ~0;
/* No printks while decoding is disabled! */
if (!dev->mmio_always_on) {
static int __init pci_proc_init(void)
{
struct pci_dev *dev = NULL;
+
+#ifdef CONFIG_GRKERNSEC_PROC_ADD
+#ifdef CONFIG_GRKERNSEC_PROC_USER
+ proc_bus_pci_dir = proc_mkdir_mode("bus/pci", S_IRUSR | S_IXUSR, NULL);
+#elif defined(CONFIG_GRKERNSEC_PROC_USERGROUP)
+ proc_bus_pci_dir = proc_mkdir_mode("bus/pci", S_IRUSR | S_IXUSR | S_IRGRP | S_IXGRP, NULL);
+#endif
+#else
proc_bus_pci_dir = proc_mkdir("bus/pci", NULL);
+#endif
proc_create("devices", 0, proc_bus_pci_dir,
&proc_bus_pci_dev_operations);
proc_initialized = 1;
.callback = chromeos_laptop_dmi_matched, \
.driver_data = (void *)&board_
-static struct dmi_system_id chromeos_laptop_dmi_table[] __initdata = {
+static struct dmi_system_id chromeos_laptop_dmi_table[] __initconst = {
{
.ident = "Samsung Series 5 550",
.matches = {
} __packed;
static struct platform_device *platform_device;
-static struct device_attribute *zone_dev_attrs;
+static device_attribute_no_const *zone_dev_attrs;
static struct attribute **zone_attrs;
static struct platform_zone *zone_data;
}
};
-static struct attribute_group zone_attribute_group = {
+static attribute_group_no_const zone_attribute_group = {
.name = "rgb_zones",
};
int err;
u32 retval = -1;
+#ifdef CONFIG_GRKERNSEC_KMEM
+ return -EPERM;
+#endif
+
err = asus_wmi_get_devstate(asus, asus->debug.dev_id, &retval);
if (err < 0)
int err;
u32 retval = -1;
+#ifdef CONFIG_GRKERNSEC_KMEM
+ return -EPERM;
+#endif
+
err = asus_wmi_set_devstate(asus->debug.dev_id, asus->debug.ctrl_param,
&retval);
union acpi_object *obj;
acpi_status status;
+#ifdef CONFIG_GRKERNSEC_KMEM
+ return -EPERM;
+#endif
+
status = wmi_evaluate_method(ASUS_WMI_MGMT_GUID,
1, asus->debug.method_id,
&input, &output);
if (!quirks->ec_read_only) {
/* allow userland write sysfs file */
- dev_attr_bluetooth.store = store_bluetooth;
- dev_attr_wlan.store = store_wlan;
- dev_attr_threeg.store = store_threeg;
- dev_attr_bluetooth.attr.mode |= S_IWUSR;
- dev_attr_wlan.attr.mode |= S_IWUSR;
- dev_attr_threeg.attr.mode |= S_IWUSR;
+ pax_open_kernel();
+ *(void **)&dev_attr_bluetooth.store = store_bluetooth;
+ *(void **)&dev_attr_wlan.store = store_wlan;
+ *(void **)&dev_attr_threeg.store = store_threeg;
+ *(umode_t *)&dev_attr_bluetooth.attr.mode |= S_IWUSR;
+ *(umode_t *)&dev_attr_wlan.attr.mode |= S_IWUSR;
+ *(umode_t *)&dev_attr_threeg.attr.mode |= S_IWUSR;
+ pax_close_kernel();
}
/* disable hardware control by fn key */
static void msi_wmi_notify(u32 value, void *context)
{
struct acpi_buffer response = { ACPI_ALLOCATE_BUFFER, NULL };
- static struct key_entry *key;
+ struct key_entry *key;
union acpi_object *obj;
acpi_status status;
}
/* High speed charging function */
-static struct device_attribute *hsc_handle;
+static device_attribute_no_const *hsc_handle;
static ssize_t sony_nc_highspeed_charging_store(struct device *dev,
struct device_attribute *attr,
}
/* low battery function */
-static struct device_attribute *lowbatt_handle;
+static device_attribute_no_const *lowbatt_handle;
static ssize_t sony_nc_lowbatt_store(struct device *dev,
struct device_attribute *attr,
}
/* fan speed function */
-static struct device_attribute *fan_handle, *hsf_handle;
+static device_attribute_no_const *fan_handle, *hsf_handle;
static ssize_t sony_nc_hsfan_store(struct device *dev,
struct device_attribute *attr,
}
/* USB charge function */
-static struct device_attribute *uc_handle;
+static device_attribute_no_const *uc_handle;
static ssize_t sony_nc_usb_charge_store(struct device *dev,
struct device_attribute *attr,
}
/* Panel ID function */
-static struct device_attribute *panel_handle;
+static device_attribute_no_const *panel_handle;
static ssize_t sony_nc_panelid_show(struct device *dev,
struct device_attribute *attr, char *buffer)
}
/* smart connect function */
-static struct device_attribute *sc_handle;
+static device_attribute_no_const *sc_handle;
static ssize_t sony_nc_smart_conn_store(struct device *dev,
struct device_attribute *attr,
return 0;
}
-void static hotkey_mask_warn_incomplete_mask(void)
+static void hotkey_mask_warn_incomplete_mask(void)
{
/* log only what the user can fix... */
const u32 wantedmask = hotkey_driver_mask &
&& !tp_features.bright_unkfw)
TPACPI_MAY_SEND_KEY(TP_ACPI_HOTKEYSCAN_FNHOME);
}
+}
#undef TPACPI_COMPARE_KEY
#undef TPACPI_MAY_SEND_KEY
-}
/*
* Polling driver
set_desc_limit(&gdt[(selname) >> 3], (size) - 1); \
} while(0)
-static struct desc_struct bad_bios_desc = GDT_ENTRY_INIT(0x4092,
+static const struct desc_struct bad_bios_desc = GDT_ENTRY_INIT(0x4093,
(unsigned long)__va(0x400UL), PAGE_SIZE - 0x400 - 1);
/*
cpu = get_cpu();
save_desc_40 = get_cpu_gdt_table(cpu)[0x40 / 8];
+
+ pax_open_kernel();
get_cpu_gdt_table(cpu)[0x40 / 8] = bad_bios_desc;
+ pax_close_kernel();
/* On some boxes IRQ's during PnP BIOS calls are deadly. */
spin_lock_irqsave(&pnp_bios_lock, flags);
:"memory");
spin_unlock_irqrestore(&pnp_bios_lock, flags);
+ pax_open_kernel();
get_cpu_gdt_table(cpu)[0x40 / 8] = save_desc_40;
+ pax_close_kernel();
+
put_cpu();
/* If we get here and this is set then the PnP BIOS faulted on us. */
return status;
}
-void pnpbios_calls_init(union pnp_bios_install_struct *header)
+void __init pnpbios_calls_init(union pnp_bios_install_struct *header)
{
int i;
pnp_bios_callpoint.offset = header->fields.pm16offset;
pnp_bios_callpoint.segment = PNP_CS16;
+ pax_open_kernel();
+
for_each_possible_cpu(i) {
struct desc_struct *gdt = get_cpu_gdt_table(i);
if (!gdt)
set_desc_base(&gdt[GDT_ENTRY_PNPBIOS_DS],
(unsigned long)__va(header->fields.pm16dseg));
}
+
+ pax_close_kernel();
}
#if IS_ENABLED(CONFIG_USB_PHY)
static struct usb_phy *transceiver;
-static struct notifier_block otg_nb;
+static int otg_handle_notification(struct notifier_block *nb,
+ unsigned long event, void *unused);
+static struct notifier_block otg_nb = {
+ .notifier_call = otg_handle_notification
+};
#endif
static struct regulator *ac_draw;
#if IS_ENABLED(CONFIG_USB_PHY)
if (!IS_ERR_OR_NULL(transceiver) && pdata->use_otg_notifier) {
- otg_nb.notifier_call = otg_handle_notification;
ret = usb_register_notifier(transceiver, &otg_nb);
if (ret) {
dev_err(dev, "failure to register otg notifier\n");
#ifdef CONFIG_SYSFS
-extern void power_supply_init_attrs(struct device_type *dev_type);
+extern void power_supply_init_attrs(void);
extern int power_supply_uevent(struct device *dev, struct kobj_uevent_env *env);
#else
-static inline void power_supply_init_attrs(struct device_type *dev_type) {}
+static inline void power_supply_init_attrs(void) {}
#define power_supply_uevent NULL
#endif /* CONFIG_SYSFS */
ATOMIC_NOTIFIER_HEAD(power_supply_notifier);
EXPORT_SYMBOL_GPL(power_supply_notifier);
-static struct device_type power_supply_dev_type;
+extern const struct attribute_group *power_supply_attr_groups[];
+static struct device_type power_supply_dev_type = {
+ .groups = power_supply_attr_groups,
+};
static bool __power_supply_is_supplied_by(struct power_supply *supplier,
struct power_supply *supply)
return PTR_ERR(power_supply_class);
power_supply_class->dev_uevent = power_supply_uevent;
- power_supply_init_attrs(&power_supply_dev_type);
+ power_supply_init_attrs();
return 0;
}
.is_visible = power_supply_attr_is_visible,
};
-static const struct attribute_group *power_supply_attr_groups[] = {
+const struct attribute_group *power_supply_attr_groups[] = {
&power_supply_attr_group,
NULL,
};
-void power_supply_init_attrs(struct device_type *dev_type)
+void power_supply_init_attrs(void)
{
int i;
- dev_type->groups = power_supply_attr_groups;
-
for (i = 0; i < ARRAY_SIZE(power_supply_attrs); i++)
__power_supply_attrs[i] = &power_supply_attrs[i].attr;
}
struct device_attribute name_attr;
};
+static ssize_t show_constraint_name(struct device *dev,
+ struct device_attribute *dev_attr,
+ char *buf);
+
static struct powercap_constraint_attr
- constraint_attrs[MAX_CONSTRAINTS_PER_ZONE];
+ constraint_attrs[MAX_CONSTRAINTS_PER_ZONE] = {
+ [0 ... MAX_CONSTRAINTS_PER_ZONE - 1] = {
+ .power_limit_attr = {
+ .attr = {
+ .name = NULL,
+ .mode = S_IWUSR | S_IRUGO
+ },
+ .show = show_constraint_power_limit_uw,
+ .store = store_constraint_power_limit_uw
+ },
+
+ .time_window_attr = {
+ .attr = {
+ .name = NULL,
+ .mode = S_IWUSR | S_IRUGO
+ },
+ .show = show_constraint_time_window_us,
+ .store = store_constraint_time_window_us
+ },
+
+ .max_power_attr = {
+ .attr = {
+ .name = NULL,
+ .mode = S_IRUGO
+ },
+ .show = show_constraint_max_power_uw,
+ .store = NULL
+ },
+
+ .min_power_attr = {
+ .attr = {
+ .name = NULL,
+ .mode = S_IRUGO
+ },
+ .show = show_constraint_min_power_uw,
+ .store = NULL
+ },
+
+ .max_time_window_attr = {
+ .attr = {
+ .name = NULL,
+ .mode = S_IRUGO
+ },
+ .show = show_constraint_max_time_window_us,
+ .store = NULL
+ },
+
+ .min_time_window_attr = {
+ .attr = {
+ .name = NULL,
+ .mode = S_IRUGO
+ },
+ .show = show_constraint_min_time_window_us,
+ .store = NULL
+ },
+
+ .name_attr = {
+ .attr = {
+ .name = NULL,
+ .mode = S_IRUGO
+ },
+ .show = show_constraint_name,
+ .store = NULL
+ }
+ }
+};
/* A list of powercap control_types */
static LIST_HEAD(powercap_cntrl_list);
}
static int create_constraint_attribute(int id, const char *name,
- int mode,
- struct device_attribute *dev_attr,
- ssize_t (*show)(struct device *,
- struct device_attribute *, char *),
- ssize_t (*store)(struct device *,
- struct device_attribute *,
- const char *, size_t)
- )
+ struct device_attribute *dev_attr)
{
+ name = kasprintf(GFP_KERNEL, "constraint_%d_%s", id, name);
- dev_attr->attr.name = kasprintf(GFP_KERNEL, "constraint_%d_%s",
- id, name);
- if (!dev_attr->attr.name)
+ if (!name)
return -ENOMEM;
- dev_attr->attr.mode = mode;
- dev_attr->show = show;
- dev_attr->store = store;
+
+ pax_open_kernel();
+ *(const char **)&dev_attr->attr.name = name;
+ pax_close_kernel();
return 0;
}
for (i = 0; i < MAX_CONSTRAINTS_PER_ZONE; ++i) {
ret = create_constraint_attribute(i, "power_limit_uw",
- S_IWUSR | S_IRUGO,
- &constraint_attrs[i].power_limit_attr,
- show_constraint_power_limit_uw,
- store_constraint_power_limit_uw);
+ &constraint_attrs[i].power_limit_attr);
if (ret)
goto err_alloc;
ret = create_constraint_attribute(i, "time_window_us",
- S_IWUSR | S_IRUGO,
- &constraint_attrs[i].time_window_attr,
- show_constraint_time_window_us,
- store_constraint_time_window_us);
+ &constraint_attrs[i].time_window_attr);
if (ret)
goto err_alloc;
- ret = create_constraint_attribute(i, "name", S_IRUGO,
- &constraint_attrs[i].name_attr,
- show_constraint_name,
- NULL);
+ ret = create_constraint_attribute(i, "name",
+ &constraint_attrs[i].name_attr);
if (ret)
goto err_alloc;
- ret = create_constraint_attribute(i, "max_power_uw", S_IRUGO,
- &constraint_attrs[i].max_power_attr,
- show_constraint_max_power_uw,
- NULL);
+ ret = create_constraint_attribute(i, "max_power_uw",
+ &constraint_attrs[i].max_power_attr);
if (ret)
goto err_alloc;
- ret = create_constraint_attribute(i, "min_power_uw", S_IRUGO,
- &constraint_attrs[i].min_power_attr,
- show_constraint_min_power_uw,
- NULL);
+ ret = create_constraint_attribute(i, "min_power_uw",
+ &constraint_attrs[i].min_power_attr);
if (ret)
goto err_alloc;
ret = create_constraint_attribute(i, "max_time_window_us",
- S_IRUGO,
- &constraint_attrs[i].max_time_window_attr,
- show_constraint_max_time_window_us,
- NULL);
+ &constraint_attrs[i].max_time_window_attr);
if (ret)
goto err_alloc;
ret = create_constraint_attribute(i, "min_time_window_us",
- S_IRUGO,
- &constraint_attrs[i].min_time_window_attr,
- show_constraint_min_time_window_us,
- NULL);
+ &constraint_attrs[i].min_time_window_attr);
if (ret)
goto err_alloc;
power_zone->zone_dev_attrs[count++] =
&dev_attr_max_energy_range_uj.attr;
if (power_zone->ops->get_energy_uj) {
+ pax_open_kernel();
if (power_zone->ops->reset_energy_uj)
- dev_attr_energy_uj.attr.mode = S_IWUSR | S_IRUGO;
+ *(umode_t *)&dev_attr_energy_uj.attr.mode = S_IWUSR | S_IRUGO;
else
- dev_attr_energy_uj.attr.mode = S_IRUGO;
+ *(umode_t *)&dev_attr_energy_uj.attr.mode = S_IRUGO;
+ pax_close_kernel();
power_zone->zone_dev_attrs[count++] =
&dev_attr_energy_uj.attr;
}
struct mutex pincfg_mux; /* protect concurrent info->pin_config access */
wait_queue_head_t tsev_wq;
int defunct; /* tells readers to go away when clock is being removed */
- struct device_attribute *pin_dev_attr;
+ device_attribute_no_const *pin_dev_attr;
struct attribute **pin_attr;
struct attribute_group pin_attr_group;
};
goto no_pin_attr;
for (i = 0; i < n_pins; i++) {
- struct device_attribute *da = &ptp->pin_dev_attr[i];
+ device_attribute_no_const *da = &ptp->pin_dev_attr[i];
sysfs_attr_init(&da->attr);
da->attr.name = info->pin_config[i].name;
da->attr.mode = 0644;
{
const struct regulation_constraints *constraints = NULL;
const struct regulator_init_data *init_data;
- static atomic_t regulator_no = ATOMIC_INIT(0);
+ static atomic_unchecked_t regulator_no = ATOMIC_INIT(0);
struct regulator_dev *rdev;
struct device *dev;
int ret, i;
rdev->dev.class = ®ulator_class;
rdev->dev.parent = dev;
dev_set_name(&rdev->dev, "regulator.%d",
- atomic_inc_return(®ulator_no) - 1);
+ atomic_inc_return_unchecked(®ulator_no) - 1);
ret = device_register(&rdev->dev);
if (ret != 0) {
put_device(&rdev->dev);
max8660->shadow_regs[MAX8660_OVER1] = 5;
} else {
/* Otherwise devices can be toggled via software */
- max8660_dcdc_ops.enable = max8660_dcdc_enable;
- max8660_dcdc_ops.disable = max8660_dcdc_disable;
+ pax_open_kernel();
+ *(void **)&max8660_dcdc_ops.enable = max8660_dcdc_enable;
+ *(void **)&max8660_dcdc_ops.disable = max8660_dcdc_disable;
+ pax_close_kernel();
}
/*
if (!pdata || !pdata->enable_ext_control) {
max->desc.enable_reg = MAX8973_VOUT;
max->desc.enable_mask = MAX8973_VOUT_ENABLE;
- max->ops.enable = regulator_enable_regmap;
- max->ops.disable = regulator_disable_regmap;
- max->ops.is_enabled = regulator_is_enabled_regmap;
+ pax_open_kernel();
+ *(void **)&max->ops.enable = regulator_enable_regmap;
+ *(void **)&max->ops.disable = regulator_disable_regmap;
+ *(void **)&max->ops.is_enabled = regulator_is_enabled_regmap;
+ pax_close_kernel();
}
if (pdata) {
mc13xxx_unlock(mc13892);
/* update mc13892_vcam ops */
- memcpy(&mc13892_vcam_ops, mc13892_regulators[MC13892_VCAM].desc.ops,
+ pax_open_kernel();
+ memcpy((void *)&mc13892_vcam_ops, mc13892_regulators[MC13892_VCAM].desc.ops,
sizeof(struct regulator_ops));
- mc13892_vcam_ops.set_mode = mc13892_vcam_set_mode,
- mc13892_vcam_ops.get_mode = mc13892_vcam_get_mode,
+ *(void **)&mc13892_vcam_ops.set_mode = mc13892_vcam_set_mode,
+ *(void **)&mc13892_vcam_ops.get_mode = mc13892_vcam_get_mode,
+ pax_close_kernel();
mc13892_regulators[MC13892_VCAM].desc.ops = &mc13892_vcam_ops;
mc13xxx_data = mc13xxx_parse_regulators_dt(pdev, mc13892_regulators,
hpet_rtc_timer_init();
/* export at least the first block of NVRAM */
- nvram.size = address_space - NVRAM_OFFSET;
+ pax_open_kernel();
+ *(size_t *)&nvram.size = address_space - NVRAM_OFFSET;
+ pax_close_kernel();
retval = sysfs_create_bin_file(&dev->kobj, &nvram);
if (retval < 0) {
dev_dbg(dev, "can't create nvram file? %d\n", retval);
#include <linux/module.h>
#include <linux/rtc.h>
#include <linux/sched.h>
+#include <linux/grsecurity.h>
#include "rtc-core.h"
static dev_t rtc_devt;
if (copy_from_user(&tm, uarg, sizeof(tm)))
return -EFAULT;
+ gr_log_timechange();
+
return rtc_set_time(rtc, &tm);
case RTC_PIE_ON:
u8 offset; /* register's offset */
u8 regs[11];
u16 nvram_offset;
- struct bin_attribute *nvram;
+ bin_attribute_no_const *nvram;
enum ds_type type;
unsigned long flags;
#define HAS_NVRAM 0 /* bit 0 == sysfs file active */
if (IS_ERR(m48t59->rtc))
return PTR_ERR(m48t59->rtc);
- m48t59_nvram_attr.size = pdata->offset;
+ pax_open_kernel();
+ *(size_t *)&m48t59_nvram_attr.size = pdata->offset;
+ pax_close_kernel();
ret = sysfs_create_bin_file(&pdev->dev.kobj, &m48t59_nvram_attr);
if (ret)
struct bfa_itn_s {
bfa_isr_func_t isr;
-};
+} __no_const;
void bfa_itn_create(struct bfa_s *bfa, struct bfa_rport_s *rport,
void (*isr)(struct bfa_s *bfa, struct bfi_msg_s *m));
#define BFA_FCS_MODULE(_mod) { _mod ## _modinit, _mod ## _modexit }
static struct bfa_fcs_mod_s fcs_modules[] = {
- { bfa_fcs_port_attach, NULL, NULL },
- { bfa_fcs_uf_attach, NULL, NULL },
- { bfa_fcs_fabric_attach, bfa_fcs_fabric_modinit,
- bfa_fcs_fabric_modexit },
+ {
+ .attach = bfa_fcs_port_attach,
+ .modinit = NULL,
+ .modexit = NULL
+ },
+ {
+ .attach = bfa_fcs_uf_attach,
+ .modinit = NULL,
+ .modexit = NULL
+ },
+ {
+ .attach = bfa_fcs_fabric_attach,
+ .modinit = bfa_fcs_fabric_modinit,
+ .modexit = bfa_fcs_fabric_modexit
+ },
};
/*
void (*offline) (struct bfa_fcs_lport_s *port);
} __port_action[] = {
{
- bfa_fcs_lport_unknown_init, bfa_fcs_lport_unknown_online,
- bfa_fcs_lport_unknown_offline}, {
- bfa_fcs_lport_fab_init, bfa_fcs_lport_fab_online,
- bfa_fcs_lport_fab_offline}, {
- bfa_fcs_lport_n2n_init, bfa_fcs_lport_n2n_online,
- bfa_fcs_lport_n2n_offline}, {
- bfa_fcs_lport_loop_init, bfa_fcs_lport_loop_online,
- bfa_fcs_lport_loop_offline},
- };
+ .init = bfa_fcs_lport_unknown_init,
+ .online = bfa_fcs_lport_unknown_online,
+ .offline = bfa_fcs_lport_unknown_offline
+ },
+ {
+ .init = bfa_fcs_lport_fab_init,
+ .online = bfa_fcs_lport_fab_online,
+ .offline = bfa_fcs_lport_fab_offline
+ },
+ {
+ .init = bfa_fcs_lport_n2n_init,
+ .online = bfa_fcs_lport_n2n_online,
+ .offline = bfa_fcs_lport_n2n_offline
+ },
+ {
+ .init = bfa_fcs_lport_loop_init,
+ .online = bfa_fcs_lport_loop_online,
+ .offline = bfa_fcs_lport_loop_offline
+ },
+};
/*
* fcs_port_sm FCS logical port state machine
bfa_ioc_disable_cbfn_t disable_cbfn;
bfa_ioc_hbfail_cbfn_t hbfail_cbfn;
bfa_ioc_reset_cbfn_t reset_cbfn;
-};
+} __no_const;
/*
* IOC event notification mechanism.
void (*ioc_set_alt_fwstate) (struct bfa_ioc_s *ioc,
enum bfi_ioc_state fwstate);
enum bfi_ioc_state (*ioc_get_alt_fwstate) (struct bfa_ioc_s *ioc);
-};
+} __no_const;
/*
* Queue element to wait for room in request queue. FIFO order is
\
extern struct bfa_module_s hal_mod_ ## __mod; \
struct bfa_module_s hal_mod_ ## __mod = { \
- bfa_ ## __mod ## _meminfo, \
- bfa_ ## __mod ## _attach, \
- bfa_ ## __mod ## _detach, \
- bfa_ ## __mod ## _start, \
- bfa_ ## __mod ## _stop, \
- bfa_ ## __mod ## _iocdisable, \
+ .meminfo = bfa_ ## __mod ## _meminfo, \
+ .attach = bfa_ ## __mod ## _attach, \
+ .detach = bfa_ ## __mod ## _detach, \
+ .start = bfa_ ## __mod ## _start, \
+ .stop = bfa_ ## __mod ## _stop, \
+ .iocdisable = bfa_ ## __mod ## _iocdisable, \
}
#define BFA_CACHELINE_SZ (256)
*/
#include "libfcoe.h"
-static atomic_t ctlr_num;
-static atomic_t fcf_num;
+static atomic_unchecked_t ctlr_num;
+static atomic_unchecked_t fcf_num;
/*
* fcoe_fcf_dev_loss_tmo: the default number of seconds that fcoe sysfs
if (!ctlr)
goto out;
- ctlr->id = atomic_inc_return(&ctlr_num) - 1;
+ ctlr->id = atomic_inc_return_unchecked(&ctlr_num) - 1;
ctlr->f = f;
ctlr->mode = FIP_CONN_TYPE_FABRIC;
INIT_LIST_HEAD(&ctlr->fcfs);
fcf->dev.parent = &ctlr->dev;
fcf->dev.bus = &fcoe_bus_type;
fcf->dev.type = &fcoe_fcf_device_type;
- fcf->id = atomic_inc_return(&fcf_num) - 1;
+ fcf->id = atomic_inc_return_unchecked(&fcf_num) - 1;
fcf->state = FCOE_FCF_STATE_UNKNOWN;
fcf->dev_loss_tmo = ctlr->fcf_dev_loss_tmo;
{
int error;
- atomic_set(&ctlr_num, 0);
- atomic_set(&fcf_num, 0);
+ atomic_set_unchecked(&ctlr_num, 0);
+ atomic_set_unchecked(&fcf_num, 0);
error = bus_register(&fcoe_bus_type);
if (error)
#include "scsi_logging.h"
-static atomic_t scsi_host_next_hn = ATOMIC_INIT(0); /* host_no for next new host */
+static atomic_unchecked_t scsi_host_next_hn = ATOMIC_INIT(0); /* host_no for next new host */
static void scsi_host_cls_release(struct device *dev)
* subtract one because we increment first then return, but we need to
* know what the next host number was before increment
*/
- shost->host_no = atomic_inc_return(&scsi_host_next_hn) - 1;
+ shost->host_no = atomic_inc_return_unchecked(&scsi_host_next_hn) - 1;
shost->dma_channel = 0xff;
/* These three are default values which can be overridden */
struct reply_queue_buffer *rq = &h->reply_queue[q];
if (h->transMethod & CFGTBL_Trans_io_accel1)
- return h->access.command_completed(h, q);
+ return h->access->command_completed(h, q);
if (unlikely(!(h->transMethod & CFGTBL_Trans_Performant)))
- return h->access.command_completed(h, q);
+ return h->access->command_completed(h, q);
if ((rq->head[rq->current_entry] & 1) == rq->wraparound) {
a = rq->head[rq->current_entry];
while (!list_empty(&h->reqQ)) {
c = list_entry(h->reqQ.next, struct CommandList, list);
/* can't do anything if fifo is full */
- if ((h->access.fifo_full(h))) {
+ if ((h->access->fifo_full(h))) {
h->fifo_recently_full = 1;
dev_warn(&h->pdev->dev, "fifo full\n");
break;
atomic_inc(&h->commands_outstanding);
spin_unlock_irqrestore(&h->lock, *flags);
/* Tell the controller execute command */
- h->access.submit_command(h, c);
+ h->access->submit_command(h, c);
spin_lock_irqsave(&h->lock, *flags);
}
}
static inline unsigned long get_next_completion(struct ctlr_info *h, u8 q)
{
- return h->access.command_completed(h, q);
+ return h->access->command_completed(h, q);
}
static inline bool interrupt_pending(struct ctlr_info *h)
{
- return h->access.intr_pending(h);
+ return h->access->intr_pending(h);
}
static inline long interrupt_not_for_us(struct ctlr_info *h)
{
- return (h->access.intr_pending(h) == 0) ||
+ return (h->access->intr_pending(h) == 0) ||
(h->interrupts_enabled == 0);
}
if (prod_index < 0)
return -ENODEV;
h->product_name = products[prod_index].product_name;
- h->access = *(products[prod_index].access);
+ h->access = products[prod_index].access;
pci_disable_link_state(h->pdev, PCIE_LINK_STATE_L0S |
PCIE_LINK_STATE_L1 | PCIE_LINK_STATE_CLKPM);
unsigned long flags;
u32 lockup_detected;
- h->access.set_intr_mask(h, HPSA_INTR_OFF);
+ h->access->set_intr_mask(h, HPSA_INTR_OFF);
spin_lock_irqsave(&h->lock, flags);
lockup_detected = readl(h->vaddr + SA5_SCRATCHPAD_OFFSET);
if (!lockup_detected) {
}
/* make sure the board interrupts are off */
- h->access.set_intr_mask(h, HPSA_INTR_OFF);
+ h->access->set_intr_mask(h, HPSA_INTR_OFF);
if (hpsa_request_irq(h, do_hpsa_intr_msi, do_hpsa_intr_intx))
goto clean2;
* fake ones to scoop up any residual completions.
*/
spin_lock_irqsave(&h->lock, flags);
- h->access.set_intr_mask(h, HPSA_INTR_OFF);
+ h->access->set_intr_mask(h, HPSA_INTR_OFF);
spin_unlock_irqrestore(&h->lock, flags);
free_irqs(h);
rc = hpsa_request_irq(h, hpsa_msix_discard_completions,
dev_info(&h->pdev->dev, "Board READY.\n");
dev_info(&h->pdev->dev,
"Waiting for stale completions to drain.\n");
- h->access.set_intr_mask(h, HPSA_INTR_ON);
+ h->access->set_intr_mask(h, HPSA_INTR_ON);
msleep(10000);
- h->access.set_intr_mask(h, HPSA_INTR_OFF);
+ h->access->set_intr_mask(h, HPSA_INTR_OFF);
rc = controller_reset_failed(h->cfgtable);
if (rc)
h->drv_req_rescan = 0;
/* Turn the interrupts on so we can service requests */
- h->access.set_intr_mask(h, HPSA_INTR_ON);
+ h->access->set_intr_mask(h, HPSA_INTR_ON);
hpsa_hba_inquiry(h);
hpsa_register_scsi(h); /* hook ourselves into SCSI subsystem */
* To write all data in the battery backed cache to disks
*/
hpsa_flush_cache(h);
- h->access.set_intr_mask(h, HPSA_INTR_OFF);
+ h->access->set_intr_mask(h, HPSA_INTR_OFF);
hpsa_free_irqs_and_disable_msix(h);
}
CFGTBL_Trans_enable_directed_msix |
(trans_support & (CFGTBL_Trans_io_accel1 |
CFGTBL_Trans_io_accel2));
- struct access_method access = SA5_performant_access;
+ struct access_method *access = &SA5_performant_access;
/* This is a bit complicated. There are 8 registers on
* the controller which we write to to tell it 8 different
* perform the superfluous readl() after each command submission.
*/
if (trans_support & (CFGTBL_Trans_io_accel1 | CFGTBL_Trans_io_accel2))
- access = SA5_performant_access_no_read;
+ access = &SA5_performant_access_no_read;
/* Controller spec: zero out this buffer. */
for (i = 0; i < h->nreply_queues; i++)
* enable outbound interrupt coalescing in accelerator mode;
*/
if (trans_support & CFGTBL_Trans_io_accel1) {
- access = SA5_ioaccel_mode1_access;
+ access = &SA5_ioaccel_mode1_access;
writel(10, &h->cfgtable->HostWrite.CoalIntDelay);
writel(4, &h->cfgtable->HostWrite.CoalIntCount);
} else {
if (trans_support & CFGTBL_Trans_io_accel2) {
- access = SA5_ioaccel_mode2_access;
+ access = &SA5_ioaccel_mode2_access;
writel(10, &h->cfgtable->HostWrite.CoalIntDelay);
writel(4, &h->cfgtable->HostWrite.CoalIntCount);
}
unsigned int msix_vector;
unsigned int msi_vector;
int intr_mode; /* either PERF_MODE_INT or SIMPLE_MODE_INT */
- struct access_method access;
+ struct access_method *access;
char hba_mode_enabled;
/* queue and queue Info */
}
static struct access_method SA5_access = {
- SA5_submit_command,
- SA5_intr_mask,
- SA5_fifo_full,
- SA5_intr_pending,
- SA5_completed,
+ .submit_command = SA5_submit_command,
+ .set_intr_mask = SA5_intr_mask,
+ .fifo_full = SA5_fifo_full,
+ .intr_pending = SA5_intr_pending,
+ .command_completed = SA5_completed,
};
static struct access_method SA5_ioaccel_mode1_access = {
- SA5_submit_command,
- SA5_performant_intr_mask,
- SA5_fifo_full,
- SA5_ioaccel_mode1_intr_pending,
- SA5_ioaccel_mode1_completed,
+ .submit_command = SA5_submit_command,
+ .set_intr_mask = SA5_performant_intr_mask,
+ .fifo_full = SA5_fifo_full,
+ .intr_pending = SA5_ioaccel_mode1_intr_pending,
+ .command_completed = SA5_ioaccel_mode1_completed,
};
static struct access_method SA5_ioaccel_mode2_access = {
- SA5_submit_command_ioaccel2,
- SA5_performant_intr_mask,
- SA5_fifo_full,
- SA5_performant_intr_pending,
- SA5_performant_completed,
+ .submit_command = SA5_submit_command_ioaccel2,
+ .set_intr_mask = SA5_performant_intr_mask,
+ .fifo_full = SA5_fifo_full,
+ .intr_pending = SA5_performant_intr_pending,
+ .command_completed = SA5_performant_completed,
};
static struct access_method SA5_performant_access = {
- SA5_submit_command,
- SA5_performant_intr_mask,
- SA5_fifo_full,
- SA5_performant_intr_pending,
- SA5_performant_completed,
+ .submit_command = SA5_submit_command,
+ .set_intr_mask = SA5_performant_intr_mask,
+ .fifo_full = SA5_fifo_full,
+ .intr_pending = SA5_performant_intr_pending,
+ .command_completed = SA5_performant_completed,
};
static struct access_method SA5_performant_access_no_read = {
- SA5_submit_command_no_read,
- SA5_performant_intr_mask,
- SA5_fifo_full,
- SA5_performant_intr_pending,
- SA5_performant_completed,
+ .submit_command = SA5_submit_command_no_read,
+ .set_intr_mask = SA5_performant_intr_mask,
+ .fifo_full = SA5_fifo_full,
+ .intr_pending = SA5_performant_intr_pending,
+ .command_completed = SA5_performant_completed,
};
struct board_type {
u16 pool_max_index;
struct {
- atomic_t no_free_exch;
- atomic_t no_free_exch_xid;
- atomic_t xid_not_found;
- atomic_t xid_busy;
- atomic_t seq_not_found;
- atomic_t non_bls_resp;
+ atomic_unchecked_t no_free_exch;
+ atomic_unchecked_t no_free_exch_xid;
+ atomic_unchecked_t xid_not_found;
+ atomic_unchecked_t xid_busy;
+ atomic_unchecked_t seq_not_found;
+ atomic_unchecked_t non_bls_resp;
} stats;
};
/* allocate memory for exchange */
ep = mempool_alloc(mp->ep_pool, GFP_ATOMIC);
if (!ep) {
- atomic_inc(&mp->stats.no_free_exch);
+ atomic_inc_unchecked(&mp->stats.no_free_exch);
goto out;
}
memset(ep, 0, sizeof(*ep));
return ep;
err:
spin_unlock_bh(&pool->lock);
- atomic_inc(&mp->stats.no_free_exch_xid);
+ atomic_inc_unchecked(&mp->stats.no_free_exch_xid);
mempool_free(ep, mp->ep_pool);
return NULL;
}
xid = ntohs(fh->fh_ox_id); /* we originated exch */
ep = fc_exch_find(mp, xid);
if (!ep) {
- atomic_inc(&mp->stats.xid_not_found);
+ atomic_inc_unchecked(&mp->stats.xid_not_found);
reject = FC_RJT_OX_ID;
goto out;
}
ep = fc_exch_find(mp, xid);
if ((f_ctl & FC_FC_FIRST_SEQ) && fc_sof_is_init(fr_sof(fp))) {
if (ep) {
- atomic_inc(&mp->stats.xid_busy);
+ atomic_inc_unchecked(&mp->stats.xid_busy);
reject = FC_RJT_RX_ID;
goto rel;
}
}
xid = ep->xid; /* get our XID */
} else if (!ep) {
- atomic_inc(&mp->stats.xid_not_found);
+ atomic_inc_unchecked(&mp->stats.xid_not_found);
reject = FC_RJT_RX_ID; /* XID not found */
goto out;
}
} else {
sp = &ep->seq;
if (sp->id != fh->fh_seq_id) {
- atomic_inc(&mp->stats.seq_not_found);
+ atomic_inc_unchecked(&mp->stats.seq_not_found);
if (f_ctl & FC_FC_END_SEQ) {
/*
* Update sequence_id based on incoming last
ep = fc_exch_find(mp, ntohs(fh->fh_ox_id));
if (!ep) {
- atomic_inc(&mp->stats.xid_not_found);
+ atomic_inc_unchecked(&mp->stats.xid_not_found);
goto out;
}
if (ep->esb_stat & ESB_ST_COMPLETE) {
- atomic_inc(&mp->stats.xid_not_found);
+ atomic_inc_unchecked(&mp->stats.xid_not_found);
goto rel;
}
if (ep->rxid == FC_XID_UNKNOWN)
ep->rxid = ntohs(fh->fh_rx_id);
if (ep->sid != 0 && ep->sid != ntoh24(fh->fh_d_id)) {
- atomic_inc(&mp->stats.xid_not_found);
+ atomic_inc_unchecked(&mp->stats.xid_not_found);
goto rel;
}
if (ep->did != ntoh24(fh->fh_s_id) &&
ep->did != FC_FID_FLOGI) {
- atomic_inc(&mp->stats.xid_not_found);
+ atomic_inc_unchecked(&mp->stats.xid_not_found);
goto rel;
}
sof = fr_sof(fp);
sp->ssb_stat |= SSB_ST_RESP;
sp->id = fh->fh_seq_id;
} else if (sp->id != fh->fh_seq_id) {
- atomic_inc(&mp->stats.seq_not_found);
+ atomic_inc_unchecked(&mp->stats.seq_not_found);
goto rel;
}
sp = fc_seq_lookup_orig(mp, fp); /* doesn't hold sequence */
if (!sp)
- atomic_inc(&mp->stats.xid_not_found);
+ atomic_inc_unchecked(&mp->stats.xid_not_found);
else
- atomic_inc(&mp->stats.non_bls_resp);
+ atomic_inc_unchecked(&mp->stats.non_bls_resp);
fc_frame_free(fp);
}
list_for_each_entry(ema, &lport->ema_list, ema_list) {
mp = ema->mp;
- st->fc_no_free_exch += atomic_read(&mp->stats.no_free_exch);
+ st->fc_no_free_exch += atomic_read_unchecked(&mp->stats.no_free_exch);
st->fc_no_free_exch_xid +=
- atomic_read(&mp->stats.no_free_exch_xid);
- st->fc_xid_not_found += atomic_read(&mp->stats.xid_not_found);
- st->fc_xid_busy += atomic_read(&mp->stats.xid_busy);
- st->fc_seq_not_found += atomic_read(&mp->stats.seq_not_found);
- st->fc_non_bls_resp += atomic_read(&mp->stats.non_bls_resp);
+ atomic_read_unchecked(&mp->stats.no_free_exch_xid);
+ st->fc_xid_not_found += atomic_read_unchecked(&mp->stats.xid_not_found);
+ st->fc_xid_busy += atomic_read_unchecked(&mp->stats.xid_busy);
+ st->fc_seq_not_found += atomic_read_unchecked(&mp->stats.seq_not_found);
+ st->fc_non_bls_resp += atomic_read_unchecked(&mp->stats.non_bls_resp);
}
}
EXPORT_SYMBOL(fc_exch_update_stats);
.postreset = ata_std_postreset,
.error_handler = ata_std_error_handler,
.post_internal_cmd = sas_ata_post_internal,
- .qc_defer = ata_std_qc_defer,
+ .qc_defer = ata_std_qc_defer,
.qc_prep = ata_noop_qc_prep,
.qc_issue = sas_ata_qc_issue,
.qc_fill_rtf = sas_ata_qc_fill_rtf,
struct dentry *debug_nodelist;
struct dentry *vport_debugfs_root;
struct lpfc_debugfs_trc *disc_trc;
- atomic_t disc_trc_cnt;
+ atomic_unchecked_t disc_trc_cnt;
#endif
uint8_t stat_data_enabled;
uint8_t stat_data_blocked;
struct timer_list fabric_block_timer;
unsigned long bit_flags;
#define FABRIC_COMANDS_BLOCKED 0
- atomic_t num_rsrc_err;
- atomic_t num_cmd_success;
+ atomic_unchecked_t num_rsrc_err;
+ atomic_unchecked_t num_cmd_success;
unsigned long last_rsrc_error_time;
unsigned long last_ramp_down_time;
#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
struct dentry *debug_slow_ring_trc;
struct lpfc_debugfs_trc *slow_ring_trc;
- atomic_t slow_ring_trc_cnt;
+ atomic_unchecked_t slow_ring_trc_cnt;
/* iDiag debugfs sub-directory */
struct dentry *idiag_root;
struct dentry *idiag_pci_cfg;
#include <linux/debugfs.h>
-static atomic_t lpfc_debugfs_seq_trc_cnt = ATOMIC_INIT(0);
+static atomic_unchecked_t lpfc_debugfs_seq_trc_cnt = ATOMIC_INIT(0);
static unsigned long lpfc_debugfs_start_time = 0L;
/* iDiag */
lpfc_debugfs_enable = 0;
len = 0;
- index = (atomic_read(&vport->disc_trc_cnt) + 1) &
+ index = (atomic_read_unchecked(&vport->disc_trc_cnt) + 1) &
(lpfc_debugfs_max_disc_trc - 1);
for (i = index; i < lpfc_debugfs_max_disc_trc; i++) {
dtp = vport->disc_trc + i;
lpfc_debugfs_enable = 0;
len = 0;
- index = (atomic_read(&phba->slow_ring_trc_cnt) + 1) &
+ index = (atomic_read_unchecked(&phba->slow_ring_trc_cnt) + 1) &
(lpfc_debugfs_max_slow_ring_trc - 1);
for (i = index; i < lpfc_debugfs_max_slow_ring_trc; i++) {
dtp = phba->slow_ring_trc + i;
!vport || !vport->disc_trc)
return;
- index = atomic_inc_return(&vport->disc_trc_cnt) &
+ index = atomic_inc_return_unchecked(&vport->disc_trc_cnt) &
(lpfc_debugfs_max_disc_trc - 1);
dtp = vport->disc_trc + index;
dtp->fmt = fmt;
dtp->data1 = data1;
dtp->data2 = data2;
dtp->data3 = data3;
- dtp->seq_cnt = atomic_inc_return(&lpfc_debugfs_seq_trc_cnt);
+ dtp->seq_cnt = atomic_inc_return_unchecked(&lpfc_debugfs_seq_trc_cnt);
dtp->jif = jiffies;
#endif
return;
!phba || !phba->slow_ring_trc)
return;
- index = atomic_inc_return(&phba->slow_ring_trc_cnt) &
+ index = atomic_inc_return_unchecked(&phba->slow_ring_trc_cnt) &
(lpfc_debugfs_max_slow_ring_trc - 1);
dtp = phba->slow_ring_trc + index;
dtp->fmt = fmt;
dtp->data1 = data1;
dtp->data2 = data2;
dtp->data3 = data3;
- dtp->seq_cnt = atomic_inc_return(&lpfc_debugfs_seq_trc_cnt);
+ dtp->seq_cnt = atomic_inc_return_unchecked(&lpfc_debugfs_seq_trc_cnt);
dtp->jif = jiffies;
#endif
return;
"slow_ring buffer\n");
goto debug_failed;
}
- atomic_set(&phba->slow_ring_trc_cnt, 0);
+ atomic_set_unchecked(&phba->slow_ring_trc_cnt, 0);
memset(phba->slow_ring_trc, 0,
(sizeof(struct lpfc_debugfs_trc) *
lpfc_debugfs_max_slow_ring_trc));
"buffer\n");
goto debug_failed;
}
- atomic_set(&vport->disc_trc_cnt, 0);
+ atomic_set_unchecked(&vport->disc_trc_cnt, 0);
snprintf(name, sizeof(name), "discovery_trace");
vport->debug_disc_trc =
"misc_register returned with status %d", error);
if (lpfc_enable_npiv) {
- lpfc_transport_functions.vport_create = lpfc_vport_create;
- lpfc_transport_functions.vport_delete = lpfc_vport_delete;
+ pax_open_kernel();
+ *(void **)&lpfc_transport_functions.vport_create = lpfc_vport_create;
+ *(void **)&lpfc_transport_functions.vport_delete = lpfc_vport_delete;
+ pax_close_kernel();
}
lpfc_transport_template =
fc_attach_transport(&lpfc_transport_functions);
unsigned long expires;
spin_lock_irqsave(&phba->hbalock, flags);
- atomic_inc(&phba->num_rsrc_err);
+ atomic_inc_unchecked(&phba->num_rsrc_err);
phba->last_rsrc_error_time = jiffies;
expires = phba->last_ramp_down_time + QUEUE_RAMP_DOWN_INTERVAL;
unsigned long num_rsrc_err, num_cmd_success;
int i;
- num_rsrc_err = atomic_read(&phba->num_rsrc_err);
- num_cmd_success = atomic_read(&phba->num_cmd_success);
+ num_rsrc_err = atomic_read_unchecked(&phba->num_rsrc_err);
+ num_cmd_success = atomic_read_unchecked(&phba->num_cmd_success);
/*
* The error and success command counters are global per
}
}
lpfc_destroy_vport_work_array(phba, vports);
- atomic_set(&phba->num_rsrc_err, 0);
- atomic_set(&phba->num_cmd_success, 0);
+ atomic_set_unchecked(&phba->num_rsrc_err, 0);
+ atomic_set_unchecked(&phba->num_cmd_success, 0);
}
/**
{
struct scsi_device *sdev = to_scsi_device(dev);
struct MPT2SAS_ADAPTER *ioc = shost_priv(sdev->host);
- static struct _raid_device *raid_device;
+ struct _raid_device *raid_device;
unsigned long flags;
Mpi2RaidVolPage0_t vol_pg0;
Mpi2ConfigReply_t mpi_reply;
{
struct scsi_device *sdev = to_scsi_device(dev);
struct MPT2SAS_ADAPTER *ioc = shost_priv(sdev->host);
- static struct _raid_device *raid_device;
+ struct _raid_device *raid_device;
unsigned long flags;
Mpi2RaidVolPage0_t vol_pg0;
Mpi2ConfigReply_t mpi_reply;
Mpi2EventDataIrOperationStatus_t *event_data =
(Mpi2EventDataIrOperationStatus_t *)
fw_event->event_data;
- static struct _raid_device *raid_device;
+ struct _raid_device *raid_device;
unsigned long flags;
u16 handle;
u64 sas_address;
struct _sas_device *sas_device;
struct _sas_node *expander_device;
- static struct _raid_device *raid_device;
+ struct _raid_device *raid_device;
u8 retry_count;
unsigned long flags;
res->scsi_dev = scsi_dev;
scsi_dev->hostdata = res;
res->change_detected = 0;
- atomic_set(&res->read_failures, 0);
- atomic_set(&res->write_failures, 0);
+ atomic_set_unchecked(&res->read_failures, 0);
+ atomic_set_unchecked(&res->write_failures, 0);
rc = 0;
}
spin_unlock_irqrestore(&pinstance->resource_lock, lock_flags);
/* If this was a SCSI read/write command keep count of errors */
if (SCSI_CMD_TYPE(scsi_cmd->cmnd[0]) == SCSI_READ_CMD)
- atomic_inc(&res->read_failures);
+ atomic_inc_unchecked(&res->read_failures);
else if (SCSI_CMD_TYPE(scsi_cmd->cmnd[0]) == SCSI_WRITE_CMD)
- atomic_inc(&res->write_failures);
+ atomic_inc_unchecked(&res->write_failures);
if (!RES_IS_GSCSI(res->cfg_entry) &&
masked_ioasc != PMCRAID_IOASC_HW_DEVICE_BUS_STATUS_ERROR) {
* block of scsi_cmd which is re-used (e.g. cancel/abort), which uses
* hrrq_id assigned here in queuecommand
*/
- ioarcb->hrrq_id = atomic_add_return(1, &(pinstance->last_message_id)) %
+ ioarcb->hrrq_id = atomic_add_return_unchecked(1, &(pinstance->last_message_id)) %
pinstance->num_hrrq;
cmd->cmd_done = pmcraid_io_done;
* block of scsi_cmd which is re-used (e.g. cancel/abort), which uses
* hrrq_id assigned here in queuecommand
*/
- ioarcb->hrrq_id = atomic_add_return(1, &(pinstance->last_message_id)) %
+ ioarcb->hrrq_id = atomic_add_return_unchecked(1, &(pinstance->last_message_id)) %
pinstance->num_hrrq;
if (request_size) {
pinstance = container_of(workp, struct pmcraid_instance, worker_q);
/* add resources only after host is added into system */
- if (!atomic_read(&pinstance->expose_resources))
+ if (!atomic_read_unchecked(&pinstance->expose_resources))
return;
fw_version = be16_to_cpu(pinstance->inq_data->fw_version);
init_waitqueue_head(&pinstance->reset_wait_q);
atomic_set(&pinstance->outstanding_cmds, 0);
- atomic_set(&pinstance->last_message_id, 0);
- atomic_set(&pinstance->expose_resources, 0);
+ atomic_set_unchecked(&pinstance->last_message_id, 0);
+ atomic_set_unchecked(&pinstance->expose_resources, 0);
INIT_LIST_HEAD(&pinstance->free_res_q);
INIT_LIST_HEAD(&pinstance->used_res_q);
/* Schedule worker thread to handle CCN and take care of adding and
* removing devices to OS
*/
- atomic_set(&pinstance->expose_resources, 1);
+ atomic_set_unchecked(&pinstance->expose_resources, 1);
schedule_work(&pinstance->worker_q);
return rc;
struct pmcraid_isr_param hrrq_vector[PMCRAID_NUM_MSIX_VECTORS];
/* Message id as filled in last fired IOARCB, used to identify HRRQ */
- atomic_t last_message_id;
+ atomic_unchecked_t last_message_id;
/* configuration table */
struct pmcraid_config_table *cfg_table;
atomic_t outstanding_cmds;
/* should add/delete resources to mid-layer now ?*/
- atomic_t expose_resources;
+ atomic_unchecked_t expose_resources;
struct pmcraid_config_table_entry_ext cfg_entry_ext;
};
struct scsi_device *scsi_dev; /* Link scsi_device structure */
- atomic_t read_failures; /* count of failed READ commands */
- atomic_t write_failures; /* count of failed WRITE commands */
+ atomic_unchecked_t read_failures; /* count of failed READ commands */
+ atomic_unchecked_t write_failures; /* count of failed WRITE commands */
/* To indicate add/delete/modify during CCN */
u8 change_detected;
return 0;
}
-struct fc_function_template qla2xxx_transport_functions = {
+fc_function_template_no_const qla2xxx_transport_functions = {
.show_host_node_name = 1,
.show_host_port_name = 1,
.bsg_timeout = qla24xx_bsg_timeout,
};
-struct fc_function_template qla2xxx_transport_vport_functions = {
+fc_function_template_no_const qla2xxx_transport_vport_functions = {
.show_host_node_name = 1,
.show_host_port_name = 1,
struct device_attribute;
extern struct device_attribute *qla2x00_host_attrs[];
struct fc_function_template;
-extern struct fc_function_template qla2xxx_transport_functions;
-extern struct fc_function_template qla2xxx_transport_vport_functions;
+extern fc_function_template_no_const qla2xxx_transport_functions;
+extern fc_function_template_no_const qla2xxx_transport_vport_functions;
extern void qla2x00_alloc_sysfs_attr(scsi_qla_host_t *);
extern void qla2x00_free_sysfs_attr(scsi_qla_host_t *, bool);
extern void qla2x00_init_host_attr(scsi_qla_host_t *);
!pci_set_consistent_dma_mask(ha->pdev, DMA_BIT_MASK(64))) {
/* Ok, a 64bit DMA mask is applicable. */
ha->flags.enable_64bit_addressing = 1;
- ha->isp_ops->calc_req_entries = qla2x00_calc_iocbs_64;
- ha->isp_ops->build_iocbs = qla2x00_build_scsi_iocbs_64;
+ pax_open_kernel();
+ *(void **)&ha->isp_ops->calc_req_entries = qla2x00_calc_iocbs_64;
+ *(void **)&ha->isp_ops->build_iocbs = qla2x00_build_scsi_iocbs_64;
+ pax_close_kernel();
return;
}
}
* (4000 only) */
atomic_t relogin_timer; /* Max Time to wait for
* relogin to complete */
- atomic_t relogin_retry_count; /* Num of times relogin has been
+ atomic_unchecked_t relogin_retry_count; /* Num of times relogin has been
* retried */
uint32_t default_time2wait; /* Default Min time between
* relogins (+aens) */
*/
if (!iscsi_is_session_online(cls_sess)) {
/* Reset retry relogin timer */
- atomic_inc(&ddb_entry->relogin_retry_count);
+ atomic_inc_unchecked(&ddb_entry->relogin_retry_count);
DEBUG2(ql4_printk(KERN_INFO, ha,
"%s: index[%d] relogin timed out-retrying"
" relogin (%d), retry (%d)\n", __func__,
ddb_entry->fw_ddb_index,
- atomic_read(&ddb_entry->relogin_retry_count),
+ atomic_read_unchecked(&ddb_entry->relogin_retry_count),
ddb_entry->default_time2wait + 4));
set_bit(DPC_RELOGIN_DEVICE, &ha->dpc_flags);
atomic_set(&ddb_entry->retry_relogin_timer,
atomic_set(&ddb_entry->retry_relogin_timer, INVALID_ENTRY);
atomic_set(&ddb_entry->relogin_timer, 0);
- atomic_set(&ddb_entry->relogin_retry_count, 0);
+ atomic_set_unchecked(&ddb_entry->relogin_retry_count, 0);
def_timeout = le16_to_cpu(ddb_entry->fw_ddb_entry.def_timeout);
ddb_entry->default_relogin_timeout =
(def_timeout > LOGIN_TOV) && (def_timeout < LOGIN_TOV * 10) ?
shost = sdev->host;
scsi_init_cmd_errh(cmd);
cmd->result = DID_NO_CONNECT << 16;
- atomic_inc(&cmd->device->iorequest_cnt);
+ atomic_inc_unchecked(&cmd->device->iorequest_cnt);
/*
* SCSI request completion path will do scsi_device_unbusy(),
INIT_LIST_HEAD(&cmd->eh_entry);
- atomic_inc(&cmd->device->iodone_cnt);
+ atomic_inc_unchecked(&cmd->device->iodone_cnt);
if (cmd->result)
- atomic_inc(&cmd->device->ioerr_cnt);
+ atomic_inc_unchecked(&cmd->device->ioerr_cnt);
disposition = scsi_decide_disposition(cmd);
if (disposition != SUCCESS &&
struct Scsi_Host *host = cmd->device->host;
int rtn = 0;
- atomic_inc(&cmd->device->iorequest_cnt);
+ atomic_inc_unchecked(&cmd->device->iorequest_cnt);
/* check if the device is still usable */
if (unlikely(cmd->device->sdev_state == SDEV_DEL)) {
char *buf) \
{ \
struct scsi_device *sdev = to_scsi_device(dev); \
- unsigned long long count = atomic_read(&sdev->field); \
+ unsigned long long count = atomic_read_unchecked(&sdev->field); \
return snprintf(buf, 20, "0x%llx\n", count); \
} \
static DEVICE_ATTR(field, S_IRUGO, show_iostat_##field, NULL)
* Netlink Infrastructure
*/
-static atomic_t fc_event_seq;
+static atomic_unchecked_t fc_event_seq;
/**
* fc_get_event_number - Obtain the next sequential FC event number
u32
fc_get_event_number(void)
{
- return atomic_add_return(1, &fc_event_seq);
+ return atomic_add_return_unchecked(1, &fc_event_seq);
}
EXPORT_SYMBOL(fc_get_event_number);
{
int error;
- atomic_set(&fc_event_seq, 0);
+ atomic_set_unchecked(&fc_event_seq, 0);
error = transport_class_register(&fc_host_class);
if (error)
char *cp;
*val = simple_strtoul(buf, &cp, 0);
- if ((*cp && (*cp != '\n')) || (*val < 0))
+ if (*cp && (*cp != '\n'))
return -EINVAL;
/*
* Check for overflow; dev_loss_tmo is u32
struct transport_container session_cont;
};
-static atomic_t iscsi_session_nr; /* sysfs session id for next new session */
+static atomic_unchecked_t iscsi_session_nr; /* sysfs session id for next new session */
static struct workqueue_struct *iscsi_eh_timer_workq;
static DEFINE_IDA(iscsi_sess_ida);
int err;
ihost = shost->shost_data;
- session->sid = atomic_add_return(1, &iscsi_session_nr);
+ session->sid = atomic_add_return_unchecked(1, &iscsi_session_nr);
if (target_id == ISCSI_MAX_TARGET) {
id = ida_simple_get(&iscsi_sess_ida, 0, 0, GFP_KERNEL);
printk(KERN_INFO "Loading iSCSI transport class v%s.\n",
ISCSI_TRANSPORT_VERSION);
- atomic_set(&iscsi_session_nr, 0);
+ atomic_set_unchecked(&iscsi_session_nr, 0);
err = class_register(&iscsi_transport_class);
if (err)
#include "scsi_priv.h"
struct srp_host_attrs {
- atomic_t next_port_id;
+ atomic_unchecked_t next_port_id;
};
#define to_srp_host_attrs(host) ((struct srp_host_attrs *)(host)->shost_data)
struct Scsi_Host *shost = dev_to_shost(dev);
struct srp_host_attrs *srp_host = to_srp_host_attrs(shost);
- atomic_set(&srp_host->next_port_id, 0);
+ atomic_set_unchecked(&srp_host->next_port_id, 0);
return 0;
}
rport_fast_io_fail_timedout);
INIT_DELAYED_WORK(&rport->dev_loss_work, rport_dev_loss_timedout);
- id = atomic_inc_return(&to_srp_host_attrs(shost)->next_port_id);
+ id = atomic_inc_return_unchecked(&to_srp_host_attrs(shost)->next_port_id);
dev_set_name(&rport->dev, "port-%d:%d", shost->host_no, id);
transport_setup_device(&rport->dev);
sdkp->disk = gd;
sdkp->index = index;
atomic_set(&sdkp->openers, 0);
- atomic_set(&sdkp->device->ioerr_cnt, 0);
+ atomic_set_unchecked(&sdkp->device->ioerr_cnt, 0);
if (!sdp->request_queue->rq_timeout) {
if (sdp->type != TYPE_MOD)
sdp->disk->disk_name,
MKDEV(SCSI_GENERIC_MAJOR, sdp->index),
NULL,
- (char *)arg);
+ (char __user *)arg);
case BLKTRACESTART:
return blk_trace_startstop(sdp->device->request_queue, 1);
case BLKTRACESTOP:
return i;
}
-static struct bin_attribute fuse_bin_attr = {
+static bin_attribute_no_const fuse_bin_attr = {
.attr = { .name = "fuse", .mode = S_IRUGO, },
.read = fuse_read,
};
EXPORT_SYMBOL_GPL(spi_bus_unlock);
/* portable code must never pass more than 32 bytes */
-#define SPI_BUFSIZ max(32, SMP_CACHE_BYTES)
+#define SPI_BUFSIZ max(32UL, SMP_CACHE_BYTES)
static u8 *buf;
#include "timed_output.h"
static struct class *timed_output_class;
-static atomic_t device_count;
+static atomic_unchecked_t device_count;
static ssize_t enable_show(struct device *dev, struct device_attribute *attr,
char *buf)
timed_output_class = class_create(THIS_MODULE, "timed_output");
if (IS_ERR(timed_output_class))
return PTR_ERR(timed_output_class);
- atomic_set(&device_count, 0);
+ atomic_set_unchecked(&device_count, 0);
timed_output_class->dev_groups = timed_output_groups;
}
if (ret < 0)
return ret;
- tdev->index = atomic_inc_return(&device_count);
+ tdev->index = atomic_inc_return_unchecked(&device_count);
tdev->dev = device_create(timed_output_class, NULL,
MKDEV(0, tdev->index), NULL, "%s", tdev->name);
if (IS_ERR(tdev->dev))
}
cfp->last_attached = dev->attached;
cfp->last_detach_count = dev->detach_count;
- ACCESS_ONCE(cfp->read_subdev) = read_s;
- ACCESS_ONCE(cfp->write_subdev) = write_s;
+ ACCESS_ONCE_RW(cfp->read_subdev) = read_s;
+ ACCESS_ONCE_RW(cfp->write_subdev) = write_s;
}
static void comedi_file_check(struct file *file)
!(s_old->async->cmd.flags & CMDF_WRITE))
return -EBUSY;
- ACCESS_ONCE(cfp->read_subdev) = s_new;
+ ACCESS_ONCE_RW(cfp->read_subdev) = s_new;
return 0;
}
(s_old->async->cmd.flags & CMDF_WRITE))
return -EBUSY;
- ACCESS_ONCE(cfp->write_subdev) = s_new;
+ ACCESS_ONCE_RW(cfp->write_subdev) = s_new;
return 0;
}
#define gdm_tty_send_control(n, r, v, d, l) (\
n->tty_dev->send_control(n->tty_dev->priv_dev, r, v, d, l))
-#define GDM_TTY_READY(gdm) (gdm && gdm->tty_dev && gdm->port.count)
+#define GDM_TTY_READY(gdm) (gdm && gdm->tty_dev && atomic_read(&gdm->port.count))
static struct tty_driver *gdm_driver[TTY_MAX_COUNT];
static struct gdm *gdm_table[TTY_MAX_COUNT][GDM_TTY_MINOR];
{
struct usb_device *usbdev = line6->usbdev;
int ret;
- unsigned char len;
+ unsigned char *plen;
/* query the serial number: */
ret = usb_control_msg(usbdev, usb_sndctrlpipe(usbdev, 0), 0x67,
return ret;
}
+ plen = kmalloc(1, GFP_KERNEL);
+ if (plen == NULL)
+ return -ENOMEM;
+
/* Wait for data length. We'll get 0xff until length arrives. */
do {
ret = usb_control_msg(usbdev, usb_rcvctrlpipe(usbdev, 0), 0x67,
USB_TYPE_VENDOR | USB_RECIP_DEVICE |
USB_DIR_IN,
- 0x0012, 0x0000, &len, 1,
+ 0x0012, 0x0000, plen, 1,
LINE6_TIMEOUT * HZ);
if (ret < 0) {
dev_err(line6->ifcdev,
"receive length failed (error %d)\n", ret);
+ kfree(plen);
return ret;
}
- } while (len == 0xff);
+ } while (*plen == 0xff);
- if (len != datalen) {
+ if (*plen != datalen) {
/* should be equal or something went wrong */
dev_err(line6->ifcdev,
"length mismatch (expected %d, got %d)\n",
- (int)datalen, (int)len);
+ (int)datalen, (int)*plen);
+ kfree(plen);
return -EINVAL;
}
+ kfree(plen);
/* receive the result: */
ret = usb_control_msg(usbdev, usb_rcvctrlpipe(usbdev, 0), 0x67,
{
struct usb_device *usbdev = line6->usbdev;
int ret;
- unsigned char status;
+ unsigned char *status;
ret = usb_control_msg(usbdev, usb_sndctrlpipe(usbdev, 0), 0x67,
USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_OUT,
return ret;
}
+ status = kmalloc(1, GFP_KERNEL);
+ if (status == NULL)
+ return -ENOMEM;
+
do {
ret = usb_control_msg(usbdev, usb_rcvctrlpipe(usbdev, 0),
0x67,
USB_TYPE_VENDOR | USB_RECIP_DEVICE |
USB_DIR_IN,
0x0012, 0x0000,
- &status, 1, LINE6_TIMEOUT * HZ);
+ status, 1, LINE6_TIMEOUT * HZ);
if (ret < 0) {
dev_err(line6->ifcdev,
"receiving status failed (error %d)\n", ret);
+ kfree(status);
return ret;
}
- } while (status == 0xff);
+ } while (*status == 0xff);
- if (status != 0) {
+ if (*status != 0) {
dev_err(line6->ifcdev, "write failed (error %d)\n", ret);
+ kfree(status);
return -EINVAL;
}
+ kfree(status);
+
return 0;
}
*/
#include <linux/wait.h>
+#include <linux/slab.h>
#include <sound/control.h>
#include "audio.h"
*/
static void toneport_setup(struct usb_line6_toneport *toneport)
{
- int ticks;
+ int *ticks;
struct usb_line6 *line6 = &toneport->line6;
struct usb_device *usbdev = line6->usbdev;
u16 idProduct = le16_to_cpu(usbdev->descriptor.idProduct);
+ ticks = kmalloc(sizeof(int), GFP_KERNEL);
+ if (ticks == NULL)
+ return;
+
/* sync time on device with host: */
- ticks = (int)get_seconds();
- line6_write_data(line6, 0x80c6, &ticks, 4);
+ *ticks = (int)get_seconds();
+ line6_write_data(line6, 0x80c6, ticks, sizeof(int));
+
+ kfree(ticks);
/* enable device: */
toneport_send_cmd(usbdev, 0x0301, 0x0000);
return 0;
}
-sfw_test_client_ops_t brw_test_client;
-void brw_init_test_client(void)
-{
- brw_test_client.tso_init = brw_client_init;
- brw_test_client.tso_fini = brw_client_fini;
- brw_test_client.tso_prep_rpc = brw_client_prep_rpc;
- brw_test_client.tso_done_rpc = brw_client_done_rpc;
+sfw_test_client_ops_t brw_test_client = {
+ .tso_init = brw_client_init,
+ .tso_fini = brw_client_fini,
+ .tso_prep_rpc = brw_client_prep_rpc,
+ .tso_done_rpc = brw_client_done_rpc,
};
srpc_service_t brw_test_service;
extern sfw_test_client_ops_t ping_test_client;
extern srpc_service_t ping_test_service;
-extern void ping_init_test_client(void);
extern void ping_init_test_service(void);
extern sfw_test_client_ops_t brw_test_client;
extern srpc_service_t brw_test_service;
-extern void brw_init_test_client(void);
extern void brw_init_test_service(void);
INIT_LIST_HEAD(&sfw_data.fw_zombie_rpcs);
INIT_LIST_HEAD(&sfw_data.fw_zombie_sessions);
- brw_init_test_client();
brw_init_test_service();
rc = sfw_register_test(&brw_test_service, &brw_test_client);
LASSERT (rc == 0);
- ping_init_test_client();
ping_init_test_service();
rc = sfw_register_test(&ping_test_service, &ping_test_client);
LASSERT (rc == 0);
return 0;
}
-sfw_test_client_ops_t ping_test_client;
-void ping_init_test_client(void)
-{
- ping_test_client.tso_init = ping_client_init;
- ping_test_client.tso_fini = ping_client_fini;
- ping_test_client.tso_prep_rpc = ping_client_prep_rpc;
- ping_test_client.tso_done_rpc = ping_client_done_rpc;
-}
+sfw_test_client_ops_t ping_test_client = {
+ .tso_init = ping_client_init,
+ .tso_fini = ping_client_fini,
+ .tso_prep_rpc = ping_client_prep_rpc,
+ .tso_done_rpc = ping_client_done_rpc,
+};
srpc_service_t ping_test_service;
void ping_init_test_service(void)
ldlm_completion_callback lcs_completion;
ldlm_blocking_callback lcs_blocking;
ldlm_glimpse_callback lcs_glimpse;
-};
+} __no_const;
/* ldlm_lockd.c */
int ldlm_del_waiting_lock(struct ldlm_lock *lock);
* lprocfs_alloc_md_stats() in obdclass/lprocfs_status.c. Also, add a
* wrapper function in include/linux/obd_class.h.
*/
-};
+} __no_const;
struct lsm_operations {
void (*lsm_free)(struct lov_stripe_md *);
int added = (mode == LCK_NL);
int overlaps = 0;
int splitted = 0;
- const struct ldlm_callback_suite null_cbs = { NULL };
+ const struct ldlm_callback_suite null_cbs = { };
CDEBUG(D_DLMTRACE,
"flags %#llx owner %llu pid %u mode %u start %llu end %llu\n",
void __user *buffer, size_t *lenp, loff_t *ppos)
{
int rc, max_delay_cs;
- struct ctl_table dummy = *table;
+ ctl_table_no_const dummy = *table;
long d;
dummy.data = &max_delay_cs;
void __user *buffer, size_t *lenp, loff_t *ppos)
{
int rc, min_delay_cs;
- struct ctl_table dummy = *table;
+ ctl_table_no_const dummy = *table;
long d;
dummy.data = &min_delay_cs;
void __user *buffer, size_t *lenp, loff_t *ppos)
{
int rc, backoff;
- struct ctl_table dummy = *table;
+ ctl_table_no_const dummy = *table;
dummy.data = &backoff;
dummy.proc_handler = &proc_dointvec;
struct cfs_psdev_ops libcfs_psdev_ops = {
- libcfs_psdev_open,
- libcfs_psdev_release,
- NULL,
- NULL,
- libcfs_ioctl
+ .p_open = libcfs_psdev_open,
+ .p_close = libcfs_psdev_release,
+ .p_read = NULL,
+ .p_write = NULL,
+ .p_ioctl = libcfs_ioctl
};
extern int insert_proc(void);
/* Increment RX stats for virtual ports */
if (work->ipprt >= CVMX_PIP_NUM_INPUT_PORTS) {
#ifdef CONFIG_64BIT
- atomic64_add(1,
+ atomic64_add_unchecked(1,
(atomic64_t *)&priv->stats.rx_packets);
- atomic64_add(skb->len,
+ atomic64_add_unchecked(skb->len,
(atomic64_t *)&priv->stats.rx_bytes);
#else
- atomic_add(1,
+ atomic_add_unchecked(1,
(atomic_t *)&priv->stats.rx_packets);
- atomic_add(skb->len,
+ atomic_add_unchecked(skb->len,
(atomic_t *)&priv->stats.rx_bytes);
#endif
}
dev->name);
*/
#ifdef CONFIG_64BIT
- atomic64_add(1,
+ atomic64_add_unchecked(1,
(atomic64_t *)&priv->stats.rx_dropped);
#else
- atomic_add(1,
+ atomic_add_unchecked(1,
(atomic_t *)&priv->stats.rx_dropped);
#endif
dev_kfree_skb_irq(skb);
* since the RX tasklet also increments it.
*/
#ifdef CONFIG_64BIT
- atomic64_add(rx_status.dropped_packets,
- (atomic64_t *)&priv->stats.rx_dropped);
+ atomic64_add_unchecked(rx_status.dropped_packets,
+ (atomic64_unchecked_t *)&priv->stats.rx_dropped);
#else
- atomic_add(rx_status.dropped_packets,
- (atomic_t *)&priv->stats.rx_dropped);
+ atomic_add_unchecked(rx_status.dropped_packets,
+ (atomic_unchecked_t *)&priv->stats.rx_dropped);
#endif
}
void (*hal_notch_filter)(struct adapter *adapter, bool enable);
void (*hal_reset_security_engine)(struct adapter *adapter);
-};
+} __no_const;
enum rt_eeprom_type {
EEPROM_93C46,
u8 *pmem);
u32 (*_write_port)(struct intf_hdl *pintfhdl, u32 addr, u32 cnt,
u8 *pmem);
-};
+} __no_const;
struct io_req {
struct list_head list;
void (*device_resume)(ulong bus_no, ulong dev_no);
int (*get_channel_info)(uuid_le type_uuid, ulong *min_size,
ulong *max_size);
-};
+} __no_const;
/* These functions live inside visorchipset, and will be called to indicate
* responses to specific events (by code outside of visorchipset).
void (*device_destroy)(ulong bus_no, ulong dev_no, int response);
void (*device_pause)(ulong bus_no, ulong dev_no, int response);
void (*device_resume)(ulong bus_no, ulong dev_no, int response);
-};
+} __no_const;
/** Register functions (in the bus driver) to get called by visorchipset
* whenever a bus or device appears for which this service partition is
#define SESSION_MAINTENANCE_INTERVAL HZ
-static atomic_t login_id = ATOMIC_INIT(0);
+static atomic_unchecked_t login_id = ATOMIC_INIT(0);
static void session_maintenance_work(struct work_struct *);
static int sbp_run_transaction(struct fw_card *, int, int, int, int,
login->lun = se_lun;
login->status_fifo_addr = sbp2_pointer_to_addr(&req->orb.status_fifo);
login->exclusive = LOGIN_ORB_EXCLUSIVE(be32_to_cpu(req->orb.misc));
- login->login_id = atomic_inc_return(&login_id);
+ login->login_id = atomic_inc_return_unchecked(&login_id);
login->tgt_agt = sbp_target_agent_register(login);
if (IS_ERR(login->tgt_agt)) {
spin_lock_init(&dev->se_tmr_lock);
spin_lock_init(&dev->qf_cmd_lock);
sema_init(&dev->caw_sem, 1);
- atomic_set(&dev->dev_ordered_id, 0);
+ atomic_set_unchecked(&dev->dev_ordered_id, 0);
INIT_LIST_HEAD(&dev->t10_wwn.t10_vpd_list);
spin_lock_init(&dev->t10_wwn.t10_vpd_lock);
INIT_LIST_HEAD(&dev->t10_pr.registration_list);
* Used to determine when ORDERED commands should go from
* Dormant to Active status.
*/
- cmd->se_ordered_id = atomic_inc_return(&dev->dev_ordered_id);
+ cmd->se_ordered_id = atomic_inc_return_unchecked(&dev->dev_ordered_id);
pr_debug("Allocated se_ordered_id: %u for Task Attr: 0x%02x on %s\n",
cmd->se_ordered_id, cmd->sam_task_attr,
dev->transport->name);
platform_set_drvdata(pdev, priv);
if (priv->uuid_bitmap & 1 << INT3400_THERMAL_PASSIVE_1) {
- int3400_thermal_ops.get_mode = int3400_thermal_get_mode;
- int3400_thermal_ops.set_mode = int3400_thermal_set_mode;
+ pax_open_kernel();
+ *(void **)&int3400_thermal_ops.get_mode = int3400_thermal_get_mode;
+ *(void **)&int3400_thermal_ops.set_mode = int3400_thermal_set_mode;
+ pax_close_kernel();
}
priv->thermal = thermal_zone_device_register("INT3400 Thermal", 0, 0,
priv, &int3400_thermal_ops,
#include <linux/export.h>
#include <linux/string.h>
#include <linux/thermal.h>
+#include <linux/mm.h>
#include "thermal_core.h"
tz->ops = ops;
tz->sensor_data = data;
- tzd->ops->get_temp = of_thermal_get_temp;
- tzd->ops->get_trend = of_thermal_get_trend;
- tzd->ops->set_emul_temp = of_thermal_set_emul_temp;
+ pax_open_kernel();
+ *(void **)&tzd->ops->get_temp = of_thermal_get_temp;
+ *(void **)&tzd->ops->get_trend = of_thermal_get_trend;
+ *(void **)&tzd->ops->set_emul_temp = of_thermal_set_emul_temp;
+ pax_close_kernel();
mutex_unlock(&tzd->lock);
return tzd;
return;
mutex_lock(&tzd->lock);
- tzd->ops->get_temp = NULL;
- tzd->ops->get_trend = NULL;
- tzd->ops->set_emul_temp = NULL;
+ pax_open_kernel();
+ *(void **)&tzd->ops->get_temp = NULL;
+ *(void **)&tzd->ops->get_trend = NULL;
+ *(void **)&tzd->ops->set_emul_temp = NULL;
+ pax_close_kernel();
tz->ops = NULL;
tz->sensor_data = NULL;
printk(KERN_DEBUG "cyc:cy_open ttyC%d, count = %d\n", info->line,
info->port.count);
#endif
- info->port.count++;
+ atomic_inc(&info->port.count);
#ifdef CY_DEBUG_COUNT
printk(KERN_DEBUG "cyc:cy_open (%d): incrementing count to %d\n",
- current->pid, info->port.count);
+ current->pid, atomic_read(&info->port.count));
#endif
/*
for (j = 0; j < cy_card[i].nports; j++) {
info = &cy_card[i].ports[j];
- if (info->port.count) {
+ if (atomic_read(&info->port.count)) {
/* XXX is the ldisc num worth this? */
struct tty_struct *tty;
struct tty_ldisc *ld;
spin_lock_irqsave(&hp->port.lock, flags);
/* Check and then increment for fast path open. */
- if (hp->port.count++ > 0) {
+ if (atomic_inc_return(&hp->port.count) > 1) {
spin_unlock_irqrestore(&hp->port.lock, flags);
hvc_kick();
return 0;
spin_lock_irqsave(&hp->port.lock, flags);
- if (--hp->port.count == 0) {
+ if (atomic_dec_return(&hp->port.count) == 0) {
spin_unlock_irqrestore(&hp->port.lock, flags);
/* We are done with the tty pointer now. */
tty_port_tty_set(&hp->port, NULL);
*/
tty_wait_until_sent_from_close(tty, HVC_CLOSE_WAIT);
} else {
- if (hp->port.count < 0)
+ if (atomic_read(&hp->port.count) < 0)
printk(KERN_ERR "hvc_close %X: oops, count is %d\n",
- hp->vtermno, hp->port.count);
+ hp->vtermno, atomic_read(&hp->port.count));
spin_unlock_irqrestore(&hp->port.lock, flags);
}
}
* open->hangup case this can be called after the final close so prevent
* that from happening for now.
*/
- if (hp->port.count <= 0) {
+ if (atomic_read(&hp->port.count) <= 0) {
spin_unlock_irqrestore(&hp->port.lock, flags);
return;
}
- hp->port.count = 0;
+ atomic_set(&hp->port.count, 0);
spin_unlock_irqrestore(&hp->port.lock, flags);
tty_port_tty_set(&hp->port, NULL);
return -EPIPE;
/* FIXME what's this (unprotected) check for? */
- if (hp->port.count <= 0)
+ if (atomic_read(&hp->port.count) <= 0)
return -EIO;
spin_lock_irqsave(&hp->lock, flags);
#include <asm/hvcserver.h>
#include <asm/uaccess.h>
#include <asm/vio.h>
+#include <asm/local.h>
/*
* 1.3.0 -> 1.3.1 In hvcs_open memset(..,0x00,..) instead of memset(..,0x3F,00).
spin_lock_irqsave(&hvcsd->lock, flags);
- if (hvcsd->port.count > 0) {
+ if (atomic_read(&hvcsd->port.count) > 0) {
spin_unlock_irqrestore(&hvcsd->lock, flags);
printk(KERN_INFO "HVCS: vterm state unchanged. "
"The hvcs device node is still in use.\n");
}
}
- hvcsd->port.count = 0;
+ atomic_set(&hvcsd->port.count, 0);
hvcsd->port.tty = tty;
tty->driver_data = hvcsd;
unsigned long flags;
spin_lock_irqsave(&hvcsd->lock, flags);
- hvcsd->port.count++;
+ atomic_inc(&hvcsd->port.count);
hvcsd->todo_mask |= HVCS_SCHED_READ;
spin_unlock_irqrestore(&hvcsd->lock, flags);
hvcsd = tty->driver_data;
spin_lock_irqsave(&hvcsd->lock, flags);
- if (--hvcsd->port.count == 0) {
+ if (atomic_dec_and_test(&hvcsd->port.count)) {
vio_disable_interrupts(hvcsd->vdev);
free_irq(irq, hvcsd);
return;
- } else if (hvcsd->port.count < 0) {
+ } else if (atomic_read(&hvcsd->port.count) < 0) {
printk(KERN_ERR "HVCS: vty-server@%X open_count: %d"
" is missmanaged.\n",
- hvcsd->vdev->unit_address, hvcsd->port.count);
+ hvcsd->vdev->unit_address, atomic_read(&hvcsd->port.count));
}
spin_unlock_irqrestore(&hvcsd->lock, flags);
spin_lock_irqsave(&hvcsd->lock, flags);
/* Preserve this so that we know how many kref refs to put */
- temp_open_count = hvcsd->port.count;
+ temp_open_count = atomic_read(&hvcsd->port.count);
/*
* Don't kref put inside the spinlock because the destruction
tty->driver_data = NULL;
hvcsd->port.tty = NULL;
- hvcsd->port.count = 0;
+ atomic_set(&hvcsd->port.count, 0);
/* This will drop any buffered data on the floor which is OK in a hangup
* scenario. */
* the middle of a write operation? This is a crummy place to do this
* but we want to keep it all in the spinlock.
*/
- if (hvcsd->port.count <= 0) {
+ if (atomic_read(&hvcsd->port.count) <= 0) {
spin_unlock_irqrestore(&hvcsd->lock, flags);
return -ENODEV;
}
{
struct hvcs_struct *hvcsd = tty->driver_data;
- if (!hvcsd || hvcsd->port.count <= 0)
+ if (!hvcsd || atomic_read(&hvcsd->port.count) <= 0)
return 0;
return HVCS_BUFF_LEN - hvcsd->chars_in_buffer;
int n_outbuf;
uint32_t vtermno;
uint32_t virq;
- atomic_t seqno; /* HVSI packet sequence number */
+ atomic_unchecked_t seqno; /* HVSI packet sequence number */
uint16_t mctrl;
uint8_t state; /* HVSI protocol state */
uint8_t flags;
packet.hdr.type = VS_QUERY_RESPONSE_PACKET_HEADER;
packet.hdr.len = sizeof(struct hvsi_query_response);
- packet.hdr.seqno = atomic_inc_return(&hp->seqno);
+ packet.hdr.seqno = atomic_inc_return_unchecked(&hp->seqno);
packet.verb = VSV_SEND_VERSION_NUMBER;
packet.u.version = HVSI_VERSION;
packet.query_seqno = query_seqno+1;
packet.hdr.type = VS_QUERY_PACKET_HEADER;
packet.hdr.len = sizeof(struct hvsi_query);
- packet.hdr.seqno = atomic_inc_return(&hp->seqno);
+ packet.hdr.seqno = atomic_inc_return_unchecked(&hp->seqno);
packet.verb = verb;
pr_debug("%s: sending %i bytes\n", __func__, packet.hdr.len);
int wrote;
packet.hdr.type = VS_CONTROL_PACKET_HEADER,
- packet.hdr.seqno = atomic_inc_return(&hp->seqno);
+ packet.hdr.seqno = atomic_inc_return_unchecked(&hp->seqno);
packet.hdr.len = sizeof(struct hvsi_control);
packet.verb = VSV_SET_MODEM_CTL;
packet.mask = HVSI_TSDTR;
BUG_ON(count > HVSI_MAX_OUTGOING_DATA);
packet.hdr.type = VS_DATA_PACKET_HEADER;
- packet.hdr.seqno = atomic_inc_return(&hp->seqno);
+ packet.hdr.seqno = atomic_inc_return_unchecked(&hp->seqno);
packet.hdr.len = count + sizeof(struct hvsi_header);
memcpy(&packet.data, buf, count);
struct hvsi_control packet __ALIGNED__;
packet.hdr.type = VS_CONTROL_PACKET_HEADER;
- packet.hdr.seqno = atomic_inc_return(&hp->seqno);
+ packet.hdr.seqno = atomic_inc_return_unchecked(&hp->seqno);
packet.hdr.len = 6;
packet.verb = VSV_CLOSE_PROTOCOL;
tty_port_tty_set(&hp->port, tty);
spin_lock_irqsave(&hp->lock, flags);
- hp->port.count++;
+ atomic_inc(&hp->port.count);
atomic_set(&hp->seqno, 0);
h_vio_signal(hp->vtermno, VIO_IRQ_ENABLE);
spin_unlock_irqrestore(&hp->lock, flags);
spin_lock_irqsave(&hp->lock, flags);
- if (--hp->port.count == 0) {
+ if (atomic_dec_return(&hp->port.count) == 0) {
tty_port_tty_set(&hp->port, NULL);
hp->inbuf_end = hp->inbuf; /* discard remaining partial packets */
spin_lock_irqsave(&hp->lock, flags);
}
- } else if (hp->port.count < 0)
+ } else if (atomic_read(&hp->port.count) < 0)
printk(KERN_ERR "hvsi_close %lu: oops, count is %d\n",
- hp - hvsi_ports, hp->port.count);
+ hp - hvsi_ports, atomic_read(&hp->port.count));
spin_unlock_irqrestore(&hp->lock, flags);
}
tty_port_tty_set(&hp->port, NULL);
spin_lock_irqsave(&hp->lock, flags);
- hp->port.count = 0;
+ atomic_set(&hp->port.count, 0);
hp->n_outbuf = 0;
spin_unlock_irqrestore(&hp->lock, flags);
}
static int hvsi_send_packet(struct hvsi_priv *pv, struct hvsi_header *packet)
{
- packet->seqno = cpu_to_be16(atomic_inc_return(&pv->seqno));
+ packet->seqno = cpu_to_be16(atomic_inc_return_unchecked(&pv->seqno));
/* Assumes that always succeeds, works in practice */
return pv->put_chars(pv->termno, (char *)packet, packet->len);
/* Reset state */
pv->established = 0;
- atomic_set(&pv->seqno, 0);
+ atomic_set_unchecked(&pv->seqno, 0);
pr_devel("HVSI@%x: Handshaking started\n", pv->termno);
#include <linux/tty_driver.h>
#include <linux/tty_flip.h>
#include <linux/uaccess.h>
+#include <asm/local.h>
#include "tty.h"
#include "network.h"
return -ENODEV;
mutex_lock(&tty->ipw_tty_mutex);
- if (tty->port.count == 0)
+ if (atomic_read(&tty->port.count) == 0)
tty->tx_bytes_queued = 0;
- tty->port.count++;
+ atomic_inc(&tty->port.count);
tty->port.tty = linux_tty;
linux_tty->driver_data = tty;
static void do_ipw_close(struct ipw_tty *tty)
{
- tty->port.count--;
-
- if (tty->port.count == 0) {
+ if (atomic_dec_return(&tty->port.count) == 0) {
struct tty_struct *linux_tty = tty->port.tty;
if (linux_tty != NULL) {
return;
mutex_lock(&tty->ipw_tty_mutex);
- if (tty->port.count == 0) {
+ if (atomic_read(&tty->port.count) == 0) {
mutex_unlock(&tty->ipw_tty_mutex);
return;
}
mutex_lock(&tty->ipw_tty_mutex);
- if (!tty->port.count) {
+ if (!atomic_read(&tty->port.count)) {
mutex_unlock(&tty->ipw_tty_mutex);
return;
}
return -ENODEV;
mutex_lock(&tty->ipw_tty_mutex);
- if (!tty->port.count) {
+ if (!atomic_read(&tty->port.count)) {
mutex_unlock(&tty->ipw_tty_mutex);
return -EINVAL;
}
if (!tty)
return -ENODEV;
- if (!tty->port.count)
+ if (!atomic_read(&tty->port.count))
return -EINVAL;
room = IPWIRELESS_TX_QUEUE_SIZE - tty->tx_bytes_queued;
if (!tty)
return 0;
- if (!tty->port.count)
+ if (!atomic_read(&tty->port.count))
return 0;
return tty->tx_bytes_queued;
if (!tty)
return -ENODEV;
- if (!tty->port.count)
+ if (!atomic_read(&tty->port.count))
return -EINVAL;
return get_control_lines(tty);
if (!tty)
return -ENODEV;
- if (!tty->port.count)
+ if (!atomic_read(&tty->port.count))
return -EINVAL;
return set_control_lines(tty, set, clear);
if (!tty)
return -ENODEV;
- if (!tty->port.count)
+ if (!atomic_read(&tty->port.count))
return -EINVAL;
/* FIXME: Exactly how is the tty object locked here .. */
* are gone */
mutex_lock(&ttyj->ipw_tty_mutex);
}
- while (ttyj->port.count)
+ while (atomic_read(&ttyj->port.count))
do_ipw_close(ttyj);
ipwireless_disassociate_network_ttys(network,
ttyj->channel_idx);
}
ch = &brd->ports[port % MAX_PORTS_PER_BOARD];
- ch->port.count++;
+ atomic_inc(&ch->port.count);
tty->driver_data = ch;
tty_port_tty_set(&ch->port, tty);
mutex_lock(&ch->port.mutex);
spin_lock_init(&dlci->lock);
mutex_init(&dlci->mutex);
dlci->fifo = &dlci->_fifo;
- if (kfifo_alloc(&dlci->_fifo, 4096, GFP_KERNEL) < 0) {
+ if (kfifo_alloc(&dlci->_fifo, 4096, GFP_KERNEL)) {
kfree(dlci);
return NULL;
}
struct gsm_dlci *dlci = tty->driver_data;
struct tty_port *port = &dlci->port;
- port->count++;
+ atomic_inc(&port->count);
tty_port_tty_set(port, tty);
dlci->modem_rx = 0;
int minimum_to_wake;
/* consumer-published */
- size_t read_tail;
+ size_t read_tail __intentional_overflow(-1);
size_t line_start;
/* protected by output lock */
{
*ops = tty_ldisc_N_TTY;
ops->owner = NULL;
- ops->refcount = ops->flags = 0;
+ atomic_set(&ops->refcount, 0);
+ ops->flags = 0;
}
EXPORT_SYMBOL_GPL(n_tty_inherit_ops);
panic("Couldn't register Unix98 pts driver");
/* Now create the /dev/ptmx special device */
+ pax_open_kernel();
tty_default_fops(&ptmx_fops);
- ptmx_fops.open = ptmx_open;
+ *(void **)&ptmx_fops.open = ptmx_open;
+ pax_close_kernel();
cdev_init(&ptmx_cdev, &ptmx_fops);
if (cdev_add(&ptmx_cdev, MKDEV(TTYAUX_MAJOR, 2), 1) ||
tty->driver_data = info;
tty_port_tty_set(port, tty);
- if (port->count++ == 0) {
+ if (atomic_inc_return(&port->count) == 1) {
atomic_inc(&rp_num_ports_open);
#ifdef ROCKET_DEBUG_OPEN
#endif
}
#ifdef ROCKET_DEBUG_OPEN
- printk(KERN_INFO "rp_open ttyR%d, count=%d\n", info->line, info->port.count);
+ printk(KERN_INFO "rp_open ttyR%d, count=%d\n", info->line, atomic-read(&info->port.count));
#endif
/*
spin_unlock_irqrestore(&info->port.lock, flags);
return;
}
- if (info->port.count)
+ if (atomic_read(&info->port.count))
atomic_dec(&rp_num_ports_open);
clear_bit((info->aiop * 8) + info->chan, (void *) &xmit_flags[info->board]);
spin_unlock_irqrestore(&info->port.lock, flags);
} is_intr_info[MAX_IOC4_INTR_ENTS];
/* Number of entries active in the above array */
- atomic_t is_num_intrs;
+ atomic_unchecked_t is_num_intrs;
} is_intr_type[IOC4_NUM_INTR_TYPES];
/* is_ir_lock must be held while
BUG_ON(!((type == IOC4_SIO_INTR_TYPE)
|| (type == IOC4_OTHER_INTR_TYPE)));
- i = atomic_inc_return(&soft-> is_intr_type[type].is_num_intrs) - 1;
+ i = atomic_inc_return_unchecked(&soft-> is_intr_type[type].is_num_intrs) - 1;
BUG_ON(!(i < MAX_IOC4_INTR_ENTS || (printk("i %d\n", i), 0)));
/* Save off the lower level interrupt handler */
soft = arg;
for (intr_type = 0; intr_type < IOC4_NUM_INTR_TYPES; intr_type++) {
- num_intrs = (int)atomic_read(
+ num_intrs = (int)atomic_read_unchecked(
&soft->is_intr_type[intr_type].is_num_intrs);
this_mir = this_ir = pending_intrs(soft, intr_type);
* I/O utilities that messages sent to the console will automatically
* be displayed on the dbg_io.
*/
- dbg_io_ops->is_console = true;
+ pax_open_kernel();
+ *(int *)&dbg_io_ops->is_console = true;
+ pax_close_kernel();
return 0;
}
#define MAX_CONFIG_LEN 40
static struct kgdb_io kgdboc_io_ops;
+static struct kgdb_io kgdboc_io_ops_console;
-/* -1 = init not run yet, 0 = unconfigured, 1 = configured. */
+/* -1 = init not run yet, 0 = unconfigured, 1/2 = configured. */
static int configured = -1;
static char config[MAX_CONFIG_LEN];
kgdboc_unregister_kbd();
if (configured == 1)
kgdb_unregister_io_module(&kgdboc_io_ops);
+ else if (configured == 2)
+ kgdb_unregister_io_module(&kgdboc_io_ops_console);
}
static int configure_kgdboc(void)
int err;
char *cptr = config;
struct console *cons;
+ int is_console = 0;
err = kgdboc_option_setup(config);
if (err || !strlen(config) || isspace(config[0]))
goto noconfig;
err = -ENODEV;
- kgdboc_io_ops.is_console = 0;
kgdb_tty_driver = NULL;
kgdboc_use_kms = 0;
int idx;
if (cons->device && cons->device(cons, &idx) == p &&
idx == tty_line) {
- kgdboc_io_ops.is_console = 1;
+ is_console = 1;
break;
}
cons = cons->next;
kgdb_tty_line = tty_line;
do_register:
- err = kgdb_register_io_module(&kgdboc_io_ops);
+ if (is_console) {
+ err = kgdb_register_io_module(&kgdboc_io_ops_console);
+ configured = 2;
+ } else {
+ err = kgdb_register_io_module(&kgdboc_io_ops);
+ configured = 1;
+ }
if (err)
goto noconfig;
if (err)
goto nmi_con_failed;
- configured = 1;
-
return 0;
nmi_con_failed:
static int __init init_kgdboc(void)
{
/* Already configured? */
- if (configured == 1)
+ if (configured >= 1)
return 0;
return configure_kgdboc();
if (config[len - 1] == '\n')
config[len - 1] = '\0';
- if (configured == 1)
+ if (configured >= 1)
cleanup_kgdboc();
/* Go and configure with the new params. */
.post_exception = kgdboc_post_exp_handler,
};
+static struct kgdb_io kgdboc_io_ops_console = {
+ .name = "kgdboc",
+ .read_char = kgdboc_get_char,
+ .write_char = kgdboc_put_char,
+ .pre_exception = kgdboc_pre_exp_handler,
+ .post_exception = kgdboc_post_exp_handler,
+ .is_console = 1
+};
+
#ifdef CONFIG_KGDB_SERIAL_CONSOLE
/* This is only available if kgdboc is a built in for early debugging */
static int __init kgdboc_early_init(char *opt)
.cons = MSM_CONSOLE,
};
-static atomic_t msm_uart_next_id = ATOMIC_INIT(0);
+static atomic_unchecked_t msm_uart_next_id = ATOMIC_INIT(0);
static const struct of_device_id msm_uartdm_table[] = {
{ .compatible = "qcom,msm-uartdm-v1.1", .data = (void *)UARTDM_1P1 },
line = pdev->id;
if (line < 0)
- line = atomic_inc_return(&msm_uart_next_id) - 1;
+ line = atomic_inc_return_unchecked(&msm_uart_next_id) - 1;
if (unlikely(line < 0 || line >= UART_NR))
return -ENXIO;
}
}
+static int s3c64xx_serial_startup(struct uart_port *port);
static int s3c24xx_serial_startup(struct uart_port *port)
{
struct s3c24xx_uart_port *ourport = to_ourport(port);
int ret;
+ /* Startup sequence is different for s3c64xx and higher SoC's */
+ if (s3c24xx_serial_has_interrupt_mask(port))
+ return s3c64xx_serial_startup(port);
+
dbg("s3c24xx_serial_startup: port=%p (%08llx,%p)\n",
port, (unsigned long long)port->mapbase, port->membase);
/* setup info for port */
port->dev = &platdev->dev;
- /* Startup sequence is different for s3c64xx and higher SoC's */
- if (s3c24xx_serial_has_interrupt_mask(port))
- s3c24xx_serial_ops.startup = s3c64xx_serial_startup;
-
port->uartclk = 1;
if (cfg->uart_flags & UPF_CONS_FLOW) {
state = drv->state + tty->index;
port = &state->port;
spin_lock_irq(&port->lock);
- --port->count;
+ atomic_dec(&port->count);
spin_unlock_irq(&port->lock);
return;
}
pr_debug("uart_close(%d) called\n", uport ? uport->line : -1);
- if (!port->count || tty_port_close_start(port, tty, filp) == 0)
+ if (!atomic_read(&port->count) || tty_port_close_start(port, tty, filp) == 0)
return;
/*
uart_flush_buffer(tty);
uart_shutdown(tty, state);
spin_lock_irqsave(&port->lock, flags);
- port->count = 0;
+ atomic_set(&port->count, 0);
clear_bit(ASYNCB_NORMAL_ACTIVE, &port->flags);
spin_unlock_irqrestore(&port->lock, flags);
tty_port_tty_set(port, NULL);
pr_debug("uart_open(%d) called\n", line);
spin_lock_irq(&port->lock);
- ++port->count;
+ atomic_inc(&port->count);
spin_unlock_irq(&port->lock);
/*
if (debug_level >= DEBUG_LEVEL_INFO)
printk("%s(%d):mgsl_close(%s) entry, count=%d\n",
- __FILE__,__LINE__, info->device_name, info->port.count);
+ __FILE__,__LINE__, info->device_name, atomic_read(&info->port.count));
if (tty_port_close_start(&info->port, tty, filp) == 0)
goto cleanup;
cleanup:
if (debug_level >= DEBUG_LEVEL_INFO)
printk("%s(%d):mgsl_close(%s) exit, count=%d\n", __FILE__,__LINE__,
- tty->driver->name, info->port.count);
+ tty->driver->name, atomic_read(&info->port.count));
} /* end of mgsl_close() */
mgsl_flush_buffer(tty);
shutdown(info);
-
- info->port.count = 0;
+
+ atomic_set(&info->port.count, 0);
info->port.flags &= ~ASYNC_NORMAL_ACTIVE;
info->port.tty = NULL;
if (debug_level >= DEBUG_LEVEL_INFO)
printk("%s(%d):block_til_ready before block on %s count=%d\n",
- __FILE__,__LINE__, tty->driver->name, port->count );
+ __FILE__,__LINE__, tty->driver->name, atomic_read(&port->count));
spin_lock_irqsave(&info->irq_spinlock, flags);
- port->count--;
+ atomic_dec(&port->count);
spin_unlock_irqrestore(&info->irq_spinlock, flags);
port->blocked_open++;
if (debug_level >= DEBUG_LEVEL_INFO)
printk("%s(%d):block_til_ready blocking on %s count=%d\n",
- __FILE__,__LINE__, tty->driver->name, port->count );
+ __FILE__,__LINE__, tty->driver->name, atomic_read(&port->count));
tty_unlock(tty);
schedule();
/* FIXME: Racy on hangup during close wait */
if (!tty_hung_up_p(filp))
- port->count++;
+ atomic_inc(&port->count);
port->blocked_open--;
if (debug_level >= DEBUG_LEVEL_INFO)
printk("%s(%d):block_til_ready after blocking on %s count=%d\n",
- __FILE__,__LINE__, tty->driver->name, port->count );
+ __FILE__,__LINE__, tty->driver->name, atomic_read(&port->count));
if (!retval)
port->flags |= ASYNC_NORMAL_ACTIVE;
if (debug_level >= DEBUG_LEVEL_INFO)
printk("%s(%d):mgsl_open(%s), old ref count = %d\n",
- __FILE__,__LINE__,tty->driver->name, info->port.count);
+ __FILE__,__LINE__,tty->driver->name, atomic_read(&info->port.count));
/* If port is closing, signal caller to try again */
if (info->port.flags & ASYNC_CLOSING){
spin_unlock_irqrestore(&info->netlock, flags);
goto cleanup;
}
- info->port.count++;
+ atomic_inc(&info->port.count);
spin_unlock_irqrestore(&info->netlock, flags);
- if (info->port.count == 1) {
+ if (atomic_read(&info->port.count) == 1) {
/* 1st open on this device, init hardware */
retval = startup(info);
if (retval < 0)
if (retval) {
if (tty->count == 1)
info->port.tty = NULL; /* tty layer will release tty struct */
- if(info->port.count)
- info->port.count--;
+ if (atomic_read(&info->port.count))
+ atomic_dec(&info->port.count);
}
return retval;
unsigned short new_crctype;
/* return error if TTY interface open */
- if (info->port.count)
+ if (atomic_read(&info->port.count))
return -EBUSY;
switch (encoding)
/* arbitrate between network and tty opens */
spin_lock_irqsave(&info->netlock, flags);
- if (info->port.count != 0 || info->netcount != 0) {
+ if (atomic_read(&info->port.count) != 0 || info->netcount != 0) {
printk(KERN_WARNING "%s: hdlc_open returning busy\n", dev->name);
spin_unlock_irqrestore(&info->netlock, flags);
return -EBUSY;
printk("%s:hdlcdev_ioctl(%s)\n",__FILE__,dev->name);
/* return error if TTY interface open */
- if (info->port.count)
+ if (atomic_read(&info->port.count))
return -EBUSY;
if (cmd != SIOCWANDEV)
tty->driver_data = info;
info->port.tty = tty;
- DBGINFO(("%s open, old ref count = %d\n", info->device_name, info->port.count));
+ DBGINFO(("%s open, old ref count = %d\n", info->device_name, atomic_read(&info->port.count)));
/* If port is closing, signal caller to try again */
if (info->port.flags & ASYNC_CLOSING){
mutex_unlock(&info->port.mutex);
goto cleanup;
}
- info->port.count++;
+ atomic_inc(&info->port.count);
spin_unlock_irqrestore(&info->netlock, flags);
- if (info->port.count == 1) {
+ if (atomic_read(&info->port.count) == 1) {
/* 1st open on this device, init hardware */
retval = startup(info);
if (retval < 0) {
if (retval) {
if (tty->count == 1)
info->port.tty = NULL; /* tty layer will release tty struct */
- if(info->port.count)
- info->port.count--;
+ if(atomic_read(&info->port.count))
+ atomic_dec(&info->port.count);
}
DBGINFO(("%s open rc=%d\n", info->device_name, retval));
if (sanity_check(info, tty->name, "close"))
return;
- DBGINFO(("%s close entry, count=%d\n", info->device_name, info->port.count));
+ DBGINFO(("%s close entry, count=%d\n", info->device_name, atomic_read(&info->port.count)));
if (tty_port_close_start(&info->port, tty, filp) == 0)
goto cleanup;
tty_port_close_end(&info->port, tty);
info->port.tty = NULL;
cleanup:
- DBGINFO(("%s close exit, count=%d\n", tty->driver->name, info->port.count));
+ DBGINFO(("%s close exit, count=%d\n", tty->driver->name, atomic_read(&info->port.count)));
}
static void hangup(struct tty_struct *tty)
shutdown(info);
spin_lock_irqsave(&info->port.lock, flags);
- info->port.count = 0;
+ atomic_set(&info->port.count, 0);
info->port.flags &= ~ASYNC_NORMAL_ACTIVE;
info->port.tty = NULL;
spin_unlock_irqrestore(&info->port.lock, flags);
unsigned short new_crctype;
/* return error if TTY interface open */
- if (info->port.count)
+ if (atomic_read(&info->port.count))
return -EBUSY;
DBGINFO(("%s hdlcdev_attach\n", info->device_name));
/* arbitrate between network and tty opens */
spin_lock_irqsave(&info->netlock, flags);
- if (info->port.count != 0 || info->netcount != 0) {
+ if (atomic_read(&info->port.count) != 0 || info->netcount != 0) {
DBGINFO(("%s hdlc_open busy\n", dev->name));
spin_unlock_irqrestore(&info->netlock, flags);
return -EBUSY;
DBGINFO(("%s hdlcdev_ioctl\n", dev->name));
/* return error if TTY interface open */
- if (info->port.count)
+ if (atomic_read(&info->port.count))
return -EBUSY;
if (cmd != SIOCWANDEV)
if (port == NULL)
continue;
spin_lock(&port->lock);
- if ((port->port.count || port->netcount) &&
+ if ((atomic_read(&port->port.count) || port->netcount) &&
port->pending_bh && !port->bh_running &&
!port->bh_requested) {
DBGISR(("%s bh queued\n", port->device_name));
add_wait_queue(&port->open_wait, &wait);
spin_lock_irqsave(&info->lock, flags);
- port->count--;
+ atomic_dec(&port->count);
spin_unlock_irqrestore(&info->lock, flags);
port->blocked_open++;
remove_wait_queue(&port->open_wait, &wait);
if (!tty_hung_up_p(filp))
- port->count++;
+ atomic_inc(&port->count);
port->blocked_open--;
if (!retval)
if (debug_level >= DEBUG_LEVEL_INFO)
printk("%s(%d):%s open(), old ref count = %d\n",
- __FILE__,__LINE__,tty->driver->name, info->port.count);
+ __FILE__,__LINE__,tty->driver->name, atomic_read(&info->port.count));
/* If port is closing, signal caller to try again */
if (info->port.flags & ASYNC_CLOSING){
spin_unlock_irqrestore(&info->netlock, flags);
goto cleanup;
}
- info->port.count++;
+ atomic_inc(&info->port.count);
spin_unlock_irqrestore(&info->netlock, flags);
- if (info->port.count == 1) {
+ if (atomic_read(&info->port.count) == 1) {
/* 1st open on this device, init hardware */
retval = startup(info);
if (retval < 0)
if (retval) {
if (tty->count == 1)
info->port.tty = NULL; /* tty layer will release tty struct */
- if(info->port.count)
- info->port.count--;
+ if(atomic_read(&info->port.count))
+ atomic_dec(&info->port.count);
}
return retval;
if (debug_level >= DEBUG_LEVEL_INFO)
printk("%s(%d):%s close() entry, count=%d\n",
- __FILE__,__LINE__, info->device_name, info->port.count);
+ __FILE__,__LINE__, info->device_name, atomic_read(&info->port.count));
if (tty_port_close_start(&info->port, tty, filp) == 0)
goto cleanup;
cleanup:
if (debug_level >= DEBUG_LEVEL_INFO)
printk("%s(%d):%s close() exit, count=%d\n", __FILE__,__LINE__,
- tty->driver->name, info->port.count);
+ tty->driver->name, atomic_read(&info->port.count));
}
/* Called by tty_hangup() when a hangup is signaled.
shutdown(info);
spin_lock_irqsave(&info->port.lock, flags);
- info->port.count = 0;
+ atomic_set(&info->port.count, 0);
info->port.flags &= ~ASYNC_NORMAL_ACTIVE;
info->port.tty = NULL;
spin_unlock_irqrestore(&info->port.lock, flags);
unsigned short new_crctype;
/* return error if TTY interface open */
- if (info->port.count)
+ if (atomic_read(&info->port.count))
return -EBUSY;
switch (encoding)
/* arbitrate between network and tty opens */
spin_lock_irqsave(&info->netlock, flags);
- if (info->port.count != 0 || info->netcount != 0) {
+ if (atomic_read(&info->port.count) != 0 || info->netcount != 0) {
printk(KERN_WARNING "%s: hdlc_open returning busy\n", dev->name);
spin_unlock_irqrestore(&info->netlock, flags);
return -EBUSY;
printk("%s:hdlcdev_ioctl(%s)\n",__FILE__,dev->name);
/* return error if TTY interface open */
- if (info->port.count)
+ if (atomic_read(&info->port.count))
return -EBUSY;
if (cmd != SIOCWANDEV)
* do not request bottom half processing if the
* device is not open in a normal mode.
*/
- if ( port && (port->port.count || port->netcount) &&
+ if ( port && (atomic_read(&port->port.count) || port->netcount) &&
port->pending_bh && !port->bh_running &&
!port->bh_requested ) {
if ( debug_level >= DEBUG_LEVEL_ISR )
if (debug_level >= DEBUG_LEVEL_INFO)
printk("%s(%d):%s block_til_ready() before block, count=%d\n",
- __FILE__,__LINE__, tty->driver->name, port->count );
+ __FILE__,__LINE__, tty->driver->name, atomic_read(&port->count));
spin_lock_irqsave(&info->lock, flags);
- port->count--;
+ atomic_dec(&port->count);
spin_unlock_irqrestore(&info->lock, flags);
port->blocked_open++;
if (debug_level >= DEBUG_LEVEL_INFO)
printk("%s(%d):%s block_til_ready() count=%d\n",
- __FILE__,__LINE__, tty->driver->name, port->count );
+ __FILE__,__LINE__, tty->driver->name, atomic_read(&port->count));
tty_unlock(tty);
schedule();
set_current_state(TASK_RUNNING);
remove_wait_queue(&port->open_wait, &wait);
if (!tty_hung_up_p(filp))
- port->count++;
+ atomic_inc(&port->count);
port->blocked_open--;
if (debug_level >= DEBUG_LEVEL_INFO)
printk("%s(%d):%s block_til_ready() after, count=%d\n",
- __FILE__,__LINE__, tty->driver->name, port->count );
+ __FILE__,__LINE__, tty->driver->name, atomic_read(&port->count));
if (!retval)
port->flags |= ASYNC_NORMAL_ACTIVE;
static ssize_t write_sysrq_trigger(struct file *file, const char __user *buf,
size_t count, loff_t *ppos)
{
- if (count) {
+ if (count && capable(CAP_SYS_ADMIN)) {
char c;
if (get_user(c, buf))
void tty_default_fops(struct file_operations *fops)
{
- *fops = tty_fops;
+ memcpy((void *)fops, &tty_fops, sizeof(tty_fops));
}
/*
raw_spin_lock_irqsave(&tty_ldiscs_lock, flags);
tty_ldiscs[disc] = new_ldisc;
new_ldisc->num = disc;
- new_ldisc->refcount = 0;
+ atomic_set(&new_ldisc->refcount, 0);
raw_spin_unlock_irqrestore(&tty_ldiscs_lock, flags);
return ret;
return -EINVAL;
raw_spin_lock_irqsave(&tty_ldiscs_lock, flags);
- if (tty_ldiscs[disc]->refcount)
+ if (atomic_read(&tty_ldiscs[disc]->refcount))
ret = -EBUSY;
else
tty_ldiscs[disc] = NULL;
if (ldops) {
ret = ERR_PTR(-EAGAIN);
if (try_module_get(ldops->owner)) {
- ldops->refcount++;
+ atomic_inc(&ldops->refcount);
ret = ldops;
}
}
unsigned long flags;
raw_spin_lock_irqsave(&tty_ldiscs_lock, flags);
- ldops->refcount--;
+ atomic_dec(&ldops->refcount);
module_put(ldops->owner);
raw_spin_unlock_irqrestore(&tty_ldiscs_lock, flags);
}
unsigned long flags;
spin_lock_irqsave(&port->lock, flags);
- port->count = 0;
+ atomic_set(&port->count, 0);
port->flags &= ~ASYNC_NORMAL_ACTIVE;
tty = port->tty;
if (tty)
/* The port lock protects the port counts */
spin_lock_irqsave(&port->lock, flags);
- port->count--;
+ atomic_dec(&port->count);
port->blocked_open++;
spin_unlock_irqrestore(&port->lock, flags);
we must not mess that up further */
spin_lock_irqsave(&port->lock, flags);
if (!tty_hung_up_p(filp))
- port->count++;
+ atomic_inc(&port->count);
port->blocked_open--;
if (retval == 0)
port->flags |= ASYNC_NORMAL_ACTIVE;
return 0;
spin_lock_irqsave(&port->lock, flags);
- if (tty->count == 1 && port->count != 1) {
+ if (tty->count == 1 && atomic_read(&port->count) != 1) {
printk(KERN_WARNING
"tty_port_close_start: tty->count = 1 port count = %d.\n",
- port->count);
- port->count = 1;
+ atomic_read(&port->count));
+ atomic_set(&port->count, 1);
}
- if (--port->count < 0) {
+ if (atomic_dec_return(&port->count) < 0) {
printk(KERN_WARNING "tty_port_close_start: count = %d\n",
- port->count);
- port->count = 0;
+ atomic_read(&port->count));
+ atomic_set(&port->count, 0);
}
- if (port->count) {
+ if (atomic_read(&port->count)) {
spin_unlock_irqrestore(&port->lock, flags);
return 0;
}
struct file *filp)
{
spin_lock_irq(&port->lock);
- ++port->count;
+ atomic_inc(&port->count);
spin_unlock_irq(&port->lock);
tty_port_tty_set(port, tty);
kbd->kbdmode == VC_OFF) &&
value != KVAL(K_SAK))
return; /* SAK is allowed even in raw mode */
+
+#if defined(CONFIG_GRKERNSEC_PROC) || defined(CONFIG_GRKERNSEC_PROC_MEMMAP)
+ {
+ void *func = fn_handler[value];
+ if (func == fn_show_state || func == fn_show_ptregs ||
+ func == fn_show_mem)
+ return;
+ }
+#endif
+
fn_handler[value](vc);
}
if (copy_from_user(&tmp, user_kbe, sizeof(struct kbentry)))
return -EFAULT;
- if (!capable(CAP_SYS_TTY_CONFIG))
- perm = 0;
-
switch (cmd) {
case KDGKBENT:
/* Ensure another thread doesn't free it under us */
spin_unlock_irqrestore(&kbd_event_lock, flags);
return put_user(val, &user_kbe->kb_value);
case KDSKBENT:
+ if (!capable(CAP_SYS_TTY_CONFIG))
+ perm = 0;
+
if (!perm)
return -EPERM;
if (!i && v == K_NOSUCHMAP) {
int i, j, k;
int ret;
- if (!capable(CAP_SYS_TTY_CONFIG))
- perm = 0;
-
kbs = kmalloc(sizeof(*kbs), GFP_KERNEL);
if (!kbs) {
ret = -ENOMEM;
kfree(kbs);
return ((p && *p) ? -EOVERFLOW : 0);
case KDSKBSENT:
+ if (!capable(CAP_SYS_TTY_CONFIG))
+ perm = 0;
+
if (!perm) {
ret = -EPERM;
goto reterr;
#include <linux/kobject.h>
#include <linux/cdev.h>
#include <linux/uio_driver.h>
+#include <asm/local.h>
#define UIO_MAX_DEVICES (1U << MINORBITS)
struct device_attribute *attr, char *buf)
{
struct uio_device *idev = dev_get_drvdata(dev);
- return sprintf(buf, "%u\n", (unsigned int)atomic_read(&idev->event));
+ return sprintf(buf, "%u\n", (unsigned int)atomic_read_unchecked(&idev->event));
}
static DEVICE_ATTR_RO(event);
{
struct uio_device *idev = info->uio_dev;
- atomic_inc(&idev->event);
+ atomic_inc_unchecked(&idev->event);
wake_up_interruptible(&idev->wait);
kill_fasync(&idev->async_queue, SIGIO, POLL_IN);
}
}
listener->dev = idev;
- listener->event_count = atomic_read(&idev->event);
+ listener->event_count = atomic_read_unchecked(&idev->event);
filep->private_data = listener;
if (idev->info->open) {
return -EIO;
poll_wait(filep, &idev->wait, wait);
- if (listener->event_count != atomic_read(&idev->event))
+ if (listener->event_count != atomic_read_unchecked(&idev->event))
return POLLIN | POLLRDNORM;
return 0;
}
do {
set_current_state(TASK_INTERRUPTIBLE);
- event_count = atomic_read(&idev->event);
+ event_count = atomic_read_unchecked(&idev->event);
if (event_count != listener->event_count) {
if (copy_to_user(buf, &event_count, count))
retval = -EFAULT;
static int uio_find_mem_index(struct vm_area_struct *vma)
{
struct uio_device *idev = vma->vm_private_data;
+ unsigned long size;
if (vma->vm_pgoff < MAX_UIO_MAPS) {
- if (idev->info->mem[vma->vm_pgoff].size == 0)
+ size = idev->info->mem[vma->vm_pgoff].size;
+ if (size == 0)
+ return -1;
+ if (vma->vm_end - vma->vm_start > size)
return -1;
return (int)vma->vm_pgoff;
}
idev->owner = owner;
idev->info = info;
init_waitqueue_head(&idev->wait);
- atomic_set(&idev->event, 0);
+ atomic_set_unchecked(&idev->event, 0);
ret = uio_get_minor(idev);
if (ret)
ret = sscanf(buf + pos, "%x=%x%n", &index, &value, &tmp);
if (ret < 2)
return -EINVAL;
- if (index < 0 || index > 0x7f)
+ if (index > 0x7f)
return -EINVAL;
pos += tmp;
if (printk_ratelimit())
atm_warn(instance, "%s: OAM not supported (vpi %d, vci %d)!\n",
__func__, vpi, vci);
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
return;
}
if (length > ATM_MAX_AAL5_PDU) {
atm_rldbg(instance, "%s: bogus length %u (vcc: 0x%p)!\n",
__func__, length, vcc);
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
goto out;
}
if (sarb->len < pdu_length) {
atm_rldbg(instance, "%s: bogus pdu_length %u (sarb->len: %u, vcc: 0x%p)!\n",
__func__, pdu_length, sarb->len, vcc);
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
goto out;
}
if (crc32_be(~0, skb_tail_pointer(sarb) - pdu_length, pdu_length) != 0xc704dd7b) {
atm_rldbg(instance, "%s: packet failed crc check (vcc: 0x%p)!\n",
__func__, vcc);
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
goto out;
}
if (printk_ratelimit())
atm_err(instance, "%s: no memory for skb (length: %u)!\n",
__func__, length);
- atomic_inc(&vcc->stats->rx_drop);
+ atomic_inc_unchecked(&vcc->stats->rx_drop);
goto out;
}
vcc->push(vcc, skb);
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
out:
skb_trim(sarb, 0);
}
struct atm_vcc *vcc = UDSL_SKB(skb)->atm.vcc;
usbatm_pop(vcc, skb);
- atomic_inc(&vcc->stats->tx);
+ atomic_inc_unchecked(&vcc->stats->tx);
skb = skb_dequeue(&instance->sndqueue);
}
if (!left--)
return sprintf(page,
"AAL5: tx %d ( %d err ), rx %d ( %d err, %d drop )\n",
- atomic_read(&atm_dev->stats.aal5.tx),
- atomic_read(&atm_dev->stats.aal5.tx_err),
- atomic_read(&atm_dev->stats.aal5.rx),
- atomic_read(&atm_dev->stats.aal5.rx_err),
- atomic_read(&atm_dev->stats.aal5.rx_drop));
+ atomic_read_unchecked(&atm_dev->stats.aal5.tx),
+ atomic_read_unchecked(&atm_dev->stats.aal5.tx_err),
+ atomic_read_unchecked(&atm_dev->stats.aal5.rx),
+ atomic_read_unchecked(&atm_dev->stats.aal5.rx_err),
+ atomic_read_unchecked(&atm_dev->stats.aal5.rx_drop));
if (!left--) {
if (instance->disconnected)
* time it gets called.
*/
static struct device_connect_event {
- atomic_t count;
+ atomic_unchecked_t count;
wait_queue_head_t wait;
} device_event = {
.count = ATOMIC_INIT(1),
void usbfs_conn_disc_event(void)
{
- atomic_add(2, &device_event.count);
+ atomic_add_unchecked(2, &device_event.count);
wake_up(&device_event.wait);
}
poll_wait(file, &device_event.wait, wait);
- event_count = atomic_read(&device_event.count);
+ event_count = atomic_read_unchecked(&device_event.count);
if (file->f_version != event_count) {
file->f_version = event_count;
return POLLIN | POLLRDNORM;
struct usb_dev_state *ps = file->private_data;
struct usb_device *dev = ps->dev;
ssize_t ret = 0;
- unsigned len;
+ size_t len;
loff_t pos;
int i;
for (i = 0; nbytes && i < dev->descriptor.bNumConfigurations; i++) {
struct usb_config_descriptor *config =
(struct usb_config_descriptor *)dev->rawdescriptors[i];
- unsigned int length = le16_to_cpu(config->wTotalLength);
+ size_t length = le16_to_cpu(config->wTotalLength);
if (*ppos < pos + length) {
/* The descriptor may claim to be longer than it
* really is. Here is the actual allocated length. */
- unsigned alloclen =
+ size_t alloclen =
le16_to_cpu(dev->config[i].desc.wTotalLength);
- len = length - (*ppos - pos);
+ len = length + pos - *ppos;
if (len > nbytes)
len = nbytes;
/* Simply don't write (skip over) unallocated parts */
if (alloclen > (*ppos - pos)) {
- alloclen -= (*ppos - pos);
+ alloclen = alloclen + pos - *ppos;
if (copy_to_user(buf,
dev->rawdescriptors[i] + (*ppos - pos),
min(len, alloclen))) {
*/
usb_get_urb(urb);
atomic_inc(&urb->use_count);
- atomic_inc(&urb->dev->urbnum);
+ atomic_inc_unchecked(&urb->dev->urbnum);
usbmon_urb_submit(&hcd->self, urb);
/* NOTE requirements on root-hub callers (usbfs and the hub
urb->hcpriv = NULL;
INIT_LIST_HEAD(&urb->urb_list);
atomic_dec(&urb->use_count);
- atomic_dec(&urb->dev->urbnum);
+ atomic_dec_unchecked(&urb->dev->urbnum);
if (atomic_read(&urb->reject))
wake_up(&usb_kill_urb_queue);
usb_put_urb(urb);
#include <linux/mutex.h>
#include <linux/random.h>
#include <linux/pm_qos.h>
+#include <linux/grsecurity.h>
#include <asm/uaccess.h>
#include <asm/byteorder.h>
goto done;
return;
}
+
+ if (gr_handle_new_usb())
+ goto done;
+
if (hub_is_superspeed(hub->hdev))
unit_load = 150;
else
* Return: If successful, the number of bytes transferred. Otherwise, a negative
* error number.
*/
-int usb_control_msg(struct usb_device *dev, unsigned int pipe, __u8 request,
+int __intentional_overflow(-1) usb_control_msg(struct usb_device *dev, unsigned int pipe, __u8 request,
__u8 requesttype, __u16 value, __u16 index, void *data,
__u16 size, int timeout)
{
* If successful, 0. Otherwise a negative error number. The number of actual
* bytes transferred will be stored in the @actual_length parameter.
*/
-int usb_interrupt_msg(struct usb_device *usb_dev, unsigned int pipe,
+int __intentional_overflow(-1) usb_interrupt_msg(struct usb_device *usb_dev, unsigned int pipe,
void *data, int len, int *actual_length, int timeout)
{
return usb_bulk_msg(usb_dev, pipe, data, len, actual_length, timeout);
* bytes transferred will be stored in the @actual_length parameter.
*
*/
-int usb_bulk_msg(struct usb_device *usb_dev, unsigned int pipe,
+int __intentional_overflow(-1) usb_bulk_msg(struct usb_device *usb_dev, unsigned int pipe,
void *data, int len, int *actual_length, int timeout)
{
struct urb *urb;
struct usb_device *udev;
udev = to_usb_device(dev);
- return sprintf(buf, "%d\n", atomic_read(&udev->urbnum));
+ return sprintf(buf, "%d\n", atomic_read_unchecked(&udev->urbnum));
}
static DEVICE_ATTR_RO(urbnum);
set_dev_node(&dev->dev, dev_to_node(bus->controller));
dev->state = USB_STATE_ATTACHED;
dev->lpm_disable_count = 1;
- atomic_set(&dev->urbnum, 0);
+ atomic_set_unchecked(&dev->urbnum, 0);
INIT_LIST_HEAD(&dev->ep0.urb_list);
dev->ep0.desc.bLength = USB_DT_ENDPOINT_SIZE;
#ifdef CONFIG_KGDB
static struct kgdb_io kgdbdbgp_io_ops;
-#define dbgp_kgdb_mode (dbg_io_ops == &kgdbdbgp_io_ops)
+static struct kgdb_io kgdbdbgp_io_ops_console;
+#define dbgp_kgdb_mode (dbg_io_ops == &kgdbdbgp_io_ops || dbg_io_ops == &kgdbdbgp_io_ops_console)
#else
#define dbgp_kgdb_mode (0)
#endif
.write_char = kgdbdbgp_write_char,
};
+static struct kgdb_io kgdbdbgp_io_ops_console = {
+ .name = "kgdbdbgp",
+ .read_char = kgdbdbgp_read_char,
+ .write_char = kgdbdbgp_write_char,
+ .is_console = 1
+};
+
static int kgdbdbgp_wait_time;
static int __init kgdbdbgp_parse_config(char *str)
ptr++;
kgdbdbgp_wait_time = simple_strtoul(ptr, &ptr, 10);
}
- kgdb_register_io_module(&kgdbdbgp_io_ops);
- kgdbdbgp_io_ops.is_console = early_dbgp_console.index != -1;
+ if (early_dbgp_console.index != -1)
+ kgdb_register_io_module(&kgdbdbgp_io_ops_console);
+ else
+ kgdb_register_io_module(&kgdbdbgp_io_ops);
return 0;
}
#include <linux/module.h>
#include <linux/device.h>
#include <linux/atomic.h>
+#include <linux/module.h>
#include "u_uac1.h"
spin_lock_irq(&port->port_lock);
/* already open? Great. */
- if (port->port.count) {
+ if (atomic_read(&port->port.count)) {
status = 0;
- port->port.count++;
+ atomic_inc(&port->port.count);
/* currently opening/closing? wait ... */
} else if (port->openclose) {
tty->driver_data = port;
port->port.tty = tty;
- port->port.count = 1;
+ atomic_set(&port->port.count, 1);
port->openclose = false;
/* if connected, start the I/O stream */
spin_lock_irq(&port->port_lock);
- if (port->port.count != 1) {
- if (port->port.count == 0)
+ if (atomic_read(&port->port.count) != 1) {
+ if (atomic_read(&port->port.count) == 0)
WARN_ON(1);
else
- --port->port.count;
+ atomic_dec(&port->port.count);
goto exit;
}
* and sleep if necessary
*/
port->openclose = true;
- port->port.count = 0;
+ atomic_set(&port->port.count, 0);
gser = port->port_usb;
if (gser && gser->disconnect)
int cond;
spin_lock_irq(&port->port_lock);
- cond = (port->port.count == 0) && !port->openclose;
+ cond = (atomic_read(&port->port.count) == 0) && !port->openclose;
spin_unlock_irq(&port->port_lock);
return cond;
}
/* if it's already open, start I/O ... and notify the serial
* protocol about open/close status (connect/disconnect).
*/
- if (port->port.count) {
+ if (atomic_read(&port->port.count)) {
pr_debug("gserial_connect: start ttyGS%d\n", port->port_num);
gs_start_io(port);
if (gser->connect)
port->port_usb = NULL;
gser->ioport = NULL;
- if (port->port.count > 0 || port->openclose) {
+ if (atomic_read(&port->port.count) > 0 || port->openclose) {
wake_up_interruptible(&port->drain_wait);
if (port->port.tty)
tty_hangup(port->port.tty);
/* finally, free any unused/unusable I/O buffers */
spin_lock_irqsave(&port->port_lock, flags);
- if (port->port.count == 0 && !port->openclose)
+ if (atomic_read(&port->port.count) == 0 && !port->openclose)
gs_buf_free(&port->port_write_buf);
gs_free_requests(gser->out, &port->read_pool, NULL);
gs_free_requests(gser->out, &port->read_queue, NULL);
#include <linux/ctype.h>
#include <linux/random.h>
#include <linux/syscalls.h>
+#include <linux/module.h>
#include "u_uac1.h"
urb->transfer_flags = URB_DIR_IN;
usb_get_urb(urb);
atomic_inc(&urb->use_count);
- atomic_inc(&urb->dev->urbnum);
+ atomic_inc_unchecked(&urb->dev->urbnum);
urb->setup_dma = dma_map_single(
hcd->self.controller,
urb->setup_packet,
urb->status = -EINPROGRESS;
usb_get_urb(urb);
atomic_inc(&urb->use_count);
- atomic_inc(&urb->dev->urbnum);
+ atomic_inc_unchecked(&urb->dev->urbnum);
retval = submit_single_step_set_feature(hcd, urb, 0);
if (!retval && !wait_for_completion_timeout(&done,
msecs_to_jiffies(2000))) {
struct hwahc *hwahc = container_of(wusbhc, struct hwahc, wusbhc);
struct wahc *wa = &hwahc->wa;
struct device *dev = &wa->usb_iface->dev;
- u8 mas_le[UWB_NUM_MAS/8];
+ u8 *mas_le = kmalloc(UWB_NUM_MAS/8, GFP_KERNEL);
+
+ if (mas_le == NULL)
+ return -ENOMEM;
/* Set the stream index */
result = usb_control_msg(wa->usb_dev, usb_sndctrlpipe(wa->usb_dev, 0),
WUSB_REQ_SET_WUSB_MAS,
USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_INTERFACE,
0, wa->usb_iface->cur_altsetting->desc.bInterfaceNumber,
- mas_le, 32, USB_CTRL_SET_TIMEOUT);
+ mas_le, UWB_NUM_MAS/8, USB_CTRL_SET_TIMEOUT);
if (result < 0)
dev_err(dev, "Cannot set WUSB MAS allocation: %d\n", result);
out:
+ kfree(mas_le);
+
return result;
}
struct mutex sysfslock; /* concurrent read and write */
};
-static atomic_t count_displays = ATOMIC_INIT(0);
+static atomic_unchecked_t count_displays = ATOMIC_INIT(0);
static struct workqueue_struct *wq;
static void appledisplay_complete(struct urb *urb)
/* Register backlight device */
snprintf(bl_name, sizeof(bl_name), "appledisplay%d",
- atomic_inc_return(&count_displays) - 1);
+ atomic_inc_return_unchecked(&count_displays) - 1);
memset(&props, 0, sizeof(struct backlight_properties));
props.type = BACKLIGHT_RAW;
props.max_brightness = 0xff;
info->port = port;
- ++port->port.count;
+ atomic_inc(&port->port.count);
if (!test_bit(ASYNCB_INITIALIZED, &port->port.flags)) {
if (serial->type->set_termios) {
/*
}
/* Now that any required fake tty operations are completed restore
* the tty port count */
- --port->port.count;
+ atomic_dec(&port->port.count);
/* The console is special in terms of closing the device so
* indicate this port is now acting as a system console. */
port->port.console = 1;
put_tty:
tty_kref_put(tty);
reset_open_count:
- port->port.count = 0;
+ atomic_set(&port->port.count, 0);
usb_autopm_put_interface(serial->interface);
error_get_interface:
usb_serial_put(serial);
static void usb_console_write(struct console *co,
const char *buf, unsigned count)
{
- static struct usbcons_info *info = &usbcons_info;
+ struct usbcons_info *info = &usbcons_info;
struct usb_serial_port *port = info->port;
struct usb_serial *serial;
int retval = -ENODEV;
__u8 useProtocol;
__u8 useTransport;
int (*initFunction)(struct us_data *);
-};
+} __do_const;
/* Dynamic bitflag definitions (us->dflags): used in set_bit() etc. */
unsigned resuming:1;
unsigned long re_timeout;
- atomic_t seqnum;
+ atomic_unchecked_t seqnum;
/*
* NOTE:
spin_lock(&vdev->priv_lock);
- priv->seqnum = atomic_inc_return(&the_controller->seqnum);
+ priv->seqnum = atomic_inc_return_unchecked(&the_controller->seqnum);
if (priv->seqnum == 0xffff)
dev_info(&urb->dev->dev, "seqnum max\n");
return -ENOMEM;
}
- unlink->seqnum = atomic_inc_return(&the_controller->seqnum);
+ unlink->seqnum = atomic_inc_return_unchecked(&the_controller->seqnum);
if (unlink->seqnum == 0xffff)
pr_info("seqnum max\n");
vdev->rhport = rhport;
}
- atomic_set(&vhci->seqnum, 0);
+ atomic_set_unchecked(&vhci->seqnum, 0);
spin_lock_init(&vhci->lock);
hcd->power_budget = 0; /* no limit */
if (!urb) {
pr_err("cannot find a urb of seqnum %u\n", pdu->base.seqnum);
pr_info("max seqnum %d\n",
- atomic_read(&the_controller->seqnum));
+ atomic_read_unchecked(&the_controller->seqnum));
usbip_event_add(ud, VDEV_EVENT_ERROR_TCP);
return;
}
spinlock_t xfer_list_lock;
struct work_struct xfer_enqueue_work;
struct work_struct xfer_error_work;
- atomic_t xfer_id_count;
+ atomic_unchecked_t xfer_id_count;
kernel_ulong_t quirks;
};
INIT_WORK(&wa->xfer_enqueue_work, wa_urb_enqueue_run);
INIT_WORK(&wa->xfer_error_work, wa_process_errored_transfers_run);
wa->dto_in_use = 0;
- atomic_set(&wa->xfer_id_count, 1);
+ atomic_set_unchecked(&wa->xfer_id_count, 1);
/* init the buf in URBs */
for (index = 0; index < WA_MAX_BUF_IN_URBS; ++index)
usb_init_urb(&(wa->buf_in_urbs[index]));
*/
static void wa_xfer_id_init(struct wa_xfer *xfer)
{
- xfer->id = atomic_add_return(1, &xfer->wa->xfer_id_count);
+ xfer->id = atomic_add_return_unchecked(1, &xfer->wa->xfer_id_count);
}
/* Return the xfer's ID. */
return 0;
/* TODO Prevent device auto probing */
- WARN("Device %s added to live group %d!\n", dev_name(dev),
+ WARN(1, "Device %s added to live group %d!\n", dev_name(dev),
iommu_group_id(group->iommu_group));
return 0;
break;
}
/* TODO: Should check and handle checksum. */
-
- hdr.num_buffers = cpu_to_vhost16(vq, headcount);
if (likely(mergeable) &&
- memcpy_toiovecend(nvq->hdr, (void *)&hdr.num_buffers,
+ memcpy_toiovecend(nvq->hdr, (unsigned char *)&headcount,
offsetof(typeof(hdr), num_buffers),
sizeof hdr.num_buffers)) {
vq_err(vq, "Failed num_buffers write");
static inline int getu16_user(const struct vringh *vrh, u16 *val, const __virtio16 *p)
{
__virtio16 v = 0;
- int rc = get_user(v, (__force __virtio16 __user *)p);
+ int rc = get_user(v, (__force_user __virtio16 *)p);
*val = vringh16_to_cpu(vrh, v);
return rc;
}
static inline int putu16_user(const struct vringh *vrh, __virtio16 *p, u16 val)
{
__virtio16 v = cpu_to_vringh16(vrh, val);
- return put_user(v, (__force __virtio16 __user *)p);
+ return put_user(v, (__force_user __virtio16 *)p);
}
static inline int copydesc_user(void *dst, const void *src, size_t len)
{
- return copy_from_user(dst, (__force void __user *)src, len) ?
+ return copy_from_user(dst, (void __force_user *)src, len) ?
-EFAULT : 0;
}
const struct vring_used_elem *src,
unsigned int num)
{
- return copy_to_user((__force void __user *)dst, src,
+ return copy_to_user((void __force_user *)dst, src,
sizeof(*dst) * num) ? -EFAULT : 0;
}
static inline int xfer_from_user(void *src, void *dst, size_t len)
{
- return copy_from_user(dst, (__force void __user *)src, len) ?
+ return copy_from_user(dst, (void __force_user *)src, len) ?
-EFAULT : 0;
}
static inline int xfer_to_user(void *dst, void *src, size_t len)
{
- return copy_to_user((__force void __user *)dst, src, len) ?
+ return copy_to_user((void __force_user *)dst, src, len) ?
-EFAULT : 0;
}
vrh->last_used_idx = 0;
vrh->vring.num = num;
/* vring expects kernel addresses, but only used via accessors. */
- vrh->vring.desc = (__force struct vring_desc *)desc;
- vrh->vring.avail = (__force struct vring_avail *)avail;
- vrh->vring.used = (__force struct vring_used *)used;
+ vrh->vring.desc = (__force_kernel struct vring_desc *)desc;
+ vrh->vring.avail = (__force_kernel struct vring_avail *)avail;
+ vrh->vring.used = (__force_kernel struct vring_used *)used;
return 0;
}
EXPORT_SYMBOL(vringh_init_user);
static inline int putu16_kern(const struct vringh *vrh, __virtio16 *p, u16 val)
{
- ACCESS_ONCE(*p) = cpu_to_vringh16(vrh, val);
+ ACCESS_ONCE_RW(*p) = cpu_to_vringh16(vrh, val);
return 0;
}
static unsigned long kb3886bl_flags;
#define KB3886BL_SUSPENDED 0x01
-static struct dmi_system_id kb3886bl_device_table[] __initdata = {
+static const struct dmi_system_id kb3886bl_device_table[] __initconst = {
{
.ident = "Sahara Touch-iT",
.matches = {
return -ENOSPC;
err = 0;
- if ((count + p) > fbmemlength) {
+ if (count > (fbmemlength - p)) {
count = fbmemlength - p;
err = -ENOSPC;
}
};
/* Must match above enum */
-static char * const r128_family[] = {
+static const char * const r128_family[] = {
"AGP",
"PCI",
"PRO AGP",
par->accel_flags = var->accel_flags; /* hack */
if (var->accel_flags) {
- info->fbops->fb_sync = atyfb_sync;
+ pax_open_kernel();
+ *(void **)&info->fbops->fb_sync = atyfb_sync;
+ pax_close_kernel();
info->flags &= ~FBINFO_HWACCEL_DISABLED;
} else {
- info->fbops->fb_sync = NULL;
+ pax_open_kernel();
+ *(void **)&info->fbops->fb_sync = NULL;
+ pax_close_kernel();
info->flags |= FBINFO_HWACCEL_DISABLED;
}
#include "../core/fb_draw.h"
#include <asm/io.h>
+#include <asm/pgtable.h>
#ifdef __sparc__
#include <asm/fbio.h>
info->sprite.buf_align = 16; /* and 64 lines tall. */
info->sprite.flags = FB_PIXMAP_IO;
- info->fbops->fb_cursor = atyfb_cursor;
+ pax_open_kernel();
+ *(void **)&info->fbops->fb_cursor = atyfb_cursor;
+ pax_close_kernel();
return 0;
}
BUG_ON(!fbdefio);
mutex_init(&fbdefio->lock);
- info->fbops->fb_mmap = fb_deferred_io_mmap;
+ pax_open_kernel();
+ *(void **)&info->fbops->fb_mmap = fb_deferred_io_mmap;
+ pax_close_kernel();
INIT_DELAYED_WORK(&info->deferred_work, fb_deferred_io_work);
INIT_LIST_HEAD(&fbdefio->pagelist);
if (fbdefio->delay == 0) /* set a default of 1 s */
page->mapping = NULL;
}
- info->fbops->fb_mmap = NULL;
+ *(void **)&info->fbops->fb_mmap = NULL;
mutex_destroy(&fbdefio->lock);
}
EXPORT_SYMBOL_GPL(fb_deferred_io_cleanup);
__u32 data;
int err;
- err = copy_to_user(&fix32->id, &fix->id, sizeof(fix32->id));
+ err = copy_to_user(fix32->id, &fix->id, sizeof(fix32->id));
data = (__u32) (unsigned long) fix->smem_start;
err |= put_user(data, &fix32->smem_start);
static inline int synthvid_send(struct hv_device *hdev,
struct synthvid_msg *msg)
{
- static atomic64_t request_id = ATOMIC64_INIT(0);
+ static atomic64_unchecked_t request_id = ATOMIC64_INIT(0);
int ret;
msg->pipe_hdr.type = PIPE_MSG_DATA;
ret = vmbus_sendpacket(hdev->channel, msg,
msg->vid_hdr.size + sizeof(struct pipe_msg_hdr),
- atomic64_inc_return(&request_id),
+ atomic64_inc_return_unchecked(&request_id),
VM_PKT_DATA_INBAND, 0);
if (ret)
}
}
printk("ringbuffer lockup!!!\n");
+ printk("head:%u tail:%u iring.size:%u space:%u\n", head, tail, par->iring.size, space);
i810_report_error(mmio);
par->dev_flags |= LOCKUP;
info->pixmap.scan_align = 1;
#ifdef CONFIG_FB_MATROX_MYSTIQUE
struct matrox_switch matrox_mystique = {
- MGA1064_preinit, MGA1064_reset, MGA1064_init, MGA1064_restore,
+ .preinit = MGA1064_preinit,
+ .reset = MGA1064_reset,
+ .init = MGA1064_init,
+ .restore = MGA1064_restore,
};
EXPORT_SYMBOL(matrox_mystique);
#endif
#ifdef CONFIG_FB_MATROX_G
struct matrox_switch matrox_G100 = {
- MGAG100_preinit, MGAG100_reset, MGAG100_init, MGAG100_restore,
+ .preinit = MGAG100_preinit,
+ .reset = MGAG100_reset,
+ .init = MGAG100_init,
+ .restore = MGAG100_restore,
};
EXPORT_SYMBOL(matrox_G100);
#endif
}
struct matrox_switch matrox_millennium = {
- Ti3026_preinit, Ti3026_reset, Ti3026_init, Ti3026_restore
+ .preinit = Ti3026_preinit,
+ .reset = Ti3026_reset,
+ .init = Ti3026_init,
+ .restore = Ti3026_restore
};
EXPORT_SYMBOL(matrox_millennium);
#endif
struct mb862xxfb_par *par = info->par;
if (info->var.bits_per_pixel == 32) {
- info->fbops->fb_fillrect = cfb_fillrect;
- info->fbops->fb_copyarea = cfb_copyarea;
- info->fbops->fb_imageblit = cfb_imageblit;
+ pax_open_kernel();
+ *(void **)&info->fbops->fb_fillrect = cfb_fillrect;
+ *(void **)&info->fbops->fb_copyarea = cfb_copyarea;
+ *(void **)&info->fbops->fb_imageblit = cfb_imageblit;
+ pax_close_kernel();
} else {
outreg(disp, GC_L0EM, 3);
- info->fbops->fb_fillrect = mb86290fb_fillrect;
- info->fbops->fb_copyarea = mb86290fb_copyarea;
- info->fbops->fb_imageblit = mb86290fb_imageblit;
+ pax_open_kernel();
+ *(void **)&info->fbops->fb_fillrect = mb86290fb_fillrect;
+ *(void **)&info->fbops->fb_copyarea = mb86290fb_copyarea;
+ *(void **)&info->fbops->fb_imageblit = mb86290fb_imageblit;
+ pax_close_kernel();
}
outreg(draw, GDC_REG_DRAW_BASE, 0);
outreg(draw, GDC_REG_MODE_MISC, 0x8000);
info->fix.line_length = (info->var.xres_virtual *
info->var.bits_per_pixel) >> 3;
if (info->var.accel_flags) {
- info->fbops->fb_imageblit = nvidiafb_imageblit;
- info->fbops->fb_fillrect = nvidiafb_fillrect;
- info->fbops->fb_copyarea = nvidiafb_copyarea;
- info->fbops->fb_sync = nvidiafb_sync;
+ pax_open_kernel();
+ *(void **)&info->fbops->fb_imageblit = nvidiafb_imageblit;
+ *(void **)&info->fbops->fb_fillrect = nvidiafb_fillrect;
+ *(void **)&info->fbops->fb_copyarea = nvidiafb_copyarea;
+ *(void **)&info->fbops->fb_sync = nvidiafb_sync;
+ pax_close_kernel();
info->pixmap.scan_align = 4;
info->flags &= ~FBINFO_HWACCEL_DISABLED;
info->flags |= FBINFO_READS_FAST;
NVResetGraphics(info);
} else {
- info->fbops->fb_imageblit = cfb_imageblit;
- info->fbops->fb_fillrect = cfb_fillrect;
- info->fbops->fb_copyarea = cfb_copyarea;
- info->fbops->fb_sync = NULL;
+ pax_open_kernel();
+ *(void **)&info->fbops->fb_imageblit = cfb_imageblit;
+ *(void **)&info->fbops->fb_fillrect = cfb_fillrect;
+ *(void **)&info->fbops->fb_copyarea = cfb_copyarea;
+ *(void **)&info->fbops->fb_sync = NULL;
+ pax_close_kernel();
info->pixmap.scan_align = 1;
info->flags |= FBINFO_HWACCEL_DISABLED;
info->flags &= ~FBINFO_READS_FAST;
info->pixmap.size = 8 * 1024;
info->pixmap.flags = FB_PIXMAP_SYSTEM;
- if (!hwcur)
- info->fbops->fb_cursor = NULL;
+ if (!hwcur) {
+ pax_open_kernel();
+ *(void **)&info->fbops->fb_cursor = NULL;
+ pax_close_kernel();
+ }
info->var.accel_flags = (!noaccel);
if (dssdev->name == NULL)
dssdev->name = dssdev->alias;
+ pax_open_kernel();
if (drv && drv->get_resolution == NULL)
- drv->get_resolution = omapdss_default_get_resolution;
+ *(void **)&drv->get_resolution = omapdss_default_get_resolution;
if (drv && drv->get_recommended_bpp == NULL)
- drv->get_recommended_bpp = omapdss_default_get_recommended_bpp;
+ *(void **)&drv->get_recommended_bpp = omapdss_default_get_recommended_bpp;
if (drv && drv->get_timings == NULL)
- drv->get_timings = omapdss_default_get_timings;
+ *(void **)&drv->get_timings = omapdss_default_get_timings;
+ pax_close_kernel();
mutex_lock(&panel_list_mutex);
list_add_tail(&dssdev->panel_list, &panel_list);
switch(prod_id) {
case S1D13506_PROD_ID: /* activate acceleration */
- s1d13xxxfb_fbops.fb_fillrect = s1d13xxxfb_bitblt_solidfill;
- s1d13xxxfb_fbops.fb_copyarea = s1d13xxxfb_bitblt_copyarea;
+ pax_open_kernel();
+ *(void **)&s1d13xxxfb_fbops.fb_fillrect = s1d13xxxfb_bitblt_solidfill;
+ *(void **)&s1d13xxxfb_fbops.fb_copyarea = s1d13xxxfb_bitblt_copyarea;
+ pax_close_kernel();
info->flags = FBINFO_DEFAULT | FBINFO_HWACCEL_YPAN |
FBINFO_HWACCEL_FILLRECT | FBINFO_HWACCEL_COPYAREA;
break;
}
static struct sh_mobile_lcdc_sys_bus_ops sh_mobile_lcdc_sys_bus_ops = {
- lcdc_sys_write_index,
- lcdc_sys_write_data,
- lcdc_sys_read_data,
+ .write_index = lcdc_sys_write_index,
+ .write_data = lcdc_sys_write_data,
+ .read_data = lcdc_sys_read_data,
};
static int sh_mobile_lcdc_sginit(struct fb_info *info,
fb_deferred_io_cleanup(info);
kfree(info->fbdefio);
info->fbdefio = NULL;
- info->fbops->fb_mmap = ufx_ops_mmap;
+ pax_open_kernel();
+ *(void **)&info->fbops->fb_mmap = ufx_ops_mmap;
+ pax_close_kernel();
}
pr_debug("released /dev/fb%d user=%d count=%d",
dlfb_urb_completion(urb);
error:
- atomic_add(bytes_sent, &dev->bytes_sent);
- atomic_add(bytes_identical, &dev->bytes_identical);
- atomic_add(width*height*2, &dev->bytes_rendered);
+ atomic_add_unchecked(bytes_sent, &dev->bytes_sent);
+ atomic_add_unchecked(bytes_identical, &dev->bytes_identical);
+ atomic_add_unchecked(width*height*2, &dev->bytes_rendered);
end_cycles = get_cycles();
- atomic_add(((unsigned int) ((end_cycles - start_cycles)
+ atomic_add_unchecked(((unsigned int) ((end_cycles - start_cycles)
>> 10)), /* Kcycles */
&dev->cpu_kcycles_used);
dlfb_urb_completion(urb);
error:
- atomic_add(bytes_sent, &dev->bytes_sent);
- atomic_add(bytes_identical, &dev->bytes_identical);
- atomic_add(bytes_rendered, &dev->bytes_rendered);
+ atomic_add_unchecked(bytes_sent, &dev->bytes_sent);
+ atomic_add_unchecked(bytes_identical, &dev->bytes_identical);
+ atomic_add_unchecked(bytes_rendered, &dev->bytes_rendered);
end_cycles = get_cycles();
- atomic_add(((unsigned int) ((end_cycles - start_cycles)
+ atomic_add_unchecked(((unsigned int) ((end_cycles - start_cycles)
>> 10)), /* Kcycles */
&dev->cpu_kcycles_used);
}
fb_deferred_io_cleanup(info);
kfree(info->fbdefio);
info->fbdefio = NULL;
- info->fbops->fb_mmap = dlfb_ops_mmap;
+ pax_open_kernel();
+ *(void **)&info->fbops->fb_mmap = dlfb_ops_mmap;
+ pax_close_kernel();
}
pr_warn("released /dev/fb%d user=%d count=%d\n",
struct fb_info *fb_info = dev_get_drvdata(fbdev);
struct dlfb_data *dev = fb_info->par;
return snprintf(buf, PAGE_SIZE, "%u\n",
- atomic_read(&dev->bytes_rendered));
+ atomic_read_unchecked(&dev->bytes_rendered));
}
static ssize_t metrics_bytes_identical_show(struct device *fbdev,
struct fb_info *fb_info = dev_get_drvdata(fbdev);
struct dlfb_data *dev = fb_info->par;
return snprintf(buf, PAGE_SIZE, "%u\n",
- atomic_read(&dev->bytes_identical));
+ atomic_read_unchecked(&dev->bytes_identical));
}
static ssize_t metrics_bytes_sent_show(struct device *fbdev,
struct fb_info *fb_info = dev_get_drvdata(fbdev);
struct dlfb_data *dev = fb_info->par;
return snprintf(buf, PAGE_SIZE, "%u\n",
- atomic_read(&dev->bytes_sent));
+ atomic_read_unchecked(&dev->bytes_sent));
}
static ssize_t metrics_cpu_kcycles_used_show(struct device *fbdev,
struct fb_info *fb_info = dev_get_drvdata(fbdev);
struct dlfb_data *dev = fb_info->par;
return snprintf(buf, PAGE_SIZE, "%u\n",
- atomic_read(&dev->cpu_kcycles_used));
+ atomic_read_unchecked(&dev->cpu_kcycles_used));
}
static ssize_t edid_show(
struct fb_info *fb_info = dev_get_drvdata(fbdev);
struct dlfb_data *dev = fb_info->par;
- atomic_set(&dev->bytes_rendered, 0);
- atomic_set(&dev->bytes_identical, 0);
- atomic_set(&dev->bytes_sent, 0);
- atomic_set(&dev->cpu_kcycles_used, 0);
+ atomic_set_unchecked(&dev->bytes_rendered, 0);
+ atomic_set_unchecked(&dev->bytes_identical, 0);
+ atomic_set_unchecked(&dev->bytes_sent, 0);
+ atomic_set_unchecked(&dev->cpu_kcycles_used, 0);
return count;
}
#include <linux/io.h>
#include <linux/mutex.h>
#include <linux/slab.h>
+#include <linux/moduleloader.h>
#include <video/edid.h>
#include <video/uvesafb.h>
#ifdef CONFIG_X86
if ((task->t.regs.eax & 0xffff) != 0x4f || task->t.regs.es < 0xc000) {
par->pmi_setpal = par->ypan = 0;
} else {
+
+#ifdef CONFIG_PAX_KERNEXEC
+#ifdef CONFIG_MODULES
+ par->pmi_code = module_alloc_exec((u16)task->t.regs.ecx);
+#endif
+ if (!par->pmi_code) {
+ par->pmi_setpal = par->ypan = 0;
+ return 0;
+ }
+#endif
+
par->pmi_base = (u16 *)phys_to_virt(((u32)task->t.regs.es << 4)
+ task->t.regs.edi);
+
+#if defined(CONFIG_MODULES) && defined(CONFIG_PAX_KERNEXEC)
+ pax_open_kernel();
+ memcpy(par->pmi_code, par->pmi_base, (u16)task->t.regs.ecx);
+ pax_close_kernel();
+
+ par->pmi_start = ktva_ktla(par->pmi_code + par->pmi_base[1]);
+ par->pmi_pal = ktva_ktla(par->pmi_code + par->pmi_base[2]);
+#else
par->pmi_start = (u8 *)par->pmi_base + par->pmi_base[1];
par->pmi_pal = (u8 *)par->pmi_base + par->pmi_base[2];
+#endif
+
printk(KERN_INFO "uvesafb: protected mode interface info at "
"%04x:%04x\n",
(u16)task->t.regs.es, (u16)task->t.regs.edi);
par->ypan = ypan;
if (par->pmi_setpal || par->ypan) {
+#if !defined(CONFIG_MODULES) || !defined(CONFIG_PAX_KERNEXEC)
if (__supported_pte_mask & _PAGE_NX) {
par->pmi_setpal = par->ypan = 0;
printk(KERN_WARNING "uvesafb: NX protection is active, "
"better not use the PMI.\n");
- } else {
+ } else
+#endif
uvesafb_vbe_getpmi(task, par);
- }
}
#else
/* The protected mode interface is not available on non-x86. */
info->fix.ywrapstep = (par->ypan > 1) ? 1 : 0;
/* Disable blanking if the user requested so. */
- if (!blank)
- info->fbops->fb_blank = NULL;
+ if (!blank) {
+ pax_open_kernel();
+ *(void **)&info->fbops->fb_blank = NULL;
+ pax_close_kernel();
+ }
/*
* Find out how much IO memory is required for the mode with
info->flags = FBINFO_FLAG_DEFAULT |
(par->ypan ? FBINFO_HWACCEL_YPAN : 0);
- if (!par->ypan)
- info->fbops->fb_pan_display = NULL;
+ if (!par->ypan) {
+ pax_open_kernel();
+ *(void **)&info->fbops->fb_pan_display = NULL;
+ pax_close_kernel();
+ }
}
static void uvesafb_init_mtrr(struct fb_info *info)
out:
kfree(par->vbe_modes);
+#if defined(CONFIG_MODULES) && defined(CONFIG_PAX_KERNEXEC)
+ if (par->pmi_code)
+ module_memfree_exec(par->pmi_code);
+#endif
+
framebuffer_release(info);
return err;
}
kfree(par->vbe_state_orig);
kfree(par->vbe_state_saved);
+#if defined(CONFIG_MODULES) && defined(CONFIG_PAX_KERNEXEC)
+ if (par->pmi_code)
+ module_memfree_exec(par->pmi_code);
+#endif
+
framebuffer_release(info);
}
return 0;
*/
#include <linux/module.h>
+#include <linux/moduleloader.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/string.h>
static int vram_total; /* Set total amount of memory */
static int pmi_setpal __read_mostly = 1; /* pmi for palette changes ??? */
static int ypan __read_mostly; /* 0..nothing, 1..ypan, 2..ywrap */
-static void (*pmi_start)(void) __read_mostly;
-static void (*pmi_pal) (void) __read_mostly;
+static void (*pmi_start)(void) __read_only;
+static void (*pmi_pal) (void) __read_only;
static int depth __read_mostly;
static int vga_compat __read_mostly;
/* --------------------------------------------------------------------- */
unsigned int size_remap;
unsigned int size_total;
char *option = NULL;
+ void *pmi_code = NULL;
/* ignore error return of fb_get_options */
fb_get_options("vesafb", &option);
size_remap = size_total;
vesafb_fix.smem_len = size_remap;
-#ifndef __i386__
- screen_info.vesapm_seg = 0;
-#endif
-
if (!request_mem_region(vesafb_fix.smem_start, size_total, "vesafb")) {
printk(KERN_WARNING
"vesafb: cannot reserve video memory at 0x%lx\n",
printk(KERN_INFO "vesafb: mode is %dx%dx%d, linelength=%d, pages=%d\n",
vesafb_defined.xres, vesafb_defined.yres, vesafb_defined.bits_per_pixel, vesafb_fix.line_length, screen_info.pages);
+#ifdef __i386__
+
+#if defined(CONFIG_MODULES) && defined(CONFIG_PAX_KERNEXEC)
+ pmi_code = module_alloc_exec(screen_info.vesapm_size);
+ if (!pmi_code)
+#elif !defined(CONFIG_PAX_KERNEXEC)
+ if (0)
+#endif
+
+#endif
+ screen_info.vesapm_seg = 0;
+
if (screen_info.vesapm_seg) {
- printk(KERN_INFO "vesafb: protected mode interface info at %04x:%04x\n",
- screen_info.vesapm_seg,screen_info.vesapm_off);
+ printk(KERN_INFO "vesafb: protected mode interface info at %04x:%04x %04x bytes\n",
+ screen_info.vesapm_seg,screen_info.vesapm_off,screen_info.vesapm_size);
}
if (screen_info.vesapm_seg < 0xc000)
if (ypan || pmi_setpal) {
unsigned short *pmi_base;
+
pmi_base = (unsigned short*)phys_to_virt(((unsigned long)screen_info.vesapm_seg << 4) + screen_info.vesapm_off);
- pmi_start = (void*)((char*)pmi_base + pmi_base[1]);
- pmi_pal = (void*)((char*)pmi_base + pmi_base[2]);
+
+#if defined(CONFIG_MODULES) && defined(CONFIG_PAX_KERNEXEC)
+ pax_open_kernel();
+ memcpy(pmi_code, pmi_base, screen_info.vesapm_size);
+#else
+ pmi_code = pmi_base;
+#endif
+
+ pmi_start = (void*)((char*)pmi_code + pmi_base[1]);
+ pmi_pal = (void*)((char*)pmi_code + pmi_base[2]);
+
+#if defined(CONFIG_MODULES) && defined(CONFIG_PAX_KERNEXEC)
+ pmi_start = ktva_ktla(pmi_start);
+ pmi_pal = ktva_ktla(pmi_pal);
+ pax_close_kernel();
+#endif
+
printk(KERN_INFO "vesafb: pmi: set display start = %p, set palette = %p\n",pmi_start,pmi_pal);
if (pmi_base[3]) {
printk(KERN_INFO "vesafb: pmi: ports = ");
info->flags = FBINFO_FLAG_DEFAULT | FBINFO_MISC_FIRMWARE |
(ypan ? FBINFO_HWACCEL_YPAN : 0);
- if (!ypan)
- info->fbops->fb_pan_display = NULL;
+ if (!ypan) {
+ pax_open_kernel();
+ *(void **)&info->fbops->fb_pan_display = NULL;
+ pax_close_kernel();
+ }
if (fb_alloc_cmap(&info->cmap, 256, 0) < 0) {
err = -ENOMEM;
fb_info(info, "%s frame buffer device\n", info->fix.id);
return 0;
err:
+
+#if defined(__i386__) && defined(CONFIG_MODULES) && defined(CONFIG_PAX_KERNEXEC)
+ module_memfree_exec(pmi_code);
+#endif
+
if (info->screen_base)
iounmap(info->screen_base);
framebuffer_release(info);
void (*set_engine_pll_state)(u8 state);
void (*set_engine_pll)(struct via_pll_config config);
-};
+} __no_const;
static inline u32 get_pll_internal_frequency(u32 ref_freq,
# Standard 224-color Linux logo
80 80
255
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 6 6 6 6 6 6 10 10 10 10 10 10
- 10 10 10 6 6 6 6 6 6 6 6 6
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 6 6 6 10 10 10 14 14 14
- 22 22 22 26 26 26 30 30 30 34 34 34
- 30 30 30 30 30 30 26 26 26 18 18 18
- 14 14 14 10 10 10 6 6 6 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 1 0 0 1 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 6 6 6 14 14 14 26 26 26 42 42 42
- 54 54 54 66 66 66 78 78 78 78 78 78
- 78 78 78 74 74 74 66 66 66 54 54 54
- 42 42 42 26 26 26 18 18 18 10 10 10
- 6 6 6 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 1 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 10 10 10
- 22 22 22 42 42 42 66 66 66 86 86 86
- 66 66 66 38 38 38 38 38 38 22 22 22
- 26 26 26 34 34 34 54 54 54 66 66 66
- 86 86 86 70 70 70 46 46 46 26 26 26
- 14 14 14 6 6 6 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 1 0 0 1 0 0 1 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 10 10 10 26 26 26
- 50 50 50 82 82 82 58 58 58 6 6 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 6 6 6 54 54 54 86 86 86 66 66 66
- 38 38 38 18 18 18 6 6 6 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 6 6 6 22 22 22 50 50 50
- 78 78 78 34 34 34 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 6 6 6 70 70 70
- 78 78 78 46 46 46 22 22 22 6 6 6
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 1 0 0 1 0 0 1 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 6 6 6 18 18 18 42 42 42 82 82 82
- 26 26 26 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 14 14 14
- 46 46 46 34 34 34 6 6 6 2 2 6
- 42 42 42 78 78 78 42 42 42 18 18 18
- 6 6 6 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 1 0 0 0 0 0 1 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 10 10 10 30 30 30 66 66 66 58 58 58
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 26 26 26
- 86 86 86 101 101 101 46 46 46 10 10 10
- 2 2 6 58 58 58 70 70 70 34 34 34
- 10 10 10 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 1 0 0 1 0 0 1 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 14 14 14 42 42 42 86 86 86 10 10 10
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 30 30 30
- 94 94 94 94 94 94 58 58 58 26 26 26
- 2 2 6 6 6 6 78 78 78 54 54 54
- 22 22 22 6 6 6 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 6 6 6
- 22 22 22 62 62 62 62 62 62 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 26 26 26
- 54 54 54 38 38 38 18 18 18 10 10 10
- 2 2 6 2 2 6 34 34 34 82 82 82
- 38 38 38 14 14 14 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 1 0 0 1 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 6 6 6
- 30 30 30 78 78 78 30 30 30 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 10 10 10
- 10 10 10 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 78 78 78
- 50 50 50 18 18 18 6 6 6 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 1 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 10 10 10
- 38 38 38 86 86 86 14 14 14 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 54 54 54
- 66 66 66 26 26 26 6 6 6 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 1 0 0 1 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 14 14 14
- 42 42 42 82 82 82 2 2 6 2 2 6
- 2 2 6 6 6 6 10 10 10 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 6 6 6
- 14 14 14 10 10 10 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 18 18 18
- 82 82 82 34 34 34 10 10 10 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 1 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 14 14 14
- 46 46 46 86 86 86 2 2 6 2 2 6
- 6 6 6 6 6 6 22 22 22 34 34 34
- 6 6 6 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 18 18 18 34 34 34
- 10 10 10 50 50 50 22 22 22 2 2 6
- 2 2 6 2 2 6 2 2 6 10 10 10
- 86 86 86 42 42 42 14 14 14 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 1 0 0 1 0 0 1 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 14 14 14
- 46 46 46 86 86 86 2 2 6 2 2 6
- 38 38 38 116 116 116 94 94 94 22 22 22
- 22 22 22 2 2 6 2 2 6 2 2 6
- 14 14 14 86 86 86 138 138 138 162 162 162
-154 154 154 38 38 38 26 26 26 6 6 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 86 86 86 46 46 46 14 14 14 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 14 14 14
- 46 46 46 86 86 86 2 2 6 14 14 14
-134 134 134 198 198 198 195 195 195 116 116 116
- 10 10 10 2 2 6 2 2 6 6 6 6
-101 98 89 187 187 187 210 210 210 218 218 218
-214 214 214 134 134 134 14 14 14 6 6 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 86 86 86 50 50 50 18 18 18 6 6 6
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 1 0 0 0
- 0 0 1 0 0 1 0 0 1 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 14 14 14
- 46 46 46 86 86 86 2 2 6 54 54 54
-218 218 218 195 195 195 226 226 226 246 246 246
- 58 58 58 2 2 6 2 2 6 30 30 30
-210 210 210 253 253 253 174 174 174 123 123 123
-221 221 221 234 234 234 74 74 74 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 70 70 70 58 58 58 22 22 22 6 6 6
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 14 14 14
- 46 46 46 82 82 82 2 2 6 106 106 106
-170 170 170 26 26 26 86 86 86 226 226 226
-123 123 123 10 10 10 14 14 14 46 46 46
-231 231 231 190 190 190 6 6 6 70 70 70
- 90 90 90 238 238 238 158 158 158 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 70 70 70 58 58 58 22 22 22 6 6 6
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 1 0 0 0
- 0 0 1 0 0 1 0 0 1 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 14 14 14
- 42 42 42 86 86 86 6 6 6 116 116 116
-106 106 106 6 6 6 70 70 70 149 149 149
-128 128 128 18 18 18 38 38 38 54 54 54
-221 221 221 106 106 106 2 2 6 14 14 14
- 46 46 46 190 190 190 198 198 198 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 74 74 74 62 62 62 22 22 22 6 6 6
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 1 0 0 0
- 0 0 1 0 0 0 0 0 1 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 14 14 14
- 42 42 42 94 94 94 14 14 14 101 101 101
-128 128 128 2 2 6 18 18 18 116 116 116
-118 98 46 121 92 8 121 92 8 98 78 10
-162 162 162 106 106 106 2 2 6 2 2 6
- 2 2 6 195 195 195 195 195 195 6 6 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 74 74 74 62 62 62 22 22 22 6 6 6
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 1 0 0 1
- 0 0 1 0 0 0 0 0 1 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 10 10 10
- 38 38 38 90 90 90 14 14 14 58 58 58
-210 210 210 26 26 26 54 38 6 154 114 10
-226 170 11 236 186 11 225 175 15 184 144 12
-215 174 15 175 146 61 37 26 9 2 2 6
- 70 70 70 246 246 246 138 138 138 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 70 70 70 66 66 66 26 26 26 6 6 6
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 10 10 10
- 38 38 38 86 86 86 14 14 14 10 10 10
-195 195 195 188 164 115 192 133 9 225 175 15
-239 182 13 234 190 10 232 195 16 232 200 30
-245 207 45 241 208 19 232 195 16 184 144 12
-218 194 134 211 206 186 42 42 42 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 50 50 50 74 74 74 30 30 30 6 6 6
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 10 10 10
- 34 34 34 86 86 86 14 14 14 2 2 6
-121 87 25 192 133 9 219 162 10 239 182 13
-236 186 11 232 195 16 241 208 19 244 214 54
-246 218 60 246 218 38 246 215 20 241 208 19
-241 208 19 226 184 13 121 87 25 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 50 50 50 82 82 82 34 34 34 10 10 10
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 10 10 10
- 34 34 34 82 82 82 30 30 30 61 42 6
-180 123 7 206 145 10 230 174 11 239 182 13
-234 190 10 238 202 15 241 208 19 246 218 74
-246 218 38 246 215 20 246 215 20 246 215 20
-226 184 13 215 174 15 184 144 12 6 6 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 26 26 26 94 94 94 42 42 42 14 14 14
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 10 10 10
- 30 30 30 78 78 78 50 50 50 104 69 6
-192 133 9 216 158 10 236 178 12 236 186 11
-232 195 16 241 208 19 244 214 54 245 215 43
-246 215 20 246 215 20 241 208 19 198 155 10
-200 144 11 216 158 10 156 118 10 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 6 6 6 90 90 90 54 54 54 18 18 18
- 6 6 6 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 10 10 10
- 30 30 30 78 78 78 46 46 46 22 22 22
-137 92 6 210 162 10 239 182 13 238 190 10
-238 202 15 241 208 19 246 215 20 246 215 20
-241 208 19 203 166 17 185 133 11 210 150 10
-216 158 10 210 150 10 102 78 10 2 2 6
- 6 6 6 54 54 54 14 14 14 2 2 6
- 2 2 6 62 62 62 74 74 74 30 30 30
- 10 10 10 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 10 10 10
- 34 34 34 78 78 78 50 50 50 6 6 6
- 94 70 30 139 102 15 190 146 13 226 184 13
-232 200 30 232 195 16 215 174 15 190 146 13
-168 122 10 192 133 9 210 150 10 213 154 11
-202 150 34 182 157 106 101 98 89 2 2 6
- 2 2 6 78 78 78 116 116 116 58 58 58
- 2 2 6 22 22 22 90 90 90 46 46 46
- 18 18 18 6 6 6 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 10 10 10
- 38 38 38 86 86 86 50 50 50 6 6 6
-128 128 128 174 154 114 156 107 11 168 122 10
-198 155 10 184 144 12 197 138 11 200 144 11
-206 145 10 206 145 10 197 138 11 188 164 115
-195 195 195 198 198 198 174 174 174 14 14 14
- 2 2 6 22 22 22 116 116 116 116 116 116
- 22 22 22 2 2 6 74 74 74 70 70 70
- 30 30 30 10 10 10 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 6 6 6 18 18 18
- 50 50 50 101 101 101 26 26 26 10 10 10
-138 138 138 190 190 190 174 154 114 156 107 11
-197 138 11 200 144 11 197 138 11 192 133 9
-180 123 7 190 142 34 190 178 144 187 187 187
-202 202 202 221 221 221 214 214 214 66 66 66
- 2 2 6 2 2 6 50 50 50 62 62 62
- 6 6 6 2 2 6 10 10 10 90 90 90
- 50 50 50 18 18 18 6 6 6 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 10 10 10 34 34 34
- 74 74 74 74 74 74 2 2 6 6 6 6
-144 144 144 198 198 198 190 190 190 178 166 146
-154 121 60 156 107 11 156 107 11 168 124 44
-174 154 114 187 187 187 190 190 190 210 210 210
-246 246 246 253 253 253 253 253 253 182 182 182
- 6 6 6 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 62 62 62
- 74 74 74 34 34 34 14 14 14 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 10 10 10 22 22 22 54 54 54
- 94 94 94 18 18 18 2 2 6 46 46 46
-234 234 234 221 221 221 190 190 190 190 190 190
-190 190 190 187 187 187 187 187 187 190 190 190
-190 190 190 195 195 195 214 214 214 242 242 242
-253 253 253 253 253 253 253 253 253 253 253 253
- 82 82 82 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 14 14 14
- 86 86 86 54 54 54 22 22 22 6 6 6
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 6 6 6 18 18 18 46 46 46 90 90 90
- 46 46 46 18 18 18 6 6 6 182 182 182
-253 253 253 246 246 246 206 206 206 190 190 190
-190 190 190 190 190 190 190 190 190 190 190 190
-206 206 206 231 231 231 250 250 250 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-202 202 202 14 14 14 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 42 42 42 86 86 86 42 42 42 18 18 18
- 6 6 6 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 6 6 6
- 14 14 14 38 38 38 74 74 74 66 66 66
- 2 2 6 6 6 6 90 90 90 250 250 250
-253 253 253 253 253 253 238 238 238 198 198 198
-190 190 190 190 190 190 195 195 195 221 221 221
-246 246 246 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 82 82 82 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 78 78 78 70 70 70 34 34 34
- 14 14 14 6 6 6 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 14 14 14
- 34 34 34 66 66 66 78 78 78 6 6 6
- 2 2 6 18 18 18 218 218 218 253 253 253
-253 253 253 253 253 253 253 253 253 246 246 246
-226 226 226 231 231 231 246 246 246 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 178 178 178 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 18 18 18 90 90 90 62 62 62
- 30 30 30 10 10 10 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 10 10 10 26 26 26
- 58 58 58 90 90 90 18 18 18 2 2 6
- 2 2 6 110 110 110 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-250 250 250 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 231 231 231 18 18 18 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 18 18 18 94 94 94
- 54 54 54 26 26 26 10 10 10 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 6 6 6 22 22 22 50 50 50
- 90 90 90 26 26 26 2 2 6 2 2 6
- 14 14 14 195 195 195 250 250 250 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-250 250 250 242 242 242 54 54 54 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 38 38 38
- 86 86 86 50 50 50 22 22 22 6 6 6
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 6 6 6 14 14 14 38 38 38 82 82 82
- 34 34 34 2 2 6 2 2 6 2 2 6
- 42 42 42 195 195 195 246 246 246 253 253 253
-253 253 253 253 253 253 253 253 253 250 250 250
-242 242 242 242 242 242 250 250 250 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 250 250 250 246 246 246 238 238 238
-226 226 226 231 231 231 101 101 101 6 6 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 38 38 38 82 82 82 42 42 42 14 14 14
- 6 6 6 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 10 10 10 26 26 26 62 62 62 66 66 66
- 2 2 6 2 2 6 2 2 6 6 6 6
- 70 70 70 170 170 170 206 206 206 234 234 234
-246 246 246 250 250 250 250 250 250 238 238 238
-226 226 226 231 231 231 238 238 238 250 250 250
-250 250 250 250 250 250 246 246 246 231 231 231
-214 214 214 206 206 206 202 202 202 202 202 202
-198 198 198 202 202 202 182 182 182 18 18 18
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 62 62 62 66 66 66 30 30 30
- 10 10 10 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 14 14 14 42 42 42 82 82 82 18 18 18
- 2 2 6 2 2 6 2 2 6 10 10 10
- 94 94 94 182 182 182 218 218 218 242 242 242
-250 250 250 253 253 253 253 253 253 250 250 250
-234 234 234 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 246 246 246
-238 238 238 226 226 226 210 210 210 202 202 202
-195 195 195 195 195 195 210 210 210 158 158 158
- 6 6 6 14 14 14 50 50 50 14 14 14
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 6 6 6 86 86 86 46 46 46
- 18 18 18 6 6 6 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 6 6 6
- 22 22 22 54 54 54 70 70 70 2 2 6
- 2 2 6 10 10 10 2 2 6 22 22 22
-166 166 166 231 231 231 250 250 250 253 253 253
-253 253 253 253 253 253 253 253 253 250 250 250
-242 242 242 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 246 246 246
-231 231 231 206 206 206 198 198 198 226 226 226
- 94 94 94 2 2 6 6 6 6 38 38 38
- 30 30 30 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 62 62 62 66 66 66
- 26 26 26 10 10 10 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 10 10 10
- 30 30 30 74 74 74 50 50 50 2 2 6
- 26 26 26 26 26 26 2 2 6 106 106 106
-238 238 238 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 246 246 246 218 218 218 202 202 202
-210 210 210 14 14 14 2 2 6 2 2 6
- 30 30 30 22 22 22 2 2 6 2 2 6
- 2 2 6 2 2 6 18 18 18 86 86 86
- 42 42 42 14 14 14 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 14 14 14
- 42 42 42 90 90 90 22 22 22 2 2 6
- 42 42 42 2 2 6 18 18 18 218 218 218
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 250 250 250 221 221 221
-218 218 218 101 101 101 2 2 6 14 14 14
- 18 18 18 38 38 38 10 10 10 2 2 6
- 2 2 6 2 2 6 2 2 6 78 78 78
- 58 58 58 22 22 22 6 6 6 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 6 6 6 18 18 18
- 54 54 54 82 82 82 2 2 6 26 26 26
- 22 22 22 2 2 6 123 123 123 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 250 250 250
-238 238 238 198 198 198 6 6 6 38 38 38
- 58 58 58 26 26 26 38 38 38 2 2 6
- 2 2 6 2 2 6 2 2 6 46 46 46
- 78 78 78 30 30 30 10 10 10 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 10 10 10 30 30 30
- 74 74 74 58 58 58 2 2 6 42 42 42
- 2 2 6 22 22 22 231 231 231 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 250 250 250
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 246 246 246 46 46 46 38 38 38
- 42 42 42 14 14 14 38 38 38 14 14 14
- 2 2 6 2 2 6 2 2 6 6 6 6
- 86 86 86 46 46 46 14 14 14 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 6 6 6 14 14 14 42 42 42
- 90 90 90 18 18 18 18 18 18 26 26 26
- 2 2 6 116 116 116 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 250 250 250 238 238 238
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 94 94 94 6 6 6
- 2 2 6 2 2 6 10 10 10 34 34 34
- 2 2 6 2 2 6 2 2 6 2 2 6
- 74 74 74 58 58 58 22 22 22 6 6 6
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 10 10 10 26 26 26 66 66 66
- 82 82 82 2 2 6 38 38 38 6 6 6
- 14 14 14 210 210 210 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 246 246 246 242 242 242
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 144 144 144 2 2 6
- 2 2 6 2 2 6 2 2 6 46 46 46
- 2 2 6 2 2 6 2 2 6 2 2 6
- 42 42 42 74 74 74 30 30 30 10 10 10
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 6 6 6 14 14 14 42 42 42 90 90 90
- 26 26 26 6 6 6 42 42 42 2 2 6
- 74 74 74 250 250 250 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 242 242 242 242 242 242
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 182 182 182 2 2 6
- 2 2 6 2 2 6 2 2 6 46 46 46
- 2 2 6 2 2 6 2 2 6 2 2 6
- 10 10 10 86 86 86 38 38 38 10 10 10
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 10 10 10 26 26 26 66 66 66 82 82 82
- 2 2 6 22 22 22 18 18 18 2 2 6
-149 149 149 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 234 234 234 242 242 242
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 206 206 206 2 2 6
- 2 2 6 2 2 6 2 2 6 38 38 38
- 2 2 6 2 2 6 2 2 6 2 2 6
- 6 6 6 86 86 86 46 46 46 14 14 14
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 6 6 6
- 18 18 18 46 46 46 86 86 86 18 18 18
- 2 2 6 34 34 34 10 10 10 6 6 6
-210 210 210 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 234 234 234 242 242 242
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 221 221 221 6 6 6
- 2 2 6 2 2 6 6 6 6 30 30 30
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 82 82 82 54 54 54 18 18 18
- 6 6 6 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 10 10 10
- 26 26 26 66 66 66 62 62 62 2 2 6
- 2 2 6 38 38 38 10 10 10 26 26 26
-238 238 238 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 231 231 231 238 238 238
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 231 231 231 6 6 6
- 2 2 6 2 2 6 10 10 10 30 30 30
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 66 66 66 58 58 58 22 22 22
- 6 6 6 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 10 10 10
- 38 38 38 78 78 78 6 6 6 2 2 6
- 2 2 6 46 46 46 14 14 14 42 42 42
-246 246 246 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 231 231 231 242 242 242
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 234 234 234 10 10 10
- 2 2 6 2 2 6 22 22 22 14 14 14
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 66 66 66 62 62 62 22 22 22
- 6 6 6 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 6 6 6 18 18 18
- 50 50 50 74 74 74 2 2 6 2 2 6
- 14 14 14 70 70 70 34 34 34 62 62 62
-250 250 250 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 231 231 231 246 246 246
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 234 234 234 14 14 14
- 2 2 6 2 2 6 30 30 30 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 66 66 66 62 62 62 22 22 22
- 6 6 6 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 6 6 6 18 18 18
- 54 54 54 62 62 62 2 2 6 2 2 6
- 2 2 6 30 30 30 46 46 46 70 70 70
-250 250 250 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 231 231 231 246 246 246
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 226 226 226 10 10 10
- 2 2 6 6 6 6 30 30 30 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 66 66 66 58 58 58 22 22 22
- 6 6 6 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 6 6 6 22 22 22
- 58 58 58 62 62 62 2 2 6 2 2 6
- 2 2 6 2 2 6 30 30 30 78 78 78
-250 250 250 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 231 231 231 246 246 246
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 206 206 206 2 2 6
- 22 22 22 34 34 34 18 14 6 22 22 22
- 26 26 26 18 18 18 6 6 6 2 2 6
- 2 2 6 82 82 82 54 54 54 18 18 18
- 6 6 6 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 6 6 6 26 26 26
- 62 62 62 106 106 106 74 54 14 185 133 11
-210 162 10 121 92 8 6 6 6 62 62 62
-238 238 238 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 231 231 231 246 246 246
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 158 158 158 18 18 18
- 14 14 14 2 2 6 2 2 6 2 2 6
- 6 6 6 18 18 18 66 66 66 38 38 38
- 6 6 6 94 94 94 50 50 50 18 18 18
- 6 6 6 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 6 6 6
- 10 10 10 10 10 10 18 18 18 38 38 38
- 78 78 78 142 134 106 216 158 10 242 186 14
-246 190 14 246 190 14 156 118 10 10 10 10
- 90 90 90 238 238 238 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 231 231 231 250 250 250
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 246 230 190
-238 204 91 238 204 91 181 142 44 37 26 9
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 38 38 38 46 46 46
- 26 26 26 106 106 106 54 54 54 18 18 18
- 6 6 6 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 6 6 6 14 14 14 22 22 22
- 30 30 30 38 38 38 50 50 50 70 70 70
-106 106 106 190 142 34 226 170 11 242 186 14
-246 190 14 246 190 14 246 190 14 154 114 10
- 6 6 6 74 74 74 226 226 226 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 231 231 231 250 250 250
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 228 184 62
-241 196 14 241 208 19 232 195 16 38 30 10
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 6 6 6 30 30 30 26 26 26
-203 166 17 154 142 90 66 66 66 26 26 26
- 6 6 6 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 6 6 6 18 18 18 38 38 38 58 58 58
- 78 78 78 86 86 86 101 101 101 123 123 123
-175 146 61 210 150 10 234 174 13 246 186 14
-246 190 14 246 190 14 246 190 14 238 190 10
-102 78 10 2 2 6 46 46 46 198 198 198
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 234 234 234 242 242 242
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 224 178 62
-242 186 14 241 196 14 210 166 10 22 18 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 6 6 6 121 92 8
-238 202 15 232 195 16 82 82 82 34 34 34
- 10 10 10 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 14 14 14 38 38 38 70 70 70 154 122 46
-190 142 34 200 144 11 197 138 11 197 138 11
-213 154 11 226 170 11 242 186 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-225 175 15 46 32 6 2 2 6 22 22 22
-158 158 158 250 250 250 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 250 250 250 242 242 242 224 178 62
-239 182 13 236 186 11 213 154 11 46 32 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 61 42 6 225 175 15
-238 190 10 236 186 11 112 100 78 42 42 42
- 14 14 14 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 6 6 6
- 22 22 22 54 54 54 154 122 46 213 154 11
-226 170 11 230 174 11 226 170 11 226 170 11
-236 178 12 242 186 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-241 196 14 184 144 12 10 10 10 2 2 6
- 6 6 6 116 116 116 242 242 242 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 231 231 231 198 198 198 214 170 54
-236 178 12 236 178 12 210 150 10 137 92 6
- 18 14 6 2 2 6 2 2 6 2 2 6
- 6 6 6 70 47 6 200 144 11 236 178 12
-239 182 13 239 182 13 124 112 88 58 58 58
- 22 22 22 6 6 6 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 10 10 10
- 30 30 30 70 70 70 180 133 36 226 170 11
-239 182 13 242 186 14 242 186 14 246 186 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 232 195 16 98 70 6 2 2 6
- 2 2 6 2 2 6 66 66 66 221 221 221
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 206 206 206 198 198 198 214 166 58
-230 174 11 230 174 11 216 158 10 192 133 9
-163 110 8 116 81 8 102 78 10 116 81 8
-167 114 7 197 138 11 226 170 11 239 182 13
-242 186 14 242 186 14 162 146 94 78 78 78
- 34 34 34 14 14 14 6 6 6 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 6 6 6
- 30 30 30 78 78 78 190 142 34 226 170 11
-239 182 13 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 241 196 14 203 166 17 22 18 6
- 2 2 6 2 2 6 2 2 6 38 38 38
-218 218 218 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-250 250 250 206 206 206 198 198 198 202 162 69
-226 170 11 236 178 12 224 166 10 210 150 10
-200 144 11 197 138 11 192 133 9 197 138 11
-210 150 10 226 170 11 242 186 14 246 190 14
-246 190 14 246 186 14 225 175 15 124 112 88
- 62 62 62 30 30 30 14 14 14 6 6 6
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 10 10 10
- 30 30 30 78 78 78 174 135 50 224 166 10
-239 182 13 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 241 196 14 139 102 15
- 2 2 6 2 2 6 2 2 6 2 2 6
- 78 78 78 250 250 250 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-250 250 250 214 214 214 198 198 198 190 150 46
-219 162 10 236 178 12 234 174 13 224 166 10
-216 158 10 213 154 11 213 154 11 216 158 10
-226 170 11 239 182 13 246 190 14 246 190 14
-246 190 14 246 190 14 242 186 14 206 162 42
-101 101 101 58 58 58 30 30 30 14 14 14
- 6 6 6 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 10 10 10
- 30 30 30 74 74 74 174 135 50 216 158 10
-236 178 12 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 241 196 14 226 184 13
- 61 42 6 2 2 6 2 2 6 2 2 6
- 22 22 22 238 238 238 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 226 226 226 187 187 187 180 133 36
-216 158 10 236 178 12 239 182 13 236 178 12
-230 174 11 226 170 11 226 170 11 230 174 11
-236 178 12 242 186 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 186 14 239 182 13
-206 162 42 106 106 106 66 66 66 34 34 34
- 14 14 14 6 6 6 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 6 6 6
- 26 26 26 70 70 70 163 133 67 213 154 11
-236 178 12 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 241 196 14
-190 146 13 18 14 6 2 2 6 2 2 6
- 46 46 46 246 246 246 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 221 221 221 86 86 86 156 107 11
-216 158 10 236 178 12 242 186 14 246 186 14
-242 186 14 239 182 13 239 182 13 242 186 14
-242 186 14 246 186 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-242 186 14 225 175 15 142 122 72 66 66 66
- 30 30 30 10 10 10 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 6 6 6
- 26 26 26 70 70 70 163 133 67 210 150 10
-236 178 12 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-232 195 16 121 92 8 34 34 34 106 106 106
-221 221 221 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-242 242 242 82 82 82 18 14 6 163 110 8
-216 158 10 236 178 12 242 186 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 242 186 14 163 133 67
- 46 46 46 18 18 18 6 6 6 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 10 10 10
- 30 30 30 78 78 78 163 133 67 210 150 10
-236 178 12 246 186 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-241 196 14 215 174 15 190 178 144 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 218 218 218
- 58 58 58 2 2 6 22 18 6 167 114 7
-216 158 10 236 178 12 246 186 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 246 186 14 242 186 14 190 150 46
- 54 54 54 22 22 22 6 6 6 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 14 14 14
- 38 38 38 86 86 86 180 133 36 213 154 11
-236 178 12 246 186 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 232 195 16 190 146 13 214 214 214
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 250 250 250 170 170 170 26 26 26
- 2 2 6 2 2 6 37 26 9 163 110 8
-219 162 10 239 182 13 246 186 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 186 14 236 178 12 224 166 10 142 122 72
- 46 46 46 18 18 18 6 6 6 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 6 6 6 18 18 18
- 50 50 50 109 106 95 192 133 9 224 166 10
-242 186 14 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-242 186 14 226 184 13 210 162 10 142 110 46
-226 226 226 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-253 253 253 253 253 253 253 253 253 253 253 253
-198 198 198 66 66 66 2 2 6 2 2 6
- 2 2 6 2 2 6 50 34 6 156 107 11
-219 162 10 239 182 13 246 186 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 242 186 14
-234 174 13 213 154 11 154 122 46 66 66 66
- 30 30 30 10 10 10 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 6 6 6 22 22 22
- 58 58 58 154 121 60 206 145 10 234 174 13
-242 186 14 246 186 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 186 14 236 178 12 210 162 10 163 110 8
- 61 42 6 138 138 138 218 218 218 250 250 250
-253 253 253 253 253 253 253 253 253 250 250 250
-242 242 242 210 210 210 144 144 144 66 66 66
- 6 6 6 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 61 42 6 163 110 8
-216 158 10 236 178 12 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 239 182 13 230 174 11 216 158 10
-190 142 34 124 112 88 70 70 70 38 38 38
- 18 18 18 6 6 6 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 6 6 6 22 22 22
- 62 62 62 168 124 44 206 145 10 224 166 10
-236 178 12 239 182 13 242 186 14 242 186 14
-246 186 14 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 236 178 12 216 158 10 175 118 6
- 80 54 7 2 2 6 6 6 6 30 30 30
- 54 54 54 62 62 62 50 50 50 38 38 38
- 14 14 14 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 6 6 6 80 54 7 167 114 7
-213 154 11 236 178 12 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 190 14 242 186 14 239 182 13 239 182 13
-230 174 11 210 150 10 174 135 50 124 112 88
- 82 82 82 54 54 54 34 34 34 18 18 18
- 6 6 6 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 6 6 6 18 18 18
- 50 50 50 158 118 36 192 133 9 200 144 11
-216 158 10 219 162 10 224 166 10 226 170 11
-230 174 11 236 178 12 239 182 13 239 182 13
-242 186 14 246 186 14 246 190 14 246 190 14
-246 190 14 246 190 14 246 190 14 246 190 14
-246 186 14 230 174 11 210 150 10 163 110 8
-104 69 6 10 10 10 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 6 6 6 91 60 6 167 114 7
-206 145 10 230 174 11 242 186 14 246 190 14
-246 190 14 246 190 14 246 186 14 242 186 14
-239 182 13 230 174 11 224 166 10 213 154 11
-180 133 36 124 112 88 86 86 86 58 58 58
- 38 38 38 22 22 22 10 10 10 6 6 6
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 14 14 14
- 34 34 34 70 70 70 138 110 50 158 118 36
-167 114 7 180 123 7 192 133 9 197 138 11
-200 144 11 206 145 10 213 154 11 219 162 10
-224 166 10 230 174 11 239 182 13 242 186 14
-246 186 14 246 186 14 246 186 14 246 186 14
-239 182 13 216 158 10 185 133 11 152 99 6
-104 69 6 18 14 6 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 2 2 6 2 2 6 2 2 6
- 2 2 6 6 6 6 80 54 7 152 99 6
-192 133 9 219 162 10 236 178 12 239 182 13
-246 186 14 242 186 14 239 182 13 236 178 12
-224 166 10 206 145 10 192 133 9 154 121 60
- 94 94 94 62 62 62 42 42 42 22 22 22
- 14 14 14 6 6 6 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 6 6 6
- 18 18 18 34 34 34 58 58 58 78 78 78
-101 98 89 124 112 88 142 110 46 156 107 11
-163 110 8 167 114 7 175 118 6 180 123 7
-185 133 11 197 138 11 210 150 10 219 162 10
-226 170 11 236 178 12 236 178 12 234 174 13
-219 162 10 197 138 11 163 110 8 130 83 6
- 91 60 6 10 10 10 2 2 6 2 2 6
- 18 18 18 38 38 38 38 38 38 38 38 38
- 38 38 38 38 38 38 38 38 38 38 38 38
- 38 38 38 38 38 38 26 26 26 2 2 6
- 2 2 6 6 6 6 70 47 6 137 92 6
-175 118 6 200 144 11 219 162 10 230 174 11
-234 174 13 230 174 11 219 162 10 210 150 10
-192 133 9 163 110 8 124 112 88 82 82 82
- 50 50 50 30 30 30 14 14 14 6 6 6
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 6 6 6 14 14 14 22 22 22 34 34 34
- 42 42 42 58 58 58 74 74 74 86 86 86
-101 98 89 122 102 70 130 98 46 121 87 25
-137 92 6 152 99 6 163 110 8 180 123 7
-185 133 11 197 138 11 206 145 10 200 144 11
-180 123 7 156 107 11 130 83 6 104 69 6
- 50 34 6 54 54 54 110 110 110 101 98 89
- 86 86 86 82 82 82 78 78 78 78 78 78
- 78 78 78 78 78 78 78 78 78 78 78 78
- 78 78 78 82 82 82 86 86 86 94 94 94
-106 106 106 101 101 101 86 66 34 124 80 6
-156 107 11 180 123 7 192 133 9 200 144 11
-206 145 10 200 144 11 192 133 9 175 118 6
-139 102 15 109 106 95 70 70 70 42 42 42
- 22 22 22 10 10 10 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 6 6 6 10 10 10
- 14 14 14 22 22 22 30 30 30 38 38 38
- 50 50 50 62 62 62 74 74 74 90 90 90
-101 98 89 112 100 78 121 87 25 124 80 6
-137 92 6 152 99 6 152 99 6 152 99 6
-138 86 6 124 80 6 98 70 6 86 66 30
-101 98 89 82 82 82 58 58 58 46 46 46
- 38 38 38 34 34 34 34 34 34 34 34 34
- 34 34 34 34 34 34 34 34 34 34 34 34
- 34 34 34 34 34 34 38 38 38 42 42 42
- 54 54 54 82 82 82 94 86 76 91 60 6
-134 86 6 156 107 11 167 114 7 175 118 6
-175 118 6 167 114 7 152 99 6 121 87 25
-101 98 89 62 62 62 34 34 34 18 18 18
- 6 6 6 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 6 6 6 6 6 6 10 10 10
- 18 18 18 22 22 22 30 30 30 42 42 42
- 50 50 50 66 66 66 86 86 86 101 98 89
-106 86 58 98 70 6 104 69 6 104 69 6
-104 69 6 91 60 6 82 62 34 90 90 90
- 62 62 62 38 38 38 22 22 22 14 14 14
- 10 10 10 10 10 10 10 10 10 10 10 10
- 10 10 10 10 10 10 6 6 6 10 10 10
- 10 10 10 10 10 10 10 10 10 14 14 14
- 22 22 22 42 42 42 70 70 70 89 81 66
- 80 54 7 104 69 6 124 80 6 137 92 6
-134 86 6 116 81 8 100 82 52 86 86 86
- 58 58 58 30 30 30 14 14 14 6 6 6
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 6 6 6 10 10 10 14 14 14
- 18 18 18 26 26 26 38 38 38 54 54 54
- 70 70 70 86 86 86 94 86 76 89 81 66
- 89 81 66 86 86 86 74 74 74 50 50 50
- 30 30 30 14 14 14 6 6 6 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 6 6 6 18 18 18 34 34 34 58 58 58
- 82 82 82 89 81 66 89 81 66 89 81 66
- 94 86 66 94 86 76 74 74 74 50 50 50
- 26 26 26 14 14 14 6 6 6 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 6 6 6 6 6 6 14 14 14 18 18 18
- 30 30 30 38 38 38 46 46 46 54 54 54
- 50 50 50 42 42 42 30 30 30 18 18 18
- 10 10 10 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 6 6 6 14 14 14 26 26 26
- 38 38 38 50 50 50 58 58 58 58 58 58
- 54 54 54 42 42 42 30 30 30 18 18 18
- 10 10 10 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 6 6 6
- 6 6 6 10 10 10 14 14 14 18 18 18
- 18 18 18 14 14 14 10 10 10 6 6 6
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 6 6 6
- 14 14 14 18 18 18 22 22 22 22 22 22
- 18 18 18 14 14 14 10 10 10 6 6 6
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
- 0 0 0 0 0 0 0 0 0 0 0 0
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 3 3 3 0 0 0 0 0 0
+0 0 0 0 0 0 0 0 0 0 0 0 3 3 3 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 1 1 1 0 0 0
+0 0 0 3 3 3 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 2 1 0 2 1 0 3 2 2
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 2 2 2 0 0 0 3 4 3 26 28 28
+37 38 37 37 38 37 14 17 19 2 2 2 0 0 0 2 2 2
+5 5 5 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 3 3 3 0 0 0 1 1 1 6 6 6
+2 2 2 0 0 0 3 3 3 4 4 4 4 4 4 4 4 4
+4 4 5 3 3 3 1 0 0 0 0 0 1 0 0 0 0 0
+1 1 1 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+2 2 2 0 0 0 0 0 0 14 17 19 60 74 84 137 136 137
+153 152 153 137 136 137 125 124 125 60 73 81 6 6 6 3 1 0
+0 0 0 3 3 3 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 0 0 0 4 4 4 41 54 63 125 124 125
+60 73 81 6 6 6 4 0 0 3 3 3 4 4 4 4 4 4
+4 4 4 0 0 0 6 9 11 41 54 63 41 65 82 22 30 35
+2 2 2 2 1 0 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 5 5 5 5 5 5 2 2 2 0 0 0
+4 0 0 6 6 6 41 54 63 137 136 137 174 174 174 167 166 167
+165 164 165 165 164 165 163 162 163 163 162 163 125 124 125 41 54 63
+1 1 1 0 0 0 0 0 0 3 3 3 5 5 5 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 5 5 5 5 5 5
+3 3 3 2 0 0 4 0 0 60 73 81 156 155 156 167 166 167
+163 162 163 85 115 134 5 7 8 0 0 0 4 4 4 5 5 5
+0 0 0 2 5 5 55 98 126 90 154 193 90 154 193 72 125 159
+37 51 59 2 0 0 1 1 1 4 5 5 4 4 4 4 4 4
+4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 5 5 5 4 4 4 1 1 1 0 0 0 3 3 3
+37 38 37 125 124 125 163 162 163 174 174 174 158 157 158 158 157 158
+156 155 156 156 155 156 158 157 158 165 164 165 174 174 174 166 165 166
+125 124 125 16 19 21 1 0 0 0 0 0 0 0 0 4 4 4
+5 5 5 5 5 5 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 5 5 5 5 5 5 1 1 1
+0 0 0 0 0 0 37 38 37 153 152 153 174 174 174 158 157 158
+174 174 174 163 162 163 37 38 37 4 3 3 4 0 0 1 1 1
+0 0 0 22 40 52 101 161 196 101 161 196 90 154 193 101 161 196
+64 123 161 14 17 19 0 0 0 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 5 5 5
+5 5 5 2 2 2 0 0 0 4 0 0 24 26 27 85 115 134
+156 155 156 174 174 174 167 166 167 156 155 156 154 153 154 157 156 157
+156 155 156 156 155 156 155 154 155 153 152 153 158 157 158 167 166 167
+174 174 174 156 155 156 60 74 84 16 19 21 0 0 0 0 0 0
+1 1 1 5 5 5 5 5 5 4 4 4 4 4 4 4 4 4
+4 4 4 5 5 5 6 6 6 3 3 3 0 0 0 4 0 0
+13 16 17 60 73 81 137 136 137 165 164 165 156 155 156 153 152 153
+174 174 174 177 184 187 60 73 81 3 1 0 0 0 0 1 1 2
+22 30 35 64 123 161 136 185 209 90 154 193 90 154 193 90 154 193
+90 154 193 21 29 34 0 0 0 3 2 2 4 4 5 4 4 4
+4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 5 5 5 3 3 3
+0 0 0 0 0 0 10 13 16 60 74 84 157 156 157 174 174 174
+174 174 174 158 157 158 153 152 153 154 153 154 156 155 156 155 154 155
+156 155 156 155 154 155 154 153 154 157 156 157 154 153 154 153 152 153
+163 162 163 174 174 174 177 184 187 137 136 137 60 73 81 13 16 17
+4 0 0 0 0 0 3 3 3 5 5 5 4 4 4 4 4 4
+5 5 5 4 4 4 1 1 1 0 0 0 3 3 3 41 54 63
+131 129 131 174 174 174 174 174 174 174 174 174 167 166 167 174 174 174
+190 197 201 137 136 137 24 26 27 4 0 0 16 21 25 50 82 103
+90 154 193 136 185 209 90 154 193 101 161 196 101 161 196 101 161 196
+31 91 132 3 6 7 0 0 0 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 2 2 2 0 0 0 4 0 0
+4 0 0 43 57 68 137 136 137 177 184 187 174 174 174 163 162 163
+155 154 155 155 154 155 156 155 156 155 154 155 158 157 158 165 164 165
+167 166 167 166 165 166 163 162 163 157 156 157 155 154 155 155 154 155
+153 152 153 156 155 156 167 166 167 174 174 174 174 174 174 131 129 131
+41 54 63 5 5 5 0 0 0 0 0 0 3 3 3 4 4 4
+1 1 1 0 0 0 1 0 0 26 28 28 125 124 125 174 174 174
+177 184 187 174 174 174 174 174 174 156 155 156 131 129 131 137 136 137
+125 124 125 24 26 27 4 0 0 41 65 82 90 154 193 136 185 209
+136 185 209 101 161 196 53 118 160 37 112 160 90 154 193 34 86 122
+7 12 15 0 0 0 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 3 3 3 0 0 0 0 0 0 5 5 5 37 38 37
+125 124 125 167 166 167 174 174 174 167 166 167 158 157 158 155 154 155
+156 155 156 156 155 156 156 155 156 163 162 163 167 166 167 155 154 155
+137 136 137 153 152 153 156 155 156 165 164 165 163 162 163 156 155 156
+156 155 156 156 155 156 155 154 155 158 157 158 166 165 166 174 174 174
+167 166 167 125 124 125 37 38 37 1 0 0 0 0 0 0 0 0
+0 0 0 24 26 27 60 74 84 158 157 158 174 174 174 174 174 174
+166 165 166 158 157 158 125 124 125 41 54 63 13 16 17 6 6 6
+6 6 6 37 38 37 80 127 157 136 185 209 101 161 196 101 161 196
+90 154 193 28 67 93 6 10 14 13 20 25 13 20 25 6 10 14
+1 1 2 4 3 3 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+1 1 1 1 0 0 4 3 3 37 38 37 60 74 84 153 152 153
+167 166 167 167 166 167 158 157 158 154 153 154 155 154 155 156 155 156
+157 156 157 158 157 158 167 166 167 167 166 167 131 129 131 43 57 68
+26 28 28 37 38 37 60 73 81 131 129 131 165 164 165 166 165 166
+158 157 158 155 154 155 156 155 156 156 155 156 156 155 156 158 157 158
+165 164 165 174 174 174 163 162 163 60 74 84 16 19 21 13 16 17
+60 73 81 131 129 131 174 174 174 174 174 174 167 166 167 165 164 165
+137 136 137 60 73 81 24 26 27 4 0 0 4 0 0 16 19 21
+52 104 138 101 161 196 136 185 209 136 185 209 90 154 193 27 99 146
+13 20 25 4 5 7 2 5 5 4 5 7 1 1 2 0 0 0
+4 4 4 4 4 4 3 3 3 2 2 2 2 2 2 4 4 4
+4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 3 3 3 0 0 0
+0 0 0 13 16 17 60 73 81 137 136 137 174 174 174 166 165 166
+158 157 158 156 155 156 157 156 157 156 155 156 155 154 155 158 157 158
+167 166 167 174 174 174 153 152 153 60 73 81 16 19 21 4 0 0
+4 0 0 4 0 0 6 6 6 26 28 28 60 74 84 158 157 158
+174 174 174 166 165 166 157 156 157 155 154 155 156 155 156 156 155 156
+155 154 155 158 157 158 167 166 167 167 166 167 131 129 131 125 124 125
+137 136 137 167 166 167 167 166 167 174 174 174 158 157 158 125 124 125
+16 19 21 4 0 0 4 0 0 10 13 16 49 76 92 107 159 188
+136 185 209 136 185 209 90 154 193 26 108 161 22 40 52 6 10 14
+2 3 3 1 1 2 1 1 2 4 4 5 4 4 5 4 4 5
+4 4 5 2 2 1 0 0 0 0 0 0 0 0 0 2 2 2
+4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 5 5 5 3 3 3 0 0 0 1 0 0 4 0 0
+37 51 59 131 129 131 167 166 167 167 166 167 163 162 163 157 156 157
+157 156 157 155 154 155 153 152 153 157 156 157 167 166 167 174 174 174
+153 152 153 125 124 125 37 38 37 4 0 0 4 0 0 4 0 0
+4 3 3 4 3 3 4 0 0 6 6 6 4 0 0 37 38 37
+125 124 125 174 174 174 174 174 174 165 164 165 156 155 156 154 153 154
+156 155 156 156 155 156 155 154 155 163 162 163 158 157 158 163 162 163
+174 174 174 174 174 174 174 174 174 125 124 125 37 38 37 0 0 0
+4 0 0 6 9 11 41 54 63 90 154 193 136 185 209 146 190 211
+136 185 209 37 112 160 22 40 52 6 10 14 3 6 7 1 1 2
+1 1 2 3 3 3 1 1 2 3 3 3 4 4 4 4 4 4
+2 2 2 2 0 0 16 19 21 37 38 37 24 26 27 0 0 0
+0 0 0 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 5 5 5 5 5 5
+4 4 4 0 0 0 0 0 0 0 0 0 26 28 28 120 125 127
+158 157 158 174 174 174 165 164 165 157 156 157 155 154 155 156 155 156
+153 152 153 153 152 153 167 166 167 174 174 174 174 174 174 125 124 125
+37 38 37 4 0 0 0 0 0 4 0 0 4 3 3 4 4 4
+4 4 4 4 4 4 5 5 5 4 0 0 4 0 0 4 0 0
+4 3 3 43 57 68 137 136 137 174 174 174 174 174 174 165 164 165
+154 153 154 153 152 153 153 152 153 153 152 153 163 162 163 174 174 174
+174 174 174 153 152 153 60 73 81 6 6 6 4 0 0 4 3 3
+32 43 50 80 127 157 136 185 209 146 190 211 146 190 211 90 154 193
+28 67 93 28 67 93 40 71 93 3 6 7 1 1 2 2 5 5
+50 82 103 79 117 143 26 37 45 0 0 0 3 3 3 1 1 1
+0 0 0 41 54 63 137 136 137 174 174 174 153 152 153 60 73 81
+2 0 0 0 0 0
+4 4 4 4 4 4 4 4 4 4 4 4 6 6 6 2 2 2
+0 0 0 2 0 0 24 26 27 60 74 84 153 152 153 174 174 174
+174 174 174 157 156 157 154 153 154 156 155 156 154 153 154 153 152 153
+165 164 165 174 174 174 177 184 187 137 136 137 43 57 68 6 6 6
+4 0 0 2 0 0 3 3 3 5 5 5 5 5 5 4 4 4
+4 4 4 4 4 4 4 4 4 5 5 5 6 6 6 4 3 3
+4 0 0 4 0 0 24 26 27 60 73 81 153 152 153 174 174 174
+174 174 174 158 157 158 158 157 158 174 174 174 174 174 174 158 157 158
+60 74 84 24 26 27 4 0 0 4 0 0 17 23 27 59 113 148
+136 185 209 191 222 234 146 190 211 136 185 209 31 91 132 7 11 13
+22 40 52 101 161 196 90 154 193 6 9 11 3 4 4 43 95 132
+136 185 209 172 205 220 55 98 126 0 0 0 0 0 0 2 0 0
+26 28 28 153 152 153 177 184 187 167 166 167 177 184 187 165 164 165
+37 38 37 0 0 0
+4 4 4 4 4 4 5 5 5 5 5 5 1 1 1 0 0 0
+13 16 17 60 73 81 137 136 137 174 174 174 174 174 174 165 164 165
+153 152 153 153 152 153 155 154 155 154 153 154 158 157 158 174 174 174
+177 184 187 163 162 163 60 73 81 16 19 21 4 0 0 4 0 0
+4 3 3 4 4 4 5 5 5 5 5 5 4 4 4 5 5 5
+5 5 5 5 5 5 5 5 5 4 4 4 4 4 4 5 5 5
+6 6 6 4 0 0 4 0 0 4 0 0 24 26 27 60 74 84
+166 165 166 174 174 174 177 184 187 165 164 165 125 124 125 24 26 27
+4 0 0 4 0 0 5 5 5 50 82 103 136 185 209 172 205 220
+146 190 211 136 185 209 26 108 161 22 40 52 7 12 15 44 81 103
+71 116 144 28 67 93 37 51 59 41 65 82 100 139 164 101 161 196
+90 154 193 90 154 193 28 67 93 0 0 0 0 0 0 26 28 28
+125 124 125 167 166 167 163 162 163 153 152 153 163 162 163 174 174 174
+85 115 134 4 0 0
+4 4 4 5 5 5 4 4 4 1 0 0 4 0 0 34 47 55
+125 124 125 174 174 174 174 174 174 167 166 167 157 156 157 153 152 153
+155 154 155 155 154 155 158 157 158 166 165 166 167 166 167 154 153 154
+125 124 125 26 28 28 4 0 0 4 0 0 4 0 0 5 5 5
+5 5 5 4 4 4 4 4 4 4 4 4 4 4 4 1 1 1
+0 0 0 0 0 0 1 1 1 4 4 4 4 4 4 4 4 4
+5 5 5 5 5 5 4 3 3 4 0 0 4 0 0 6 6 6
+37 38 37 131 129 131 137 136 137 37 38 37 0 0 0 4 0 0
+4 5 5 43 61 72 90 154 193 172 205 220 146 190 211 136 185 209
+90 154 193 28 67 93 13 20 25 43 61 72 71 116 144 44 81 103
+2 5 5 7 11 13 59 113 148 101 161 196 90 154 193 28 67 93
+13 20 25 6 10 14 0 0 0 13 16 17 60 73 81 137 136 137
+166 165 166 158 157 158 156 155 156 154 153 154 167 166 167 174 174 174
+60 73 81 4 0 0
+4 4 4 4 4 4 0 0 0 3 3 3 60 74 84 174 174 174
+174 174 174 167 166 167 163 162 163 155 154 155 157 156 157 155 154 155
+156 155 156 163 162 163 167 166 167 158 157 158 125 124 125 37 38 37
+4 3 3 4 0 0 4 0 0 6 6 6 6 6 6 5 5 5
+4 4 4 4 4 4 4 4 4 1 1 1 0 0 0 2 3 3
+10 13 16 7 11 13 1 0 0 0 0 0 2 2 1 4 4 4
+4 4 4 4 4 4 4 4 4 5 5 5 4 3 3 4 0 0
+4 0 0 7 11 13 13 16 17 4 0 0 3 3 3 34 47 55
+80 127 157 146 190 211 172 205 220 136 185 209 136 185 209 136 185 209
+28 67 93 22 40 52 55 98 126 55 98 126 21 29 34 7 11 13
+50 82 103 101 161 196 101 161 196 35 83 115 13 20 25 2 2 1
+1 1 2 1 1 2 37 51 59 131 129 131 174 174 174 174 174 174
+167 166 167 163 162 163 163 162 163 167 166 167 174 174 174 125 124 125
+16 19 21 4 0 0
+4 4 4 4 0 0 4 0 0 60 74 84 174 174 174 174 174 174
+158 157 158 155 154 155 155 154 155 156 155 156 155 154 155 158 157 158
+167 166 167 165 164 165 131 129 131 60 73 81 13 16 17 4 0 0
+4 0 0 4 3 3 6 6 6 4 3 3 5 5 5 4 4 4
+4 4 4 3 2 2 0 0 0 0 0 0 7 11 13 45 69 86
+80 127 157 71 116 144 43 61 72 7 11 13 0 0 0 1 1 1
+4 3 3 4 4 4 4 4 4 4 4 4 6 6 6 5 5 5
+3 2 2 4 0 0 1 0 0 21 29 34 59 113 148 136 185 209
+146 190 211 136 185 209 136 185 209 136 185 209 136 185 209 136 185 209
+68 124 159 44 81 103 22 40 52 13 16 17 43 61 72 90 154 193
+136 185 209 59 113 148 21 29 34 3 4 3 1 1 1 0 0 0
+24 26 27 125 124 125 163 162 163 174 174 174 166 165 166 165 164 165
+163 162 163 125 124 125 125 124 125 125 124 125 125 124 125 26 28 28
+4 0 0 4 3 3
+3 3 3 0 0 0 24 26 27 153 152 153 177 184 187 158 157 158
+156 155 156 156 155 156 155 154 155 155 154 155 165 164 165 174 174 174
+155 154 155 60 74 84 26 28 28 4 0 0 4 0 0 3 1 0
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 3 3
+2 0 0 0 0 0 0 0 0 32 43 50 72 125 159 101 161 196
+136 185 209 101 161 196 101 161 196 79 117 143 32 43 50 0 0 0
+0 0 0 2 2 2 4 4 4 4 4 4 3 3 3 1 0 0
+0 0 0 4 5 5 49 76 92 101 161 196 146 190 211 146 190 211
+136 185 209 136 185 209 136 185 209 136 185 209 136 185 209 90 154 193
+28 67 93 13 16 17 37 51 59 80 127 157 136 185 209 90 154 193
+22 40 52 6 9 11 3 4 3 2 2 1 16 19 21 60 73 81
+137 136 137 163 162 163 158 157 158 166 165 166 167 166 167 153 152 153
+60 74 84 37 38 37 6 6 6 13 16 17 4 0 0 1 0 0
+3 2 2 4 4 4
+3 2 2 4 0 0 37 38 37 137 136 137 167 166 167 158 157 158
+157 156 157 154 153 154 157 156 157 167 166 167 174 174 174 125 124 125
+37 38 37 4 0 0 4 0 0 4 0 0 4 3 3 4 4 4
+4 4 4 4 4 4 5 5 5 5 5 5 1 1 1 0 0 0
+0 0 0 16 21 25 55 98 126 90 154 193 136 185 209 101 161 196
+101 161 196 101 161 196 136 185 209 136 185 209 101 161 196 55 98 126
+14 17 19 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
+22 40 52 90 154 193 146 190 211 146 190 211 136 185 209 136 185 209
+136 185 209 136 185 209 136 185 209 101 161 196 35 83 115 7 11 13
+17 23 27 59 113 148 136 185 209 101 161 196 34 86 122 7 12 15
+2 5 5 3 4 3 6 6 6 60 73 81 131 129 131 163 162 163
+166 165 166 174 174 174 174 174 174 163 162 163 125 124 125 41 54 63
+13 16 17 4 0 0 4 0 0 4 0 0 1 0 0 2 2 2
+4 4 4 4 4 4
+1 1 1 2 1 0 43 57 68 137 136 137 153 152 153 153 152 153
+163 162 163 156 155 156 165 164 165 167 166 167 60 74 84 6 6 6
+4 0 0 4 0 0 5 5 5 4 4 4 4 4 4 4 4 4
+4 5 5 6 6 6 4 3 3 0 0 0 0 0 0 11 15 18
+40 71 93 100 139 164 101 161 196 101 161 196 101 161 196 101 161 196
+101 161 196 101 161 196 101 161 196 101 161 196 136 185 209 136 185 209
+101 161 196 45 69 86 6 6 6 0 0 0 17 23 27 55 98 126
+136 185 209 146 190 211 136 185 209 136 185 209 136 185 209 136 185 209
+136 185 209 136 185 209 90 154 193 22 40 52 7 11 13 50 82 103
+136 185 209 136 185 209 53 118 160 22 40 52 7 11 13 2 5 5
+3 4 3 37 38 37 125 124 125 157 156 157 166 165 166 167 166 167
+174 174 174 174 174 174 137 136 137 60 73 81 4 0 0 4 0 0
+4 0 0 4 0 0 5 5 5 3 3 3 3 3 3 4 4 4
+4 4 4 4 4 4
+4 0 0 4 0 0 41 54 63 137 136 137 125 124 125 131 129 131
+155 154 155 167 166 167 174 174 174 60 74 84 6 6 6 4 0 0
+4 3 3 6 6 6 4 4 4 4 4 4 4 4 4 5 5 5
+4 4 4 1 1 1 0 0 0 3 6 7 41 65 82 72 125 159
+101 161 196 101 161 196 101 161 196 90 154 193 90 154 193 101 161 196
+101 161 196 101 161 196 101 161 196 101 161 196 101 161 196 136 185 209
+136 185 209 136 185 209 80 127 157 55 98 126 101 161 196 146 190 211
+136 185 209 136 185 209 136 185 209 101 161 196 136 185 209 101 161 196
+136 185 209 101 161 196 35 83 115 22 30 35 101 161 196 172 205 220
+90 154 193 28 67 93 7 11 13 2 5 5 3 4 3 13 16 17
+85 115 134 167 166 167 174 174 174 174 174 174 174 174 174 174 174 174
+167 166 167 60 74 84 13 16 17 4 0 0 4 0 0 4 3 3
+6 6 6 5 5 5 4 4 4 5 5 5 4 4 4 5 5 5
+5 5 5 5 5 5
+1 1 1 4 0 0 41 54 63 137 136 137 137 136 137 125 124 125
+131 129 131 167 166 167 157 156 157 37 38 37 6 6 6 4 0 0
+6 6 6 5 5 5 4 4 4 4 4 4 4 5 5 2 2 1
+0 0 0 0 0 0 26 37 45 58 111 146 101 161 196 101 161 196
+101 161 196 90 154 193 90 154 193 90 154 193 101 161 196 101 161 196
+101 161 196 101 161 196 101 161 196 101 161 196 101 161 196 101 161 196
+101 161 196 136 185 209 136 185 209 136 185 209 146 190 211 136 185 209
+136 185 209 101 161 196 136 185 209 136 185 209 101 161 196 136 185 209
+101 161 196 136 185 209 136 185 209 136 185 209 136 185 209 16 89 141
+7 11 13 2 5 5 2 5 5 13 16 17 60 73 81 154 154 154
+174 174 174 174 174 174 174 174 174 174 174 174 163 162 163 125 124 125
+24 26 27 4 0 0 4 0 0 4 0 0 5 5 5 5 5 5
+4 4 4 4 4 4 4 4 4 5 5 5 5 5 5 5 5 5
+5 5 5 4 4 4
+4 0 0 6 6 6 37 38 37 137 136 137 137 136 137 131 129 131
+131 129 131 153 152 153 131 129 131 26 28 28 4 0 0 4 3 3
+6 6 6 4 4 4 4 4 4 4 4 4 0 0 0 0 0 0
+13 20 25 51 88 114 90 154 193 101 161 196 101 161 196 90 154 193
+90 154 193 90 154 193 90 154 193 90 154 193 90 154 193 101 161 196
+101 161 196 101 161 196 101 161 196 101 161 196 136 185 209 101 161 196
+101 161 196 136 185 209 101 161 196 136 185 209 136 185 209 101 161 196
+136 185 209 101 161 196 136 185 209 101 161 196 101 161 196 101 161 196
+136 185 209 136 185 209 136 185 209 37 112 160 21 29 34 5 7 8
+2 5 5 13 16 17 43 57 68 131 129 131 174 174 174 174 174 174
+174 174 174 167 166 167 157 156 157 125 124 125 37 38 37 4 0 0
+4 0 0 4 0 0 5 5 5 5 5 5 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+1 1 1 4 0 0 41 54 63 153 152 153 137 136 137 137 136 137
+137 136 137 153 152 153 125 124 125 24 26 27 4 0 0 3 2 2
+4 4 4 4 4 4 4 3 3 4 0 0 3 6 7 43 61 72
+64 123 161 101 161 196 90 154 193 90 154 193 90 154 193 90 154 193
+90 154 193 90 154 193 90 154 193 90 154 193 101 161 196 90 154 193
+101 161 196 101 161 196 101 161 196 101 161 196 101 161 196 101 161 196
+101 161 196 101 161 196 101 161 196 101 161 196 101 161 196 101 161 196
+136 185 209 101 161 196 101 161 196 136 185 209 136 185 209 101 161 196
+101 161 196 90 154 193 28 67 93 13 16 17 7 11 13 3 6 7
+37 51 59 125 124 125 163 162 163 174 174 174 167 166 167 166 165 166
+167 166 167 131 129 131 60 73 81 4 0 0 4 0 0 4 0 0
+3 3 3 5 5 5 6 6 6 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 0 0 4 0 0 41 54 63 137 136 137 153 152 153 137 136 137
+153 152 153 157 156 157 125 124 125 24 26 27 0 0 0 2 2 2
+4 4 4 4 4 4 2 0 0 0 0 0 28 67 93 90 154 193
+90 154 193 90 154 193 90 154 193 90 154 193 64 123 161 90 154 193
+90 154 193 90 154 193 90 154 193 90 154 193 90 154 193 101 161 196
+90 154 193 101 161 196 101 161 196 101 161 196 90 154 193 136 185 209
+101 161 196 101 161 196 136 185 209 101 161 196 136 185 209 101 161 196
+101 161 196 101 161 196 136 185 209 101 161 196 101 161 196 90 154 193
+35 83 115 13 16 17 3 6 7 2 5 5 13 16 17 60 74 84
+154 154 154 166 165 166 165 164 165 158 157 158 163 162 163 157 156 157
+60 74 84 13 16 17 4 0 0 4 0 0 3 2 2 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+1 1 1 4 0 0 41 54 63 157 156 157 155 154 155 137 136 137
+153 152 153 158 157 158 137 136 137 26 28 28 2 0 0 2 2 2
+4 4 4 4 4 4 1 0 0 6 10 14 34 86 122 90 154 193
+64 123 161 90 154 193 64 123 161 90 154 193 90 154 193 90 154 193
+64 123 161 90 154 193 90 154 193 90 154 193 90 154 193 90 154 193
+101 161 196 101 161 196 101 161 196 101 161 196 101 161 196 101 161 196
+101 161 196 101 161 196 101 161 196 101 161 196 101 161 196 101 161 196
+136 185 209 101 161 196 136 185 209 90 154 193 26 108 161 22 40 52
+13 16 17 5 7 8 2 5 5 2 5 5 37 38 37 165 164 165
+174 174 174 163 162 163 154 154 154 165 164 165 167 166 167 60 73 81
+6 6 6 4 0 0 4 0 0 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 0 0 6 6 6 41 54 63 156 155 156 158 157 158 153 152 153
+156 155 156 165 164 165 137 136 137 26 28 28 0 0 0 2 2 2
+4 4 5 4 4 4 2 0 0 7 12 15 31 96 139 64 123 161
+90 154 193 64 123 161 90 154 193 90 154 193 64 123 161 90 154 193
+90 154 193 90 154 193 90 154 193 90 154 193 90 154 193 90 154 193
+90 154 193 90 154 193 90 154 193 101 161 196 101 161 196 101 161 196
+101 161 196 101 161 196 101 161 196 101 161 196 101 161 196 136 185 209
+101 161 196 136 185 209 26 108 161 22 40 52 7 11 13 5 7 8
+2 5 5 2 5 5 2 5 5 2 2 1 37 38 37 158 157 158
+174 174 174 154 154 154 156 155 156 167 166 167 165 164 165 37 38 37
+4 0 0 4 3 3 5 5 5 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+3 1 0 4 0 0 60 73 81 157 156 157 163 162 163 153 152 153
+158 157 158 167 166 167 137 136 137 26 28 28 2 0 0 2 2 2
+4 5 5 4 4 4 4 0 0 7 12 15 24 86 132 26 108 161
+37 112 160 64 123 161 90 154 193 64 123 161 90 154 193 90 154 193
+90 154 193 90 154 193 90 154 193 90 154 193 90 154 193 90 154 193
+90 154 193 101 161 196 90 154 193 101 161 196 101 161 196 101 161 196
+101 161 196 101 161 196 101 161 196 136 185 209 101 161 196 136 185 209
+90 154 193 35 83 115 13 16 17 13 16 17 7 11 13 3 6 7
+5 7 8 6 6 6 3 4 3 2 2 1 30 32 34 154 154 154
+167 166 167 154 154 154 154 154 154 174 174 174 165 164 165 37 38 37
+6 6 6 4 0 0 6 6 6 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 0 0 4 0 0 41 54 63 163 162 163 166 165 166 154 154 154
+163 162 163 174 174 174 137 136 137 26 28 28 0 0 0 2 2 2
+4 5 5 4 4 5 1 1 2 6 10 14 28 67 93 18 97 151
+18 97 151 18 97 151 26 108 161 37 112 160 37 112 160 90 154 193
+64 123 161 90 154 193 90 154 193 90 154 193 90 154 193 101 161 196
+90 154 193 101 161 196 101 161 196 90 154 193 101 161 196 101 161 196
+101 161 196 101 161 196 101 161 196 136 185 209 90 154 193 16 89 141
+13 20 25 7 11 13 5 7 8 5 7 8 2 5 5 4 5 5
+3 4 3 4 5 5 3 4 3 0 0 0 37 38 37 158 157 158
+174 174 174 158 157 158 158 157 158 167 166 167 174 174 174 41 54 63
+4 0 0 3 2 2 5 5 5 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+1 1 1 4 0 0 60 73 81 165 164 165 174 174 174 158 157 158
+167 166 167 174 174 174 153 152 153 26 28 28 2 0 0 2 2 2
+4 5 5 4 4 4 4 0 0 7 12 15 10 87 144 10 87 144
+18 97 151 18 97 151 18 97 151 26 108 161 26 108 161 26 108 161
+26 108 161 37 112 160 53 118 160 90 154 193 90 154 193 90 154 193
+90 154 193 90 154 193 101 161 196 101 161 196 101 161 196 101 161 196
+101 161 196 136 185 209 90 154 193 26 108 161 22 40 52 13 16 17
+7 11 13 3 6 7 5 7 8 5 7 8 2 5 5 4 5 5
+4 5 5 6 6 6 3 4 3 0 0 0 30 32 34 158 157 158
+174 174 174 156 155 156 155 154 155 165 164 165 154 153 154 37 38 37
+4 0 0 4 3 3 5 5 5 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 0 0 4 0 0 60 73 81 167 166 167 174 174 174 163 162 163
+174 174 174 174 174 174 153 152 153 26 28 28 0 0 0 3 3 3
+5 5 5 4 4 4 1 1 2 7 12 15 28 67 93 18 97 151
+18 97 151 18 97 151 18 97 151 18 97 151 18 97 151 26 108 161
+26 108 161 26 108 161 26 108 161 26 108 161 26 108 161 26 108 161
+90 154 193 26 108 161 90 154 193 90 154 193 90 154 193 101 161 196
+101 161 196 26 108 161 22 40 52 13 16 17 7 11 13 2 5 5
+2 5 5 6 6 6 2 5 5 4 5 5 4 5 5 4 5 5
+3 4 3 5 5 5 3 4 3 2 0 0 30 32 34 137 136 137
+153 152 153 137 136 137 131 129 131 137 136 137 131 129 131 37 38 37
+4 0 0 4 3 3 5 5 5 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+1 1 1 4 0 0 60 73 81 167 166 167 174 174 174 166 165 166
+174 174 174 177 184 187 153 152 153 30 32 34 1 0 0 3 3 3
+5 5 5 4 3 3 4 0 0 7 12 15 10 87 144 10 87 144
+18 97 151 18 97 151 18 97 151 26 108 161 26 108 161 26 108 161
+26 108 161 26 108 161 26 108 161 26 108 161 26 108 161 26 108 161
+26 108 161 26 108 161 26 108 161 90 154 193 90 154 193 26 108 161
+35 83 115 13 16 17 7 11 13 5 7 8 3 6 7 5 7 8
+2 5 5 6 6 6 4 5 5 4 5 5 3 4 3 4 5 5
+3 4 3 6 6 6 3 4 3 0 0 0 26 28 28 125 124 125
+131 129 131 125 124 125 125 124 125 131 129 131 131 129 131 37 38 37
+4 0 0 3 3 3 5 5 5 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+3 1 0 4 0 0 60 73 81 174 174 174 177 184 187 167 166 167
+174 174 174 177 184 187 153 152 153 30 32 34 0 0 0 3 3 3
+5 5 5 4 4 4 1 1 2 6 10 14 28 67 93 18 97 151
+18 97 151 18 97 151 18 97 151 18 97 151 18 97 151 26 108 161
+26 108 161 26 108 161 26 108 161 26 108 161 26 108 161 26 108 161
+26 108 161 90 154 193 26 108 161 26 108 161 24 86 132 13 20 25
+7 11 13 13 20 25 22 40 52 5 7 8 3 4 3 3 4 3
+4 5 5 3 4 3 4 5 5 3 4 3 4 5 5 3 4 3
+4 4 4 5 5 5 3 3 3 2 0 0 26 28 28 125 124 125
+137 136 137 125 124 125 125 124 125 137 136 137 131 129 131 37 38 37
+0 0 0 3 3 3 5 5 5 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+1 1 1 4 0 0 60 73 81 174 174 174 177 184 187 174 174 174
+174 174 174 190 197 201 157 156 157 30 32 34 1 0 0 3 3 3
+5 5 5 4 3 3 4 0 0 7 12 15 10 87 144 10 87 144
+18 97 151 19 95 150 19 95 150 18 97 151 18 97 151 26 108 161
+18 97 151 26 108 161 26 108 161 26 108 161 26 108 161 90 154 193
+26 108 161 26 108 161 26 108 161 22 40 52 2 5 5 3 4 3
+28 67 93 37 112 160 34 86 122 2 5 5 3 4 3 3 4 3
+3 4 3 3 4 3 3 4 3 2 2 1 3 4 3 4 4 4
+4 5 5 5 5 5 3 3 3 0 0 0 26 28 28 131 129 131
+137 136 137 125 124 125 125 124 125 137 136 137 131 129 131 37 38 37
+0 0 0 3 3 3 5 5 5 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 0 0 4 0 0 60 73 81 174 174 174 177 184 187 174 174 174
+174 174 174 190 197 201 158 157 158 30 32 34 0 0 0 2 2 2
+5 5 5 4 4 4 1 1 2 6 10 14 28 67 93 18 97 151
+10 87 144 19 95 150 19 95 150 18 97 151 18 97 151 18 97 151
+26 108 161 26 108 161 26 108 161 26 108 161 26 108 161 26 108 161
+18 97 151 22 40 52 2 5 5 2 2 1 22 40 52 26 108 161
+90 154 193 37 112 160 22 40 52 3 4 3 13 20 25 22 30 35
+3 6 7 1 1 1 2 2 2 6 9 11 5 5 5 4 3 3
+4 4 4 5 5 5 3 3 3 2 0 0 26 28 28 131 129 131
+137 136 137 125 124 125 125 124 125 137 136 137 131 129 131 37 38 37
+0 0 0 3 3 3 5 5 5 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+1 1 1 4 0 0 60 73 81 177 184 187 193 200 203 174 174 174
+177 184 187 193 200 203 163 162 163 30 32 34 4 0 0 2 2 2
+5 5 5 4 3 3 4 0 0 6 10 14 24 86 132 10 87 144
+10 87 144 10 87 144 19 95 150 19 95 150 19 95 150 18 97 151
+26 108 161 26 108 161 26 108 161 90 154 193 26 108 161 28 67 93
+6 10 14 2 5 5 13 20 25 24 86 132 37 112 160 90 154 193
+10 87 144 7 12 15 2 5 5 28 67 93 37 112 160 28 67 93
+2 2 1 7 12 15 35 83 115 28 67 93 3 6 7 1 0 0
+4 4 4 5 5 5 3 3 3 0 0 0 26 28 28 131 129 131
+137 136 137 125 124 125 125 124 125 137 136 137 131 129 131 37 38 37
+0 0 0 3 3 3 5 5 5 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 0 0 4 0 0 60 73 81 174 174 174 190 197 201 174 174 174
+177 184 187 193 200 203 163 162 163 30 32 34 0 0 0 2 2 2
+5 5 5 4 4 4 1 1 2 6 10 14 28 67 93 10 87 144
+10 87 144 16 89 141 19 95 150 10 87 144 26 108 161 26 108 161
+26 108 161 26 108 161 26 108 161 28 67 93 6 10 14 1 1 2
+7 12 15 28 67 93 26 108 161 16 89 141 24 86 132 21 29 34
+3 4 3 21 29 34 37 112 160 37 112 160 27 99 146 21 29 34
+21 29 34 26 108 161 90 154 193 35 83 115 1 1 2 2 0 0
+4 4 4 5 5 5 3 3 3 2 0 0 26 28 28 125 124 125
+137 136 137 125 124 125 125 124 125 137 136 137 131 129 131 37 38 37
+0 0 0 3 3 3 5 5 5 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+3 1 0 4 0 0 60 73 81 193 200 203 193 200 203 174 174 174
+190 197 201 193 200 203 165 164 165 37 38 37 4 0 0 2 2 2
+5 5 5 4 3 3 4 0 0 6 10 14 24 86 132 10 87 144
+10 87 144 10 87 144 16 89 141 18 97 151 18 97 151 10 87 144
+24 86 132 24 86 132 13 20 25 4 5 7 4 5 7 22 40 52
+18 97 151 37 112 160 26 108 161 7 12 15 1 1 1 0 0 0
+28 67 93 37 112 160 26 108 161 28 67 93 22 40 52 28 67 93
+26 108 161 90 154 193 26 108 161 10 87 144 0 0 0 2 0 0
+4 4 4 5 5 5 3 3 3 0 0 0 26 28 28 131 129 131
+137 136 137 125 124 125 125 124 125 137 136 137 131 129 131 37 38 37
+0 0 0 3 3 3 5 5 5 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 0 0 6 6 6 60 73 81 174 174 174 193 200 203 174 174 174
+190 197 201 193 200 203 165 164 165 30 32 34 0 0 0 2 2 2
+5 5 5 4 4 4 1 1 2 6 10 14 28 67 93 10 87 144
+10 87 144 10 87 144 10 87 144 18 97 151 28 67 93 6 10 14
+0 0 0 1 1 2 4 5 7 13 20 25 16 89 141 26 108 161
+26 108 161 26 108 161 24 86 132 6 9 11 2 3 3 22 40 52
+37 112 160 16 89 141 22 40 52 28 67 93 26 108 161 26 108 161
+90 154 193 26 108 161 26 108 161 28 67 93 1 1 1 4 0 0
+4 4 4 5 5 5 3 3 3 4 0 0 26 28 28 124 126 130
+137 136 137 125 124 125 125 124 125 137 136 137 131 129 131 37 38 37
+0 0 0 3 3 3 5 5 5 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 0 0 4 0 0 60 73 81 193 200 203 193 200 203 174 174 174
+193 200 203 193 200 203 167 166 167 37 38 37 4 0 0 2 2 2
+5 5 5 4 4 4 4 0 0 6 10 14 28 67 93 10 87 144
+10 87 144 10 87 144 18 97 151 10 87 144 13 20 25 4 5 7
+1 1 2 1 1 1 22 40 52 26 108 161 26 108 161 26 108 161
+26 108 161 26 108 161 26 108 161 24 86 132 22 40 52 22 40 52
+22 40 52 22 40 52 10 87 144 26 108 161 26 108 161 26 108 161
+26 108 161 26 108 161 90 154 193 10 87 144 0 0 0 4 0 0
+4 4 4 5 5 5 3 3 3 0 0 0 26 28 28 131 129 131
+137 136 137 125 124 125 125 124 125 137 136 137 131 129 131 37 38 37
+0 0 0 3 3 3 5 5 5 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 0 0 6 6 6 60 73 81 174 174 174 220 221 221 174 174 174
+190 197 201 205 212 215 167 166 167 30 32 34 0 0 0 2 2 2
+5 5 5 4 4 4 1 1 2 6 10 14 28 67 93 10 87 144
+10 87 144 10 87 144 10 87 144 10 87 144 22 40 52 1 1 2
+2 0 0 1 1 2 24 86 132 26 108 161 26 108 161 26 108 161
+26 108 161 19 95 150 16 89 141 10 87 144 22 40 52 22 40 52
+10 87 144 26 108 161 37 112 160 26 108 161 26 108 161 26 108 161
+26 108 161 26 108 161 26 108 161 28 67 93 2 0 0 3 1 0
+4 4 4 5 5 5 3 3 3 2 0 0 26 28 28 131 129 131
+137 136 137 125 124 125 125 124 125 137 136 137 131 129 131 37 38 37
+0 0 0 3 3 3 5 5 5 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 0 0 4 0 0 60 73 81 220 221 221 190 197 201 174 174 174
+193 200 203 193 200 203 174 174 174 37 38 37 4 0 0 2 2 2
+5 5 5 4 4 4 3 2 2 1 1 2 13 20 25 10 87 144
+10 87 144 10 87 144 10 87 144 10 87 144 10 87 144 13 20 25
+13 20 25 22 40 52 10 87 144 18 97 151 18 97 151 26 108 161
+10 87 144 13 20 25 6 10 14 21 29 34 24 86 132 18 97 151
+26 108 161 26 108 161 26 108 161 26 108 161 26 108 161 26 108 161
+26 108 161 90 154 193 18 97 151 13 20 25 0 0 0 4 3 3
+4 4 4 5 5 5 3 3 3 0 0 0 26 28 28 131 129 131
+137 136 137 125 124 125 125 124 125 137 136 137 131 129 131 37 38 37
+0 0 0 3 3 3 5 5 5 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 0 0 6 6 6 60 73 81 174 174 174 220 221 221 174 174 174
+190 197 201 220 221 221 167 166 167 30 32 34 1 0 0 2 2 2
+5 5 5 4 4 4 4 4 5 2 5 5 4 5 7 13 20 25
+28 67 93 10 87 144 10 87 144 10 87 144 10 87 144 10 87 144
+10 87 144 10 87 144 18 97 151 10 87 144 18 97 151 18 97 151
+28 67 93 2 3 3 0 0 0 28 67 93 26 108 161 26 108 161
+26 108 161 26 108 161 26 108 161 26 108 161 26 108 161 26 108 161
+26 108 161 10 87 144 13 20 25 1 1 2 3 2 2 4 4 4
+4 4 4 5 5 5 3 3 3 2 0 0 26 28 28 131 129 131
+137 136 137 125 124 125 125 124 125 137 136 137 131 129 131 37 38 37
+0 0 0 3 3 3 5 5 5 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 0 0 4 0 0 60 73 81 220 221 221 190 197 201 174 174 174
+193 200 203 193 200 203 174 174 174 26 28 28 4 0 0 4 3 3
+5 5 5 4 4 4 4 4 4 4 4 5 1 1 2 2 5 5
+4 5 7 22 40 52 10 87 144 10 87 144 18 97 151 10 87 144
+10 87 144 10 87 144 10 87 144 10 87 144 10 87 144 18 97 151
+10 87 144 28 67 93 22 40 52 10 87 144 26 108 161 18 97 151
+18 97 151 18 97 151 26 108 161 26 108 161 26 108 161 26 108 161
+22 40 52 1 1 2 0 0 0 2 3 3 4 4 4 4 4 4
+4 4 4 5 5 5 4 4 4 0 0 0 26 28 28 131 129 131
+137 136 137 125 124 125 125 124 125 137 136 137 131 129 131 37 38 37
+0 0 0 3 3 3 5 5 5 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 0 0 6 6 6 60 73 81 174 174 174 220 221 221 174 174 174
+190 197 201 220 221 221 190 197 201 41 54 63 4 0 0 2 2 2
+6 6 6 4 4 4 4 4 4 4 4 5 4 4 5 3 3 3
+1 1 2 1 1 2 6 10 14 22 40 52 10 87 144 18 97 151
+18 97 151 10 87 144 10 87 144 10 87 144 18 97 151 10 87 144
+10 87 144 18 97 151 26 108 161 18 97 151 18 97 151 10 87 144
+26 108 161 26 108 161 26 108 161 10 87 144 28 67 93 6 10 14
+1 1 2 1 1 2 4 3 3 4 4 5 4 4 4 4 4 4
+5 5 5 5 5 5 1 1 1 4 0 0 37 51 59 137 136 137
+137 136 137 125 124 125 125 124 125 137 136 137 131 129 131 37 38 37
+0 0 0 3 3 3 5 5 5 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 0 0 4 0 0 60 73 81 220 221 221 193 200 203 174 174 174
+193 200 203 193 200 203 220 221 221 137 136 137 13 16 17 4 0 0
+2 2 2 4 4 4 4 4 4 4 4 4 4 4 4 4 4 5
+4 4 5 4 3 3 1 1 2 4 5 7 13 20 25 28 67 93
+10 87 144 10 87 144 10 87 144 10 87 144 10 87 144 10 87 144
+10 87 144 18 97 151 18 97 151 10 87 144 18 97 151 26 108 161
+26 108 161 18 97 151 28 67 93 6 10 14 0 0 0 0 0 0
+2 3 3 4 5 5 4 4 5 4 4 4 4 4 4 5 5 5
+3 3 3 1 1 1 0 0 0 16 19 21 125 124 125 137 136 137
+131 129 131 125 124 125 125 124 125 137 136 137 131 129 131 37 38 37
+0 0 0 3 3 3 5 5 5 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 0 0 6 6 6 60 73 81 174 174 174 220 221 221 174 174 174
+193 200 203 190 197 201 220 221 221 220 221 221 153 152 153 30 32 34
+0 0 0 0 0 0 2 2 2 4 4 4 4 4 4 4 4 4
+4 4 4 4 5 5 4 5 7 1 1 2 1 1 2 4 5 7
+13 20 25 28 67 93 10 87 144 18 97 151 10 87 144 10 87 144
+10 87 144 10 87 144 10 87 144 18 97 151 26 108 161 18 97 151
+28 67 93 7 12 15 0 0 0 0 0 0 2 2 1 4 4 4
+4 5 5 4 5 5 4 4 4 4 4 4 3 3 3 0 0 0
+0 0 0 0 0 0 37 38 37 125 124 125 158 157 158 131 129 131
+125 124 125 125 124 125 125 124 125 137 136 137 131 129 131 37 38 37
+0 0 0 3 3 3 5 5 5 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 3 3 4 0 0 41 54 63 193 200 203 220 221 221 174 174 174
+193 200 203 193 200 203 193 200 203 220 221 221 244 246 246 193 200 203
+120 125 127 5 5 5 1 0 0 0 0 0 1 1 1 4 4 4
+4 4 4 4 4 4 4 5 5 4 5 5 4 4 5 1 1 2
+4 5 7 4 5 7 22 40 52 10 87 144 10 87 144 10 87 144
+10 87 144 10 87 144 18 97 151 10 87 144 10 87 144 13 20 25
+4 5 7 2 3 3 1 1 2 4 4 4 4 5 5 4 4 4
+4 4 4 4 4 4 4 4 4 1 1 1 0 0 0 1 1 2
+24 26 27 60 74 84 153 152 153 163 162 163 137 136 137 125 124 125
+125 124 125 125 124 125 125 124 125 137 136 137 125 124 125 26 28 28
+0 0 0 3 3 3 5 5 5 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 0 0 6 6 6 26 28 28 156 155 156 220 221 221 220 221 221
+174 174 174 193 200 203 193 200 203 193 200 203 205 212 215 220 221 221
+220 221 221 167 166 167 60 73 81 7 11 13 0 0 0 0 0 0
+3 3 3 4 4 4 4 4 4 4 4 4 4 4 5 4 4 5
+4 4 5 1 1 2 1 1 2 4 5 7 22 40 52 10 87 144
+10 87 144 10 87 144 10 87 144 22 40 52 4 5 7 1 1 2
+1 1 2 4 4 5 4 4 4 4 4 4 4 4 4 4 4 4
+5 5 5 2 2 2 0 0 0 4 0 0 16 19 21 60 73 81
+137 136 137 167 166 167 158 157 158 137 136 137 131 129 131 131 129 131
+125 124 125 125 124 125 131 129 131 155 154 155 60 74 84 5 7 8
+0 0 0 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+5 5 5 4 0 0 4 0 0 60 73 81 193 200 203 220 221 221
+193 200 203 193 200 203 193 200 203 193 200 203 205 212 215 220 221 221
+220 221 221 220 221 221 220 221 221 137 136 137 43 57 68 6 6 6
+4 0 0 1 1 1 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 5 4 4 5 3 2 2 1 1 2 2 5 5 13 20 25
+22 40 52 22 40 52 13 20 25 2 3 3 1 1 2 3 3 3
+4 5 7 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+1 1 1 0 0 0 2 3 3 41 54 63 131 129 131 166 165 166
+166 165 166 155 154 155 153 152 153 137 136 137 137 136 137 125 124 125
+125 124 125 137 136 137 137 136 137 125 124 125 37 38 37 4 3 3
+4 3 3 5 5 5 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 3 3 6 6 6 6 6 6 13 16 17 60 73 81 167 166 167
+220 221 221 220 221 221 220 221 221 193 200 203 193 200 203 193 200 203
+205 212 215 220 221 221 220 221 221 244 246 246 205 212 215 125 124 125
+24 26 27 0 0 0 0 0 0 2 2 2 5 5 5 5 5 5
+4 4 4 4 4 4 4 4 4 4 4 5 1 1 2 4 5 7
+4 5 7 4 5 7 1 1 2 3 2 2 4 4 5 4 4 4
+4 4 4 4 4 4 5 5 5 4 4 4 0 0 0 0 0 0
+2 0 0 26 28 28 125 124 125 174 174 174 174 174 174 166 165 166
+156 155 156 153 152 153 137 136 137 137 136 137 131 129 131 137 136 137
+137 136 137 137 136 137 60 74 84 30 32 34 4 0 0 4 0 0
+5 5 5 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+5 5 5 6 6 6 4 0 0 4 0 0 6 6 6 26 28 28
+125 124 125 174 174 174 220 221 221 220 221 221 220 221 221 193 200 203
+205 212 215 220 221 221 205 212 215 220 221 221 220 221 221 244 246 246
+193 200 203 60 74 84 13 16 17 4 0 0 0 0 0 3 3 3
+5 5 5 5 5 5 4 4 4 4 4 4 4 4 5 3 3 3
+1 1 2 3 3 3 4 4 5 4 4 5 4 4 4 4 4 4
+5 5 5 5 5 5 2 2 2 0 0 0 0 0 0 13 16 17
+60 74 84 174 174 174 193 200 203 174 174 174 167 166 167 163 162 163
+153 152 153 153 152 153 137 136 137 137 136 137 153 152 153 137 136 137
+125 124 125 41 54 63 24 26 27 4 0 0 4 0 0 5 5 5
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 3 3 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6
+6 6 6 37 38 37 131 129 131 220 221 221 220 221 221 220 221 221
+193 200 203 193 200 203 220 221 221 205 212 215 220 221 221 244 246 246
+244 246 246 244 246 246 174 174 174 41 54 63 0 0 0 0 0 0
+0 0 0 4 4 4 5 5 5 5 5 5 4 4 4 4 4 5
+4 4 5 4 4 5 4 4 4 4 4 4 6 6 6 6 6 6
+3 3 3 0 0 0 2 0 0 13 16 17 60 73 81 156 155 156
+220 221 221 193 200 203 174 174 174 165 164 165 163 162 163 154 153 154
+153 152 153 153 152 153 158 157 158 163 162 163 137 136 137 60 73 81
+13 16 17 4 0 0 4 0 0 4 3 3 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+5 5 5 4 3 3 4 3 3 6 6 6 6 6 6 6 6 6
+6 6 6 6 6 6 6 6 6 37 38 37 167 166 167 244 246 246
+244 246 246 220 221 221 205 212 215 205 212 215 220 221 221 193 200 203
+220 221 221 244 246 246 244 246 246 244 246 246 137 136 137 37 38 37
+3 2 2 0 0 0 1 1 1 5 5 5 5 5 5 4 4 4
+4 4 4 4 4 4 4 4 4 5 5 5 4 4 4 1 1 1
+0 0 0 5 5 5 43 57 68 153 152 153 193 200 203 220 221 221
+177 184 187 174 174 174 167 166 167 166 165 166 158 157 158 157 156 157
+158 157 158 166 165 166 156 155 156 85 115 134 13 16 17 4 0 0
+4 0 0 4 0 0 5 5 5 5 5 5 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+5 5 5 4 3 3 6 6 6 6 6 6 4 0 0 6 6 6
+6 6 6 6 6 6 6 6 6 6 6 6 13 16 17 60 73 81
+177 184 187 220 221 221 220 221 221 220 221 221 205 212 215 220 221 221
+220 221 221 205 212 215 220 221 221 244 246 246 244 246 246 205 212 215
+125 124 125 30 32 34 0 0 0 0 0 0 2 2 2 5 5 5
+4 4 4 4 4 4 4 4 4 1 1 1 0 0 0 1 0 0
+37 38 37 131 129 131 205 212 215 220 221 221 193 200 203 174 174 174
+174 174 174 174 174 174 167 166 167 165 164 165 166 165 166 167 166 167
+158 157 158 125 124 125 37 38 37 4 0 0 4 0 0 4 0 0
+4 3 3 5 5 5 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 4 4 5 5 5 4 3 3 4 3 3 6 6 6 6 6 6
+4 0 0 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6
+26 28 28 125 124 125 205 212 215 220 221 221 220 221 221 220 221 221
+205 212 215 220 221 221 205 212 215 220 221 221 220 221 221 244 246 246
+244 246 246 190 197 201 60 74 84 16 19 21 4 0 0 0 0 0
+0 0 0 0 0 0 0 0 0 0 0 0 16 19 21 120 125 127
+177 184 187 220 221 221 205 212 215 177 184 187 174 174 174 177 184 187
+174 174 174 174 174 174 167 166 167 174 174 174 166 165 166 137 136 137
+60 73 81 13 16 17 4 0 0 4 0 0 4 3 3 6 6 6
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+5 5 5 4 3 3 5 5 5 4 3 3 6 6 6 4 0 0
+6 6 6 6 6 6 4 0 0 6 6 6 4 0 0 6 6 6
+6 6 6 6 6 6 37 38 37 137 136 137 193 200 203 220 221 221
+220 221 221 205 212 215 220 221 221 205 212 215 205 212 215 220 221 221
+220 221 221 220 221 221 244 246 246 166 165 166 43 57 68 2 2 2
+0 0 0 4 0 0 16 19 21 60 73 81 157 156 157 202 210 214
+220 221 221 193 200 203 177 184 187 177 184 187 177 184 187 174 174 174
+174 174 174 174 174 174 174 174 174 157 156 157 60 74 84 24 26 27
+4 0 0 4 0 0 4 0 0 6 6 6 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 4 4 4 4 4 5 5 5 4 3 3 5 5 5 6 6 6
+6 6 6 4 0 0 6 6 6 6 6 6 6 6 6 4 0 0
+4 0 0 4 0 0 6 6 6 24 26 27 60 73 81 167 166 167
+220 221 221 220 221 221 220 221 221 205 212 215 205 212 215 205 212 215
+205 212 215 220 221 221 220 221 221 220 221 221 205 212 215 137 136 137
+60 74 84 125 124 125 137 136 137 190 197 201 220 221 221 193 200 203
+177 184 187 177 184 187 177 184 187 174 174 174 174 174 174 177 184 187
+190 197 201 174 174 174 125 124 125 37 38 37 6 6 6 4 0 0
+4 0 0 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 4 4 4 4 4 5 5 5 5 5 5 4 3 3 6 6 6
+4 0 0 6 6 6 6 6 6 6 6 6 4 0 0 6 6 6
+6 6 6 6 6 6 4 0 0 4 0 0 6 6 6 6 6 6
+125 124 125 193 200 203 244 246 246 220 221 221 205 212 215 205 212 215
+205 212 215 193 200 203 205 212 215 205 212 215 220 221 221 220 221 221
+193 200 203 193 200 203 205 212 215 193 200 203 193 200 203 177 184 187
+190 197 201 190 197 201 174 174 174 190 197 201 193 200 203 190 197 201
+153 152 153 60 73 81 4 0 0 4 0 0 4 0 0 3 2 2
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 5 5 5 4 3 3
+6 6 6 4 3 3 4 3 3 4 3 3 6 6 6 6 6 6
+4 0 0 6 6 6 6 6 6 6 6 6 4 0 0 4 0 0
+4 0 0 26 28 28 131 129 131 220 221 221 244 246 246 220 221 221
+205 212 215 193 200 203 205 212 215 193 200 203 193 200 203 205 212 215
+220 221 221 193 200 203 193 200 203 193 200 203 190 197 201 174 174 174
+174 174 174 190 197 201 193 200 203 193 200 203 167 166 167 125 124 125
+6 6 6 4 0 0 4 0 0 4 3 3 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 5 5 5
+5 5 5 4 3 3 5 5 5 6 6 6 4 3 3 5 5 5
+6 6 6 6 6 6 4 0 0 6 6 6 6 6 6 6 6 6
+4 0 0 4 0 0 6 6 6 41 54 63 158 157 158 220 221 221
+220 221 221 220 221 221 193 200 203 193 200 203 193 200 203 190 197 201
+190 197 201 190 197 201 190 197 201 190 197 201 174 174 174 193 200 203
+193 200 203 220 221 221 174 174 174 125 124 125 37 38 37 4 0 0
+4 0 0 4 3 3 6 6 6 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 5 5 5 4 3 3 4 3 3 4 3 3 5 5 5
+4 3 3 6 6 6 5 5 5 4 3 3 6 6 6 6 6 6
+6 6 6 6 6 6 4 0 0 4 0 0 13 16 17 60 73 81
+174 174 174 220 221 221 220 221 221 205 212 215 190 197 201 174 174 174
+193 200 203 174 174 174 190 197 201 174 174 174 193 200 203 220 221 221
+193 200 203 131 129 131 37 38 37 6 6 6 4 0 0 4 0 0
+6 6 6 6 6 6 4 3 3 5 5 5 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 5 5 5 5 5 5 5 5 5
+5 5 5 4 3 3 4 3 3 5 5 5 4 3 3 4 3 3
+5 5 5 6 6 6 6 6 6 4 0 0 6 6 6 6 6 6
+6 6 6 125 124 125 174 174 174 220 221 221 220 221 221 193 200 203
+193 200 203 193 200 203 193 200 203 193 200 203 220 221 221 158 157 158
+60 73 81 6 6 6 4 0 0 4 0 0 5 5 5 6 6 6
+5 5 5 5 5 5 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 5 5 5 5 5 5 4 3 3 5 5 5 4 3 3
+5 5 5 5 5 5 6 6 6 6 6 6 4 0 0 4 0 0
+4 0 0 4 0 0 26 28 28 125 124 125 174 174 174 193 200 203
+193 200 203 174 174 174 193 200 203 167 166 167 125 124 125 6 6 6
+6 6 6 6 6 6 4 0 0 6 6 6 6 6 6 5 5 5
+4 3 3 5 5 5 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 5 5 5
+4 3 3 6 6 6 4 0 0 6 6 6 6 6 6 6 6 6
+6 6 6 4 0 0 4 0 0 6 6 6 37 38 37 125 124 125
+153 152 153 131 129 131 125 124 125 37 38 37 6 6 6 6 6 6
+6 6 6 4 0 0 6 6 6 6 6 6 4 3 3 5 5 5
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 5 5 5 5 5 5 4 3 3 5 5 5 4 3 3
+6 6 6 6 6 6 4 0 0 4 0 0 6 6 6 6 6 6
+24 26 27 24 26 27 6 6 6 6 6 6 6 6 6 4 0 0
+6 6 6 6 6 6 4 0 0 6 6 6 5 5 5 4 3 3
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 5 5 5 4 3 3 5 5 5 6 6 6
+4 0 0 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6
+6 6 6 6 6 6 6 6 6 4 0 0 6 6 6 6 6 6
+4 0 0 6 6 6 6 6 6 4 3 3 5 5 5 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 5 5 5 4 3 3 5 5 5
+5 5 5 5 5 5 4 0 0 6 6 6 4 0 0 6 6 6
+6 6 6 6 6 6 6 6 6 4 0 0 6 6 6 4 0 0
+6 6 6 4 3 3 5 5 5 4 3 3 5 5 5 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 5 5 5
+4 3 3 6 6 6 4 3 3 6 6 6 6 6 6 6 6 6
+4 0 0 6 6 6 4 0 0 6 6 6 6 6 6 6 6 6
+6 6 6 4 3 3 5 5 5 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 5 5 5 4 3 3 5 5 5 4 0 0 6 6 6
+6 6 6 4 0 0 6 6 6 6 6 6 4 0 0 6 6 6
+4 3 3 5 5 5 5 5 5 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 5 5 5 4 3 3 5 5 5 6 6 6 4 3 3
+4 3 3 6 6 6 6 6 6 4 3 3 6 6 6 4 3 3
+5 5 5 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 5 5 5 4 3 3 6 6 6
+5 5 5 4 3 3 4 3 3 4 3 3 5 5 5 5 5 5
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 5 5 5 4 3 3
+5 5 5 4 3 3 5 5 5 5 5 5 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4
+4 4 4 4 4 4
static int xsd_kva_open(struct inode *inode, struct file *file)
{
file->private_data = (void *)kasprintf(GFP_KERNEL, "0x%p",
+#ifdef CONFIG_GRKERNSEC_HIDESYM
+ NULL);
+#else
xen_store_interface);
+#endif
+
if (!file->private_data)
return -ENOMEM;
return 0;
retval = v9fs_file_write_internal(inode,
v9inode->writeback_fid,
- (__force const char __user *)buffer,
+ (const char __force_user *)buffer,
len, &offset, 0);
if (retval > 0)
retval = 0;
void
v9fs_vfs_put_link(struct dentry *dentry, struct nameidata *nd, void *p)
{
- char *s = nd_get_link(nd);
+ const char *s = nd_get_link(nd);
p9_debug(P9_DEBUG_VFS, " %pd %s\n",
dentry, IS_ERR(s) ? "<error>" : s);
config BINFMT_AOUT
tristate "Kernel support for a.out and ECOFF binaries"
- depends on HAVE_AOUT
+ depends on HAVE_AOUT && BROKEN
---help---
A.out (Assembler.OUTput) is a set of formats for libraries and
executables used in the earliest versions of UNIX. Linux used
struct afs_vnode *vnode;
struct super_block *sb;
struct inode *inode;
- static atomic_t afs_autocell_ino;
+ static atomic_unchecked_t afs_autocell_ino;
_enter("{%x:%u},%*.*s,",
AFS_FS_I(dir)->fid.vid, AFS_FS_I(dir)->fid.vnode,
data.fid.unique = 0;
data.fid.vnode = 0;
- inode = iget5_locked(sb, atomic_inc_return(&afs_autocell_ino),
+ inode = iget5_locked(sb, atomic_inc_return_unchecked(&afs_autocell_ino),
afs_iget5_autocell_test, afs_iget5_set,
&data);
if (!inode) {
size += sizeof(struct io_event) * nr_events;
nr_pages = PFN_UP(size);
- if (nr_pages < 0)
+ if (nr_pages <= 0)
return -EINVAL;
file = aio_private_file(ctx, nr_pages);
unsigned long limit;
limit = rlimit(RLIMIT_FSIZE);
+ gr_learn_resource(current, RLIMIT_FSIZE, (unsigned long)offset, 1);
if (limit != RLIM_INFINITY && offset > limit)
goto out_sig;
if (offset > inode->i_sb->s_maxbytes)
{
unsigned long sigpipe, flags;
mm_segment_t fs;
- const char *data = (const char *)addr;
+ const char __user *data = (const char __force_user *)addr;
ssize_t wr = 0;
sigpipe = sigismember(¤t->pending.signal, SIGPIPE);
return 1;
}
+#ifdef CONFIG_GRKERNSEC_HIDESYM
+static atomic_unchecked_t autofs_dummy_name_id = ATOMIC_INIT(0);
+#endif
+
int autofs4_wait(struct autofs_sb_info *sbi, struct dentry *dentry,
enum autofs_notify notify)
{
/* If this is a direct mount request create a dummy name */
if (IS_ROOT(dentry) && autofs_type_trigger(sbi->type))
+#ifdef CONFIG_GRKERNSEC_HIDESYM
+ /* this name does get written to userland via autofs4_write() */
+ qstr.len = sprintf(name, "%08x", atomic_inc_return_unchecked(&autofs_dummy_name_id));
+#else
qstr.len = sprintf(name, "%p", dentry);
+#endif
else {
qstr.len = autofs4_getpath(sbi, dentry, &name);
if (!qstr.len) {
#include <asm/byteorder.h>
-static inline u64
+static inline u64 __intentional_overflow(-1)
fs64_to_cpu(const struct super_block *sb, fs64 n)
{
if (BEFS_SB(sb)->byte_order == BEFS_BYTESEX_LE)
return (__force fs64)cpu_to_be64(n);
}
-static inline u32
+static inline u32 __intentional_overflow(-1)
fs32_to_cpu(const struct super_block *sb, fs32 n)
{
if (BEFS_SB(sb)->byte_order == BEFS_BYTESEX_LE)
return (__force fs32)cpu_to_be32(n);
}
-static inline u16
+static inline u16 __intentional_overflow(-1)
fs16_to_cpu(const struct super_block *sb, fs16 n)
{
if (BEFS_SB(sb)->byte_order == BEFS_BYTESEX_LE)
#include <linux/string.h>
#include <linux/fs.h>
#include <linux/file.h>
+#include <linux/security.h>
#include <linux/stat.h>
#include <linux/fcntl.h>
#include <linux/ptrace.h>
#endif
# define START_STACK(u) ((void __user *)u.start_stack)
+ memset(&dump, 0, sizeof(dump));
+
fs = get_fs();
set_fs(KERNEL_DS);
has_dumped = 1;
/* If the size of the dump file exceeds the rlimit, then see what would happen
if we wrote the stack, but not the data area. */
+ gr_learn_resource(current, RLIMIT_CORE, (dump.u_dsize + dump.u_ssize+1) * PAGE_SIZE, 1);
if ((dump.u_dsize + dump.u_ssize+1) * PAGE_SIZE > cprm->limit)
dump.u_dsize = 0;
/* Make sure we have enough room to write the stack and data areas. */
+ gr_learn_resource(current, RLIMIT_CORE, (dump.u_ssize + 1) * PAGE_SIZE, 1);
if ((dump.u_ssize + 1) * PAGE_SIZE > cprm->limit)
dump.u_ssize = 0;
rlim = rlimit(RLIMIT_DATA);
if (rlim >= RLIM_INFINITY)
rlim = ~0;
+
+ gr_learn_resource(current, RLIMIT_DATA, ex.a_data + ex.a_bss, 1);
if (ex.a_data + ex.a_bss > rlim)
return -ENOMEM;
install_exec_creds(bprm);
+#if defined(CONFIG_PAX_NOEXEC) || defined(CONFIG_PAX_ASLR)
+ current->mm->pax_flags = 0UL;
+#endif
+
+#ifdef CONFIG_PAX_PAGEEXEC
+ if (!(N_FLAGS(ex) & F_PAX_PAGEEXEC)) {
+ current->mm->pax_flags |= MF_PAX_PAGEEXEC;
+
+#ifdef CONFIG_PAX_EMUTRAMP
+ if (N_FLAGS(ex) & F_PAX_EMUTRAMP)
+ current->mm->pax_flags |= MF_PAX_EMUTRAMP;
+#endif
+
+#ifdef CONFIG_PAX_MPROTECT
+ if (!(N_FLAGS(ex) & F_PAX_MPROTECT))
+ current->mm->pax_flags |= MF_PAX_MPROTECT;
+#endif
+
+ }
+#endif
+
if (N_MAGIC(ex) == OMAGIC) {
unsigned long text_addr, map_size;
loff_t pos;
return error;
error = vm_mmap(bprm->file, N_DATADDR(ex), ex.a_data,
- PROT_READ | PROT_WRITE | PROT_EXEC,
+ PROT_READ | PROT_WRITE,
MAP_FIXED | MAP_PRIVATE | MAP_DENYWRITE | MAP_EXECUTABLE,
fd_offset + ex.a_text);
if (error != N_DATADDR(ex))
#include <linux/utsname.h>
#include <linux/coredump.h>
#include <linux/sched.h>
+#include <linux/xattr.h>
#include <asm/uaccess.h>
#include <asm/param.h>
#include <asm/page.h>
static int load_elf_binary(struct linux_binprm *bprm);
static unsigned long elf_map(struct file *, unsigned long, struct elf_phdr *,
- int, int, unsigned long);
+ int, int, unsigned long) __intentional_overflow(-1);
#ifdef CONFIG_USELIB
static int load_elf_library(struct file *);
#define elf_core_dump NULL
#endif
+#ifdef CONFIG_PAX_MPROTECT
+static void elf_handle_mprotect(struct vm_area_struct *vma, unsigned long newflags);
+#endif
+
+#ifdef CONFIG_GRKERNSEC_RWXMAP_LOG
+static void elf_handle_mmap(struct file *file);
+#endif
+
#if ELF_EXEC_PAGESIZE > PAGE_SIZE
#define ELF_MIN_ALIGN ELF_EXEC_PAGESIZE
#else
.load_binary = load_elf_binary,
.load_shlib = load_elf_library,
.core_dump = elf_core_dump,
+
+#ifdef CONFIG_PAX_MPROTECT
+ .handle_mprotect= elf_handle_mprotect,
+#endif
+
+#ifdef CONFIG_GRKERNSEC_RWXMAP_LOG
+ .handle_mmap = elf_handle_mmap,
+#endif
+
.min_coredump = ELF_EXEC_PAGESIZE,
};
static int set_brk(unsigned long start, unsigned long end)
{
+ unsigned long e = end;
+
start = ELF_PAGEALIGN(start);
end = ELF_PAGEALIGN(end);
if (end > start) {
if (BAD_ADDR(addr))
return addr;
}
- current->mm->start_brk = current->mm->brk = end;
+ current->mm->start_brk = current->mm->brk = e;
return 0;
}
elf_addr_t __user *u_rand_bytes;
const char *k_platform = ELF_PLATFORM;
const char *k_base_platform = ELF_BASE_PLATFORM;
- unsigned char k_rand_bytes[16];
+ u32 k_rand_bytes[4];
int items;
elf_addr_t *elf_info;
int ei_index = 0;
const struct cred *cred = current_cred();
struct vm_area_struct *vma;
+ unsigned long saved_auxv[AT_VECTOR_SIZE];
/*
* In some cases (e.g. Hyper-Threading), we want to avoid L1
* Generate 16 random bytes for userspace PRNG seeding.
*/
get_random_bytes(k_rand_bytes, sizeof(k_rand_bytes));
- u_rand_bytes = (elf_addr_t __user *)
- STACK_ALLOC(p, sizeof(k_rand_bytes));
+ prandom_seed(k_rand_bytes[0] ^ prandom_u32());
+ prandom_seed(k_rand_bytes[1] ^ prandom_u32());
+ prandom_seed(k_rand_bytes[2] ^ prandom_u32());
+ prandom_seed(k_rand_bytes[3] ^ prandom_u32());
+ p = STACK_ROUND(p, sizeof(k_rand_bytes));
+ u_rand_bytes = (elf_addr_t __user *) p;
if (__copy_to_user(u_rand_bytes, k_rand_bytes, sizeof(k_rand_bytes)))
return -EFAULT;
return -EFAULT;
current->mm->env_end = p;
+ memcpy(saved_auxv, elf_info, ei_index * sizeof(elf_addr_t));
+
/* Put the elf_info on the stack in the right place. */
sp = (elf_addr_t __user *)envp + 1;
- if (copy_to_user(sp, elf_info, ei_index * sizeof(elf_addr_t)))
+ if (copy_to_user(sp, saved_auxv, ei_index * sizeof(elf_addr_t)))
return -EFAULT;
return 0;
}
an ELF header */
static unsigned long load_elf_interp(struct elfhdr *interp_elf_ex,
- struct file *interpreter, unsigned long *interp_map_addr,
+ struct file *interpreter,
unsigned long no_base, struct elf_phdr *interp_elf_phdata)
{
struct elf_phdr *eppnt;
- unsigned long load_addr = 0;
+ unsigned long load_addr = 0, pax_task_size = TASK_SIZE;
int load_addr_set = 0;
unsigned long last_bss = 0, elf_bss = 0;
- unsigned long error = ~0UL;
+ unsigned long error = -EINVAL;
unsigned long total_size;
int i;
goto out;
}
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (current->mm->pax_flags & MF_PAX_SEGMEXEC)
+ pax_task_size = SEGMEXEC_TASK_SIZE;
+#endif
+
eppnt = interp_elf_phdata;
for (i = 0; i < interp_elf_ex->e_phnum; i++, eppnt++) {
if (eppnt->p_type == PT_LOAD) {
map_addr = elf_map(interpreter, load_addr + vaddr,
eppnt, elf_prot, elf_type, total_size);
total_size = 0;
- if (!*interp_map_addr)
- *interp_map_addr = map_addr;
error = map_addr;
if (BAD_ADDR(map_addr))
goto out;
k = load_addr + eppnt->p_vaddr;
if (BAD_ADDR(k) ||
eppnt->p_filesz > eppnt->p_memsz ||
- eppnt->p_memsz > TASK_SIZE ||
- TASK_SIZE - eppnt->p_memsz < k) {
+ eppnt->p_memsz > pax_task_size ||
+ pax_task_size - eppnt->p_memsz < k) {
error = -ENOMEM;
goto out;
}
elf_bss = ELF_PAGESTART(elf_bss + ELF_MIN_ALIGN - 1);
/* Map the last of the bss segment */
- error = vm_brk(elf_bss, last_bss - elf_bss);
- if (BAD_ADDR(error))
- goto out;
+ if (last_bss > elf_bss) {
+ error = vm_brk(elf_bss, last_bss - elf_bss);
+ if (BAD_ADDR(error))
+ goto out;
+ }
}
error = load_addr;
return error;
}
+#ifdef CONFIG_PAX_PT_PAX_FLAGS
+#ifdef CONFIG_PAX_SOFTMODE
+static unsigned long pax_parse_pt_pax_softmode(const struct elf_phdr * const elf_phdata)
+{
+ unsigned long pax_flags = 0UL;
+
+#ifdef CONFIG_PAX_PAGEEXEC
+ if (elf_phdata->p_flags & PF_PAGEEXEC)
+ pax_flags |= MF_PAX_PAGEEXEC;
+#endif
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (elf_phdata->p_flags & PF_SEGMEXEC)
+ pax_flags |= MF_PAX_SEGMEXEC;
+#endif
+
+#ifdef CONFIG_PAX_EMUTRAMP
+ if ((elf_phdata->p_flags & PF_EMUTRAMP) && (pax_flags & (MF_PAX_PAGEEXEC | MF_PAX_SEGMEXEC)))
+ pax_flags |= MF_PAX_EMUTRAMP;
+#endif
+
+#ifdef CONFIG_PAX_MPROTECT
+ if (elf_phdata->p_flags & PF_MPROTECT)
+ pax_flags |= MF_PAX_MPROTECT;
+#endif
+
+#if defined(CONFIG_PAX_RANDMMAP) || defined(CONFIG_PAX_RANDUSTACK)
+ if (randomize_va_space && (elf_phdata->p_flags & PF_RANDMMAP))
+ pax_flags |= MF_PAX_RANDMMAP;
+#endif
+
+ return pax_flags;
+}
+#endif
+
+static unsigned long pax_parse_pt_pax_hardmode(const struct elf_phdr * const elf_phdata)
+{
+ unsigned long pax_flags = 0UL;
+
+#ifdef CONFIG_PAX_PAGEEXEC
+ if (!(elf_phdata->p_flags & PF_NOPAGEEXEC))
+ pax_flags |= MF_PAX_PAGEEXEC;
+#endif
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (!(elf_phdata->p_flags & PF_NOSEGMEXEC))
+ pax_flags |= MF_PAX_SEGMEXEC;
+#endif
+
+#ifdef CONFIG_PAX_EMUTRAMP
+ if (!(elf_phdata->p_flags & PF_NOEMUTRAMP))
+ pax_flags |= MF_PAX_EMUTRAMP;
+#endif
+
+#ifdef CONFIG_PAX_MPROTECT
+ if (!(elf_phdata->p_flags & PF_NOMPROTECT))
+ pax_flags |= MF_PAX_MPROTECT;
+#endif
+
+#if defined(CONFIG_PAX_RANDMMAP) || defined(CONFIG_PAX_RANDUSTACK)
+ if (randomize_va_space && !(elf_phdata->p_flags & PF_NORANDMMAP))
+ pax_flags |= MF_PAX_RANDMMAP;
+#endif
+
+ return pax_flags;
+}
+#endif
+
+#ifdef CONFIG_PAX_XATTR_PAX_FLAGS
+#ifdef CONFIG_PAX_SOFTMODE
+static unsigned long pax_parse_xattr_pax_softmode(unsigned long pax_flags_softmode)
+{
+ unsigned long pax_flags = 0UL;
+
+#ifdef CONFIG_PAX_PAGEEXEC
+ if (pax_flags_softmode & MF_PAX_PAGEEXEC)
+ pax_flags |= MF_PAX_PAGEEXEC;
+#endif
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (pax_flags_softmode & MF_PAX_SEGMEXEC)
+ pax_flags |= MF_PAX_SEGMEXEC;
+#endif
+
+#ifdef CONFIG_PAX_EMUTRAMP
+ if ((pax_flags_softmode & MF_PAX_EMUTRAMP) && (pax_flags & (MF_PAX_PAGEEXEC | MF_PAX_SEGMEXEC)))
+ pax_flags |= MF_PAX_EMUTRAMP;
+#endif
+
+#ifdef CONFIG_PAX_MPROTECT
+ if (pax_flags_softmode & MF_PAX_MPROTECT)
+ pax_flags |= MF_PAX_MPROTECT;
+#endif
+
+#if defined(CONFIG_PAX_RANDMMAP) || defined(CONFIG_PAX_RANDUSTACK)
+ if (randomize_va_space && (pax_flags_softmode & MF_PAX_RANDMMAP))
+ pax_flags |= MF_PAX_RANDMMAP;
+#endif
+
+ return pax_flags;
+}
+#endif
+
+static unsigned long pax_parse_xattr_pax_hardmode(unsigned long pax_flags_hardmode)
+{
+ unsigned long pax_flags = 0UL;
+
+#ifdef CONFIG_PAX_PAGEEXEC
+ if (!(pax_flags_hardmode & MF_PAX_PAGEEXEC))
+ pax_flags |= MF_PAX_PAGEEXEC;
+#endif
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (!(pax_flags_hardmode & MF_PAX_SEGMEXEC))
+ pax_flags |= MF_PAX_SEGMEXEC;
+#endif
+
+#ifdef CONFIG_PAX_EMUTRAMP
+ if (!(pax_flags_hardmode & MF_PAX_EMUTRAMP))
+ pax_flags |= MF_PAX_EMUTRAMP;
+#endif
+
+#ifdef CONFIG_PAX_MPROTECT
+ if (!(pax_flags_hardmode & MF_PAX_MPROTECT))
+ pax_flags |= MF_PAX_MPROTECT;
+#endif
+
+#if defined(CONFIG_PAX_RANDMMAP) || defined(CONFIG_PAX_RANDUSTACK)
+ if (randomize_va_space && !(pax_flags_hardmode & MF_PAX_RANDMMAP))
+ pax_flags |= MF_PAX_RANDMMAP;
+#endif
+
+ return pax_flags;
+}
+#endif
+
+#if defined(CONFIG_PAX_NOEXEC) || defined(CONFIG_PAX_ASLR)
+static unsigned long pax_parse_defaults(void)
+{
+ unsigned long pax_flags = 0UL;
+
+#ifdef CONFIG_PAX_SOFTMODE
+ if (pax_softmode)
+ return pax_flags;
+#endif
+
+#ifdef CONFIG_PAX_PAGEEXEC
+ pax_flags |= MF_PAX_PAGEEXEC;
+#endif
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ pax_flags |= MF_PAX_SEGMEXEC;
+#endif
+
+#ifdef CONFIG_PAX_MPROTECT
+ pax_flags |= MF_PAX_MPROTECT;
+#endif
+
+#ifdef CONFIG_PAX_RANDMMAP
+ if (randomize_va_space)
+ pax_flags |= MF_PAX_RANDMMAP;
+#endif
+
+ return pax_flags;
+}
+
+static unsigned long pax_parse_ei_pax(const struct elfhdr * const elf_ex)
+{
+ unsigned long pax_flags = PAX_PARSE_FLAGS_FALLBACK;
+
+#ifdef CONFIG_PAX_EI_PAX
+
+#ifdef CONFIG_PAX_SOFTMODE
+ if (pax_softmode)
+ return pax_flags;
+#endif
+
+ pax_flags = 0UL;
+
+#ifdef CONFIG_PAX_PAGEEXEC
+ if (!(elf_ex->e_ident[EI_PAX] & EF_PAX_PAGEEXEC))
+ pax_flags |= MF_PAX_PAGEEXEC;
+#endif
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (!(elf_ex->e_ident[EI_PAX] & EF_PAX_SEGMEXEC))
+ pax_flags |= MF_PAX_SEGMEXEC;
+#endif
+
+#ifdef CONFIG_PAX_EMUTRAMP
+ if ((pax_flags & (MF_PAX_PAGEEXEC | MF_PAX_SEGMEXEC)) && (elf_ex->e_ident[EI_PAX] & EF_PAX_EMUTRAMP))
+ pax_flags |= MF_PAX_EMUTRAMP;
+#endif
+
+#ifdef CONFIG_PAX_MPROTECT
+ if ((pax_flags & (MF_PAX_PAGEEXEC | MF_PAX_SEGMEXEC)) && !(elf_ex->e_ident[EI_PAX] & EF_PAX_MPROTECT))
+ pax_flags |= MF_PAX_MPROTECT;
+#endif
+
+#ifdef CONFIG_PAX_ASLR
+ if (randomize_va_space && !(elf_ex->e_ident[EI_PAX] & EF_PAX_RANDMMAP))
+ pax_flags |= MF_PAX_RANDMMAP;
+#endif
+
+#endif
+
+ return pax_flags;
+
+}
+
+static unsigned long pax_parse_pt_pax(const struct elfhdr * const elf_ex, const struct elf_phdr * const elf_phdata)
+{
+
+#ifdef CONFIG_PAX_PT_PAX_FLAGS
+ unsigned long i;
+
+ for (i = 0UL; i < elf_ex->e_phnum; i++)
+ if (elf_phdata[i].p_type == PT_PAX_FLAGS) {
+ if (((elf_phdata[i].p_flags & PF_PAGEEXEC) && (elf_phdata[i].p_flags & PF_NOPAGEEXEC)) ||
+ ((elf_phdata[i].p_flags & PF_SEGMEXEC) && (elf_phdata[i].p_flags & PF_NOSEGMEXEC)) ||
+ ((elf_phdata[i].p_flags & PF_EMUTRAMP) && (elf_phdata[i].p_flags & PF_NOEMUTRAMP)) ||
+ ((elf_phdata[i].p_flags & PF_MPROTECT) && (elf_phdata[i].p_flags & PF_NOMPROTECT)) ||
+ ((elf_phdata[i].p_flags & PF_RANDMMAP) && (elf_phdata[i].p_flags & PF_NORANDMMAP)))
+ return PAX_PARSE_FLAGS_FALLBACK;
+
+#ifdef CONFIG_PAX_SOFTMODE
+ if (pax_softmode)
+ return pax_parse_pt_pax_softmode(&elf_phdata[i]);
+ else
+#endif
+
+ return pax_parse_pt_pax_hardmode(&elf_phdata[i]);
+ break;
+ }
+#endif
+
+ return PAX_PARSE_FLAGS_FALLBACK;
+}
+
+static unsigned long pax_parse_xattr_pax(struct file * const file)
+{
+
+#ifdef CONFIG_PAX_XATTR_PAX_FLAGS
+ ssize_t xattr_size, i;
+ unsigned char xattr_value[sizeof("pemrs") - 1];
+ unsigned long pax_flags_hardmode = 0UL, pax_flags_softmode = 0UL;
+
+ xattr_size = pax_getxattr(file->f_path.dentry, xattr_value, sizeof xattr_value);
+ if (xattr_size < 0 || xattr_size > sizeof xattr_value)
+ return PAX_PARSE_FLAGS_FALLBACK;
+
+ for (i = 0; i < xattr_size; i++)
+ switch (xattr_value[i]) {
+ default:
+ return PAX_PARSE_FLAGS_FALLBACK;
+
+#define parse_flag(option1, option2, flag) \
+ case option1: \
+ if (pax_flags_hardmode & MF_PAX_##flag) \
+ return PAX_PARSE_FLAGS_FALLBACK;\
+ pax_flags_hardmode |= MF_PAX_##flag; \
+ break; \
+ case option2: \
+ if (pax_flags_softmode & MF_PAX_##flag) \
+ return PAX_PARSE_FLAGS_FALLBACK;\
+ pax_flags_softmode |= MF_PAX_##flag; \
+ break;
+
+ parse_flag('p', 'P', PAGEEXEC);
+ parse_flag('e', 'E', EMUTRAMP);
+ parse_flag('m', 'M', MPROTECT);
+ parse_flag('r', 'R', RANDMMAP);
+ parse_flag('s', 'S', SEGMEXEC);
+
+#undef parse_flag
+ }
+
+ if (pax_flags_hardmode & pax_flags_softmode)
+ return PAX_PARSE_FLAGS_FALLBACK;
+
+#ifdef CONFIG_PAX_SOFTMODE
+ if (pax_softmode)
+ return pax_parse_xattr_pax_softmode(pax_flags_softmode);
+ else
+#endif
+
+ return pax_parse_xattr_pax_hardmode(pax_flags_hardmode);
+#else
+ return PAX_PARSE_FLAGS_FALLBACK;
+#endif
+
+}
+
+static long pax_parse_pax_flags(const struct elfhdr * const elf_ex, const struct elf_phdr * const elf_phdata, struct file * const file)
+{
+ unsigned long pax_flags, ei_pax_flags, pt_pax_flags, xattr_pax_flags;
+
+ pax_flags = pax_parse_defaults();
+ ei_pax_flags = pax_parse_ei_pax(elf_ex);
+ pt_pax_flags = pax_parse_pt_pax(elf_ex, elf_phdata);
+ xattr_pax_flags = pax_parse_xattr_pax(file);
+
+ if (pt_pax_flags != PAX_PARSE_FLAGS_FALLBACK &&
+ xattr_pax_flags != PAX_PARSE_FLAGS_FALLBACK &&
+ pt_pax_flags != xattr_pax_flags)
+ return -EINVAL;
+ if (xattr_pax_flags != PAX_PARSE_FLAGS_FALLBACK)
+ pax_flags = xattr_pax_flags;
+ else if (pt_pax_flags != PAX_PARSE_FLAGS_FALLBACK)
+ pax_flags = pt_pax_flags;
+ else if (ei_pax_flags != PAX_PARSE_FLAGS_FALLBACK)
+ pax_flags = ei_pax_flags;
+
+#if defined(CONFIG_PAX_PAGEEXEC) && defined(CONFIG_PAX_SEGMEXEC)
+ if ((pax_flags & (MF_PAX_PAGEEXEC | MF_PAX_SEGMEXEC)) == (MF_PAX_PAGEEXEC | MF_PAX_SEGMEXEC)) {
+ if ((__supported_pte_mask & _PAGE_NX))
+ pax_flags &= ~MF_PAX_SEGMEXEC;
+ else
+ pax_flags &= ~MF_PAX_PAGEEXEC;
+ }
+#endif
+
+ if (0 > pax_check_flags(&pax_flags))
+ return -EINVAL;
+
+ current->mm->pax_flags = pax_flags;
+ return 0;
+}
+#endif
+
/*
* These are the functions used to load ELF style executables and shared
* libraries. There is no binary dependent code anywhere else.
{
unsigned long random_variable = 0;
+#ifdef CONFIG_PAX_RANDUSTACK
+ if (current->mm->pax_flags & MF_PAX_RANDMMAP)
+ return stack_top - current->mm->delta_stack;
+#endif
+
if ((current->flags & PF_RANDOMIZE) &&
!(current->personality & ADDR_NO_RANDOMIZE)) {
random_variable = (unsigned long) get_random_int();
unsigned long load_addr = 0, load_bias = 0;
int load_addr_set = 0;
char * elf_interpreter = NULL;
- unsigned long error;
+ unsigned long error = 0;
struct elf_phdr *elf_ppnt, *elf_phdata, *interp_elf_phdata = NULL;
unsigned long elf_bss, elf_brk;
int retval, i;
struct elfhdr interp_elf_ex;
} *loc;
struct arch_elf_state arch_state = INIT_ARCH_ELF_STATE;
+ unsigned long pax_task_size;
loc = kmalloc(sizeof(*loc), GFP_KERNEL);
if (!loc) {
/* Do this immediately, since STACK_TOP as used in setup_arg_pages
may depend on the personality. */
SET_PERSONALITY2(loc->elf_ex, &arch_state);
+
+#if defined(CONFIG_PAX_NOEXEC) || defined(CONFIG_PAX_ASLR)
+ current->mm->pax_flags = 0UL;
+#endif
+
+#ifdef CONFIG_PAX_DLRESOLVE
+ current->mm->call_dl_resolve = 0UL;
+#endif
+
+#if defined(CONFIG_PPC32) && defined(CONFIG_PAX_EMUSIGRT)
+ current->mm->call_syscall = 0UL;
+#endif
+
+#ifdef CONFIG_PAX_ASLR
+ current->mm->delta_mmap = 0UL;
+ current->mm->delta_stack = 0UL;
+#endif
+
+#if defined(CONFIG_PAX_NOEXEC) || defined(CONFIG_PAX_ASLR)
+ if (0 > pax_parse_pax_flags(&loc->elf_ex, elf_phdata, bprm->file)) {
+ send_sig(SIGKILL, current, 0);
+ goto out_free_dentry;
+ }
+#endif
+
+#ifdef CONFIG_PAX_HAVE_ACL_FLAGS
+ pax_set_initial_flags(bprm);
+#elif defined(CONFIG_PAX_HOOK_ACL_FLAGS)
+ if (pax_set_initial_flags_func)
+ (pax_set_initial_flags_func)(bprm);
+#endif
+
+#ifdef CONFIG_ARCH_TRACK_EXEC_LIMIT
+ if ((current->mm->pax_flags & MF_PAX_PAGEEXEC) && !(__supported_pte_mask & _PAGE_NX)) {
+ current->mm->context.user_cs_limit = PAGE_SIZE;
+ current->mm->def_flags |= VM_PAGEEXEC | VM_NOHUGEPAGE;
+ }
+#endif
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (current->mm->pax_flags & MF_PAX_SEGMEXEC) {
+ current->mm->context.user_cs_base = SEGMEXEC_TASK_SIZE;
+ current->mm->context.user_cs_limit = TASK_SIZE-SEGMEXEC_TASK_SIZE;
+ pax_task_size = SEGMEXEC_TASK_SIZE;
+ current->mm->def_flags |= VM_NOHUGEPAGE;
+ } else
+#endif
+
+ pax_task_size = TASK_SIZE;
+
+#if defined(CONFIG_ARCH_TRACK_EXEC_LIMIT) || defined(CONFIG_PAX_SEGMEXEC)
+ if (current->mm->pax_flags & (MF_PAX_PAGEEXEC | MF_PAX_SEGMEXEC)) {
+ set_user_cs(current->mm->context.user_cs_base, current->mm->context.user_cs_limit, get_cpu());
+ put_cpu();
+ }
+#endif
+
+#ifdef CONFIG_PAX_ASLR
+ if (current->mm->pax_flags & MF_PAX_RANDMMAP) {
+ current->mm->delta_mmap = (pax_get_random_long() & ((1UL << PAX_DELTA_MMAP_LEN)-1)) << PAGE_SHIFT;
+ current->mm->delta_stack = (pax_get_random_long() & ((1UL << PAX_DELTA_STACK_LEN)-1)) << PAGE_SHIFT;
+ }
+#endif
+
+#if defined(CONFIG_PAX_PAGEEXEC) || defined(CONFIG_PAX_SEGMEXEC)
+ if (current->mm->pax_flags & (MF_PAX_PAGEEXEC | MF_PAX_SEGMEXEC)) {
+ executable_stack = EXSTACK_DISABLE_X;
+ current->personality &= ~READ_IMPLIES_EXEC;
+ } else
+#endif
+
if (elf_read_implies_exec(loc->elf_ex, executable_stack))
current->personality |= READ_IMPLIES_EXEC;
#else
load_bias = ELF_PAGESTART(ELF_ET_DYN_BASE - vaddr);
#endif
+
+#ifdef CONFIG_PAX_RANDMMAP
+ /* PaX: randomize base address at the default exe base if requested */
+ if ((current->mm->pax_flags & MF_PAX_RANDMMAP) && elf_interpreter) {
+#ifdef CONFIG_SPARC64
+ load_bias = (pax_get_random_long() & ((1UL << PAX_DELTA_MMAP_LEN) - 1)) << (PAGE_SHIFT+1);
+#else
+ load_bias = (pax_get_random_long() & ((1UL << PAX_DELTA_MMAP_LEN) - 1)) << PAGE_SHIFT;
+#endif
+ load_bias = ELF_PAGESTART(PAX_ELF_ET_DYN_BASE - vaddr + load_bias);
+ elf_flags |= MAP_FIXED;
+ }
+#endif
+
}
error = elf_map(bprm->file, load_bias + vaddr, elf_ppnt,
* allowed task size. Note that p_filesz must always be
* <= p_memsz so it is only necessary to check p_memsz.
*/
- if (BAD_ADDR(k) || elf_ppnt->p_filesz > elf_ppnt->p_memsz ||
- elf_ppnt->p_memsz > TASK_SIZE ||
- TASK_SIZE - elf_ppnt->p_memsz < k) {
+ if (k >= pax_task_size || elf_ppnt->p_filesz > elf_ppnt->p_memsz ||
+ elf_ppnt->p_memsz > pax_task_size ||
+ pax_task_size - elf_ppnt->p_memsz < k) {
/* set_brk can never work. Avoid overflows. */
retval = -EINVAL;
goto out_free_dentry;
if (retval)
goto out_free_dentry;
if (likely(elf_bss != elf_brk) && unlikely(padzero(elf_bss))) {
- retval = -EFAULT; /* Nobody gets to see this, but.. */
- goto out_free_dentry;
+ /*
+ * This bss-zeroing can fail if the ELF
+ * file specifies odd protections. So
+ * we don't check the return value
+ */
}
- if (elf_interpreter) {
- unsigned long interp_map_addr = 0;
+#ifdef CONFIG_PAX_RANDMMAP
+ if (current->mm->pax_flags & MF_PAX_RANDMMAP) {
+ unsigned long start, size, flags;
+ vm_flags_t vm_flags;
+ start = ELF_PAGEALIGN(elf_brk);
+ size = PAGE_SIZE + ((pax_get_random_long() & ((1UL << 22) - 1UL)) << 4);
+ flags = MAP_FIXED | MAP_PRIVATE;
+ vm_flags = VM_DONTEXPAND | VM_DONTDUMP;
+
+ down_write(¤t->mm->mmap_sem);
+ start = get_unmapped_area(NULL, start, PAGE_ALIGN(size), 0, flags);
+ retval = -ENOMEM;
+ if (!IS_ERR_VALUE(start) && !find_vma_intersection(current->mm, start, start + size + PAGE_SIZE)) {
+// if (current->personality & ADDR_NO_RANDOMIZE)
+// vm_flags |= VM_READ | VM_MAYREAD;
+ start = mmap_region(NULL, start, PAGE_ALIGN(size), vm_flags, 0);
+ retval = IS_ERR_VALUE(start) ? start : 0;
+ }
+ up_write(¤t->mm->mmap_sem);
+ if (retval == 0)
+ retval = set_brk(start + size, start + size + PAGE_SIZE);
+ if (retval < 0)
+ goto out_free_dentry;
+ }
+#endif
+
+ if (elf_interpreter) {
elf_entry = load_elf_interp(&loc->interp_elf_ex,
interpreter,
- &interp_map_addr,
load_bias, interp_elf_phdata);
if (!IS_ERR((void *)elf_entry)) {
/*
* Decide what to dump of a segment, part, all or none.
*/
static unsigned long vma_dump_size(struct vm_area_struct *vma,
- unsigned long mm_flags)
+ unsigned long mm_flags, long signr)
{
#define FILTER(type) (mm_flags & (1UL << MMF_DUMP_##type))
if (vma->vm_file == NULL)
return 0;
- if (FILTER(MAPPED_PRIVATE))
+ if (signr == SIGKILL || FILTER(MAPPED_PRIVATE))
goto whole;
/*
{
elf_addr_t *auxv = (elf_addr_t *) mm->saved_auxv;
int i = 0;
- do
+ do {
i += 2;
- while (auxv[i - 2] != AT_NULL);
+ } while (auxv[i - 2] != AT_NULL);
fill_note(note, "CORE", NT_AUXV, i * sizeof(elf_addr_t), auxv);
}
{
mm_segment_t old_fs = get_fs();
set_fs(KERNEL_DS);
- copy_siginfo_to_user((user_siginfo_t __user *) csigdata, siginfo);
+ copy_siginfo_to_user((user_siginfo_t __force_user *) csigdata, siginfo);
set_fs(old_fs);
fill_note(note, "CORE", NT_SIGINFO, sizeof(*csigdata), csigdata);
}
vma = next_vma(vma, gate_vma)) {
unsigned long dump_size;
- dump_size = vma_dump_size(vma, cprm->mm_flags);
+ dump_size = vma_dump_size(vma, cprm->mm_flags, cprm->siginfo->si_signo);
vma_filesz[i++] = dump_size;
vma_data_size += dump_size;
}
#endif /* CONFIG_ELF_CORE */
+#ifdef CONFIG_PAX_MPROTECT
+/* PaX: non-PIC ELF libraries need relocations on their executable segments
+ * therefore we'll grant them VM_MAYWRITE once during their life. Similarly
+ * we'll remove VM_MAYWRITE for good on RELRO segments.
+ *
+ * The checks favour ld-linux.so behaviour which operates on a per ELF segment
+ * basis because we want to allow the common case and not the special ones.
+ */
+static void elf_handle_mprotect(struct vm_area_struct *vma, unsigned long newflags)
+{
+ struct elfhdr elf_h;
+ struct elf_phdr elf_p;
+ unsigned long i;
+ unsigned long oldflags;
+ bool is_textrel_rw, is_textrel_rx, is_relro;
+
+ if (!(vma->vm_mm->pax_flags & MF_PAX_MPROTECT) || !vma->vm_file)
+ return;
+
+ oldflags = vma->vm_flags & (VM_MAYEXEC | VM_MAYWRITE | VM_MAYREAD | VM_EXEC | VM_WRITE | VM_READ);
+ newflags &= VM_MAYEXEC | VM_MAYWRITE | VM_MAYREAD | VM_EXEC | VM_WRITE | VM_READ;
+
+#ifdef CONFIG_PAX_ELFRELOCS
+ /* possible TEXTREL */
+ is_textrel_rw = !vma->anon_vma && oldflags == (VM_MAYEXEC | VM_MAYREAD | VM_EXEC | VM_READ) && newflags == (VM_WRITE | VM_READ);
+ is_textrel_rx = vma->anon_vma && oldflags == (VM_MAYEXEC | VM_MAYWRITE | VM_MAYREAD | VM_WRITE | VM_READ) && newflags == (VM_EXEC | VM_READ);
+#else
+ is_textrel_rw = false;
+ is_textrel_rx = false;
+#endif
+
+ /* possible RELRO */
+ is_relro = vma->anon_vma && oldflags == (VM_MAYWRITE | VM_MAYREAD | VM_READ) && newflags == (VM_MAYWRITE | VM_MAYREAD | VM_READ);
+
+ if (!is_textrel_rw && !is_textrel_rx && !is_relro)
+ return;
+
+ if (sizeof(elf_h) != kernel_read(vma->vm_file, 0UL, (char *)&elf_h, sizeof(elf_h)) ||
+ memcmp(elf_h.e_ident, ELFMAG, SELFMAG) ||
+
+#ifdef CONFIG_PAX_ETEXECRELOCS
+ ((is_textrel_rw || is_textrel_rx) && (elf_h.e_type != ET_DYN && elf_h.e_type != ET_EXEC)) ||
+#else
+ ((is_textrel_rw || is_textrel_rx) && elf_h.e_type != ET_DYN) ||
+#endif
+
+ (is_relro && (elf_h.e_type != ET_DYN && elf_h.e_type != ET_EXEC)) ||
+ !elf_check_arch(&elf_h) ||
+ elf_h.e_phentsize != sizeof(struct elf_phdr) ||
+ elf_h.e_phnum > 65536UL / sizeof(struct elf_phdr))
+ return;
+
+ for (i = 0UL; i < elf_h.e_phnum; i++) {
+ if (sizeof(elf_p) != kernel_read(vma->vm_file, elf_h.e_phoff + i*sizeof(elf_p), (char *)&elf_p, sizeof(elf_p)))
+ return;
+ switch (elf_p.p_type) {
+ case PT_DYNAMIC:
+ if (!is_textrel_rw && !is_textrel_rx)
+ continue;
+ i = 0UL;
+ while ((i+1) * sizeof(elf_dyn) <= elf_p.p_filesz) {
+ elf_dyn dyn;
+
+ if (sizeof(dyn) != kernel_read(vma->vm_file, elf_p.p_offset + i*sizeof(dyn), (char *)&dyn, sizeof(dyn)))
+ break;
+ if (dyn.d_tag == DT_NULL)
+ break;
+ if (dyn.d_tag == DT_TEXTREL || (dyn.d_tag == DT_FLAGS && (dyn.d_un.d_val & DF_TEXTREL))) {
+ gr_log_textrel(vma);
+ if (is_textrel_rw)
+ vma->vm_flags |= VM_MAYWRITE;
+ else
+ /* PaX: disallow write access after relocs are done, hopefully noone else needs it... */
+ vma->vm_flags &= ~VM_MAYWRITE;
+ break;
+ }
+ i++;
+ }
+ is_textrel_rw = false;
+ is_textrel_rx = false;
+ continue;
+
+ case PT_GNU_RELRO:
+ if (!is_relro)
+ continue;
+ if ((elf_p.p_offset >> PAGE_SHIFT) == vma->vm_pgoff && ELF_PAGEALIGN(elf_p.p_memsz) == vma->vm_end - vma->vm_start)
+ vma->vm_flags &= ~VM_MAYWRITE;
+ is_relro = false;
+ continue;
+
+#ifdef CONFIG_PAX_PT_PAX_FLAGS
+ case PT_PAX_FLAGS: {
+ const char *msg_mprotect = "", *msg_emutramp = "";
+ char *buffer_lib, *buffer_exe;
+
+ if (elf_p.p_flags & PF_NOMPROTECT)
+ msg_mprotect = "MPROTECT disabled";
+
+#ifdef CONFIG_PAX_EMUTRAMP
+ if (!(vma->vm_mm->pax_flags & MF_PAX_EMUTRAMP) && !(elf_p.p_flags & PF_NOEMUTRAMP))
+ msg_emutramp = "EMUTRAMP enabled";
+#endif
+
+ if (!msg_mprotect[0] && !msg_emutramp[0])
+ continue;
+
+ if (!printk_ratelimit())
+ continue;
+
+ buffer_lib = (char *)__get_free_page(GFP_KERNEL);
+ buffer_exe = (char *)__get_free_page(GFP_KERNEL);
+ if (buffer_lib && buffer_exe) {
+ char *path_lib, *path_exe;
+
+ path_lib = pax_get_path(&vma->vm_file->f_path, buffer_lib, PAGE_SIZE);
+ path_exe = pax_get_path(&vma->vm_mm->exe_file->f_path, buffer_exe, PAGE_SIZE);
+
+ pr_info("PAX: %s wants %s%s%s on %s\n", path_lib, msg_mprotect,
+ (msg_mprotect[0] && msg_emutramp[0] ? " and " : ""), msg_emutramp, path_exe);
+
+ }
+ free_page((unsigned long)buffer_exe);
+ free_page((unsigned long)buffer_lib);
+ continue;
+ }
+#endif
+
+ }
+ }
+}
+#endif
+
+#ifdef CONFIG_GRKERNSEC_RWXMAP_LOG
+
+extern int grsec_enable_log_rwxmaps;
+
+static void elf_handle_mmap(struct file *file)
+{
+ struct elfhdr elf_h;
+ struct elf_phdr elf_p;
+ unsigned long i;
+
+ if (!grsec_enable_log_rwxmaps)
+ return;
+
+ if (sizeof(elf_h) != kernel_read(file, 0UL, (char *)&elf_h, sizeof(elf_h)) ||
+ memcmp(elf_h.e_ident, ELFMAG, SELFMAG) ||
+ (elf_h.e_type != ET_DYN && elf_h.e_type != ET_EXEC) || !elf_check_arch(&elf_h) ||
+ elf_h.e_phentsize != sizeof(struct elf_phdr) ||
+ elf_h.e_phnum > 65536UL / sizeof(struct elf_phdr))
+ return;
+
+ for (i = 0UL; i < elf_h.e_phnum; i++) {
+ if (sizeof(elf_p) != kernel_read(file, elf_h.e_phoff + i*sizeof(elf_p), (char *)&elf_p, sizeof(elf_p)))
+ return;
+ if (elf_p.p_type == PT_GNU_STACK && (elf_p.p_flags & PF_X))
+ gr_log_ptgnustack(file);
+ }
+}
+#endif
+
static int __init init_elf_binfmt(void)
{
register_binfmt(&elf_format);
else if (bdev->bd_contains == bdev)
return true; /* is a whole device which isn't held */
- else if (whole->bd_holder == bd_may_claim)
+ else if (whole->bd_holder == (void *)bd_may_claim)
return true; /* is a partition of a device that is being partitioned */
else if (whole->bd_holder != NULL)
return false; /* is a partition of a held device */
free_extent_buffer(buf);
add_root_to_dirty_list(root);
} else {
- if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID)
- parent_start = parent->start;
- else
+ if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) {
+ if (parent)
+ parent_start = parent->start;
+ else
+ parent_start = 0;
+ } else
parent_start = 0;
WARN_ON(trans->transid != btrfs_header_generation(parent));
static void finish_one_item(struct btrfs_delayed_root *delayed_root)
{
- int seq = atomic_inc_return(&delayed_root->items_seq);
+ int seq = atomic_inc_return_unchecked(&delayed_root->items_seq);
if ((atomic_dec_return(&delayed_root->items) <
BTRFS_DELAYED_BACKGROUND || seq % BTRFS_DELAYED_BATCH == 0) &&
waitqueue_active(&delayed_root->wait))
static int could_end_wait(struct btrfs_delayed_root *delayed_root, int seq)
{
- int val = atomic_read(&delayed_root->items_seq);
+ int val = atomic_read_unchecked(&delayed_root->items_seq);
if (val < seq || val >= seq + BTRFS_DELAYED_BATCH)
return 1;
int seq;
int ret;
- seq = atomic_read(&delayed_root->items_seq);
+ seq = atomic_read_unchecked(&delayed_root->items_seq);
ret = btrfs_wq_run_delayed_node(delayed_root, root, 0);
if (ret)
*/
struct list_head prepare_list;
atomic_t items; /* for delayed items */
- atomic_t items_seq; /* for delayed items */
+ atomic_unchecked_t items_seq; /* for delayed items */
int nodes; /* for delayed nodes */
wait_queue_head_t wait;
};
struct btrfs_delayed_root *delayed_root)
{
atomic_set(&delayed_root->items, 0);
- atomic_set(&delayed_root->items_seq, 0);
+ atomic_set_unchecked(&delayed_root->items_seq, 0);
delayed_root->nodes = 0;
spin_lock_init(&delayed_root->lock);
init_waitqueue_head(&delayed_root->wait);
for (i = 0; i < num_types; i++) {
struct btrfs_space_info *tmp;
+ /* Don't copy in more than we allocated */
if (!slot_count)
break;
+ slot_count--;
+
info = NULL;
rcu_read_lock();
list_for_each_entry_rcu(tmp, &root->fs_info->space_info,
memcpy(dest, &space, sizeof(space));
dest++;
space_args.total_spaces++;
- slot_count--;
}
- if (!slot_count)
- break;
}
up_read(&info->groups_sem);
}
function, line, errstr);
return;
}
- ACCESS_ONCE(trans->transaction->aborted) = errno;
+ ACCESS_ONCE_RW(trans->transaction->aborted) = errno;
/* Wake up anybody who may be waiting on this transaction */
wake_up(&root->fs_info->transaction_wait);
wake_up(&root->fs_info->transaction_blocked_wait);
for (set = 0; set < FEAT_MAX; set++) {
int i;
struct attribute *attrs[2];
- struct attribute_group agroup = {
+ attribute_group_no_const agroup = {
.name = "features",
.attrs = attrs,
};
* extent entry.
*/
use_bitmap_op = cache->free_space_ctl->op->use_bitmap;
- cache->free_space_ctl->op->use_bitmap = test_use_bitmap;
+ pax_open_kernel();
+ *(void **)&cache->free_space_ctl->op->use_bitmap = test_use_bitmap;
+ pax_close_kernel();
/*
* Extent entry covering free space range [128Mb - 256Kb, 128Mb - 128Kb[
if (ret)
return ret;
- cache->free_space_ctl->op->use_bitmap = use_bitmap_op;
+ pax_open_kernel();
+ *(void **)&cache->free_space_ctl->op->use_bitmap = use_bitmap_op;
+ pax_close_kernel();
__btrfs_remove_free_space_cache(cache->free_space_ctl);
return 0;
static inline void btrfs_set_log_full_commit(struct btrfs_fs_info *fs_info,
struct btrfs_trans_handle *trans)
{
- ACCESS_ONCE(fs_info->last_trans_log_full_commit) = trans->transid;
+ ACCESS_ONCE_RW(fs_info->last_trans_log_full_commit) = trans->transid;
}
static inline int btrfs_need_log_full_commit(struct btrfs_fs_info *fs_info,
bh_cachep = kmem_cache_create("buffer_head",
sizeof(struct buffer_head), 0,
(SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
- SLAB_MEM_SPREAD),
+ SLAB_MEM_SPREAD|SLAB_NO_SANITIZE),
NULL);
/*
args);
/* start by checking things over */
- ASSERT(cache->fstop_percent >= 0 &&
- cache->fstop_percent < cache->fcull_percent &&
+ ASSERT(cache->fstop_percent < cache->fcull_percent &&
cache->fcull_percent < cache->frun_percent &&
cache->frun_percent < 100);
- ASSERT(cache->bstop_percent >= 0 &&
- cache->bstop_percent < cache->bcull_percent &&
+ ASSERT(cache->bstop_percent < cache->bcull_percent &&
cache->bcull_percent < cache->brun_percent &&
cache->brun_percent < 100);
if (n > buflen)
return -EMSGSIZE;
- if (copy_to_user(_buffer, buffer, n) != 0)
+ if (n > sizeof(buffer) || copy_to_user(_buffer, buffer, n) != 0)
return -EFAULT;
return n;
if (test_bit(CACHEFILES_DEAD, &cache->flags))
return -EIO;
- if (datalen < 0 || datalen > PAGE_SIZE - 1)
+ if (datalen > PAGE_SIZE - 1)
return -EOPNOTSUPP;
/* drag the command string into the kernel so we can parse it */
if (args[0] != '%' || args[1] != '\0')
return -EINVAL;
- if (fstop < 0 || fstop >= cache->fcull_percent)
+ if (fstop >= cache->fcull_percent)
return cachefiles_daemon_range_error(cache, args);
cache->fstop_percent = fstop;
if (args[0] != '%' || args[1] != '\0')
return -EINVAL;
- if (bstop < 0 || bstop >= cache->bcull_percent)
+ if (bstop >= cache->bcull_percent)
return cachefiles_daemon_range_error(cache, args);
cache->bstop_percent = bstop;
wait_queue_head_t daemon_pollwq; /* poll waitqueue for daemon */
struct rb_root active_nodes; /* active nodes (can't be culled) */
rwlock_t active_lock; /* lock for active_nodes */
- atomic_t gravecounter; /* graveyard uniquifier */
+ atomic_unchecked_t gravecounter; /* graveyard uniquifier */
unsigned frun_percent; /* when to stop culling (% files) */
unsigned fcull_percent; /* when to start culling (% files) */
unsigned fstop_percent; /* when to stop allocating (% files) */
* proc.c
*/
#ifdef CONFIG_CACHEFILES_HISTOGRAM
-extern atomic_t cachefiles_lookup_histogram[HZ];
-extern atomic_t cachefiles_mkdir_histogram[HZ];
-extern atomic_t cachefiles_create_histogram[HZ];
+extern atomic_unchecked_t cachefiles_lookup_histogram[HZ];
+extern atomic_unchecked_t cachefiles_mkdir_histogram[HZ];
+extern atomic_unchecked_t cachefiles_create_histogram[HZ];
extern int __init cachefiles_proc_init(void);
extern void cachefiles_proc_cleanup(void);
static inline
-void cachefiles_hist(atomic_t histogram[], unsigned long start_jif)
+void cachefiles_hist(atomic_unchecked_t histogram[], unsigned long start_jif)
{
unsigned long jif = jiffies - start_jif;
if (jif >= HZ)
jif = HZ - 1;
- atomic_inc(&histogram[jif]);
+ atomic_inc_unchecked(&histogram[jif]);
}
#else
/* first step is to make up a grave dentry in the graveyard */
sprintf(nbuffer, "%08x%08x",
(uint32_t) get_seconds(),
- (uint32_t) atomic_inc_return(&cache->gravecounter));
+ (uint32_t) atomic_inc_return_unchecked(&cache->gravecounter));
/* do the multiway lock magic */
trap = lock_rename(cache->graveyard, dir);
#include <linux/seq_file.h>
#include "internal.h"
-atomic_t cachefiles_lookup_histogram[HZ];
-atomic_t cachefiles_mkdir_histogram[HZ];
-atomic_t cachefiles_create_histogram[HZ];
+atomic_unchecked_t cachefiles_lookup_histogram[HZ];
+atomic_unchecked_t cachefiles_mkdir_histogram[HZ];
+atomic_unchecked_t cachefiles_create_histogram[HZ];
/*
* display the latency histogram
return 0;
default:
index = (unsigned long) v - 3;
- x = atomic_read(&cachefiles_lookup_histogram[index]);
- y = atomic_read(&cachefiles_mkdir_histogram[index]);
- z = atomic_read(&cachefiles_create_histogram[index]);
+ x = atomic_read_unchecked(&cachefiles_lookup_histogram[index]);
+ y = atomic_read_unchecked(&cachefiles_mkdir_histogram[index]);
+ z = atomic_read_unchecked(&cachefiles_create_histogram[index]);
if (x == 0 && y == 0 && z == 0)
return 0;
struct dentry *dentry, *last;
struct ceph_dentry_info *di;
int err = 0;
+ char d_name[DNAME_INLINE_LEN];
+ const unsigned char *name;
/* claim ref on last dentry we returned */
last = fi->dentry;
dout(" %llu (%llu) dentry %p %pd %p\n", di->offset, ctx->pos,
dentry, dentry, dentry->d_inode);
- if (!dir_emit(ctx, dentry->d_name.name,
+ name = dentry->d_name.name;
+ if (name == dentry->d_iname) {
+ memcpy(d_name, name, dentry->d_name.len);
+ name = d_name;
+ }
+ if (!dir_emit(ctx, name,
dentry->d_name.len,
ceph_translate_ino(dentry->d_sb, dentry->d_inode->i_ino),
dentry->d_inode->i_mode >> 12)) {
struct ceph_fs_client *fsc = ceph_inode_to_client(inode);
struct ceph_mds_client *mdsc = fsc->mdsc;
unsigned frag = fpos_frag(ctx->pos);
- int off = fpos_off(ctx->pos);
+ unsigned int off = fpos_off(ctx->pos);
int err;
u32 ftype;
struct ceph_mds_reply_info_parsed *rinfo;
/*
* construct our own bdi so we can control readahead, etc.
*/
-static atomic_long_t bdi_seq = ATOMIC_LONG_INIT(0);
+static atomic_long_unchecked_t bdi_seq = ATOMIC_LONG_INIT(0);
static int ceph_register_bdi(struct super_block *sb,
struct ceph_fs_client *fsc)
default_backing_dev_info.ra_pages;
err = bdi_register(&fsc->backing_dev_info, NULL, "ceph-%ld",
- atomic_long_inc_return(&bdi_seq));
+ atomic_long_inc_return_unchecked(&bdi_seq));
if (!err)
sb->s_bdi = &fsc->backing_dev_info;
return err;
if (strtobool(&c, &bv) == 0) {
#ifdef CONFIG_CIFS_STATS2
- atomic_set(&totBufAllocCount, 0);
- atomic_set(&totSmBufAllocCount, 0);
+ atomic_set_unchecked(&totBufAllocCount, 0);
+ atomic_set_unchecked(&totSmBufAllocCount, 0);
#endif /* CONFIG_CIFS_STATS2 */
spin_lock(&cifs_tcp_ses_lock);
list_for_each(tmp1, &cifs_tcp_ses_list) {
tcon = list_entry(tmp3,
struct cifs_tcon,
tcon_list);
- atomic_set(&tcon->num_smbs_sent, 0);
+ atomic_set_unchecked(&tcon->num_smbs_sent, 0);
if (server->ops->clear_stats)
server->ops->clear_stats(tcon);
}
smBufAllocCount.counter, cifs_min_small);
#ifdef CONFIG_CIFS_STATS2
seq_printf(m, "Total Large %d Small %d Allocations\n",
- atomic_read(&totBufAllocCount),
- atomic_read(&totSmBufAllocCount));
+ atomic_read_unchecked(&totBufAllocCount),
+ atomic_read_unchecked(&totSmBufAllocCount));
#endif /* CONFIG_CIFS_STATS2 */
seq_printf(m, "Operations (MIDs): %d\n", atomic_read(&midCount));
if (tcon->need_reconnect)
seq_puts(m, "\tDISCONNECTED ");
seq_printf(m, "\nSMBs: %d",
- atomic_read(&tcon->num_smbs_sent));
+ atomic_read_unchecked(&tcon->num_smbs_sent));
if (server->ops->print_stats)
server->ops->print_stats(m, tcon);
}
*/
cifs_req_cachep = kmem_cache_create("cifs_request",
CIFSMaxBufSize + max_hdr_size, 0,
- SLAB_HWCACHE_ALIGN, NULL);
+ SLAB_HWCACHE_ALIGN | SLAB_USERCOPY, NULL);
if (cifs_req_cachep == NULL)
return -ENOMEM;
efficient to alloc 1 per page off the slab compared to 17K (5page)
alloc of large cifs buffers even when page debugging is on */
cifs_sm_req_cachep = kmem_cache_create("cifs_small_rq",
- MAX_CIFS_SMALL_BUFFER_SIZE, 0, SLAB_HWCACHE_ALIGN,
+ MAX_CIFS_SMALL_BUFFER_SIZE, 0, SLAB_HWCACHE_ALIGN | SLAB_USERCOPY,
NULL);
if (cifs_sm_req_cachep == NULL) {
mempool_destroy(cifs_req_poolp);
atomic_set(&bufAllocCount, 0);
atomic_set(&smBufAllocCount, 0);
#ifdef CONFIG_CIFS_STATS2
- atomic_set(&totBufAllocCount, 0);
- atomic_set(&totSmBufAllocCount, 0);
+ atomic_set_unchecked(&totBufAllocCount, 0);
+ atomic_set_unchecked(&totSmBufAllocCount, 0);
#endif /* CONFIG_CIFS_STATS2 */
atomic_set(&midCount, 0);
__u16 Flags; /* optional support bits */
enum statusEnum tidStatus;
#ifdef CONFIG_CIFS_STATS
- atomic_t num_smbs_sent;
+ atomic_unchecked_t num_smbs_sent;
union {
struct {
- atomic_t num_writes;
- atomic_t num_reads;
- atomic_t num_flushes;
- atomic_t num_oplock_brks;
- atomic_t num_opens;
- atomic_t num_closes;
- atomic_t num_deletes;
- atomic_t num_mkdirs;
- atomic_t num_posixopens;
- atomic_t num_posixmkdirs;
- atomic_t num_rmdirs;
- atomic_t num_renames;
- atomic_t num_t2renames;
- atomic_t num_ffirst;
- atomic_t num_fnext;
- atomic_t num_fclose;
- atomic_t num_hardlinks;
- atomic_t num_symlinks;
- atomic_t num_locks;
- atomic_t num_acl_get;
- atomic_t num_acl_set;
+ atomic_unchecked_t num_writes;
+ atomic_unchecked_t num_reads;
+ atomic_unchecked_t num_flushes;
+ atomic_unchecked_t num_oplock_brks;
+ atomic_unchecked_t num_opens;
+ atomic_unchecked_t num_closes;
+ atomic_unchecked_t num_deletes;
+ atomic_unchecked_t num_mkdirs;
+ atomic_unchecked_t num_posixopens;
+ atomic_unchecked_t num_posixmkdirs;
+ atomic_unchecked_t num_rmdirs;
+ atomic_unchecked_t num_renames;
+ atomic_unchecked_t num_t2renames;
+ atomic_unchecked_t num_ffirst;
+ atomic_unchecked_t num_fnext;
+ atomic_unchecked_t num_fclose;
+ atomic_unchecked_t num_hardlinks;
+ atomic_unchecked_t num_symlinks;
+ atomic_unchecked_t num_locks;
+ atomic_unchecked_t num_acl_get;
+ atomic_unchecked_t num_acl_set;
} cifs_stats;
#ifdef CONFIG_CIFS_SMB2
struct {
- atomic_t smb2_com_sent[NUMBER_OF_SMB2_COMMANDS];
- atomic_t smb2_com_failed[NUMBER_OF_SMB2_COMMANDS];
+ atomic_unchecked_t smb2_com_sent[NUMBER_OF_SMB2_COMMANDS];
+ atomic_unchecked_t smb2_com_failed[NUMBER_OF_SMB2_COMMANDS];
} smb2_stats;
#endif /* CONFIG_CIFS_SMB2 */
} stats;
}
#ifdef CONFIG_CIFS_STATS
-#define cifs_stats_inc atomic_inc
+#define cifs_stats_inc atomic_inc_unchecked
static inline void cifs_stats_bytes_written(struct cifs_tcon *tcon,
unsigned int bytes)
/* Various Debug counters */
GLOBAL_EXTERN atomic_t bufAllocCount; /* current number allocated */
#ifdef CONFIG_CIFS_STATS2
-GLOBAL_EXTERN atomic_t totBufAllocCount; /* total allocated over all time */
-GLOBAL_EXTERN atomic_t totSmBufAllocCount;
+GLOBAL_EXTERN atomic_unchecked_t totBufAllocCount; /* total allocated over all time */
+GLOBAL_EXTERN atomic_unchecked_t totSmBufAllocCount;
#endif
GLOBAL_EXTERN atomic_t smBufAllocCount;
GLOBAL_EXTERN atomic_t midCount;
index = mapping->writeback_index; /* Start from prev offset */
end = -1;
} else {
- index = wbc->range_start >> PAGE_CACHE_SHIFT;
- end = wbc->range_end >> PAGE_CACHE_SHIFT;
- if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
+ if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) {
range_whole = true;
+ index = 0;
+ end = ULONG_MAX;
+ } else {
+ index = wbc->range_start >> PAGE_CACHE_SHIFT;
+ end = wbc->range_end >> PAGE_CACHE_SHIFT;
+ }
scanned = true;
}
server = cifs_sb_master_tcon(cifs_sb)->ses->server;
memset(ret_buf, 0, buf_size + 3);
atomic_inc(&bufAllocCount);
#ifdef CONFIG_CIFS_STATS2
- atomic_inc(&totBufAllocCount);
+ atomic_inc_unchecked(&totBufAllocCount);
#endif /* CONFIG_CIFS_STATS2 */
}
/* memset(ret_buf, 0, sizeof(struct smb_hdr) + 27);*/
atomic_inc(&smBufAllocCount);
#ifdef CONFIG_CIFS_STATS2
- atomic_inc(&totSmBufAllocCount);
+ atomic_inc_unchecked(&totSmBufAllocCount);
#endif /* CONFIG_CIFS_STATS2 */
}
cifs_clear_stats(struct cifs_tcon *tcon)
{
#ifdef CONFIG_CIFS_STATS
- atomic_set(&tcon->stats.cifs_stats.num_writes, 0);
- atomic_set(&tcon->stats.cifs_stats.num_reads, 0);
- atomic_set(&tcon->stats.cifs_stats.num_flushes, 0);
- atomic_set(&tcon->stats.cifs_stats.num_oplock_brks, 0);
- atomic_set(&tcon->stats.cifs_stats.num_opens, 0);
- atomic_set(&tcon->stats.cifs_stats.num_posixopens, 0);
- atomic_set(&tcon->stats.cifs_stats.num_posixmkdirs, 0);
- atomic_set(&tcon->stats.cifs_stats.num_closes, 0);
- atomic_set(&tcon->stats.cifs_stats.num_deletes, 0);
- atomic_set(&tcon->stats.cifs_stats.num_mkdirs, 0);
- atomic_set(&tcon->stats.cifs_stats.num_rmdirs, 0);
- atomic_set(&tcon->stats.cifs_stats.num_renames, 0);
- atomic_set(&tcon->stats.cifs_stats.num_t2renames, 0);
- atomic_set(&tcon->stats.cifs_stats.num_ffirst, 0);
- atomic_set(&tcon->stats.cifs_stats.num_fnext, 0);
- atomic_set(&tcon->stats.cifs_stats.num_fclose, 0);
- atomic_set(&tcon->stats.cifs_stats.num_hardlinks, 0);
- atomic_set(&tcon->stats.cifs_stats.num_symlinks, 0);
- atomic_set(&tcon->stats.cifs_stats.num_locks, 0);
- atomic_set(&tcon->stats.cifs_stats.num_acl_get, 0);
- atomic_set(&tcon->stats.cifs_stats.num_acl_set, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_writes, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_reads, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_flushes, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_oplock_brks, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_opens, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_posixopens, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_posixmkdirs, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_closes, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_deletes, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_mkdirs, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_rmdirs, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_renames, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_t2renames, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_ffirst, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_fnext, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_fclose, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_hardlinks, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_symlinks, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_locks, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_acl_get, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_acl_set, 0);
#endif
}
{
#ifdef CONFIG_CIFS_STATS
seq_printf(m, " Oplocks breaks: %d",
- atomic_read(&tcon->stats.cifs_stats.num_oplock_brks));
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_oplock_brks));
seq_printf(m, "\nReads: %d Bytes: %llu",
- atomic_read(&tcon->stats.cifs_stats.num_reads),
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_reads),
(long long)(tcon->bytes_read));
seq_printf(m, "\nWrites: %d Bytes: %llu",
- atomic_read(&tcon->stats.cifs_stats.num_writes),
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_writes),
(long long)(tcon->bytes_written));
seq_printf(m, "\nFlushes: %d",
- atomic_read(&tcon->stats.cifs_stats.num_flushes));
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_flushes));
seq_printf(m, "\nLocks: %d HardLinks: %d Symlinks: %d",
- atomic_read(&tcon->stats.cifs_stats.num_locks),
- atomic_read(&tcon->stats.cifs_stats.num_hardlinks),
- atomic_read(&tcon->stats.cifs_stats.num_symlinks));
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_locks),
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_hardlinks),
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_symlinks));
seq_printf(m, "\nOpens: %d Closes: %d Deletes: %d",
- atomic_read(&tcon->stats.cifs_stats.num_opens),
- atomic_read(&tcon->stats.cifs_stats.num_closes),
- atomic_read(&tcon->stats.cifs_stats.num_deletes));
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_opens),
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_closes),
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_deletes));
seq_printf(m, "\nPosix Opens: %d Posix Mkdirs: %d",
- atomic_read(&tcon->stats.cifs_stats.num_posixopens),
- atomic_read(&tcon->stats.cifs_stats.num_posixmkdirs));
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_posixopens),
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_posixmkdirs));
seq_printf(m, "\nMkdirs: %d Rmdirs: %d",
- atomic_read(&tcon->stats.cifs_stats.num_mkdirs),
- atomic_read(&tcon->stats.cifs_stats.num_rmdirs));
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_mkdirs),
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_rmdirs));
seq_printf(m, "\nRenames: %d T2 Renames %d",
- atomic_read(&tcon->stats.cifs_stats.num_renames),
- atomic_read(&tcon->stats.cifs_stats.num_t2renames));
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_renames),
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_t2renames));
seq_printf(m, "\nFindFirst: %d FNext %d FClose %d",
- atomic_read(&tcon->stats.cifs_stats.num_ffirst),
- atomic_read(&tcon->stats.cifs_stats.num_fnext),
- atomic_read(&tcon->stats.cifs_stats.num_fclose));
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_ffirst),
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_fnext),
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_fclose));
#endif
}
#ifdef CONFIG_CIFS_STATS
int i;
for (i = 0; i < NUMBER_OF_SMB2_COMMANDS; i++) {
- atomic_set(&tcon->stats.smb2_stats.smb2_com_sent[i], 0);
- atomic_set(&tcon->stats.smb2_stats.smb2_com_failed[i], 0);
+ atomic_set_unchecked(&tcon->stats.smb2_stats.smb2_com_sent[i], 0);
+ atomic_set_unchecked(&tcon->stats.smb2_stats.smb2_com_failed[i], 0);
}
#endif
}
smb2_print_stats(struct seq_file *m, struct cifs_tcon *tcon)
{
#ifdef CONFIG_CIFS_STATS
- atomic_t *sent = tcon->stats.smb2_stats.smb2_com_sent;
- atomic_t *failed = tcon->stats.smb2_stats.smb2_com_failed;
+ atomic_unchecked_t *sent = tcon->stats.smb2_stats.smb2_com_sent;
+ atomic_unchecked_t *failed = tcon->stats.smb2_stats.smb2_com_failed;
seq_printf(m, "\nNegotiates: %d sent %d failed",
- atomic_read(&sent[SMB2_NEGOTIATE_HE]),
- atomic_read(&failed[SMB2_NEGOTIATE_HE]));
+ atomic_read_unchecked(&sent[SMB2_NEGOTIATE_HE]),
+ atomic_read_unchecked(&failed[SMB2_NEGOTIATE_HE]));
seq_printf(m, "\nSessionSetups: %d sent %d failed",
- atomic_read(&sent[SMB2_SESSION_SETUP_HE]),
- atomic_read(&failed[SMB2_SESSION_SETUP_HE]));
+ atomic_read_unchecked(&sent[SMB2_SESSION_SETUP_HE]),
+ atomic_read_unchecked(&failed[SMB2_SESSION_SETUP_HE]));
seq_printf(m, "\nLogoffs: %d sent %d failed",
- atomic_read(&sent[SMB2_LOGOFF_HE]),
- atomic_read(&failed[SMB2_LOGOFF_HE]));
+ atomic_read_unchecked(&sent[SMB2_LOGOFF_HE]),
+ atomic_read_unchecked(&failed[SMB2_LOGOFF_HE]));
seq_printf(m, "\nTreeConnects: %d sent %d failed",
- atomic_read(&sent[SMB2_TREE_CONNECT_HE]),
- atomic_read(&failed[SMB2_TREE_CONNECT_HE]));
+ atomic_read_unchecked(&sent[SMB2_TREE_CONNECT_HE]),
+ atomic_read_unchecked(&failed[SMB2_TREE_CONNECT_HE]));
seq_printf(m, "\nTreeDisconnects: %d sent %d failed",
- atomic_read(&sent[SMB2_TREE_DISCONNECT_HE]),
- atomic_read(&failed[SMB2_TREE_DISCONNECT_HE]));
+ atomic_read_unchecked(&sent[SMB2_TREE_DISCONNECT_HE]),
+ atomic_read_unchecked(&failed[SMB2_TREE_DISCONNECT_HE]));
seq_printf(m, "\nCreates: %d sent %d failed",
- atomic_read(&sent[SMB2_CREATE_HE]),
- atomic_read(&failed[SMB2_CREATE_HE]));
+ atomic_read_unchecked(&sent[SMB2_CREATE_HE]),
+ atomic_read_unchecked(&failed[SMB2_CREATE_HE]));
seq_printf(m, "\nCloses: %d sent %d failed",
- atomic_read(&sent[SMB2_CLOSE_HE]),
- atomic_read(&failed[SMB2_CLOSE_HE]));
+ atomic_read_unchecked(&sent[SMB2_CLOSE_HE]),
+ atomic_read_unchecked(&failed[SMB2_CLOSE_HE]));
seq_printf(m, "\nFlushes: %d sent %d failed",
- atomic_read(&sent[SMB2_FLUSH_HE]),
- atomic_read(&failed[SMB2_FLUSH_HE]));
+ atomic_read_unchecked(&sent[SMB2_FLUSH_HE]),
+ atomic_read_unchecked(&failed[SMB2_FLUSH_HE]));
seq_printf(m, "\nReads: %d sent %d failed",
- atomic_read(&sent[SMB2_READ_HE]),
- atomic_read(&failed[SMB2_READ_HE]));
+ atomic_read_unchecked(&sent[SMB2_READ_HE]),
+ atomic_read_unchecked(&failed[SMB2_READ_HE]));
seq_printf(m, "\nWrites: %d sent %d failed",
- atomic_read(&sent[SMB2_WRITE_HE]),
- atomic_read(&failed[SMB2_WRITE_HE]));
+ atomic_read_unchecked(&sent[SMB2_WRITE_HE]),
+ atomic_read_unchecked(&failed[SMB2_WRITE_HE]));
seq_printf(m, "\nLocks: %d sent %d failed",
- atomic_read(&sent[SMB2_LOCK_HE]),
- atomic_read(&failed[SMB2_LOCK_HE]));
+ atomic_read_unchecked(&sent[SMB2_LOCK_HE]),
+ atomic_read_unchecked(&failed[SMB2_LOCK_HE]));
seq_printf(m, "\nIOCTLs: %d sent %d failed",
- atomic_read(&sent[SMB2_IOCTL_HE]),
- atomic_read(&failed[SMB2_IOCTL_HE]));
+ atomic_read_unchecked(&sent[SMB2_IOCTL_HE]),
+ atomic_read_unchecked(&failed[SMB2_IOCTL_HE]));
seq_printf(m, "\nCancels: %d sent %d failed",
- atomic_read(&sent[SMB2_CANCEL_HE]),
- atomic_read(&failed[SMB2_CANCEL_HE]));
+ atomic_read_unchecked(&sent[SMB2_CANCEL_HE]),
+ atomic_read_unchecked(&failed[SMB2_CANCEL_HE]));
seq_printf(m, "\nEchos: %d sent %d failed",
- atomic_read(&sent[SMB2_ECHO_HE]),
- atomic_read(&failed[SMB2_ECHO_HE]));
+ atomic_read_unchecked(&sent[SMB2_ECHO_HE]),
+ atomic_read_unchecked(&failed[SMB2_ECHO_HE]));
seq_printf(m, "\nQueryDirectories: %d sent %d failed",
- atomic_read(&sent[SMB2_QUERY_DIRECTORY_HE]),
- atomic_read(&failed[SMB2_QUERY_DIRECTORY_HE]));
+ atomic_read_unchecked(&sent[SMB2_QUERY_DIRECTORY_HE]),
+ atomic_read_unchecked(&failed[SMB2_QUERY_DIRECTORY_HE]));
seq_printf(m, "\nChangeNotifies: %d sent %d failed",
- atomic_read(&sent[SMB2_CHANGE_NOTIFY_HE]),
- atomic_read(&failed[SMB2_CHANGE_NOTIFY_HE]));
+ atomic_read_unchecked(&sent[SMB2_CHANGE_NOTIFY_HE]),
+ atomic_read_unchecked(&failed[SMB2_CHANGE_NOTIFY_HE]));
seq_printf(m, "\nQueryInfos: %d sent %d failed",
- atomic_read(&sent[SMB2_QUERY_INFO_HE]),
- atomic_read(&failed[SMB2_QUERY_INFO_HE]));
+ atomic_read_unchecked(&sent[SMB2_QUERY_INFO_HE]),
+ atomic_read_unchecked(&failed[SMB2_QUERY_INFO_HE]));
seq_printf(m, "\nSetInfos: %d sent %d failed",
- atomic_read(&sent[SMB2_SET_INFO_HE]),
- atomic_read(&failed[SMB2_SET_INFO_HE]));
+ atomic_read_unchecked(&sent[SMB2_SET_INFO_HE]),
+ atomic_read_unchecked(&failed[SMB2_SET_INFO_HE]));
seq_printf(m, "\nOplockBreaks: %d sent %d failed",
- atomic_read(&sent[SMB2_OPLOCK_BREAK_HE]),
- atomic_read(&failed[SMB2_OPLOCK_BREAK_HE]));
+ atomic_read_unchecked(&sent[SMB2_OPLOCK_BREAK_HE]),
+ atomic_read_unchecked(&failed[SMB2_OPLOCK_BREAK_HE]));
#endif
}
default:
cifs_dbg(VFS, "info level %u isn't supported\n",
srch_inf->info_level);
- rc = -EINVAL;
- goto qdir_exit;
+ return -EINVAL;
}
req->FileIndex = cpu_to_le32(index);
#include "coda_linux.h"
#include "coda_cache.h"
-static atomic_t permission_epoch = ATOMIC_INIT(0);
+static atomic_unchecked_t permission_epoch = ATOMIC_INIT(0);
/* replace or extend an acl cache hit */
void coda_cache_enter(struct inode *inode, int mask)
struct coda_inode_info *cii = ITOC(inode);
spin_lock(&cii->c_lock);
- cii->c_cached_epoch = atomic_read(&permission_epoch);
+ cii->c_cached_epoch = atomic_read_unchecked(&permission_epoch);
if (!uid_eq(cii->c_uid, current_fsuid())) {
cii->c_uid = current_fsuid();
cii->c_cached_perm = mask;
{
struct coda_inode_info *cii = ITOC(inode);
spin_lock(&cii->c_lock);
- cii->c_cached_epoch = atomic_read(&permission_epoch) - 1;
+ cii->c_cached_epoch = atomic_read_unchecked(&permission_epoch) - 1;
spin_unlock(&cii->c_lock);
}
/* remove all acl caches */
void coda_cache_clear_all(struct super_block *sb)
{
- atomic_inc(&permission_epoch);
+ atomic_inc_unchecked(&permission_epoch);
}
spin_lock(&cii->c_lock);
hit = (mask & cii->c_cached_perm) == mask &&
uid_eq(cii->c_uid, current_fsuid()) &&
- cii->c_cached_epoch == atomic_read(&permission_epoch);
+ cii->c_cached_epoch == atomic_read_unchecked(&permission_epoch);
spin_unlock(&cii->c_lock);
return hit;
#include <asm/ioctls.h>
#include "internal.h"
-int compat_log = 1;
+int compat_log = 0;
int compat_printk(const char *fmt, ...)
{
set_fs(KERNEL_DS);
/* The __user pointer cast is valid because of the set_fs() */
- ret = sys_io_setup(nr_reqs, (aio_context_t __user *) &ctx64);
+ ret = sys_io_setup(nr_reqs, (aio_context_t __force_user *) &ctx64);
set_fs(oldfs);
/* truncating is ok because it's a user address */
if (!ret)
goto out;
ret = -EINVAL;
- if (nr_segs > UIO_MAXIOV || nr_segs < 0)
+ if (nr_segs > UIO_MAXIOV)
goto out;
if (nr_segs > fast_segs) {
ret = -ENOMEM;
struct compat_readdir_callback {
struct dir_context ctx;
struct compat_old_linux_dirent __user *dirent;
+ struct file * file;
int result;
};
buf->result = -EOVERFLOW;
return -EOVERFLOW;
}
+
+ if (!gr_acl_handle_filldir(buf->file, name, namlen, ino))
+ return 0;
+
buf->result++;
dirent = buf->dirent;
if (!access_ok(VERIFY_WRITE, dirent,
if (!f.file)
return -EBADF;
+ buf.file = f.file;
error = iterate_dir(f.file, &buf.ctx);
if (buf.result)
error = buf.result;
struct dir_context ctx;
struct compat_linux_dirent __user *current_dir;
struct compat_linux_dirent __user *previous;
+ struct file * file;
int count;
int error;
};
buf->error = -EOVERFLOW;
return -EOVERFLOW;
}
+
+ if (!gr_acl_handle_filldir(buf->file, name, namlen, ino))
+ return 0;
+
dirent = buf->previous;
if (dirent) {
if (__put_user(offset, &dirent->d_off))
if (!f.file)
return -EBADF;
+ buf.file = f.file;
error = iterate_dir(f.file, &buf.ctx);
if (error >= 0)
error = buf.error;
struct dir_context ctx;
struct linux_dirent64 __user *current_dir;
struct linux_dirent64 __user *previous;
+ struct file * file;
int count;
int error;
};
buf->error = -EINVAL; /* only used if we fail.. */
if (reclen > buf->count)
return -EINVAL;
+
+ if (!gr_acl_handle_filldir(buf->file, name, namlen, ino))
+ return 0;
+
dirent = buf->previous;
if (dirent) {
if (!f.file)
return -EBADF;
+ buf.file = f.file;
error = iterate_dir(f.file, &buf.ctx);
if (error >= 0)
error = buf.error;
#undef elf_phdr
#undef elf_shdr
#undef elf_note
+#undef elf_dyn
#undef elf_addr_t
#define elfhdr elf32_hdr
#define elf_phdr elf32_phdr
#define elf_shdr elf32_shdr
#define elf_note elf32_note
+#define elf_dyn Elf32_Dyn
#define elf_addr_t Elf32_Addr
/*
return -EFAULT;
if (__get_user(udata, &ss32->iomem_base))
return -EFAULT;
- ss.iomem_base = compat_ptr(udata);
+ ss.iomem_base = (unsigned char __force_kernel *)compat_ptr(udata);
if (__get_user(ss.iomem_reg_shift, &ss32->iomem_reg_shift) ||
__get_user(ss.port_high, &ss32->port_high))
return -EFAULT;
for (i = 0; i < nmsgs; i++) {
if (copy_in_user(&tmsgs[i].addr, &umsgs[i].addr, 3*sizeof(u16)))
return -EFAULT;
- if (get_user(datap, &umsgs[i].buf) ||
- put_user(compat_ptr(datap), &tmsgs[i].buf))
+ if (get_user(datap, (compat_caddr_t __user *)&umsgs[i].buf) ||
+ put_user(compat_ptr(datap), (u8 __user * __user *)&tmsgs[i].buf))
return -EFAULT;
}
return sys_ioctl(fd, cmd, (unsigned long)tdata);
copy_in_user(&p->l_len, &p32->l_len, sizeof(s64)) ||
copy_in_user(&p->l_sysid, &p32->l_sysid, sizeof(s32)) ||
copy_in_user(&p->l_pid, &p32->l_pid, sizeof(u32)) ||
- copy_in_user(&p->l_pad, &p32->l_pad, 4*sizeof(u32)))
+ copy_in_user(p->l_pad, p32->l_pad, 4*sizeof(u32)))
return -EFAULT;
return ioctl_preallocate(file, p);
static int __init init_sys32_ioctl_cmp(const void *p, const void *q)
{
unsigned int a, b;
- a = *(unsigned int *)p;
- b = *(unsigned int *)q;
+ a = *(const unsigned int *)p;
+ b = *(const unsigned int *)q;
if (a > b)
return 1;
if (a < b)
}
for (p = q->next; p != &parent_sd->s_children; p = p->next) {
struct configfs_dirent *next;
- const char *name;
+ const unsigned char * name;
+ char d_name[sizeof(next->s_dentry->d_iname)];
int len;
struct inode *inode = NULL;
continue;
name = configfs_get_name(next);
- len = strlen(name);
+ if (next->s_dentry && name == next->s_dentry->d_iname) {
+ len = next->s_dentry->d_name.len;
+ memcpy(d_name, name, len);
+ name = d_name;
+ } else
+ len = strlen(name);
/*
* We'll have a dentry and an inode for
struct pipe_inode_info *pipe = file->private_data;
pipe_lock(pipe);
- pipe->readers++;
- pipe->writers--;
+ atomic_inc(&pipe->readers);
+ atomic_dec(&pipe->writers);
wake_up_interruptible_sync(&pipe->wait);
kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
pipe_unlock(pipe);
* We actually want wait_event_freezable() but then we need
* to clear TIF_SIGPENDING and improve dump_interrupted().
*/
- wait_event_interruptible(pipe->wait, pipe->readers == 1);
+ wait_event_interruptible(pipe->wait, atomic_read(&pipe->readers) == 1);
pipe_lock(pipe);
- pipe->readers--;
- pipe->writers++;
+ atomic_dec(&pipe->readers);
+ atomic_inc(&pipe->writers);
pipe_unlock(pipe);
}
struct files_struct *displaced;
bool need_nonrelative = false;
bool core_dumped = false;
- static atomic_t core_dump_count = ATOMIC_INIT(0);
+ static atomic_unchecked_t core_dump_count = ATOMIC_INIT(0);
+ long signr = siginfo->si_signo;
+ int dumpable;
struct coredump_params cprm = {
.siginfo = siginfo,
.regs = signal_pt_regs(),
.mm_flags = mm->flags,
};
- audit_core_dumps(siginfo->si_signo);
+ audit_core_dumps(signr);
+
+ dumpable = __get_dumpable(cprm.mm_flags);
+
+ if (signr == SIGSEGV || signr == SIGBUS || signr == SIGKILL || signr == SIGILL)
+ gr_handle_brute_attach(dumpable);
binfmt = mm->binfmt;
if (!binfmt || !binfmt->core_dump)
goto fail;
- if (!__get_dumpable(cprm.mm_flags))
+ if (!dumpable)
goto fail;
cred = prepare_creds();
need_nonrelative = true;
}
- retval = coredump_wait(siginfo->si_signo, &core_state);
+ retval = coredump_wait(signr, &core_state);
if (retval < 0)
goto fail_creds;
}
cprm.limit = RLIM_INFINITY;
- dump_count = atomic_inc_return(&core_dump_count);
+ dump_count = atomic_inc_return_unchecked(&core_dump_count);
if (core_pipe_limit && (core_pipe_limit < dump_count)) {
printk(KERN_WARNING "Pid %d(%s) over core_pipe_limit\n",
task_tgid_vnr(current), current->comm);
} else {
struct inode *inode;
+ gr_learn_resource(current, RLIMIT_CORE, binfmt->min_coredump, 1);
+
if (cprm.limit < binfmt->min_coredump)
goto fail_unlock;
filp_close(cprm.file, NULL);
fail_dropcount:
if (ispipe)
- atomic_dec(&core_dump_count);
+ atomic_dec_unchecked(&core_dump_count);
fail_unlock:
kfree(cn.corename);
coredump_finish(mm, core_dumped);
struct file *file = cprm->file;
loff_t pos = file->f_pos;
ssize_t n;
+
+ gr_learn_resource(current, RLIMIT_CORE, cprm->written + nr, 1);
if (cprm->written + nr > cprm->limit)
return 0;
while (nr) {
* dentry_iput drops the locks, at which point nobody (except
* transient RCU lookups) can reach this dentry.
*/
- BUG_ON((int)dentry->d_lockref.count > 0);
+ BUG_ON((int)__lockref_read(&dentry->d_lockref) > 0);
this_cpu_dec(nr_dentry);
if (dentry->d_op && dentry->d_op->d_release)
dentry->d_op->d_release(dentry);
struct dentry *parent = dentry->d_parent;
if (IS_ROOT(dentry))
return NULL;
- if (unlikely((int)dentry->d_lockref.count < 0))
+ if (unlikely((int)__lockref_read(&dentry->d_lockref) < 0))
return NULL;
if (likely(spin_trylock(&parent->d_lock)))
return parent;
dentry->d_flags |= DCACHE_REFERENCED;
dentry_lru_add(dentry);
- dentry->d_lockref.count--;
+ __lockref_dec(&dentry->d_lockref);
spin_unlock(&dentry->d_lock);
return;
/* This must be called with d_lock held */
static inline void __dget_dlock(struct dentry *dentry)
{
- dentry->d_lockref.count++;
+ __lockref_inc(&dentry->d_lockref);
}
static inline void __dget(struct dentry *dentry)
goto repeat;
}
rcu_read_unlock();
- BUG_ON(!ret->d_lockref.count);
- ret->d_lockref.count++;
+ BUG_ON(!__lockref_read(&ret->d_lockref));
+ __lockref_inc(&ret->d_lockref);
spin_unlock(&ret->d_lock);
return ret;
}
spin_lock(&inode->i_lock);
hlist_for_each_entry(dentry, &inode->i_dentry, d_u.d_alias) {
spin_lock(&dentry->d_lock);
- if (!dentry->d_lockref.count) {
+ if (!__lockref_read(&dentry->d_lockref)) {
struct dentry *parent = lock_parent(dentry);
- if (likely(!dentry->d_lockref.count)) {
+ if (likely(!__lockref_read(&dentry->d_lockref))) {
__dentry_kill(dentry);
dput(parent);
goto restart;
* We found an inuse dentry which was not removed from
* the LRU because of laziness during lookup. Do not free it.
*/
- if ((int)dentry->d_lockref.count > 0) {
+ if ((int)__lockref_read(&dentry->d_lockref) > 0) {
spin_unlock(&dentry->d_lock);
if (parent)
spin_unlock(&parent->d_lock);
dentry = parent;
while (dentry && !lockref_put_or_lock(&dentry->d_lockref)) {
parent = lock_parent(dentry);
- if (dentry->d_lockref.count != 1) {
- dentry->d_lockref.count--;
+ if (__lockref_read(&dentry->d_lockref) != 1) {
+ __lockref_inc(&dentry->d_lockref);
spin_unlock(&dentry->d_lock);
if (parent)
spin_unlock(&parent->d_lock);
* counts, just remove them from the LRU. Otherwise give them
* another pass through the LRU.
*/
- if (dentry->d_lockref.count) {
+ if (__lockref_read(&dentry->d_lockref) > 0) {
d_lru_isolate(dentry);
spin_unlock(&dentry->d_lock);
return LRU_REMOVED;
} else {
if (dentry->d_flags & DCACHE_LRU_LIST)
d_lru_del(dentry);
- if (!dentry->d_lockref.count) {
+ if (!__lockref_read(&dentry->d_lockref)) {
d_shrink_add(dentry, &data->dispose);
data->found++;
}
return D_WALK_CONTINUE;
/* root with refcount 1 is fine */
- if (dentry == _data && dentry->d_lockref.count == 1)
+ if (dentry == _data && __lockref_read(&dentry->d_lockref) == 1)
return D_WALK_CONTINUE;
printk(KERN_ERR "BUG: Dentry %p{i=%lx,n=%pd} "
dentry->d_inode ?
dentry->d_inode->i_ino : 0UL,
dentry,
- dentry->d_lockref.count,
+ __lockref_read(&dentry->d_lockref),
dentry->d_sb->s_type->name,
dentry->d_sb->s_id);
WARN_ON(1);
dentry->d_iname[DNAME_INLINE_LEN-1] = 0;
if (name->len > DNAME_INLINE_LEN-1) {
size_t size = offsetof(struct external_name, name[1]);
- struct external_name *p = kmalloc(size + name->len, GFP_KERNEL);
+ struct external_name *p = kmalloc(round_up(size + name->len, sizeof(unsigned long)), GFP_KERNEL);
if (!p) {
kmem_cache_free(dentry_cache, dentry);
return NULL;
smp_wmb();
dentry->d_name.name = dname;
- dentry->d_lockref.count = 1;
+ __lockref_set(&dentry->d_lockref, 1);
dentry->d_flags = 0;
spin_lock_init(&dentry->d_lock);
seqcount_init(&dentry->d_seq);
dentry->d_sb = sb;
dentry->d_op = NULL;
dentry->d_fsdata = NULL;
+#ifdef CONFIG_GRKERNSEC_CHROOT_RENAME
+ atomic_set(&dentry->chroot_refcnt, 0);
+#endif
INIT_HLIST_BL_NODE(&dentry->d_hash);
INIT_LIST_HEAD(&dentry->d_lru);
INIT_LIST_HEAD(&dentry->d_subdirs);
goto next;
}
- dentry->d_lockref.count++;
+ __lockref_inc(&dentry->d_lockref);
found = dentry;
spin_unlock(&dentry->d_lock);
break;
spin_lock(&dentry->d_lock);
inode = dentry->d_inode;
isdir = S_ISDIR(inode->i_mode);
- if (dentry->d_lockref.count == 1) {
+ if (__lockref_read(&dentry->d_lockref) == 1) {
if (!spin_trylock(&inode->i_lock)) {
spin_unlock(&dentry->d_lock);
cpu_relax();
if (!(dentry->d_flags & DCACHE_GENOCIDE)) {
dentry->d_flags |= DCACHE_GENOCIDE;
- dentry->d_lockref.count--;
+ __lockref_dec(&dentry->d_lockref);
}
}
return D_WALK_CONTINUE;
mempages -= reserve;
names_cachep = kmem_cache_create("names_cache", PATH_MAX, 0,
- SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
+ SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_USERCOPY|
+ SLAB_NO_SANITIZE, NULL);
dcache_init();
inode_init();
*/
struct dentry *debugfs_create_dir(const char *name, struct dentry *parent)
{
+#ifdef CONFIG_GRKERNSEC_SYSFS_RESTRICT
+ return __create_file(name, S_IFDIR | S_IRWXU,
+#else
return __create_file(name, S_IFDIR | S_IRWXU | S_IRUGO | S_IXUGO,
+#endif
parent, NULL, NULL);
}
EXPORT_SYMBOL_GPL(debugfs_create_dir);
old_fs = get_fs();
set_fs(get_ds());
rc = lower_dentry->d_inode->i_op->readlink(lower_dentry,
- (char __user *)lower_buf,
+ (char __force_user *)lower_buf,
PATH_MAX);
set_fs(old_fs);
if (rc < 0)
goto out_unlock_msg_ctx;
i = PKT_TYPE_SIZE + PKT_CTR_SIZE;
if (msg_ctx->msg) {
- if (copy_to_user(&buf[i], packet_length, packet_length_size))
+ if (packet_length_size > sizeof(packet_length) || copy_to_user(&buf[i], packet_length, packet_length_size))
goto out_unlock_msg_ctx;
i += packet_length_size;
if (copy_to_user(&buf[i], msg_ctx->msg, msg_ctx->msg_size))
#include <linux/pipe_fs_i.h>
#include <linux/oom.h>
#include <linux/compat.h>
+#include <linux/random.h>
+#include <linux/seq_file.h>
+#include <linux/coredump.h>
+#include <linux/mman.h>
+
+#ifdef CONFIG_PAX_REFCOUNT
+#include <linux/kallsyms.h>
+#include <linux/kdebug.h>
+#endif
+
+#include <trace/events/fs.h>
#include <asm/uaccess.h>
+#include <asm/sections.h>
#include <asm/mmu_context.h>
#include <asm/tlb.h>
#include <trace/events/sched.h>
+#ifdef CONFIG_PAX_HAVE_ACL_FLAGS
+void __weak pax_set_initial_flags(struct linux_binprm *bprm)
+{
+ pr_warn_once("PAX: PAX_HAVE_ACL_FLAGS was enabled without providing the pax_set_initial_flags callback, this is probably not what you wanted.\n");
+}
+#endif
+
+#ifdef CONFIG_PAX_HOOK_ACL_FLAGS
+void (*pax_set_initial_flags_func)(struct linux_binprm *bprm);
+EXPORT_SYMBOL(pax_set_initial_flags_func);
+#endif
+
int suid_dumpable = 0;
static LIST_HEAD(formats);
static DEFINE_RWLOCK(binfmt_lock);
+extern int gr_process_kernel_exec_ban(void);
+extern int gr_process_suid_exec_ban(const struct linux_binprm *bprm);
+
void __register_binfmt(struct linux_binfmt * fmt, int insert)
{
BUG_ON(!fmt);
if (WARN_ON(!fmt->load_binary))
return;
write_lock(&binfmt_lock);
- insert ? list_add(&fmt->lh, &formats) :
- list_add_tail(&fmt->lh, &formats);
+ insert ? pax_list_add((struct list_head *)&fmt->lh, &formats) :
+ pax_list_add_tail((struct list_head *)&fmt->lh, &formats);
write_unlock(&binfmt_lock);
}
void unregister_binfmt(struct linux_binfmt * fmt)
{
write_lock(&binfmt_lock);
- list_del(&fmt->lh);
+ pax_list_del((struct list_head *)&fmt->lh);
write_unlock(&binfmt_lock);
}
int write)
{
struct page *page;
- int ret;
-#ifdef CONFIG_STACK_GROWSUP
- if (write) {
- ret = expand_downwards(bprm->vma, pos);
- if (ret < 0)
- return NULL;
- }
-#endif
- ret = get_user_pages(current, bprm->mm, pos,
- 1, write, 1, &page, NULL);
- if (ret <= 0)
+ if (0 > expand_downwards(bprm->vma, pos))
+ return NULL;
+ if (0 >= get_user_pages(current, bprm->mm, pos, 1, write, 1, &page, NULL))
return NULL;
if (write) {
if (size <= ARG_MAX)
return page;
+#ifdef CONFIG_GRKERNSEC_PROC_MEMMAP
+ // only allow 512KB for argv+env on suid/sgid binaries
+ // to prevent easy ASLR exhaustion
+ if (((!uid_eq(bprm->cred->euid, current_euid())) ||
+ (!gid_eq(bprm->cred->egid, current_egid()))) &&
+ (size > (512 * 1024))) {
+ put_page(page);
+ return NULL;
+ }
+#endif
+
/*
* Limit to 1/4-th the stack size for the argv+env strings.
* This ensures that:
vma->vm_end = STACK_TOP_MAX;
vma->vm_start = vma->vm_end - PAGE_SIZE;
vma->vm_flags = VM_SOFTDIRTY | VM_STACK_FLAGS | VM_STACK_INCOMPLETE_SETUP;
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ vma->vm_flags &= ~(VM_EXEC | VM_MAYEXEC);
+#endif
+
vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
INIT_LIST_HEAD(&vma->anon_vma_chain);
arch_bprm_mm_init(mm, vma);
up_write(&mm->mmap_sem);
bprm->p = vma->vm_end - sizeof(void *);
+
+#ifdef CONFIG_PAX_RANDUSTACK
+ if (randomize_va_space)
+ bprm->p ^= prandom_u32() & ~PAGE_MASK;
+#endif
+
return 0;
err:
up_write(&mm->mmap_sem);
} ptr;
};
-static const char __user *get_user_arg_ptr(struct user_arg_ptr argv, int nr)
+const char __user *get_user_arg_ptr(struct user_arg_ptr argv, int nr)
{
const char __user *native;
compat_uptr_t compat;
if (get_user(compat, argv.ptr.compat + nr))
- return ERR_PTR(-EFAULT);
+ return (const char __force_user *)ERR_PTR(-EFAULT);
return compat_ptr(compat);
}
#endif
if (get_user(native, argv.ptr.native + nr))
- return ERR_PTR(-EFAULT);
+ return (const char __force_user *)ERR_PTR(-EFAULT);
return native;
}
if (!p)
break;
- if (IS_ERR(p))
+ if (IS_ERR((const char __force_kernel *)p))
return -EFAULT;
if (i >= max)
ret = -EFAULT;
str = get_user_arg_ptr(argv, argc);
- if (IS_ERR(str))
+ if (IS_ERR((const char __force_kernel *)str))
goto out;
len = strnlen_user(str, MAX_ARG_STRLEN);
int r;
mm_segment_t oldfs = get_fs();
struct user_arg_ptr argv = {
- .ptr.native = (const char __user *const __user *)__argv,
+ .ptr.native = (const char __user * const __force_user *)__argv,
};
set_fs(KERNEL_DS);
unsigned long new_end = old_end - shift;
struct mmu_gather tlb;
- BUG_ON(new_start > new_end);
+ if (new_start >= new_end || new_start < mmap_min_addr)
+ return -ENOMEM;
/*
* ensure there are no vmas between where we want to go
if (vma != find_vma(mm, new_start))
return -EFAULT;
+#ifdef CONFIG_PAX_SEGMEXEC
+ BUG_ON(pax_find_mirror_vma(vma));
+#endif
+
/*
* cover the whole range: [new_start, old_end)
*/
stack_top = arch_align_stack(stack_top);
stack_top = PAGE_ALIGN(stack_top);
- if (unlikely(stack_top < mmap_min_addr) ||
- unlikely(vma->vm_end - vma->vm_start >= stack_top - mmap_min_addr))
- return -ENOMEM;
-
stack_shift = vma->vm_end - stack_top;
bprm->p -= stack_shift;
bprm->exec -= stack_shift;
down_write(&mm->mmap_sem);
+
+ /* Move stack pages down in memory. */
+ if (stack_shift) {
+ ret = shift_arg_pages(vma, stack_shift);
+ if (ret)
+ goto out_unlock;
+ }
+
vm_flags = VM_STACK_FLAGS;
+#if defined(CONFIG_PAX_PAGEEXEC) || defined(CONFIG_PAX_SEGMEXEC)
+ if (mm->pax_flags & (MF_PAX_PAGEEXEC | MF_PAX_SEGMEXEC)) {
+ vm_flags &= ~VM_EXEC;
+
+#ifdef CONFIG_PAX_MPROTECT
+ if (mm->pax_flags & MF_PAX_MPROTECT)
+ vm_flags &= ~VM_MAYEXEC;
+#endif
+
+ }
+#endif
+
/*
* Adjust stack execute permissions; explicitly enable for
* EXSTACK_ENABLE_X, disable for EXSTACK_DISABLE_X and leave alone
goto out_unlock;
BUG_ON(prev != vma);
- /* Move stack pages down in memory. */
- if (stack_shift) {
- ret = shift_arg_pages(vma, stack_shift);
- if (ret)
- goto out_unlock;
- }
-
/* mprotect_fixup is overkill to remove the temporary stack flags */
vma->vm_flags &= ~VM_STACK_INCOMPLETE_SETUP;
#endif
current->mm->start_stack = bprm->p;
ret = expand_stack(vma, stack_base);
+
+#if !defined(CONFIG_STACK_GROWSUP) && defined(CONFIG_PAX_RANDMMAP)
+ if (!ret && (mm->pax_flags & MF_PAX_RANDMMAP) && STACK_TOP <= 0xFFFFFFFFU && STACK_TOP > vma->vm_end) {
+ unsigned long size;
+ vm_flags_t vm_flags;
+
+ size = STACK_TOP - vma->vm_end;
+ vm_flags = VM_NONE | VM_DONTEXPAND | VM_DONTDUMP;
+
+ ret = vma->vm_end != mmap_region(NULL, vma->vm_end, size, vm_flags, 0);
+
+#ifdef CONFIG_X86
+ if (!ret) {
+ size = PAGE_SIZE + mmap_min_addr + ((mm->delta_mmap ^ mm->delta_stack) & (0xFFUL << PAGE_SHIFT));
+ ret = 0 != mmap_region(NULL, 0, PAGE_ALIGN(size), vm_flags, 0);
+ }
+#endif
+
+ }
+#endif
+
if (ret)
ret = -EFAULT;
if (err)
goto exit;
- if (name->name[0] != '\0')
+ if (name->name[0] != '\0') {
fsnotify_open(file);
+ trace_open_exec(name->name);
+ }
out:
return file;
old_fs = get_fs();
set_fs(get_ds());
/* The cast to a user pointer is valid due to the set_fs() */
- result = vfs_read(file, (void __user *)addr, count, &pos);
+ result = vfs_read(file, (void __force_user *)addr, count, &pos);
set_fs(old_fs);
return result;
}
tsk->mm = mm;
tsk->active_mm = mm;
activate_mm(active_mm, mm);
+ populate_stack();
tsk->mm->vmacache_seqnum = 0;
vmacache_flush(tsk);
task_unlock(tsk);
}
rcu_read_unlock();
- if (p->fs->users > n_fs)
+ if (atomic_read(&p->fs->users) > n_fs)
bprm->unsafe |= LSM_UNSAFE_SHARE;
else
p->fs->in_exec = 1;
return ret;
}
+#ifdef CONFIG_GRKERNSEC_PROC_MEMMAP
+static DEFINE_PER_CPU(u64, exec_counter);
+static int __init init_exec_counters(void)
+{
+ unsigned int cpu;
+
+ for_each_possible_cpu(cpu) {
+ per_cpu(exec_counter, cpu) = (u64)cpu;
+ }
+
+ return 0;
+}
+early_initcall(init_exec_counters);
+static inline void increment_exec_counter(void)
+{
+ BUILD_BUG_ON(NR_CPUS > (1 << 16));
+ current->exec_id = this_cpu_add_return(exec_counter, 1 << 16);
+}
+#else
+static inline void increment_exec_counter(void) {}
+#endif
+
+extern void gr_handle_exec_args(struct linux_binprm *bprm,
+ struct user_arg_ptr argv);
+
/*
* sys_execve() executes a new program.
*/
struct user_arg_ptr envp,
int flags)
{
+#ifdef CONFIG_GRKERNSEC
+ struct file *old_exec_file;
+ struct acl_subject_label *old_acl;
+ struct rlimit old_rlim[RLIM_NLIMITS];
+#endif
char *pathbuf = NULL;
struct linux_binprm *bprm;
struct file *file;
if (IS_ERR(filename))
return PTR_ERR(filename);
+ gr_learn_resource(current, RLIMIT_NPROC, atomic_read(¤t_user()->processes), 1);
+
/*
* We move the actual failure in case of RLIMIT_NPROC excess from
* set*uid() to execve() because too many poorly written programs
if (IS_ERR(file))
goto out_unmark;
+ if (gr_ptrace_readexec(file, bprm->unsafe)) {
+ retval = -EPERM;
+ goto out_unmark;
+ }
+
sched_exec();
bprm->file = file;
}
bprm->interp = bprm->filename;
+ if (!gr_acl_handle_execve(file->f_path.dentry, file->f_path.mnt)) {
+ retval = -EACCES;
+ goto out_unmark;
+ }
+
retval = bprm_mm_init(bprm);
if (retval)
goto out_unmark;
if (retval < 0)
goto out;
+#ifdef CONFIG_GRKERNSEC
+ old_acl = current->acl;
+ memcpy(old_rlim, current->signal->rlim, sizeof(old_rlim));
+ old_exec_file = current->exec_file;
+ get_file(file);
+ current->exec_file = file;
+#endif
+#ifdef CONFIG_GRKERNSEC_PROC_MEMMAP
+ /* limit suid stack to 8MB
+ * we saved the old limits above and will restore them if this exec fails
+ */
+ if (((!uid_eq(bprm->cred->euid, current_euid())) || (!gid_eq(bprm->cred->egid, current_egid()))) &&
+ (old_rlim[RLIMIT_STACK].rlim_cur > (8 * 1024 * 1024)))
+ current->signal->rlim[RLIMIT_STACK].rlim_cur = 8 * 1024 * 1024;
+#endif
+
+ if (gr_process_kernel_exec_ban() || gr_process_suid_exec_ban(bprm)) {
+ retval = -EPERM;
+ goto out_fail;
+ }
+
+ if (!gr_tpe_allow(file)) {
+ retval = -EACCES;
+ goto out_fail;
+ }
+
+ if (gr_check_crash_exec(file)) {
+ retval = -EACCES;
+ goto out_fail;
+ }
+
+ retval = gr_set_proc_label(file->f_path.dentry, file->f_path.mnt,
+ bprm->unsafe);
+ if (retval < 0)
+ goto out_fail;
+
retval = copy_strings_kernel(1, &bprm->filename, bprm);
if (retval < 0)
- goto out;
+ goto out_fail;
bprm->exec = bprm->p;
retval = copy_strings(bprm->envc, envp, bprm);
if (retval < 0)
- goto out;
+ goto out_fail;
retval = copy_strings(bprm->argc, argv, bprm);
if (retval < 0)
- goto out;
+ goto out_fail;
+
+ gr_log_chroot_exec(file->f_path.dentry, file->f_path.mnt);
+
+ gr_handle_exec_args(bprm, argv);
retval = exec_binprm(bprm);
if (retval < 0)
- goto out;
+ goto out_fail;
+#ifdef CONFIG_GRKERNSEC
+ if (old_exec_file)
+ fput(old_exec_file);
+#endif
/* execve succeeded */
+
+ increment_exec_counter();
current->fs->in_exec = 0;
current->in_execve = 0;
acct_update_integrals(current);
put_files_struct(displaced);
return retval;
+out_fail:
+#ifdef CONFIG_GRKERNSEC
+ current->acl = old_acl;
+ memcpy(current->signal->rlim, old_rlim, sizeof(old_rlim));
+ fput(current->exec_file);
+ current->exec_file = old_exec_file;
+#endif
+
out:
if (bprm->mm) {
acct_arg_size(bprm, 0);
argv, envp, flags);
}
#endif
+
+int pax_check_flags(unsigned long *flags)
+{
+ int retval = 0;
+
+#if !defined(CONFIG_X86_32) || !defined(CONFIG_PAX_SEGMEXEC)
+ if (*flags & MF_PAX_SEGMEXEC)
+ {
+ *flags &= ~MF_PAX_SEGMEXEC;
+ retval = -EINVAL;
+ }
+#endif
+
+ if ((*flags & MF_PAX_PAGEEXEC)
+
+#ifdef CONFIG_PAX_PAGEEXEC
+ && (*flags & MF_PAX_SEGMEXEC)
+#endif
+
+ )
+ {
+ *flags &= ~MF_PAX_PAGEEXEC;
+ retval = -EINVAL;
+ }
+
+ if ((*flags & MF_PAX_MPROTECT)
+
+#ifdef CONFIG_PAX_MPROTECT
+ && !(*flags & (MF_PAX_PAGEEXEC | MF_PAX_SEGMEXEC))
+#endif
+
+ )
+ {
+ *flags &= ~MF_PAX_MPROTECT;
+ retval = -EINVAL;
+ }
+
+ if ((*flags & MF_PAX_EMUTRAMP)
+
+#ifdef CONFIG_PAX_EMUTRAMP
+ && !(*flags & (MF_PAX_PAGEEXEC | MF_PAX_SEGMEXEC))
+#endif
+
+ )
+ {
+ *flags &= ~MF_PAX_EMUTRAMP;
+ retval = -EINVAL;
+ }
+
+ return retval;
+}
+
+EXPORT_SYMBOL(pax_check_flags);
+
+#if defined(CONFIG_PAX_PAGEEXEC) || defined(CONFIG_PAX_SEGMEXEC)
+char *pax_get_path(const struct path *path, char *buf, int buflen)
+{
+ char *pathname = d_path(path, buf, buflen);
+
+ if (IS_ERR(pathname))
+ goto toolong;
+
+ pathname = mangle_path(buf, pathname, "\t\n\\");
+ if (!pathname)
+ goto toolong;
+
+ *pathname = 0;
+ return buf;
+
+toolong:
+ return "<path too long>";
+}
+EXPORT_SYMBOL(pax_get_path);
+
+void pax_report_fault(struct pt_regs *regs, void *pc, void *sp)
+{
+ struct task_struct *tsk = current;
+ struct mm_struct *mm = current->mm;
+ char *buffer_exec = (char *)__get_free_page(GFP_KERNEL);
+ char *buffer_fault = (char *)__get_free_page(GFP_KERNEL);
+ char *path_exec = NULL;
+ char *path_fault = NULL;
+ unsigned long start = 0UL, end = 0UL, offset = 0UL;
+ siginfo_t info = { };
+
+ if (buffer_exec && buffer_fault) {
+ struct vm_area_struct *vma, *vma_exec = NULL, *vma_fault = NULL;
+
+ down_read(&mm->mmap_sem);
+ vma = mm->mmap;
+ while (vma && (!vma_exec || !vma_fault)) {
+ if (vma->vm_file && mm->exe_file == vma->vm_file && (vma->vm_flags & VM_EXEC))
+ vma_exec = vma;
+ if (vma->vm_start <= (unsigned long)pc && (unsigned long)pc < vma->vm_end)
+ vma_fault = vma;
+ vma = vma->vm_next;
+ }
+ if (vma_exec)
+ path_exec = pax_get_path(&vma_exec->vm_file->f_path, buffer_exec, PAGE_SIZE);
+ if (vma_fault) {
+ start = vma_fault->vm_start;
+ end = vma_fault->vm_end;
+ offset = vma_fault->vm_pgoff << PAGE_SHIFT;
+ if (vma_fault->vm_file)
+ path_fault = pax_get_path(&vma_fault->vm_file->f_path, buffer_fault, PAGE_SIZE);
+ else if ((unsigned long)pc >= mm->start_brk && (unsigned long)pc < mm->brk)
+ path_fault = "<heap>";
+ else if (vma_fault->vm_flags & (VM_GROWSDOWN | VM_GROWSUP))
+ path_fault = "<stack>";
+ else
+ path_fault = "<anonymous mapping>";
+ }
+ up_read(&mm->mmap_sem);
+ }
+ if (tsk->signal->curr_ip)
+ printk(KERN_ERR "PAX: From %pI4: execution attempt in: %s, %08lx-%08lx %08lx\n", &tsk->signal->curr_ip, path_fault, start, end, offset);
+ else
+ printk(KERN_ERR "PAX: execution attempt in: %s, %08lx-%08lx %08lx\n", path_fault, start, end, offset);
+ printk(KERN_ERR "PAX: terminating task: %s(%s):%d, uid/euid: %u/%u, PC: %p, SP: %p\n", path_exec, tsk->comm, task_pid_nr(tsk),
+ from_kuid_munged(&init_user_ns, task_uid(tsk)), from_kuid_munged(&init_user_ns, task_euid(tsk)), pc, sp);
+ free_page((unsigned long)buffer_exec);
+ free_page((unsigned long)buffer_fault);
+ pax_report_insns(regs, pc, sp);
+ info.si_signo = SIGKILL;
+ info.si_errno = 0;
+ info.si_code = SI_KERNEL;
+ info.si_pid = 0;
+ info.si_uid = 0;
+ do_coredump(&info);
+}
+#endif
+
+#ifdef CONFIG_PAX_REFCOUNT
+void pax_report_refcount_overflow(struct pt_regs *regs)
+{
+ if (current->signal->curr_ip)
+ printk(KERN_ERR "PAX: From %pI4: refcount overflow detected in: %s:%d, uid/euid: %u/%u\n",
+ ¤t->signal->curr_ip, current->comm, task_pid_nr(current),
+ from_kuid_munged(&init_user_ns, current_uid()), from_kuid_munged(&init_user_ns, current_euid()));
+ else
+ printk(KERN_ERR "PAX: refcount overflow detected in: %s:%d, uid/euid: %u/%u\n", current->comm, task_pid_nr(current),
+ from_kuid_munged(&init_user_ns, current_uid()), from_kuid_munged(&init_user_ns, current_euid()));
+ print_symbol(KERN_ERR "PAX: refcount overflow occured at: %s\n", instruction_pointer(regs));
+ preempt_disable();
+ show_regs(regs);
+ preempt_enable();
+ force_sig_info(SIGKILL, SEND_SIG_FORCED, current);
+}
+#endif
+
+#ifdef CONFIG_PAX_USERCOPY
+/* 0: not at all, 1: fully, 2: fully inside frame, -1: partially (implies an error) */
+static noinline int check_stack_object(const void *obj, unsigned long len)
+{
+ const void * const stack = task_stack_page(current);
+ const void * const stackend = stack + THREAD_SIZE;
+
+#if defined(CONFIG_FRAME_POINTER) && defined(CONFIG_X86)
+ const void *frame = NULL;
+ const void *oldframe;
+#endif
+
+ if (obj + len < obj)
+ return -1;
+
+ if (obj + len <= stack || stackend <= obj)
+ return 0;
+
+ if (obj < stack || stackend < obj + len)
+ return -1;
+
+#if defined(CONFIG_FRAME_POINTER) && defined(CONFIG_X86)
+ oldframe = __builtin_frame_address(1);
+ if (oldframe)
+ frame = __builtin_frame_address(2);
+ /*
+ low ----------------------------------------------> high
+ [saved bp][saved ip][args][local vars][saved bp][saved ip]
+ ^----------------^
+ allow copies only within here
+ */
+ while (stack <= frame && frame < stackend) {
+ /* if obj + len extends past the last frame, this
+ check won't pass and the next frame will be 0,
+ causing us to bail out and correctly report
+ the copy as invalid
+ */
+ if (obj + len <= frame)
+ return obj >= oldframe + 2 * sizeof(void *) ? 2 : -1;
+ oldframe = frame;
+ frame = *(const void * const *)frame;
+ }
+ return -1;
+#else
+ return 1;
+#endif
+}
+
+static __noreturn void pax_report_usercopy(const void *ptr, unsigned long len, bool to_user, const char *type)
+{
+ if (current->signal->curr_ip)
+ printk(KERN_ERR "PAX: From %pI4: kernel memory %s attempt detected %s %p (%s) (%lu bytes)\n",
+ ¤t->signal->curr_ip, to_user ? "leak" : "overwrite", to_user ? "from" : "to", ptr, type ? : "unknown", len);
+ else
+ printk(KERN_ERR "PAX: kernel memory %s attempt detected %s %p (%s) (%lu bytes)\n",
+ to_user ? "leak" : "overwrite", to_user ? "from" : "to", ptr, type ? : "unknown", len);
+ dump_stack();
+ gr_handle_kernel_exploit();
+ do_group_exit(SIGKILL);
+}
+#endif
+
+#ifdef CONFIG_PAX_USERCOPY
+
+static inline bool check_kernel_text_object(unsigned long low, unsigned long high)
+{
+#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_KERNEXEC)
+ unsigned long textlow = ktla_ktva((unsigned long)_stext);
+#ifdef CONFIG_MODULES
+ unsigned long texthigh = (unsigned long)MODULES_EXEC_VADDR;
+#else
+ unsigned long texthigh = ktla_ktva((unsigned long)_etext);
+#endif
+
+#else
+ unsigned long textlow = (unsigned long)_stext;
+ unsigned long texthigh = (unsigned long)_etext;
+
+#ifdef CONFIG_X86_64
+ /* check against linear mapping as well */
+ if (high > (unsigned long)__va(__pa(textlow)) &&
+ low < (unsigned long)__va(__pa(texthigh)))
+ return true;
+#endif
+
+#endif
+
+ if (high <= textlow || low >= texthigh)
+ return false;
+ else
+ return true;
+}
+#endif
+
+void __check_object_size(const void *ptr, unsigned long n, bool to_user, bool const_size)
+{
+#ifdef CONFIG_PAX_USERCOPY
+ const char *type;
+#endif
+
+#if !defined(CONFIG_STACK_GROWSUP) && !defined(CONFIG_X86_64)
+ unsigned long stackstart = (unsigned long)task_stack_page(current);
+ unsigned long currentsp = (unsigned long)&stackstart;
+ if (unlikely((currentsp < stackstart + 512 ||
+ currentsp >= stackstart + THREAD_SIZE) && !in_interrupt()))
+ BUG();
+#endif
+
+#ifndef CONFIG_PAX_USERCOPY_DEBUG
+ if (const_size)
+ return;
+#endif
+
+#ifdef CONFIG_PAX_USERCOPY
+ if (!n)
+ return;
+
+ type = check_heap_object(ptr, n);
+ if (!type) {
+ int ret = check_stack_object(ptr, n);
+ if (ret == 1 || ret == 2)
+ return;
+ if (ret == 0) {
+ if (check_kernel_text_object((unsigned long)ptr, (unsigned long)ptr + n))
+ type = "<kernel text>";
+ else
+ return;
+ } else
+ type = "<process stack>";
+ }
+
+ pax_report_usercopy(ptr, n, to_user, type);
+#endif
+
+}
+EXPORT_SYMBOL(__check_object_size);
+
+#ifdef CONFIG_PAX_MEMORY_STACKLEAK
+void pax_track_stack(void)
+{
+ unsigned long sp = (unsigned long)&sp;
+ if (sp < current_thread_info()->lowest_stack &&
+ sp >= (unsigned long)task_stack_page(current) + 2 * sizeof(unsigned long))
+ current_thread_info()->lowest_stack = sp;
+ if (unlikely((sp & ~(THREAD_SIZE - 1)) < (THREAD_SIZE/16)))
+ BUG();
+}
+EXPORT_SYMBOL(pax_track_stack);
+#endif
+
+#ifdef CONFIG_PAX_SIZE_OVERFLOW
+void report_size_overflow(const char *file, unsigned int line, const char *func, const char *ssa_name)
+{
+ printk(KERN_ERR "PAX: size overflow detected in function %s %s:%u %s", func, file, line, ssa_name);
+ dump_stack();
+ do_group_exit(SIGKILL);
+}
+EXPORT_SYMBOL(report_size_overflow);
+#endif
free_blocks = percpu_counter_read_positive(&sbi->s_freeblocks_counter);
root_blocks = le32_to_cpu(sbi->s_es->s_r_blocks_count);
- if (free_blocks < root_blocks + 1 && !capable(CAP_SYS_RESOURCE) &&
+ if (free_blocks < root_blocks + 1 &&
!uid_eq(sbi->s_resuid, current_fsuid()) &&
(gid_eq(sbi->s_resgid, GLOBAL_ROOT_GID) ||
- !in_group_p (sbi->s_resgid))) {
+ !in_group_p (sbi->s_resgid)) && !capable_nolog(CAP_SYS_RESOURCE)) {
return 0;
}
return 1;
#ifdef CONFIG_EXT2_FS_XATTR
if (test_opt(sb, XATTR_USER))
seq_puts(seq, ",user_xattr");
- if (!test_opt(sb, XATTR_USER) &&
- (def_mount_opts & EXT2_DEFM_XATTR_USER)) {
+ if (!test_opt(sb, XATTR_USER))
seq_puts(seq, ",nouser_xattr");
- }
#endif
#ifdef CONFIG_EXT2_FS_POSIX_ACL
if (def_mount_opts & EXT2_DEFM_UID16)
set_opt(sbi->s_mount_opt, NO_UID32);
#ifdef CONFIG_EXT2_FS_XATTR
- if (def_mount_opts & EXT2_DEFM_XATTR_USER)
- set_opt(sbi->s_mount_opt, XATTR_USER);
+ /* always enable user xattrs */
+ set_opt(sbi->s_mount_opt, XATTR_USER);
#endif
#ifdef CONFIG_EXT2_FS_POSIX_ACL
if (def_mount_opts & EXT2_DEFM_ACL)
struct buffer_head *bh = NULL;
struct ext2_xattr_entry *entry;
char *end;
- size_t rest = buffer_size;
+ size_t rest = buffer_size, total_size = 0;
int error;
ea_idebug(inode, "buffer=%p, buffer_size=%ld",
buffer += size;
}
rest -= size;
+ total_size += size;
}
}
- error = buffer_size - rest; /* total size */
+ error = total_size;
cleanup:
brelse(bh);
free_blocks = percpu_counter_read_positive(&sbi->s_freeblocks_counter);
root_blocks = le32_to_cpu(sbi->s_es->s_r_blocks_count);
- if (free_blocks < root_blocks + 1 && !capable(CAP_SYS_RESOURCE) &&
+ if (free_blocks < root_blocks + 1 &&
!use_reservation && !uid_eq(sbi->s_resuid, current_fsuid()) &&
(gid_eq(sbi->s_resgid, GLOBAL_ROOT_GID) ||
- !in_group_p (sbi->s_resgid))) {
+ !in_group_p (sbi->s_resgid)) && !capable_nolog(CAP_SYS_RESOURCE)) {
return 0;
}
return 1;
#ifdef CONFIG_EXT3_FS_XATTR
if (test_opt(sb, XATTR_USER))
seq_puts(seq, ",user_xattr");
- if (!test_opt(sb, XATTR_USER) &&
- (def_mount_opts & EXT3_DEFM_XATTR_USER)) {
+ if (!test_opt(sb, XATTR_USER))
seq_puts(seq, ",nouser_xattr");
- }
#endif
#ifdef CONFIG_EXT3_FS_POSIX_ACL
if (test_opt(sb, POSIX_ACL))
if (def_mount_opts & EXT3_DEFM_UID16)
set_opt(sbi->s_mount_opt, NO_UID32);
#ifdef CONFIG_EXT3_FS_XATTR
- if (def_mount_opts & EXT3_DEFM_XATTR_USER)
- set_opt(sbi->s_mount_opt, XATTR_USER);
+ /* always enable user xattrs */
+ set_opt(sbi->s_mount_opt, XATTR_USER);
#endif
#ifdef CONFIG_EXT3_FS_POSIX_ACL
if (def_mount_opts & EXT3_DEFM_ACL)
ext3_xattr_list_entries(struct dentry *dentry, struct ext3_xattr_entry *entry,
char *buffer, size_t buffer_size)
{
- size_t rest = buffer_size;
+ size_t rest = buffer_size, total_size = 0;
for (; !IS_LAST_ENTRY(entry); entry = EXT3_XATTR_NEXT(entry)) {
const struct xattr_handler *handler =
buffer += size;
}
rest -= size;
+ total_size += size;
}
}
- return buffer_size - rest;
+ return total_size;
}
static int
/* Hm, nope. Are (enough) root reserved clusters available? */
if (uid_eq(sbi->s_resuid, current_fsuid()) ||
(!gid_eq(sbi->s_resgid, GLOBAL_ROOT_GID) && in_group_p(sbi->s_resgid)) ||
- capable(CAP_SYS_RESOURCE) ||
- (flags & EXT4_MB_USE_ROOT_BLOCKS)) {
+ (flags & EXT4_MB_USE_ROOT_BLOCKS) ||
+ capable_nolog(CAP_SYS_RESOURCE)) {
if (free_clusters >= (nclusters + dirty_clusters +
resv_clusters))
unsigned long s_mb_last_start;
/* stats for buddy allocator */
- atomic_t s_bal_reqs; /* number of reqs with len > 1 */
- atomic_t s_bal_success; /* we found long enough chunks */
- atomic_t s_bal_allocated; /* in blocks */
- atomic_t s_bal_ex_scanned; /* total extents scanned */
- atomic_t s_bal_goals; /* goal hits */
- atomic_t s_bal_breaks; /* too long searches */
- atomic_t s_bal_2orders; /* 2^order hits */
+ atomic_unchecked_t s_bal_reqs; /* number of reqs with len > 1 */
+ atomic_unchecked_t s_bal_success; /* we found long enough chunks */
+ atomic_unchecked_t s_bal_allocated; /* in blocks */
+ atomic_unchecked_t s_bal_ex_scanned; /* total extents scanned */
+ atomic_unchecked_t s_bal_goals; /* goal hits */
+ atomic_unchecked_t s_bal_breaks; /* too long searches */
+ atomic_unchecked_t s_bal_2orders; /* 2^order hits */
spinlock_t s_bal_lock;
unsigned long s_mb_buddies_generated;
unsigned long long s_mb_generation_time;
- atomic_t s_mb_lost_chunks;
- atomic_t s_mb_preallocated;
- atomic_t s_mb_discarded;
+ atomic_unchecked_t s_mb_lost_chunks;
+ atomic_unchecked_t s_mb_preallocated;
+ atomic_unchecked_t s_mb_discarded;
atomic_t s_lock_busy;
/* locality groups */
BUG_ON(ac->ac_b_ex.fe_len != ac->ac_g_ex.fe_len);
if (EXT4_SB(sb)->s_mb_stats)
- atomic_inc(&EXT4_SB(sb)->s_bal_2orders);
+ atomic_inc_unchecked(&EXT4_SB(sb)->s_bal_2orders);
break;
}
ac->ac_status = AC_STATUS_CONTINUE;
ac->ac_flags |= EXT4_MB_HINT_FIRST;
cr = 3;
- atomic_inc(&sbi->s_mb_lost_chunks);
+ atomic_inc_unchecked(&sbi->s_mb_lost_chunks);
goto repeat;
}
}
if (sbi->s_mb_stats) {
ext4_msg(sb, KERN_INFO,
"mballoc: %u blocks %u reqs (%u success)",
- atomic_read(&sbi->s_bal_allocated),
- atomic_read(&sbi->s_bal_reqs),
- atomic_read(&sbi->s_bal_success));
+ atomic_read_unchecked(&sbi->s_bal_allocated),
+ atomic_read_unchecked(&sbi->s_bal_reqs),
+ atomic_read_unchecked(&sbi->s_bal_success));
ext4_msg(sb, KERN_INFO,
"mballoc: %u extents scanned, %u goal hits, "
"%u 2^N hits, %u breaks, %u lost",
- atomic_read(&sbi->s_bal_ex_scanned),
- atomic_read(&sbi->s_bal_goals),
- atomic_read(&sbi->s_bal_2orders),
- atomic_read(&sbi->s_bal_breaks),
- atomic_read(&sbi->s_mb_lost_chunks));
+ atomic_read_unchecked(&sbi->s_bal_ex_scanned),
+ atomic_read_unchecked(&sbi->s_bal_goals),
+ atomic_read_unchecked(&sbi->s_bal_2orders),
+ atomic_read_unchecked(&sbi->s_bal_breaks),
+ atomic_read_unchecked(&sbi->s_mb_lost_chunks));
ext4_msg(sb, KERN_INFO,
"mballoc: %lu generated and it took %Lu",
sbi->s_mb_buddies_generated,
sbi->s_mb_generation_time);
ext4_msg(sb, KERN_INFO,
"mballoc: %u preallocated, %u discarded",
- atomic_read(&sbi->s_mb_preallocated),
- atomic_read(&sbi->s_mb_discarded));
+ atomic_read_unchecked(&sbi->s_mb_preallocated),
+ atomic_read_unchecked(&sbi->s_mb_discarded));
}
free_percpu(sbi->s_locality_groups);
struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
if (sbi->s_mb_stats && ac->ac_g_ex.fe_len > 1) {
- atomic_inc(&sbi->s_bal_reqs);
- atomic_add(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated);
+ atomic_inc_unchecked(&sbi->s_bal_reqs);
+ atomic_add_unchecked(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated);
if (ac->ac_b_ex.fe_len >= ac->ac_o_ex.fe_len)
- atomic_inc(&sbi->s_bal_success);
- atomic_add(ac->ac_found, &sbi->s_bal_ex_scanned);
+ atomic_inc_unchecked(&sbi->s_bal_success);
+ atomic_add_unchecked(ac->ac_found, &sbi->s_bal_ex_scanned);
if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start &&
ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group)
- atomic_inc(&sbi->s_bal_goals);
+ atomic_inc_unchecked(&sbi->s_bal_goals);
if (ac->ac_found > sbi->s_mb_max_to_scan)
- atomic_inc(&sbi->s_bal_breaks);
+ atomic_inc_unchecked(&sbi->s_bal_breaks);
}
if (ac->ac_op == EXT4_MB_HISTORY_ALLOC)
trace_ext4_mb_new_inode_pa(ac, pa);
ext4_mb_use_inode_pa(ac, pa);
- atomic_add(pa->pa_free, &sbi->s_mb_preallocated);
+ atomic_add_unchecked(pa->pa_free, &sbi->s_mb_preallocated);
ei = EXT4_I(ac->ac_inode);
grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group);
trace_ext4_mb_new_group_pa(ac, pa);
ext4_mb_use_group_pa(ac, pa);
- atomic_add(pa->pa_free, &EXT4_SB(sb)->s_mb_preallocated);
+ atomic_add_unchecked(pa->pa_free, &EXT4_SB(sb)->s_mb_preallocated);
grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group);
lg = ac->ac_lg;
* from the bitmap and continue.
*/
}
- atomic_add(free, &sbi->s_mb_discarded);
+ atomic_add_unchecked(free, &sbi->s_mb_discarded);
return err;
}
ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit);
BUG_ON(group != e4b->bd_group && pa->pa_len != 0);
mb_free_blocks(pa->pa_inode, e4b, bit, pa->pa_len);
- atomic_add(pa->pa_len, &EXT4_SB(sb)->s_mb_discarded);
+ atomic_add_unchecked(pa->pa_len, &EXT4_SB(sb)->s_mb_discarded);
trace_ext4_mballoc_discard(sb, NULL, group, bit, pa->pa_len);
return 0;
void __dump_mmp_msg(struct super_block *sb, struct mmp_struct *mmp,
const char *function, unsigned int line, const char *msg)
{
- __ext4_warning(sb, function, line, msg);
+ __ext4_warning(sb, function, line, "%s", msg);
__ext4_warning(sb, function, line,
"MMP failure info: last update time: %llu, last update "
"node: %s, last update device: %s\n",
ext4_debug("mark blocks [%llu/%u] used\n", block, count);
for (count2 = count; count > 0; count -= count2, block += count2) {
- ext4_fsblk_t start;
+ ext4_fsblk_t start, diff;
struct buffer_head *bh;
ext4_group_t group;
int err;
start = ext4_group_first_block_no(sb, group);
group -= flex_gd->groups[0].group;
- count2 = EXT4_BLOCKS_PER_GROUP(sb) - (block - start);
- if (count2 > count)
- count2 = count;
-
if (flex_gd->bg_flags[group] & EXT4_BG_BLOCK_UNINIT) {
BUG_ON(flex_gd->count > 1);
continue;
err = ext4_journal_get_write_access(handle, bh);
if (err)
return err;
+
+ diff = block - start;
+ count2 = EXT4_BLOCKS_PER_GROUP(sb) - diff;
+ if (count2 > count)
+ count2 = count;
+
ext4_debug("mark block bitmap %#04llx (+%llu/%u)\n", block,
- block - start, count2);
- ext4_set_bits(bh->b_data, block - start, count2);
+ diff, count2);
+ ext4_set_bits(bh->b_data, diff, count2);
err = ext4_handle_dirty_metadata(handle, NULL, bh);
if (unlikely(err))
}
#define DEFAULT_JOURNAL_IOPRIO (IOPRIO_PRIO_VALUE(IOPRIO_CLASS_BE, 3))
-static char deprecated_msg[] = "Mount option \"%s\" will be removed by %s\n"
+static const char deprecated_msg[] = "Mount option \"%s\" will be removed by %s\n"
"Contact linux-ext4@vger.kernel.org if you think we should keep it.\n";
#ifdef CONFIG_QUOTA
int offset;
int deprecated_val;
} u;
-};
+} __do_const;
static int parse_strtoull(const char *buf,
unsigned long long max, unsigned long long *value)
ext4_xattr_list_entries(struct dentry *dentry, struct ext4_xattr_entry *entry,
char *buffer, size_t buffer_size)
{
- size_t rest = buffer_size;
+ size_t rest = buffer_size, total_size = 0;
for (; !IS_LAST_ENTRY(entry); entry = EXT4_XATTR_NEXT(entry)) {
const struct xattr_handler *handler =
buffer += size;
}
rest -= size;
+ total_size += size;
}
}
- return buffer_size - rest;
+ return total_size;
}
static int
int force)
{
security_file_set_fowner(filp);
+ if (gr_handle_chroot_fowner(pid, type))
+ return;
+ if (gr_check_protected_task_fowner(pid, type))
+ return;
f_modown(filp, pid, type, force);
}
EXPORT_SYMBOL(__f_setown);
#include <linux/fs_struct.h>
#include <linux/fsnotify.h>
#include <linux/personality.h>
+#include <linux/grsecurity.h>
#include <asm/uaccess.h>
#include "internal.h"
#include "mount.h"
} else
retval = 0;
/* copy the mount id */
- if (copy_to_user(mnt_id, &real_mount(path->mnt)->mnt_id,
- sizeof(*mnt_id)) ||
+ if (put_user(real_mount(path->mnt)->mnt_id, mnt_id) ||
copy_to_user(ufh, handle,
sizeof(struct file_handle) + handle_bytes))
retval = -EFAULT;
* the directory. Ideally we would like CAP_DAC_SEARCH.
* But we don't have that
*/
- if (!capable(CAP_DAC_READ_SEARCH)) {
+ if (!capable(CAP_DAC_READ_SEARCH) || !gr_chroot_fhandle()) {
retval = -EPERM;
goto out_err;
}
goto out_err;
}
/* copy the full handle */
- if (copy_from_user(handle, ufh,
- sizeof(struct file_handle) +
+ *handle = f_handle;
+ if (copy_from_user(&handle->f_handle,
+ &ufh->f_handle,
f_handle.handle_bytes)) {
retval = -EFAULT;
goto out_handle;
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/file.h>
+#include <linux/security.h>
#include <linux/fdtable.h>
#include <linux/bitops.h>
#include <linux/interrupt.h>
* Return <0 error code on error; 1 on successful completion.
* The files->file_lock should be held on entry, and will be held on exit.
*/
-static int expand_fdtable(struct files_struct *files, int nr)
+static int expand_fdtable(struct files_struct *files, unsigned int nr)
__releases(files->file_lock)
__acquires(files->file_lock)
{
* expanded and execution may have blocked.
* The files->file_lock should be held on entry, and will be held on exit.
*/
-static int expand_files(struct files_struct *files, int nr)
+static int expand_files(struct files_struct *files, unsigned int nr)
{
struct fdtable *fdt;
if (!file)
return __close_fd(files, fd);
+ gr_learn_resource(current, RLIMIT_NOFILE, fd, 0);
if (fd >= rlimit(RLIMIT_NOFILE))
return -EBADF;
if (unlikely(oldfd == newfd))
return -EINVAL;
+ gr_learn_resource(current, RLIMIT_NOFILE, newfd, 0);
if (newfd >= rlimit(RLIMIT_NOFILE))
return -EBADF;
int f_dupfd(unsigned int from, struct file *file, unsigned flags)
{
int err;
+ gr_learn_resource(current, RLIMIT_NOFILE, from, 0);
if (from >= rlimit(RLIMIT_NOFILE))
return -EINVAL;
err = alloc_fd(from, flags);
int len = dot ? dot - name : strlen(name);
fs = __get_fs_type(name, len);
+#ifdef CONFIG_GRKERNSEC_MODHARDEN
+ if (!fs && (___request_module(true, "grsec_modharden_fs", "fs-%.*s", len, name) == 0))
+#else
if (!fs && (request_module("fs-%.*s", len, name) == 0))
+#endif
fs = __get_fs_type(name, len);
if (dot && fs && !(fs->fs_flags & FS_HAS_SUBTYPE)) {
#include <linux/path.h>
#include <linux/slab.h>
#include <linux/fs_struct.h>
+#include <linux/grsecurity.h>
#include "internal.h"
/*
struct path old_root;
path_get(path);
+ gr_inc_chroot_refcnts(path->dentry, path->mnt);
spin_lock(&fs->lock);
write_seqcount_begin(&fs->seq);
old_root = fs->root;
fs->root = *path;
+ gr_set_chroot_entries(current, path);
write_seqcount_end(&fs->seq);
spin_unlock(&fs->lock);
- if (old_root.dentry)
+ if (old_root.dentry) {
+ gr_dec_chroot_refcnts(old_root.dentry, old_root.mnt);
path_put(&old_root);
+ }
}
/*
int hits = 0;
spin_lock(&fs->lock);
write_seqcount_begin(&fs->seq);
+ /* this root replacement is only done by pivot_root,
+ leave grsec's chroot tagging alone for this task
+ so that a pivoted root isn't treated as a chroot
+ */
hits += replace_path(&fs->root, old_root, new_root);
hits += replace_path(&fs->pwd, old_root, new_root);
write_seqcount_end(&fs->seq);
void free_fs_struct(struct fs_struct *fs)
{
+ gr_dec_chroot_refcnts(fs->root.dentry, fs->root.mnt);
path_put(&fs->root);
path_put(&fs->pwd);
kmem_cache_free(fs_cachep, fs);
task_lock(tsk);
spin_lock(&fs->lock);
tsk->fs = NULL;
- kill = !--fs->users;
+ gr_clear_chroot_entries(tsk);
+ kill = !atomic_dec_return(&fs->users);
spin_unlock(&fs->lock);
task_unlock(tsk);
if (kill)
struct fs_struct *fs = kmem_cache_alloc(fs_cachep, GFP_KERNEL);
/* We don't need to lock fs - think why ;-) */
if (fs) {
- fs->users = 1;
+ atomic_set(&fs->users, 1);
fs->in_exec = 0;
spin_lock_init(&fs->lock);
seqcount_init(&fs->seq);
spin_lock(&old->lock);
fs->root = old->root;
path_get(&fs->root);
+ /* instead of calling gr_set_chroot_entries here,
+ we call it from every caller of this function
+ */
fs->pwd = old->pwd;
path_get(&fs->pwd);
spin_unlock(&old->lock);
task_lock(current);
spin_lock(&fs->lock);
- kill = !--fs->users;
+ kill = !atomic_dec_return(&fs->users);
current->fs = new_fs;
+ gr_set_chroot_entries(current, &new_fs->root);
spin_unlock(&fs->lock);
task_unlock(current);
int current_umask(void)
{
- return current->fs->umask;
+ return current->fs->umask | gr_acl_umask();
}
EXPORT_SYMBOL(current_umask);
/* to be mentioned only in INIT_TASK */
struct fs_struct init_fs = {
- .users = 1,
+ .users = ATOMIC_INIT(1),
.lock = __SPIN_LOCK_UNLOCKED(init_fs.lock),
.seq = SEQCNT_ZERO(init_fs.seq),
.umask = 0022,
struct kmem_cache *fscache_cookie_jar;
-static atomic_t fscache_object_debug_id = ATOMIC_INIT(0);
+static atomic_unchecked_t fscache_object_debug_id = ATOMIC_INIT(0);
static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie);
static int fscache_alloc_object(struct fscache_cache *cache,
parent ? (char *) parent->def->name : "<no-parent>",
def->name, netfs_data, enable);
- fscache_stat(&fscache_n_acquires);
+ fscache_stat_unchecked(&fscache_n_acquires);
/* if there's no parent cookie, then we don't create one here either */
if (!parent) {
- fscache_stat(&fscache_n_acquires_null);
+ fscache_stat_unchecked(&fscache_n_acquires_null);
_leave(" [no parent]");
return NULL;
}
/* allocate and initialise a cookie */
cookie = kmem_cache_alloc(fscache_cookie_jar, GFP_KERNEL);
if (!cookie) {
- fscache_stat(&fscache_n_acquires_oom);
+ fscache_stat_unchecked(&fscache_n_acquires_oom);
_leave(" [ENOMEM]");
return NULL;
}
switch (cookie->def->type) {
case FSCACHE_COOKIE_TYPE_INDEX:
- fscache_stat(&fscache_n_cookie_index);
+ fscache_stat_unchecked(&fscache_n_cookie_index);
break;
case FSCACHE_COOKIE_TYPE_DATAFILE:
- fscache_stat(&fscache_n_cookie_data);
+ fscache_stat_unchecked(&fscache_n_cookie_data);
break;
default:
- fscache_stat(&fscache_n_cookie_special);
+ fscache_stat_unchecked(&fscache_n_cookie_special);
break;
}
} else {
atomic_dec(&parent->n_children);
__fscache_cookie_put(cookie);
- fscache_stat(&fscache_n_acquires_nobufs);
+ fscache_stat_unchecked(&fscache_n_acquires_nobufs);
_leave(" = NULL");
return NULL;
}
}
}
- fscache_stat(&fscache_n_acquires_ok);
+ fscache_stat_unchecked(&fscache_n_acquires_ok);
_leave(" = %p", cookie);
return cookie;
}
cache = fscache_select_cache_for_object(cookie->parent);
if (!cache) {
up_read(&fscache_addremove_sem);
- fscache_stat(&fscache_n_acquires_no_cache);
+ fscache_stat_unchecked(&fscache_n_acquires_no_cache);
_leave(" = -ENOMEDIUM [no cache]");
return -ENOMEDIUM;
}
object = cache->ops->alloc_object(cache, cookie);
fscache_stat_d(&fscache_n_cop_alloc_object);
if (IS_ERR(object)) {
- fscache_stat(&fscache_n_object_no_alloc);
+ fscache_stat_unchecked(&fscache_n_object_no_alloc);
ret = PTR_ERR(object);
goto error;
}
- fscache_stat(&fscache_n_object_alloc);
+ fscache_stat_unchecked(&fscache_n_object_alloc);
- object->debug_id = atomic_inc_return(&fscache_object_debug_id);
+ object->debug_id = atomic_inc_return_unchecked(&fscache_object_debug_id);
_debug("ALLOC OBJ%x: %s {%lx}",
object->debug_id, cookie->def->name, object->events);
_enter("{%s}", cookie->def->name);
- fscache_stat(&fscache_n_invalidates);
+ fscache_stat_unchecked(&fscache_n_invalidates);
/* Only permit invalidation of data files. Invalidating an index will
* require the caller to release all its attachments to the tree rooted
{
struct fscache_object *object;
- fscache_stat(&fscache_n_updates);
+ fscache_stat_unchecked(&fscache_n_updates);
if (!cookie) {
- fscache_stat(&fscache_n_updates_null);
+ fscache_stat_unchecked(&fscache_n_updates_null);
_leave(" [no cookie]");
return;
}
*/
void __fscache_relinquish_cookie(struct fscache_cookie *cookie, bool retire)
{
- fscache_stat(&fscache_n_relinquishes);
+ fscache_stat_unchecked(&fscache_n_relinquishes);
if (retire)
- fscache_stat(&fscache_n_relinquishes_retire);
+ fscache_stat_unchecked(&fscache_n_relinquishes_retire);
if (!cookie) {
- fscache_stat(&fscache_n_relinquishes_null);
+ fscache_stat_unchecked(&fscache_n_relinquishes_null);
_leave(" [no cookie]");
return;
}
if (test_bit(FSCACHE_IOERROR, &object->cache->flags))
goto inconsistent;
- op->debug_id = atomic_inc_return(&fscache_op_debug_id);
+ op->debug_id = atomic_inc_return_unchecked(&fscache_op_debug_id);
__fscache_use_cookie(cookie);
if (fscache_submit_op(object, op) < 0)
extern int fscache_wait_for_deferred_lookup(struct fscache_cookie *);
extern int fscache_wait_for_operation_activation(struct fscache_object *,
struct fscache_operation *,
- atomic_t *,
- atomic_t *,
+ atomic_unchecked_t *,
+ atomic_unchecked_t *,
void (*)(struct fscache_operation *));
extern void fscache_invalidate_writes(struct fscache_cookie *);
* stats.c
*/
#ifdef CONFIG_FSCACHE_STATS
-extern atomic_t fscache_n_ops_processed[FSCACHE_MAX_THREADS];
-extern atomic_t fscache_n_objs_processed[FSCACHE_MAX_THREADS];
-
-extern atomic_t fscache_n_op_pend;
-extern atomic_t fscache_n_op_run;
-extern atomic_t fscache_n_op_enqueue;
-extern atomic_t fscache_n_op_deferred_release;
-extern atomic_t fscache_n_op_release;
-extern atomic_t fscache_n_op_gc;
-extern atomic_t fscache_n_op_cancelled;
-extern atomic_t fscache_n_op_rejected;
-
-extern atomic_t fscache_n_attr_changed;
-extern atomic_t fscache_n_attr_changed_ok;
-extern atomic_t fscache_n_attr_changed_nobufs;
-extern atomic_t fscache_n_attr_changed_nomem;
-extern atomic_t fscache_n_attr_changed_calls;
-
-extern atomic_t fscache_n_allocs;
-extern atomic_t fscache_n_allocs_ok;
-extern atomic_t fscache_n_allocs_wait;
-extern atomic_t fscache_n_allocs_nobufs;
-extern atomic_t fscache_n_allocs_intr;
-extern atomic_t fscache_n_allocs_object_dead;
-extern atomic_t fscache_n_alloc_ops;
-extern atomic_t fscache_n_alloc_op_waits;
-
-extern atomic_t fscache_n_retrievals;
-extern atomic_t fscache_n_retrievals_ok;
-extern atomic_t fscache_n_retrievals_wait;
-extern atomic_t fscache_n_retrievals_nodata;
-extern atomic_t fscache_n_retrievals_nobufs;
-extern atomic_t fscache_n_retrievals_intr;
-extern atomic_t fscache_n_retrievals_nomem;
-extern atomic_t fscache_n_retrievals_object_dead;
-extern atomic_t fscache_n_retrieval_ops;
-extern atomic_t fscache_n_retrieval_op_waits;
-
-extern atomic_t fscache_n_stores;
-extern atomic_t fscache_n_stores_ok;
-extern atomic_t fscache_n_stores_again;
-extern atomic_t fscache_n_stores_nobufs;
-extern atomic_t fscache_n_stores_oom;
-extern atomic_t fscache_n_store_ops;
-extern atomic_t fscache_n_store_calls;
-extern atomic_t fscache_n_store_pages;
-extern atomic_t fscache_n_store_radix_deletes;
-extern atomic_t fscache_n_store_pages_over_limit;
-
-extern atomic_t fscache_n_store_vmscan_not_storing;
-extern atomic_t fscache_n_store_vmscan_gone;
-extern atomic_t fscache_n_store_vmscan_busy;
-extern atomic_t fscache_n_store_vmscan_cancelled;
-extern atomic_t fscache_n_store_vmscan_wait;
-
-extern atomic_t fscache_n_marks;
-extern atomic_t fscache_n_uncaches;
-
-extern atomic_t fscache_n_acquires;
-extern atomic_t fscache_n_acquires_null;
-extern atomic_t fscache_n_acquires_no_cache;
-extern atomic_t fscache_n_acquires_ok;
-extern atomic_t fscache_n_acquires_nobufs;
-extern atomic_t fscache_n_acquires_oom;
-
-extern atomic_t fscache_n_invalidates;
-extern atomic_t fscache_n_invalidates_run;
-
-extern atomic_t fscache_n_updates;
-extern atomic_t fscache_n_updates_null;
-extern atomic_t fscache_n_updates_run;
-
-extern atomic_t fscache_n_relinquishes;
-extern atomic_t fscache_n_relinquishes_null;
-extern atomic_t fscache_n_relinquishes_waitcrt;
-extern atomic_t fscache_n_relinquishes_retire;
-
-extern atomic_t fscache_n_cookie_index;
-extern atomic_t fscache_n_cookie_data;
-extern atomic_t fscache_n_cookie_special;
-
-extern atomic_t fscache_n_object_alloc;
-extern atomic_t fscache_n_object_no_alloc;
-extern atomic_t fscache_n_object_lookups;
-extern atomic_t fscache_n_object_lookups_negative;
-extern atomic_t fscache_n_object_lookups_positive;
-extern atomic_t fscache_n_object_lookups_timed_out;
-extern atomic_t fscache_n_object_created;
-extern atomic_t fscache_n_object_avail;
-extern atomic_t fscache_n_object_dead;
-
-extern atomic_t fscache_n_checkaux_none;
-extern atomic_t fscache_n_checkaux_okay;
-extern atomic_t fscache_n_checkaux_update;
-extern atomic_t fscache_n_checkaux_obsolete;
+extern atomic_unchecked_t fscache_n_ops_processed[FSCACHE_MAX_THREADS];
+extern atomic_unchecked_t fscache_n_objs_processed[FSCACHE_MAX_THREADS];
+
+extern atomic_unchecked_t fscache_n_op_pend;
+extern atomic_unchecked_t fscache_n_op_run;
+extern atomic_unchecked_t fscache_n_op_enqueue;
+extern atomic_unchecked_t fscache_n_op_deferred_release;
+extern atomic_unchecked_t fscache_n_op_release;
+extern atomic_unchecked_t fscache_n_op_gc;
+extern atomic_unchecked_t fscache_n_op_cancelled;
+extern atomic_unchecked_t fscache_n_op_rejected;
+
+extern atomic_unchecked_t fscache_n_attr_changed;
+extern atomic_unchecked_t fscache_n_attr_changed_ok;
+extern atomic_unchecked_t fscache_n_attr_changed_nobufs;
+extern atomic_unchecked_t fscache_n_attr_changed_nomem;
+extern atomic_unchecked_t fscache_n_attr_changed_calls;
+
+extern atomic_unchecked_t fscache_n_allocs;
+extern atomic_unchecked_t fscache_n_allocs_ok;
+extern atomic_unchecked_t fscache_n_allocs_wait;
+extern atomic_unchecked_t fscache_n_allocs_nobufs;
+extern atomic_unchecked_t fscache_n_allocs_intr;
+extern atomic_unchecked_t fscache_n_allocs_object_dead;
+extern atomic_unchecked_t fscache_n_alloc_ops;
+extern atomic_unchecked_t fscache_n_alloc_op_waits;
+
+extern atomic_unchecked_t fscache_n_retrievals;
+extern atomic_unchecked_t fscache_n_retrievals_ok;
+extern atomic_unchecked_t fscache_n_retrievals_wait;
+extern atomic_unchecked_t fscache_n_retrievals_nodata;
+extern atomic_unchecked_t fscache_n_retrievals_nobufs;
+extern atomic_unchecked_t fscache_n_retrievals_intr;
+extern atomic_unchecked_t fscache_n_retrievals_nomem;
+extern atomic_unchecked_t fscache_n_retrievals_object_dead;
+extern atomic_unchecked_t fscache_n_retrieval_ops;
+extern atomic_unchecked_t fscache_n_retrieval_op_waits;
+
+extern atomic_unchecked_t fscache_n_stores;
+extern atomic_unchecked_t fscache_n_stores_ok;
+extern atomic_unchecked_t fscache_n_stores_again;
+extern atomic_unchecked_t fscache_n_stores_nobufs;
+extern atomic_unchecked_t fscache_n_stores_oom;
+extern atomic_unchecked_t fscache_n_store_ops;
+extern atomic_unchecked_t fscache_n_store_calls;
+extern atomic_unchecked_t fscache_n_store_pages;
+extern atomic_unchecked_t fscache_n_store_radix_deletes;
+extern atomic_unchecked_t fscache_n_store_pages_over_limit;
+
+extern atomic_unchecked_t fscache_n_store_vmscan_not_storing;
+extern atomic_unchecked_t fscache_n_store_vmscan_gone;
+extern atomic_unchecked_t fscache_n_store_vmscan_busy;
+extern atomic_unchecked_t fscache_n_store_vmscan_cancelled;
+extern atomic_unchecked_t fscache_n_store_vmscan_wait;
+
+extern atomic_unchecked_t fscache_n_marks;
+extern atomic_unchecked_t fscache_n_uncaches;
+
+extern atomic_unchecked_t fscache_n_acquires;
+extern atomic_unchecked_t fscache_n_acquires_null;
+extern atomic_unchecked_t fscache_n_acquires_no_cache;
+extern atomic_unchecked_t fscache_n_acquires_ok;
+extern atomic_unchecked_t fscache_n_acquires_nobufs;
+extern atomic_unchecked_t fscache_n_acquires_oom;
+
+extern atomic_unchecked_t fscache_n_invalidates;
+extern atomic_unchecked_t fscache_n_invalidates_run;
+
+extern atomic_unchecked_t fscache_n_updates;
+extern atomic_unchecked_t fscache_n_updates_null;
+extern atomic_unchecked_t fscache_n_updates_run;
+
+extern atomic_unchecked_t fscache_n_relinquishes;
+extern atomic_unchecked_t fscache_n_relinquishes_null;
+extern atomic_unchecked_t fscache_n_relinquishes_waitcrt;
+extern atomic_unchecked_t fscache_n_relinquishes_retire;
+
+extern atomic_unchecked_t fscache_n_cookie_index;
+extern atomic_unchecked_t fscache_n_cookie_data;
+extern atomic_unchecked_t fscache_n_cookie_special;
+
+extern atomic_unchecked_t fscache_n_object_alloc;
+extern atomic_unchecked_t fscache_n_object_no_alloc;
+extern atomic_unchecked_t fscache_n_object_lookups;
+extern atomic_unchecked_t fscache_n_object_lookups_negative;
+extern atomic_unchecked_t fscache_n_object_lookups_positive;
+extern atomic_unchecked_t fscache_n_object_lookups_timed_out;
+extern atomic_unchecked_t fscache_n_object_created;
+extern atomic_unchecked_t fscache_n_object_avail;
+extern atomic_unchecked_t fscache_n_object_dead;
+
+extern atomic_unchecked_t fscache_n_checkaux_none;
+extern atomic_unchecked_t fscache_n_checkaux_okay;
+extern atomic_unchecked_t fscache_n_checkaux_update;
+extern atomic_unchecked_t fscache_n_checkaux_obsolete;
extern atomic_t fscache_n_cop_alloc_object;
extern atomic_t fscache_n_cop_lookup_object;
atomic_inc(stat);
}
+static inline void fscache_stat_unchecked(atomic_unchecked_t *stat)
+{
+ atomic_inc_unchecked(stat);
+}
+
static inline void fscache_stat_d(atomic_t *stat)
{
atomic_dec(stat);
#define __fscache_stat(stat) (NULL)
#define fscache_stat(stat) do {} while (0)
+#define fscache_stat_unchecked(stat) do {} while (0)
#define fscache_stat_d(stat) do {} while (0)
#endif
_debug("LOOKUP \"%s\" in \"%s\"",
cookie->def->name, object->cache->tag->name);
- fscache_stat(&fscache_n_object_lookups);
+ fscache_stat_unchecked(&fscache_n_object_lookups);
fscache_stat(&fscache_n_cop_lookup_object);
ret = object->cache->ops->lookup_object(object);
fscache_stat_d(&fscache_n_cop_lookup_object);
if (ret == -ETIMEDOUT) {
/* probably stuck behind another object, so move this one to
* the back of the queue */
- fscache_stat(&fscache_n_object_lookups_timed_out);
+ fscache_stat_unchecked(&fscache_n_object_lookups_timed_out);
_leave(" [timeout]");
return NO_TRANSIT;
}
_enter("{OBJ%x,%s}", object->debug_id, object->state->name);
if (!test_and_set_bit(FSCACHE_OBJECT_IS_LOOKED_UP, &object->flags)) {
- fscache_stat(&fscache_n_object_lookups_negative);
+ fscache_stat_unchecked(&fscache_n_object_lookups_negative);
/* Allow write requests to begin stacking up and read requests to begin
* returning ENODATA.
/* if we were still looking up, then we must have a positive lookup
* result, in which case there may be data available */
if (!test_and_set_bit(FSCACHE_OBJECT_IS_LOOKED_UP, &object->flags)) {
- fscache_stat(&fscache_n_object_lookups_positive);
+ fscache_stat_unchecked(&fscache_n_object_lookups_positive);
/* We do (presumably) have data */
clear_bit_unlock(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);
clear_bit_unlock(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags);
wake_up_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP);
} else {
- fscache_stat(&fscache_n_object_created);
+ fscache_stat_unchecked(&fscache_n_object_created);
}
set_bit(FSCACHE_OBJECT_IS_AVAILABLE, &object->flags);
fscache_stat_d(&fscache_n_cop_lookup_complete);
fscache_hist(fscache_obj_instantiate_histogram, object->lookup_jif);
- fscache_stat(&fscache_n_object_avail);
+ fscache_stat_unchecked(&fscache_n_object_avail);
_leave("");
return transit_to(JUMPSTART_DEPS);
/* this just shifts the object release to the work processor */
fscache_put_object(object);
- fscache_stat(&fscache_n_object_dead);
+ fscache_stat_unchecked(&fscache_n_object_dead);
_leave("");
return transit_to(OBJECT_DEAD);
enum fscache_checkaux result;
if (!object->cookie->def->check_aux) {
- fscache_stat(&fscache_n_checkaux_none);
+ fscache_stat_unchecked(&fscache_n_checkaux_none);
return FSCACHE_CHECKAUX_OKAY;
}
switch (result) {
/* entry okay as is */
case FSCACHE_CHECKAUX_OKAY:
- fscache_stat(&fscache_n_checkaux_okay);
+ fscache_stat_unchecked(&fscache_n_checkaux_okay);
break;
/* entry requires update */
case FSCACHE_CHECKAUX_NEEDS_UPDATE:
- fscache_stat(&fscache_n_checkaux_update);
+ fscache_stat_unchecked(&fscache_n_checkaux_update);
break;
/* entry requires deletion */
case FSCACHE_CHECKAUX_OBSOLETE:
- fscache_stat(&fscache_n_checkaux_obsolete);
+ fscache_stat_unchecked(&fscache_n_checkaux_obsolete);
break;
default:
{
const struct fscache_state *s;
- fscache_stat(&fscache_n_invalidates_run);
+ fscache_stat_unchecked(&fscache_n_invalidates_run);
fscache_stat(&fscache_n_cop_invalidate_object);
s = _fscache_invalidate_object(object, event);
fscache_stat_d(&fscache_n_cop_invalidate_object);
{
_enter("{OBJ%x},%d", object->debug_id, event);
- fscache_stat(&fscache_n_updates_run);
+ fscache_stat_unchecked(&fscache_n_updates_run);
fscache_stat(&fscache_n_cop_update_object);
object->cache->ops->update_object(object);
fscache_stat_d(&fscache_n_cop_update_object);
#include <linux/slab.h>
#include "internal.h"
-atomic_t fscache_op_debug_id;
+atomic_unchecked_t fscache_op_debug_id;
EXPORT_SYMBOL(fscache_op_debug_id);
/**
ASSERTCMP(atomic_read(&op->usage), >, 0);
ASSERTCMP(op->state, ==, FSCACHE_OP_ST_IN_PROGRESS);
- fscache_stat(&fscache_n_op_enqueue);
+ fscache_stat_unchecked(&fscache_n_op_enqueue);
switch (op->flags & FSCACHE_OP_TYPE) {
case FSCACHE_OP_ASYNC:
_debug("queue async");
wake_up_bit(&op->flags, FSCACHE_OP_WAITING);
if (op->processor)
fscache_enqueue_operation(op);
- fscache_stat(&fscache_n_op_run);
+ fscache_stat_unchecked(&fscache_n_op_run);
}
/*
if (object->n_in_progress > 0) {
atomic_inc(&op->usage);
list_add_tail(&op->pend_link, &object->pending_ops);
- fscache_stat(&fscache_n_op_pend);
+ fscache_stat_unchecked(&fscache_n_op_pend);
} else if (!list_empty(&object->pending_ops)) {
atomic_inc(&op->usage);
list_add_tail(&op->pend_link, &object->pending_ops);
- fscache_stat(&fscache_n_op_pend);
+ fscache_stat_unchecked(&fscache_n_op_pend);
fscache_start_operations(object);
} else {
ASSERTCMP(object->n_in_progress, ==, 0);
object->n_exclusive++; /* reads and writes must wait */
atomic_inc(&op->usage);
list_add_tail(&op->pend_link, &object->pending_ops);
- fscache_stat(&fscache_n_op_pend);
+ fscache_stat_unchecked(&fscache_n_op_pend);
ret = 0;
} else {
/* If we're in any other state, there must have been an I/O
if (object->n_exclusive > 0) {
atomic_inc(&op->usage);
list_add_tail(&op->pend_link, &object->pending_ops);
- fscache_stat(&fscache_n_op_pend);
+ fscache_stat_unchecked(&fscache_n_op_pend);
} else if (!list_empty(&object->pending_ops)) {
atomic_inc(&op->usage);
list_add_tail(&op->pend_link, &object->pending_ops);
- fscache_stat(&fscache_n_op_pend);
+ fscache_stat_unchecked(&fscache_n_op_pend);
fscache_start_operations(object);
} else {
ASSERTCMP(object->n_exclusive, ==, 0);
object->n_ops++;
atomic_inc(&op->usage);
list_add_tail(&op->pend_link, &object->pending_ops);
- fscache_stat(&fscache_n_op_pend);
+ fscache_stat_unchecked(&fscache_n_op_pend);
ret = 0;
} else if (fscache_object_is_dying(object)) {
- fscache_stat(&fscache_n_op_rejected);
+ fscache_stat_unchecked(&fscache_n_op_rejected);
op->state = FSCACHE_OP_ST_CANCELLED;
ret = -ENOBUFS;
} else if (!test_bit(FSCACHE_IOERROR, &object->cache->flags)) {
ret = -EBUSY;
if (op->state == FSCACHE_OP_ST_PENDING) {
ASSERT(!list_empty(&op->pend_link));
- fscache_stat(&fscache_n_op_cancelled);
+ fscache_stat_unchecked(&fscache_n_op_cancelled);
list_del_init(&op->pend_link);
if (do_cancel)
do_cancel(op);
while (!list_empty(&object->pending_ops)) {
op = list_entry(object->pending_ops.next,
struct fscache_operation, pend_link);
- fscache_stat(&fscache_n_op_cancelled);
+ fscache_stat_unchecked(&fscache_n_op_cancelled);
list_del_init(&op->pend_link);
ASSERTCMP(op->state, ==, FSCACHE_OP_ST_PENDING);
op->state, ==, FSCACHE_OP_ST_CANCELLED);
op->state = FSCACHE_OP_ST_DEAD;
- fscache_stat(&fscache_n_op_release);
+ fscache_stat_unchecked(&fscache_n_op_release);
if (op->release) {
op->release(op);
* lock, and defer it otherwise */
if (!spin_trylock(&object->lock)) {
_debug("defer put");
- fscache_stat(&fscache_n_op_deferred_release);
+ fscache_stat_unchecked(&fscache_n_op_deferred_release);
cache = object->cache;
spin_lock(&cache->op_gc_list_lock);
_debug("GC DEFERRED REL OBJ%x OP%x",
object->debug_id, op->debug_id);
- fscache_stat(&fscache_n_op_gc);
+ fscache_stat_unchecked(&fscache_n_op_gc);
ASSERTCMP(atomic_read(&op->usage), ==, 0);
ASSERTCMP(op->state, ==, FSCACHE_OP_ST_DEAD);
val = radix_tree_lookup(&cookie->stores, page->index);
if (!val) {
rcu_read_unlock();
- fscache_stat(&fscache_n_store_vmscan_not_storing);
+ fscache_stat_unchecked(&fscache_n_store_vmscan_not_storing);
__fscache_uncache_page(cookie, page);
return true;
}
spin_unlock(&cookie->stores_lock);
if (xpage) {
- fscache_stat(&fscache_n_store_vmscan_cancelled);
- fscache_stat(&fscache_n_store_radix_deletes);
+ fscache_stat_unchecked(&fscache_n_store_vmscan_cancelled);
+ fscache_stat_unchecked(&fscache_n_store_radix_deletes);
ASSERTCMP(xpage, ==, page);
} else {
- fscache_stat(&fscache_n_store_vmscan_gone);
+ fscache_stat_unchecked(&fscache_n_store_vmscan_gone);
}
wake_up_bit(&cookie->flags, 0);
* sleeping on memory allocation, so we may need to impose a timeout
* too. */
if (!(gfp & __GFP_WAIT) || !(gfp & __GFP_FS)) {
- fscache_stat(&fscache_n_store_vmscan_busy);
+ fscache_stat_unchecked(&fscache_n_store_vmscan_busy);
return false;
}
- fscache_stat(&fscache_n_store_vmscan_wait);
+ fscache_stat_unchecked(&fscache_n_store_vmscan_wait);
if (!release_page_wait_timeout(cookie, page))
_debug("fscache writeout timeout page: %p{%lx}",
page, page->index);
FSCACHE_COOKIE_STORING_TAG);
if (!radix_tree_tag_get(&cookie->stores, page->index,
FSCACHE_COOKIE_PENDING_TAG)) {
- fscache_stat(&fscache_n_store_radix_deletes);
+ fscache_stat_unchecked(&fscache_n_store_radix_deletes);
xpage = radix_tree_delete(&cookie->stores, page->index);
}
spin_unlock(&cookie->stores_lock);
_enter("{OBJ%x OP%x}", object->debug_id, op->debug_id);
- fscache_stat(&fscache_n_attr_changed_calls);
+ fscache_stat_unchecked(&fscache_n_attr_changed_calls);
if (fscache_object_is_active(object)) {
fscache_stat(&fscache_n_cop_attr_changed);
ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
- fscache_stat(&fscache_n_attr_changed);
+ fscache_stat_unchecked(&fscache_n_attr_changed);
op = kzalloc(sizeof(*op), GFP_KERNEL);
if (!op) {
- fscache_stat(&fscache_n_attr_changed_nomem);
+ fscache_stat_unchecked(&fscache_n_attr_changed_nomem);
_leave(" = -ENOMEM");
return -ENOMEM;
}
if (fscache_submit_exclusive_op(object, op) < 0)
goto nobufs_dec;
spin_unlock(&cookie->lock);
- fscache_stat(&fscache_n_attr_changed_ok);
+ fscache_stat_unchecked(&fscache_n_attr_changed_ok);
fscache_put_operation(op);
_leave(" = 0");
return 0;
kfree(op);
if (wake_cookie)
__fscache_wake_unused_cookie(cookie);
- fscache_stat(&fscache_n_attr_changed_nobufs);
+ fscache_stat_unchecked(&fscache_n_attr_changed_nobufs);
_leave(" = %d", -ENOBUFS);
return -ENOBUFS;
}
/* allocate a retrieval operation and attempt to submit it */
op = kzalloc(sizeof(*op), GFP_NOIO);
if (!op) {
- fscache_stat(&fscache_n_retrievals_nomem);
+ fscache_stat_unchecked(&fscache_n_retrievals_nomem);
return NULL;
}
return 0;
}
- fscache_stat(&fscache_n_retrievals_wait);
+ fscache_stat_unchecked(&fscache_n_retrievals_wait);
jif = jiffies;
if (wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,
TASK_INTERRUPTIBLE) != 0) {
- fscache_stat(&fscache_n_retrievals_intr);
+ fscache_stat_unchecked(&fscache_n_retrievals_intr);
_leave(" = -ERESTARTSYS");
return -ERESTARTSYS;
}
*/
int fscache_wait_for_operation_activation(struct fscache_object *object,
struct fscache_operation *op,
- atomic_t *stat_op_waits,
- atomic_t *stat_object_dead,
+ atomic_unchecked_t *stat_op_waits,
+ atomic_unchecked_t *stat_object_dead,
void (*do_cancel)(struct fscache_operation *))
{
int ret;
_debug(">>> WT");
if (stat_op_waits)
- fscache_stat(stat_op_waits);
+ fscache_stat_unchecked(stat_op_waits);
if (wait_on_bit(&op->flags, FSCACHE_OP_WAITING,
TASK_INTERRUPTIBLE) != 0) {
ret = fscache_cancel_op(op, do_cancel);
check_if_dead:
if (op->state == FSCACHE_OP_ST_CANCELLED) {
if (stat_object_dead)
- fscache_stat(stat_object_dead);
+ fscache_stat_unchecked(stat_object_dead);
_leave(" = -ENOBUFS [cancelled]");
return -ENOBUFS;
}
pr_err("%s() = -ENOBUFS [obj dead %d]\n", __func__, op->state);
fscache_cancel_op(op, do_cancel);
if (stat_object_dead)
- fscache_stat(stat_object_dead);
+ fscache_stat_unchecked(stat_object_dead);
return -ENOBUFS;
}
return 0;
_enter("%p,%p,,,", cookie, page);
- fscache_stat(&fscache_n_retrievals);
+ fscache_stat_unchecked(&fscache_n_retrievals);
if (hlist_empty(&cookie->backing_objects))
goto nobufs;
goto nobufs_unlock_dec;
spin_unlock(&cookie->lock);
- fscache_stat(&fscache_n_retrieval_ops);
+ fscache_stat_unchecked(&fscache_n_retrieval_ops);
/* pin the netfs read context in case we need to do the actual netfs
* read because we've encountered a cache read failure */
error:
if (ret == -ENOMEM)
- fscache_stat(&fscache_n_retrievals_nomem);
+ fscache_stat_unchecked(&fscache_n_retrievals_nomem);
else if (ret == -ERESTARTSYS)
- fscache_stat(&fscache_n_retrievals_intr);
+ fscache_stat_unchecked(&fscache_n_retrievals_intr);
else if (ret == -ENODATA)
- fscache_stat(&fscache_n_retrievals_nodata);
+ fscache_stat_unchecked(&fscache_n_retrievals_nodata);
else if (ret < 0)
- fscache_stat(&fscache_n_retrievals_nobufs);
+ fscache_stat_unchecked(&fscache_n_retrievals_nobufs);
else
- fscache_stat(&fscache_n_retrievals_ok);
+ fscache_stat_unchecked(&fscache_n_retrievals_ok);
fscache_put_retrieval(op);
_leave(" = %d", ret);
__fscache_wake_unused_cookie(cookie);
kfree(op);
nobufs:
- fscache_stat(&fscache_n_retrievals_nobufs);
+ fscache_stat_unchecked(&fscache_n_retrievals_nobufs);
_leave(" = -ENOBUFS");
return -ENOBUFS;
}
_enter("%p,,%d,,,", cookie, *nr_pages);
- fscache_stat(&fscache_n_retrievals);
+ fscache_stat_unchecked(&fscache_n_retrievals);
if (hlist_empty(&cookie->backing_objects))
goto nobufs;
goto nobufs_unlock_dec;
spin_unlock(&cookie->lock);
- fscache_stat(&fscache_n_retrieval_ops);
+ fscache_stat_unchecked(&fscache_n_retrieval_ops);
/* pin the netfs read context in case we need to do the actual netfs
* read because we've encountered a cache read failure */
error:
if (ret == -ENOMEM)
- fscache_stat(&fscache_n_retrievals_nomem);
+ fscache_stat_unchecked(&fscache_n_retrievals_nomem);
else if (ret == -ERESTARTSYS)
- fscache_stat(&fscache_n_retrievals_intr);
+ fscache_stat_unchecked(&fscache_n_retrievals_intr);
else if (ret == -ENODATA)
- fscache_stat(&fscache_n_retrievals_nodata);
+ fscache_stat_unchecked(&fscache_n_retrievals_nodata);
else if (ret < 0)
- fscache_stat(&fscache_n_retrievals_nobufs);
+ fscache_stat_unchecked(&fscache_n_retrievals_nobufs);
else
- fscache_stat(&fscache_n_retrievals_ok);
+ fscache_stat_unchecked(&fscache_n_retrievals_ok);
fscache_put_retrieval(op);
_leave(" = %d", ret);
if (wake_cookie)
__fscache_wake_unused_cookie(cookie);
nobufs:
- fscache_stat(&fscache_n_retrievals_nobufs);
+ fscache_stat_unchecked(&fscache_n_retrievals_nobufs);
_leave(" = -ENOBUFS");
return -ENOBUFS;
}
_enter("%p,%p,,,", cookie, page);
- fscache_stat(&fscache_n_allocs);
+ fscache_stat_unchecked(&fscache_n_allocs);
if (hlist_empty(&cookie->backing_objects))
goto nobufs;
goto nobufs_unlock_dec;
spin_unlock(&cookie->lock);
- fscache_stat(&fscache_n_alloc_ops);
+ fscache_stat_unchecked(&fscache_n_alloc_ops);
ret = fscache_wait_for_operation_activation(
object, &op->op,
error:
if (ret == -ERESTARTSYS)
- fscache_stat(&fscache_n_allocs_intr);
+ fscache_stat_unchecked(&fscache_n_allocs_intr);
else if (ret < 0)
- fscache_stat(&fscache_n_allocs_nobufs);
+ fscache_stat_unchecked(&fscache_n_allocs_nobufs);
else
- fscache_stat(&fscache_n_allocs_ok);
+ fscache_stat_unchecked(&fscache_n_allocs_ok);
fscache_put_retrieval(op);
_leave(" = %d", ret);
if (wake_cookie)
__fscache_wake_unused_cookie(cookie);
nobufs:
- fscache_stat(&fscache_n_allocs_nobufs);
+ fscache_stat_unchecked(&fscache_n_allocs_nobufs);
_leave(" = -ENOBUFS");
return -ENOBUFS;
}
spin_lock(&cookie->stores_lock);
- fscache_stat(&fscache_n_store_calls);
+ fscache_stat_unchecked(&fscache_n_store_calls);
/* find a page to store */
page = NULL;
page = results[0];
_debug("gang %d [%lx]", n, page->index);
if (page->index > op->store_limit) {
- fscache_stat(&fscache_n_store_pages_over_limit);
+ fscache_stat_unchecked(&fscache_n_store_pages_over_limit);
goto superseded;
}
spin_unlock(&cookie->stores_lock);
spin_unlock(&object->lock);
- fscache_stat(&fscache_n_store_pages);
+ fscache_stat_unchecked(&fscache_n_store_pages);
fscache_stat(&fscache_n_cop_write_page);
ret = object->cache->ops->write_page(op, page);
fscache_stat_d(&fscache_n_cop_write_page);
ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
ASSERT(PageFsCache(page));
- fscache_stat(&fscache_n_stores);
+ fscache_stat_unchecked(&fscache_n_stores);
if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
_leave(" = -ENOBUFS [invalidating]");
spin_unlock(&cookie->stores_lock);
spin_unlock(&object->lock);
- op->op.debug_id = atomic_inc_return(&fscache_op_debug_id);
+ op->op.debug_id = atomic_inc_return_unchecked(&fscache_op_debug_id);
op->store_limit = object->store_limit;
__fscache_use_cookie(cookie);
spin_unlock(&cookie->lock);
radix_tree_preload_end();
- fscache_stat(&fscache_n_store_ops);
- fscache_stat(&fscache_n_stores_ok);
+ fscache_stat_unchecked(&fscache_n_store_ops);
+ fscache_stat_unchecked(&fscache_n_stores_ok);
/* the work queue now carries its own ref on the object */
fscache_put_operation(&op->op);
return 0;
already_queued:
- fscache_stat(&fscache_n_stores_again);
+ fscache_stat_unchecked(&fscache_n_stores_again);
already_pending:
spin_unlock(&cookie->stores_lock);
spin_unlock(&object->lock);
spin_unlock(&cookie->lock);
radix_tree_preload_end();
kfree(op);
- fscache_stat(&fscache_n_stores_ok);
+ fscache_stat_unchecked(&fscache_n_stores_ok);
_leave(" = 0");
return 0;
kfree(op);
if (wake_cookie)
__fscache_wake_unused_cookie(cookie);
- fscache_stat(&fscache_n_stores_nobufs);
+ fscache_stat_unchecked(&fscache_n_stores_nobufs);
_leave(" = -ENOBUFS");
return -ENOBUFS;
nomem_free:
kfree(op);
nomem:
- fscache_stat(&fscache_n_stores_oom);
+ fscache_stat_unchecked(&fscache_n_stores_oom);
_leave(" = -ENOMEM");
return -ENOMEM;
}
ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
ASSERTCMP(page, !=, NULL);
- fscache_stat(&fscache_n_uncaches);
+ fscache_stat_unchecked(&fscache_n_uncaches);
/* cache withdrawal may beat us to it */
if (!PageFsCache(page))
struct fscache_cookie *cookie = op->op.object->cookie;
#ifdef CONFIG_FSCACHE_STATS
- atomic_inc(&fscache_n_marks);
+ atomic_inc_unchecked(&fscache_n_marks);
#endif
_debug("- mark %p{%lx}", page, page->index);
/*
* operation counters
*/
-atomic_t fscache_n_op_pend;
-atomic_t fscache_n_op_run;
-atomic_t fscache_n_op_enqueue;
-atomic_t fscache_n_op_requeue;
-atomic_t fscache_n_op_deferred_release;
-atomic_t fscache_n_op_release;
-atomic_t fscache_n_op_gc;
-atomic_t fscache_n_op_cancelled;
-atomic_t fscache_n_op_rejected;
-
-atomic_t fscache_n_attr_changed;
-atomic_t fscache_n_attr_changed_ok;
-atomic_t fscache_n_attr_changed_nobufs;
-atomic_t fscache_n_attr_changed_nomem;
-atomic_t fscache_n_attr_changed_calls;
-
-atomic_t fscache_n_allocs;
-atomic_t fscache_n_allocs_ok;
-atomic_t fscache_n_allocs_wait;
-atomic_t fscache_n_allocs_nobufs;
-atomic_t fscache_n_allocs_intr;
-atomic_t fscache_n_allocs_object_dead;
-atomic_t fscache_n_alloc_ops;
-atomic_t fscache_n_alloc_op_waits;
-
-atomic_t fscache_n_retrievals;
-atomic_t fscache_n_retrievals_ok;
-atomic_t fscache_n_retrievals_wait;
-atomic_t fscache_n_retrievals_nodata;
-atomic_t fscache_n_retrievals_nobufs;
-atomic_t fscache_n_retrievals_intr;
-atomic_t fscache_n_retrievals_nomem;
-atomic_t fscache_n_retrievals_object_dead;
-atomic_t fscache_n_retrieval_ops;
-atomic_t fscache_n_retrieval_op_waits;
-
-atomic_t fscache_n_stores;
-atomic_t fscache_n_stores_ok;
-atomic_t fscache_n_stores_again;
-atomic_t fscache_n_stores_nobufs;
-atomic_t fscache_n_stores_oom;
-atomic_t fscache_n_store_ops;
-atomic_t fscache_n_store_calls;
-atomic_t fscache_n_store_pages;
-atomic_t fscache_n_store_radix_deletes;
-atomic_t fscache_n_store_pages_over_limit;
-
-atomic_t fscache_n_store_vmscan_not_storing;
-atomic_t fscache_n_store_vmscan_gone;
-atomic_t fscache_n_store_vmscan_busy;
-atomic_t fscache_n_store_vmscan_cancelled;
-atomic_t fscache_n_store_vmscan_wait;
-
-atomic_t fscache_n_marks;
-atomic_t fscache_n_uncaches;
-
-atomic_t fscache_n_acquires;
-atomic_t fscache_n_acquires_null;
-atomic_t fscache_n_acquires_no_cache;
-atomic_t fscache_n_acquires_ok;
-atomic_t fscache_n_acquires_nobufs;
-atomic_t fscache_n_acquires_oom;
-
-atomic_t fscache_n_invalidates;
-atomic_t fscache_n_invalidates_run;
-
-atomic_t fscache_n_updates;
-atomic_t fscache_n_updates_null;
-atomic_t fscache_n_updates_run;
-
-atomic_t fscache_n_relinquishes;
-atomic_t fscache_n_relinquishes_null;
-atomic_t fscache_n_relinquishes_waitcrt;
-atomic_t fscache_n_relinquishes_retire;
-
-atomic_t fscache_n_cookie_index;
-atomic_t fscache_n_cookie_data;
-atomic_t fscache_n_cookie_special;
-
-atomic_t fscache_n_object_alloc;
-atomic_t fscache_n_object_no_alloc;
-atomic_t fscache_n_object_lookups;
-atomic_t fscache_n_object_lookups_negative;
-atomic_t fscache_n_object_lookups_positive;
-atomic_t fscache_n_object_lookups_timed_out;
-atomic_t fscache_n_object_created;
-atomic_t fscache_n_object_avail;
-atomic_t fscache_n_object_dead;
-
-atomic_t fscache_n_checkaux_none;
-atomic_t fscache_n_checkaux_okay;
-atomic_t fscache_n_checkaux_update;
-atomic_t fscache_n_checkaux_obsolete;
+atomic_unchecked_t fscache_n_op_pend;
+atomic_unchecked_t fscache_n_op_run;
+atomic_unchecked_t fscache_n_op_enqueue;
+atomic_unchecked_t fscache_n_op_requeue;
+atomic_unchecked_t fscache_n_op_deferred_release;
+atomic_unchecked_t fscache_n_op_release;
+atomic_unchecked_t fscache_n_op_gc;
+atomic_unchecked_t fscache_n_op_cancelled;
+atomic_unchecked_t fscache_n_op_rejected;
+
+atomic_unchecked_t fscache_n_attr_changed;
+atomic_unchecked_t fscache_n_attr_changed_ok;
+atomic_unchecked_t fscache_n_attr_changed_nobufs;
+atomic_unchecked_t fscache_n_attr_changed_nomem;
+atomic_unchecked_t fscache_n_attr_changed_calls;
+
+atomic_unchecked_t fscache_n_allocs;
+atomic_unchecked_t fscache_n_allocs_ok;
+atomic_unchecked_t fscache_n_allocs_wait;
+atomic_unchecked_t fscache_n_allocs_nobufs;
+atomic_unchecked_t fscache_n_allocs_intr;
+atomic_unchecked_t fscache_n_allocs_object_dead;
+atomic_unchecked_t fscache_n_alloc_ops;
+atomic_unchecked_t fscache_n_alloc_op_waits;
+
+atomic_unchecked_t fscache_n_retrievals;
+atomic_unchecked_t fscache_n_retrievals_ok;
+atomic_unchecked_t fscache_n_retrievals_wait;
+atomic_unchecked_t fscache_n_retrievals_nodata;
+atomic_unchecked_t fscache_n_retrievals_nobufs;
+atomic_unchecked_t fscache_n_retrievals_intr;
+atomic_unchecked_t fscache_n_retrievals_nomem;
+atomic_unchecked_t fscache_n_retrievals_object_dead;
+atomic_unchecked_t fscache_n_retrieval_ops;
+atomic_unchecked_t fscache_n_retrieval_op_waits;
+
+atomic_unchecked_t fscache_n_stores;
+atomic_unchecked_t fscache_n_stores_ok;
+atomic_unchecked_t fscache_n_stores_again;
+atomic_unchecked_t fscache_n_stores_nobufs;
+atomic_unchecked_t fscache_n_stores_oom;
+atomic_unchecked_t fscache_n_store_ops;
+atomic_unchecked_t fscache_n_store_calls;
+atomic_unchecked_t fscache_n_store_pages;
+atomic_unchecked_t fscache_n_store_radix_deletes;
+atomic_unchecked_t fscache_n_store_pages_over_limit;
+
+atomic_unchecked_t fscache_n_store_vmscan_not_storing;
+atomic_unchecked_t fscache_n_store_vmscan_gone;
+atomic_unchecked_t fscache_n_store_vmscan_busy;
+atomic_unchecked_t fscache_n_store_vmscan_cancelled;
+atomic_unchecked_t fscache_n_store_vmscan_wait;
+
+atomic_unchecked_t fscache_n_marks;
+atomic_unchecked_t fscache_n_uncaches;
+
+atomic_unchecked_t fscache_n_acquires;
+atomic_unchecked_t fscache_n_acquires_null;
+atomic_unchecked_t fscache_n_acquires_no_cache;
+atomic_unchecked_t fscache_n_acquires_ok;
+atomic_unchecked_t fscache_n_acquires_nobufs;
+atomic_unchecked_t fscache_n_acquires_oom;
+
+atomic_unchecked_t fscache_n_invalidates;
+atomic_unchecked_t fscache_n_invalidates_run;
+
+atomic_unchecked_t fscache_n_updates;
+atomic_unchecked_t fscache_n_updates_null;
+atomic_unchecked_t fscache_n_updates_run;
+
+atomic_unchecked_t fscache_n_relinquishes;
+atomic_unchecked_t fscache_n_relinquishes_null;
+atomic_unchecked_t fscache_n_relinquishes_waitcrt;
+atomic_unchecked_t fscache_n_relinquishes_retire;
+
+atomic_unchecked_t fscache_n_cookie_index;
+atomic_unchecked_t fscache_n_cookie_data;
+atomic_unchecked_t fscache_n_cookie_special;
+
+atomic_unchecked_t fscache_n_object_alloc;
+atomic_unchecked_t fscache_n_object_no_alloc;
+atomic_unchecked_t fscache_n_object_lookups;
+atomic_unchecked_t fscache_n_object_lookups_negative;
+atomic_unchecked_t fscache_n_object_lookups_positive;
+atomic_unchecked_t fscache_n_object_lookups_timed_out;
+atomic_unchecked_t fscache_n_object_created;
+atomic_unchecked_t fscache_n_object_avail;
+atomic_unchecked_t fscache_n_object_dead;
+
+atomic_unchecked_t fscache_n_checkaux_none;
+atomic_unchecked_t fscache_n_checkaux_okay;
+atomic_unchecked_t fscache_n_checkaux_update;
+atomic_unchecked_t fscache_n_checkaux_obsolete;
atomic_t fscache_n_cop_alloc_object;
atomic_t fscache_n_cop_lookup_object;
seq_puts(m, "FS-Cache statistics\n");
seq_printf(m, "Cookies: idx=%u dat=%u spc=%u\n",
- atomic_read(&fscache_n_cookie_index),
- atomic_read(&fscache_n_cookie_data),
- atomic_read(&fscache_n_cookie_special));
+ atomic_read_unchecked(&fscache_n_cookie_index),
+ atomic_read_unchecked(&fscache_n_cookie_data),
+ atomic_read_unchecked(&fscache_n_cookie_special));
seq_printf(m, "Objects: alc=%u nal=%u avl=%u ded=%u\n",
- atomic_read(&fscache_n_object_alloc),
- atomic_read(&fscache_n_object_no_alloc),
- atomic_read(&fscache_n_object_avail),
- atomic_read(&fscache_n_object_dead));
+ atomic_read_unchecked(&fscache_n_object_alloc),
+ atomic_read_unchecked(&fscache_n_object_no_alloc),
+ atomic_read_unchecked(&fscache_n_object_avail),
+ atomic_read_unchecked(&fscache_n_object_dead));
seq_printf(m, "ChkAux : non=%u ok=%u upd=%u obs=%u\n",
- atomic_read(&fscache_n_checkaux_none),
- atomic_read(&fscache_n_checkaux_okay),
- atomic_read(&fscache_n_checkaux_update),
- atomic_read(&fscache_n_checkaux_obsolete));
+ atomic_read_unchecked(&fscache_n_checkaux_none),
+ atomic_read_unchecked(&fscache_n_checkaux_okay),
+ atomic_read_unchecked(&fscache_n_checkaux_update),
+ atomic_read_unchecked(&fscache_n_checkaux_obsolete));
seq_printf(m, "Pages : mrk=%u unc=%u\n",
- atomic_read(&fscache_n_marks),
- atomic_read(&fscache_n_uncaches));
+ atomic_read_unchecked(&fscache_n_marks),
+ atomic_read_unchecked(&fscache_n_uncaches));
seq_printf(m, "Acquire: n=%u nul=%u noc=%u ok=%u nbf=%u"
" oom=%u\n",
- atomic_read(&fscache_n_acquires),
- atomic_read(&fscache_n_acquires_null),
- atomic_read(&fscache_n_acquires_no_cache),
- atomic_read(&fscache_n_acquires_ok),
- atomic_read(&fscache_n_acquires_nobufs),
- atomic_read(&fscache_n_acquires_oom));
+ atomic_read_unchecked(&fscache_n_acquires),
+ atomic_read_unchecked(&fscache_n_acquires_null),
+ atomic_read_unchecked(&fscache_n_acquires_no_cache),
+ atomic_read_unchecked(&fscache_n_acquires_ok),
+ atomic_read_unchecked(&fscache_n_acquires_nobufs),
+ atomic_read_unchecked(&fscache_n_acquires_oom));
seq_printf(m, "Lookups: n=%u neg=%u pos=%u crt=%u tmo=%u\n",
- atomic_read(&fscache_n_object_lookups),
- atomic_read(&fscache_n_object_lookups_negative),
- atomic_read(&fscache_n_object_lookups_positive),
- atomic_read(&fscache_n_object_created),
- atomic_read(&fscache_n_object_lookups_timed_out));
+ atomic_read_unchecked(&fscache_n_object_lookups),
+ atomic_read_unchecked(&fscache_n_object_lookups_negative),
+ atomic_read_unchecked(&fscache_n_object_lookups_positive),
+ atomic_read_unchecked(&fscache_n_object_created),
+ atomic_read_unchecked(&fscache_n_object_lookups_timed_out));
seq_printf(m, "Invals : n=%u run=%u\n",
- atomic_read(&fscache_n_invalidates),
- atomic_read(&fscache_n_invalidates_run));
+ atomic_read_unchecked(&fscache_n_invalidates),
+ atomic_read_unchecked(&fscache_n_invalidates_run));
seq_printf(m, "Updates: n=%u nul=%u run=%u\n",
- atomic_read(&fscache_n_updates),
- atomic_read(&fscache_n_updates_null),
- atomic_read(&fscache_n_updates_run));
+ atomic_read_unchecked(&fscache_n_updates),
+ atomic_read_unchecked(&fscache_n_updates_null),
+ atomic_read_unchecked(&fscache_n_updates_run));
seq_printf(m, "Relinqs: n=%u nul=%u wcr=%u rtr=%u\n",
- atomic_read(&fscache_n_relinquishes),
- atomic_read(&fscache_n_relinquishes_null),
- atomic_read(&fscache_n_relinquishes_waitcrt),
- atomic_read(&fscache_n_relinquishes_retire));
+ atomic_read_unchecked(&fscache_n_relinquishes),
+ atomic_read_unchecked(&fscache_n_relinquishes_null),
+ atomic_read_unchecked(&fscache_n_relinquishes_waitcrt),
+ atomic_read_unchecked(&fscache_n_relinquishes_retire));
seq_printf(m, "AttrChg: n=%u ok=%u nbf=%u oom=%u run=%u\n",
- atomic_read(&fscache_n_attr_changed),
- atomic_read(&fscache_n_attr_changed_ok),
- atomic_read(&fscache_n_attr_changed_nobufs),
- atomic_read(&fscache_n_attr_changed_nomem),
- atomic_read(&fscache_n_attr_changed_calls));
+ atomic_read_unchecked(&fscache_n_attr_changed),
+ atomic_read_unchecked(&fscache_n_attr_changed_ok),
+ atomic_read_unchecked(&fscache_n_attr_changed_nobufs),
+ atomic_read_unchecked(&fscache_n_attr_changed_nomem),
+ atomic_read_unchecked(&fscache_n_attr_changed_calls));
seq_printf(m, "Allocs : n=%u ok=%u wt=%u nbf=%u int=%u\n",
- atomic_read(&fscache_n_allocs),
- atomic_read(&fscache_n_allocs_ok),
- atomic_read(&fscache_n_allocs_wait),
- atomic_read(&fscache_n_allocs_nobufs),
- atomic_read(&fscache_n_allocs_intr));
+ atomic_read_unchecked(&fscache_n_allocs),
+ atomic_read_unchecked(&fscache_n_allocs_ok),
+ atomic_read_unchecked(&fscache_n_allocs_wait),
+ atomic_read_unchecked(&fscache_n_allocs_nobufs),
+ atomic_read_unchecked(&fscache_n_allocs_intr));
seq_printf(m, "Allocs : ops=%u owt=%u abt=%u\n",
- atomic_read(&fscache_n_alloc_ops),
- atomic_read(&fscache_n_alloc_op_waits),
- atomic_read(&fscache_n_allocs_object_dead));
+ atomic_read_unchecked(&fscache_n_alloc_ops),
+ atomic_read_unchecked(&fscache_n_alloc_op_waits),
+ atomic_read_unchecked(&fscache_n_allocs_object_dead));
seq_printf(m, "Retrvls: n=%u ok=%u wt=%u nod=%u nbf=%u"
" int=%u oom=%u\n",
- atomic_read(&fscache_n_retrievals),
- atomic_read(&fscache_n_retrievals_ok),
- atomic_read(&fscache_n_retrievals_wait),
- atomic_read(&fscache_n_retrievals_nodata),
- atomic_read(&fscache_n_retrievals_nobufs),
- atomic_read(&fscache_n_retrievals_intr),
- atomic_read(&fscache_n_retrievals_nomem));
+ atomic_read_unchecked(&fscache_n_retrievals),
+ atomic_read_unchecked(&fscache_n_retrievals_ok),
+ atomic_read_unchecked(&fscache_n_retrievals_wait),
+ atomic_read_unchecked(&fscache_n_retrievals_nodata),
+ atomic_read_unchecked(&fscache_n_retrievals_nobufs),
+ atomic_read_unchecked(&fscache_n_retrievals_intr),
+ atomic_read_unchecked(&fscache_n_retrievals_nomem));
seq_printf(m, "Retrvls: ops=%u owt=%u abt=%u\n",
- atomic_read(&fscache_n_retrieval_ops),
- atomic_read(&fscache_n_retrieval_op_waits),
- atomic_read(&fscache_n_retrievals_object_dead));
+ atomic_read_unchecked(&fscache_n_retrieval_ops),
+ atomic_read_unchecked(&fscache_n_retrieval_op_waits),
+ atomic_read_unchecked(&fscache_n_retrievals_object_dead));
seq_printf(m, "Stores : n=%u ok=%u agn=%u nbf=%u oom=%u\n",
- atomic_read(&fscache_n_stores),
- atomic_read(&fscache_n_stores_ok),
- atomic_read(&fscache_n_stores_again),
- atomic_read(&fscache_n_stores_nobufs),
- atomic_read(&fscache_n_stores_oom));
+ atomic_read_unchecked(&fscache_n_stores),
+ atomic_read_unchecked(&fscache_n_stores_ok),
+ atomic_read_unchecked(&fscache_n_stores_again),
+ atomic_read_unchecked(&fscache_n_stores_nobufs),
+ atomic_read_unchecked(&fscache_n_stores_oom));
seq_printf(m, "Stores : ops=%u run=%u pgs=%u rxd=%u olm=%u\n",
- atomic_read(&fscache_n_store_ops),
- atomic_read(&fscache_n_store_calls),
- atomic_read(&fscache_n_store_pages),
- atomic_read(&fscache_n_store_radix_deletes),
- atomic_read(&fscache_n_store_pages_over_limit));
+ atomic_read_unchecked(&fscache_n_store_ops),
+ atomic_read_unchecked(&fscache_n_store_calls),
+ atomic_read_unchecked(&fscache_n_store_pages),
+ atomic_read_unchecked(&fscache_n_store_radix_deletes),
+ atomic_read_unchecked(&fscache_n_store_pages_over_limit));
seq_printf(m, "VmScan : nos=%u gon=%u bsy=%u can=%u wt=%u\n",
- atomic_read(&fscache_n_store_vmscan_not_storing),
- atomic_read(&fscache_n_store_vmscan_gone),
- atomic_read(&fscache_n_store_vmscan_busy),
- atomic_read(&fscache_n_store_vmscan_cancelled),
- atomic_read(&fscache_n_store_vmscan_wait));
+ atomic_read_unchecked(&fscache_n_store_vmscan_not_storing),
+ atomic_read_unchecked(&fscache_n_store_vmscan_gone),
+ atomic_read_unchecked(&fscache_n_store_vmscan_busy),
+ atomic_read_unchecked(&fscache_n_store_vmscan_cancelled),
+ atomic_read_unchecked(&fscache_n_store_vmscan_wait));
seq_printf(m, "Ops : pend=%u run=%u enq=%u can=%u rej=%u\n",
- atomic_read(&fscache_n_op_pend),
- atomic_read(&fscache_n_op_run),
- atomic_read(&fscache_n_op_enqueue),
- atomic_read(&fscache_n_op_cancelled),
- atomic_read(&fscache_n_op_rejected));
+ atomic_read_unchecked(&fscache_n_op_pend),
+ atomic_read_unchecked(&fscache_n_op_run),
+ atomic_read_unchecked(&fscache_n_op_enqueue),
+ atomic_read_unchecked(&fscache_n_op_cancelled),
+ atomic_read_unchecked(&fscache_n_op_rejected));
seq_printf(m, "Ops : dfr=%u rel=%u gc=%u\n",
- atomic_read(&fscache_n_op_deferred_release),
- atomic_read(&fscache_n_op_release),
- atomic_read(&fscache_n_op_gc));
+ atomic_read_unchecked(&fscache_n_op_deferred_release),
+ atomic_read_unchecked(&fscache_n_op_release),
+ atomic_read_unchecked(&fscache_n_op_gc));
seq_printf(m, "CacheOp: alo=%d luo=%d luc=%d gro=%d\n",
atomic_read(&fscache_n_cop_alloc_object),
INIT_LIST_HEAD(&cuse_conntbl[i]);
/* inherit and extend fuse_dev_operations */
- cuse_channel_fops = fuse_dev_operations;
- cuse_channel_fops.owner = THIS_MODULE;
- cuse_channel_fops.open = cuse_channel_open;
- cuse_channel_fops.release = cuse_channel_release;
+ pax_open_kernel();
+ memcpy((void *)&cuse_channel_fops, &fuse_dev_operations, sizeof(fuse_dev_operations));
+ *(void **)&cuse_channel_fops.owner = THIS_MODULE;
+ *(void **)&cuse_channel_fops.open = cuse_channel_open;
+ *(void **)&cuse_channel_fops.release = cuse_channel_release;
+ pax_close_kernel();
cuse_class = class_create(THIS_MODULE, "cuse");
if (IS_ERR(cuse_class))
ret = 0;
pipe_lock(pipe);
- if (!pipe->readers) {
+ if (!atomic_read(&pipe->readers)) {
send_sig(SIGPIPE, current, 0);
if (!ret)
ret = -EPIPE;
page_nr++;
ret += buf->len;
- if (pipe->files)
+ if (atomic_read(&pipe->files))
do_wakeup = 1;
}
return link;
}
-static void free_link(char *link)
+static void free_link(const char *link)
{
if (!IS_ERR(link))
free_page((unsigned long) link);
static void hostfs_put_link(struct dentry *dentry, struct nameidata *nd, void *cookie)
{
- char *s = nd_get_link(nd);
+ const char *s = nd_get_link(nd);
if (!IS_ERR(s))
__putname(s);
}
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
struct hstate *h = hstate_file(file);
+ unsigned long offset = gr_rand_threadstack_offset(mm, file, flags);
struct vm_unmapped_area_info info;
if (len & ~huge_page_mask(h))
return addr;
}
+#ifdef CONFIG_PAX_RANDMMAP
+ if (!(mm->pax_flags & MF_PAX_RANDMMAP))
+#endif
+
if (addr) {
addr = ALIGN(addr, huge_page_size(h));
vma = find_vma(mm, addr);
- if (TASK_SIZE - len >= addr &&
- (!vma || addr + len <= vma->vm_start))
+ if (TASK_SIZE - len >= addr && check_heap_stack_gap(vma, addr, len, offset))
return addr;
}
info.flags = 0;
info.length = len;
info.low_limit = TASK_UNMAPPED_BASE;
+
+#ifdef CONFIG_PAX_RANDMMAP
+ if (mm->pax_flags & MF_PAX_RANDMMAP)
+ info.low_limit += mm->delta_mmap;
+#endif
+
info.high_limit = TASK_SIZE;
info.align_mask = PAGE_MASK & ~huge_page_mask(h);
info.align_offset = 0;
};
MODULE_ALIAS_FS("hugetlbfs");
-static struct vfsmount *hugetlbfs_vfsmount[HUGE_MAX_HSTATE];
+struct vfsmount *hugetlbfs_vfsmount[HUGE_MAX_HSTATE];
static int can_do_hugetlb_shm(void)
{
unsigned int *p = &get_cpu_var(last_ino);
unsigned int res = *p;
+start:
+
#ifdef CONFIG_SMP
if (unlikely((res & (LAST_INO_BATCH-1)) == 0)) {
- static atomic_t shared_last_ino;
- int next = atomic_add_return(LAST_INO_BATCH, &shared_last_ino);
+ static atomic_unchecked_t shared_last_ino;
+ int next = atomic_add_return_unchecked(LAST_INO_BATCH, &shared_last_ino);
res = next - LAST_INO_BATCH;
}
#endif
- *p = ++res;
+ if (unlikely(!++res))
+ goto start; /* never zero */
+ *p = res;
put_cpu_var(last_ino);
return res;
}
struct jffs2_unknown_node marker = {
.magic = cpu_to_je16(JFFS2_MAGIC_BITMASK),
.nodetype = cpu_to_je16(JFFS2_NODETYPE_CLEANMARKER),
- .totlen = cpu_to_je32(c->cleanmarker_size)
+ .totlen = cpu_to_je32(c->cleanmarker_size),
+ .hdr_crc = cpu_to_je32(0)
};
jffs2_prealloc_raw_node_refs(c, jeb, 1);
{
.magic = constant_cpu_to_je16(JFFS2_MAGIC_BITMASK),
.nodetype = constant_cpu_to_je16(JFFS2_NODETYPE_CLEANMARKER),
- .totlen = constant_cpu_to_je32(8)
+ .totlen = constant_cpu_to_je32(8),
+ .hdr_crc = constant_cpu_to_je32(0)
};
/*
jfs_inode_cachep =
kmem_cache_create("jfs_ip", sizeof(struct jfs_inode_info), 0,
- SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD,
+ SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD|SLAB_USERCOPY,
init_once);
if (jfs_inode_cachep == NULL)
return -ENOMEM;
*
* Returns 31 bit hash of ns + name (so it fits in an off_t )
*/
-static unsigned int kernfs_name_hash(const char *name, const void *ns)
+static unsigned int kernfs_name_hash(const unsigned char *name, const void *ns)
{
unsigned long hash = init_name_hash();
unsigned int len = strlen(name);
ret = scops->mkdir(parent, dentry->d_name.name, mode);
kernfs_put_active(parent);
+
+ if (!ret) {
+ struct dentry *dentry_ret = kernfs_iop_lookup(dir, dentry, 0);
+ ret = PTR_ERR_OR_ZERO(dentry_ret);
+ }
+
return ret;
}
struct kernfs_open_node {
atomic_t refcnt;
- atomic_t event;
+ atomic_unchecked_t event;
wait_queue_head_t poll;
struct list_head files; /* goes through kernfs_open_file.list */
};
{
struct kernfs_open_file *of = sf->private;
- of->event = atomic_read(&of->kn->attr.open->event);
+ of->event = atomic_read_unchecked(&of->kn->attr.open->event);
return of->kn->attr.ops->seq_show(sf, v);
}
{
struct kernfs_open_file *of = kernfs_of(file);
const struct kernfs_ops *ops;
- size_t len;
+ ssize_t len;
char *buf;
if (of->atomic_write_len) {
return ret;
}
-static int kernfs_vma_access(struct vm_area_struct *vma, unsigned long addr,
- void *buf, int len, int write)
+static ssize_t kernfs_vma_access(struct vm_area_struct *vma, unsigned long addr,
+ void *buf, size_t len, int write)
{
struct file *file = vma->vm_file;
struct kernfs_open_file *of = kernfs_of(file);
- int ret;
+ ssize_t ret;
if (!of->vm_ops)
return -EINVAL;
return -ENOMEM;
atomic_set(&new_on->refcnt, 0);
- atomic_set(&new_on->event, 1);
+ atomic_set_unchecked(&new_on->event, 1);
init_waitqueue_head(&new_on->poll);
INIT_LIST_HEAD(&new_on->files);
goto retry;
kernfs_put_active(kn);
- if (of->event != atomic_read(&on->event))
+ if (of->event != atomic_read_unchecked(&on->event))
goto trigger;
return DEFAULT_POLLMASK;
on = kn->attr.open;
if (on) {
- atomic_inc(&on->event);
+ atomic_inc_unchecked(&on->event);
wake_up_interruptible(&on->poll);
}
static void kernfs_iop_put_link(struct dentry *dentry, struct nameidata *nd,
void *cookie)
{
- char *page = nd_get_link(nd);
+ const char *page = nd_get_link(nd);
if (!IS_ERR(page))
free_page((unsigned long)page);
}
for (p = q->next; p != &dentry->d_subdirs; p = p->next) {
struct dentry *next = list_entry(p, struct dentry, d_child);
+ char d_name[sizeof(next->d_iname)];
+ const unsigned char *name;
+
spin_lock_nested(&next->d_lock, DENTRY_D_LOCK_NESTED);
if (!simple_positive(next)) {
spin_unlock(&next->d_lock);
spin_unlock(&next->d_lock);
spin_unlock(&dentry->d_lock);
- if (!dir_emit(ctx, next->d_name.name, next->d_name.len,
+ name = next->d_name.name;
+ if (name == next->d_iname) {
+ memcpy(d_name, name, next->d_name.len);
+ name = d_name;
+ }
+ if (!dir_emit(ctx, name, next->d_name.len,
next->d_inode->i_ino, dt_type(next->d_inode)))
return 0;
spin_lock(&dentry->d_lock);
void kfree_put_link(struct dentry *dentry, struct nameidata *nd,
void *cookie)
{
- char *s = nd_get_link(nd);
+ const char *s = nd_get_link(nd);
if (!IS_ERR(s))
kfree(s);
}
/*
* Cookie counter for NLM requests
*/
-static atomic_t nlm_cookie = ATOMIC_INIT(0x1234);
+static atomic_unchecked_t nlm_cookie = ATOMIC_INIT(0x1234);
void nlmclnt_next_cookie(struct nlm_cookie *c)
{
- u32 cookie = atomic_inc_return(&nlm_cookie);
+ u32 cookie = atomic_inc_return_unchecked(&nlm_cookie);
memcpy(c->data, &cookie, 4);
c->len=4;
locks_remove_posix(filp, filp);
if (filp->f_op->flock) {
- struct file_lock fl = {
+ struct file_lock flock = {
.fl_owner = filp,
.fl_pid = current->tgid,
.fl_file = filp,
.fl_type = F_UNLCK,
.fl_end = OFFSET_MAX,
};
- filp->f_op->flock(filp, F_SETLKW, &fl);
- if (fl.fl_ops && fl.fl_ops->fl_release_private)
- fl.fl_ops->fl_release_private(&fl);
+ filp->f_op->flock(filp, F_SETLKW, &flock);
+ if (flock.fl_ops && flock.fl_ops->fl_release_private)
+ flock.fl_ops->fl_release_private(&flock);
}
spin_lock(&inode->i_lock);
u64 seq; /* Sequence number to prevent loops */
wait_queue_head_t poll;
u64 event;
-};
+} __randomize_layout;
struct mnt_pcp {
int mnt_count;
int mnt_expiry_mark; /* true if marked for expiry */
struct hlist_head mnt_pins;
struct path mnt_ex_mountpoint;
-};
+} __randomize_layout;
#define MNT_NS_INTERNAL ERR_PTR(-EINVAL) /* distinct from any mnt_namespace */
if (ret != -EACCES)
return ret;
+#ifdef CONFIG_GRKERNSEC
+ /* we'll block if we have to log due to a denied capability use */
+ if (mask & MAY_NOT_BLOCK)
+ return -ECHILD;
+#endif
+
if (S_ISDIR(inode->i_mode)) {
/* DACs are overridable for directories */
- if (capable_wrt_inode_uidgid(inode, CAP_DAC_OVERRIDE))
- return 0;
if (!(mask & MAY_WRITE))
- if (capable_wrt_inode_uidgid(inode,
- CAP_DAC_READ_SEARCH))
+ if (capable_wrt_inode_uidgid_nolog(inode, CAP_DAC_OVERRIDE) ||
+ capable_wrt_inode_uidgid(inode, CAP_DAC_READ_SEARCH))
return 0;
+ if (capable_wrt_inode_uidgid(inode, CAP_DAC_OVERRIDE))
+ return 0;
return -EACCES;
}
+ /*
+ * Searching includes executable on directories, else just read.
+ */
+ mask &= MAY_READ | MAY_WRITE | MAY_EXEC;
+ if (mask == MAY_READ)
+ if (capable_wrt_inode_uidgid_nolog(inode, CAP_DAC_OVERRIDE) ||
+ capable_wrt_inode_uidgid(inode, CAP_DAC_READ_SEARCH))
+ return 0;
+
/*
* Read/write DACs are always overridable.
* Executable DACs are overridable when there is
if (capable_wrt_inode_uidgid(inode, CAP_DAC_OVERRIDE))
return 0;
- /*
- * Searching includes executable on directories, else just read.
- */
- mask &= MAY_READ | MAY_WRITE | MAY_EXEC;
- if (mask == MAY_READ)
- if (capable_wrt_inode_uidgid(inode, CAP_DAC_READ_SEARCH))
- return 0;
-
return -EACCES;
}
EXPORT_SYMBOL(generic_permission);
int last_type;
unsigned depth;
struct file *base;
- char *saved_names[MAX_NESTED_LINKS + 1];
+ const char *saved_names[MAX_NESTED_LINKS + 1];
};
/*
nd->flags |= LOOKUP_JUMPED;
}
-void nd_set_link(struct nameidata *nd, char *path)
+void nd_set_link(struct nameidata *nd, const char *path)
{
nd->saved_names[nd->depth] = path;
}
EXPORT_SYMBOL(nd_set_link);
-char *nd_get_link(struct nameidata *nd)
+const char *nd_get_link(const struct nameidata *nd)
{
return nd->saved_names[nd->depth];
}
{
struct dentry *dentry = link->dentry;
int error;
- char *s;
+ const char *s;
BUG_ON(nd->flags & LOOKUP_RCU);
if (error)
goto out_put_nd_path;
+ if (gr_handle_follow_link(dentry->d_parent->d_inode,
+ dentry->d_inode, dentry, nd->path.mnt)) {
+ error = -EACCES;
+ goto out_put_nd_path;
+ }
+
nd->last_type = LAST_BIND;
*p = dentry->d_inode->i_op->follow_link(dentry, nd);
error = PTR_ERR(*p);
if (res)
break;
res = walk_component(nd, path, LOOKUP_FOLLOW);
+ if (res >= 0 && gr_handle_symlink_owner(&link, nd->inode))
+ res = -EACCES;
put_link(nd, &link, cookie);
} while (res > 0);
static inline u64 hash_name(const char *name)
{
unsigned long a, b, adata, bdata, mask, hash, len;
- const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS;
+ static const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS;
hash = a = 0;
len = -sizeof(unsigned long);
if (err)
break;
err = lookup_last(nd, &path);
+ if (!err && gr_handle_symlink_owner(&link, nd->inode))
+ err = -EACCES;
put_link(nd, &link, cookie);
}
}
if (!err)
err = complete_walk(nd);
+ if (!err && !(nd->flags & LOOKUP_PARENT)) {
+ if (!gr_acl_handle_hidden_file(nd->path.dentry, nd->path.mnt)) {
+ path_put(&nd->path);
+ err = -ENOENT;
+ }
+ }
+
if (!err && nd->flags & LOOKUP_DIRECTORY) {
if (!d_can_lookup(nd->path.dentry)) {
path_put(&nd->path);
retval = path_lookupat(dfd, name->name,
flags | LOOKUP_REVAL, nd);
- if (likely(!retval))
+ if (likely(!retval)) {
audit_inode(name, nd->path.dentry, flags & LOOKUP_PARENT);
+ if (name->name[0] != '/' && nd->path.dentry && nd->inode) {
+ if (!gr_chroot_fchdir(nd->path.dentry, nd->path.mnt)) {
+ path_put(&nd->path);
+ return -ENOENT;
+ }
+ }
+ }
return retval;
}
if (flag & O_NOATIME && !inode_owner_or_capable(inode))
return -EPERM;
+ if (gr_handle_rofs_blockwrite(dentry, path->mnt, acc_mode))
+ return -EPERM;
+ if (gr_handle_rawio(inode))
+ return -EPERM;
+ if (!gr_acl_handle_open(dentry, path->mnt, acc_mode))
+ return -EACCES;
+
return 0;
}
* cleared otherwise prior to returning.
*/
static int lookup_open(struct nameidata *nd, struct path *path,
- struct file *file,
+ struct path *link, struct file *file,
const struct open_flags *op,
bool got_write, int *opened)
{
/* Negative dentry, just create the file */
if (!dentry->d_inode && (op->open_flag & O_CREAT)) {
umode_t mode = op->mode;
+
+ if (link && gr_handle_symlink_owner(link, dir->d_inode)) {
+ error = -EACCES;
+ goto out_dput;
+ }
+
+ if (!gr_acl_handle_creat(dentry, dir, nd->path.mnt, op->open_flag, op->acc_mode, mode)) {
+ error = -EACCES;
+ goto out_dput;
+ }
+
if (!IS_POSIXACL(dir->d_inode))
mode &= ~current_umask();
/*
nd->flags & LOOKUP_EXCL);
if (error)
goto out_dput;
+ else
+ gr_handle_create(dentry, nd->path.mnt);
}
out_no_open:
path->dentry = dentry;
/*
* Handle the last step of open()
*/
-static int do_last(struct nameidata *nd, struct path *path,
+static int do_last(struct nameidata *nd, struct path *path, struct path *link,
struct file *file, const struct open_flags *op,
int *opened, struct filename *name)
{
if (error)
return error;
+ if (!gr_acl_handle_hidden_file(dir, nd->path.mnt)) {
+ error = -ENOENT;
+ goto out;
+ }
+ if (link && gr_handle_symlink_owner(link, nd->inode)) {
+ error = -EACCES;
+ goto out;
+ }
+
audit_inode(name, dir, LOOKUP_PARENT);
error = -EISDIR;
/* trailing slashes? */
*/
}
mutex_lock(&dir->d_inode->i_mutex);
- error = lookup_open(nd, path, file, op, got_write, opened);
+ error = lookup_open(nd, path, link, file, op, got_write, opened);
mutex_unlock(&dir->d_inode->i_mutex);
if (error <= 0) {
goto finish_open_created;
}
+ if (!gr_acl_handle_hidden_file(path->dentry, nd->path.mnt)) {
+ error = -ENOENT;
+ goto exit_dput;
+ }
+ if (link && gr_handle_symlink_owner(link, path->dentry->d_inode)) {
+ error = -EACCES;
+ goto exit_dput;
+ }
+
/*
* create/update audit record if it already exists.
*/
- if (d_is_positive(path->dentry))
+ if (d_is_positive(path->dentry)) {
+ /* only check if O_CREAT is specified, all other checks need to go
+ into may_open */
+ if (gr_handle_fifo(path->dentry, path->mnt, dir, open_flag, acc_mode)) {
+ error = -EACCES;
+ goto exit_dput;
+ }
+
audit_inode(name, path->dentry, 0);
+ }
/*
* If atomic_open() acquired write access it is dropped now due to
}
}
BUG_ON(inode != path->dentry->d_inode);
+ /* if we're resolving a symlink to another symlink */
+ if (link && gr_handle_symlink_owner(link, inode)) {
+ error = -EACCES;
+ goto out;
+ }
return 1;
}
path_put(&save_parent);
return error;
}
+
+ if (!gr_acl_handle_hidden_file(nd->path.dentry, nd->path.mnt)) {
+ error = -ENOENT;
+ goto out;
+ }
+ if (link && gr_handle_symlink_owner(link, nd->inode)) {
+ error = -EACCES;
+ goto out;
+ }
+
audit_inode(name, nd->path.dentry, 0);
+
error = -EISDIR;
if ((open_flag & O_CREAT) && d_is_dir(nd->path.dentry))
goto out;
if (unlikely(error))
goto out;
- error = do_last(nd, &path, file, op, &opened, pathname);
+ error = do_last(nd, &path, NULL, file, op, &opened, pathname);
while (unlikely(error > 0)) { /* trailing symlink */
struct path link = path;
void *cookie;
error = follow_link(&link, nd, &cookie);
if (unlikely(error))
break;
- error = do_last(nd, &path, file, op, &opened, pathname);
+ error = do_last(nd, &path, &link, file, op, &opened, pathname);
put_link(nd, &link, cookie);
}
out:
goto unlock;
error = -EEXIST;
- if (d_is_positive(dentry))
+ if (d_is_positive(dentry)) {
+ if (!gr_acl_handle_hidden_file(dentry, nd.path.mnt))
+ error = -ENOENT;
goto fail;
-
+ }
/*
* Special case - lookup gave negative, but... we had foo/bar/
* From the vfs_mknod() POV we just have a negative dentry -
}
EXPORT_SYMBOL(user_path_create);
+static struct dentry *user_path_create_with_name(int dfd, const char __user *pathname, struct path *path, struct filename **to, unsigned int lookup_flags)
+{
+ struct filename *tmp = getname(pathname);
+ struct dentry *res;
+ if (IS_ERR(tmp))
+ return ERR_CAST(tmp);
+ res = kern_path_create(dfd, tmp->name, path, lookup_flags);
+ if (IS_ERR(res))
+ putname(tmp);
+ else
+ *to = tmp;
+ return res;
+}
+
int vfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev)
{
int error = may_create(dir, dentry);
if (!IS_POSIXACL(path.dentry->d_inode))
mode &= ~current_umask();
+
+ if (gr_handle_chroot_mknod(dentry, path.mnt, mode)) {
+ error = -EPERM;
+ goto out;
+ }
+
+ if (!gr_acl_handle_mknod(dentry, path.dentry, path.mnt, mode)) {
+ error = -EACCES;
+ goto out;
+ }
+
error = security_path_mknod(&path, dentry, mode, dev);
if (error)
goto out;
error = vfs_mknod(path.dentry->d_inode,dentry,mode,0);
break;
}
+ if (!error)
+ gr_handle_create(dentry, path.mnt);
out:
done_path_create(&path, dentry);
if (retry_estale(error, lookup_flags)) {
if (!IS_POSIXACL(path.dentry->d_inode))
mode &= ~current_umask();
+ if (!gr_acl_handle_mkdir(dentry, path.dentry, path.mnt)) {
+ error = -EACCES;
+ goto out;
+ }
error = security_path_mkdir(&path, dentry, mode);
if (!error)
error = vfs_mkdir(path.dentry->d_inode, dentry, mode);
+ if (!error)
+ gr_handle_create(dentry, path.mnt);
+out:
done_path_create(&path, dentry);
if (retry_estale(error, lookup_flags)) {
lookup_flags |= LOOKUP_REVAL;
struct filename *name;
struct dentry *dentry;
struct nameidata nd;
+ u64 saved_ino = 0;
+ dev_t saved_dev = 0;
unsigned int lookup_flags = 0;
retry:
name = user_path_parent(dfd, pathname, &nd, lookup_flags);
error = -ENOENT;
goto exit3;
}
+
+ saved_ino = gr_get_ino_from_dentry(dentry);
+ saved_dev = gr_get_dev_from_dentry(dentry);
+
+ if (!gr_acl_handle_rmdir(dentry, nd.path.mnt)) {
+ error = -EACCES;
+ goto exit3;
+ }
+
error = security_path_rmdir(&nd.path, dentry);
if (error)
goto exit3;
error = vfs_rmdir(nd.path.dentry->d_inode, dentry);
+ if (!error && (saved_dev || saved_ino))
+ gr_handle_delete(saved_ino, saved_dev);
exit3:
dput(dentry);
exit2:
struct nameidata nd;
struct inode *inode = NULL;
struct inode *delegated_inode = NULL;
+ u64 saved_ino = 0;
+ dev_t saved_dev = 0;
unsigned int lookup_flags = 0;
retry:
name = user_path_parent(dfd, pathname, &nd, lookup_flags);
if (d_is_negative(dentry))
goto slashes;
ihold(inode);
+
+ if (inode->i_nlink <= 1) {
+ saved_ino = gr_get_ino_from_dentry(dentry);
+ saved_dev = gr_get_dev_from_dentry(dentry);
+ }
+ if (!gr_acl_handle_unlink(dentry, nd.path.mnt)) {
+ error = -EACCES;
+ goto exit2;
+ }
+
error = security_path_unlink(&nd.path, dentry);
if (error)
goto exit2;
error = vfs_unlink(nd.path.dentry->d_inode, dentry, &delegated_inode);
+ if (!error && (saved_ino || saved_dev))
+ gr_handle_delete(saved_ino, saved_dev);
exit2:
dput(dentry);
}
if (IS_ERR(dentry))
goto out_putname;
+ if (!gr_acl_handle_symlink(dentry, path.dentry, path.mnt, from)) {
+ error = -EACCES;
+ goto out;
+ }
+
error = security_path_symlink(&path, dentry, from->name);
if (!error)
error = vfs_symlink(path.dentry->d_inode, dentry, from->name);
+ if (!error)
+ gr_handle_create(dentry, path.mnt);
+out:
done_path_create(&path, dentry);
if (retry_estale(error, lookup_flags)) {
lookup_flags |= LOOKUP_REVAL;
struct dentry *new_dentry;
struct path old_path, new_path;
struct inode *delegated_inode = NULL;
+ struct filename *to = NULL;
int how = 0;
int error;
if (error)
return error;
- new_dentry = user_path_create(newdfd, newname, &new_path,
+ new_dentry = user_path_create_with_name(newdfd, newname, &new_path, &to,
(how & LOOKUP_REVAL));
error = PTR_ERR(new_dentry);
if (IS_ERR(new_dentry))
error = may_linkat(&old_path);
if (unlikely(error))
goto out_dput;
+
+ if (gr_handle_hardlink(old_path.dentry, old_path.mnt,
+ old_path.dentry->d_inode,
+ old_path.dentry->d_inode->i_mode, to)) {
+ error = -EACCES;
+ goto out_dput;
+ }
+
+ if (!gr_acl_handle_link(new_dentry, new_path.dentry, new_path.mnt,
+ old_path.dentry, old_path.mnt, to)) {
+ error = -EACCES;
+ goto out_dput;
+ }
+
error = security_path_link(old_path.dentry, &new_path, new_dentry);
if (error)
goto out_dput;
error = vfs_link(old_path.dentry, new_path.dentry->d_inode, new_dentry, &delegated_inode);
+ if (!error)
+ gr_handle_create(new_dentry, new_path.mnt);
out_dput:
+ putname(to);
done_path_create(&new_path, new_dentry);
if (delegated_inode) {
error = break_deleg_wait(&delegated_inode);
if (new_dentry == trap)
goto exit5;
+ if (gr_bad_chroot_rename(old_dentry, oldnd.path.mnt, new_dentry, newnd.path.mnt)) {
+ /* use EXDEV error to cause 'mv' to switch to an alternative
+ * method for usability
+ */
+ error = -EXDEV;
+ goto exit5;
+ }
+
+ error = gr_acl_handle_rename(new_dentry, new_dir, newnd.path.mnt,
+ old_dentry, old_dir->d_inode, oldnd.path.mnt,
+ to, flags);
+ if (error)
+ goto exit5;
+
error = security_path_rename(&oldnd.path, old_dentry,
&newnd.path, new_dentry, flags);
if (error)
error = vfs_rename(old_dir->d_inode, old_dentry,
new_dir->d_inode, new_dentry,
&delegated_inode, flags);
+ if (!error)
+ gr_handle_rename(old_dir->d_inode, new_dir->d_inode, old_dentry,
+ new_dentry, oldnd.path.mnt, new_dentry->d_inode ? 1 : 0, flags);
exit5:
dput(new_dentry);
exit4:
int readlink_copy(char __user *buffer, int buflen, const char *link)
{
+ char tmpbuf[64];
+ const char *newlink;
int len = PTR_ERR(link);
+
if (IS_ERR(link))
goto out;
len = strlen(link);
if (len > (unsigned) buflen)
len = buflen;
- if (copy_to_user(buffer, link, len))
+
+ if (len < sizeof(tmpbuf)) {
+ memcpy(tmpbuf, link, len);
+ newlink = tmpbuf;
+ } else
+ newlink = link;
+
+ if (copy_to_user(buffer, newlink, len))
len = -EFAULT;
out:
return len;
if (!(sb->s_flags & MS_RDONLY))
retval = do_remount_sb(sb, MS_RDONLY, NULL, 0);
up_write(&sb->s_umount);
+
+ gr_log_remount(mnt->mnt_devname, retval);
+
return retval;
}
}
unlock_mount_hash();
namespace_unlock();
+
+ gr_log_unmount(mnt->mnt_devname, retval);
+
return retval;
}
* unixes. Our API is identical to OSF/1 to avoid making a mess of AMD
*/
-SYSCALL_DEFINE2(umount, char __user *, name, int, flags)
+SYSCALL_DEFINE2(umount, const char __user *, name, int, flags)
{
struct path path;
struct mount *mnt;
/*
* The 2.0 compatible umount. No flags.
*/
-SYSCALL_DEFINE1(oldumount, char __user *, name)
+SYSCALL_DEFINE1(oldumount, const char __user *, name)
{
return sys_umount(name, 0);
}
MS_NOATIME | MS_NODIRATIME | MS_RELATIME| MS_KERNMOUNT |
MS_STRICTATIME);
+ if (gr_handle_rofs_mount(path.dentry, path.mnt, mnt_flags)) {
+ retval = -EPERM;
+ goto dput_out;
+ }
+
+ if (gr_handle_chroot_mount(path.dentry, path.mnt, dev_name)) {
+ retval = -EPERM;
+ goto dput_out;
+ }
+
if (flags & MS_REMOUNT)
retval = do_remount(&path, flags & ~MS_REMOUNT, mnt_flags,
data_page);
retval = do_new_mount(&path, type_page, flags, mnt_flags,
dev_name, data_page);
dput_out:
+ gr_log_mount(dev_name, &path, retval);
+
path_put(&path);
+
return retval;
}
* number incrementing at 10Ghz will take 12,427 years to wrap which
* is effectively never, so we can ignore the possibility.
*/
-static atomic64_t mnt_ns_seq = ATOMIC64_INIT(1);
+static atomic64_unchecked_t mnt_ns_seq = ATOMIC64_INIT(1);
static struct mnt_namespace *alloc_mnt_ns(struct user_namespace *user_ns)
{
return ERR_PTR(ret);
}
new_ns->ns.ops = &mntns_operations;
- new_ns->seq = atomic64_add_return(1, &mnt_ns_seq);
+ new_ns->seq = atomic64_add_return_unchecked(1, &mnt_ns_seq);
atomic_set(&new_ns->count, 1);
new_ns->root = NULL;
INIT_LIST_HEAD(&new_ns->list);
return new_ns;
}
-struct mnt_namespace *copy_mnt_ns(unsigned long flags, struct mnt_namespace *ns,
+__latent_entropy struct mnt_namespace *copy_mnt_ns(unsigned long flags, struct mnt_namespace *ns,
struct user_namespace *user_ns, struct fs_struct *new_fs)
{
struct mnt_namespace *new_ns;
}
EXPORT_SYMBOL(mount_subtree);
-SYSCALL_DEFINE5(mount, char __user *, dev_name, char __user *, dir_name,
- char __user *, type, unsigned long, flags, void __user *, data)
+SYSCALL_DEFINE5(mount, const char __user *, dev_name, const char __user *, dir_name,
+ const char __user *, type, unsigned long, flags, void __user *, data)
{
int ret;
char *kernel_type;
if (error)
goto out2;
+ if (gr_handle_chroot_pivot()) {
+ error = -EPERM;
+ goto out2;
+ }
+
get_fs_root(current->fs, &root);
old_mp = lock_mount(&old);
error = PTR_ERR(old_mp);
!ns_capable(current_user_ns(), CAP_SYS_ADMIN))
return -EPERM;
- if (fs->users != 1)
+ if (atomic_read(&fs->users) != 1)
return -EINVAL;
get_mnt_ns(mnt_ns);
callback_decode_arg_t decode_args;
callback_encode_res_t encode_res;
long res_maxsize;
-};
+} __do_const;
static struct callback_op callback_ops[];
return nfs_size_to_loff_t(fattr->size) > i_size_read(inode);
}
-static atomic_long_t nfs_attr_generation_counter;
+static atomic_long_unchecked_t nfs_attr_generation_counter;
static unsigned long nfs_read_attr_generation_counter(void)
{
- return atomic_long_read(&nfs_attr_generation_counter);
+ return atomic_long_read_unchecked(&nfs_attr_generation_counter);
}
unsigned long nfs_inc_attr_generation_counter(void)
{
- return atomic_long_inc_return(&nfs_attr_generation_counter);
+ return atomic_long_inc_return_unchecked(&nfs_attr_generation_counter);
}
void nfs_fattr_init(struct nfs_fattr *fattr)
nfsd4op_rsize op_rsize_bop;
stateid_getter op_get_currentstateid;
stateid_setter op_set_currentstateid;
-};
+} __do_const;
static struct nfsd4_operation nfsd4_ops[];
typedef __be32(*nfsd4_dec)(struct nfsd4_compoundargs *argp, void *);
-static nfsd4_dec nfsd4_dec_ops[] = {
+static const nfsd4_dec nfsd4_dec_ops[] = {
[OP_ACCESS] = (nfsd4_dec)nfsd4_decode_access,
[OP_CLOSE] = (nfsd4_dec)nfsd4_decode_close,
[OP_COMMIT] = (nfsd4_dec)nfsd4_decode_commit,
struct kvec *resv = &rqstp->rq_res.head[0], *cachv;
u32 hash;
struct nfsd_drc_bucket *b;
- int len;
+ long len;
size_t bufsize = 0;
if (!rp)
hash = nfsd_cache_hash(rp->c_xid);
b = &drc_hashtbl[hash];
- len = resv->iov_len - ((char*)statp - (char*)resv->iov_base);
- len >>= 2;
+ if (statp) {
+ len = (char*)statp - (char*)resv->iov_base;
+ len = resv->iov_len - len;
+ len >>= 2;
+ }
/* Don't cache excessive amounts of data and XDR failures */
- if (!statp || len > (256 >> 2)) {
+ if (!statp || len > (256 >> 2) || len < 0) {
nfsd_reply_cache_free(b, rp);
return;
}
switch (cachetype) {
case RC_REPLSTAT:
if (len != 1)
- printk("nfsd: RC_REPLSTAT/reply len %d!\n",len);
+ printk("nfsd: RC_REPLSTAT/reply len %ld!\n",len);
rp->c_replstat = *statp;
break;
case RC_REPLBUFF:
oldfs = get_fs();
set_fs(KERNEL_DS);
- host_err = vfs_readv(file, (struct iovec __user *)vec, vlen, &offset);
+ host_err = vfs_readv(file, (struct iovec __force_user *)vec, vlen, &offset);
set_fs(oldfs);
return nfsd_finish_read(file, count, host_err);
}
/* Write the data. */
oldfs = get_fs(); set_fs(KERNEL_DS);
- host_err = vfs_writev(file, (struct iovec __user *)vec, vlen, &pos);
+ host_err = vfs_writev(file, (struct iovec __force_user *)vec, vlen, &pos);
set_fs(oldfs);
if (host_err < 0)
goto out_nfserr;
*/
oldfs = get_fs(); set_fs(KERNEL_DS);
- host_err = inode->i_op->readlink(path.dentry, (char __user *)buf, *lenp);
+ host_err = inode->i_op->readlink(path.dentry, (char __force_user *)buf, *lenp);
set_fs(oldfs);
if (host_err < 0)
int __register_nls(struct nls_table *nls, struct module *owner)
{
- struct nls_table ** tmp = &tables;
+ struct nls_table *tmp = tables;
if (nls->next)
return -EBUSY;
- nls->owner = owner;
+ pax_open_kernel();
+ *(void **)&nls->owner = owner;
+ pax_close_kernel();
spin_lock(&nls_lock);
- while (*tmp) {
- if (nls == *tmp) {
+ while (tmp) {
+ if (nls == tmp) {
spin_unlock(&nls_lock);
return -EBUSY;
}
- tmp = &(*tmp)->next;
+ tmp = tmp->next;
}
- nls->next = tables;
+ pax_open_kernel();
+ *(struct nls_table **)&nls->next = tables;
+ pax_close_kernel();
tables = nls;
spin_unlock(&nls_lock);
return 0;
int unregister_nls(struct nls_table * nls)
{
- struct nls_table ** tmp = &tables;
+ struct nls_table * const * tmp = &tables;
spin_lock(&nls_lock);
while (*tmp) {
if (nls == *tmp) {
- *tmp = nls->next;
+ pax_open_kernel();
+ *(struct nls_table **)tmp = nls->next;
+ pax_close_kernel();
spin_unlock(&nls_lock);
return 0;
}
return -EINVAL;
}
-static struct nls_table *find_nls(char *charset)
+static struct nls_table *find_nls(const char *charset)
{
struct nls_table *nls;
spin_lock(&nls_lock);
return nls;
}
-struct nls_table *load_nls(char *charset)
+struct nls_table *load_nls(const char *charset)
{
return try_then_request_module(find_nls(charset), "nls_%s", charset);
}
p_nls = load_nls("cp932");
if (p_nls) {
- table.charset2upper = p_nls->charset2upper;
- table.charset2lower = p_nls->charset2lower;
+ pax_open_kernel();
+ *(const unsigned char **)&table.charset2upper = p_nls->charset2upper;
+ *(const unsigned char **)&table.charset2lower = p_nls->charset2lower;
+ pax_close_kernel();
return register_nls(&table);
}
p_nls = load_nls("koi8-u");
if (p_nls) {
- table.charset2upper = p_nls->charset2upper;
- table.charset2lower = p_nls->charset2lower;
+ pax_open_kernel();
+ *(const unsigned char **)&table.charset2upper = p_nls->charset2upper;
+ *(const unsigned char **)&table.charset2lower = p_nls->charset2lower;
+ pax_close_kernel();
return register_nls(&table);
}
fd = fanotify_event_metadata.fd;
ret = -EFAULT;
- if (copy_to_user(buf, &fanotify_event_metadata,
- fanotify_event_metadata.event_len))
+ if (fanotify_event_metadata.event_len > sizeof fanotify_event_metadata ||
+ copy_to_user(buf, &fanotify_event_metadata, fanotify_event_metadata.event_len))
goto out_close_fd;
#ifdef CONFIG_FANOTIFY_ACCESS_PERMISSIONS
#include <linux/fsnotify_backend.h>
#include "fsnotify.h"
-static atomic_t fsnotify_sync_cookie = ATOMIC_INIT(0);
+static atomic_unchecked_t fsnotify_sync_cookie = ATOMIC_INIT(0);
/**
* fsnotify_get_cookie - return a unique cookie for use in synchronizing events.
*/
u32 fsnotify_get_cookie(void)
{
- return atomic_inc_return(&fsnotify_sync_cookie);
+ return atomic_inc_return_unchecked(&fsnotify_sync_cookie);
}
EXPORT_SYMBOL_GPL(fsnotify_get_cookie);
ia = (INDEX_ALLOCATION*)(kaddr + (ia_pos & ~PAGE_CACHE_MASK &
~(s64)(ndir->itype.index.block_size - 1)));
/* Bounds checks. */
- if (unlikely((u8*)ia < kaddr || (u8*)ia > kaddr + PAGE_CACHE_SIZE)) {
+ if (unlikely(!kaddr || (u8*)ia < kaddr || (u8*)ia > kaddr + PAGE_CACHE_SIZE)) {
ntfs_error(sb, "Out of bounds check failed. Corrupt directory "
"inode 0x%lx or driver bug.", vdir->i_ino);
goto err_out;
char *addr;
size_t total = 0;
unsigned len;
- int left;
+ unsigned left;
do {
len = PAGE_CACHE_SIZE - ofs;
if (!silent)
ntfs_error(sb, "Primary boot sector is invalid.");
} else if (!silent)
- ntfs_error(sb, read_err_str, "primary");
+ ntfs_error(sb, read_err_str, "%s", "primary");
if (!(NTFS_SB(sb)->on_errors & ON_ERRORS_RECOVER)) {
if (bh_primary)
brelse(bh_primary);
goto hotfix_primary_boot_sector;
brelse(bh_backup);
} else if (!silent)
- ntfs_error(sb, read_err_str, "backup");
+ ntfs_error(sb, read_err_str, "%s", "backup");
/* Try to read NT3.51- backup boot sector. */
if ((bh_backup = sb_bread(sb, nr_blocks >> 1))) {
if (is_boot_sector_ntfs(sb, (NTFS_BOOT_SECTOR*)
"sector.");
brelse(bh_backup);
} else if (!silent)
- ntfs_error(sb, read_err_str, "backup");
+ ntfs_error(sb, read_err_str, "%s", "backup");
/* We failed. Cleanup and return. */
if (bh_primary)
brelse(bh_primary);
goto bail;
}
- atomic_inc(&osb->alloc_stats.moves);
+ atomic_inc_unchecked(&osb->alloc_stats.moves);
bail:
if (handle)
struct ocfs2_alloc_stats
{
- atomic_t moves;
- atomic_t local_data;
- atomic_t bitmap_data;
- atomic_t bg_allocs;
- atomic_t bg_extends;
+ atomic_unchecked_t moves;
+ atomic_unchecked_t local_data;
+ atomic_unchecked_t bitmap_data;
+ atomic_unchecked_t bg_allocs;
+ atomic_unchecked_t bg_extends;
};
enum ocfs2_local_alloc_state
mlog_errno(status);
goto bail;
}
- atomic_inc(&osb->alloc_stats.bg_extends);
+ atomic_inc_unchecked(&osb->alloc_stats.bg_extends);
/* You should never ask for this much metadata */
BUG_ON(bits_wanted >
mlog_errno(status);
goto bail;
}
- atomic_inc(&OCFS2_SB(ac->ac_inode->i_sb)->alloc_stats.bg_allocs);
+ atomic_inc_unchecked(&OCFS2_SB(ac->ac_inode->i_sb)->alloc_stats.bg_allocs);
*suballoc_loc = res.sr_bg_blkno;
*suballoc_bit_start = res.sr_bit_offset;
trace_ocfs2_claim_new_inode_at_loc((unsigned long long)di_blkno,
res->sr_bits);
- atomic_inc(&OCFS2_SB(ac->ac_inode->i_sb)->alloc_stats.bg_allocs);
+ atomic_inc_unchecked(&OCFS2_SB(ac->ac_inode->i_sb)->alloc_stats.bg_allocs);
BUG_ON(res->sr_bits != 1);
mlog_errno(status);
goto bail;
}
- atomic_inc(&OCFS2_SB(ac->ac_inode->i_sb)->alloc_stats.bg_allocs);
+ atomic_inc_unchecked(&OCFS2_SB(ac->ac_inode->i_sb)->alloc_stats.bg_allocs);
BUG_ON(res.sr_bits != 1);
cluster_start,
num_clusters);
if (!status)
- atomic_inc(&osb->alloc_stats.local_data);
+ atomic_inc_unchecked(&osb->alloc_stats.local_data);
} else {
if (min_clusters > (osb->bitmap_cpg - 1)) {
/* The only paths asking for contiguousness
ocfs2_desc_bitmap_to_cluster_off(ac->ac_inode,
res.sr_bg_blkno,
res.sr_bit_offset);
- atomic_inc(&osb->alloc_stats.bitmap_data);
+ atomic_inc_unchecked(&osb->alloc_stats.bitmap_data);
*num_clusters = res.sr_bits;
}
}
"%10s => GlobalAllocs: %d LocalAllocs: %d "
"SubAllocs: %d LAWinMoves: %d SAExtends: %d\n",
"Stats",
- atomic_read(&osb->alloc_stats.bitmap_data),
- atomic_read(&osb->alloc_stats.local_data),
- atomic_read(&osb->alloc_stats.bg_allocs),
- atomic_read(&osb->alloc_stats.moves),
- atomic_read(&osb->alloc_stats.bg_extends));
+ atomic_read_unchecked(&osb->alloc_stats.bitmap_data),
+ atomic_read_unchecked(&osb->alloc_stats.local_data),
+ atomic_read_unchecked(&osb->alloc_stats.bg_allocs),
+ atomic_read_unchecked(&osb->alloc_stats.moves),
+ atomic_read_unchecked(&osb->alloc_stats.bg_extends));
out += snprintf(buf + out, len - out,
"%10s => State: %u Descriptor: %llu Size: %u bits "
mutex_init(&osb->system_file_mutex);
- atomic_set(&osb->alloc_stats.moves, 0);
- atomic_set(&osb->alloc_stats.local_data, 0);
- atomic_set(&osb->alloc_stats.bitmap_data, 0);
- atomic_set(&osb->alloc_stats.bg_allocs, 0);
- atomic_set(&osb->alloc_stats.bg_extends, 0);
+ atomic_set_unchecked(&osb->alloc_stats.moves, 0);
+ atomic_set_unchecked(&osb->alloc_stats.local_data, 0);
+ atomic_set_unchecked(&osb->alloc_stats.bitmap_data, 0);
+ atomic_set_unchecked(&osb->alloc_stats.bg_allocs, 0);
+ atomic_set_unchecked(&osb->alloc_stats.bg_extends, 0);
/* Copy the blockcheck stats from the superblock probe */
osb->osb_ecc_stats = *stats;
#include <linux/dnotify.h>
#include <linux/compat.h>
+#define CREATE_TRACE_POINTS
+#include <trace/events/fs.h>
#include "internal.h"
int do_truncate(struct dentry *dentry, loff_t length, unsigned int time_attrs,
error = locks_verify_truncate(inode, NULL, length);
if (!error)
error = security_path_truncate(path);
+ if (!error && !gr_acl_handle_truncate(path->dentry, path->mnt))
+ error = -EACCES;
if (!error)
error = do_truncate(path->dentry, length, 0, NULL);
error = locks_verify_truncate(inode, f.file, length);
if (!error)
error = security_path_truncate(&f.file->f_path);
+ if (!error && !gr_acl_handle_truncate(f.file->f_path.dentry, f.file->f_path.mnt))
+ error = -EACCES;
if (!error)
error = do_truncate(dentry, length, ATTR_MTIME|ATTR_CTIME, f.file);
sb_end_write(inode->i_sb);
if (__mnt_is_readonly(path.mnt))
res = -EROFS;
+ if (!res && !gr_acl_handle_access(path.dentry, path.mnt, mode))
+ res = -EACCES;
+
out_path_release:
path_put(&path);
if (retry_estale(res, lookup_flags)) {
if (error)
goto dput_and_out;
+ gr_log_chdir(path.dentry, path.mnt);
+
set_fs_pwd(current->fs, &path);
dput_and_out:
goto out_putf;
error = inode_permission(inode, MAY_EXEC | MAY_CHDIR);
+
+ if (!error && !gr_chroot_fchdir(f.file->f_path.dentry, f.file->f_path.mnt))
+ error = -EPERM;
+
+ if (!error)
+ gr_log_chdir(f.file->f_path.dentry, f.file->f_path.mnt);
+
if (!error)
set_fs_pwd(current->fs, &f.file->f_path);
out_putf:
if (error)
goto dput_and_out;
+ if (gr_handle_chroot_chroot(path.dentry, path.mnt))
+ goto dput_and_out;
+
set_fs_root(current->fs, &path);
+
+ gr_handle_chroot_chdir(&path);
+
error = 0;
dput_and_out:
path_put(&path);
return error;
retry_deleg:
mutex_lock(&inode->i_mutex);
+
+ if (!gr_acl_handle_chmod(path->dentry, path->mnt, &mode)) {
+ error = -EACCES;
+ goto out_unlock;
+ }
+ if (gr_handle_chroot_chmod(path->dentry, path->mnt, mode)) {
+ error = -EACCES;
+ goto out_unlock;
+ }
+
error = security_path_chmod(path, mode);
if (error)
goto out_unlock;
uid = make_kuid(current_user_ns(), user);
gid = make_kgid(current_user_ns(), group);
+ if (!gr_acl_handle_chown(path->dentry, path->mnt))
+ return -EACCES;
+
newattrs.ia_valid = ATTR_CTIME;
if (user != (uid_t) -1) {
if (!uid_valid(uid))
} else {
fsnotify_open(f);
fd_install(fd, f);
+ trace_do_sys_open(tmp->name, flags, mode);
}
}
putname(tmp);
static void pipe_lock_nested(struct pipe_inode_info *pipe, int subclass)
{
- if (pipe->files)
+ if (atomic_read(&pipe->files))
mutex_lock_nested(&pipe->mutex, subclass);
}
void pipe_unlock(struct pipe_inode_info *pipe)
{
- if (pipe->files)
+ if (atomic_read(&pipe->files))
mutex_unlock(&pipe->mutex);
}
EXPORT_SYMBOL(pipe_unlock);
}
if (bufs) /* More to do? */
continue;
- if (!pipe->writers)
+ if (!atomic_read(&pipe->writers))
break;
- if (!pipe->waiting_writers) {
+ if (!atomic_read(&pipe->waiting_writers)) {
/* syscall merging: Usually we must not sleep
* if O_NONBLOCK is set, or if we got some data.
* But if a writer sleeps in kernel space, then
__pipe_lock(pipe);
- if (!pipe->readers) {
+ if (!atomic_read(&pipe->readers)) {
send_sig(SIGPIPE, current, 0);
ret = -EPIPE;
goto out;
for (;;) {
int bufs;
- if (!pipe->readers) {
+ if (!atomic_read(&pipe->readers)) {
send_sig(SIGPIPE, current, 0);
if (!ret)
ret = -EPIPE;
kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
do_wakeup = 0;
}
- pipe->waiting_writers++;
+ atomic_inc(&pipe->waiting_writers);
pipe_wait(pipe);
- pipe->waiting_writers--;
+ atomic_dec(&pipe->waiting_writers);
}
out:
__pipe_unlock(pipe);
mask = 0;
if (filp->f_mode & FMODE_READ) {
mask = (nrbufs > 0) ? POLLIN | POLLRDNORM : 0;
- if (!pipe->writers && filp->f_version != pipe->w_counter)
+ if (!atomic_read(&pipe->writers) && filp->f_version != pipe->w_counter)
mask |= POLLHUP;
}
* Most Unices do not set POLLERR for FIFOs but on Linux they
* behave exactly like pipes for poll().
*/
- if (!pipe->readers)
+ if (!atomic_read(&pipe->readers))
mask |= POLLERR;
}
int kill = 0;
spin_lock(&inode->i_lock);
- if (!--pipe->files) {
+ if (atomic_dec_and_test(&pipe->files)) {
inode->i_pipe = NULL;
kill = 1;
}
__pipe_lock(pipe);
if (file->f_mode & FMODE_READ)
- pipe->readers--;
+ atomic_dec(&pipe->readers);
if (file->f_mode & FMODE_WRITE)
- pipe->writers--;
+ atomic_dec(&pipe->writers);
- if (pipe->readers || pipe->writers) {
+ if (atomic_read(&pipe->readers) || atomic_read(&pipe->writers)) {
wake_up_interruptible_sync_poll(&pipe->wait, POLLIN | POLLOUT | POLLRDNORM | POLLWRNORM | POLLERR | POLLHUP);
kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
kfree(pipe);
}
-static struct vfsmount *pipe_mnt __read_mostly;
+struct vfsmount *pipe_mnt __read_mostly;
/*
* pipefs_dname() is called from d_path().
goto fail_iput;
inode->i_pipe = pipe;
- pipe->files = 2;
- pipe->readers = pipe->writers = 1;
+ atomic_set(&pipe->files, 2);
+ atomic_set(&pipe->readers, 1);
+ atomic_set(&pipe->writers, 1);
inode->i_fop = &pipefifo_fops;
/*
spin_lock(&inode->i_lock);
if (inode->i_pipe) {
pipe = inode->i_pipe;
- pipe->files++;
+ atomic_inc(&pipe->files);
spin_unlock(&inode->i_lock);
} else {
spin_unlock(&inode->i_lock);
pipe = alloc_pipe_info();
if (!pipe)
return -ENOMEM;
- pipe->files = 1;
+ atomic_set(&pipe->files, 1);
spin_lock(&inode->i_lock);
if (unlikely(inode->i_pipe)) {
- inode->i_pipe->files++;
+ atomic_inc(&inode->i_pipe->files);
spin_unlock(&inode->i_lock);
free_pipe_info(pipe);
pipe = inode->i_pipe;
* opened, even when there is no process writing the FIFO.
*/
pipe->r_counter++;
- if (pipe->readers++ == 0)
+ if (atomic_inc_return(&pipe->readers) == 1)
wake_up_partner(pipe);
- if (!is_pipe && !pipe->writers) {
+ if (!is_pipe && !atomic_read(&pipe->writers)) {
if ((filp->f_flags & O_NONBLOCK)) {
/* suppress POLLHUP until we have
* seen a writer */
* errno=ENXIO when there is no process reading the FIFO.
*/
ret = -ENXIO;
- if (!is_pipe && (filp->f_flags & O_NONBLOCK) && !pipe->readers)
+ if (!is_pipe && (filp->f_flags & O_NONBLOCK) && !atomic_read(&pipe->readers))
goto err;
pipe->w_counter++;
- if (!pipe->writers++)
+ if (atomic_inc_return(&pipe->writers) == 1)
wake_up_partner(pipe);
- if (!is_pipe && !pipe->readers) {
+ if (!is_pipe && !atomic_read(&pipe->readers)) {
if (wait_for_partner(pipe, &pipe->r_counter))
goto err_wr;
}
* the process can at least talk to itself.
*/
- pipe->readers++;
- pipe->writers++;
+ atomic_inc(&pipe->readers);
+ atomic_inc(&pipe->writers);
pipe->r_counter++;
pipe->w_counter++;
- if (pipe->readers == 1 || pipe->writers == 1)
+ if (atomic_read(&pipe->readers) == 1 || atomic_read(&pipe->writers) == 1)
wake_up_partner(pipe);
break;
return 0;
err_rd:
- if (!--pipe->readers)
+ if (atomic_dec_and_test(&pipe->readers))
wake_up_interruptible(&pipe->wait);
ret = -ERESTARTSYS;
goto err;
err_wr:
- if (!--pipe->writers)
+ if (atomic_dec_and_test(&pipe->writers))
wake_up_interruptible(&pipe->wait);
ret = -ERESTARTSYS;
goto err;
#include <linux/xattr.h>
#include <linux/export.h>
#include <linux/user_namespace.h>
+#include <linux/grsecurity.h>
struct posix_acl **acl_by_type(struct inode *inode, int type)
{
}
}
if (mode_p)
- *mode_p = (*mode_p & ~S_IRWXUGO) | mode;
+ *mode_p = ((*mode_p & ~S_IRWXUGO) | mode) & ~gr_acl_umask();
return not_equiv;
}
EXPORT_SYMBOL(posix_acl_equiv_mode);
mode &= (group_obj->e_perm << 3) | ~S_IRWXG;
}
- *mode_p = (*mode_p & ~S_IRWXUGO) | mode;
+ *mode_p = ((*mode_p & ~S_IRWXUGO) | mode) & ~gr_acl_umask();
return not_equiv;
}
struct posix_acl *clone = posix_acl_clone(*acl, gfp);
int err = -ENOMEM;
if (clone) {
+ *mode_p &= ~gr_acl_umask();
+
err = posix_acl_create_masq(clone, mode_p);
if (err < 0) {
posix_acl_release(clone);
posix_acl_from_xattr(struct user_namespace *user_ns,
const void *value, size_t size)
{
- posix_acl_xattr_header *header = (posix_acl_xattr_header *)value;
- posix_acl_xattr_entry *entry = (posix_acl_xattr_entry *)(header+1), *end;
+ const posix_acl_xattr_header *header = (const posix_acl_xattr_header *)value;
+ const posix_acl_xattr_entry *entry = (const posix_acl_xattr_entry *)(header+1), *end;
int count;
struct posix_acl *acl;
struct posix_acl_entry *acl_e;
+ umode_t umask = gr_acl_umask();
if (!value)
return NULL;
switch(acl_e->e_tag) {
case ACL_USER_OBJ:
+ acl_e->e_perm &= ~((umask & S_IRWXU) >> 6);
+ break;
case ACL_GROUP_OBJ:
case ACL_MASK:
+ acl_e->e_perm &= ~((umask & S_IRWXG) >> 3);
+ break;
case ACL_OTHER:
+ acl_e->e_perm &= ~(umask & S_IRWXO);
break;
case ACL_USER:
+ acl_e->e_perm &= ~((umask & S_IRWXU) >> 6);
acl_e->e_uid =
make_kuid(user_ns,
le32_to_cpu(entry->e_id));
goto fail;
break;
case ACL_GROUP:
+ acl_e->e_perm &= ~((umask & S_IRWXG) >> 3);
acl_e->e_gid =
make_kgid(user_ns,
le32_to_cpu(entry->e_id));
config PROC_KCORE
bool "/proc/kcore support" if !ARM
- depends on PROC_FS && MMU
+ depends on PROC_FS && MMU && !GRKERNSEC_PROC_ADD
help
Provides a virtual ELF core file of the live kernel. This can
be read with gdb and other ELF tools. No modifications can be
config PROC_VMCORE
bool "/proc/vmcore support"
- depends on PROC_FS && CRASH_DUMP
- default y
+ depends on PROC_FS && CRASH_DUMP && !GRKERNSEC
+ default n
help
Exports the dump image of crashed kernel in ELF format.
limited in memory.
config PROC_PAGE_MONITOR
- default y
- depends on PROC_FS && MMU
+ default n
+ depends on PROC_FS && MMU && !GRKERNSEC
bool "Enable /proc page monitoring" if EXPERT
help
Various /proc files exist to monitor process memory utilization:
#include <linux/tty.h>
#include <linux/string.h>
#include <linux/mman.h>
+#include <linux/grsecurity.h>
#include <linux/proc_fs.h>
#include <linux/ioport.h>
#include <linux/uaccess.h>
seq_putc(m, '\n');
}
+#if defined(CONFIG_PAX_NOEXEC) || defined(CONFIG_PAX_ASLR)
+static inline void task_pax(struct seq_file *m, struct task_struct *p)
+{
+ if (p->mm)
+ seq_printf(m, "PaX:\t%c%c%c%c%c\n",
+ p->mm->pax_flags & MF_PAX_PAGEEXEC ? 'P' : 'p',
+ p->mm->pax_flags & MF_PAX_EMUTRAMP ? 'E' : 'e',
+ p->mm->pax_flags & MF_PAX_MPROTECT ? 'M' : 'm',
+ p->mm->pax_flags & MF_PAX_RANDMMAP ? 'R' : 'r',
+ p->mm->pax_flags & MF_PAX_SEGMEXEC ? 'S' : 's');
+ else
+ seq_printf(m, "PaX:\t-----\n");
+}
+#endif
+
int proc_pid_status(struct seq_file *m, struct pid_namespace *ns,
struct pid *pid, struct task_struct *task)
{
task_cpus_allowed(m, task);
cpuset_task_status_allowed(m, task);
task_context_switch_counts(m, task);
+
+#if defined(CONFIG_PAX_NOEXEC) || defined(CONFIG_PAX_ASLR)
+ task_pax(m, task);
+#endif
+
+#if defined(CONFIG_GRKERNSEC) && !defined(CONFIG_GRKERNSEC_NO_RBAC)
+ task_grsec_rbac(m, task);
+#endif
+
return 0;
}
+#ifdef CONFIG_GRKERNSEC_PROC_MEMMAP
+#define PAX_RAND_FLAGS(_mm) (_mm != NULL && _mm != current->mm && \
+ (_mm->pax_flags & MF_PAX_RANDMMAP || \
+ _mm->pax_flags & MF_PAX_SEGMEXEC))
+#endif
+
static int do_task_stat(struct seq_file *m, struct pid_namespace *ns,
struct pid *pid, struct task_struct *task, int whole)
{
char tcomm[sizeof(task->comm)];
unsigned long flags;
+#ifdef CONFIG_GRKERNSEC_PROC_MEMMAP
+ if (current->exec_id != m->exec_id) {
+ gr_log_badprocpid("stat");
+ return 0;
+ }
+#endif
+
state = *get_task_state(task);
vsize = eip = esp = 0;
permitted = ptrace_may_access(task, PTRACE_MODE_READ | PTRACE_MODE_NOAUDIT);
gtime = task_gtime(task);
}
+#ifdef CONFIG_GRKERNSEC_PROC_MEMMAP
+ if (PAX_RAND_FLAGS(mm)) {
+ eip = 0;
+ esp = 0;
+ wchan = 0;
+ }
+#endif
+#ifdef CONFIG_GRKERNSEC_HIDESYM
+ wchan = 0;
+ eip =0;
+ esp =0;
+#endif
+
/* scale priority and nice values from timeslices to -20..20 */
/* to make it look like a "normal" Unix priority/nice value */
priority = task_prio(task);
seq_put_decimal_ull(m, ' ', vsize);
seq_put_decimal_ull(m, ' ', mm ? get_mm_rss(mm) : 0);
seq_put_decimal_ull(m, ' ', rsslim);
+#ifdef CONFIG_GRKERNSEC_PROC_MEMMAP
+ seq_put_decimal_ull(m, ' ', PAX_RAND_FLAGS(mm) ? 1 : (mm ? (permitted ? mm->start_code : 1) : 0));
+ seq_put_decimal_ull(m, ' ', PAX_RAND_FLAGS(mm) ? 1 : (mm ? (permitted ? mm->end_code : 1) : 0));
+ seq_put_decimal_ull(m, ' ', PAX_RAND_FLAGS(mm) ? 0 : ((permitted && mm) ? mm->start_stack : 0));
+#else
seq_put_decimal_ull(m, ' ', mm ? (permitted ? mm->start_code : 1) : 0);
seq_put_decimal_ull(m, ' ', mm ? (permitted ? mm->end_code : 1) : 0);
seq_put_decimal_ull(m, ' ', (permitted && mm) ? mm->start_stack : 0);
+#endif
seq_put_decimal_ull(m, ' ', esp);
seq_put_decimal_ull(m, ' ', eip);
/* The signal information here is obsolete.
seq_put_decimal_ull(m, ' ', cputime_to_clock_t(gtime));
seq_put_decimal_ll(m, ' ', cputime_to_clock_t(cgtime));
- if (mm && permitted) {
+ if (mm && permitted
+#ifdef CONFIG_GRKERNSEC_PROC_MEMMAP
+ && !PAX_RAND_FLAGS(mm)
+#endif
+ ) {
seq_put_decimal_ull(m, ' ', mm->start_data);
seq_put_decimal_ull(m, ' ', mm->end_data);
seq_put_decimal_ull(m, ' ', mm->start_brk);
struct pid *pid, struct task_struct *task)
{
unsigned long size = 0, resident = 0, shared = 0, text = 0, data = 0;
- struct mm_struct *mm = get_task_mm(task);
+ struct mm_struct *mm;
+#ifdef CONFIG_GRKERNSEC_PROC_MEMMAP
+ if (current->exec_id != m->exec_id) {
+ gr_log_badprocpid("statm");
+ return 0;
+ }
+#endif
+ mm = get_task_mm(task);
if (mm) {
size = task_statm(mm, &shared, &text, &data, &resident);
mmput(mm);
return 0;
}
+#ifdef CONFIG_GRKERNSEC_PROC_IPADDR
+int proc_pid_ipaddr(struct seq_file *m, struct pid_namespace *ns, struct pid *pid, struct task_struct *task)
+{
+ unsigned long flags;
+ u32 curr_ip = 0;
+
+ if (lock_task_sighand(task, &flags)) {
+ curr_ip = task->signal->curr_ip;
+ unlock_task_sighand(task, &flags);
+ }
+ return seq_printf(m, "%pI4\n", &curr_ip);
+}
+#endif
+
#ifdef CONFIG_CHECKPOINT_RESTORE
static struct pid *
get_children_pid(struct inode *inode, struct pid *pid_prev, loff_t pos)
union proc_op op;
};
+struct getdents_callback {
+ struct linux_dirent __user * current_dir;
+ struct linux_dirent __user * previous;
+ struct file * file;
+ int count;
+ int error;
+};
+
#define NOD(NAME, MODE, IOP, FOP, OP) { \
.name = (NAME), \
.len = sizeof(NAME) - 1, \
return 0;
}
+#ifdef CONFIG_GRKERNSEC_PROC_MEMMAP
+#define PAX_RAND_FLAGS(_mm) (_mm != NULL && _mm != current->mm && \
+ (_mm->pax_flags & MF_PAX_RANDMMAP || \
+ _mm->pax_flags & MF_PAX_SEGMEXEC))
+#endif
+
static int proc_pid_auxv(struct seq_file *m, struct pid_namespace *ns,
struct pid *pid, struct task_struct *task)
{
struct mm_struct *mm = mm_access(task, PTRACE_MODE_READ);
if (mm && !IS_ERR(mm)) {
unsigned int nwords = 0;
+
+#ifdef CONFIG_GRKERNSEC_PROC_MEMMAP
+ /* allow if we're currently ptracing this task */
+ if (PAX_RAND_FLAGS(mm) &&
+ (!(task->ptrace & PT_PTRACED) || (task->parent != current))) {
+ mmput(mm);
+ return 0;
+ }
+#endif
+
do {
nwords += 2;
} while (mm->saved_auxv[nwords - 2] != 0); /* AT_NULL */
}
-#ifdef CONFIG_KALLSYMS
+#if defined(CONFIG_KALLSYMS) && !defined(CONFIG_GRKERNSEC_HIDESYM)
/*
* Provides a wchan file via kallsyms in a proper one-value-per-file format.
* Returns the resolved symbol. If that fails, simply return the address.
mutex_unlock(&task->signal->cred_guard_mutex);
}
-#ifdef CONFIG_STACKTRACE
+#if defined(CONFIG_STACKTRACE) && !defined(CONFIG_GRKERNSEC_HIDESYM)
#define MAX_STACK_TRACE_DEPTH 64
return 0;
}
-#ifdef CONFIG_HAVE_ARCH_TRACEHOOK
+#if defined(CONFIG_HAVE_ARCH_TRACEHOOK) && !defined(CONFIG_GRKERNSEC_PROC_MEMMAP)
static int proc_pid_syscall(struct seq_file *m, struct pid_namespace *ns,
struct pid *pid, struct task_struct *task)
{
/************************************************************************/
/* permission checks */
-static int proc_fd_access_allowed(struct inode *inode)
+static int proc_fd_access_allowed(struct inode *inode, unsigned int log)
{
struct task_struct *task;
int allowed = 0;
*/
task = get_proc_task(inode);
if (task) {
- allowed = ptrace_may_access(task, PTRACE_MODE_READ);
+ if (log)
+ allowed = ptrace_may_access(task, PTRACE_MODE_READ);
+ else
+ allowed = ptrace_may_access(task, PTRACE_MODE_READ | PTRACE_MODE_NOAUDIT);
put_task_struct(task);
}
return allowed;
struct task_struct *task,
int hide_pid_min)
{
+ if (gr_pid_is_chrooted(task) || gr_check_hidden_task(task))
+ return false;
+
+#if defined(CONFIG_GRKERNSEC_PROC_USER) || defined(CONFIG_GRKERNSEC_PROC_USERGROUP)
+ rcu_read_lock();
+ {
+ const struct cred *tmpcred = current_cred();
+ const struct cred *cred = __task_cred(task);
+
+ if (uid_eq(tmpcred->uid, GLOBAL_ROOT_UID) || uid_eq(tmpcred->uid, cred->uid)
+#ifdef CONFIG_GRKERNSEC_PROC_USERGROUP
+ || in_group_p(grsec_proc_gid)
+#endif
+ ) {
+ rcu_read_unlock();
+ return true;
+ }
+ }
+ rcu_read_unlock();
+
+ if (!pid->hide_pid)
+ return false;
+#endif
+
if (pid->hide_pid < hide_pid_min)
return true;
if (in_group_p(pid->pid_gid))
return true;
+
return ptrace_may_access(task, PTRACE_MODE_READ);
}
put_task_struct(task);
if (!has_perms) {
+#if defined(CONFIG_GRKERNSEC_PROC_USER) || defined(CONFIG_GRKERNSEC_PROC_USERGROUP)
+ {
+#else
if (pid->hide_pid == 2) {
+#endif
/*
* Let's make getdents(), stat(), and open()
* consistent with each other. If a process
if (task) {
mm = mm_access(task, mode);
+ if (!IS_ERR_OR_NULL(mm) && gr_acl_handle_procpidmem(task)) {
+ mmput(mm);
+ mm = ERR_PTR(-EPERM);
+ }
put_task_struct(task);
if (!IS_ERR_OR_NULL(mm)) {
return PTR_ERR(mm);
file->private_data = mm;
+
+#ifdef CONFIG_GRKERNSEC_PROC_MEMMAP
+ file->f_version = current->exec_id;
+#endif
+
return 0;
}
ssize_t copied;
char *page;
+#ifdef CONFIG_GRKERNSEC
+ if (write)
+ return -EPERM;
+#endif
+#ifdef CONFIG_GRKERNSEC_PROC_MEMMAP
+ if (file->f_version != current->exec_id) {
+ gr_log_badprocpid("mem");
+ return 0;
+ }
+#endif
+
if (!mm)
return 0;
goto free;
while (count > 0) {
- int this_len = min_t(int, count, PAGE_SIZE);
+ ssize_t this_len = min_t(ssize_t, count, PAGE_SIZE);
if (write && copy_from_user(page, buf, this_len)) {
copied = -EFAULT;
if (!mm)
return 0;
+#ifdef CONFIG_GRKERNSEC_PROC_MEMMAP
+ if (file->f_version != current->exec_id) {
+ gr_log_badprocpid("environ");
+ return 0;
+ }
+#endif
+
page = (char *)__get_free_page(GFP_TEMPORARY);
if (!page)
return -ENOMEM;
goto free;
while (count > 0) {
size_t this_len, max_len;
- int retval;
+ ssize_t retval;
if (src >= (mm->env_end - mm->env_start))
break;
int error = -EACCES;
/* Are we allowed to snoop on the tasks file descriptors? */
- if (!proc_fd_access_allowed(inode))
+ if (!proc_fd_access_allowed(inode, 0))
goto out;
error = PROC_I(inode)->op.proc_get_link(dentry, &path);
struct path path;
/* Are we allowed to snoop on the tasks file descriptors? */
- if (!proc_fd_access_allowed(inode))
- goto out;
+ /* logging this is needed for learning on chromium to work properly,
+ but we don't want to flood the logs from 'ps' which does a readlink
+ on /proc/fd/2 of tasks in the listing, nor do we want 'ps' to learn
+ CAP_SYS_PTRACE as it's not necessary for its basic functionality
+ */
+ if (dentry->d_name.name[0] == '2' && dentry->d_name.name[1] == '\0') {
+ if (!proc_fd_access_allowed(inode,0))
+ goto out;
+ } else {
+ if (!proc_fd_access_allowed(inode,1))
+ goto out;
+ }
error = PROC_I(inode)->op.proc_get_link(dentry, &path);
if (error)
rcu_read_lock();
cred = __task_cred(task);
inode->i_uid = cred->euid;
+#ifdef CONFIG_GRKERNSEC_PROC_USERGROUP
+ inode->i_gid = grsec_proc_gid;
+#else
inode->i_gid = cred->egid;
+#endif
rcu_read_unlock();
}
security_task_to_inode(task, inode);
return -ENOENT;
}
if ((inode->i_mode == (S_IFDIR|S_IRUGO|S_IXUGO)) ||
+#ifdef CONFIG_GRKERNSEC_PROC_USER
+ (inode->i_mode == (S_IFDIR|S_IRUSR|S_IXUSR)) ||
+#elif defined(CONFIG_GRKERNSEC_PROC_USERGROUP)
+ (inode->i_mode == (S_IFDIR|S_IRUSR|S_IRGRP|S_IXUSR|S_IXGRP)) ||
+#endif
task_dumpable(task)) {
cred = __task_cred(task);
stat->uid = cred->euid;
+#ifdef CONFIG_GRKERNSEC_PROC_USERGROUP
+ stat->gid = grsec_proc_gid;
+#else
stat->gid = cred->egid;
+#endif
}
}
rcu_read_unlock();
if (task) {
if ((inode->i_mode == (S_IFDIR|S_IRUGO|S_IXUGO)) ||
+#ifdef CONFIG_GRKERNSEC_PROC_USER
+ (inode->i_mode == (S_IFDIR|S_IRUSR|S_IXUSR)) ||
+#elif defined(CONFIG_GRKERNSEC_PROC_USERGROUP)
+ (inode->i_mode == (S_IFDIR|S_IRUSR|S_IRGRP|S_IXUSR|S_IXGRP)) ||
+#endif
task_dumpable(task)) {
rcu_read_lock();
cred = __task_cred(task);
inode->i_uid = cred->euid;
+#ifdef CONFIG_GRKERNSEC_PROC_USERGROUP
+ inode->i_gid = grsec_proc_gid;
+#else
inode->i_gid = cred->egid;
+#endif
rcu_read_unlock();
} else {
inode->i_uid = GLOBAL_ROOT_UID;
if (!task)
goto out_no_task;
+ if (gr_pid_is_chrooted(task) || gr_check_hidden_task(task))
+ goto out;
+
/*
* Yes, it does not scale. And it should not. Don't add
* new entries into /proc/<tgid>/ without very good reasons.
if (!task)
return -ENOENT;
+ if (gr_pid_is_chrooted(task) || gr_check_hidden_task(task))
+ goto out;
+
if (!dir_emit_dots(file, ctx))
goto out;
REG("autogroup", S_IRUGO|S_IWUSR, proc_pid_sched_autogroup_operations),
#endif
REG("comm", S_IRUGO|S_IWUSR, proc_pid_set_comm_operations),
-#ifdef CONFIG_HAVE_ARCH_TRACEHOOK
+#if defined(CONFIG_HAVE_ARCH_TRACEHOOK) && !defined(CONFIG_GRKERNSEC_PROC_MEMMAP)
ONE("syscall", S_IRUSR, proc_pid_syscall),
#endif
ONE("cmdline", S_IRUGO, proc_pid_cmdline),
#ifdef CONFIG_SECURITY
DIR("attr", S_IRUGO|S_IXUGO, proc_attr_dir_inode_operations, proc_attr_dir_operations),
#endif
-#ifdef CONFIG_KALLSYMS
+#if defined(CONFIG_KALLSYMS) && !defined(CONFIG_GRKERNSEC_HIDESYM)
ONE("wchan", S_IRUGO, proc_pid_wchan),
#endif
-#ifdef CONFIG_STACKTRACE
+#if defined(CONFIG_STACKTRACE) && !defined(CONFIG_GRKERNSEC_HIDESYM)
ONE("stack", S_IRUSR, proc_pid_stack),
#endif
#ifdef CONFIG_SCHEDSTATS
#ifdef CONFIG_HARDWALL
ONE("hardwall", S_IRUGO, proc_pid_hardwall),
#endif
+#ifdef CONFIG_GRKERNSEC_PROC_IPADDR
+ ONE("ipaddr", S_IRUSR, proc_pid_ipaddr),
+#endif
#ifdef CONFIG_USER_NS
REG("uid_map", S_IRUGO|S_IWUSR, proc_uid_map_operations),
REG("gid_map", S_IRUGO|S_IWUSR, proc_gid_map_operations),
if (!inode)
goto out;
+#ifdef CONFIG_GRKERNSEC_PROC_USER
+ inode->i_mode = S_IFDIR|S_IRUSR|S_IXUSR;
+#elif defined(CONFIG_GRKERNSEC_PROC_USERGROUP)
+ inode->i_gid = grsec_proc_gid;
+ inode->i_mode = S_IFDIR|S_IRUSR|S_IRGRP|S_IXUSR|S_IXGRP;
+#else
inode->i_mode = S_IFDIR|S_IRUGO|S_IXUGO;
+#endif
inode->i_op = &proc_tgid_base_inode_operations;
inode->i_fop = &proc_tgid_base_operations;
inode->i_flags|=S_IMMUTABLE;
if (!task)
goto out;
+ if (gr_pid_is_chrooted(task) || gr_check_hidden_task(task))
+ goto out_put_task;
+
result = proc_pid_instantiate(dir, dentry, task, NULL);
+out_put_task:
put_task_struct(task);
out:
return ERR_PTR(result);
REG("sched", S_IRUGO|S_IWUSR, proc_pid_sched_operations),
#endif
REG("comm", S_IRUGO|S_IWUSR, proc_pid_set_comm_operations),
-#ifdef CONFIG_HAVE_ARCH_TRACEHOOK
+#if defined(CONFIG_HAVE_ARCH_TRACEHOOK) && !defined(CONFIG_GRKERNSEC_PROC_MEMMAP)
ONE("syscall", S_IRUSR, proc_pid_syscall),
#endif
ONE("cmdline", S_IRUGO, proc_pid_cmdline),
#ifdef CONFIG_SECURITY
DIR("attr", S_IRUGO|S_IXUGO, proc_attr_dir_inode_operations, proc_attr_dir_operations),
#endif
-#ifdef CONFIG_KALLSYMS
+#if defined(CONFIG_KALLSYMS) && !defined(CONFIG_GRKERNSEC_HIDESYM)
ONE("wchan", S_IRUGO, proc_pid_wchan),
#endif
-#ifdef CONFIG_STACKTRACE
+#if defined(CONFIG_STACKTRACE) && !defined(CONFIG_GRKERNSEC_HIDESYM)
ONE("stack", S_IRUSR, proc_pid_stack),
#endif
#ifdef CONFIG_SCHEDSTATS
static int __init proc_cmdline_init(void)
{
+#ifdef CONFIG_GRKERNSEC_PROC_ADD
+ proc_create_grsec("cmdline", 0, NULL, &cmdline_proc_fops);
+#else
proc_create("cmdline", 0, NULL, &cmdline_proc_fops);
+#endif
return 0;
}
fs_initcall(proc_cmdline_init);
static int __init proc_devices_init(void)
{
+#ifdef CONFIG_GRKERNSEC_PROC_ADD
+ proc_create_grsec("devices", 0, NULL, &proc_devinfo_operations);
+#else
proc_create("devices", 0, NULL, &proc_devinfo_operations);
+#endif
return 0;
}
fs_initcall(proc_devices_init);
if (!task)
return -ENOENT;
- files = get_files_struct(task);
+ if (!gr_acl_handle_procpidmem(task))
+ files = get_files_struct(task);
put_task_struct(task);
if (files) {
*/
int proc_fd_permission(struct inode *inode, int mask)
{
+ struct task_struct *task;
int rv = generic_permission(inode, mask);
- if (rv == 0)
- return 0;
+
if (task_tgid(current) == proc_pid(inode))
rv = 0;
+
+ task = get_proc_task(inode);
+ if (task == NULL)
+ return rv;
+
+ if (gr_acl_handle_procpidmem(task))
+ rv = -EACCES;
+
+ put_task_struct(task);
+
return rv;
}
#include <linux/bitops.h>
#include <linux/spinlock.h>
#include <linux/completion.h>
+#include <linux/grsecurity.h>
#include <asm/uaccess.h>
#include "internal.h"
return proc_lookup_de(PDE(dir), dir, dentry);
}
+struct dentry *proc_lookup_restrict(struct inode *dir, struct dentry *dentry,
+ unsigned int flags)
+{
+ if (gr_proc_is_restricted())
+ return ERR_PTR(-EACCES);
+
+ return proc_lookup_de(PDE(dir), dir, dentry);
+}
+
/*
* This returns non-zero if at EOF, so that the /proc
* root directory can use this and check if it should
return proc_readdir_de(PDE(inode), file, ctx);
}
+int proc_readdir_restrict(struct file *file, struct dir_context *ctx)
+{
+ struct inode *inode = file_inode(file);
+
+ if (gr_proc_is_restricted())
+ return -EACCES;
+
+ return proc_readdir_de(PDE(inode), file, ctx);
+}
+
/*
* These are the generic /proc directory operations. They
* use the in-memory "struct proc_dir_entry" tree to parse
.iterate = proc_readdir,
};
+static const struct file_operations proc_dir_restricted_operations = {
+ .llseek = generic_file_llseek,
+ .read = generic_read_dir,
+ .iterate = proc_readdir_restrict,
+};
+
/*
* proc directories can do almost nothing..
*/
.setattr = proc_notify_change,
};
+static const struct inode_operations proc_dir_restricted_inode_operations = {
+ .lookup = proc_lookup_restrict,
+ .getattr = proc_getattr,
+ .setattr = proc_notify_change,
+};
+
static int proc_register(struct proc_dir_entry * dir, struct proc_dir_entry * dp)
{
int ret;
return ret;
if (S_ISDIR(dp->mode)) {
- dp->proc_fops = &proc_dir_operations;
- dp->proc_iops = &proc_dir_inode_operations;
+ if (dp->restricted) {
+ dp->proc_fops = &proc_dir_restricted_operations;
+ dp->proc_iops = &proc_dir_restricted_inode_operations;
+ } else {
+ dp->proc_fops = &proc_dir_operations;
+ dp->proc_iops = &proc_dir_inode_operations;
+ }
dir->nlink++;
} else if (S_ISLNK(dp->mode)) {
dp->proc_iops = &proc_link_inode_operations;
}
EXPORT_SYMBOL_GPL(proc_mkdir_data);
+struct proc_dir_entry *proc_mkdir_data_restrict(const char *name, umode_t mode,
+ struct proc_dir_entry *parent, void *data)
+{
+ struct proc_dir_entry *ent;
+
+ if (mode == 0)
+ mode = S_IRUGO | S_IXUGO;
+
+ ent = __proc_create(&parent, name, S_IFDIR | mode, 2);
+ if (ent) {
+ ent->data = data;
+ ent->restricted = 1;
+ if (proc_register(parent, ent) < 0) {
+ kfree(ent);
+ ent = NULL;
+ }
+ }
+ return ent;
+}
+EXPORT_SYMBOL_GPL(proc_mkdir_data_restrict);
+
struct proc_dir_entry *proc_mkdir_mode(const char *name, umode_t mode,
struct proc_dir_entry *parent)
{
}
EXPORT_SYMBOL(proc_mkdir);
+struct proc_dir_entry *proc_mkdir_restrict(const char *name,
+ struct proc_dir_entry *parent)
+{
+ return proc_mkdir_data_restrict(name, 0, parent, NULL);
+}
+EXPORT_SYMBOL(proc_mkdir_restrict);
+
struct proc_dir_entry *proc_create_data(const char *name, umode_t mode,
struct proc_dir_entry *parent,
const struct file_operations *proc_fops,
#include <linux/mount.h>
#include <linux/magic.h>
#include <linux/namei.h>
+#include <linux/grsecurity.h>
#include <asm/uaccess.h>
#include "internal.h"
+#ifdef CONFIG_PROC_SYSCTL
+extern const struct inode_operations proc_sys_inode_operations;
+extern const struct inode_operations proc_sys_dir_operations;
+#endif
+
static void proc_evict_inode(struct inode *inode)
{
struct proc_dir_entry *de;
RCU_INIT_POINTER(PROC_I(inode)->sysctl, NULL);
sysctl_head_put(head);
}
+
+#ifdef CONFIG_PROC_SYSCTL
+ if (inode->i_op == &proc_sys_inode_operations ||
+ inode->i_op == &proc_sys_dir_operations)
+ gr_handle_delete(inode->i_ino, inode->i_sb->s_dev);
+#endif
+
}
static struct kmem_cache * proc_inode_cachep;
if (de->mode) {
inode->i_mode = de->mode;
inode->i_uid = de->uid;
+#ifdef CONFIG_GRKERNSEC_PROC_USERGROUP
+ inode->i_gid = grsec_proc_gid;
+#else
inode->i_gid = de->gid;
+#endif
}
if (de->size)
inode->i_size = de->size;
struct completion *pde_unload_completion;
struct list_head pde_openers; /* who did ->open, but not ->release */
spinlock_t pde_unload_lock; /* proc_fops checks and pde_users bumps */
+ u8 restricted; /* a directory in /proc/net that should be restricted via GRKERNSEC_PROC */
u8 namelen;
char name[];
-};
+} __randomize_layout;
union proc_op {
int (*proc_get_link)(struct dentry *, struct path *);
struct ctl_table *sysctl_entry;
const struct proc_ns_operations *ns_ops;
struct inode vfs_inode;
-};
+} __randomize_layout;
/*
* General functions
struct pid *, struct task_struct *);
extern int proc_pid_statm(struct seq_file *, struct pid_namespace *,
struct pid *, struct task_struct *);
+#ifdef CONFIG_GRKERNSEC_PROC_IPADDR
+extern int proc_pid_ipaddr(struct seq_file *, struct pid_namespace *,
+ struct pid *, struct task_struct *);
+#endif
/*
* base.c
* generic.c
*/
extern struct dentry *proc_lookup(struct inode *, struct dentry *, unsigned int);
+extern struct dentry *proc_lookup_restrict(struct inode *, struct dentry *, unsigned int);
extern struct dentry *proc_lookup_de(struct proc_dir_entry *, struct inode *,
struct dentry *);
extern int proc_readdir(struct file *, struct dir_context *);
+extern int proc_readdir_restrict(struct file *, struct dir_context *);
extern int proc_readdir_de(struct proc_dir_entry *, struct file *, struct dir_context *);
static inline struct proc_dir_entry *pde_get(struct proc_dir_entry *pde)
static int __init proc_interrupts_init(void)
{
+#ifdef CONFIG_GRKERNSEC_PROC_ADD
+ proc_create_grsec("interrupts", 0, NULL, &proc_interrupts_operations);
+#else
proc_create("interrupts", 0, NULL, &proc_interrupts_operations);
+#endif
return 0;
}
fs_initcall(proc_interrupts_init);
* the addresses in the elf_phdr on our list.
*/
start = kc_offset_to_vaddr(*fpos - elf_buflen);
- if ((tsz = (PAGE_SIZE - (start & ~PAGE_MASK))) > buflen)
+ tsz = PAGE_SIZE - (start & ~PAGE_MASK);
+ if (tsz > buflen)
tsz = buflen;
-
+
while (buflen) {
struct kcore_list *m;
kfree(elf_buf);
} else {
if (kern_addr_valid(start)) {
- unsigned long n;
-
- n = copy_to_user(buffer, (char *)start, tsz);
- /*
- * We cannot distinguish between fault on source
- * and fault on destination. When this happens
- * we clear too and hope it will trigger the
- * EFAULT again.
- */
- if (n) {
- if (clear_user(buffer + tsz - n,
- n))
+ char *elf_buf;
+ mm_segment_t oldfs;
+
+ elf_buf = kmalloc(tsz, GFP_KERNEL);
+ if (!elf_buf)
+ return -ENOMEM;
+ oldfs = get_fs();
+ set_fs(KERNEL_DS);
+ if (!__copy_from_user(elf_buf, (const void __user *)start, tsz)) {
+ set_fs(oldfs);
+ if (copy_to_user(buffer, elf_buf, tsz)) {
+ kfree(elf_buf);
return -EFAULT;
+ }
}
+ set_fs(oldfs);
+ kfree(elf_buf);
} else {
if (clear_user(buffer, tsz))
return -EFAULT;
static int open_kcore(struct inode *inode, struct file *filp)
{
+#if defined(CONFIG_GRKERNSEC_PROC_ADD) || defined(CONFIG_GRKERNSEC_HIDESYM)
+ return -EPERM;
+#endif
if (!capable(CAP_SYS_RAWIO))
return -EPERM;
if (kcore_need_update)
vmi.used >> 10,
vmi.largest_chunk >> 10
#ifdef CONFIG_MEMORY_FAILURE
- , atomic_long_read(&num_poisoned_pages) << (PAGE_SHIFT - 10)
+ , atomic_long_read_unchecked(&num_poisoned_pages) << (PAGE_SHIFT - 10)
#endif
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
, K(global_page_state(NR_ANON_TRANSPARENT_HUGEPAGES) *
if (file) {
seq_pad(m, ' ');
- seq_path(m, &file->f_path, "");
+ seq_path(m, &file->f_path, "\n\\");
}
seq_putc(m, '\n');
#include <linux/nsproxy.h>
#include <net/net_namespace.h>
#include <linux/seq_file.h>
+#include <linux/grsecurity.h>
#include "internal.h"
+#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE)
+static struct seq_operations *ipv6_seq_ops_addr;
+
+void register_ipv6_seq_ops_addr(struct seq_operations *addr)
+{
+ ipv6_seq_ops_addr = addr;
+}
+
+void unregister_ipv6_seq_ops_addr(void)
+{
+ ipv6_seq_ops_addr = NULL;
+}
+
+EXPORT_SYMBOL_GPL(register_ipv6_seq_ops_addr);
+EXPORT_SYMBOL_GPL(unregister_ipv6_seq_ops_addr);
+#endif
+
static inline struct net *PDE_NET(struct proc_dir_entry *pde)
{
return pde->parent->data;
return maybe_get_net(PDE_NET(PDE(inode)));
}
+extern const struct seq_operations dev_seq_ops;
+
int seq_open_net(struct inode *ino, struct file *f,
const struct seq_operations *ops, int size)
{
BUG_ON(size < sizeof(*p));
+ /* only permit access to /proc/net/dev */
+ if (
+#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE)
+ ops != ipv6_seq_ops_addr &&
+#endif
+ ops != &dev_seq_ops && gr_proc_is_restricted())
+ return -EACCES;
+
net = get_proc_net(ino);
if (net == NULL)
return -ENXIO;
int err;
struct net *net;
+ if (gr_proc_is_restricted())
+ return -EACCES;
+
err = -ENXIO;
net = get_proc_net(inode);
if (net == NULL)
#include <linux/namei.h>
#include <linux/mm.h>
#include <linux/module.h>
+#include <linux/nsproxy.h>
+#ifdef CONFIG_GRKERNSEC
+#include <net/net_namespace.h>
+#endif
#include "internal.h"
+extern int gr_handle_chroot_sysctl(const int op);
+extern int gr_handle_sysctl_mod(const char *dirname, const char *name,
+ const int op);
+
static const struct dentry_operations proc_sys_dentry_operations;
static const struct file_operations proc_sys_file_operations;
-static const struct inode_operations proc_sys_inode_operations;
+const struct inode_operations proc_sys_inode_operations;
static const struct file_operations proc_sys_dir_file_operations;
-static const struct inode_operations proc_sys_dir_operations;
+const struct inode_operations proc_sys_dir_operations;
void proc_sys_poll_notify(struct ctl_table_poll *poll)
{
err = NULL;
d_set_d_op(dentry, &proc_sys_dentry_operations);
+
+ gr_handle_proc_create(dentry, inode);
+
d_add(dentry, inode);
out:
struct inode *inode = file_inode(filp);
struct ctl_table_header *head = grab_header(inode);
struct ctl_table *table = PROC_I(inode)->sysctl_entry;
+ int op = write ? MAY_WRITE : MAY_READ;
ssize_t error;
size_t res;
* and won't be until we finish.
*/
error = -EPERM;
- if (sysctl_perm(head, table, write ? MAY_WRITE : MAY_READ))
+ if (sysctl_perm(head, table, op))
goto out;
/* if that can happen at all, it should be -EINVAL, not -EISDIR */
if (!table->proc_handler)
goto out;
+#ifdef CONFIG_GRKERNSEC
+ error = -EPERM;
+ if (gr_handle_chroot_sysctl(op))
+ goto out;
+ dget(filp->f_path.dentry);
+ if (gr_handle_sysctl_mod(filp->f_path.dentry->d_parent->d_name.name, table->procname, op)) {
+ dput(filp->f_path.dentry);
+ goto out;
+ }
+ dput(filp->f_path.dentry);
+ if (!gr_acl_handle_open(filp->f_path.dentry, filp->f_path.mnt, op))
+ goto out;
+ if (write) {
+ if (current->nsproxy->net_ns != table->extra2) {
+ if (!capable(CAP_SYS_ADMIN))
+ goto out;
+ } else if (!ns_capable(current->nsproxy->net_ns->user_ns, CAP_NET_ADMIN))
+ goto out;
+ }
+#endif
+
/* careful: calling conventions are nasty here */
res = count;
error = table->proc_handler(table, write, buf, &res, ppos);
return false;
} else {
d_set_d_op(child, &proc_sys_dentry_operations);
+
+ gr_handle_proc_create(child, inode);
+
d_add(child, inode);
}
} else {
if ((*pos)++ < ctx->pos)
return true;
+ if (!gr_acl_handle_hidden_file(file->f_path.dentry, file->f_path.mnt))
+ return 0;
+
if (unlikely(S_ISLNK(table->mode)))
res = proc_sys_link_fill_cache(file, ctx, head, table);
else
if (IS_ERR(head))
return PTR_ERR(head);
+ if (table && !gr_acl_handle_hidden_file(dentry, mnt))
+ return -ENOENT;
+
generic_fillattr(inode, stat);
if (table)
stat->mode = (stat->mode & S_IFMT) | table->mode;
.llseek = generic_file_llseek,
};
-static const struct inode_operations proc_sys_inode_operations = {
+const struct inode_operations proc_sys_inode_operations = {
.permission = proc_sys_permission,
.setattr = proc_sys_setattr,
.getattr = proc_sys_getattr,
};
-static const struct inode_operations proc_sys_dir_operations = {
+const struct inode_operations proc_sys_dir_operations = {
.lookup = proc_sys_lookup,
.permission = proc_sys_permission,
.setattr = proc_sys_setattr,
static struct ctl_dir *new_dir(struct ctl_table_set *set,
const char *name, int namelen)
{
- struct ctl_table *table;
+ ctl_table_no_const *table;
struct ctl_dir *new;
struct ctl_node *node;
char *new_name;
return NULL;
node = (struct ctl_node *)(new + 1);
- table = (struct ctl_table *)(node + 1);
+ table = (ctl_table_no_const *)(node + 1);
new_name = (char *)(table + 2);
memcpy(new_name, name, namelen);
new_name[namelen] = '\0';
static struct ctl_table_header *new_links(struct ctl_dir *dir, struct ctl_table *table,
struct ctl_table_root *link_root)
{
- struct ctl_table *link_table, *entry, *link;
+ ctl_table_no_const *link_table, *link;
+ struct ctl_table *entry;
struct ctl_table_header *links;
struct ctl_node *node;
char *link_name;
return NULL;
node = (struct ctl_node *)(links + 1);
- link_table = (struct ctl_table *)(node + nr_entries);
+ link_table = (ctl_table_no_const *)(node + nr_entries);
link_name = (char *)&link_table[nr_entries + 1];
for (link = link_table, entry = table; entry->procname; link++, entry++) {
struct ctl_table_header ***subheader, struct ctl_table_set *set,
struct ctl_table *table)
{
- struct ctl_table *ctl_table_arg = NULL;
- struct ctl_table *entry, *files;
+ ctl_table_no_const *ctl_table_arg = NULL, *files = NULL;
+ struct ctl_table *entry;
int nr_files = 0;
int nr_dirs = 0;
int err = -ENOMEM;
nr_files++;
}
- files = table;
/* If there are mixed files and directories we need a new table */
if (nr_dirs && nr_files) {
- struct ctl_table *new;
+ ctl_table_no_const *new;
files = kzalloc(sizeof(struct ctl_table) * (nr_files + 1),
GFP_KERNEL);
if (!files)
/* Register everything except a directory full of subdirectories */
if (nr_files || !nr_dirs) {
struct ctl_table_header *header;
- header = __register_sysctl_table(set, path, files);
+ header = __register_sysctl_table(set, path, files ? files : table);
if (!header) {
kfree(ctl_table_arg);
goto out;
proc_mkdir("openprom", NULL);
#endif
proc_tty_init();
+#ifdef CONFIG_GRKERNSEC_PROC_ADD
+#ifdef CONFIG_GRKERNSEC_PROC_USER
+ proc_mkdir_mode("bus", S_IRUSR | S_IXUSR, NULL);
+#elif defined(CONFIG_GRKERNSEC_PROC_USERGROUP)
+ proc_mkdir_mode("bus", S_IRUSR | S_IXUSR | S_IRGRP | S_IXGRP, NULL);
+#endif
+#else
proc_mkdir("bus", NULL);
+#endif
proc_sys_init();
}
#include <linux/irqnr.h>
#include <linux/cputime.h>
#include <linux/tick.h>
+#include <linux/grsecurity.h>
#ifndef arch_irq_stat_cpu
#define arch_irq_stat_cpu(cpu) 0
u64 sum_softirq = 0;
unsigned int per_softirq_sums[NR_SOFTIRQS] = {0};
struct timespec boottime;
+ int unrestricted = 1;
+
+#ifdef CONFIG_GRKERNSEC_PROC_ADD
+#if defined(CONFIG_GRKERNSEC_PROC_USER) || defined(CONFIG_GRKERNSEC_PROC_USERGROUP)
+ if (!uid_eq(current_uid(), GLOBAL_ROOT_UID)
+#ifdef CONFIG_GRKERNSEC_PROC_USERGROUP
+ && !in_group_p(grsec_proc_gid)
+#endif
+ )
+ unrestricted = 0;
+#endif
+#endif
user = nice = system = idle = iowait =
irq = softirq = steal = 0;
nice += kcpustat_cpu(i).cpustat[CPUTIME_NICE];
system += kcpustat_cpu(i).cpustat[CPUTIME_SYSTEM];
idle += get_idle_time(i);
- iowait += get_iowait_time(i);
- irq += kcpustat_cpu(i).cpustat[CPUTIME_IRQ];
- softirq += kcpustat_cpu(i).cpustat[CPUTIME_SOFTIRQ];
- steal += kcpustat_cpu(i).cpustat[CPUTIME_STEAL];
- guest += kcpustat_cpu(i).cpustat[CPUTIME_GUEST];
- guest_nice += kcpustat_cpu(i).cpustat[CPUTIME_GUEST_NICE];
- sum += kstat_cpu_irqs_sum(i);
- sum += arch_irq_stat_cpu(i);
-
- for (j = 0; j < NR_SOFTIRQS; j++) {
- unsigned int softirq_stat = kstat_softirqs_cpu(j, i);
-
- per_softirq_sums[j] += softirq_stat;
- sum_softirq += softirq_stat;
+ if (unrestricted) {
+ iowait += get_iowait_time(i);
+ irq += kcpustat_cpu(i).cpustat[CPUTIME_IRQ];
+ softirq += kcpustat_cpu(i).cpustat[CPUTIME_SOFTIRQ];
+ steal += kcpustat_cpu(i).cpustat[CPUTIME_STEAL];
+ guest += kcpustat_cpu(i).cpustat[CPUTIME_GUEST];
+ guest_nice += kcpustat_cpu(i).cpustat[CPUTIME_GUEST_NICE];
+ sum += kstat_cpu_irqs_sum(i);
+ sum += arch_irq_stat_cpu(i);
+ for (j = 0; j < NR_SOFTIRQS; j++) {
+ unsigned int softirq_stat = kstat_softirqs_cpu(j, i);
+
+ per_softirq_sums[j] += softirq_stat;
+ sum_softirq += softirq_stat;
+ }
}
}
- sum += arch_irq_stat();
+ if (unrestricted)
+ sum += arch_irq_stat();
seq_puts(p, "cpu ");
seq_put_decimal_ull(p, ' ', cputime64_to_clock_t(user));
nice = kcpustat_cpu(i).cpustat[CPUTIME_NICE];
system = kcpustat_cpu(i).cpustat[CPUTIME_SYSTEM];
idle = get_idle_time(i);
- iowait = get_iowait_time(i);
- irq = kcpustat_cpu(i).cpustat[CPUTIME_IRQ];
- softirq = kcpustat_cpu(i).cpustat[CPUTIME_SOFTIRQ];
- steal = kcpustat_cpu(i).cpustat[CPUTIME_STEAL];
- guest = kcpustat_cpu(i).cpustat[CPUTIME_GUEST];
- guest_nice = kcpustat_cpu(i).cpustat[CPUTIME_GUEST_NICE];
+ if (unrestricted) {
+ iowait = get_iowait_time(i);
+ irq = kcpustat_cpu(i).cpustat[CPUTIME_IRQ];
+ softirq = kcpustat_cpu(i).cpustat[CPUTIME_SOFTIRQ];
+ steal = kcpustat_cpu(i).cpustat[CPUTIME_STEAL];
+ guest = kcpustat_cpu(i).cpustat[CPUTIME_GUEST];
+ guest_nice = kcpustat_cpu(i).cpustat[CPUTIME_GUEST_NICE];
+ }
seq_printf(p, "cpu%d", i);
seq_put_decimal_ull(p, ' ', cputime64_to_clock_t(user));
seq_put_decimal_ull(p, ' ', cputime64_to_clock_t(nice));
/* sum again ? it could be updated? */
for_each_irq_nr(j)
- seq_put_decimal_ull(p, ' ', kstat_irqs_usr(j));
+ seq_put_decimal_ull(p, ' ', unrestricted ? kstat_irqs_usr(j) : 0ULL);
seq_printf(p,
"\nctxt %llu\n"
"processes %lu\n"
"procs_running %lu\n"
"procs_blocked %lu\n",
- nr_context_switches(),
+ unrestricted ? nr_context_switches() : 0ULL,
(unsigned long)jif,
- total_forks,
- nr_running(),
- nr_iowait());
+ unrestricted ? total_forks : 0UL,
+ unrestricted ? nr_running() : 0UL,
+ unrestricted ? nr_iowait() : 0UL);
seq_printf(p, "softirq %llu", (unsigned long long)sum_softirq);
#include <linux/swap.h>
#include <linux/swapops.h>
#include <linux/mmu_notifier.h>
+#include <linux/grsecurity.h>
#include <asm/elf.h>
#include <asm/uaccess.h>
#include <asm/tlbflush.h>
#include "internal.h"
+#ifdef CONFIG_GRKERNSEC_PROC_MEMMAP
+#define PAX_RAND_FLAGS(_mm) (_mm != NULL && _mm != current->mm && \
+ (_mm->pax_flags & MF_PAX_RANDMMAP || \
+ _mm->pax_flags & MF_PAX_SEGMEXEC))
+#endif
+
void task_mem(struct seq_file *m, struct mm_struct *mm)
{
unsigned long data, text, lib, swap;
"VmExe:\t%8lu kB\n"
"VmLib:\t%8lu kB\n"
"VmPTE:\t%8lu kB\n"
- "VmSwap:\t%8lu kB\n",
- hiwater_vm << (PAGE_SHIFT-10),
+ "VmSwap:\t%8lu kB\n"
+
+#ifdef CONFIG_ARCH_TRACK_EXEC_LIMIT
+ "CsBase:\t%8lx\nCsLim:\t%8lx\n"
+#endif
+
+ ,hiwater_vm << (PAGE_SHIFT-10),
total_vm << (PAGE_SHIFT-10),
mm->locked_vm << (PAGE_SHIFT-10),
mm->pinned_vm << (PAGE_SHIFT-10),
mm->stack_vm << (PAGE_SHIFT-10), text, lib,
(PTRS_PER_PTE * sizeof(pte_t) *
atomic_long_read(&mm->nr_ptes)) >> 10,
- swap << (PAGE_SHIFT-10));
+ swap << (PAGE_SHIFT-10)
+
+#ifdef CONFIG_ARCH_TRACK_EXEC_LIMIT
+#ifdef CONFIG_GRKERNSEC_PROC_MEMMAP
+ , PAX_RAND_FLAGS(mm) ? 0 : mm->context.user_cs_base
+ , PAX_RAND_FLAGS(mm) ? 0 : mm->context.user_cs_limit
+#else
+ , mm->context.user_cs_base
+ , mm->context.user_cs_limit
+#endif
+#endif
+
+ );
}
unsigned long task_vsize(struct mm_struct *mm)
pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT;
}
- /* We don't show the stack guard page in /proc/maps */
+#ifdef CONFIG_GRKERNSEC_PROC_MEMMAP
+ start = PAX_RAND_FLAGS(mm) ? 0UL : vma->vm_start;
+ end = PAX_RAND_FLAGS(mm) ? 0UL : vma->vm_end;
+#else
start = vma->vm_start;
- if (stack_guard_page_start(vma, start))
- start += PAGE_SIZE;
end = vma->vm_end;
- if (stack_guard_page_end(vma, end))
- end -= PAGE_SIZE;
+#endif
seq_setwidth(m, 25 + sizeof(void *) * 6 - 1);
seq_printf(m, "%08lx-%08lx %c%c%c%c %08llx %02x:%02x %lu ",
flags & VM_WRITE ? 'w' : '-',
flags & VM_EXEC ? 'x' : '-',
flags & VM_MAYSHARE ? 's' : 'p',
+#ifdef CONFIG_GRKERNSEC_PROC_MEMMAP
+ PAX_RAND_FLAGS(mm) ? 0UL : pgoff,
+#else
pgoff,
+#endif
MAJOR(dev), MINOR(dev), ino);
/*
*/
if (file) {
seq_pad(m, ' ');
- seq_path(m, &file->f_path, "\n");
+ seq_path(m, &file->f_path, "\n\\");
goto done;
}
* Thread stack in /proc/PID/task/TID/maps or
* the main process stack.
*/
- if (!is_pid || (vma->vm_start <= mm->start_stack &&
- vma->vm_end >= mm->start_stack)) {
+ if (!is_pid || (vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP)) ||
+ (vma->vm_start <= mm->start_stack &&
+ vma->vm_end >= mm->start_stack)) {
name = "[stack]";
} else {
/* Thread stack in /proc/PID/maps */
static int show_map(struct seq_file *m, void *v, int is_pid)
{
+#ifdef CONFIG_GRKERNSEC_PROC_MEMMAP
+ if (current->exec_id != m->exec_id) {
+ gr_log_badprocpid("maps");
+ return 0;
+ }
+#endif
show_map_vma(m, v, is_pid);
m_cache_vma(m, v);
return 0;
.private = &mss,
};
+#ifdef CONFIG_GRKERNSEC_PROC_MEMMAP
+ if (current->exec_id != m->exec_id) {
+ gr_log_badprocpid("smaps");
+ return 0;
+ }
+#endif
memset(&mss, 0, sizeof mss);
- mss.vma = vma;
- /* mmap_sem is held in m_start */
- if (vma->vm_mm && !is_vm_hugetlb_page(vma))
- walk_page_range(vma->vm_start, vma->vm_end, &smaps_walk);
-
+#ifdef CONFIG_GRKERNSEC_PROC_MEMMAP
+ if (!PAX_RAND_FLAGS(vma->vm_mm)) {
+#endif
+ mss.vma = vma;
+ /* mmap_sem is held in m_start */
+ if (vma->vm_mm && !is_vm_hugetlb_page(vma))
+ walk_page_range(vma->vm_start, vma->vm_end, &smaps_walk);
+#ifdef CONFIG_GRKERNSEC_PROC_MEMMAP
+ }
+#endif
show_map_vma(m, vma, is_pid);
seq_printf(m,
"KernelPageSize: %8lu kB\n"
"MMUPageSize: %8lu kB\n"
"Locked: %8lu kB\n",
+#ifdef CONFIG_GRKERNSEC_PROC_MEMMAP
+ PAX_RAND_FLAGS(vma->vm_mm) ? 0UL : (vma->vm_end - vma->vm_start) >> 10,
+#else
(vma->vm_end - vma->vm_start) >> 10,
+#endif
mss.resident >> 10,
(unsigned long)(mss.pss >> (10 + PSS_SHIFT)),
mss.shared_clean >> 10,
char buffer[64];
int nid;
+#ifdef CONFIG_GRKERNSEC_PROC_MEMMAP
+ if (current->exec_id != m->exec_id) {
+ gr_log_badprocpid("numa_maps");
+ return 0;
+ }
+#endif
+
if (!mm)
return 0;
mpol_to_str(buffer, sizeof(buffer), proc_priv->task_mempolicy);
}
+#ifdef CONFIG_GRKERNSEC_PROC_MEMMAP
+ seq_printf(m, "%08lx %s", PAX_RAND_FLAGS(vma->vm_mm) ? 0UL : vma->vm_start, buffer);
+#else
seq_printf(m, "%08lx %s", vma->vm_start, buffer);
+#endif
if (file) {
seq_puts(m, " file=");
- seq_path(m, &file->f_path, "\n\t= ");
+ seq_path(m, &file->f_path, "\n\t\\= ");
} else if (vma->vm_start <= mm->brk && vma->vm_end >= mm->start_brk) {
seq_puts(m, " heap");
} else {
else
bytes += kobjsize(mm);
- if (current->fs && current->fs->users > 1)
+ if (current->fs && atomic_read(¤t->fs->users) > 1)
sbytes += kobjsize(current->fs);
else
bytes += kobjsize(current->fs);
if (file) {
seq_pad(m, ' ');
- seq_path(m, &file->f_path, "");
+ seq_path(m, &file->f_path, "\n\\");
} else if (mm) {
pid_t tid = pid_of_stack(priv, vma, is_pid);
nr_bytes = count;
/* If pfn is not ram, return zeros for sparse dump files */
- if (pfn_is_ram(pfn) == 0)
- memset(buf, 0, nr_bytes);
- else {
+ if (pfn_is_ram(pfn) == 0) {
+ if (userbuf) {
+ if (clear_user((char __force_user *)buf, nr_bytes))
+ return -EFAULT;
+ } else
+ memset(buf, 0, nr_bytes);
+ } else {
tmp = copy_oldmem_page(pfn, buf, nr_bytes,
offset, userbuf);
if (tmp < 0)
static int copy_to(void *target, void *src, size_t size, int userbuf)
{
if (userbuf) {
- if (copy_to_user((char __user *) target, src, size))
+ if (copy_to_user((char __force_user *) target, src, size))
return -EFAULT;
} else {
memcpy(target, src, size);
if (*fpos < m->offset + m->size) {
tsz = min_t(size_t, m->offset + m->size - *fpos, buflen);
start = m->paddr + *fpos - m->offset;
- tmp = read_from_oldmem(buffer, tsz, &start, userbuf);
+ tmp = read_from_oldmem((char __force_kernel *)buffer, tsz, &start, userbuf);
if (tmp < 0)
return tmp;
buflen -= tsz;
static ssize_t read_vmcore(struct file *file, char __user *buffer,
size_t buflen, loff_t *fpos)
{
- return __read_vmcore((__force char *) buffer, buflen, fpos, 1);
+ return __read_vmcore((__force_kernel char *) buffer, buflen, fpos, 1);
}
/*
BYTESEX_BE,
};
-static inline __u64 fs64_to_cpu(struct qnx6_sb_info *sbi, __fs64 n)
+static inline __u64 __intentional_overflow(-1) fs64_to_cpu(struct qnx6_sb_info *sbi, __fs64 n)
{
if (sbi->s_bytesex == BYTESEX_LE)
return le64_to_cpu((__force __le64)n);
return (__force __fs64)cpu_to_be64(n);
}
-static inline __u32 fs32_to_cpu(struct qnx6_sb_info *sbi, __fs32 n)
+static inline __u32 __intentional_overflow(-1) fs32_to_cpu(struct qnx6_sb_info *sbi, __fs32 n)
{
if (sbi->s_bytesex == BYTESEX_LE)
return le32_to_cpu((__force __le32)n);
void quota_send_warning(struct kqid qid, dev_t dev,
const char warntype)
{
- static atomic_t seq;
+ static atomic_unchecked_t seq;
struct sk_buff *skb;
void *msg_head;
int ret;
"VFS: Not enough memory to send quota warning.\n");
return;
}
- msg_head = genlmsg_put(skb, 0, atomic_add_return(1, &seq),
+ msg_head = genlmsg_put(skb, 0, atomic_add_return_unchecked(1, &seq),
"a_genl_family, 0, QUOTA_NL_C_WARNING);
if (!msg_head) {
printk(KERN_ERR
old_fs = get_fs();
set_fs(get_ds());
- p = (__force const char __user *)buf;
+ p = (const char __force_user *)buf;
if (count > MAX_RW_COUNT)
count = MAX_RW_COUNT;
if (file->f_op->write)
#include <linux/security.h>
#include <linux/syscalls.h>
#include <linux/unistd.h>
+#include <linux/namei.h>
#include <asm/uaccess.h>
struct readdir_callback {
struct dir_context ctx;
struct old_linux_dirent __user * dirent;
+ struct file * file;
int result;
};
buf->result = -EOVERFLOW;
return -EOVERFLOW;
}
+
+ if (!gr_acl_handle_filldir(buf->file, name, namlen, ino))
+ return 0;
+
buf->result++;
dirent = buf->dirent;
if (!access_ok(VERIFY_WRITE, dirent,
if (!f.file)
return -EBADF;
+ buf.file = f.file;
error = iterate_dir(f.file, &buf.ctx);
if (buf.result)
error = buf.result;
struct dir_context ctx;
struct linux_dirent __user * current_dir;
struct linux_dirent __user * previous;
+ struct file * file;
int count;
int error;
};
buf->error = -EOVERFLOW;
return -EOVERFLOW;
}
+
+ if (!gr_acl_handle_filldir(buf->file, name, namlen, ino))
+ return 0;
+
dirent = buf->previous;
if (dirent) {
if (__put_user(offset, &dirent->d_off))
if (!f.file)
return -EBADF;
+ buf.file = f.file;
error = iterate_dir(f.file, &buf.ctx);
if (error >= 0)
error = buf.error;
struct dir_context ctx;
struct linux_dirent64 __user * current_dir;
struct linux_dirent64 __user * previous;
+ struct file *file;
int count;
int error;
};
buf->error = -EINVAL; /* only used if we fail.. */
if (reclen > buf->count)
return -EINVAL;
+
+ if (!gr_acl_handle_filldir(buf->file, name, namlen, ino))
+ return 0;
+
dirent = buf->previous;
if (dirent) {
if (__put_user(offset, &dirent->d_off))
if (!f.file)
return -EBADF;
+ buf.file = f.file;
error = iterate_dir(f.file, &buf.ctx);
if (error >= 0)
error = buf.error;
return;
}
- atomic_inc(&fs_generation(tb->tb_sb));
+ atomic_inc_unchecked(&fs_generation(tb->tb_sb));
do_balance_starts(tb);
/*
}
static struct item_operations errcatch_ops = {
- errcatch_bytes_number,
- errcatch_decrement_key,
- errcatch_is_left_mergeable,
- errcatch_print_item,
- errcatch_check_item,
-
- errcatch_create_vi,
- errcatch_check_left,
- errcatch_check_right,
- errcatch_part_size,
- errcatch_unit_num,
- errcatch_print_vi
+ .bytes_number = errcatch_bytes_number,
+ .decrement_key = errcatch_decrement_key,
+ .is_left_mergeable = errcatch_is_left_mergeable,
+ .print_item = errcatch_print_item,
+ .check_item = errcatch_check_item,
+
+ .create_vi = errcatch_create_vi,
+ .check_left = errcatch_check_left,
+ .check_right = errcatch_check_right,
+ .part_size = errcatch_part_size,
+ .unit_num = errcatch_unit_num,
+ .print_vi = errcatch_print_vi
};
#if ! (TYPE_STAT_DATA == 0 && TYPE_INDIRECT == 1 && TYPE_DIRECT == 2 && TYPE_DIRENTRY == 3)
"SMALL_TAILS " : "NO_TAILS ",
replay_only(sb) ? "REPLAY_ONLY " : "",
convert_reiserfs(sb) ? "CONV " : "",
- atomic_read(&r->s_generation_counter),
+ atomic_read_unchecked(&r->s_generation_counter),
SF(s_disk_reads), SF(s_disk_writes), SF(s_fix_nodes),
SF(s_do_balance), SF(s_unneeded_left_neighbor),
SF(s_good_search_by_key_reada), SF(s_bmaps),
/* Comment? -Hans */
wait_queue_head_t s_wait;
/* increased by one every time the tree gets re-balanced */
- atomic_t s_generation_counter;
+ atomic_unchecked_t s_generation_counter;
/* File system properties. Currently holds on-disk FS format */
unsigned long s_properties;
#define REISERFS_USER_MEM 1 /* user memory mode */
#define fs_generation(s) (REISERFS_SB(s)->s_generation_counter)
-#define get_generation(s) atomic_read (&fs_generation(s))
+#define get_generation(s) atomic_read_unchecked (&fs_generation(s))
#define FILESYSTEM_CHANGED_TB(tb) (get_generation((tb)->tb_sb) != (tb)->fs_gen)
#define __fs_changed(gen,s) (gen != get_generation (s))
#define fs_changed(gen,s) \
sbi->s_mount_opt |= (1 << REISERFS_SMALLTAIL);
sbi->s_mount_opt |= (1 << REISERFS_ERROR_RO);
sbi->s_mount_opt |= (1 << REISERFS_BARRIER_FLUSH);
+#ifdef CONFIG_REISERFS_FS_XATTR
+ /* turn on user xattrs by default */
+ sbi->s_mount_opt |= (1 << REISERFS_XATTRS_USER);
+#endif
/* no preallocation minimum, be smart in reiserfs_file_write instead */
sbi->s_alloc_options.preallocmin = 0;
/* Preallocate by 16 blocks (17-1) at once */
#include <linux/export.h>
#include <linux/slab.h>
#include <linux/poll.h>
+#include <linux/security.h>
#include <linux/personality.h> /* for STICKY_TIMEOUTS */
#include <linux/file.h>
#include <linux/fdtable.h>
struct poll_list *walk = head;
unsigned long todo = nfds;
+ gr_learn_resource(current, RLIMIT_NOFILE, nfds, 1);
if (nfds > rlimit(RLIMIT_NOFILE))
return -EINVAL;
#include <linux/slab.h>
#include <linux/cred.h>
#include <linux/mm.h>
+#include <linux/sched.h>
+#include <linux/grsecurity.h>
#include <asm/uaccess.h>
#include <asm/page.h>
static void *seq_buf_alloc(unsigned long size)
{
- void *buf;
-
- /*
- * __GFP_NORETRY to avoid oom-killings with high-order allocations -
- * it's better to fall back to vmalloc() than to kill things.
- */
- buf = kmalloc(size, GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN);
- if (!buf && size > PAGE_SIZE)
- buf = vmalloc(size);
- return buf;
+ return kmalloc(size, GFP_KERNEL | GFP_USERCOPY);
}
/**
#ifdef CONFIG_USER_NS
p->user_ns = file->f_cred->user_ns;
#endif
+#ifdef CONFIG_GRKERNSEC_PROC_MEMMAP
+ p->exec_id = current->exec_id;
+#endif
/*
* Wrappers around seq_open(e.g. swaps_open) need to be
}
EXPORT_SYMBOL(seq_open);
+
+int seq_open_restrict(struct file *file, const struct seq_operations *op)
+{
+ if (gr_proc_is_restricted())
+ return -EACCES;
+
+ return seq_open(file, op);
+}
+EXPORT_SYMBOL(seq_open_restrict);
+
static int traverse(struct seq_file *m, loff_t offset)
{
loff_t pos = 0, index;
ssize_t seq_read(struct file *file, char __user *buf, size_t size, loff_t *ppos)
{
struct seq_file *m = file->private_data;
- size_t copied = 0;
+ ssize_t copied = 0;
loff_t pos;
size_t n;
void *p;
int single_open(struct file *file, int (*show)(struct seq_file *, void *),
void *data)
{
- struct seq_operations *op = kmalloc(sizeof(*op), GFP_KERNEL);
+ seq_operations_no_const *op = kzalloc(sizeof(*op), GFP_KERNEL);
int res = -ENOMEM;
if (op) {
}
EXPORT_SYMBOL(single_open_size);
+int single_open_restrict(struct file *file, int (*show)(struct seq_file *, void *),
+ void *data)
+{
+ if (gr_proc_is_restricted())
+ return -EACCES;
+
+ return single_open(file, show, data);
+}
+EXPORT_SYMBOL(single_open_restrict);
+
+
int single_release(struct inode *inode, struct file *file)
{
const struct seq_operations *op = ((struct seq_file *)file->private_data)->op;
pipe_lock(pipe);
for (;;) {
- if (!pipe->readers) {
+ if (!atomic_read(&pipe->readers)) {
send_sig(SIGPIPE, current, 0);
if (!ret)
ret = -EPIPE;
page_nr++;
ret += buf->len;
- if (pipe->files)
+ if (atomic_read(&pipe->files))
do_wakeup = 1;
if (!--spd->nr_pages)
do_wakeup = 0;
}
- pipe->waiting_writers++;
+ atomic_inc(&pipe->waiting_writers);
pipe_wait(pipe);
- pipe->waiting_writers--;
+ atomic_dec(&pipe->waiting_writers);
}
pipe_unlock(pipe);
old_fs = get_fs();
set_fs(get_ds());
/* The cast to a user pointer is valid due to the set_fs() */
- res = vfs_readv(file, (const struct iovec __user *)vec, vlen, &pos);
+ res = vfs_readv(file, (const struct iovec __force_user *)vec, vlen, &pos);
set_fs(old_fs);
return res;
old_fs = get_fs();
set_fs(get_ds());
/* The cast to a user pointer is valid due to the set_fs() */
- res = vfs_write(file, (__force const char __user *)buf, count, &pos);
+ res = vfs_write(file, (const char __force_user *)buf, count, &pos);
set_fs(old_fs);
return res;
goto err;
this_len = min_t(size_t, len, PAGE_CACHE_SIZE - offset);
- vec[i].iov_base = (void __user *) page_address(page);
+ vec[i].iov_base = (void __force_user *) page_address(page);
vec[i].iov_len = this_len;
spd.pages[i] = page;
spd.nr_pages++;
ops->release(pipe, buf);
pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1);
pipe->nrbufs--;
- if (pipe->files)
+ if (atomic_read(&pipe->files))
sd->need_wakeup = true;
}
static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd)
{
while (!pipe->nrbufs) {
- if (!pipe->writers)
+ if (!atomic_read(&pipe->writers))
return 0;
- if (!pipe->waiting_writers && sd->num_spliced)
+ if (!atomic_read(&pipe->waiting_writers) && sd->num_spliced)
return 0;
if (sd->flags & SPLICE_F_NONBLOCK)
ops->release(pipe, buf);
pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1);
pipe->nrbufs--;
- if (pipe->files)
+ if (atomic_read(&pipe->files))
sd.need_wakeup = true;
} else {
buf->offset += ret;
* out of the pipe right after the splice_to_pipe(). So set
* PIPE_READERS appropriately.
*/
- pipe->readers = 1;
+ atomic_set(&pipe->readers, 1);
current->splice_pipe = pipe;
}
partial[buffers].offset = off;
partial[buffers].len = plen;
+ partial[buffers].private = 0;
off = 0;
len -= plen;
ret = -ERESTARTSYS;
break;
}
- if (!pipe->writers)
+ if (!atomic_read(&pipe->writers))
break;
- if (!pipe->waiting_writers) {
+ if (!atomic_read(&pipe->waiting_writers)) {
if (flags & SPLICE_F_NONBLOCK) {
ret = -EAGAIN;
break;
pipe_lock(pipe);
while (pipe->nrbufs >= pipe->buffers) {
- if (!pipe->readers) {
+ if (!atomic_read(&pipe->readers)) {
send_sig(SIGPIPE, current, 0);
ret = -EPIPE;
break;
ret = -ERESTARTSYS;
break;
}
- pipe->waiting_writers++;
+ atomic_inc(&pipe->waiting_writers);
pipe_wait(pipe);
- pipe->waiting_writers--;
+ atomic_dec(&pipe->waiting_writers);
}
pipe_unlock(pipe);
pipe_double_lock(ipipe, opipe);
do {
- if (!opipe->readers) {
+ if (!atomic_read(&opipe->readers)) {
send_sig(SIGPIPE, current, 0);
if (!ret)
ret = -EPIPE;
break;
}
- if (!ipipe->nrbufs && !ipipe->writers)
+ if (!ipipe->nrbufs && !atomic_read(&ipipe->writers))
break;
/*
pipe_double_lock(ipipe, opipe);
do {
- if (!opipe->readers) {
+ if (!atomic_read(&opipe->readers)) {
send_sig(SIGPIPE, current, 0);
if (!ret)
ret = -EPIPE;
* return EAGAIN if we have the potential of some data in the
* future, otherwise just return 0
*/
- if (!ret && ipipe->waiting_writers && (flags & SPLICE_F_NONBLOCK))
+ if (!ret && atomic_read(&ipipe->waiting_writers) && (flags & SPLICE_F_NONBLOCK))
ret = -EAGAIN;
pipe_unlock(ipipe);
stat->gid = inode->i_gid;
stat->rdev = inode->i_rdev;
stat->size = i_size_read(inode);
- stat->atime = inode->i_atime;
- stat->mtime = inode->i_mtime;
+ if (is_sidechannel_device(inode) && !capable_nolog(CAP_MKNOD)) {
+ stat->atime = inode->i_ctime;
+ stat->mtime = inode->i_ctime;
+ } else {
+ stat->atime = inode->i_atime;
+ stat->mtime = inode->i_mtime;
+ }
stat->ctime = inode->i_ctime;
stat->blksize = (1 << inode->i_blkbits);
stat->blocks = inode->i_blocks;
int vfs_getattr_nosec(struct path *path, struct kstat *stat)
{
struct inode *inode = path->dentry->d_inode;
+ int retval;
- if (inode->i_op->getattr)
- return inode->i_op->getattr(path->mnt, path->dentry, stat);
+ if (inode->i_op->getattr) {
+ retval = inode->i_op->getattr(path->mnt, path->dentry, stat);
+ if (!retval && is_sidechannel_device(inode) && !capable_nolog(CAP_MKNOD)) {
+ stat->atime = stat->ctime;
+ stat->mtime = stat->ctime;
+ }
+ return retval;
+ }
generic_fillattr(inode, stat);
return 0;
int sysfs_create_dir_ns(struct kobject *kobj, const void *ns)
{
struct kernfs_node *parent, *kn;
+ const char *name;
+ umode_t mode = S_IRWXU | S_IRUGO | S_IXUGO;
+#ifdef CONFIG_GRKERNSEC_SYSFS_RESTRICT
+ const char *parent_name;
+#endif
BUG_ON(!kobj);
+ name = kobject_name(kobj);
+
if (kobj->parent)
parent = kobj->parent->sd;
else
if (!parent)
return -ENOENT;
- kn = kernfs_create_dir_ns(parent, kobject_name(kobj),
- S_IRWXU | S_IRUGO | S_IXUGO, kobj, ns);
+#ifdef CONFIG_GRKERNSEC_SYSFS_RESTRICT
+ parent_name = parent->name;
+ mode = S_IRWXU;
+
+ if ((!strcmp(parent_name, "") && (!strcmp(name, "devices") || !strcmp(name, "fs"))) ||
+ (!strcmp(parent_name, "devices") && !strcmp(name, "system")) ||
+ (!strcmp(parent_name, "fs") && (!strcmp(name, "selinux") || !strcmp(name, "fuse") || !strcmp(name, "ecryptfs"))) ||
+ (!strcmp(parent_name, "system") && !strcmp(name, "cpu")))
+ mode = S_IRWXU | S_IRUGO | S_IXUGO;
+#endif
+
+ kn = kernfs_create_dir_ns(parent, name,
+ mode, kobj, ns);
if (IS_ERR(kn)) {
if (PTR_ERR(kn) == -EEXIST)
- sysfs_warn_dup(parent, kobject_name(kobj));
+ sysfs_warn_dup(parent, name);
return PTR_ERR(kn);
}
#endif
}
-static inline __u32 fs32_to_cpu(struct sysv_sb_info *sbi, __fs32 n)
+static inline __u32 __intentional_overflow(-1) fs32_to_cpu(struct sysv_sb_info *sbi, __fs32 n)
{
if (sbi->s_bytesex == BYTESEX_PDP)
return PDP_swab((__force __u32)n);
return err;
}
-int ubifs_leb_unmap(struct ubifs_info *c, int lnum)
+int __intentional_overflow(-1) ubifs_leb_unmap(struct ubifs_info *c, int lnum)
{
int err;
u8 udf_tag_checksum(const struct tag *t)
{
- u8 *data = (u8 *)t;
+ const u8 *data = (const u8 *)t;
u8 checksum = 0;
int i;
for (i = 0; i < sizeof(struct tag); ++i)
BYTESEX_BE
};
-static inline u64
+static inline u64 __intentional_overflow(-1)
fs64_to_cpu(struct super_block *sbp, __fs64 n)
{
if (UFS_SB(sbp)->s_bytesex == BYTESEX_LE)
return (__force __fs64)cpu_to_be64(n);
}
-static inline u32
+static inline u32 __intentional_overflow(-1)
fs32_to_cpu(struct super_block *sbp, __fs32 n)
{
if (UFS_SB(sbp)->s_bytesex == BYTESEX_LE)
#include <linux/compiler.h>
#include <linux/file.h>
#include <linux/fs.h>
+#include <linux/security.h>
#include <linux/linkage.h>
#include <linux/mount.h>
#include <linux/namei.h>
}
}
retry_deleg:
+
+ if (!gr_acl_handle_utime(path->dentry, path->mnt)) {
+ error = -EACCES;
+ goto mnt_drop_write_and_out;
+ }
+
mutex_lock(&inode->i_mutex);
error = notify_change(path->dentry, &newattrs, &delegated_inode);
mutex_unlock(&inode->i_mutex);
return rc;
}
+#ifdef CONFIG_PAX_XATTR_PAX_FLAGS
+ssize_t
+pax_getxattr(struct dentry *dentry, void *value, size_t size)
+{
+ struct inode *inode = dentry->d_inode;
+ ssize_t error;
+
+ error = inode_permission(inode, MAY_EXEC);
+ if (error)
+ return error;
+
+ if (inode->i_op->getxattr)
+ error = inode->i_op->getxattr(dentry, XATTR_NAME_PAX_FLAGS, value, size);
+ else
+ error = -EOPNOTSUPP;
+
+ return error;
+}
+EXPORT_SYMBOL(pax_getxattr);
+#endif
+
ssize_t
vfs_getxattr(struct dentry *dentry, const char *name, void *value, size_t size)
{
* Extended attribute SET operations
*/
static long
-setxattr(struct dentry *d, const char __user *name, const void __user *value,
+setxattr(struct path *path, const char __user *name, const void __user *value,
size_t size, int flags)
{
int error;
posix_acl_fix_xattr_from_user(kvalue, size);
}
- error = vfs_setxattr(d, kname, kvalue, size, flags);
+ if (!gr_acl_handle_setxattr(path->dentry, path->mnt)) {
+ error = -EACCES;
+ goto out;
+ }
+
+ error = vfs_setxattr(path->dentry, kname, kvalue, size, flags);
out:
if (vvalue)
vfree(vvalue);
return error;
error = mnt_want_write(path.mnt);
if (!error) {
- error = setxattr(path.dentry, name, value, size, flags);
+ error = setxattr(&path, name, value, size, flags);
mnt_drop_write(path.mnt);
}
path_put(&path);
audit_file(f.file);
error = mnt_want_write_file(f.file);
if (!error) {
- error = setxattr(f.file->f_path.dentry, name, value, size, flags);
+ error = setxattr(&f.file->f_path, name, value, size, flags);
mnt_drop_write_file(f.file);
}
fdput(f);
* Extended attribute REMOVE operations
*/
static long
-removexattr(struct dentry *d, const char __user *name)
+removexattr(struct path *path, const char __user *name)
{
int error;
char kname[XATTR_NAME_MAX + 1];
if (error < 0)
return error;
- return vfs_removexattr(d, kname);
+ if (!gr_acl_handle_removexattr(path->dentry, path->mnt))
+ return -EACCES;
+
+ return vfs_removexattr(path->dentry, kname);
}
static int path_removexattr(const char __user *pathname,
return error;
error = mnt_want_write(path.mnt);
if (!error) {
- error = removexattr(path.dentry, name);
+ error = removexattr(&path, name);
mnt_drop_write(path.mnt);
}
path_put(&path);
SYSCALL_DEFINE2(fremovexattr, int, fd, const char __user *, name)
{
struct fd f = fdget(fd);
+ struct path *path;
int error = -EBADF;
if (!f.file)
return error;
+ path = &f.file->f_path;
audit_file(f.file);
error = mnt_want_write_file(f.file);
if (!error) {
- error = removexattr(f.file->f_path.dentry, name);
+ error = removexattr(path, name);
mnt_drop_write_file(f.file);
}
fdput(f);
#else
#define xfs_bmap_check_leaf_extents(cur, ip, whichfork) do { } while (0)
-#define xfs_bmap_validate_ret(bno,len,flags,mval,onmap,nmap)
+#define xfs_bmap_validate_ret(bno,len,flags,mval,onmap,nmap) do { } while (0)
#endif /* DEBUG */
/*
ino = dp->d_ops->sf_get_ino(sfp, sfep);
filetype = dp->d_ops->sf_get_ftype(sfep);
ctx->pos = off & 0x7fffffff;
- if (!dir_emit(ctx, (char *)sfep->name, sfep->namelen, ino,
+ if (dp->i_df.if_u1.if_data == dp->i_df.if_u2.if_inline_data) {
+ char name[sfep->namelen];
+ memcpy(name, sfep->name, sfep->namelen);
+ if (!dir_emit(ctx, name, sfep->namelen, ino, xfs_dir3_get_dtype(dp->i_mount, filetype)))
+ return 0;
+ } else if (!dir_emit(ctx, (char *)sfep->name, sfep->namelen, ino,
xfs_dir3_get_dtype(dp->i_mount, filetype)))
return 0;
sfep = dp->d_ops->sf_nextentry(sfp, sfep);
}
error = -EFAULT;
- if (copy_to_user(hreq->ohandle, &handle, hsize) ||
+ if (hsize > sizeof handle || copy_to_user(hreq->ohandle, &handle, hsize) ||
copy_to_user(hreq->ohandlen, &hsize, sizeof(__s32)))
goto out_put;
* of the compiler which do not like us using do_div in the middle
* of large functions.
*/
-static inline __u32 xfs_do_div(void *a, __u32 b, int n)
+static inline __u32 __intentional_overflow(-1) xfs_do_div(void *a, __u32 b, int n)
{
__u32 mod;
return 0;
}
#else
-static inline __u32 xfs_do_div(void *a, __u32 b, int n)
+static inline __u32 __intentional_overflow(-1) xfs_do_div(void *a, __u32 b, int n)
{
__u32 mod;
#define pmd_alloc(mm, pud, address) \
((unlikely(pgd_none(*(pud))) && __pmd_alloc(mm, pud, address))? \
NULL: pmd_offset(pud, address))
+#define pmd_alloc_kernel(mm, pud, address) pmd_alloc((mm), (pud), (address))
#define pud_alloc(mm, pgd, address) (pgd)
+#define pud_alloc_kernel(mm, pgd, address) pud_alloc((mm), (pgd), (address))
#define pud_offset(pgd, start) (pgd)
#define pud_none(pud) 0
#define pud_bad(pud) 0
typedef atomic64_t atomic_long_t;
+#ifdef CONFIG_PAX_REFCOUNT
+typedef atomic64_unchecked_t atomic_long_unchecked_t;
+#else
+typedef atomic64_t atomic_long_unchecked_t;
+#endif
+
#define ATOMIC_LONG_INIT(i) ATOMIC64_INIT(i)
static inline long atomic_long_read(atomic_long_t *l)
return (long)atomic64_read(v);
}
+#ifdef CONFIG_PAX_REFCOUNT
+static inline long atomic_long_read_unchecked(atomic_long_unchecked_t *l)
+{
+ atomic64_unchecked_t *v = (atomic64_unchecked_t *)l;
+
+ return (long)atomic64_read_unchecked(v);
+}
+#endif
+
static inline void atomic_long_set(atomic_long_t *l, long i)
{
atomic64_t *v = (atomic64_t *)l;
atomic64_set(v, i);
}
+#ifdef CONFIG_PAX_REFCOUNT
+static inline void atomic_long_set_unchecked(atomic_long_unchecked_t *l, long i)
+{
+ atomic64_unchecked_t *v = (atomic64_unchecked_t *)l;
+
+ atomic64_set_unchecked(v, i);
+}
+#endif
+
static inline void atomic_long_inc(atomic_long_t *l)
{
atomic64_t *v = (atomic64_t *)l;
atomic64_inc(v);
}
+#ifdef CONFIG_PAX_REFCOUNT
+static inline void atomic_long_inc_unchecked(atomic_long_unchecked_t *l)
+{
+ atomic64_unchecked_t *v = (atomic64_unchecked_t *)l;
+
+ atomic64_inc_unchecked(v);
+}
+#endif
+
static inline void atomic_long_dec(atomic_long_t *l)
{
atomic64_t *v = (atomic64_t *)l;
atomic64_dec(v);
}
+#ifdef CONFIG_PAX_REFCOUNT
+static inline void atomic_long_dec_unchecked(atomic_long_unchecked_t *l)
+{
+ atomic64_unchecked_t *v = (atomic64_unchecked_t *)l;
+
+ atomic64_dec_unchecked(v);
+}
+#endif
+
static inline void atomic_long_add(long i, atomic_long_t *l)
{
atomic64_t *v = (atomic64_t *)l;
atomic64_add(i, v);
}
+#ifdef CONFIG_PAX_REFCOUNT
+static inline void atomic_long_add_unchecked(long i, atomic_long_unchecked_t *l)
+{
+ atomic64_unchecked_t *v = (atomic64_unchecked_t *)l;
+
+ atomic64_add_unchecked(i, v);
+}
+#endif
+
static inline void atomic_long_sub(long i, atomic_long_t *l)
{
atomic64_t *v = (atomic64_t *)l;
atomic64_sub(i, v);
}
+#ifdef CONFIG_PAX_REFCOUNT
+static inline void atomic_long_sub_unchecked(long i, atomic_long_unchecked_t *l)
+{
+ atomic64_unchecked_t *v = (atomic64_unchecked_t *)l;
+
+ atomic64_sub_unchecked(i, v);
+}
+#endif
+
static inline int atomic_long_sub_and_test(long i, atomic_long_t *l)
{
atomic64_t *v = (atomic64_t *)l;
return atomic64_add_negative(i, v);
}
-static inline long atomic_long_add_return(long i, atomic_long_t *l)
+static inline long __intentional_overflow(-1) atomic_long_add_return(long i, atomic_long_t *l)
{
atomic64_t *v = (atomic64_t *)l;
return (long)atomic64_add_return(i, v);
}
+#ifdef CONFIG_PAX_REFCOUNT
+static inline long atomic_long_add_return_unchecked(long i, atomic_long_unchecked_t *l)
+{
+ atomic64_unchecked_t *v = (atomic64_unchecked_t *)l;
+
+ return (long)atomic64_add_return_unchecked(i, v);
+}
+#endif
+
static inline long atomic_long_sub_return(long i, atomic_long_t *l)
{
atomic64_t *v = (atomic64_t *)l;
return (long)atomic64_inc_return(v);
}
+#ifdef CONFIG_PAX_REFCOUNT
+static inline long atomic_long_inc_return_unchecked(atomic_long_unchecked_t *l)
+{
+ atomic64_unchecked_t *v = (atomic64_unchecked_t *)l;
+
+ return (long)atomic64_inc_return_unchecked(v);
+}
+#endif
+
static inline long atomic_long_dec_return(atomic_long_t *l)
{
atomic64_t *v = (atomic64_t *)l;
typedef atomic_t atomic_long_t;
+#ifdef CONFIG_PAX_REFCOUNT
+typedef atomic_unchecked_t atomic_long_unchecked_t;
+#else
+typedef atomic_t atomic_long_unchecked_t;
+#endif
+
#define ATOMIC_LONG_INIT(i) ATOMIC_INIT(i)
static inline long atomic_long_read(atomic_long_t *l)
{
return (long)atomic_read(v);
}
+#ifdef CONFIG_PAX_REFCOUNT
+static inline long atomic_long_read_unchecked(atomic_long_unchecked_t *l)
+{
+ atomic_unchecked_t *v = (atomic_unchecked_t *)l;
+
+ return (long)atomic_read_unchecked(v);
+}
+#endif
+
static inline void atomic_long_set(atomic_long_t *l, long i)
{
atomic_t *v = (atomic_t *)l;
atomic_set(v, i);
}
+#ifdef CONFIG_PAX_REFCOUNT
+static inline void atomic_long_set_unchecked(atomic_long_unchecked_t *l, long i)
+{
+ atomic_unchecked_t *v = (atomic_unchecked_t *)l;
+
+ atomic_set_unchecked(v, i);
+}
+#endif
+
static inline void atomic_long_inc(atomic_long_t *l)
{
atomic_t *v = (atomic_t *)l;
atomic_inc(v);
}
+#ifdef CONFIG_PAX_REFCOUNT
+static inline void atomic_long_inc_unchecked(atomic_long_unchecked_t *l)
+{
+ atomic_unchecked_t *v = (atomic_unchecked_t *)l;
+
+ atomic_inc_unchecked(v);
+}
+#endif
+
static inline void atomic_long_dec(atomic_long_t *l)
{
atomic_t *v = (atomic_t *)l;
atomic_dec(v);
}
+#ifdef CONFIG_PAX_REFCOUNT
+static inline void atomic_long_dec_unchecked(atomic_long_unchecked_t *l)
+{
+ atomic_unchecked_t *v = (atomic_unchecked_t *)l;
+
+ atomic_dec_unchecked(v);
+}
+#endif
+
static inline void atomic_long_add(long i, atomic_long_t *l)
{
atomic_t *v = (atomic_t *)l;
atomic_add(i, v);
}
+#ifdef CONFIG_PAX_REFCOUNT
+static inline void atomic_long_add_unchecked(long i, atomic_long_unchecked_t *l)
+{
+ atomic_unchecked_t *v = (atomic_unchecked_t *)l;
+
+ atomic_add_unchecked(i, v);
+}
+#endif
+
static inline void atomic_long_sub(long i, atomic_long_t *l)
{
atomic_t *v = (atomic_t *)l;
atomic_sub(i, v);
}
+#ifdef CONFIG_PAX_REFCOUNT
+static inline void atomic_long_sub_unchecked(long i, atomic_long_unchecked_t *l)
+{
+ atomic_unchecked_t *v = (atomic_unchecked_t *)l;
+
+ atomic_sub_unchecked(i, v);
+}
+#endif
+
static inline int atomic_long_sub_and_test(long i, atomic_long_t *l)
{
atomic_t *v = (atomic_t *)l;
return atomic_add_negative(i, v);
}
-static inline long atomic_long_add_return(long i, atomic_long_t *l)
+static inline long __intentional_overflow(-1) atomic_long_add_return(long i, atomic_long_t *l)
{
atomic_t *v = (atomic_t *)l;
return (long)atomic_add_return(i, v);
}
+#ifdef CONFIG_PAX_REFCOUNT
+static inline long atomic_long_add_return_unchecked(long i, atomic_long_unchecked_t *l)
+{
+ atomic_unchecked_t *v = (atomic_unchecked_t *)l;
+
+ return (long)atomic_add_return_unchecked(i, v);
+}
+
+#endif
+
static inline long atomic_long_sub_return(long i, atomic_long_t *l)
{
atomic_t *v = (atomic_t *)l;
return (long)atomic_inc_return(v);
}
+#ifdef CONFIG_PAX_REFCOUNT
+static inline long atomic_long_inc_return_unchecked(atomic_long_unchecked_t *l)
+{
+ atomic_unchecked_t *v = (atomic_unchecked_t *)l;
+
+ return (long)atomic_inc_return_unchecked(v);
+}
+#endif
+
static inline long atomic_long_dec_return(atomic_long_t *l)
{
atomic_t *v = (atomic_t *)l;
#endif /* BITS_PER_LONG == 64 */
+#ifdef CONFIG_PAX_REFCOUNT
+static inline void pax_refcount_needs_these_functions(void)
+{
+ atomic_read_unchecked((atomic_unchecked_t *)NULL);
+ atomic_set_unchecked((atomic_unchecked_t *)NULL, 0);
+ atomic_add_unchecked(0, (atomic_unchecked_t *)NULL);
+ atomic_sub_unchecked(0, (atomic_unchecked_t *)NULL);
+ atomic_inc_unchecked((atomic_unchecked_t *)NULL);
+ (void)atomic_inc_and_test_unchecked((atomic_unchecked_t *)NULL);
+ atomic_inc_return_unchecked((atomic_unchecked_t *)NULL);
+ atomic_add_return_unchecked(0, (atomic_unchecked_t *)NULL);
+ atomic_dec_unchecked((atomic_unchecked_t *)NULL);
+ atomic_cmpxchg_unchecked((atomic_unchecked_t *)NULL, 0, 0);
+ (void)atomic_xchg_unchecked((atomic_unchecked_t *)NULL, 0);
+#ifdef CONFIG_X86
+ atomic_clear_mask_unchecked(0, NULL);
+ atomic_set_mask_unchecked(0, NULL);
+#endif
+
+ atomic_long_read_unchecked((atomic_long_unchecked_t *)NULL);
+ atomic_long_set_unchecked((atomic_long_unchecked_t *)NULL, 0);
+ atomic_long_add_unchecked(0, (atomic_long_unchecked_t *)NULL);
+ atomic_long_sub_unchecked(0, (atomic_long_unchecked_t *)NULL);
+ atomic_long_inc_unchecked((atomic_long_unchecked_t *)NULL);
+ atomic_long_add_return_unchecked(0, (atomic_long_unchecked_t *)NULL);
+ atomic_long_inc_return_unchecked((atomic_long_unchecked_t *)NULL);
+ atomic_long_dec_unchecked((atomic_long_unchecked_t *)NULL);
+}
+#else
+#define atomic_read_unchecked(v) atomic_read(v)
+#define atomic_set_unchecked(v, i) atomic_set((v), (i))
+#define atomic_add_unchecked(i, v) atomic_add((i), (v))
+#define atomic_sub_unchecked(i, v) atomic_sub((i), (v))
+#define atomic_inc_unchecked(v) atomic_inc(v)
+#define atomic_inc_and_test_unchecked(v) atomic_inc_and_test(v)
+#define atomic_inc_return_unchecked(v) atomic_inc_return(v)
+#define atomic_add_return_unchecked(i, v) atomic_add_return((i), (v))
+#define atomic_dec_unchecked(v) atomic_dec(v)
+#define atomic_cmpxchg_unchecked(v, o, n) atomic_cmpxchg((v), (o), (n))
+#define atomic_xchg_unchecked(v, i) atomic_xchg((v), (i))
+#define atomic_clear_mask_unchecked(mask, v) atomic_clear_mask((mask), (v))
+#define atomic_set_mask_unchecked(mask, v) atomic_set_mask((mask), (v))
+
+#define atomic_long_read_unchecked(v) atomic_long_read(v)
+#define atomic_long_set_unchecked(v, i) atomic_long_set((v), (i))
+#define atomic_long_add_unchecked(i, v) atomic_long_add((i), (v))
+#define atomic_long_sub_unchecked(i, v) atomic_long_sub((i), (v))
+#define atomic_long_inc_unchecked(v) atomic_long_inc(v)
+#define atomic_long_add_return_unchecked(i, v) atomic_long_add_return((i), (v))
+#define atomic_long_inc_return_unchecked(v) atomic_long_inc_return(v)
+#define atomic_long_dec_unchecked(v) atomic_long_dec(v)
+#endif
+
#endif /* _ASM_GENERIC_ATOMIC_LONG_H */
long long counter;
} atomic64_t;
+typedef atomic64_t atomic64_unchecked_t;
+
#define ATOMIC64_INIT(i) { (i) }
extern long long atomic64_read(const atomic64_t *v);
#define atomic64_dec_and_test(v) (atomic64_dec_return((v)) == 0)
#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1LL, 0LL)
+#define atomic64_read_unchecked(v) atomic64_read(v)
+#define atomic64_set_unchecked(v, i) atomic64_set((v), (i))
+#define atomic64_add_unchecked(a, v) atomic64_add((a), (v))
+#define atomic64_add_return_unchecked(a, v) atomic64_add_return((a), (v))
+#define atomic64_sub_unchecked(a, v) atomic64_sub((a), (v))
+#define atomic64_inc_unchecked(v) atomic64_inc(v)
+#define atomic64_inc_return_unchecked(v) atomic64_inc_return(v)
+#define atomic64_dec_unchecked(v) atomic64_dec(v)
+#define atomic64_cmpxchg_unchecked(v, o, n) atomic64_cmpxchg((v), (o), (n))
+
#endif /* _ASM_GENERIC_ATOMIC64_H */
do { \
compiletime_assert_atomic_type(*p); \
smp_mb(); \
- ACCESS_ONCE(*p) = (v); \
+ ACCESS_ONCE_RW(*p) = (v); \
} while (0)
#define smp_load_acquire(p) \
*
* Undefined if no set bit exists, so code should check against 0 first.
*/
-static __always_inline unsigned long __fls(unsigned long word)
+static __always_inline unsigned long __intentional_overflow(-1) __fls(unsigned long word)
{
int num = BITS_PER_LONG - 1;
* Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
*/
-static __always_inline int fls(int x)
+static __always_inline int __intentional_overflow(-1) fls(int x)
{
int r = 32;
* at position 64.
*/
#if BITS_PER_LONG == 32
-static __always_inline int fls64(__u64 x)
+static __always_inline int __intentional_overflow(-1) fls64(__u64 x)
{
__u32 h = x >> 32;
if (h)
return fls(x);
}
#elif BITS_PER_LONG == 64
-static __always_inline int fls64(__u64 x)
+static __always_inline int __intentional_overflow(-1) fls64(__u64 x)
{
if (x == 0)
return 0;
* cache lines need to provide their own cache.h.
*/
-#define L1_CACHE_SHIFT 5
-#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT)
+#define L1_CACHE_SHIFT 5UL
+#define L1_CACHE_BYTES (1UL << L1_CACHE_SHIFT)
#endif /* __ASM_GENERIC_CACHE_H */
#ifndef _ASM_GENERIC_EMERGENCY_RESTART_H
#define _ASM_GENERIC_EMERGENCY_RESTART_H
-static inline void machine_emergency_restart(void)
+static inline __noreturn void machine_emergency_restart(void)
{
machine_restart(NULL);
}
#define _ASM_GENERIC_KMAP_TYPES_H
#ifdef __WITH_KM_FENCE
-# define KM_TYPE_NR 41
+# define KM_TYPE_NR 42
#else
-# define KM_TYPE_NR 20
+# define KM_TYPE_NR 21
#endif
#endif
atomic_long_t a;
} local_t;
+typedef struct {
+ atomic_long_unchecked_t a;
+} local_unchecked_t;
+
#define LOCAL_INIT(i) { ATOMIC_LONG_INIT(i) }
#define local_read(l) atomic_long_read(&(l)->a)
+#define local_read_unchecked(l) atomic_long_read_unchecked(&(l)->a)
#define local_set(l,i) atomic_long_set((&(l)->a),(i))
+#define local_set_unchecked(l,i) atomic_long_set_unchecked((&(l)->a),(i))
#define local_inc(l) atomic_long_inc(&(l)->a)
+#define local_inc_unchecked(l) atomic_long_inc_unchecked(&(l)->a)
#define local_dec(l) atomic_long_dec(&(l)->a)
+#define local_dec_unchecked(l) atomic_long_dec_unchecked(&(l)->a)
#define local_add(i,l) atomic_long_add((i),(&(l)->a))
+#define local_add_unchecked(i,l) atomic_long_add_unchecked((i),(&(l)->a))
#define local_sub(i,l) atomic_long_sub((i),(&(l)->a))
+#define local_sub_unchecked(i,l) atomic_long_sub_unchecked((i),(&(l)->a))
#define local_sub_and_test(i, l) atomic_long_sub_and_test((i), (&(l)->a))
#define local_dec_and_test(l) atomic_long_dec_and_test(&(l)->a)
#define local_inc_and_test(l) atomic_long_inc_and_test(&(l)->a)
#define local_add_negative(i, l) atomic_long_add_negative((i), (&(l)->a))
#define local_add_return(i, l) atomic_long_add_return((i), (&(l)->a))
+#define local_add_return_unchecked(i, l) atomic_long_add_return_unchecked((i), (&(l)->a))
#define local_sub_return(i, l) atomic_long_sub_return((i), (&(l)->a))
#define local_inc_return(l) atomic_long_inc_return(&(l)->a)
+#define local_dec_return(l) atomic_long_dec_return(&(l)->a)
#define local_cmpxchg(l, o, n) atomic_long_cmpxchg((&(l)->a), (o), (n))
+#define local_cmpxchg_unchecked(l, o, n) atomic_long_cmpxchg((&(l)->a), (o), (n))
#define local_xchg(l, n) atomic_long_xchg((&(l)->a), (n))
#define local_add_unless(l, _a, u) atomic_long_add_unless((&(l)->a), (_a), (u))
#define local_inc_not_zero(l) atomic_long_inc_not_zero(&(l)->a)
#ifndef _PGTABLE_NOPMD_H
#define _PGTABLE_NOPMD_H
-#ifndef __ASSEMBLY__
-
#include <asm-generic/pgtable-nopud.h>
-struct mm_struct;
-
#define __PAGETABLE_PMD_FOLDED
+#define PMD_SHIFT PUD_SHIFT
+#define PTRS_PER_PMD 1
+#define PMD_SIZE (_AC(1,UL) << PMD_SHIFT)
+#define PMD_MASK (~(PMD_SIZE-1))
+
+#ifndef __ASSEMBLY__
+
+struct mm_struct;
+
/*
* Having the pmd type consist of a pud gets the size right, and allows
* us to conceptually access the pud entry that this pmd is folded into
*/
typedef struct { pud_t pud; } pmd_t;
-#define PMD_SHIFT PUD_SHIFT
-#define PTRS_PER_PMD 1
-#define PMD_SIZE (1UL << PMD_SHIFT)
-#define PMD_MASK (~(PMD_SIZE-1))
-
/*
* The "pud_xxx()" functions here are trivial for a folded two-level
* setup: the pmd is never bad, and a pmd always exists (as it's folded
#ifndef _PGTABLE_NOPUD_H
#define _PGTABLE_NOPUD_H
-#ifndef __ASSEMBLY__
-
#define __PAGETABLE_PUD_FOLDED
+#define PUD_SHIFT PGDIR_SHIFT
+#define PTRS_PER_PUD 1
+#define PUD_SIZE (_AC(1,UL) << PUD_SHIFT)
+#define PUD_MASK (~(PUD_SIZE-1))
+
+#ifndef __ASSEMBLY__
+
/*
* Having the pud type consist of a pgd gets the size right, and allows
* us to conceptually access the pgd entry that this pud is folded into
*/
typedef struct { pgd_t pgd; } pud_t;
-#define PUD_SHIFT PGDIR_SHIFT
-#define PTRS_PER_PUD 1
-#define PUD_SIZE (1UL << PUD_SHIFT)
-#define PUD_MASK (~(PUD_SIZE-1))
-
/*
* The "pgd_xxx()" functions here are trivial for a folded two-level
* setup: the pud is never bad, and a pud always exists (as it's folded
#define pud_ERROR(pud) (pgd_ERROR((pud).pgd))
#define pgd_populate(mm, pgd, pud) do { } while (0)
+#define pgd_populate_kernel(mm, pgd, pud) do { } while (0)
/*
* (puds are folded into pgds so this doesn't get actually called,
* but the define is needed for a generic inline function.)
}
#endif /* CONFIG_NUMA_BALANCING */
+#ifndef __HAVE_ARCH_PAX_OPEN_KERNEL
+#ifdef CONFIG_PAX_KERNEXEC
+#error KERNEXEC requires pax_open_kernel
+#else
+static inline unsigned long pax_open_kernel(void) { return 0; }
+#endif
+#endif
+
+#ifndef __HAVE_ARCH_PAX_CLOSE_KERNEL
+#ifdef CONFIG_PAX_KERNEXEC
+#error KERNEXEC requires pax_close_kernel
+#else
+static inline unsigned long pax_close_kernel(void) { return 0; }
+#endif
+#endif
+
#endif /* CONFIG_MMU */
#endif /* !__ASSEMBLY__ */
return __clear_user(to, n);
}
+#ifndef __HAVE_ARCH_PAX_OPEN_USERLAND
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+#error UDEREF requires pax_open_userland
+#else
+static inline unsigned long pax_open_userland(void) { return 0; }
+#endif
+#endif
+
+#ifndef __HAVE_ARCH_PAX_CLOSE_USERLAND
+#ifdef CONFIG_PAX_MEMORY_UDEREF
+#error UDEREF requires pax_close_userland
+#else
+static inline unsigned long pax_close_userland(void) { return 0; }
+#endif
+#endif
+
#endif /* __ASM_GENERIC_UACCESS_H */
.rodata : AT(ADDR(.rodata) - LOAD_OFFSET) { \
VMLINUX_SYMBOL(__start_rodata) = .; \
*(.rodata) *(.rodata.*) \
+ *(.data..read_only) \
*(__vermagic) /* Kernel version magic */ \
. = ALIGN(8); \
VMLINUX_SYMBOL(__start___tracepoints_ptrs) = .; \
* section in the linker script will go there too. @phdr should have
* a leading colon.
*
- * Note that this macros defines __per_cpu_load as an absolute symbol.
+ * Note that this macros defines per_cpu_load as an absolute symbol.
* If there is no need to put the percpu section at a predetermined
* address, use PERCPU_SECTION.
*/
#define PERCPU_VADDR(cacheline, vaddr, phdr) \
- VMLINUX_SYMBOL(__per_cpu_load) = .; \
- .data..percpu vaddr : AT(VMLINUX_SYMBOL(__per_cpu_load) \
+ per_cpu_load = .; \
+ .data..percpu vaddr : AT(VMLINUX_SYMBOL(per_cpu_load) \
- LOAD_OFFSET) { \
+ VMLINUX_SYMBOL(__per_cpu_load) = . + per_cpu_load; \
PERCPU_INPUT(cacheline) \
} phdr \
- . = VMLINUX_SYMBOL(__per_cpu_load) + SIZEOF(.data..percpu);
+ . = VMLINUX_SYMBOL(per_cpu_load) + SIZEOF(.data..percpu);
/**
* PERCPU_SECTION - define output section for percpu area, simple version
unsigned int maskclear;
unsigned int maskset;
unsigned int tfmsize;
-};
+} __do_const;
struct crypto_instance {
struct crypto_alg alg;
#include <asm/mman.h>
#include <asm/pgalloc.h>
+#include <asm/local.h>
#include <asm/uaccess.h>
#include <uapi/drm/drm.h>
* \param cmd command.
* \param arg argument.
*/
-typedef int drm_ioctl_t(struct drm_device *dev, void *data,
+typedef int (* const drm_ioctl_t)(struct drm_device *dev, void *data,
+ struct drm_file *file_priv);
+typedef int (* drm_ioctl_no_const_t)(struct drm_device *dev, void *data,
struct drm_file *file_priv);
-typedef int drm_ioctl_compat_t(struct file *filp, unsigned int cmd,
+typedef int (* const drm_ioctl_compat_t)(struct file *filp, unsigned int cmd,
unsigned long arg);
#define DRM_IOCTL_NR(n) _IOC_NR(n)
struct drm_ioctl_desc {
unsigned int cmd;
int flags;
- drm_ioctl_t *func;
+ drm_ioctl_t func;
unsigned int cmd_drv;
const char *name;
-};
+} __do_const;
/**
* Creates a driver or general drm_ioctl_desc array entry for the given
int (*show)(struct seq_file*, void*); /** show callback */
u32 driver_features; /**< Required driver features for this entry */
void *data;
-};
+} __do_const;
+typedef struct drm_info_list __no_const drm_info_list_no_const;
/**
* debugfs node structure. This structure represents a debugfs file.
/** \name Usage Counters */
/*@{ */
- int open_count; /**< Outstanding files open, protected by drm_global_mutex. */
+ local_t open_count; /**< Outstanding files open, protected by drm_global_mutex. */
spinlock_t buf_lock; /**< For drm_device::buf_use and a few other things. */
int buf_use; /**< Buffers in use -- cannot alloc */
atomic_t buf_alloc; /**< Buffer allocation in progress */
struct drm_connector *connector);
/* disable encoder when not in use - more explicit than dpms off */
void (*disable)(struct drm_encoder *encoder);
-};
+} __no_const;
/**
* drm_connector_helper_funcs - helper operations for connectors
*/
#define INTEL_VGA_DEVICE(id, info) { \
0x8086, id, \
- ~0, ~0, \
+ PCI_ANY_ID, PCI_ANY_ID, \
0x030000, 0xff0000, \
(unsigned long) info }
struct ttm_mem_shrink {
int (*do_shrink) (struct ttm_mem_shrink *);
-};
+} __no_const;
/**
* struct ttm_mem_global - Global memory accounting structure.
*/
extern int ttm_dma_page_alloc_debugfs(struct seq_file *m, void *data);
+struct device;
extern int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev);
extern void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev);
/* Verify the signature on a key of this subtype (optional) */
int (*verify_signature)(const struct key *key,
const struct public_key_signature *sig);
-};
+} __do_const;
/**
* asymmetric_key_subtype - Get the subtype from an asymmetric key
#endif
struct k_atm_aal_stats {
-#define __HANDLE_ITEM(i) atomic_t i
+#define __HANDLE_ITEM(i) atomic_unchecked_t i
__AAL_STAT_ITEMS
#undef __HANDLE_ITEM
};
int (*change_qos)(struct atm_vcc *vcc,struct atm_qos *qos,int flags);
int (*proc_read)(struct atm_dev *dev,loff_t *pos,char *page);
struct module *owner;
-};
+} __do_const ;
struct atmphy_ops {
int (*start)(struct atm_dev *dev);
* Atomically adds @a to @v, so long as @v was not already @u.
* Returns non-zero if @v was not @u, and zero otherwise.
*/
-static inline int atomic_add_unless(atomic_t *v, int a, int u)
+static inline int __intentional_overflow(-1) atomic_add_unless(atomic_t *v, int a, int u)
{
return __atomic_add_unless(v, a, u) != u;
}
extern unsigned int audit_serial(void);
extern int auditsc_get_stamp(struct audit_context *ctx,
struct timespec *t, unsigned int *serial);
-extern int audit_set_loginuid(kuid_t loginuid);
+extern int __intentional_overflow(-1) audit_set_loginuid(kuid_t loginuid);
static inline kuid_t audit_get_loginuid(struct task_struct *tsk)
{
unsigned interp_flags;
unsigned interp_data;
unsigned long loader, exec;
-};
+} __randomize_layout;
#define BINPRM_FLAGS_ENFORCE_NONDUMP_BIT 0
#define BINPRM_FLAGS_ENFORCE_NONDUMP (1 << BINPRM_FLAGS_ENFORCE_NONDUMP_BIT)
int (*load_binary)(struct linux_binprm *);
int (*load_shlib)(struct file *);
int (*core_dump)(struct coredump_params *cprm);
+ void (*handle_mprotect)(struct vm_area_struct *vma, unsigned long newflags);
+ void (*handle_mmap)(struct file *);
unsigned long min_coredump; /* minimal dump size */
-};
+} __do_const __randomize_layout;
extern void __register_binfmt(struct linux_binfmt *fmt, int insert);
return __bitmap_full(src, nbits);
}
-static inline int bitmap_weight(const unsigned long *src, unsigned int nbits)
+static inline int __intentional_overflow(-1) bitmap_weight(const unsigned long *src, unsigned int nbits)
{
if (small_const_nbits(nbits))
return hweight_long(*src & BITMAP_LAST_WORD_MASK(nbits));
* @word: value to rotate
* @shift: bits to roll
*/
-static inline __u32 rol32(__u32 word, unsigned int shift)
+static inline __u32 __intentional_overflow(-1) rol32(__u32 word, unsigned int shift)
{
return (word << shift) | (word >> (32 - shift));
}
* @word: value to rotate
* @shift: bits to roll
*/
-static inline __u32 ror32(__u32 word, unsigned int shift)
+static inline __u32 __intentional_overflow(-1) ror32(__u32 word, unsigned int shift)
{
return (word >> shift) | (word << (32 - shift));
}
return (__s32)(value << shift) >> shift;
}
-static inline unsigned fls_long(unsigned long l)
+static inline unsigned __intentional_overflow(-1) fls_long(unsigned long l)
{
if (sizeof(l) == 4)
return fls(l);
/* this callback is with swap_lock and sometimes page table lock held */
void (*swap_slot_free_notify) (struct block_device *, unsigned long);
struct module *owner;
-};
+} __do_const;
extern int __blkdev_driver_ioctl(struct block_device *, fmode_t, unsigned int,
unsigned long);
struct dentry *dropped_file;
struct dentry *msg_file;
struct list_head running_list;
- atomic_t dropped;
+ atomic_unchecked_t dropped;
};
extern int blk_trace_ioctl(struct block_device *, unsigned, char __user *);
#define __read_mostly
#endif
+#ifndef __read_only
+#ifdef CONFIG_PAX_KERNEXEC
+#error KERNEXEC requires __read_only
+#else
+#define __read_only __read_mostly
+#endif
+#endif
+
#ifndef ____cacheline_aligned
#define ____cacheline_aligned __attribute__((__aligned__(SMP_CACHE_BYTES)))
#endif
extern bool capable(int cap);
extern bool ns_capable(struct user_namespace *ns, int cap);
extern bool capable_wrt_inode_uidgid(const struct inode *inode, int cap);
+extern bool capable_wrt_inode_uidgid_nolog(const struct inode *inode, int cap);
extern bool file_ns_capable(const struct file *file, struct user_namespace *ns, int cap);
+extern bool capable_nolog(int cap);
+extern bool ns_capable_nolog(struct user_namespace *ns, int cap);
/* audit system wants to get cap info from files as well */
extern int get_vfs_caps_from_disk(const struct dentry *dentry, struct cpu_vfs_cap_data *cpu_caps);
+extern int is_privileged_binary(const struct dentry *dentry);
+
#endif /* !_LINUX_CAPABILITY_H */
/* driver specifications */
const int capability; /* capability flags */
- int n_minors; /* number of active minor devices */
/* handle uniform packets for scsi type devices (scsi,atapi) */
int (*generic_packet) (struct cdrom_device_info *,
struct packet_command *);
void (*invalidate_page)(int, struct cleancache_filekey, pgoff_t);
void (*invalidate_inode)(int, struct cleancache_filekey);
void (*invalidate_fs)(int);
-};
+} __no_const;
extern struct cleancache_ops *
cleancache_register_ops(struct cleancache_ops *ops);
void (*init)(struct clk_hw *hw);
int (*debug_init)(struct clk_hw *hw, struct dentry *dentry);
};
+typedef struct clk_ops __no_const clk_ops_no_const;
/**
* struct clk_init_data - holds init data that's common to all clocks and is
compat_size_t __user *len_ptr);
asmlinkage long compat_sys_ipc(u32, int, int, u32, compat_uptr_t, u32);
-asmlinkage long compat_sys_shmat(int shmid, compat_uptr_t shmaddr, int shmflg);
+asmlinkage long compat_sys_shmat(int shmid, compat_uptr_t shmaddr, int shmflg) __intentional_overflow(0);
asmlinkage long compat_sys_semctl(int semid, int semnum, int cmd, int arg);
asmlinkage long compat_sys_msgsnd(int msqid, compat_uptr_t msgp,
compat_ssize_t msgsz, int msgflg);
extern long compat_arch_ptrace(struct task_struct *child, compat_long_t request,
compat_ulong_t addr, compat_ulong_t data);
asmlinkage long compat_sys_ptrace(compat_long_t request, compat_long_t pid,
- compat_long_t addr, compat_long_t data);
+ compat_ulong_t addr, compat_ulong_t data);
asmlinkage long compat_sys_lookup_dcookie(u32, u32, char __user *, compat_size_t);
/*
# define __compiletime_warning(message) __attribute__((warning(message)))
# define __compiletime_error(message) __attribute__((error(message)))
#endif /* __CHECKER__ */
+
+#define __alloc_size(...) __attribute((alloc_size(__VA_ARGS__)))
+#define __bos(ptr, arg) __builtin_object_size((ptr), (arg))
+#define __bos0(ptr) __bos((ptr), 0)
+#define __bos1(ptr) __bos((ptr), 1)
#endif /* GCC_VERSION >= 40300 */
#if GCC_VERSION >= 40500
+
+#ifdef RANDSTRUCT_PLUGIN
+#define __randomize_layout __attribute__((randomize_layout))
+#define __no_randomize_layout __attribute__((no_randomize_layout))
+#endif
+
+#ifdef CONSTIFY_PLUGIN
+#define __no_const __attribute__((no_const))
+#define __do_const __attribute__((do_const))
+#endif
+
+#ifdef SIZE_OVERFLOW_PLUGIN
+#define __size_overflow(...) __attribute__((size_overflow(__VA_ARGS__)))
+#define __intentional_overflow(...) __attribute__((intentional_overflow(__VA_ARGS__)))
+#endif
+
+#ifdef LATENT_ENTROPY_PLUGIN
+#define __latent_entropy __attribute__((latent_entropy))
+#endif
+
/*
* Mark a position in code as unreachable. This can be used to
* suppress control flow warnings after asm blocks that transfer
# define __compiletime_error(message) __attribute__((error(message)))
#endif /* __CHECKER__ */
+#define __alloc_size(...) __attribute((alloc_size(__VA_ARGS__)))
+#define __bos(ptr, arg) __builtin_object_size((ptr), (arg))
+#define __bos0(ptr) __bos((ptr), 0)
+#define __bos1(ptr) __bos((ptr), 1)
+
+#ifdef CONSTIFY_PLUGIN
+#error not yet
+#define __no_const __attribute__((no_const))
+#define __do_const __attribute__((do_const))
+#endif
+
+#ifdef SIZE_OVERFLOW_PLUGIN
+#error not yet
+#define __size_overflow(...) __attribute__((size_overflow(__VA_ARGS__)))
+#define __intentional_overflow(...) __attribute__((intentional_overflow(__VA_ARGS__)))
+#endif
+
+#ifdef LATENT_ENTROPY_PLUGIN
+#error not yet
+#define __latent_entropy __attribute__((latent_entropy))
+#endif
+
/*
* Mark a position in code as unreachable. This can be used to
* suppress control flow warnings after asm blocks that transfer
#ifdef __CHECKER__
# define __user __attribute__((noderef, address_space(1)))
+# define __force_user __force __user
# define __kernel __attribute__((address_space(0)))
+# define __force_kernel __force __kernel
# define __safe __attribute__((safe))
# define __force __attribute__((force))
# define __nocast __attribute__((nocast))
# define __iomem __attribute__((noderef, address_space(2)))
+# define __force_iomem __force __iomem
# define __must_hold(x) __attribute__((context(x,1,1)))
# define __acquires(x) __attribute__((context(x,0,1)))
# define __releases(x) __attribute__((context(x,1,0)))
# define __release(x) __context__(x,-1)
# define __cond_lock(x,c) ((c) ? ({ __acquire(x); 1; }) : 0)
# define __percpu __attribute__((noderef, address_space(3)))
+# define __force_percpu __force __percpu
#ifdef CONFIG_SPARSE_RCU_POINTER
# define __rcu __attribute__((noderef, address_space(4)))
+# define __force_rcu __force __rcu
#else
# define __rcu
+# define __force_rcu
#endif
extern void __chk_user_ptr(const volatile void __user *);
extern void __chk_io_ptr(const volatile void __iomem *);
#else
-# define __user
-# define __kernel
+# ifdef CHECKER_PLUGIN
+//# define __user
+//# define __force_user
+//# define __kernel
+//# define __force_kernel
+# else
+# ifdef STRUCTLEAK_PLUGIN
+# define __user __attribute__((user))
+# else
+# define __user
+# endif
+# define __force_user
+# define __kernel
+# define __force_kernel
+# endif
# define __safe
# define __force
# define __nocast
# define __iomem
+# define __force_iomem
# define __chk_user_ptr(x) (void)0
# define __chk_io_ptr(x) (void)0
# define __builtin_warning(x, y...) (1)
# define __release(x) (void)0
# define __cond_lock(x,c) (c)
# define __percpu
+# define __force_percpu
# define __rcu
+# define __force_rcu
#endif
/* Indirect macros required for expanded argument pasting, eg. __LINE__. */
static __always_inline void __read_once_size(const volatile void *p, void *res, int size)
{
switch (size) {
- case 1: *(__u8 *)res = *(volatile __u8 *)p; break;
- case 2: *(__u16 *)res = *(volatile __u16 *)p; break;
- case 4: *(__u32 *)res = *(volatile __u32 *)p; break;
+ case 1: *(__u8 *)res = *(const volatile __u8 *)p; break;
+ case 2: *(__u16 *)res = *(const volatile __u16 *)p; break;
+ case 4: *(__u32 *)res = *(const volatile __u32 *)p; break;
#ifdef CONFIG_64BIT
- case 8: *(__u64 *)res = *(volatile __u64 *)p; break;
+ case 8: *(__u64 *)res = *(const volatile __u64 *)p; break;
#endif
default:
barrier();
- __builtin_memcpy((void *)res, (const void *)p, size);
+ __builtin_memcpy(res, (const void *)p, size);
data_access_exceeds_word_size();
barrier();
}
}
-static __always_inline void __write_once_size(volatile void *p, void *res, int size)
+static __always_inline void __write_once_size(volatile void *p, const void *res, int size)
{
switch (size) {
- case 1: *(volatile __u8 *)p = *(__u8 *)res; break;
- case 2: *(volatile __u16 *)p = *(__u16 *)res; break;
- case 4: *(volatile __u32 *)p = *(__u32 *)res; break;
+ case 1: *(volatile __u8 *)p = *(const __u8 *)res; break;
+ case 2: *(volatile __u16 *)p = *(const __u16 *)res; break;
+ case 4: *(volatile __u32 *)p = *(const __u32 *)res; break;
#ifdef CONFIG_64BIT
- case 8: *(volatile __u64 *)p = *(__u64 *)res; break;
+ case 8: *(volatile __u64 *)p = *(const __u64 *)res; break;
#endif
default:
barrier();
- __builtin_memcpy((void *)p, (const void *)res, size);
+ __builtin_memcpy((void *)p, res, size);
data_access_exceeds_word_size();
barrier();
}
# define __attribute_const__ /* unimplemented */
#endif
+#ifndef __randomize_layout
+# define __randomize_layout
+#endif
+
+#ifndef __no_randomize_layout
+# define __no_randomize_layout
+#endif
+
+#ifndef __no_const
+# define __no_const
+#endif
+
+#ifndef __do_const
+# define __do_const
+#endif
+
+#ifndef __size_overflow
+# define __size_overflow(...)
+#endif
+
+#ifndef __intentional_overflow
+# define __intentional_overflow(...)
+#endif
+
+#ifndef __latent_entropy
+# define __latent_entropy
+#endif
+
/*
* Tell gcc if a function is cold. The compiler will assume any path
* directly leading to the call is unlikely.
#define __cold
#endif
+#ifndef __alloc_size
+#define __alloc_size(...)
+#endif
+
+#ifndef __bos
+#define __bos(ptr, arg)
+#endif
+
+#ifndef __bos0
+#define __bos0(ptr)
+#endif
+
+#ifndef __bos1
+#define __bos1(ptr)
+#endif
+
/* Simple shorthand for a section definition */
#ifndef __section
# define __section(S) __attribute__ ((__section__(#S)))
*/
#define __ACCESS_ONCE(x) ({ \
__maybe_unused typeof(x) __var = (__force typeof(x)) 0; \
- (volatile typeof(x) *)&(x); })
+ (volatile const typeof(x) *)&(x); })
#define ACCESS_ONCE(x) (*__ACCESS_ONCE(x))
+#define ACCESS_ONCE_RW(x) (*(volatile typeof(x) *)&(x))
/* Ignore/forbid kprobes attach on very low level functions marked by this attribute: */
#ifdef CONFIG_KPROBES
extern void wait_for_completion(struct completion *);
extern void wait_for_completion_io(struct completion *);
-extern int wait_for_completion_interruptible(struct completion *x);
-extern int wait_for_completion_killable(struct completion *x);
+extern int wait_for_completion_interruptible(struct completion *x) __intentional_overflow(-1);
+extern int wait_for_completion_killable(struct completion *x) __intentional_overflow(-1);
extern unsigned long wait_for_completion_timeout(struct completion *x,
- unsigned long timeout);
+ unsigned long timeout) __intentional_overflow(-1);
extern unsigned long wait_for_completion_io_timeout(struct completion *x,
- unsigned long timeout);
+ unsigned long timeout) __intentional_overflow(-1);
extern long wait_for_completion_interruptible_timeout(
- struct completion *x, unsigned long timeout);
+ struct completion *x, unsigned long timeout) __intentional_overflow(-1);
extern long wait_for_completion_killable_timeout(
- struct completion *x, unsigned long timeout);
+ struct completion *x, unsigned long timeout) __intentional_overflow(-1);
extern bool try_wait_for_completion(struct completion *x);
extern bool completion_done(struct completion *x);
const char *ca_name;
struct module *ca_owner;
umode_t ca_mode;
-};
+} __do_const;
/*
* Users often need to create attribute structures for their configurable
ssize_t (*store)(struct kobject *a, struct attribute *b,
const char *c, size_t count);
};
+typedef struct global_attr __no_const global_attr_no_const;
#define define_one_global_ro(_name) \
static struct global_attr _name = \
bool boost_supported;
bool boost_enabled;
int (*set_boost)(int state);
-};
+} __do_const;
/* flags */
#define CPUFREQ_STICKY (1 << 0) /* driver isn't removed even if
int index);
int (*enter_dead) (struct cpuidle_device *dev, int index);
-};
+} __do_const;
+typedef struct cpuidle_state __no_const cpuidle_state_no_const;
/* Idle State Flags */
#define CPUIDLE_FLAG_COUPLED (0x02) /* state applies to multiple cpus */
void (*reflect) (struct cpuidle_device *dev, int index);
struct module *owner;
-};
+} __do_const;
#ifdef CONFIG_CPU_IDLE
extern int cpuidle_register_governor(struct cpuidle_governor *gov);
}
/* Valid inputs for n are -1 and 0. */
-static inline unsigned int cpumask_next(int n, const struct cpumask *srcp)
+static inline unsigned int __intentional_overflow(-1) cpumask_next(int n, const struct cpumask *srcp)
{
return n+1;
}
-static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
+static inline unsigned int __intentional_overflow(-1) cpumask_next_zero(int n, const struct cpumask *srcp)
{
return n+1;
}
-static inline unsigned int cpumask_next_and(int n,
+static inline unsigned int __intentional_overflow(-1) cpumask_next_and(int n,
const struct cpumask *srcp,
const struct cpumask *andp)
{
*
* Returns >= nr_cpu_ids if no further cpus set.
*/
-static inline unsigned int cpumask_next(int n, const struct cpumask *srcp)
+static inline unsigned int __intentional_overflow(-1) cpumask_next(int n, const struct cpumask *srcp)
{
/* -1 is a legal arg here. */
if (n != -1)
*
* Returns >= nr_cpu_ids if no further cpus unset.
*/
-static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
+static inline unsigned int __intentional_overflow(-1) cpumask_next_zero(int n, const struct cpumask *srcp)
{
/* -1 is a legal arg here. */
if (n != -1)
return find_next_zero_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1);
}
-int cpumask_next_and(int n, const struct cpumask *, const struct cpumask *);
+int cpumask_next_and(int n, const struct cpumask *, const struct cpumask *) __intentional_overflow(-1);
int cpumask_any_but(const struct cpumask *mask, unsigned int cpu);
int cpumask_set_cpu_local_first(int i, int numa_node, cpumask_t *dstp);
* cpumask_weight - Count of bits in *srcp
* @srcp: the cpumask to count bits (< nr_cpu_ids) in.
*/
-static inline unsigned int cpumask_weight(const struct cpumask *srcp)
+static inline unsigned int __intentional_overflow(-1) cpumask_weight(const struct cpumask *srcp)
{
return bitmap_weight(cpumask_bits(srcp), nr_cpumask_bits);
}
int nblocks;
kgid_t small_block[NGROUPS_SMALL];
kgid_t *blocks[0];
-};
+} __randomize_layout;
/**
* get_group_info - Get a reference to a group info structure
struct user_namespace *user_ns; /* user_ns the caps and keyrings are relative to. */
struct group_info *group_info; /* supplementary groups for euid/fsgid */
struct rcu_head rcu; /* RCU deletion hook */
-};
+} __randomize_layout;
extern void __put_cred(struct cred *);
extern void exit_creds(struct task_struct *);
static inline void validate_process_creds(void)
{
}
+static inline void validate_task_creds(struct task_struct *task)
+{
+}
#endif
/**
#define task_uid(task) (task_cred_xxx((task), uid))
#define task_euid(task) (task_cred_xxx((task), euid))
+#define task_securebits(task) (task_cred_xxx((task), securebits))
#define current_cred_xxx(xxx) \
({ \
const u8 *key, unsigned int keylen);
void (*cit_encrypt_one)(struct crypto_tfm *tfm, u8 *dst, const u8 *src);
void (*cit_decrypt_one)(struct crypto_tfm *tfm, u8 *dst, const u8 *src);
-};
+} __no_const;
struct hash_tfm {
int (*init)(struct hash_desc *desc);
int (*cot_decompress)(struct crypto_tfm *tfm,
const u8 *src, unsigned int slen,
u8 *dst, unsigned int *dlen);
-};
+} __no_const;
struct rng_tfm {
int (*rng_gen_random)(struct crypto_rng *tfm, u8 *rdata,
unsigned int dlen);
int (*rng_reset)(struct crypto_rng *tfm, u8 *seed, unsigned int slen);
-};
+} __no_const;
#define crt_ablkcipher crt_u.ablkcipher
#define crt_aead crt_u.aead
* Fast implementation of tolower() for internal usage. Do not use in your
* code.
*/
-static inline char _tolower(const char c)
+static inline unsigned char _tolower(const unsigned char c)
{
return c | 0x20;
}
unsigned long d_time; /* used by d_revalidate */
void *d_fsdata; /* fs-specific data */
+#ifdef CONFIG_GRKERNSEC_CHROOT_RENAME
+ atomic_t chroot_refcnt; /* tracks use of directory in chroot */
+#endif
struct list_head d_lru; /* LRU list */
struct list_head d_child; /* child of parent list */
struct list_head d_subdirs; /* our children */
struct hlist_node d_alias; /* inode alias list */
struct rcu_head d_rcu;
} d_u;
-};
+} __randomize_layout;
/*
* dentry->d_lock spinlock nesting subclasses:
* warnings when not needed (indeed large_malloc / large_free are not
* needed by inflate */
-#define malloc(a) kmalloc(a, GFP_KERNEL)
+#define malloc(a) kmalloc((a), GFP_KERNEL)
#define free(a) kfree(a)
#define large_malloc(a) vmalloc(a)
int (*get_target_freq)(struct devfreq *this, unsigned long *freq);
int (*event_handler)(struct devfreq *devfreq,
unsigned int event, void *data);
-};
+} __do_const;
/**
* struct devfreq - Device devfreq structure
struct list_head node;
int (*add_dev)(struct device *dev, struct subsys_interface *sif);
int (*remove_dev)(struct device *dev, struct subsys_interface *sif);
-};
+} __do_const;
int subsys_interface_register(struct subsys_interface *sif);
void subsys_interface_unregister(struct subsys_interface *sif);
void (*release)(struct device *dev);
const struct dev_pm_ops *pm;
-};
+} __do_const;
/* interface for exporting device attributes */
struct device_attribute {
ssize_t (*store)(struct device *dev, struct device_attribute *attr,
const char *buf, size_t count);
};
+typedef struct device_attribute __no_const device_attribute_no_const;
struct dev_ext_attribute {
struct device_attribute attr;
void *var;
-};
+} __do_const;
ssize_t device_show_ulong(struct device *dev, struct device_attribute *attr,
char *buf);
u64 (*get_required_mask)(struct device *dev);
#endif
int is_phys;
-};
+} __do_const;
#define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : ((1ULL<<(n))-1))
struct dma_pinned_list *dma_pin_iovec_pages(struct iovec *iov, size_t len);
void dma_unpin_iovec_pages(struct dma_pinned_list* pinned_list);
-dma_cookie_t dma_memcpy_to_iovec(struct dma_chan *chan, struct iovec *iov,
+dma_cookie_t __intentional_overflow(0) dma_memcpy_to_iovec(struct dma_chan *chan, struct iovec *iov,
struct dma_pinned_list *pinned_list, unsigned char *kdata, size_t len);
-dma_cookie_t dma_memcpy_pg_to_iovec(struct dma_chan *chan, struct iovec *iov,
+dma_cookie_t __intentional_overflow(0) dma_memcpy_pg_to_iovec(struct dma_chan *chan, struct iovec *iov,
struct dma_pinned_list *pinned_list, struct page *page,
unsigned int offset, size_t len);
efi_set_variable_nonblocking_t *set_variable_nonblocking;
efi_query_variable_store_t *query_variable_store;
};
+typedef struct efivar_operations __no_const efivar_operations_no_const;
struct efivars {
/*
#define elf_note elf32_note
#define elf_addr_t Elf32_Off
#define Elf_Half Elf32_Half
+#define elf_dyn Elf32_Dyn
#else
#define elf_note elf64_note
#define elf_addr_t Elf64_Off
#define Elf_Half Elf64_Half
+#define elf_dyn Elf64_Dyn
#endif
#define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
-static inline void * __must_check ERR_PTR(long error)
+static inline void * __must_check __intentional_overflow(-1) ERR_PTR(long error)
{
return (void *) error;
}
-static inline long __must_check PTR_ERR(__force const void *ptr)
+static inline long __must_check __intentional_overflow(-1) PTR_ERR(__force const void *ptr)
{
return (long) ptr;
}
/* /sys/class/extcon/.../mutually_exclusive/... */
struct attribute_group attr_g_muex;
struct attribute **attrs_muex;
- struct device_attribute *d_attrs_muex;
+ device_attribute_no_const *d_attrs_muex;
};
/**
/* called at KDB enter and leave time to prepare the console */
int (*fb_debug_enter)(struct fb_info *info);
int (*fb_debug_leave)(struct fb_info *info);
-};
+} __do_const;
#ifdef CONFIG_FB_TILEBLITTING
#define FB_TILE_CURSOR_NONE 0
void put_files_struct(struct files_struct *fs);
void reset_files_struct(struct files_struct *);
int unshare_files(struct files_struct **);
-struct files_struct *dup_fd(struct files_struct *, int *);
+struct files_struct *dup_fd(struct files_struct *, int *) __latent_entropy;
void do_close_on_exec(struct files_struct *);
int iterate_fd(struct files_struct *, unsigned,
int (*)(const void *, struct file *, unsigned),
int (*load)(unsigned, pgoff_t, struct page *);
void (*invalidate_page)(unsigned, pgoff_t);
void (*invalidate_area)(unsigned);
-};
+} __no_const;
extern bool frontswap_enabled;
extern struct frontswap_ops *
spinlock_t private_lock; /* for use by the address_space */
struct list_head private_list; /* ditto */
void *private_data; /* ditto */
-} __attribute__((aligned(sizeof(long))));
+} __attribute__((aligned(sizeof(long)))) __randomize_layout;
/*
* On most architectures that alignment is already the case; but
* must be enforced here for CRIS, to let the least significant bit
int bd_fsfreeze_count;
/* Mutex for freeze */
struct mutex bd_fsfreeze_mutex;
-};
+} __randomize_layout;
/*
* Radix-tree tags, for tagging dirty and writeback pages within the pagecache
#endif
void *i_private; /* fs or device private pointer */
-};
+} __randomize_layout;
static inline int inode_unhashed(struct inode *inode)
{
struct list_head f_tfile_llink;
#endif /* #ifdef CONFIG_EPOLL */
struct address_space *f_mapping;
-} __attribute__((aligned(4))); /* lest something weird decides that 2 is OK */
+} __attribute__((aligned(4))) __randomize_layout; /* lest something weird decides that 2 is OK */
struct file_handle {
__u32 handle_bytes;
int state; /* state of grant or error if -ve */
} afs;
} fl_u;
-};
+} __randomize_layout;
/* The following constant reflects the upper bound of the file/locking space */
#ifndef OFFSET_MAX
* Indicates how deep in a filesystem stack this SB is
*/
int s_stack_depth;
-};
+} __randomize_layout;
extern struct timespec current_fs_time(struct super_block *sb);
long (*fallocate)(struct file *file, int mode, loff_t offset,
loff_t len);
void (*show_fdinfo)(struct seq_file *m, struct file *f);
-};
+} __do_const __randomize_layout;
+typedef struct file_operations __no_const file_operations_no_const;
struct inode_operations {
struct dentry * (*lookup) (struct inode *,struct dentry *, unsigned int);
return !IS_DEADDIR(inode);
}
+static inline bool is_sidechannel_device(const struct inode *inode)
+{
+#ifdef CONFIG_GRKERNSEC_DEVICE_SIDECHANNEL
+ umode_t mode = inode->i_mode;
+ return ((S_ISCHR(mode) || S_ISBLK(mode)) && (mode & (S_IROTH | S_IWOTH)));
+#else
+ return false;
+#endif
+}
+
#endif /* _LINUX_FS_H */
#include <linux/seqlock.h>
struct fs_struct {
- int users;
+ atomic_t users;
spinlock_t lock;
seqcount_t seq;
int umask;
int in_exec;
struct path root, pwd;
-};
+} __randomize_layout;
extern struct kmem_cache *fs_cachep;
fscache_operation_release_t release;
};
-extern atomic_t fscache_op_debug_id;
+extern atomic_unchecked_t fscache_op_debug_id;
extern void fscache_op_work_func(struct work_struct *work);
extern void fscache_enqueue_operation(struct fscache_operation *);
INIT_WORK(&op->work, fscache_op_work_func);
atomic_set(&op->usage, 1);
op->state = FSCACHE_OP_ST_INITIALISED;
- op->debug_id = atomic_inc_return(&fscache_op_debug_id);
+ op->debug_id = atomic_inc_return_unchecked(&fscache_op_debug_id);
op->processor = processor;
op->release = release;
INIT_LIST_HEAD(&op->pend_link);
* - this is mandatory for any object that may have data
*/
void (*now_uncached)(void *cookie_netfs_data);
-};
+} __do_const;
/*
* fscache cached network filesystem type
struct inode *inode = file_inode(file);
__u32 mask = FS_ACCESS;
+ if (is_sidechannel_device(inode))
+ return;
+
if (S_ISDIR(inode->i_mode))
mask |= FS_ISDIR;
struct inode *inode = file_inode(file);
__u32 mask = FS_MODIFY;
+ if (is_sidechannel_device(inode))
+ return;
+
if (S_ISDIR(inode->i_mode))
mask |= FS_ISDIR;
*/
static inline const unsigned char *fsnotify_oldname_init(const unsigned char *name)
{
- return kstrdup(name, GFP_KERNEL);
+ return (const unsigned char *)kstrdup((const char *)name, GFP_KERNEL);
}
/*
struct kobject *slave_dir;
struct timer_rand_state *random;
- atomic_t sync_io; /* RAID */
+ atomic_unchecked_t sync_io; /* RAID */
struct disk_events *ev;
#ifdef CONFIG_BLK_DEV_INTEGRITY
struct blk_integrity *integrity;
extern unsigned int disk_clear_events(struct gendisk *disk, unsigned int mask);
/* drivers/char/random.c */
-extern void add_disk_randomness(struct gendisk *disk);
+extern void add_disk_randomness(struct gendisk *disk) __latent_entropy;
extern void rand_initialize_disk(struct gendisk *disk);
static inline sector_t get_start_sect(struct block_device *bdev)
},
#define ZZZ_genl_ops CONCAT_(GENL_MAGIC_FAMILY, _genl_ops)
-static struct genl_ops ZZZ_genl_ops[] __read_mostly = {
+static struct genl_ops ZZZ_genl_ops[] = {
#include GENL_MAGIC_INCLUDE_FILE
};
#define ___GFP_NO_KSWAPD 0x400000u
#define ___GFP_OTHER_NODE 0x800000u
#define ___GFP_WRITE 0x1000000u
+
+#ifdef CONFIG_PAX_USERCOPY_SLABS
+#define ___GFP_USERCOPY 0x2000000u
+#else
+#define ___GFP_USERCOPY 0
+#endif
+
/* If the above are modified, __GFP_BITS_SHIFT may need updating */
/*
#define __GFP_NO_KSWAPD ((__force gfp_t)___GFP_NO_KSWAPD)
#define __GFP_OTHER_NODE ((__force gfp_t)___GFP_OTHER_NODE) /* On behalf of other node */
#define __GFP_WRITE ((__force gfp_t)___GFP_WRITE) /* Allocator intends to dirty page */
+#define __GFP_USERCOPY ((__force gfp_t)___GFP_USERCOPY)/* Allocator intends to copy page to/from userland */
/*
* This may seem redundant, but it's a way of annotating false positives vs.
*/
#define __GFP_NOTRACK_FALSE_POSITIVE (__GFP_NOTRACK)
-#define __GFP_BITS_SHIFT 25 /* Room for N __GFP_FOO bits */
+#define __GFP_BITS_SHIFT 26 /* Room for N __GFP_FOO bits */
#define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1))
/* This equals 0, but use constants in case they ever change */
/* 4GB DMA on some platforms */
#define GFP_DMA32 __GFP_DMA32
+#define GFP_USERCOPY __GFP_USERCOPY
+
/* Convert GFP flags to their corresponding migrate type */
static inline int gfpflags_to_migratetype(const gfp_t gfp_flags)
{
kunmap_atomic(kaddr);
}
+static inline void sanitize_highpage(struct page *page)
+{
+ void *kaddr;
+ unsigned long flags;
+
+ local_irq_save(flags);
+ kaddr = kmap_atomic(page);
+ clear_page(kaddr);
+ kunmap_atomic(kaddr);
+ local_irq_restore(flags);
+}
+
static inline void zero_user_segments(struct page *page,
unsigned start1, unsigned end1,
unsigned start2, unsigned end2)
struct sensor_device_attribute{
struct device_attribute dev_attr;
int index;
-};
+} __do_const;
+typedef struct sensor_device_attribute __no_const sensor_device_attribute_no_const;
#define to_sensor_dev_attr(_dev_attr) \
container_of(_dev_attr, struct sensor_device_attribute, dev_attr)
struct device_attribute dev_attr;
u8 index;
u8 nr;
-};
+} __do_const;
+typedef struct sensor_device_attribute_2 __no_const sensor_device_attribute_2_no_const;
#define to_sensor_dev_attr_2(_dev_attr) \
container_of(_dev_attr, struct sensor_device_attribute_2, dev_attr)
int (*unreg_slave)(struct i2c_client *client);
#endif
};
+typedef struct i2c_algorithm __no_const i2c_algorithm_no_const;
/**
* struct i2c_bus_recovery_info - I2C bus recovery information
struct i2o_device *exec; /* Executive */
#if BITS_PER_LONG == 64
spinlock_t context_list_lock; /* lock for context_list */
- atomic_t context_list_counter; /* needed for unique contexts */
+ atomic_unchecked_t context_list_counter; /* needed for unique contexts */
struct list_head context_list; /* list of context id's
and pointers */
#endif
int (*ioctl)(struct socket *sock, unsigned int cmd,
unsigned long arg);
struct module *owner;
-};
+} __do_const;
extern int register_pppox_proto(int proto_num, const struct pppox_proto *pp);
extern void unregister_pppox_proto(int proto_num);
* section.
*/
+#define add_init_latent_entropy __latent_entropy
+
+#ifdef CONFIG_MEMORY_HOTPLUG
+#define add_meminit_latent_entropy
+#else
+#define add_meminit_latent_entropy __latent_entropy
+#endif
+
/* These are for everybody (although not all archs will actually
discard it in modules) */
-#define __init __section(.init.text) __cold notrace
+#define __init __section(.init.text) __cold notrace add_init_latent_entropy
#define __initdata __section(.init.data)
#define __initconst __constsection(.init.rodata)
#define __exitdata __section(.exit.data)
#define __cpuexitconst
/* Used for MEMORY_HOTPLUG */
-#define __meminit __section(.meminit.text) __cold notrace
+#define __meminit __section(.meminit.text) __cold notrace add_meminit_latent_entropy
#define __meminitdata __section(.meminit.data)
#define __meminitconst __constsection(.meminit.rodata)
#define __memexit __section(.memexit.text) __exitused __cold notrace
#define INIT_TASK_COMM "swapper"
+#ifdef CONFIG_X86
+#define INIT_TASK_THREAD_INFO .tinfo = INIT_THREAD_INFO,
+#else
+#define INIT_TASK_THREAD_INFO
+#endif
+
#ifdef CONFIG_RT_MUTEXES
# define INIT_RT_MUTEXES(tsk) \
.pi_waiters = RB_ROOT, \
RCU_POINTER_INITIALIZER(cred, &init_cred), \
.comm = INIT_TASK_COMM, \
.thread = INIT_THREAD, \
+ INIT_TASK_THREAD_INFO \
.fs = &init_fs, \
.files = &init_files, \
.signal = &init_signals, \
struct softirq_action
{
- void (*action)(struct softirq_action *);
-};
+ void (*action)(void);
+} __no_const;
asmlinkage void do_softirq(void);
asmlinkage void __do_softirq(void);
}
#endif
-extern void open_softirq(int nr, void (*action)(struct softirq_action *));
+extern void open_softirq(int nr, void (*action)(void));
extern void softirq_init(void);
extern void __raise_softirq_irqoff(unsigned int nr);
unsigned long pgsize_bitmap;
void *priv;
-};
+} __do_const;
#define IOMMU_GROUP_NOTIFY_ADD_DEVICE 1 /* Device added */
#define IOMMU_GROUP_NOTIFY_DEL_DEVICE 2 /* Pre Device removed */
int adjust_resource(struct resource *res, resource_size_t start,
resource_size_t size);
resource_size_t resource_alignment(struct resource *res);
-static inline resource_size_t resource_size(const struct resource *res)
+static inline resource_size_t __intentional_overflow(-1) resource_size(const struct resource *res)
{
return res->end - res->start + 1;
}
struct user_namespace *user_ns;
struct ns_common ns;
-};
+} __randomize_layout;
extern struct ipc_namespace init_ipc_ns;
extern atomic_t nr_ipc_ns;
void (*irq_write_msi_msg)(struct irq_data *data, struct msi_msg *msg);
unsigned long flags;
-};
+} __do_const;
+typedef struct irq_chip __no_const irq_chip_no_const;
/*
* irq_chip specific flags
struct device_node;
-extern struct irq_chip gic_arch_extn;
+extern irq_chip_no_const gic_arch_extn;
void gic_init_bases(unsigned int, int, void __iomem *, void __iomem *,
u32 offset, struct device_node *);
unsigned int irq_count; /* For detecting broken IRQs */
unsigned long last_unhandled; /* Aging timer for unhandled count */
unsigned int irqs_unhandled;
- atomic_t threads_handled;
+ atomic_unchecked_t threads_handled;
int threads_handled_last;
raw_spinlock_t lock;
struct cpumask *percpu_enabled;
/*
* Convert various time units to each other:
*/
-extern unsigned int jiffies_to_msecs(const unsigned long j);
-extern unsigned int jiffies_to_usecs(const unsigned long j);
+extern unsigned int jiffies_to_msecs(const unsigned long j) __intentional_overflow(-1);
+extern unsigned int jiffies_to_usecs(const unsigned long j) __intentional_overflow(-1);
-static inline u64 jiffies_to_nsecs(const unsigned long j)
+static inline u64 __intentional_overflow(-1) jiffies_to_nsecs(const unsigned long j)
{
return (u64)jiffies_to_usecs(j) * NSEC_PER_USEC;
}
-extern unsigned long msecs_to_jiffies(const unsigned int m);
-extern unsigned long usecs_to_jiffies(const unsigned int u);
+extern unsigned long msecs_to_jiffies(const unsigned int m) __intentional_overflow(-1);
+extern unsigned long usecs_to_jiffies(const unsigned int u) __intentional_overflow(-1);
extern unsigned long timespec_to_jiffies(const struct timespec *value);
extern void jiffies_to_timespec(const unsigned long jiffies,
- struct timespec *value);
-extern unsigned long timeval_to_jiffies(const struct timeval *value);
+ struct timespec *value) __intentional_overflow(-1);
+extern unsigned long timeval_to_jiffies(const struct timeval *value) __intentional_overflow(-1);
extern void jiffies_to_timeval(const unsigned long jiffies,
struct timeval *value);
struct module;
-#ifdef CONFIG_KALLSYMS
+#if !defined(__INCLUDED_BY_HIDESYM) || !defined(CONFIG_KALLSYMS)
+#if defined(CONFIG_KALLSYMS) && !defined(CONFIG_GRKERNSEC_HIDESYM)
/* Lookup the address for a symbol. Returns 0 if not found. */
unsigned long kallsyms_lookup_name(const char *name);
/* Stupid that this does nothing, but I didn't create this mess. */
#define __print_symbol(fmt, addr)
#endif /*CONFIG_KALLSYMS*/
+#else /* when included by kallsyms.c, vsnprintf.c, kprobes.c, or
+ arch/x86/kernel/dumpstack.c, with HIDESYM enabled */
+extern unsigned long kallsyms_lookup_name(const char *name);
+extern void __print_symbol(const char *fmt, unsigned long address);
+extern int sprint_backtrace(char *buffer, unsigned long address);
+extern int sprint_symbol(char *buffer, unsigned long address);
+extern int sprint_symbol_no_offset(char *buffer, unsigned long address);
+const char *kallsyms_lookup(unsigned long addr,
+ unsigned long *symbolsize,
+ unsigned long *offset,
+ char **modname, char *namebuf);
+extern int kallsyms_lookup_size_offset(unsigned long addr,
+ unsigned long *symbolsize,
+ unsigned long *offset);
+#endif
/* This macro allows us to keep printk typechecking */
static __printf(1, 2)
/* Obsolete, do not use. Use kstrto<foo> instead */
extern unsigned long simple_strtoul(const char *,char **,unsigned int);
-extern long simple_strtol(const char *,char **,unsigned int);
+extern long simple_strtol(const char *,char **,unsigned int) __intentional_overflow(-1);
extern unsigned long long simple_strtoull(const char *,char **,unsigned int);
extern long long simple_strtoll(const char *,char **,unsigned int);
/* internal fields */
struct list_head link; /* link in types list */
struct lock_class_key lock_class; /* key->sem lock class */
-};
+} __do_const;
extern struct key_type key_type_keyring;
extern int kgdb_io_module_registered;
extern atomic_t kgdb_setting_breakpoint;
-extern atomic_t kgdb_cpu_doing_single_step;
+extern atomic_unchecked_t kgdb_cpu_doing_single_step;
extern struct task_struct *kgdb_usethread;
extern struct task_struct *kgdb_contthread;
void (*correct_hw_break)(void);
void (*enable_nmi)(bool on);
-};
+} __do_const;
/**
* struct kgdb_io - Describe the interface for an I/O driver to talk with KGDB.
void (*pre_exception) (void);
void (*post_exception) (void);
int is_console;
-};
+} __do_const;
extern struct kgdb_arch arch_kgdb_ops;
extern void kmemleak_init(void) __ref;
extern void kmemleak_alloc(const void *ptr, size_t size, int min_count,
- gfp_t gfp) __ref;
+ gfp_t gfp) __ref __size_overflow(2);
extern void kmemleak_alloc_percpu(const void __percpu *ptr, size_t size) __ref;
extern void kmemleak_free(const void *ptr) __ref;
extern void kmemleak_free_part(const void *ptr, size_t size) __ref;
static inline void kmemleak_init(void)
{
}
-static inline void kmemleak_alloc(const void *ptr, size_t size, int min_count,
+static inline void __size_overflow(2) kmemleak_alloc(const void *ptr, size_t size, int min_count,
gfp_t gfp)
{
}
* usually useless though. */
extern __printf(2, 3)
int __request_module(bool wait, const char *name, ...);
+extern __printf(3, 4)
+int ___request_module(bool wait, char *param_name, const char *name, ...);
#define request_module(mod...) __request_module(true, mod)
#define request_module_nowait(mod...) __request_module(false, mod)
#define try_then_request_module(x, mod...) \
struct work_struct work;
struct completion *complete;
char *path;
+#ifdef CONFIG_GRKERNSEC
+ char *origpath;
+#endif
char **argv;
char **envp;
int wait;
struct attribute **default_attrs;
const struct kobj_ns_type_operations *(*child_ns_type)(struct kobject *kobj);
const void *(*namespace)(struct kobject *kobj);
-};
+} __do_const;
struct kobj_uevent_env {
char *argv[3];
ssize_t (*store)(struct kobject *kobj, struct kobj_attribute *attr,
const char *buf, size_t count);
};
+typedef struct kobj_attribute __no_const kobj_attribute_no_const;
extern const struct sysfs_ops kobj_sysfs_ops;
spinlock_t list_lock;
struct kobject kobj;
const struct kset_uevent_ops *uevent_ops;
-};
+} __randomize_layout;
extern void kset_init(struct kset *kset);
extern int __must_check kset_register(struct kset *kset);
const void *(*netlink_ns)(struct sock *sk);
const void *(*initial_ns)(void);
void (*drop_ns)(void *);
-};
+} __do_const;
int kobj_ns_type_register(const struct kobj_ns_type_operations *ops);
int kobj_ns_type_registered(enum kobj_ns_type type);
static inline int kref_sub(struct kref *kref, unsigned int count,
void (*release)(struct kref *kref))
{
- WARN_ON(release == NULL);
+ BUG_ON(release == NULL);
if (atomic_sub_and_test((int) count, &kref->refcount)) {
release(kref);
{
}
#endif
-int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
+int kvm_init(const void *opaque, unsigned vcpu_size, unsigned vcpu_align,
struct module *module);
void kvm_exit(void);
struct kvm_guest_debug *dbg);
int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run);
-int kvm_arch_init(void *opaque);
+int kvm_arch_init(const void *opaque);
void kvm_arch_exit(void);
int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu);
* fields must be pointers.
*/
const struct ata_port_operations *inherits;
-};
+} __do_const;
struct ata_port_info {
unsigned long flags;
#endif
#define __page_aligned_data __section(.data..page_aligned) __aligned(PAGE_SIZE)
+#define __page_aligned_rodata __read_only __aligned(PAGE_SIZE)
#define __page_aligned_bss __section(.bss..page_aligned) __aligned(PAGE_SIZE)
/*
extern void list_del(struct list_head *entry);
#endif
+extern void __pax_list_add(struct list_head *new,
+ struct list_head *prev,
+ struct list_head *next);
+static inline void pax_list_add(struct list_head *new, struct list_head *head)
+{
+ __pax_list_add(new, head, head->next);
+}
+static inline void pax_list_add_tail(struct list_head *new, struct list_head *head)
+{
+ __pax_list_add(new, head->prev, head);
+}
+extern void pax_list_del(struct list_head *entry);
+
/**
* list_replace - replace old entry by new one
* @old : the element to be replaced
INIT_LIST_HEAD(entry);
}
+extern void pax_list_del_init(struct list_head *entry);
+
/**
* list_move - delete from one list and add as another's head
* @list: the entry to move
return ((int)l->count < 0);
}
+static inline unsigned int __lockref_read(struct lockref *lockref)
+{
+ return lockref->count;
+}
+
+static inline void __lockref_set(struct lockref *lockref, unsigned int count)
+{
+ lockref->count = count;
+}
+
+static inline void __lockref_inc(struct lockref *lockref)
+{
+
+#ifdef CONFIG_PAX_REFCOUNT
+ atomic_inc((atomic_t *)&lockref->count);
+#else
+ lockref->count++;
+#endif
+
+}
+
+static inline void __lockref_dec(struct lockref *lockref)
+{
+
+#ifdef CONFIG_PAX_REFCOUNT
+ atomic_dec((atomic_t *)&lockref->count);
+#else
+ lockref->count--;
+#endif
+
+}
+
#endif /* __LINUX_LOCKREF_H */
* This is commonly provided by 32bit archs to provide an optimized 64bit
* divide.
*/
-static inline u64 div_u64_rem(u64 dividend, u32 divisor, u32 *remainder)
+static inline u64 __intentional_overflow(-1) div_u64_rem(u64 dividend, u32 divisor, u32 *remainder)
{
*remainder = dividend % divisor;
return dividend / divisor;
/**
* div64_u64 - unsigned 64bit divide with 64bit divisor
*/
-static inline u64 div64_u64(u64 dividend, u64 divisor)
+static inline u64 __intentional_overflow(-1) div64_u64(u64 dividend, u64 divisor)
{
return dividend / divisor;
}
#define div64_ul(x, y) div_u64((x), (y))
#ifndef div_u64_rem
-static inline u64 div_u64_rem(u64 dividend, u32 divisor, u32 *remainder)
+static inline u64 __intentional_overflow(-1) div_u64_rem(u64 dividend, u32 divisor, u32 *remainder)
{
*remainder = do_div(dividend, divisor);
return dividend;
#endif
#ifndef div64_u64
-extern u64 div64_u64(u64 dividend, u64 divisor);
+extern u64 __intentional_overflow(-1) div64_u64(u64 dividend, u64 divisor);
#endif
#ifndef div64_s64
* divide.
*/
#ifndef div_u64
-static inline u64 div_u64(u64 dividend, u32 divisor)
+static inline u64 __intentional_overflow(-1) div_u64(u64 dividend, u32 divisor)
{
u32 remainder;
return div_u64_rem(dividend, divisor, &remainder);
}
#define vma_policy(vma) ((vma)->vm_policy)
+static inline void set_vma_policy(struct vm_area_struct *vma, struct mempolicy *pol)
+{
+ vma->vm_policy = pol;
+}
static inline void mpol_get(struct mempolicy *pol)
{
}
#define vma_policy(vma) NULL
+static inline void set_vma_policy(struct vm_area_struct *vma, struct mempolicy *pol)
+{
+}
static inline int
vma_dup_policy(struct vm_area_struct *src, struct vm_area_struct *dst)
#define VM_DONTCOPY 0x00020000 /* Do not copy this vma on fork */
#define VM_DONTEXPAND 0x00040000 /* Cannot expand with mremap() */
+
+#if defined(CONFIG_PAX_PAGEEXEC) && defined(CONFIG_X86_32)
+#define VM_PAGEEXEC 0x00080000 /* vma->vm_page_prot needs special handling */
+#endif
+
#define VM_ACCOUNT 0x00100000 /* Is a VM accounted object */
#define VM_NORESERVE 0x00200000 /* should the VM suppress accounting */
#define VM_HUGETLB 0x00400000 /* Huge TLB Page VM */
/* called by access_process_vm when get_user_pages() fails, typically
* for use by special VMAs that can switch between memory and hardware
*/
- int (*access)(struct vm_area_struct *vma, unsigned long addr,
- void *buf, int len, int write);
+ ssize_t (*access)(struct vm_area_struct *vma, unsigned long addr,
+ void *buf, size_t len, int write);
/* Called by the /proc/PID/maps code to ask the vma whether it
* has a special name. Returning non-NULL will also cause this
int (*remap_pages)(struct vm_area_struct *vma, unsigned long addr,
unsigned long size, pgoff_t pgoff);
};
+typedef struct vm_operations_struct __no_const vm_operations_struct_no_const;
struct mmu_gather;
struct inode;
unsigned long *pfn);
int follow_phys(struct vm_area_struct *vma, unsigned long address,
unsigned int flags, unsigned long *prot, resource_size_t *phys);
-int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
- void *buf, int len, int write);
+ssize_t generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
+ void *buf, size_t len, int write);
static inline void unmap_shared_mapping_range(struct address_space *mapping,
loff_t const holebegin, loff_t const holelen)
}
#endif
-extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write);
-extern int access_remote_vm(struct mm_struct *mm, unsigned long addr,
- void *buf, int len, int write);
+extern ssize_t access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, size_t len, int write);
+extern ssize_t access_remote_vm(struct mm_struct *mm, unsigned long addr,
+ void *buf, size_t len, int write);
long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
unsigned long start, unsigned long nr_pages,
int clear_page_dirty_for_io(struct page *page);
int get_cmdline(struct task_struct *task, char *buffer, int buflen);
-/* Is the vma a continuation of the stack vma above it? */
-static inline int vma_growsdown(struct vm_area_struct *vma, unsigned long addr)
-{
- return vma && (vma->vm_end == addr) && (vma->vm_flags & VM_GROWSDOWN);
-}
-
-static inline int stack_guard_page_start(struct vm_area_struct *vma,
- unsigned long addr)
-{
- return (vma->vm_flags & VM_GROWSDOWN) &&
- (vma->vm_start == addr) &&
- !vma_growsdown(vma->vm_prev, addr);
-}
-
-/* Is the vma a continuation of the stack vma below it? */
-static inline int vma_growsup(struct vm_area_struct *vma, unsigned long addr)
-{
- return vma && (vma->vm_start == addr) && (vma->vm_flags & VM_GROWSUP);
-}
-
-static inline int stack_guard_page_end(struct vm_area_struct *vma,
- unsigned long addr)
-{
- return (vma->vm_flags & VM_GROWSUP) &&
- (vma->vm_end == addr) &&
- !vma_growsup(vma->vm_next, addr);
-}
-
extern struct task_struct *task_of_stack(struct task_struct *task,
struct vm_area_struct *vma, bool in_group);
{
return 0;
}
+
+static inline int __pud_alloc_kernel(struct mm_struct *mm, pgd_t *pgd,
+ unsigned long address)
+{
+ return 0;
+}
#else
int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address);
+int __pud_alloc_kernel(struct mm_struct *mm, pgd_t *pgd, unsigned long address);
#endif
#ifdef __PAGETABLE_PMD_FOLDED
{
return 0;
}
+
+static inline int __pmd_alloc_kernel(struct mm_struct *mm, pud_t *pud,
+ unsigned long address)
+{
+ return 0;
+}
#else
int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address);
+int __pmd_alloc_kernel(struct mm_struct *mm, pud_t *pud, unsigned long address);
#endif
int __pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
NULL: pud_offset(pgd, address);
}
+static inline pud_t *pud_alloc_kernel(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
+{
+ return (unlikely(pgd_none(*pgd)) && __pud_alloc_kernel(mm, pgd, address))?
+ NULL: pud_offset(pgd, address);
+}
+
static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
{
return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))?
NULL: pmd_offset(pud, address);
}
+
+static inline pmd_t *pmd_alloc_kernel(struct mm_struct *mm, pud_t *pud, unsigned long address)
+{
+ return (unlikely(pud_none(*pud)) && __pmd_alloc_kernel(mm, pud, address))?
+ NULL: pmd_offset(pud, address);
+}
#endif /* CONFIG_MMU && !__ARCH_HAS_4LEVEL_HACK */
#if USE_SPLIT_PTE_PTLOCKS
bool *need_rmap_locks);
extern void exit_mmap(struct mm_struct *);
+#if defined(CONFIG_GRKERNSEC) && (defined(CONFIG_GRKERNSEC_RESLOG) || !defined(CONFIG_GRKERNSEC_NO_RBAC))
+extern void gr_learn_resource(const struct task_struct *task, const int res,
+ const unsigned long wanted, const int gt);
+#else
+static inline void gr_learn_resource(const struct task_struct *task, const int res,
+ const unsigned long wanted, const int gt)
+{
+}
+#endif
+
static inline int check_data_rlimit(unsigned long rlim,
unsigned long new,
unsigned long start,
unsigned long end_data,
unsigned long start_data)
{
+ gr_learn_resource(current, RLIMIT_DATA, (new - start) + (end_data - start_data), 1);
if (rlim < RLIM_INFINITY) {
if (((new - start) + (end_data - start_data)) > rlim)
return -ENOSPC;
unsigned long addr, unsigned long len,
unsigned long flags, struct page **pages);
-extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long);
+extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long) __intentional_overflow(-1);
extern unsigned long mmap_region(struct file *file, unsigned long addr,
unsigned long len, vm_flags_t vm_flags, unsigned long pgoff);
unsigned long len, unsigned long prot, unsigned long flags,
unsigned long pgoff, unsigned long *populate);
extern int do_munmap(struct mm_struct *, unsigned long, size_t);
+extern int __do_munmap(struct mm_struct *, unsigned long, size_t);
#ifdef CONFIG_MMU
extern int __mm_populate(unsigned long addr, unsigned long len,
unsigned long high_limit;
unsigned long align_mask;
unsigned long align_offset;
+ unsigned long threadstack_offset;
};
-extern unsigned long unmapped_area(struct vm_unmapped_area_info *info);
-extern unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info);
+extern unsigned long unmapped_area(const struct vm_unmapped_area_info *info);
+extern unsigned long unmapped_area_topdown(const struct vm_unmapped_area_info *info);
/*
* Search for an unmapped address range.
* - satisfies (begin_addr & align_mask) == (align_offset & align_mask)
*/
static inline unsigned long
-vm_unmapped_area(struct vm_unmapped_area_info *info)
+vm_unmapped_area(const struct vm_unmapped_area_info *info)
{
if (!(info->flags & VM_UNMAPPED_AREA_TOPDOWN))
return unmapped_area(info);
extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr,
struct vm_area_struct **pprev);
+extern struct vm_area_struct *pax_find_mirror_vma(struct vm_area_struct *vma);
+extern __must_check long pax_mirror_vma(struct vm_area_struct *vma_m, struct vm_area_struct *vma);
+extern void pax_mirror_file_pte(struct vm_area_struct *vma, unsigned long address, struct page *page_m, spinlock_t *ptl);
+
/* Look up the first VMA which intersects the interval start_addr..end_addr-1,
NULL if none. Assume start_addr < end_addr. */
static inline struct vm_area_struct * find_vma_intersection(struct mm_struct * mm, unsigned long start_addr, unsigned long end_addr)
}
#ifdef CONFIG_MMU
-pgprot_t vm_get_page_prot(unsigned long vm_flags);
+pgprot_t vm_get_page_prot(vm_flags_t vm_flags);
void vma_set_page_prot(struct vm_area_struct *vma);
#else
-static inline pgprot_t vm_get_page_prot(unsigned long vm_flags)
+static inline pgprot_t vm_get_page_prot(vm_flags_t vm_flags)
{
return __pgprot(0);
}
static inline void vm_stat_account(struct mm_struct *mm,
unsigned long flags, struct file *file, long pages)
{
+
+#ifdef CONFIG_PAX_RANDMMAP
+ if (!(mm->pax_flags & MF_PAX_RANDMMAP) || (flags & (VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC)))
+#endif
+
mm->total_vm += pages;
}
#endif /* CONFIG_PROC_FS */
extern int sysctl_memory_failure_early_kill;
extern int sysctl_memory_failure_recovery;
extern void shake_page(struct page *p, int access);
-extern atomic_long_t num_poisoned_pages;
+extern atomic_long_unchecked_t num_poisoned_pages;
extern int soft_offline_page(struct page *page, int flags);
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
static inline void setup_nr_node_ids(void) {}
#endif
+#ifdef CONFIG_ARCH_TRACK_EXEC_LIMIT
+extern void track_exec_limit(struct mm_struct *mm, unsigned long start, unsigned long end, unsigned long prot);
+#else
+static inline void track_exec_limit(struct mm_struct *mm, unsigned long start, unsigned long end, unsigned long prot) {}
+#endif
+
#endif /* __KERNEL__ */
#endif /* _LINUX_MM_H */
#ifdef CONFIG_NUMA
struct mempolicy *vm_policy; /* NUMA policy for the VMA */
#endif
-};
+
+ struct vm_area_struct *vm_mirror;/* PaX: mirror vma or NULL */
+} __randomize_layout;
struct core_thread {
struct task_struct *task;
/* address of the bounds directory */
void __user *bd_addr;
#endif
-};
+
+#if defined(CONFIG_PAX_NOEXEC) || defined(CONFIG_PAX_ASLR)
+ unsigned long pax_flags;
+#endif
+
+#ifdef CONFIG_PAX_DLRESOLVE
+ unsigned long call_dl_resolve;
+#endif
+
+#if defined(CONFIG_PPC32) && defined(CONFIG_PAX_EMUSIGRT)
+ unsigned long call_syscall;
+#endif
+
+#ifdef CONFIG_PAX_ASLR
+ unsigned long delta_mmap; /* randomized offset */
+ unsigned long delta_stack; /* randomized offset */
+#endif
+
+} __randomize_layout;
static inline void mm_init_cpumask(struct mm_struct *mm)
{
/* Called from ioremap.c */
extern void mmiotrace_ioremap(resource_size_t offset, unsigned long size,
void __iomem *addr);
-extern void mmiotrace_iounmap(volatile void __iomem *addr);
+extern void mmiotrace_iounmap(const volatile void __iomem *addr);
/* For anyone to insert markers. Remember trailing newline. */
extern __printf(1, 2) int mmiotrace_printk(const char *fmt, ...);
{
}
-static inline void mmiotrace_iounmap(volatile void __iomem *addr)
+static inline void mmiotrace_iounmap(const volatile void __iomem *addr)
{
}
ZONE_PADDING(_pad3_)
/* Zone statistics */
- atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS];
+ atomic_long_unchecked_t vm_stat[NR_VM_ZONE_STAT_ITEMS];
} ____cacheline_internodealigned_in_smp;
enum zone_flags {
#define USB_DEVICE_ID_MATCH_INT_PROTOCOL 0x0200
#define USB_DEVICE_ID_MATCH_INT_NUMBER 0x0400
-#define HID_ANY_ID (~0)
+#define HID_ANY_ID (~0U)
#define HID_BUS_ANY 0xffff
#define HID_GROUP_ANY 0x0000
const char *ident;
struct dmi_strmatch matches[4];
void *driver_data;
-};
+} __do_const;
/*
* struct dmi_device_id appears during expansion of
* "MODULE_DEVICE_TABLE(dmi, x)". Compiler doesn't look inside it
#include <linux/moduleparam.h>
#include <linux/jump_label.h>
#include <linux/export.h>
+#include <linux/fs.h>
#include <linux/percpu.h>
#include <asm/module.h>
+#include <asm/pgtable.h>
/* In stripped ARM and x86-64 modules, ~ is surprisingly rare. */
#define MODULE_SIG_STRING "~Module signature appended~\n"
struct kobject *drivers_dir;
struct module_param_attrs *mp;
struct completion *kobj_completion;
-};
+} __randomize_layout;
struct module_attribute {
struct attribute attr;
int (*test)(struct module *);
void (*free)(struct module *);
};
+typedef struct module_attribute __no_const module_attribute_no_const;
struct module_version_attribute {
struct module_attribute mattr;
const char *module_name;
const char *version;
-} __attribute__ ((__aligned__(sizeof(void *))));
+} __do_const __attribute__ ((__aligned__(sizeof(void *))));
extern ssize_t __modver_version_show(struct module_attribute *,
struct module_kobject *, char *);
/* Sysfs stuff. */
struct module_kobject mkobj;
- struct module_attribute *modinfo_attrs;
+ module_attribute_no_const *modinfo_attrs;
const char *version;
const char *srcversion;
struct kobject *holders_dir;
int (*init)(void);
/* If this is non-NULL, vfree after init() returns */
- void *module_init;
+ void *module_init_rx, *module_init_rw;
/* Here is the actual code + data, vfree'd on unload. */
- void *module_core;
+ void *module_core_rx, *module_core_rw;
/* Here are the sizes of the init and core sections */
- unsigned int init_size, core_size;
+ unsigned int init_size_rw, core_size_rw;
/* The size of the executable code in each section. */
- unsigned int init_text_size, core_text_size;
-
- /* Size of RO sections of the module (text+rodata) */
- unsigned int init_ro_size, core_ro_size;
+ unsigned int init_size_rx, core_size_rx;
/* Arch-specific module values */
struct mod_arch_specific arch;
#ifdef CONFIG_EVENT_TRACING
struct ftrace_event_call **trace_events;
unsigned int num_trace_events;
+ struct file_operations trace_id;
+ struct file_operations trace_enable;
+ struct file_operations trace_format;
+ struct file_operations trace_filter;
#endif
#ifdef CONFIG_FTRACE_MCOUNT_RECORD
unsigned int num_ftrace_callsites;
ctor_fn_t *ctors;
unsigned int num_ctors;
#endif
-};
+} __randomize_layout;
#ifndef MODULE_ARCH_INIT
#define MODULE_ARCH_INIT {}
#endif
bool is_module_percpu_address(unsigned long addr);
bool is_module_text_address(unsigned long addr);
+static inline int within_module_range(unsigned long addr, void *start, unsigned long size)
+{
+
+#ifdef CONFIG_PAX_KERNEXEC
+ if (ktla_ktva(addr) >= (unsigned long)start &&
+ ktla_ktva(addr) < (unsigned long)start + size)
+ return 1;
+#endif
+
+ return ((void *)addr >= start && (void *)addr < start + size);
+}
+
+static inline int within_module_core_rx(unsigned long addr, const struct module *mod)
+{
+ return within_module_range(addr, mod->module_core_rx, mod->core_size_rx);
+}
+
+static inline int within_module_core_rw(unsigned long addr, const struct module *mod)
+{
+ return within_module_range(addr, mod->module_core_rw, mod->core_size_rw);
+}
+
+static inline int within_module_init_rx(unsigned long addr, const struct module *mod)
+{
+ return within_module_range(addr, mod->module_init_rx, mod->init_size_rx);
+}
+
+static inline int within_module_init_rw(unsigned long addr, const struct module *mod)
+{
+ return within_module_range(addr, mod->module_init_rw, mod->init_size_rw);
+}
+
static inline bool within_module_core(unsigned long addr,
const struct module *mod)
{
- return (unsigned long)mod->module_core <= addr &&
- addr < (unsigned long)mod->module_core + mod->core_size;
+ return within_module_core_rx(addr, mod) || within_module_core_rw(addr, mod);
}
static inline bool within_module_init(unsigned long addr,
const struct module *mod)
{
- return (unsigned long)mod->module_init <= addr &&
- addr < (unsigned long)mod->module_init + mod->init_size;
+ return within_module_init_rx(addr, mod) || within_module_init_rw(addr, mod);
}
static inline bool within_module(unsigned long addr, const struct module *mod)
sections. Returns NULL on failure. */
void *module_alloc(unsigned long size);
+#ifdef CONFIG_PAX_KERNEXEC
+void *module_alloc_exec(unsigned long size);
+#else
+#define module_alloc_exec(x) module_alloc(x)
+#endif
+
/* Free memory returned from module_alloc. */
void module_memfree(void *module_region);
+#ifdef CONFIG_PAX_KERNEXEC
+void module_memfree_exec(void *module_region);
+#else
+#define module_memfree_exec(x) module_memfree((x))
+#endif
+
/*
* Apply the given relocation to the (simplified) ELF. Return -error
* or 0.
unsigned int relsec,
struct module *me)
{
+#ifdef CONFIG_MODULES
printk(KERN_ERR "module %s: REL relocation unsupported\n",
module_name(me));
+#endif
return -ENOEXEC;
}
#endif
unsigned int relsec,
struct module *me)
{
+#ifdef CONFIG_MODULES
printk(KERN_ERR "module %s: REL relocation unsupported\n",
module_name(me));
+#endif
return -ENOEXEC;
}
#endif
* @len is usually just sizeof(string).
*/
#define module_param_string(name, string, len, perm) \
- static const struct kparam_string __param_string_##name \
+ static const struct kparam_string __param_string_##name __used \
= { len, string }; \
__module_param_call(MODULE_PARAM_PREFIX, name, \
¶m_ops_string, \
*/
#define module_param_array_named(name, array, type, nump, perm) \
param_check_##type(name, &(array)[0]); \
- static const struct kparam_array __param_arr_##name \
+ static const struct kparam_array __param_arr_##name __used \
= { .max = ARRAY_SIZE(array), .num = nump, \
.ops = ¶m_ops_##type, \
.elemsize = sizeof(array[0]), .elem = array }; \
struct dentry *mnt_root; /* root of the mounted tree */
struct super_block *mnt_sb; /* pointer to superblock */
int mnt_flags;
-};
+} __randomize_layout;
struct file; /* forward dec */
struct path;
extern void unlock_rename(struct dentry *, struct dentry *);
extern void nd_jump_link(struct nameidata *nd, struct path *path);
-extern void nd_set_link(struct nameidata *nd, char *path);
-extern char *nd_get_link(struct nameidata *nd);
+extern void nd_set_link(struct nameidata *nd, const char *path);
+extern const char *nd_get_link(const struct nameidata *nd);
static inline void nd_terminate_link(void *name, size_t len, size_t maxlen)
{
int (*create)(struct net *net, struct socket *sock,
int protocol, int kern);
struct module *owner;
-};
+} __do_const;
struct iovec;
struct kvec;
u8 state);
#endif
};
+typedef struct net_device_ops __no_const net_device_ops_no_const;
/**
* enum net_device_priv_flags - &struct net_device priv_flags
struct net_device_stats stats;
- atomic_long_t rx_dropped;
- atomic_long_t tx_dropped;
+ atomic_long_unchecked_t rx_dropped;
+ atomic_long_unchecked_t tx_dropped;
- atomic_t carrier_changes;
+ atomic_unchecked_t carrier_changes;
#ifdef CONFIG_WIRELESS_EXT
const struct iw_handler_def * wireless_handlers;
#endif
/* Use the module struct to lock set/get code in place */
struct module *owner;
-};
+} __do_const;
/* Function to register/unregister hook points. */
int nf_register_hook(struct nf_hook_ops *reg);
const struct nlattr * const cda[]);
const struct nla_policy *policy; /* netlink attribute policy */
const u_int16_t attr_count; /* number of nlattr's */
-};
+} __do_const;
struct nfnetlink_subsystem {
const char *name;
const unsigned char *charset2upper;
struct module *owner;
struct nls_table *next;
-};
+} __do_const;
/* this value hold the maximum octet of charset */
#define NLS_MAX_CHARSET_SIZE 6 /* for UTF-8 */
/* nls_base.c */
extern int __register_nls(struct nls_table *, struct module *);
extern int unregister_nls(struct nls_table *);
-extern struct nls_table *load_nls(char *);
+extern struct nls_table *load_nls(const char *);
extern void unload_nls(struct nls_table *);
extern struct nls_table *load_nls_default(void);
#define register_nls(nls) __register_nls((nls), THIS_MODULE)
notifier_fn_t notifier_call;
struct notifier_block __rcu *next;
int priority;
-};
+} __do_const;
+typedef struct notifier_block __no_const notifier_block_no_const;
struct atomic_notifier_head {
spinlock_t lock;
int oprofilefs_create_ro_ulong(struct dentry * root,
char const * name, ulong * val);
-/** Create a file for read-only access to an atomic_t. */
+/** Create a file for read-only access to an atomic_unchecked_t. */
int oprofilefs_create_ro_atomic(struct dentry * root,
- char const * name, atomic_t * val);
+ char const * name, atomic_unchecked_t * val);
/** create a directory */
struct dentry *oprofilefs_mkdir(struct dentry *parent, char const *name);
struct padata_serial_queue __percpu *squeue;
atomic_t reorder_objects;
atomic_t refcnt;
- atomic_t seq_nr;
+ atomic_unchecked_t seq_nr;
struct padata_cpumask cpumask;
spinlock_t lock ____cacheline_aligned;
unsigned int processed;
#ifndef _LINUX_PATH_H
#define _LINUX_PATH_H
+#include <linux/compiler.h>
+
struct dentry;
struct vfsmount;
struct path {
struct vfsmount *mnt;
struct dentry *dentry;
-};
+} __randomize_layout;
extern void path_get(const struct path *);
extern void path_put(const struct path *);
int (*get_latch_status) (struct hotplug_slot *slot, u8 *value);
int (*get_adapter_status) (struct hotplug_slot *slot, u8 *value);
int (*reset_slot) (struct hotplug_slot *slot, int probe);
-};
+} __do_const;
+typedef struct hotplug_slot_ops __no_const hotplug_slot_ops_no_const;
/**
* struct hotplug_slot_info - used to notify the hotplug pci core of the state of the slot
* preallocate for this. Keep PERCPU_DYNAMIC_RESERVE equal to or
* larger than PERCPU_DYNAMIC_EARLY_SIZE.
*/
-#define PERCPU_DYNAMIC_EARLY_SLOTS 128
+#define PERCPU_DYNAMIC_EARLY_SLOTS 256
#define PERCPU_DYNAMIC_EARLY_SIZE (12 << 10)
/*
enum perf_event_active_state state;
unsigned int attach_state;
- local64_t count;
- atomic64_t child_count;
+ local64_t count; /* PaX: fix it one day */
+ atomic64_unchecked_t child_count;
/*
* These are the total time in nanoseconds that the event
* These accumulate total time (in nanoseconds) that children
* events have been enabled and running, respectively.
*/
- atomic64_t child_total_time_enabled;
- atomic64_t child_total_time_running;
+ atomic64_unchecked_t child_total_time_enabled;
+ atomic64_unchecked_t child_total_time_running;
/*
* Protect attach/detach and child_list:
entry->ip[entry->nr++] = ip;
}
-extern int sysctl_perf_event_paranoid;
+extern int sysctl_perf_event_legitimately_concerned;
extern int sysctl_perf_event_mlock;
extern int sysctl_perf_event_sample_rate;
extern int sysctl_perf_cpu_time_max_percent;
loff_t *ppos);
+static inline bool perf_paranoid_any(void)
+{
+ return sysctl_perf_event_legitimately_concerned > 2;
+}
+
static inline bool perf_paranoid_tracepoint_raw(void)
{
- return sysctl_perf_event_paranoid > -1;
+ return sysctl_perf_event_legitimately_concerned > -1;
}
static inline bool perf_paranoid_cpu(void)
{
- return sysctl_perf_event_paranoid > 0;
+ return sysctl_perf_event_legitimately_concerned > 0;
}
static inline bool perf_paranoid_kernel(void)
{
- return sysctl_perf_event_paranoid > 1;
+ return sysctl_perf_event_legitimately_concerned > 1;
}
extern void perf_event_init(void);
struct device_attribute attr;
u64 id;
const char *event_str;
-};
+} __do_const;
#define PMU_EVENT_ATTR(_name, _var, _id, _show) \
static struct perf_pmu_events_attr _var = { \
int hide_pid;
int reboot; /* group exit code if this pidns was rebooted */
struct ns_common ns;
-};
+} __randomize_layout;
extern struct pid_namespace init_pid_ns;
struct mutex mutex;
wait_queue_head_t wait;
unsigned int nrbufs, curbuf, buffers;
- unsigned int readers;
- unsigned int writers;
- unsigned int files;
- unsigned int waiting_writers;
+ atomic_t readers;
+ atomic_t writers;
+ atomic_t files;
+ atomic_t waiting_writers;
unsigned int r_counter;
unsigned int w_counter;
struct page *tmp_page;
struct dev_pm_ops ops;
void (*detach)(struct device *dev, bool power_off);
};
+typedef struct dev_pm_domain __no_const dev_pm_domain_no_const;
/*
* The PM_EVENT_ messages are also used by drivers implementing the legacy
int (*save_state)(struct device *dev);
int (*restore_state)(struct device *dev);
bool (*active_wakeup)(struct device *dev);
-};
+} __no_const;
struct gpd_cpuidle_data {
unsigned int saved_exit_latency;
- struct cpuidle_state *idle_state;
+ cpuidle_state_no_const *idle_state;
};
struct generic_pm_domain {
static inline void pm_runtime_mark_last_busy(struct device *dev)
{
- ACCESS_ONCE(dev->power.last_busy) = jiffies;
+ ACCESS_ONCE_RW(dev->power.last_busy) = jiffies;
}
static inline bool pm_runtime_is_irq_safe(struct device *dev)
struct pnp_fixup {
char id[7];
void (*quirk_function) (struct pnp_dev * dev); /* fixup function */
-};
+} __do_const;
/* config parameters */
#define PNP_CONFIG_NORMAL 0x0001
* under normal circumstances, used to verify that nobody uses
* non-initialized list entries.
*/
-#define LIST_POISON1 ((void *) 0x00100100 + POISON_POINTER_DELTA)
-#define LIST_POISON2 ((void *) 0x00200200 + POISON_POINTER_DELTA)
+#define LIST_POISON1 ((void *) (long)0xFFFFFF01)
+#define LIST_POISON2 ((void *) (long)0xFFFFFF02)
/********** include/linux/timer.h **********/
/*
int (*notify)(struct omap_sr *sr, u32 status);
u8 notify_flags;
u8 class_type;
-};
+} __do_const;
/**
* struct omap_sr_nvalue_table - Smartreflex n-target value info
struct module *owner;
/* Extra skb space needed by the compressor algorithm */
unsigned int comp_extra;
-};
+} __do_const;
/*
* The return value from decompress routine is the length of the
#define preempt_count_dec_and_test() __preempt_count_dec_and_test()
#endif
+#define raw_preempt_count_add(val) __preempt_count_add(val)
+#define raw_preempt_count_sub(val) __preempt_count_sub(val)
+
#define __preempt_count_inc() __preempt_count_add(1)
#define __preempt_count_dec() __preempt_count_sub(1)
#define preempt_count_inc() preempt_count_add(1)
+#define raw_preempt_count_inc() raw_preempt_count_add(1)
#define preempt_count_dec() preempt_count_sub(1)
+#define raw_preempt_count_dec() raw_preempt_count_sub(1)
#ifdef CONFIG_PREEMPT_COUNT
barrier(); \
} while (0)
+#define raw_preempt_disable() \
+do { \
+ raw_preempt_count_inc(); \
+ barrier(); \
+} while (0)
+
#define sched_preempt_enable_no_resched() \
do { \
barrier(); \
#define preempt_enable_no_resched() sched_preempt_enable_no_resched()
+#define raw_preempt_enable_no_resched() \
+do { \
+ barrier(); \
+ raw_preempt_count_dec(); \
+} while (0)
+
#ifdef CONFIG_PREEMPT
#define preempt_enable() \
do { \
* region.
*/
#define preempt_disable() barrier()
+#define raw_preempt_disable() barrier()
#define sched_preempt_enable_no_resched() barrier()
#define preempt_enable_no_resched() barrier()
+#define raw_preempt_enable_no_resched() barrier()
#define preempt_enable() barrier()
#define preempt_check_resched() do { } while (0)
/*
* Modules have no business playing preemption tricks.
*/
+#ifndef CONFIG_PAX_KERNEXEC
#undef sched_preempt_enable_no_resched
#undef preempt_enable_no_resched
#undef preempt_enable_no_resched_notrace
#undef preempt_check_resched
#endif
+#endif
#define preempt_set_need_resched() \
do { \
#endif
typedef int(*printk_func_t)(const char *fmt, va_list args);
+extern int kptr_restrict;
#ifdef CONFIG_PRINTK
asmlinkage __printf(5, 0)
extern int printk_delay_msec;
extern int dmesg_restrict;
-extern int kptr_restrict;
extern void wake_up_klogd(void);
extern struct proc_dir_entry *proc_symlink(const char *,
struct proc_dir_entry *, const char *);
extern struct proc_dir_entry *proc_mkdir(const char *, struct proc_dir_entry *);
+extern struct proc_dir_entry *proc_mkdir_restrict(const char *, struct proc_dir_entry *);
extern struct proc_dir_entry *proc_mkdir_data(const char *, umode_t,
struct proc_dir_entry *, void *);
+extern struct proc_dir_entry *proc_mkdir_data_restrict(const char *, umode_t,
+ struct proc_dir_entry *, void *);
extern struct proc_dir_entry *proc_mkdir_mode(const char *, umode_t,
struct proc_dir_entry *);
return proc_create_data(name, mode, parent, proc_fops, NULL);
}
+static inline struct proc_dir_entry *proc_create_grsec(const char *name, umode_t mode,
+ struct proc_dir_entry *parent, const struct file_operations *proc_fops)
+{
+#ifdef CONFIG_GRKERNSEC_PROC_USER
+ return proc_create_data(name, S_IRUSR, parent, proc_fops, NULL);
+#elif defined(CONFIG_GRKERNSEC_PROC_USERGROUP)
+ return proc_create_data(name, S_IRUSR | S_IRGRP, parent, proc_fops, NULL);
+#else
+ return proc_create_data(name, mode, parent, proc_fops, NULL);
+#endif
+}
+
+
extern void proc_set_size(struct proc_dir_entry *, loff_t);
extern void proc_set_user(struct proc_dir_entry *, kuid_t, kgid_t);
extern void *PDE_DATA(const struct inode *);
struct proc_dir_entry *parent,const char *dest) { return NULL;}
static inline struct proc_dir_entry *proc_mkdir(const char *name,
struct proc_dir_entry *parent) {return NULL;}
+static inline struct proc_dir_entry *proc_mkdir_restrict(const char *name,
+ struct proc_dir_entry *parent) { return NULL; }
static inline struct proc_dir_entry *proc_mkdir_data(const char *name,
umode_t mode, struct proc_dir_entry *parent, void *data) { return NULL; }
+static inline struct proc_dir_entry *proc_mkdir_data_restrict(const char *name,
+ umode_t mode, struct proc_dir_entry *parent, void *data) { return NULL; }
static inline struct proc_dir_entry *proc_mkdir_mode(const char *name,
umode_t mode, struct proc_dir_entry *parent) { return NULL; }
#define proc_create(name, mode, parent, proc_fops) ({NULL;})
static inline struct proc_dir_entry *proc_net_mkdir(
struct net *net, const char *name, struct proc_dir_entry *parent)
{
- return proc_mkdir_data(name, 0, parent, net);
+ return proc_mkdir_data_restrict(name, 0, parent, net);
}
#endif /* _LINUX_PROC_FS_H */
struct ns_common *(*get)(struct task_struct *task);
void (*put)(struct ns_common *ns);
int (*install)(struct nsproxy *nsproxy, struct ns_common *ns);
-};
+} __do_const __randomize_layout;
extern const struct proc_ns_operations netns_operations;
extern const struct proc_ns_operations utsns_operations;
extern bool qid_eq(struct kqid left, struct kqid right);
extern bool qid_lt(struct kqid left, struct kqid right);
-extern qid_t from_kqid(struct user_namespace *to, struct kqid qid);
+extern qid_t from_kqid(struct user_namespace *to, struct kqid qid) __intentional_overflow(-1);
extern qid_t from_kqid_munged(struct user_namespace *to, struct kqid qid);
extern bool qid_valid(struct kqid qid);
#include <uapi/linux/random.h>
extern void add_device_randomness(const void *, unsigned int);
+
+static inline void add_latent_entropy(void)
+{
+
+#ifdef LATENT_ENTROPY_PLUGIN
+ add_device_randomness((const void *)&latent_entropy, sizeof(latent_entropy));
+#endif
+
+}
+
extern void add_input_randomness(unsigned int type, unsigned int code,
- unsigned int value);
-extern void add_interrupt_randomness(int irq, int irq_flags);
+ unsigned int value) __latent_entropy;
+extern void add_interrupt_randomness(int irq, int irq_flags) __latent_entropy;
extern void get_random_bytes(void *buf, int nbytes);
extern void get_random_bytes_arch(void *buf, int nbytes);
extern const struct file_operations random_fops, urandom_fops;
#endif
-unsigned int get_random_int(void);
+unsigned int __intentional_overflow(-1) get_random_int(void);
unsigned long randomize_range(unsigned long start, unsigned long end, unsigned long len);
-u32 prandom_u32(void);
+u32 prandom_u32(void) __intentional_overflow(-1);
void prandom_bytes(void *buf, size_t nbytes);
void prandom_seed(u32 seed);
void prandom_reseed_late(void);
u32 prandom_u32_state(struct rnd_state *state);
void prandom_bytes_state(struct rnd_state *state, void *buf, size_t nbytes);
+static inline unsigned long __intentional_overflow(-1) pax_get_random_long(void)
+{
+ return prandom_u32() + (sizeof(long) > 4 ? (unsigned long)prandom_u32() << 32 : 0);
+}
+
/**
* prandom_u32_max - returns a pseudo-random number in interval [0, ep_ro)
* @ep_ro: right open interval endpoint
*
* Returns: pseudo-random number in interval [0, ep_ro)
*/
-static inline u32 prandom_u32_max(u32 ep_ro)
+static inline u32 __intentional_overflow(-1) prandom_u32_max(u32 ep_ro)
{
return (u32)(((u64) prandom_u32() * ep_ro) >> 32);
}
old->rbaugmented = rbcompute(old); \
} \
rbstatic const struct rb_augment_callbacks rbname = { \
- rbname ## _propagate, rbname ## _copy, rbname ## _rotate \
+ .propagate = rbname ## _propagate, \
+ .copy = rbname ## _copy, \
+ .rotate = rbname ## _rotate \
};
*/
static inline void INIT_LIST_HEAD_RCU(struct list_head *list)
{
- ACCESS_ONCE(list->next) = list;
- ACCESS_ONCE(list->prev) = list;
+ ACCESS_ONCE_RW(list->next) = list;
+ ACCESS_ONCE_RW(list->prev) = list;
}
/*
struct list_head *prev, struct list_head *next);
#endif
+void __pax_list_add_rcu(struct list_head *new,
+ struct list_head *prev, struct list_head *next);
+
/**
* list_add_rcu - add a new entry to rcu-protected list
* @new: new entry to be added
__list_add_rcu(new, head, head->next);
}
+static inline void pax_list_add_rcu(struct list_head *new, struct list_head *head)
+{
+ __pax_list_add_rcu(new, head, head->next);
+}
+
/**
* list_add_tail_rcu - add a new entry to rcu-protected list
* @new: new entry to be added
__list_add_rcu(new, head->prev, head);
}
+static inline void pax_list_add_tail_rcu(struct list_head *new,
+ struct list_head *head)
+{
+ __pax_list_add_rcu(new, head->prev, head);
+}
+
/**
* list_del_rcu - deletes entry from list without re-initialization
* @entry: the element to delete from the list.
entry->prev = LIST_POISON2;
}
+extern void pax_list_del_rcu(struct list_head *entry);
+
/**
* hlist_del_init_rcu - deletes entry from hash list with re-initialization
* @n: the element to delete from the hash list.
#define rcu_note_voluntary_context_switch(t) \
do { \
if (ACCESS_ONCE((t)->rcu_tasks_holdout)) \
- ACCESS_ONCE((t)->rcu_tasks_holdout) = false; \
+ ACCESS_ONCE_RW((t)->rcu_tasks_holdout) = false; \
} while (0)
#else /* #ifdef CONFIG_TASKS_RCU */
#define TASKS_RCU(x) do { } while (0)
*/
extern void migrate_to_reboot_cpu(void);
-extern void machine_restart(char *cmd);
-extern void machine_halt(void);
-extern void machine_power_off(void);
+extern void machine_restart(char *cmd) __noreturn;
+extern void machine_halt(void) __noreturn;
+extern void machine_power_off(void) __noreturn;
extern void machine_shutdown(void);
struct pt_regs;
*/
extern void kernel_restart_prepare(char *cmd);
-extern void kernel_restart(char *cmd);
-extern void kernel_halt(void);
-extern void kernel_power_off(void);
+extern void kernel_restart(char *cmd) __noreturn;
+extern void kernel_halt(void) __noreturn;
+extern void kernel_power_off(void) __noreturn;
extern int C_A_D; /* for sysctl */
void ctrl_alt_del(void);
* Emergency restart, callable from an interrupt handler.
*/
-extern void emergency_restart(void);
+extern void emergency_restart(void) __noreturn;
#include <asm/emergency-restart.h>
#endif /* _LINUX_REBOOT_H */
unsigned int align;
unsigned int bias;
unsigned int core_note_type;
-};
+} __do_const;
+typedef struct user_regset __no_const user_regset_no_const;
/**
* struct user_regset_view - available regsets
* The callback should return 0 if successful, negative if not.
*/
int (*remove_buf_file)(struct dentry *dentry);
-};
+} __no_const;
/*
* CONFIG_RELAY kernel API, kernel/relay.c
int (*map_inb)(struct rio_mport *mport, dma_addr_t lstart,
u64 rstart, u32 size, u32 flags);
void (*unmap_inb)(struct rio_mport *mport, dma_addr_t lstart);
-};
+} __no_const;
#define RIO_RESOURCE_MEM 0x00000100
#define RIO_RESOURCE_DOORBELL 0x00000200
void anon_vma_init(void); /* create anon_vma_cachep */
int anon_vma_prepare(struct vm_area_struct *);
void unlink_anon_vmas(struct vm_area_struct *);
-int anon_vma_clone(struct vm_area_struct *, struct vm_area_struct *);
-int anon_vma_fork(struct vm_area_struct *, struct vm_area_struct *);
+int anon_vma_clone(struct vm_area_struct *, const struct vm_area_struct *);
+int anon_vma_fork(struct vm_area_struct *, const struct vm_area_struct *);
static inline void anon_vma_merge(struct vm_area_struct *vma,
struct vm_area_struct *next)
#ifndef _LINUX_SCATTERLIST_H
#define _LINUX_SCATTERLIST_H
+#include <linux/sched.h>
#include <linux/string.h>
#include <linux/bug.h>
#include <linux/mm.h>
{
#ifdef CONFIG_DEBUG_SG
BUG_ON(!virt_addr_valid(buf));
+#endif
+#ifdef CONFIG_GRKERNSEC_KSTACKOVERFLOW
+ if (object_starts_on_stack(buf)) {
+ void *adjbuf = buf - current->stack + current->lowmem_stack;
+ sg_set_page(sg, virt_to_page(adjbuf), buflen, offset_in_page(adjbuf));
+ } else
#endif
sg_set_page(sg, virt_to_page(buf), buflen, offset_in_page(buf));
}
struct perf_event_context;
struct blk_plug;
struct filename;
+struct linux_binprm;
#define VMACACHE_BITS 2
#define VMACACHE_SIZE (1U << VMACACHE_BITS)
extern int in_sched_functions(unsigned long addr);
#define MAX_SCHEDULE_TIMEOUT LONG_MAX
-extern signed long schedule_timeout(signed long timeout);
+extern signed long schedule_timeout(signed long timeout) __intentional_overflow(-1);
extern signed long schedule_timeout_interruptible(signed long timeout);
extern signed long schedule_timeout_killable(signed long timeout);
extern signed long schedule_timeout_uninterruptible(signed long timeout);
struct user_namespace;
#ifdef CONFIG_MMU
+
+#ifdef CONFIG_GRKERNSEC_RAND_THREADSTACK
+extern unsigned long gr_rand_threadstack_offset(const struct mm_struct *mm, const struct file *filp, unsigned long flags);
+#else
+static inline unsigned long gr_rand_threadstack_offset(const struct mm_struct *mm, const struct file *filp, unsigned long flags)
+{
+ return 0;
+}
+#endif
+
+extern bool check_heap_stack_gap(const struct vm_area_struct *vma, unsigned long addr, unsigned long len, unsigned long offset);
+extern unsigned long skip_heap_stack_gap(const struct vm_area_struct *vma, unsigned long len, unsigned long offset);
+
extern void arch_pick_mmap_layout(struct mm_struct *mm);
extern unsigned long
arch_get_unmapped_area(struct file *, unsigned long, unsigned long,
#ifdef CONFIG_TASKSTATS
struct taskstats *stats;
#endif
+
+#ifdef CONFIG_GRKERNSEC
+ u32 curr_ip;
+ u32 saved_ip;
+ u32 gr_saddr;
+ u32 gr_daddr;
+ u16 gr_sport;
+ u16 gr_dport;
+ u8 used_accept:1;
+#endif
+
#ifdef CONFIG_AUDIT
unsigned audit_tty;
unsigned audit_tty_log_passwd;
struct mutex cred_guard_mutex; /* guard against foreign influences on
* credential calculations
* (notably. ptrace) */
-};
+} __randomize_layout;
/*
* Bits in flags field of signal_struct.
struct key *session_keyring; /* UID's default session keyring */
#endif
+#ifdef CONFIG_GRKERNSEC_KERN_LOCKOUT
+ unsigned char kernel_banned;
+#endif
+#ifdef CONFIG_GRKERNSEC_BRUTE
+ unsigned char suid_banned;
+ unsigned long suid_ban_expires;
+#endif
+
/* Hash table maintenance information */
struct hlist_node uidhash_node;
kuid_t uid;
#ifdef CONFIG_PERF_EVENTS
atomic_long_t locked_vm;
#endif
-};
+} __randomize_layout;
extern int uids_sysfs_init(void);
struct task_struct {
volatile long state; /* -1 unrunnable, 0 runnable, >0 stopped */
void *stack;
+#ifdef CONFIG_GRKERNSEC_KSTACKOVERFLOW
+ void *lowmem_stack;
+#endif
atomic_t usage;
unsigned int flags; /* per process flags, defined below */
unsigned int ptrace;
struct list_head thread_node;
struct completion *vfork_done; /* for vfork() */
- int __user *set_child_tid; /* CLONE_CHILD_SETTID */
- int __user *clear_child_tid; /* CLONE_CHILD_CLEARTID */
+ pid_t __user *set_child_tid; /* CLONE_CHILD_SETTID */
+ pid_t __user *clear_child_tid; /* CLONE_CHILD_CLEARTID */
cputime_t utime, stime, utimescaled, stimescaled;
cputime_t gtime;
struct task_cputime cputime_expires;
struct list_head cpu_timers[3];
-/* process credentials */
- const struct cred __rcu *real_cred; /* objective and real subjective task
- * credentials (COW) */
- const struct cred __rcu *cred; /* effective (overridable) subjective task
- * credentials (COW) */
char comm[TASK_COMM_LEN]; /* executable name excluding path
- access with [gs]et_task_comm (which lock
it with task_lock())
#endif
/* CPU-specific state of this task */
struct thread_struct thread;
+/* thread_info moved to task_struct */
+#ifdef CONFIG_X86
+ struct thread_info tinfo;
+#endif
/* filesystem information */
struct fs_struct *fs;
/* open file information */
gfp_t lockdep_reclaim_gfp;
#endif
+/* process credentials */
+ const struct cred __rcu *real_cred; /* objective and real subjective task
+ * credentials (COW) */
+
/* journalling filesystem info */
void *journal_info;
/* cg_list protected by css_set_lock and tsk->alloc_lock */
struct list_head cg_list;
#endif
+
+ const struct cred __rcu *cred; /* effective (overridable) subjective task
+ * credentials (COW) */
+
#ifdef CONFIG_FUTEX
struct robust_list_head __user *robust_list;
#ifdef CONFIG_COMPAT
* Number of functions that haven't been traced
* because of depth overrun.
*/
- atomic_t trace_overrun;
+ atomic_unchecked_t trace_overrun;
/* Pause for the tracing */
atomic_t tracing_graph_pause;
#endif
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
unsigned long task_state_change;
#endif
-};
+
+#ifdef CONFIG_GRKERNSEC
+ /* grsecurity */
+#ifdef CONFIG_GRKERNSEC_PROC_MEMMAP
+ u64 exec_id;
+#endif
+#ifdef CONFIG_GRKERNSEC_SETXID
+ const struct cred *delayed_cred;
+#endif
+ struct dentry *gr_chroot_dentry;
+ struct acl_subject_label *acl;
+ struct acl_subject_label *tmpacl;
+ struct acl_role_label *role;
+ struct file *exec_file;
+ unsigned long brute_expires;
+ u16 acl_role_id;
+ u8 inherited;
+ /* is this the task that authenticated to the special role */
+ u8 acl_sp_role;
+ u8 is_writable;
+ u8 brute;
+ u8 gr_is_chrooted;
+#endif
+
+} __randomize_layout;
+
+#define MF_PAX_PAGEEXEC 0x01000000 /* Paging based non-executable pages */
+#define MF_PAX_EMUTRAMP 0x02000000 /* Emulate trampolines */
+#define MF_PAX_MPROTECT 0x04000000 /* Restrict mprotect() */
+#define MF_PAX_RANDMMAP 0x08000000 /* Randomize mmap() base */
+/*#define MF_PAX_RANDEXEC 0x10000000*/ /* Randomize ET_EXEC base */
+#define MF_PAX_SEGMEXEC 0x20000000 /* Segmentation based non-executable pages */
+
+#ifdef CONFIG_PAX_SOFTMODE
+extern int pax_softmode;
+#endif
+
+extern int pax_check_flags(unsigned long *);
+#define PAX_PARSE_FLAGS_FALLBACK (~0UL)
+
+/* if tsk != current then task_lock must be held on it */
+#if defined(CONFIG_PAX_NOEXEC) || defined(CONFIG_PAX_ASLR)
+static inline unsigned long pax_get_flags(struct task_struct *tsk)
+{
+ if (likely(tsk->mm))
+ return tsk->mm->pax_flags;
+ else
+ return 0UL;
+}
+
+/* if tsk != current then task_lock must be held on it */
+static inline long pax_set_flags(struct task_struct *tsk, unsigned long flags)
+{
+ if (likely(tsk->mm)) {
+ tsk->mm->pax_flags = flags;
+ return 0;
+ }
+ return -EINVAL;
+}
+#endif
+
+#ifdef CONFIG_PAX_HAVE_ACL_FLAGS
+extern void pax_set_initial_flags(struct linux_binprm *bprm);
+#elif defined(CONFIG_PAX_HOOK_ACL_FLAGS)
+extern void (*pax_set_initial_flags_func)(struct linux_binprm *bprm);
+#endif
+
+struct path;
+extern char *pax_get_path(const struct path *path, char *buf, int buflen);
+extern void pax_report_fault(struct pt_regs *regs, void *pc, void *sp);
+extern void pax_report_insns(struct pt_regs *regs, void *pc, void *sp);
+extern void pax_report_refcount_overflow(struct pt_regs *regs);
/* Future-safe accessor for struct task_struct's cpus_allowed. */
#define tsk_cpus_allowed(tsk) (&(tsk)->cpus_allowed)
pid_t __task_pid_nr_ns(struct task_struct *task, enum pid_type type,
struct pid_namespace *ns);
-static inline pid_t task_pid_nr(struct task_struct *tsk)
+static inline pid_t task_pid_nr(const struct task_struct *tsk)
{
return tsk->pid;
}
extern void sched_clock_init(void);
+#ifdef CONFIG_GRKERNSEC_KSTACKOVERFLOW
+static inline void populate_stack(void)
+{
+ struct task_struct *curtask = current;
+ int c;
+ int *ptr = curtask->stack;
+ int *end = curtask->stack + THREAD_SIZE;
+
+ while (ptr < end) {
+ c = *(volatile int *)ptr;
+ ptr += PAGE_SIZE/sizeof(int);
+ }
+}
+#else
+static inline void populate_stack(void)
+{
+}
+#endif
+
#ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
static inline void sched_clock_tick(void)
{
extern struct exec_domain default_exec_domain;
union thread_union {
+#ifndef CONFIG_X86
struct thread_info thread_info;
+#endif
unsigned long stack[THREAD_SIZE/sizeof(long)];
};
*/
extern struct task_struct *find_task_by_vpid(pid_t nr);
+extern struct task_struct *find_task_by_vpid_unrestricted(pid_t nr);
extern struct task_struct *find_task_by_pid_ns(pid_t nr,
struct pid_namespace *ns);
extern void exit_itimers(struct signal_struct *);
extern void flush_itimer_signals(void);
-extern void do_group_exit(int);
+extern __noreturn void do_group_exit(int);
extern int do_execve(struct filename *,
const char __user * const __user *,
#define task_stack_end_corrupted(task) \
(*(end_of_stack(task)) != STACK_END_MAGIC)
-static inline int object_is_on_stack(void *obj)
+static inline int object_starts_on_stack(const void *obj)
{
- void *stack = task_stack_page(current);
+ const void *stack = task_stack_page(current);
return (obj >= stack) && (obj < (stack + THREAD_SIZE));
}
#define DEFAULT_MAX_MAP_COUNT (USHRT_MAX - MAPCOUNT_ELF_CORE_MARGIN)
extern int sysctl_max_map_count;
+extern unsigned long sysctl_heap_stack_gap;
extern unsigned int sysctl_sched_latency;
extern unsigned int sysctl_sched_min_granularity;
#include <linux/slab.h>
#include <linux/err.h>
#include <linux/string.h>
+#include <linux/grsecurity.h>
struct linux_binprm;
struct cred;
extern int cap_netlink_send(struct sock *sk, struct sk_buff *skb);
-void reset_security_ops(void);
-
#ifdef CONFIG_MMU
extern unsigned long mmap_min_addr;
extern unsigned long dac_mmap_min_addr;
struct audit_context *actx);
void (*audit_rule_free) (void *lsmrule);
#endif /* CONFIG_AUDIT */
-};
+} __randomize_layout;
/* prototypes */
extern int security_init(void);
}
extern void down(struct semaphore *sem);
-extern int __must_check down_interruptible(struct semaphore *sem);
+extern int __must_check down_interruptible(struct semaphore *sem) __intentional_overflow(-1);
extern int __must_check down_killable(struct semaphore *sem);
extern int __must_check down_trylock(struct semaphore *sem);
extern int __must_check down_timeout(struct semaphore *sem, long jiffies);
struct mutex lock;
const struct seq_operations *op;
int poll_event;
+#ifdef CONFIG_GRKERNSEC_PROC_MEMMAP
+ u64 exec_id;
+#endif
#ifdef CONFIG_USER_NS
struct user_namespace *user_ns;
#endif
void * (*next) (struct seq_file *m, void *v, loff_t *pos);
int (*show) (struct seq_file *m, void *v);
};
+typedef struct seq_operations __no_const seq_operations_no_const;
#define SEQ_SKIP 1
char *mangle_path(char *s, const char *p, const char *esc);
int seq_open(struct file *, const struct seq_operations *);
+int seq_open_restrict(struct file *, const struct seq_operations *);
ssize_t seq_read(struct file *, char __user *, size_t, loff_t *);
loff_t seq_lseek(struct file *, loff_t, int);
int seq_release(struct inode *, struct file *);
}
int single_open(struct file *, int (*)(struct seq_file *, void *), void *);
+int single_open_restrict(struct file *, int (*)(struct seq_file *, void *), void *);
int single_open_size(struct file *, int (*)(struct seq_file *, void *), void *, size_t);
int single_release(struct inode *, struct file *);
void *__seq_open_private(struct file *, const struct seq_operations *, int);
/* The task created the shm object. NULL if the task is dead. */
struct task_struct *shm_creator;
struct list_head shm_clist; /* list by creator */
+#ifdef CONFIG_GRKERNSEC
+ u64 shm_createtime;
+ pid_t shm_lapid;
+#endif
};
/* shm_mode upper byte flags */
* know it'll be handled, so that they don't get converted to
* SIGKILL or just silently dropped.
*/
- kernel_sigaction(sig, (__force __sighandler_t)2);
+ kernel_sigaction(sig, (__force_user __sighandler_t)2);
}
static inline void disallow_signal(int sig)
struct sk_buff *__alloc_skb(unsigned int size, gfp_t priority, int flags,
int node);
struct sk_buff *build_skb(void *data, unsigned int frag_size);
-static inline struct sk_buff *alloc_skb(unsigned int size,
+static inline struct sk_buff * __intentional_overflow(0) alloc_skb(unsigned int size,
gfp_t priority)
{
return __alloc_skb(size, priority, 0, NUMA_NO_NODE);
return skb->inner_transport_header - skb->inner_network_header;
}
-static inline int skb_network_offset(const struct sk_buff *skb)
+static inline int __intentional_overflow(0) skb_network_offset(const struct sk_buff *skb)
{
return skb_network_header(skb) - skb->data;
}
* NET_IP_ALIGN(2) + ethernet_header(14) + IP_header(20/40) + ports(8)
*/
#ifndef NET_SKB_PAD
-#define NET_SKB_PAD max(32, L1_CACHE_BYTES)
+#define NET_SKB_PAD max(_AC(32,UL), L1_CACHE_BYTES)
#endif
int ___pskb_trim(struct sk_buff *skb, unsigned int len);
int *err);
unsigned int datagram_poll(struct file *file, struct socket *sock,
struct poll_table_struct *wait);
-int skb_copy_datagram_iter(const struct sk_buff *from, int offset,
+int __intentional_overflow(0) skb_copy_datagram_iter(const struct sk_buff *from, int offset,
struct iov_iter *to, int size);
-static inline int skb_copy_datagram_msg(const struct sk_buff *from, int offset,
+static inline int __intentional_overflow(2,4) skb_copy_datagram_msg(const struct sk_buff *from, int offset,
struct msghdr *msg, int size)
{
return skb_copy_datagram_iter(from, offset, &msg->msg_iter, size);
nf_bridge_put(skb->nf_bridge);
skb->nf_bridge = NULL;
#endif
+#if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE)
+ skb->nf_trace = 0;
+#endif
}
static inline void nf_reset_trace(struct sk_buff *skb)
#include <linux/gfp.h>
#include <linux/types.h>
#include <linux/workqueue.h>
-
+#include <linux/err.h>
/*
* Flags to pass to kmem_cache_create().
* The ones marked DEBUG are only valid if CONFIG_SLAB_DEBUG is set.
*/
#define SLAB_DEBUG_FREE 0x00000100UL /* DEBUG: Perform (expensive) checks on free */
+
+#ifdef CONFIG_PAX_USERCOPY_SLABS
+#define SLAB_USERCOPY 0x00000200UL /* PaX: Allow copying objs to/from userland */
+#else
+#define SLAB_USERCOPY 0x00000000UL
+#endif
+
#define SLAB_RED_ZONE 0x00000400UL /* DEBUG: Red zone objs in a cache */
#define SLAB_POISON 0x00000800UL /* DEBUG: Poison objects */
+
+#ifdef CONFIG_PAX_MEMORY_SANITIZE
+#define SLAB_NO_SANITIZE 0x00001000UL /* PaX: Do not sanitize objs on free */
+#else
+#define SLAB_NO_SANITIZE 0x00000000UL
+#endif
+
#define SLAB_HWCACHE_ALIGN 0x00002000UL /* Align objs on cache lines */
#define SLAB_CACHE_DMA 0x00004000UL /* Use GFP_DMA memory */
#define SLAB_STORE_USER 0x00010000UL /* DEBUG: Store the last owner for bug hunting */
* ZERO_SIZE_PTR can be passed to kfree though in the same way that NULL can.
* Both make kfree a no-op.
*/
-#define ZERO_SIZE_PTR ((void *)16)
+#define ZERO_SIZE_PTR \
+({ \
+ BUILD_BUG_ON(!(MAX_ERRNO & ~PAGE_MASK));\
+ (void *)(-MAX_ERRNO-1L); \
+})
-#define ZERO_OR_NULL_PTR(x) ((unsigned long)(x) <= \
- (unsigned long)ZERO_SIZE_PTR)
+#define ZERO_OR_NULL_PTR(x) ((unsigned long)(x) - 1 >= (unsigned long)ZERO_SIZE_PTR - 1)
#include <linux/kmemleak.h>
void kfree(const void *);
void kzfree(const void *);
size_t ksize(const void *);
+const char *check_heap_object(const void *ptr, unsigned long n);
+bool is_usercopy_object(const void *ptr);
/*
* Some archs want to perform DMA into kmalloc caches and need a guaranteed
extern struct kmem_cache *kmalloc_dma_caches[KMALLOC_SHIFT_HIGH + 1];
#endif
+#ifdef CONFIG_PAX_USERCOPY_SLABS
+extern struct kmem_cache *kmalloc_usercopy_caches[KMALLOC_SHIFT_HIGH + 1];
+#endif
+
/*
* Figure out which kmalloc slab an allocation of a certain size
* belongs to.
* 2 = 120 .. 192 bytes
* n = 2^(n-1) .. 2^n -1
*/
-static __always_inline int kmalloc_index(size_t size)
+static __always_inline __size_overflow(1) int kmalloc_index(size_t size)
{
if (!size)
return 0;
}
#endif /* !CONFIG_SLOB */
-void *__kmalloc(size_t size, gfp_t flags);
+void *__kmalloc(size_t size, gfp_t flags) __alloc_size(1) __size_overflow(1);
void *kmem_cache_alloc(struct kmem_cache *, gfp_t flags);
#ifdef CONFIG_NUMA
-void *__kmalloc_node(size_t size, gfp_t flags, int node);
+void *__kmalloc_node(size_t size, gfp_t flags, int node) __alloc_size(1) __size_overflow(1);
void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);
#else
-static __always_inline void *__kmalloc_node(size_t size, gfp_t flags, int node)
+static __always_inline void * __size_overflow(1) __kmalloc_node(size_t size, gfp_t flags, int node)
{
return __kmalloc(size, flags);
}
/* 4) cache creation/removal */
const char *name;
struct list_head list;
- int refcount;
+ atomic_t refcount;
int object_size;
int align;
unsigned long node_allocs;
unsigned long node_frees;
unsigned long node_overflow;
- atomic_t allochit;
- atomic_t allocmiss;
- atomic_t freehit;
- atomic_t freemiss;
+ atomic_unchecked_t allochit;
+ atomic_unchecked_t allocmiss;
+ atomic_unchecked_t freehit;
+ atomic_unchecked_t freemiss;
+#ifdef CONFIG_PAX_MEMORY_SANITIZE
+ atomic_unchecked_t sanitized;
+ atomic_unchecked_t not_sanitized;
+#endif
/*
* If debugging is enabled, then the allocator can add additional
struct kmem_cache_order_objects max;
struct kmem_cache_order_objects min;
gfp_t allocflags; /* gfp flags to use on each alloc */
- int refcount; /* Refcount for slab cache destroy */
+ atomic_t refcount; /* Refcount for slab cache destroy */
void (*ctor)(void *);
int inuse; /* Offset to metadata */
int align; /* Alignment */
#endif
#define get_cpu() ({ preempt_disable(); smp_processor_id(); })
+#define raw_get_cpu() ({ raw_preempt_disable(); raw_smp_processor_id(); })
#define put_cpu() preempt_enable()
+#define raw_put_cpu_no_resched() raw_preempt_enable_no_resched()
/*
* Callback to arch code if there's nosmp or maxcpus=0 on the
struct sock_diag_handler {
__u8 family;
int (*dump)(struct sk_buff *skb, struct nlmsghdr *nlh);
-};
+} __do_const;
int sock_diag_register(const struct sock_diag_handler *h);
void sock_diag_unregister(const struct sock_diag_handler *h);
#include <uapi/linux/sonet.h>
struct k_sonet_stats {
-#define __HANDLE_ITEM(i) atomic_t i
+#define __HANDLE_ITEM(i) atomic_unchecked_t i
__SONET_ITEMS
#undef __HANDLE_ITEM
};
{
switch (sap->sa_family) {
case AF_INET:
- return ntohs(((struct sockaddr_in *)sap)->sin_port);
+ return ntohs(((const struct sockaddr_in *)sap)->sin_port);
case AF_INET6:
- return ntohs(((struct sockaddr_in6 *)sap)->sin6_port);
+ return ntohs(((const struct sockaddr_in6 *)sap)->sin6_port);
}
return 0;
}
static inline bool __rpc_copy_addr4(struct sockaddr *dst,
const struct sockaddr *src)
{
- const struct sockaddr_in *ssin = (struct sockaddr_in *) src;
+ const struct sockaddr_in *ssin = (const struct sockaddr_in *) src;
struct sockaddr_in *dsin = (struct sockaddr_in *) dst;
dsin->sin_family = ssin->sin_family;
if (sa->sa_family != AF_INET6)
return 0;
- return ((struct sockaddr_in6 *) sa)->sin6_scope_id;
+ return ((const struct sockaddr_in6 *) sa)->sin6_scope_id;
}
#endif /* _LINUX_SUNRPC_ADDR_H */
unsigned int p_timer; /* Which RTT timer to use */
u32 p_statidx; /* Which procedure to account */
const char * p_name; /* name of procedure */
-};
+} __do_const;
#ifdef __KERNEL__
unsigned int pc_count; /* call count */
unsigned int pc_cachetype; /* cache info (NFS) */
unsigned int pc_xdrressize; /* maximum size of XDR reply */
-};
+} __do_const;
/*
* Function prototypes.
extern unsigned int svcrdma_max_requests;
extern unsigned int svcrdma_max_req_size;
-extern atomic_t rdma_stat_recv;
-extern atomic_t rdma_stat_read;
-extern atomic_t rdma_stat_write;
-extern atomic_t rdma_stat_sq_starve;
-extern atomic_t rdma_stat_rq_starve;
-extern atomic_t rdma_stat_rq_poll;
-extern atomic_t rdma_stat_rq_prod;
-extern atomic_t rdma_stat_sq_poll;
-extern atomic_t rdma_stat_sq_prod;
+extern atomic_unchecked_t rdma_stat_recv;
+extern atomic_unchecked_t rdma_stat_read;
+extern atomic_unchecked_t rdma_stat_write;
+extern atomic_unchecked_t rdma_stat_sq_starve;
+extern atomic_unchecked_t rdma_stat_rq_starve;
+extern atomic_unchecked_t rdma_stat_rq_poll;
+extern atomic_unchecked_t rdma_stat_rq_prod;
+extern atomic_unchecked_t rdma_stat_sq_poll;
+extern atomic_unchecked_t rdma_stat_sq_prod;
#define RPCRDMA_VERSION 1
int (*release)(struct svc_rqst *rq);
void (*domain_release)(struct auth_domain *);
int (*set_client)(struct svc_rqst *rq);
-};
+} __do_const;
#define SVC_GARBAGE 1
#define SVC_SYSERR 2
extern void
swiotlb_free_coherent(struct device *hwdev, size_t size,
- void *vaddr, dma_addr_t dma_handle);
+ void *vaddr, dma_addr_t dma_handle,
+ struct dma_attrs *attrs);
extern dma_addr_t swiotlb_map_page(struct device *dev, struct page *page,
unsigned long offset, size_t size,
#define __MAP(n,...) __MAP##n(__VA_ARGS__)
#define __SC_DECL(t, a) t a
+#define __TYPE_IS_U(t) (__same_type((t)0, 0UL) || __same_type((t)0, 0U) || __same_type((t)0, (unsigned short)0) || __same_type((t)0, (unsigned char)0))
#define __TYPE_IS_L(t) (__same_type((t)0, 0L))
#define __TYPE_IS_UL(t) (__same_type((t)0, 0UL))
#define __TYPE_IS_LL(t) (__same_type((t)0, 0LL) || __same_type((t)0, 0ULL))
-#define __SC_LONG(t, a) __typeof(__builtin_choose_expr(__TYPE_IS_LL(t), 0LL, 0L)) a
+#define __SC_LONG(t, a) __typeof( \
+ __builtin_choose_expr( \
+ sizeof(t) > sizeof(int), \
+ (t) 0, \
+ __builtin_choose_expr(__TYPE_IS_U(t), 0UL, 0L) \
+ )) a
#define __SC_CAST(t, a) (t) a
#define __SC_ARGS(t, a) a
#define __SC_TEST(t, a) (void)BUILD_BUG_ON_ZERO(!__TYPE_IS_LL(t) && sizeof(t) > sizeof(long))
asmlinkage long sys_fsync(unsigned int fd);
asmlinkage long sys_fdatasync(unsigned int fd);
asmlinkage long sys_bdflush(int func, long data);
-asmlinkage long sys_mount(char __user *dev_name, char __user *dir_name,
- char __user *type, unsigned long flags,
+asmlinkage long sys_mount(const char __user *dev_name, const char __user *dir_name,
+ const char __user *type, unsigned long flags,
void __user *data);
-asmlinkage long sys_umount(char __user *name, int flags);
-asmlinkage long sys_oldumount(char __user *name);
+asmlinkage long sys_umount(const char __user *name, int flags);
+asmlinkage long sys_oldumount(const char __user *name);
asmlinkage long sys_truncate(const char __user *path, long length);
asmlinkage long sys_ftruncate(unsigned int fd, unsigned long length);
asmlinkage long sys_stat(const char __user *filename,
asmlinkage long sys_getpeername(int, struct sockaddr __user *, int __user *);
asmlinkage long sys_send(int, void __user *, size_t, unsigned);
asmlinkage long sys_sendto(int, void __user *, size_t, unsigned,
- struct sockaddr __user *, int);
+ struct sockaddr __user *, int) __intentional_overflow(0);
asmlinkage long sys_sendmsg(int fd, struct user_msghdr __user *msg, unsigned flags);
asmlinkage long sys_sendmmsg(int fd, struct mmsghdr __user *msg,
unsigned int vlen, unsigned flags);
int (*suspend)(void);
void (*resume)(void);
void (*shutdown)(void);
-};
+} __do_const;
extern void register_syscore_ops(struct syscore_ops *ops);
extern void unregister_syscore_ops(struct syscore_ops *ops);
extern int proc_dostring(struct ctl_table *, int,
void __user *, size_t *, loff_t *);
+extern int proc_dostring_modpriv(struct ctl_table *, int,
+ void __user *, size_t *, loff_t *);
extern int proc_dointvec(struct ctl_table *, int,
void __user *, size_t *, loff_t *);
extern int proc_dointvec_minmax(struct ctl_table *, int,
struct ctl_table_poll *poll;
void *extra1;
void *extra2;
-};
+} __do_const __randomize_layout;
+typedef struct ctl_table __no_const ctl_table_no_const;
struct ctl_node {
struct rb_node node;
struct lock_class_key *key;
struct lock_class_key skey;
#endif
-};
+} __do_const;
+typedef struct attribute __no_const attribute_no_const;
/**
* sysfs_attr_init - initialize a dynamically allocated sysfs attribute
struct attribute *, int);
struct attribute **attrs;
struct bin_attribute **bin_attrs;
-};
+} __do_const;
+typedef struct attribute_group __no_const attribute_group_no_const;
/**
* Use these macros to make defining attributes easier. See include/linux/device.h
char *, loff_t, size_t);
int (*mmap)(struct file *, struct kobject *, struct bin_attribute *attr,
struct vm_area_struct *vma);
-};
+} __do_const;
+typedef struct bin_attribute __no_const bin_attribute_no_const;
/**
* sysfs_bin_attr_init - initialize a dynamically allocated bin_attribute
#include <linux/errno.h>
#include <linux/types.h>
+#include <linux/compiler.h>
/* Possible values of bitmask for enabling sysrq functions */
/* 0x0001 is reserved for enable everything */
char *help_msg;
char *action_msg;
int enable_mask;
-};
+} __do_const;
#ifdef CONFIG_MAGIC_SYSRQ
#error "no set_restore_sigmask() provided and default one won't work"
#endif
+extern void __check_object_size(const void *ptr, unsigned long n, bool to_user, bool const_size);
+
+static inline void check_object_size(const void *ptr, unsigned long n, bool to_user)
+{
+ __check_object_size(ptr, n, to_user, __builtin_constant_p(n));
+}
+
#endif /* __KERNEL__ */
#endif /* _LINUX_THREAD_INFO_H */
const struct tty_port_operations *ops; /* Port operations */
spinlock_t lock; /* Lock protecting tty field */
int blocked_open; /* Waiting to open */
- int count; /* Usage count */
+ atomic_t count; /* Usage count */
wait_queue_head_t open_wait; /* Open waiters */
wait_queue_head_t close_wait; /* Close waiters */
wait_queue_head_t delta_msr_wait; /* Modem status change */
/* If the tty has a pending do_SAK, queue it here - akpm */
struct work_struct SAK_work;
struct tty_port *port;
-};
+} __randomize_layout;
/* Each of a tty's open files has private_data pointing to tty_file_private */
struct tty_file_private {
struct tty_struct *tty, struct file *filp);
static inline int tty_port_users(struct tty_port *port)
{
- return port->count + port->blocked_open;
+ return atomic_read(&port->count) + port->blocked_open;
}
extern int tty_register_ldisc(int disc, struct tty_ldisc_ops *new_ldisc);
void (*poll_put_char)(struct tty_driver *driver, int line, char ch);
#endif
const struct file_operations *proc_fops;
-};
+} __do_const __randomize_layout;
struct tty_driver {
int magic; /* magic number for this structure */
const struct tty_operations *ops;
struct list_head tty_drivers;
-};
+} __randomize_layout;
extern struct list_head tty_drivers;
struct module *owner;
- int refcount;
+ atomic_t refcount;
};
struct tty_ldisc {
int counter;
} atomic_t;
+#ifdef CONFIG_PAX_REFCOUNT
+typedef struct {
+ int counter;
+} atomic_unchecked_t;
+#else
+typedef atomic_t atomic_unchecked_t;
+#endif
+
#ifdef CONFIG_64BIT
typedef struct {
long counter;
} atomic64_t;
+
+#ifdef CONFIG_PAX_REFCOUNT
+typedef struct {
+ long counter;
+} atomic64_unchecked_t;
+#else
+typedef atomic64_t atomic64_unchecked_t;
+#endif
#endif
struct list_head {
long ret; \
mm_segment_t old_fs = get_fs(); \
\
- set_fs(KERNEL_DS); \
pagefault_disable(); \
- ret = __copy_from_user_inatomic(&(retval), (__force typeof(retval) __user *)(addr), sizeof(retval)); \
- pagefault_enable(); \
+ set_fs(KERNEL_DS); \
+ ret = __copy_from_user_inatomic(&(retval), (typeof(retval) __force_user *)(addr), sizeof(retval)); \
set_fs(old_fs); \
+ pagefault_enable(); \
ret; \
})
#endif /* CONFIG_USER_NS */
+#define GR_GLOBAL_UID(x) from_kuid_munged(&init_user_ns, (x))
+#define GR_GLOBAL_GID(x) from_kgid_munged(&init_user_ns, (x))
+#define gr_is_global_root(x) uid_eq((x), GLOBAL_ROOT_UID)
+#define gr_is_global_nonroot(x) (!uid_eq((x), GLOBAL_ROOT_UID))
+
#endif /* _LINUX_UIDGID_H */
struct module *owner;
struct device *dev;
int minor;
- atomic_t event;
+ atomic_unchecked_t event;
struct fasync_struct *async_queue;
wait_queue_head_t wait;
struct uio_info *info;
#include <linux/kernel.h>
#include <asm/byteorder.h>
-static inline u16 get_unaligned_le16(const void *p)
+static inline u16 __intentional_overflow(-1) get_unaligned_le16(const void *p)
{
- return le16_to_cpup((__le16 *)p);
+ return le16_to_cpup((const __le16 *)p);
}
-static inline u32 get_unaligned_le32(const void *p)
+static inline u32 __intentional_overflow(-1) get_unaligned_le32(const void *p)
{
- return le32_to_cpup((__le32 *)p);
+ return le32_to_cpup((const __le32 *)p);
}
-static inline u64 get_unaligned_le64(const void *p)
+static inline u64 __intentional_overflow(-1) get_unaligned_le64(const void *p)
{
- return le64_to_cpup((__le64 *)p);
+ return le64_to_cpup((const __le64 *)p);
}
-static inline u16 get_unaligned_be16(const void *p)
+static inline u16 __intentional_overflow(-1) get_unaligned_be16(const void *p)
{
- return be16_to_cpup((__be16 *)p);
+ return be16_to_cpup((const __be16 *)p);
}
-static inline u32 get_unaligned_be32(const void *p)
+static inline u32 __intentional_overflow(-1) get_unaligned_be32(const void *p)
{
- return be32_to_cpup((__be32 *)p);
+ return be32_to_cpup((const __be32 *)p);
}
-static inline u64 get_unaligned_be64(const void *p)
+static inline u64 __intentional_overflow(-1) get_unaligned_be64(const void *p)
{
- return be64_to_cpup((__be64 *)p);
+ return be64_to_cpup((const __be64 *)p);
}
static inline void put_unaligned_le16(u16 val, void *p)
int maxchild;
u32 quirks;
- atomic_t urbnum;
+ atomic_unchecked_t urbnum;
unsigned long active_duration;
extern int usb_control_msg(struct usb_device *dev, unsigned int pipe,
__u8 request, __u8 requesttype, __u16 value, __u16 index,
- void *data, __u16 size, int timeout);
+ void *data, __u16 size, int timeout) __intentional_overflow(-1);
extern int usb_interrupt_msg(struct usb_device *usb_dev, unsigned int pipe,
void *data, int len, int *actual_length, int timeout);
extern int usb_bulk_msg(struct usb_device *usb_dev, unsigned int pipe,
*/
struct renesas_usbhs_driver_callback {
int (*notify_hotplug)(struct platform_device *pdev);
-};
+} __no_const;
/*
* callback functions for platform
struct key *persistent_keyring_register;
struct rw_semaphore persistent_keyring_register_sem;
#endif
-};
+} __randomize_layout;
extern struct user_namespace init_user_ns;
struct new_utsname name;
struct user_namespace *user_ns;
struct ns_common ns;
-};
+} __randomize_layout;
extern struct uts_namespace init_uts_ns;
#ifdef CONFIG_UTS_NS
#define MODULE_ARCH_VERMAGIC ""
#endif
+#ifdef CONFIG_PAX_REFCOUNT
+#define MODULE_PAX_REFCOUNT "REFCOUNT "
+#else
+#define MODULE_PAX_REFCOUNT ""
+#endif
+
+#ifdef CONSTIFY_PLUGIN
+#define MODULE_CONSTIFY_PLUGIN "CONSTIFY_PLUGIN "
+#else
+#define MODULE_CONSTIFY_PLUGIN ""
+#endif
+
+#ifdef STACKLEAK_PLUGIN
+#define MODULE_STACKLEAK_PLUGIN "STACKLEAK_PLUGIN "
+#else
+#define MODULE_STACKLEAK_PLUGIN ""
+#endif
+
+#ifdef RANDSTRUCT_PLUGIN
+#include <generated/randomize_layout_hash.h>
+#define MODULE_RANDSTRUCT_PLUGIN "RANDSTRUCT_PLUGIN_" RANDSTRUCT_HASHED_SEED
+#else
+#define MODULE_RANDSTRUCT_PLUGIN
+#endif
+
+#ifdef CONFIG_GRKERNSEC
+#define MODULE_GRSEC "GRSEC "
+#else
+#define MODULE_GRSEC ""
+#endif
+
#define VERMAGIC_STRING \
UTS_RELEASE " " \
MODULE_VERMAGIC_SMP MODULE_VERMAGIC_PREEMPT \
MODULE_VERMAGIC_MODULE_UNLOAD MODULE_VERMAGIC_MODVERSIONS \
- MODULE_ARCH_VERMAGIC
+ MODULE_ARCH_VERMAGIC \
+ MODULE_PAX_REFCOUNT MODULE_CONSTIFY_PLUGIN MODULE_STACKLEAK_PLUGIN \
+ MODULE_GRSEC MODULE_RANDSTRUCT_PLUGIN
void vga_switcheroo_set_dynamic_switch(struct pci_dev *pdev, enum vga_switcheroo_state dynamic);
-int vga_switcheroo_init_domain_pm_ops(struct device *dev, struct dev_pm_domain *domain);
+int vga_switcheroo_init_domain_pm_ops(struct device *dev, dev_pm_domain_no_const *domain);
void vga_switcheroo_fini_domain_pm_ops(struct device *dev);
-int vga_switcheroo_init_domain_pm_optimus_hdmi_audio(struct device *dev, struct dev_pm_domain *domain);
+int vga_switcheroo_init_domain_pm_optimus_hdmi_audio(struct device *dev, dev_pm_domain_no_const *domain);
#else
static inline void vga_switcheroo_unregister_client(struct pci_dev *dev) {}
static inline void vga_switcheroo_set_dynamic_switch(struct pci_dev *pdev, enum vga_switcheroo_state dynamic) {}
-static inline int vga_switcheroo_init_domain_pm_ops(struct device *dev, struct dev_pm_domain *domain) { return -EINVAL; }
+static inline int vga_switcheroo_init_domain_pm_ops(struct device *dev, dev_pm_domain_no_const *domain) { return -EINVAL; }
static inline void vga_switcheroo_fini_domain_pm_ops(struct device *dev) {}
-static inline int vga_switcheroo_init_domain_pm_optimus_hdmi_audio(struct device *dev, struct dev_pm_domain *domain) { return -EINVAL; }
+static inline int vga_switcheroo_init_domain_pm_optimus_hdmi_audio(struct device *dev, dev_pm_domain_no_const *domain) { return -EINVAL; }
#endif
#endif /* _LINUX_VGA_SWITCHEROO_H_ */
#define VM_USERMAP 0x00000008 /* suitable for remap_vmalloc_range */
#define VM_VPAGES 0x00000010 /* buffer for pages was vmalloc'ed */
#define VM_UNINITIALIZED 0x00000020 /* vm_struct is not fully initialized */
+
+#if defined(CONFIG_X86) && defined(CONFIG_PAX_KERNEXEC)
+#define VM_KERNEXEC 0x00000040 /* allocate from executable kernel memory range */
+#endif
+
/* bits [20..32] reserved for arch specific ioremap internals */
/*
unsigned long flags, pgprot_t prot);
extern void vunmap(const void *addr);
+#ifdef CONFIG_GRKERNSEC_KSTACKOVERFLOW
+extern void unmap_process_stacks(struct task_struct *task);
+#endif
+
extern int remap_vmalloc_range_partial(struct vm_area_struct *vma,
unsigned long uaddr, void *kaddr,
unsigned long size);
/* for /dev/kmem */
extern long vread(char *buf, char *addr, unsigned long count);
-extern long vwrite(char *buf, char *addr, unsigned long count);
+extern long vwrite(char *buf, char *addr, unsigned long count) __size_overflow(3);
/*
* Internals. Dont't use..
/*
* Zone based page accounting with per cpu differentials.
*/
-extern atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS];
+extern atomic_long_unchecked_t vm_stat[NR_VM_ZONE_STAT_ITEMS];
static inline void zone_page_state_add(long x, struct zone *zone,
enum zone_stat_item item)
{
- atomic_long_add(x, &zone->vm_stat[item]);
- atomic_long_add(x, &vm_stat[item]);
+ atomic_long_add_unchecked(x, &zone->vm_stat[item]);
+ atomic_long_add_unchecked(x, &vm_stat[item]);
}
-static inline unsigned long global_page_state(enum zone_stat_item item)
+static inline unsigned long __intentional_overflow(-1) global_page_state(enum zone_stat_item item)
{
- long x = atomic_long_read(&vm_stat[item]);
+ long x = atomic_long_read_unchecked(&vm_stat[item]);
#ifdef CONFIG_SMP
if (x < 0)
x = 0;
return x;
}
-static inline unsigned long zone_page_state(struct zone *zone,
+static inline unsigned long __intentional_overflow(-1) zone_page_state(struct zone *zone,
enum zone_stat_item item)
{
- long x = atomic_long_read(&zone->vm_stat[item]);
+ long x = atomic_long_read_unchecked(&zone->vm_stat[item]);
#ifdef CONFIG_SMP
if (x < 0)
x = 0;
static inline unsigned long zone_page_state_snapshot(struct zone *zone,
enum zone_stat_item item)
{
- long x = atomic_long_read(&zone->vm_stat[item]);
+ long x = atomic_long_read_unchecked(&zone->vm_stat[item]);
#ifdef CONFIG_SMP
int cpu;
static inline void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
{
- atomic_long_inc(&zone->vm_stat[item]);
- atomic_long_inc(&vm_stat[item]);
+ atomic_long_inc_unchecked(&zone->vm_stat[item]);
+ atomic_long_inc_unchecked(&vm_stat[item]);
}
static inline void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
{
- atomic_long_dec(&zone->vm_stat[item]);
- atomic_long_dec(&vm_stat[item]);
+ atomic_long_dec_unchecked(&zone->vm_stat[item]);
+ atomic_long_dec_unchecked(&vm_stat[item]);
}
static inline void __inc_zone_page_state(struct page *page,
size_t size, int handler_flags);
int (*set)(struct dentry *dentry, const char *name, const void *buffer,
size_t size, int flags, int handler_flags);
-};
+} __do_const;
struct xattr {
const char *name;
};
ssize_t xattr_getsecurity(struct inode *, const char *, void *, size_t);
+#ifdef CONFIG_PAX_XATTR_PAX_FLAGS
+ssize_t pax_getxattr(struct dentry *, void *, size_t);
+#endif
ssize_t vfs_getxattr(struct dentry *, const char *, void *, size_t);
ssize_t vfs_listxattr(struct dentry *d, char *list, size_t size);
int __vfs_setxattr_noperm(struct dentry *, const char *, const void *, size_t, int);
#define _ZLIB_H
#include <linux/zconf.h>
+#include <linux/compiler.h>
/* zlib deflate based on ZLIB_VERSION "1.1.3" */
/* zlib inflate based on ZLIB_VERSION "1.2.3" */
/* basic functions */
-extern int zlib_deflate_workspacesize (int windowBits, int memLevel);
+extern int zlib_deflate_workspacesize (int windowBits, int memLevel) __intentional_overflow(0);
/*
Returns the number of bytes that needs to be allocated for a per-
stream workspace with the specified parameters. A pointer to this
int (*mmap) (struct file *, struct vm_area_struct *);
int (*open) (struct file *);
int (*release) (struct file *);
-};
+} __do_const;
/*
* Newer version of video_device, handled by videodev2.c
this function returns 0. If the name ends with a digit (e.g. cx18),
then the name will be set to cx18-0 since cx180 looks really odd. */
int v4l2_device_set_name(struct v4l2_device *v4l2_dev, const char *basename,
- atomic_t *instance);
+ atomic_unchecked_t *instance);
/* Set v4l2_dev->dev to NULL. Call when the USB parent disconnects.
Since the parent disappears this ensures that v4l2_dev doesn't have an
int (*cancelled)(struct p9_client *, struct p9_req_t *req);
int (*zc_request)(struct p9_client *, struct p9_req_t *,
char *, char *, int , int, int, int);
-};
+} __do_const;
void v9fs_register_trans(struct p9_trans_module *m);
void v9fs_unregister_trans(struct p9_trans_module *m);
u32 secid; /* Security ID */
#endif
u32 consumed;
-};
+} __randomize_layout;
#define UNIXCB(skb) (*(struct unix_skb_parms *)&((skb)->cb))
#define UNIXSID(skb) (&UNIXCB((skb)).secid)
struct sk_buff *(*alloc_skb) (struct l2cap_chan *chan,
unsigned long hdr_len,
unsigned long len, int nb);
-};
+} __do_const;
struct l2cap_conn {
struct hci_conn *hcon;
static inline void bond_tx_drop(struct net_device *dev, struct sk_buff *skb)
{
- atomic_long_inc(&dev->tx_dropped);
+ atomic_long_inc_unchecked(&dev->tx_dropped);
dev_kfree_skb_any(skb);
}
void (*radioset_rsp)(void);
void (*reject_rsp)(struct cflayer *layer, u8 linkid,
struct cflayer *client_layer);
-};
+} __no_const;
/* Link Setup Parameters for CAIF-Links. */
struct cfctrl_link_param {
struct cfctrl {
struct cfsrvl serv;
struct cfctrl_rsp res;
- atomic_t req_seq_no;
- atomic_t rsp_seq_no;
+ atomic_unchecked_t req_seq_no;
+ atomic_unchecked_t rsp_seq_no;
struct list_head list;
/* Protects from simultaneous access to first_req list */
spinlock_t info_list_lock;
void flow_cache_flush(struct net *net);
void flow_cache_flush_deferred(struct net *net);
-extern atomic_t flow_cache_genid;
+extern atomic_unchecked_t flow_cache_genid;
#endif
u8 cmd;
u8 internal_flags;
u8 flags;
-};
+} __do_const;
int __genl_register_family(struct genl_family *family);
cell += skb_get_rx_queue(skb) & gcells->gro_cells_mask;
if (skb_queue_len(&cell->napi_skbs) > netdev_max_backlog) {
- atomic_long_inc(&dev->rx_dropped);
+ atomic_long_inc_unchecked(&dev->rx_dropped);
kfree_skb(skb);
return;
}
int (*bind_conflict)(const struct sock *sk,
const struct inet_bind_bucket *tb, bool relax);
void (*mtu_reduced)(struct sock *sk);
-};
+} __do_const;
/** inet_connection_sock - INET connection oriented sock
*
*/
union {
struct {
- atomic_t rid; /* Frag reception counter */
+ atomic_unchecked_t rid; /* Frag reception counter */
};
struct rcu_head rcu;
struct inet_peer *gc_next;
}
}
-u32 ip_idents_reserve(u32 hash, int segs);
+u32 ip_idents_reserve(u32 hash, int segs) __intentional_overflow(-1);
void __ip_select_ident(struct iphdr *iph, int segs);
static inline void ip_select_ident_segs(struct sk_buff *skb, struct sock *sk, int segs)
#define FIB_RES_SADDR(net, res) \
((FIB_RES_NH(res).nh_saddr_genid == \
- atomic_read(&(net)->ipv4.dev_addr_genid)) ? \
+ atomic_read_unchecked(&(net)->ipv4.dev_addr_genid)) ? \
FIB_RES_NH(res).nh_saddr : \
fib_info_update_nh_saddr((net), &FIB_RES_NH(res)))
#define FIB_RES_GW(res) (FIB_RES_NH(res).nh_gw)
struct ip_vs_conn *control; /* Master control connection */
atomic_t n_control; /* Number of controlled ones */
struct ip_vs_dest *dest; /* real server */
- atomic_t in_pkts; /* incoming packet counter */
+ atomic_unchecked_t in_pkts; /* incoming packet counter */
/* Packet transmitter for different forwarding methods. If it
* mangles the packet, it must return NF_DROP or better NF_STOLEN,
__be16 port; /* port number of the server */
union nf_inet_addr addr; /* IP address of the server */
volatile unsigned int flags; /* dest status flags */
- atomic_t conn_flags; /* flags to copy to conn */
+ atomic_unchecked_t conn_flags; /* flags to copy to conn */
atomic_t weight; /* server weight */
atomic_t refcnt; /* reference counter */
/* ip_vs_lblc */
int sysctl_lblc_expiration;
struct ctl_table_header *lblc_ctl_header;
- struct ctl_table *lblc_ctl_table;
+ ctl_table_no_const *lblc_ctl_table;
/* ip_vs_lblcr */
int sysctl_lblcr_expiration;
struct ctl_table_header *lblcr_ctl_header;
- struct ctl_table *lblcr_ctl_table;
+ ctl_table_no_const *lblcr_ctl_table;
/* ip_vs_est */
struct list_head est_list; /* estimator list */
spinlock_t est_lock;
#include <linux/termios.h>
#include <linux/timer.h>
#include <linux/tty.h> /* struct tty_struct */
+#include <asm/local.h>
#include <net/irda/irias_object.h>
#include <net/irda/ircomm_core.h>
struct iucv_sock_list {
struct hlist_head head;
rwlock_t lock;
- atomic_t autobind_name;
+ atomic_unchecked_t autobind_name;
};
unsigned int iucv_sock_poll(struct file *file, struct socket *sock,
#define LLC_CONN_AC_STOP_SENDACK_TMR 70
#define LLC_CONN_AC_START_SENDACK_TMR_IF_NOT_RUNNING 71
-typedef int (*llc_conn_action_t)(struct sock *sk, struct sk_buff *skb);
+typedef int (* const llc_conn_action_t)(struct sock *sk, struct sk_buff *skb);
int llc_conn_ac_clear_remote_busy(struct sock *sk, struct sk_buff *skb);
int llc_conn_ac_conn_ind(struct sock *sk, struct sk_buff *skb);
return (struct llc_conn_state_ev *)skb->cb;
}
-typedef int (*llc_conn_ev_t)(struct sock *sk, struct sk_buff *skb);
-typedef int (*llc_conn_ev_qfyr_t)(struct sock *sk, struct sk_buff *skb);
+typedef int (* const llc_conn_ev_t)(struct sock *sk, struct sk_buff *skb);
+typedef int (* const llc_conn_ev_qfyr_t)(struct sock *sk, struct sk_buff *skb);
int llc_conn_ev_conn_req(struct sock *sk, struct sk_buff *skb);
int llc_conn_ev_data_req(struct sock *sk, struct sk_buff *skb);
u8 next_state;
const llc_conn_ev_qfyr_t *ev_qualifiers;
const llc_conn_action_t *ev_actions;
-};
+} __do_const;
struct llc_conn_state {
u8 current_state;
#define SAP_ACT_TEST_IND 9
/* All action functions must look like this */
-typedef int (*llc_sap_action_t)(struct llc_sap *sap, struct sk_buff *skb);
+typedef int (* const llc_sap_action_t)(struct llc_sap *sap, struct sk_buff *skb);
int llc_sap_action_unitdata_ind(struct llc_sap *sap, struct sk_buff *skb);
int llc_sap_action_send_ui(struct llc_sap *sap, struct sk_buff *skb);
llc_sap_ev_t ev;
u8 next_state;
const llc_sap_action_t *ev_actions;
-};
+} __do_const;
struct llc_sap_state {
u8 curr_state;
void (*remove_sta_debugfs)(void *priv, void *priv_sta);
u32 (*get_expected_throughput)(void *priv_sta);
-};
+} __do_const;
static inline int rate_supported(struct ieee80211_sta *sta,
enum ieee80211_band band,
void (*error_report)(struct neighbour *, struct sk_buff *);
int (*output)(struct neighbour *, struct sk_buff *);
int (*connected_output)(struct neighbour *, struct sk_buff *);
-};
+} __do_const;
struct pneigh_entry {
struct pneigh_entry *next;
struct neigh_statistics __percpu *stats;
struct neigh_hash_table __rcu *nht;
struct pneigh_entry **phash_buckets;
-};
+} __randomize_layout;
enum {
NEIGH_ARP_TABLE = 0,
struct netns_ipvs *ipvs;
#endif
struct sock *diag_nlsk;
- atomic_t fnhe_genid;
-};
+ atomic_unchecked_t fnhe_genid;
+} __randomize_layout;
#include <linux/seq_file_net.h>
#define __net_init __init
#define __net_exit __exit_refok
#define __net_initdata __initdata
+#ifdef CONSTIFY_PLUGIN
#define __net_initconst __initconst
+#else
+#define __net_initconst __initdata
+#endif
#endif
struct pernet_operations {
void (*exit_batch)(struct list_head *net_exit_list);
int *id;
size_t size;
-};
+} __do_const;
/*
* Use these carefully. If you implement a network device and it
static inline int rt_genid_ipv4(struct net *net)
{
- return atomic_read(&net->ipv4.rt_genid);
+ return atomic_read_unchecked(&net->ipv4.rt_genid);
}
static inline void rt_genid_bump_ipv4(struct net *net)
{
- atomic_inc(&net->ipv4.rt_genid);
+ atomic_inc_unchecked(&net->ipv4.rt_genid);
}
extern void (*__fib6_flush_trees)(struct net *net);
static inline int fnhe_genid(struct net *net)
{
- return atomic_read(&net->fnhe_genid);
+ return atomic_read_unchecked(&net->fnhe_genid);
}
static inline void fnhe_genid_bump(struct net *net)
{
- atomic_inc(&net->fnhe_genid);
+ atomic_inc_unchecked(&net->fnhe_genid);
}
#endif /* __NET_NET_NAMESPACE_H */
static inline void nlmsg_trim(struct sk_buff *skb, const void *mark)
{
if (mark)
- skb_trim(skb, (unsigned char *) mark - skb->data);
+ skb_trim(skb, (const unsigned char *) mark - skb->data);
}
/**
struct nf_proto_net {
#ifdef CONFIG_SYSCTL
struct ctl_table_header *ctl_table_header;
- struct ctl_table *ctl_table;
+ ctl_table_no_const *ctl_table;
#ifdef CONFIG_NF_CONNTRACK_PROC_COMPAT
struct ctl_table_header *ctl_compat_header;
- struct ctl_table *ctl_compat_table;
+ ctl_table_no_const *ctl_compat_table;
#endif
#endif
unsigned int users;
struct nf_icmp_net icmpv6;
#if defined(CONFIG_SYSCTL) && defined(CONFIG_NF_CONNTRACK_PROC_COMPAT)
struct ctl_table_header *ctl_table_header;
- struct ctl_table *ctl_table;
+ ctl_table_no_const *ctl_table;
#endif
};
struct ping_group_range ping_group_range;
- atomic_t dev_addr_genid;
+ atomic_unchecked_t dev_addr_genid;
#ifdef CONFIG_SYSCTL
unsigned long *sysctl_local_reserved_ports;
struct fib_rules_ops *mr_rules_ops;
#endif
#endif
- atomic_t rt_genid;
+ atomic_unchecked_t rt_genid;
};
#endif
struct fib_rules_ops *mr6_rules_ops;
#endif
#endif
- atomic_t dev_addr_genid;
- atomic_t fib6_sernum;
+ atomic_unchecked_t dev_addr_genid;
+ atomic_unchecked_t fib6_sernum;
};
#if IS_ENABLED(CONFIG_NF_DEFRAG_IPV6)
/* flow cache part */
struct flow_cache flow_cache_global;
- atomic_t flow_cache_genid;
+ atomic_unchecked_t flow_cache_genid;
struct list_head flow_cache_gc_list;
spinlock_t flow_cache_gc_lock;
struct work_struct flow_cache_gc_work;
extern struct proto ping_prot;
#if IS_ENABLED(CONFIG_IPV6)
-extern struct pingv6_ops pingv6_ops;
+extern struct pingv6_ops *pingv6_ops;
#endif
struct pingfakehdr {
* socket lookup?
*/
icmp_strict_tag_validation:1;
-};
+} __do_const;
#if IS_ENABLED(CONFIG_IPV6)
struct inet6_protocol {
u8 type, u8 code, int offset,
__be32 info);
unsigned int flags; /* INET6_PROTO_xxx */
-};
+} __do_const;
#define INET6_PROTO_NOPOLICY 0x1
#define INET6_PROTO_FINAL 0x2
int (*fill_slave_info)(struct sk_buff *skb,
const struct net_device *dev,
const struct net_device *slave_dev);
-};
+} __do_const;
int __rtnl_link_register(struct rtnl_link_ops *ops);
void __rtnl_link_unregister(struct rtnl_link_ops *ops);
unsigned int offset)
{
struct sctphdr *sh = sctp_hdr(skb);
- __le32 ret, old = sh->checksum;
- const struct skb_checksum_ops ops = {
+ __le32 ret, old = sh->checksum;
+ static const struct skb_checksum_ops ops = {
.update = sctp_csum_update,
.combine = sctp_csum_combine,
};
typedef struct {
sctp_state_fn_t *fn;
const char *name;
-} sctp_sm_table_entry_t;
+} __do_const sctp_sm_table_entry_t;
/* A naming convention of "sctp_sf_xxx" applies to all the state functions
* currently in use.
__u32 sctp_generate_tsn(const struct sctp_endpoint *);
/* Extern declarations for major data structures. */
-extern sctp_timer_event_t *sctp_timer_events[SCTP_NUM_TIMEOUT_TYPES];
+extern sctp_timer_event_t * const sctp_timer_events[SCTP_NUM_TIMEOUT_TYPES];
/* Get the size of a DATA chunk payload. */
void (*to_sk_saddr)(union sctp_addr *, struct sock *sk);
void (*to_sk_daddr)(union sctp_addr *, struct sock *sk);
struct sctp_af *af;
-};
+} __do_const;
/* Structure to track chunk fragments that have been acked, but peer
unsigned int sk_napi_id;
unsigned int sk_ll_usec;
#endif
- atomic_t sk_drops;
+ atomic_unchecked_t sk_drops;
int sk_rcvbuf;
struct sk_filter __rcu *sk_filter;
void (*destroy_cgroup)(struct mem_cgroup *memcg);
struct cg_proto *(*proto_cgroup)(struct mem_cgroup *memcg);
#endif
-};
+} __randomize_layout;
/*
* Bits in struct cg_proto.flags
page_counter_uncharge(&prot->memory_allocated, amt);
}
-static inline long
+static inline long __intentional_overflow(-1)
sk_memory_allocated(const struct sock *sk)
{
struct proto *prot = sk->sk_prot;
struct scm_cookie *scm;
struct msghdr *msg, async_msg;
struct kiocb *kiocb;
-};
+} __randomize_layout;
static inline struct sock_iocb *kiocb_to_siocb(struct kiocb *iocb)
{
}
static inline int skb_do_copy_data_nocache(struct sock *sk, struct sk_buff *skb,
- char __user *from, char *to,
+ char __user *from, unsigned char *to,
int copy, int offset)
{
if (skb->ip_summed == CHECKSUM_NONE) {
}
}
-struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp);
+struct sk_buff * __intentional_overflow(0) sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp);
/**
* sk_page_frag - return an appropriate page_frag
void tcp_xmit_retransmit_queue(struct sock *);
void tcp_simple_retransmit(struct sock *);
int tcp_trim_head(struct sock *, struct sk_buff *, u32);
-int tcp_fragment(struct sock *, struct sk_buff *, u32, unsigned int, gfp_t);
+int __intentional_overflow(3) tcp_fragment(struct sock *, struct sk_buff *, u32, unsigned int, gfp_t);
void tcp_send_probe0(struct sock *);
void tcp_send_partial(struct sock *);
* If this grows please adjust skbuff.h:skbuff->cb[xxx] size appropriately.
*/
struct tcp_skb_cb {
- __u32 seq; /* Starting sequence number */
- __u32 end_seq; /* SEQ + FIN + SYN + datalen */
+ __u32 seq __intentional_overflow(0); /* Starting sequence number */
+ __u32 end_seq __intentional_overflow(0); /* SEQ + FIN + SYN + datalen */
union {
/* Note : tcp_tw_isn is used in input path only
* (isn chosen by tcp_timewait_state_process())
__u8 ip_dsfield; /* IPv4 tos or IPv6 dsfield */
/* 1 byte hole */
- __u32 ack_seq; /* Sequence number ACK'd */
+ __u32 ack_seq __intentional_overflow(0); /* Sequence number ACK'd */
union {
struct inet_skb_parm h4;
#if IS_ENABLED(CONFIG_IPV6)
struct xfrm_policy_afinfo {
unsigned short family;
struct dst_ops *dst_ops;
- void (*garbage_collect)(struct net *net);
struct dst_entry *(*dst_lookup)(struct net *net, int tos,
const xfrm_address_t *saddr,
const xfrm_address_t *daddr);
struct net_device *dev,
const struct flowi *fl);
struct dst_entry *(*blackhole_route)(struct net *net, struct dst_entry *orig);
-};
+} __do_const;
int xfrm_policy_register_afinfo(struct xfrm_policy_afinfo *afinfo);
int xfrm_policy_unregister_afinfo(struct xfrm_policy_afinfo *afinfo);
int (*transport_finish)(struct sk_buff *skb,
int async);
void (*local_error)(struct sk_buff *skb, u32 mtu);
-};
+} __do_const;
int xfrm_state_register_afinfo(struct xfrm_state_afinfo *afinfo);
int xfrm_state_unregister_afinfo(struct xfrm_state_afinfo *afinfo);
struct module *owner;
unsigned int encap;
int flags;
-};
+} __do_const;
/* Flags for xfrm_mode. */
enum {
struct timer_list timer;
struct flow_cache_object flo;
- atomic_t genid;
+ atomic_unchecked_t genid;
u32 priority;
u32 index;
struct xfrm_mark mark;
}
void xfrm_garbage_collect(struct net *net);
+void xfrm_garbage_collect_deferred(struct net *net);
#else
static inline void xfrm_garbage_collect(struct net *net)
{
}
+static inline void xfrm_garbage_collect_deferred(struct net *net)
+{
+}
#endif
static __inline__
int backlog);
int (*destroy_listen)(struct iw_cm_id *cm_id);
-};
+} __no_const;
/**
* iw_create_cm_id - Create an IW CM identifier.
*/
void (*disc_stop_final) (struct fc_lport *);
};
+typedef struct libfc_function_template __no_const libfc_function_template_no_const;
/**
* struct fc_disc - Discovery context
struct fc_vport *vport;
/* Operational Information */
- struct libfc_function_template tt;
+ libfc_function_template_no_const tt;
u8 link_up;
u8 qfull;
enum fc_lport_state state;
unsigned int max_device_blocked; /* what device_blocked counts down from */
#define SCSI_DEFAULT_DEVICE_BLOCKED 3
- atomic_t iorequest_cnt;
- atomic_t iodone_cnt;
- atomic_t ioerr_cnt;
+ atomic_unchecked_t iorequest_cnt;
+ atomic_unchecked_t iodone_cnt;
+ atomic_unchecked_t ioerr_cnt;
struct device sdev_gendev,
sdev_dev;
unsigned long show_host_system_hostname:1;
unsigned long disable_target_scan:1;
-};
+} __do_const;
+typedef struct fc_function_template __no_const fc_function_template_no_const;
/**
struct snd_compr_caps *caps);
int (*get_codec_caps) (struct snd_compr_stream *stream,
struct snd_compr_codec_caps *codec);
-};
+} __no_const;
/**
* struct snd_compr: Compressed device
enum snd_soc_dapm_type, int);
bool ignore_pmdown_time; /* Doesn't benefit from pmdown delay */
-};
+} __do_const;
/* SoC platform interface */
struct snd_soc_platform_driver {
const struct snd_compr_ops *compr_ops;
int (*bespoke_trigger)(struct snd_pcm_substream *, int);
-};
+} __do_const;
struct snd_soc_dai_link_component {
const char *name;
atomic_long_t write_bytes;
/* Active commands on this virtual SE device */
atomic_t simple_cmds;
- atomic_t dev_ordered_id;
+ atomic_unchecked_t dev_ordered_id;
atomic_t dev_ordered_sync;
atomic_t dev_qf_count;
int export_count;
*/
TRACE_EVENT(irq_handler_entry,
- TP_PROTO(int irq, struct irqaction *action),
+ TP_PROTO(int irq, const struct irqaction *action),
TP_ARGS(irq, action),
*/
TRACE_EVENT(irq_handler_exit,
- TP_PROTO(int irq, struct irqaction *action, int ret),
+ TP_PROTO(int irq, const struct irqaction *action, int ret),
TP_ARGS(irq, action, ret),
M_MIPS2 = 152 /* MIPS R6000/R4000 binary */
};
+/* Constants for the N_FLAGS field */
+#define F_PAX_PAGEEXEC 1 /* Paging based non-executable pages */
+#define F_PAX_EMUTRAMP 2 /* Emulate trampolines */
+#define F_PAX_MPROTECT 4 /* Restrict mprotect() */
+#define F_PAX_RANDMMAP 8 /* Randomize mmap() base */
+/*#define F_PAX_RANDEXEC 16*/ /* Randomize ET_EXEC base */
+#define F_PAX_SEGMEXEC 32 /* Segmentation based non-executable pages */
+
#if !defined (N_MAGIC)
#define N_MAGIC(exec) ((exec).a_info & 0xffff)
#endif
* Bcache on disk data structures
*/
+#include <linux/compiler.h>
#include <asm/types.h>
#define BITMASK(name, type, field, offset, size) \
/* Btree keys - all units are in sectors */
struct bkey {
- __u64 high;
- __u64 low;
+ __u64 high __intentional_overflow(-1);
+ __u64 low __intentional_overflow(-1);
__u64 ptr[];
};
static inline __le64 __cpu_to_le64p(const __u64 *p)
{
- return (__force __le64)*p;
+ return (__force const __le64)*p;
}
-static inline __u64 __le64_to_cpup(const __le64 *p)
+static inline __u64 __intentional_overflow(-1) __le64_to_cpup(const __le64 *p)
{
- return (__force __u64)*p;
+ return (__force const __u64)*p;
}
static inline __le32 __cpu_to_le32p(const __u32 *p)
{
- return (__force __le32)*p;
+ return (__force const __le32)*p;
}
static inline __u32 __le32_to_cpup(const __le32 *p)
{
- return (__force __u32)*p;
+ return (__force const __u32)*p;
}
static inline __le16 __cpu_to_le16p(const __u16 *p)
{
- return (__force __le16)*p;
+ return (__force const __le16)*p;
}
static inline __u16 __le16_to_cpup(const __le16 *p)
{
- return (__force __u16)*p;
+ return (__force const __u16)*p;
}
static inline __be64 __cpu_to_be64p(const __u64 *p)
{
- return (__force __be64)__swab64p(p);
+ return (__force const __be64)__swab64p(p);
}
static inline __u64 __be64_to_cpup(const __be64 *p)
{
- return __swab64p((__u64 *)p);
+ return __swab64p((const __u64 *)p);
}
static inline __be32 __cpu_to_be32p(const __u32 *p)
{
- return (__force __be32)__swab32p(p);
+ return (__force const __be32)__swab32p(p);
}
-static inline __u32 __be32_to_cpup(const __be32 *p)
+static inline __u32 __intentional_overflow(-1) __be32_to_cpup(const __be32 *p)
{
- return __swab32p((__u32 *)p);
+ return __swab32p((const __u32 *)p);
}
static inline __be16 __cpu_to_be16p(const __u16 *p)
{
- return (__force __be16)__swab16p(p);
+ return (__force const __be16)__swab16p(p);
}
static inline __u16 __be16_to_cpup(const __be16 *p)
{
- return __swab16p((__u16 *)p);
+ return __swab16p((const __u16 *)p);
}
#define __cpu_to_le64s(x) do { (void)(x); } while (0)
#define __le64_to_cpus(x) do { (void)(x); } while (0)
#define PT_GNU_EH_FRAME 0x6474e550
#define PT_GNU_STACK (PT_LOOS + 0x474e551)
+#define PT_GNU_RELRO (PT_LOOS + 0x474e552)
+
+#define PT_PAX_FLAGS (PT_LOOS + 0x5041580)
+
+/* Constants for the e_flags field */
+#define EF_PAX_PAGEEXEC 1 /* Paging based non-executable pages */
+#define EF_PAX_EMUTRAMP 2 /* Emulate trampolines */
+#define EF_PAX_MPROTECT 4 /* Restrict mprotect() */
+#define EF_PAX_RANDMMAP 8 /* Randomize mmap() base */
+/*#define EF_PAX_RANDEXEC 16*/ /* Randomize ET_EXEC base */
+#define EF_PAX_SEGMEXEC 32 /* Segmentation based non-executable pages */
/*
* Extended Numbering
#define DT_DEBUG 21
#define DT_TEXTREL 22
#define DT_JMPREL 23
+#define DT_FLAGS 30
+ #define DF_TEXTREL 0x00000004
#define DT_ENCODING 32
#define OLD_DT_LOOS 0x60000000
#define DT_LOOS 0x6000000d
#define PF_W 0x2
#define PF_X 0x1
+#define PF_PAGEEXEC (1U << 4) /* Enable PAGEEXEC */
+#define PF_NOPAGEEXEC (1U << 5) /* Disable PAGEEXEC */
+#define PF_SEGMEXEC (1U << 6) /* Enable SEGMEXEC */
+#define PF_NOSEGMEXEC (1U << 7) /* Disable SEGMEXEC */
+#define PF_MPROTECT (1U << 8) /* Enable MPROTECT */
+#define PF_NOMPROTECT (1U << 9) /* Disable MPROTECT */
+/*#define PF_RANDEXEC (1U << 10)*/ /* Enable RANDEXEC */
+/*#define PF_NORANDEXEC (1U << 11)*/ /* Disable RANDEXEC */
+#define PF_EMUTRAMP (1U << 12) /* Enable EMUTRAMP */
+#define PF_NOEMUTRAMP (1U << 13) /* Disable EMUTRAMP */
+#define PF_RANDMMAP (1U << 14) /* Enable RANDMMAP */
+#define PF_NORANDMMAP (1U << 15) /* Disable RANDMMAP */
+
typedef struct elf32_phdr{
Elf32_Word p_type;
Elf32_Off p_offset;
#define EI_OSABI 7
#define EI_PAD 8
+#define EI_PAX 14
+
#define ELFMAG0 0x7f /* EI_MAG */
#define ELFMAG1 'E'
#define ELFMAG2 'L'
#define PER_CLEAR_ON_SETID (READ_IMPLIES_EXEC | \
ADDR_NO_RANDOMIZE | \
ADDR_COMPAT_LAYOUT | \
+ ADDR_LIMIT_3GB | \
MMAP_PAGE_ZERO)
/*
__u16 pages; /* 0x32 */
__u16 vesa_attributes; /* 0x34 */
__u32 capabilities; /* 0x36 */
- __u8 _reserved[6]; /* 0x3a */
+ __u16 vesapm_size; /* 0x3a */
+ __u8 _reserved[4]; /* 0x3c */
} __attribute__((packed));
#define VIDEO_TYPE_MDA 0x10 /* Monochrome Text Display */
* ___swab16, ___swab32, ___swab64, ___swahw32, ___swahb32
*/
-static inline __attribute_const__ __u16 __fswab16(__u16 val)
+static inline __intentional_overflow(-1) __attribute_const__ __u16 __fswab16(__u16 val)
{
#ifdef __HAVE_BUILTIN_BSWAP16__
return __builtin_bswap16(val);
#endif
}
-static inline __attribute_const__ __u32 __fswab32(__u32 val)
+static inline __intentional_overflow(-1) __attribute_const__ __u32 __fswab32(__u32 val)
{
#ifdef __HAVE_BUILTIN_BSWAP32__
return __builtin_bswap32(val);
#endif
}
-static inline __attribute_const__ __u64 __fswab64(__u64 val)
+static inline __intentional_overflow(-1) __attribute_const__ __u64 __fswab64(__u64 val)
{
#ifdef __HAVE_BUILTIN_BSWAP64__
return __builtin_bswap64(val);
#define XATTR_POSIX_ACL_DEFAULT "posix_acl_default"
#define XATTR_NAME_POSIX_ACL_DEFAULT XATTR_SYSTEM_PREFIX XATTR_POSIX_ACL_DEFAULT
+/* User namespace */
+#define XATTR_PAX_PREFIX XATTR_USER_PREFIX "pax."
+#define XATTR_PAX_FLAGS_SUFFIX "flags"
+#define XATTR_NAME_PAX_FLAGS XATTR_PAX_PREFIX XATTR_PAX_FLAGS_SUFFIX
#endif /* _UAPI_LINUX_XATTR_H */
u32 pseudo_palette[256];
int blank_mode; /*one of FB_BLANK_ */
/* blit-only rendering path metrics, exposed through sysfs */
- atomic_t bytes_rendered; /* raw pixel-bytes driver asked to render */
- atomic_t bytes_identical; /* saved effort with backbuffer comparison */
- atomic_t bytes_sent; /* to usb, after compression including overhead */
- atomic_t cpu_kcycles_used; /* transpired during pixel processing */
+ atomic_unchecked_t bytes_rendered; /* raw pixel-bytes driver asked to render */
+ atomic_unchecked_t bytes_identical; /* saved effort with backbuffer comparison */
+ atomic_unchecked_t bytes_sent; /* to usb, after compression including overhead */
+ atomic_unchecked_t cpu_kcycles_used; /* transpired during pixel processing */
};
#define NR_USB_REQUEST_I2C_SUB_IO 0x02
u8 ypan; /* 0 - nothing, 1 - ypan, 2 - ywrap */
u8 pmi_setpal; /* PMI for palette changes */
u16 *pmi_base; /* protected mode interface location */
+ u8 *pmi_code; /* protected mode code location */
void *pmi_start;
void *pmi_pal;
u8 *vbe_state_orig; /*
config CHECKPOINT_RESTORE
bool "Checkpoint/restore support" if EXPERT
+ depends on !GRKERNSEC
default n
help
Enables additional kernel features in a sake of checkpoint/restore.
config COMPAT_BRK
bool "Disable heap randomization"
- default y
+ default n
help
Randomizing heap placement makes heap exploits harder, but it
also breaks ancient binaries (including anything libc5 based).
config STOP_MACHINE
bool
default y
- depends on (SMP && MODULE_UNLOAD) || HOTPLUG_CPU
+ depends on (SMP && MODULE_UNLOAD) || HOTPLUG_CPU || GRKERNSEC
help
Need stop_machine() primitive.
# Makefile for the linux kernel.
#
+ccflags-y := $(GCC_PLUGINS_CFLAGS)
+asflags-y := $(GCC_PLUGINS_AFLAGS)
+
obj-y := main.o version.o mounts.o
ifneq ($(CONFIG_BLK_DEV_INITRD),y)
obj-y += noinitramfs.o
static int __init do_mount_root(char *name, char *fs, int flags, void *data)
{
struct super_block *s;
- int err = sys_mount(name, "/root", fs, flags, data);
+ int err = sys_mount((char __force_user *)name, (char __force_user *)"/root", (char __force_user *)fs, flags, (void __force_user *)data);
if (err)
return err;
- sys_chdir("/root");
+ sys_chdir((const char __force_user *)"/root");
s = current->fs->pwd.dentry->d_sb;
ROOT_DEV = s->s_dev;
printk(KERN_INFO
va_start(args, fmt);
vsprintf(buf, fmt, args);
va_end(args);
- fd = sys_open("/dev/root", O_RDWR | O_NDELAY, 0);
+ fd = sys_open((char __user *)"/dev/root", O_RDWR | O_NDELAY, 0);
if (fd >= 0) {
sys_ioctl(fd, FDEJECT, 0);
sys_close(fd);
}
printk(KERN_NOTICE "VFS: Insert %s and press ENTER\n", buf);
- fd = sys_open("/dev/console", O_RDWR, 0);
+ fd = sys_open((__force const char __user *)"/dev/console", O_RDWR, 0);
if (fd >= 0) {
sys_ioctl(fd, TCGETS, (long)&termios);
termios.c_lflag &= ~ICANON;
sys_ioctl(fd, TCSETSF, (long)&termios);
- sys_read(fd, &c, 1);
+ sys_read(fd, (char __user *)&c, 1);
termios.c_lflag |= ICANON;
sys_ioctl(fd, TCSETSF, (long)&termios);
sys_close(fd);
mount_root();
out:
devtmpfs_mount("dev");
- sys_mount(".", "/", NULL, MS_MOVE, NULL);
- sys_chroot(".");
+ sys_mount((char __force_user *)".", (char __force_user *)"/", NULL, MS_MOVE, NULL);
+ sys_chroot((const char __force_user *)".");
}
static bool is_tmpfs;
static inline int create_dev(char *name, dev_t dev)
{
- sys_unlink(name);
- return sys_mknod(name, S_IFBLK|0600, new_encode_dev(dev));
+ sys_unlink((char __force_user *)name);
+ return sys_mknod((char __force_user *)name, S_IFBLK|0600, new_encode_dev(dev));
}
#if BITS_PER_LONG == 32
static inline u32 bstat(char *name)
{
struct stat64 stat;
- if (sys_stat64(name, &stat) != 0)
+ if (sys_stat64((char __force_user *)name, (struct stat64 __force_user *)&stat) != 0)
return 0;
if (!S_ISBLK(stat.st_mode))
return 0;
static inline u32 bstat(char *name)
{
struct stat stat;
- if (sys_newstat(name, &stat) != 0)
+ if (sys_newstat((const char __force_user *)name, (struct stat __force_user *)&stat) != 0)
return 0;
if (!S_ISBLK(stat.st_mode))
return 0;
{
sys_unshare(CLONE_FS | CLONE_FILES);
/* stdin/stdout/stderr for /linuxrc */
- sys_open("/dev/console", O_RDWR, 0);
+ sys_open((const char __force_user *)"/dev/console", O_RDWR, 0);
sys_dup(0);
sys_dup(0);
/* move initrd over / and chdir/chroot in initrd root */
- sys_chdir("/root");
- sys_mount(".", "/", NULL, MS_MOVE, NULL);
- sys_chroot(".");
+ sys_chdir((const char __force_user *)"/root");
+ sys_mount((char __force_user *)".", (char __force_user *)"/", NULL, MS_MOVE, NULL);
+ sys_chroot((const char __force_user *)".");
sys_setsid();
return 0;
}
create_dev("/dev/root.old", Root_RAM0);
/* mount initrd on rootfs' /root */
mount_block_root("/dev/root.old", root_mountflags & ~MS_RDONLY);
- sys_mkdir("/old", 0700);
- sys_chdir("/old");
+ sys_mkdir((const char __force_user *)"/old", 0700);
+ sys_chdir((const char __force_user *)"/old");
/* try loading default modules from initrd */
load_default_modules();
current->flags &= ~PF_FREEZER_SKIP;
/* move initrd to rootfs' /old */
- sys_mount("..", ".", NULL, MS_MOVE, NULL);
+ sys_mount((char __force_user *)"..", (char __force_user *)".", NULL, MS_MOVE, NULL);
/* switch root and cwd back to / of rootfs */
- sys_chroot("..");
+ sys_chroot((const char __force_user *)"..");
if (new_decode_dev(real_root_dev) == Root_RAM0) {
- sys_chdir("/old");
+ sys_chdir((const char __force_user *)"/old");
return;
}
- sys_chdir("/");
+ sys_chdir((const char __force_user *)"/");
ROOT_DEV = new_decode_dev(real_root_dev);
mount_root();
printk(KERN_NOTICE "Trying to move old root to /initrd ... ");
- error = sys_mount("/old", "/root/initrd", NULL, MS_MOVE, NULL);
+ error = sys_mount((char __force_user *)"/old", (char __force_user *)"/root/initrd", NULL, MS_MOVE, NULL);
if (!error)
printk("okay\n");
else {
- int fd = sys_open("/dev/root.old", O_RDWR, 0);
+ int fd = sys_open((const char __force_user *)"/dev/root.old", O_RDWR, 0);
if (error == -ENOENT)
printk("/initrd does not exist. Ignored.\n");
else
printk("failed\n");
printk(KERN_NOTICE "Unmounting old root\n");
- sys_umount("/old", MNT_DETACH);
+ sys_umount((char __force_user *)"/old", MNT_DETACH);
printk(KERN_NOTICE "Trying to free ramdisk memory ... ");
if (fd < 0) {
error = fd;
* mounted in the normal path.
*/
if (rd_load_image("/initrd.image") && ROOT_DEV != Root_RAM0) {
- sys_unlink("/initrd.image");
+ sys_unlink((const char __force_user *)"/initrd.image");
handle_initrd();
return 1;
}
}
- sys_unlink("/initrd.image");
+ sys_unlink((const char __force_user *)"/initrd.image");
return 0;
}
partitioned ? "_d" : "", minor,
md_setup_args[ent].device_names);
- fd = sys_open(name, 0, 0);
+ fd = sys_open((char __force_user *)name, 0, 0);
if (fd < 0) {
printk(KERN_ERR "md: open failed - cannot start "
"array %s\n", name);
* array without it
*/
sys_close(fd);
- fd = sys_open(name, 0, 0);
+ fd = sys_open((char __force_user *)name, 0, 0);
sys_ioctl(fd, BLKRRPART, 0);
}
sys_close(fd);
wait_for_device_probe();
- fd = sys_open("/dev/md0", 0, 0);
+ fd = sys_open((const char __force_user *) "/dev/md0", 0, 0);
if (fd >= 0) {
sys_ioctl(fd, RAID_AUTORUN, raid_autopart);
sys_close(fd);
* Initial thread structure. Alignment of this is handled by a special
* linker map entry.
*/
+#ifdef CONFIG_X86
+union thread_union init_thread_union __init_task_data;
+#else
union thread_union init_thread_union __init_task_data =
{ INIT_THREAD_INFO(init_task) };
+#endif
/* sys_write only can write MAX_RW_COUNT aka 2G-4K bytes at most */
while (count) {
- ssize_t rv = sys_write(fd, p, count);
+ ssize_t rv = sys_write(fd, (char __force_user *)p, count);
if (rv < 0) {
if (rv == -EINTR || rv == -EAGAIN)
}
}
-static long __init do_utime(char *filename, time_t mtime)
+static long __init do_utime(char __force_user *filename, time_t mtime)
{
struct timespec t[2];
struct dir_entry *de, *tmp;
list_for_each_entry_safe(de, tmp, &dir_list, list) {
list_del(&de->list);
- do_utime(de->name, de->mtime);
+ do_utime((char __force_user *)de->name, de->mtime);
kfree(de->name);
kfree(de);
}
if (nlink >= 2) {
char *old = find_link(major, minor, ino, mode, collected);
if (old)
- return (sys_link(old, collected) < 0) ? -1 : 1;
+ return (sys_link((char __force_user *)old, (char __force_user *)collected) < 0) ? -1 : 1;
}
return 0;
}
{
struct stat st;
- if (!sys_newlstat(path, &st) && (st.st_mode ^ fmode) & S_IFMT) {
+ if (!sys_newlstat((char __force_user *)path, (struct stat __force_user *)&st) && (st.st_mode ^ fmode) & S_IFMT) {
if (S_ISDIR(st.st_mode))
- sys_rmdir(path);
+ sys_rmdir((char __force_user *)path);
else
- sys_unlink(path);
+ sys_unlink((char __force_user *)path);
}
}
int openflags = O_WRONLY|O_CREAT;
if (ml != 1)
openflags |= O_TRUNC;
- wfd = sys_open(collected, openflags, mode);
+ wfd = sys_open((char __force_user *)collected, openflags, mode);
if (wfd >= 0) {
sys_fchown(wfd, uid, gid);
}
}
} else if (S_ISDIR(mode)) {
- sys_mkdir(collected, mode);
- sys_chown(collected, uid, gid);
- sys_chmod(collected, mode);
+ sys_mkdir((char __force_user *)collected, mode);
+ sys_chown((char __force_user *)collected, uid, gid);
+ sys_chmod((char __force_user *)collected, mode);
dir_add(collected, mtime);
} else if (S_ISBLK(mode) || S_ISCHR(mode) ||
S_ISFIFO(mode) || S_ISSOCK(mode)) {
if (maybe_link() == 0) {
- sys_mknod(collected, mode, rdev);
- sys_chown(collected, uid, gid);
- sys_chmod(collected, mode);
- do_utime(collected, mtime);
+ sys_mknod((char __force_user *)collected, mode, rdev);
+ sys_chown((char __force_user *)collected, uid, gid);
+ sys_chmod((char __force_user *)collected, mode);
+ do_utime((char __force_user *)collected, mtime);
}
}
return 0;
if (xwrite(wfd, victim, body_len) != body_len)
error("write error");
sys_close(wfd);
- do_utime(vcollected, mtime);
+ do_utime((char __force_user *)vcollected, mtime);
kfree(vcollected);
eat(body_len);
state = SkipIt;
{
collected[N_ALIGN(name_len) + body_len] = '\0';
clean_path(collected, 0);
- sys_symlink(collected + N_ALIGN(name_len), collected);
- sys_lchown(collected, uid, gid);
- do_utime(collected, mtime);
+ sys_symlink((char __force_user *)collected + N_ALIGN(name_len), (char __force_user *)collected);
+ sys_lchown((char __force_user *)collected, uid, gid);
+ do_utime((char __force_user *)collected, mtime);
state = SkipIt;
next_state = Reset;
return 0;
static inline void mark_rodata_ro(void) { }
#endif
+extern void grsecurity_init(void);
+
/*
* Debug helper: via this flag we know that we are in 'early bootup code'
* where only the boot processor is running with IRQ disabled. This means
__setup("reset_devices", set_reset_devices);
+#ifdef CONFIG_GRKERNSEC_PROC_USERGROUP
+kgid_t grsec_proc_gid = KGIDT_INIT(CONFIG_GRKERNSEC_PROC_GID);
+static int __init setup_grsec_proc_gid(char *str)
+{
+ grsec_proc_gid = KGIDT_INIT(simple_strtol(str, NULL, 0));
+ return 1;
+}
+__setup("grsec_proc_gid=", setup_grsec_proc_gid);
+#endif
+
+#if defined(CONFIG_X86_64) && defined(CONFIG_PAX_MEMORY_UDEREF)
+unsigned long pax_user_shadow_base __read_only;
+EXPORT_SYMBOL(pax_user_shadow_base);
+extern char pax_enter_kernel_user[];
+extern char pax_exit_kernel_user[];
+#endif
+
+#if defined(CONFIG_X86) && defined(CONFIG_PAX_MEMORY_UDEREF)
+static int __init setup_pax_nouderef(char *str)
+{
+#ifdef CONFIG_X86_32
+ unsigned int cpu;
+ struct desc_struct *gdt;
+
+ for (cpu = 0; cpu < nr_cpu_ids; cpu++) {
+ gdt = get_cpu_gdt_table(cpu);
+ gdt[GDT_ENTRY_KERNEL_DS].type = 3;
+ gdt[GDT_ENTRY_KERNEL_DS].limit = 0xf;
+ gdt[GDT_ENTRY_DEFAULT_USER_CS].limit = 0xf;
+ gdt[GDT_ENTRY_DEFAULT_USER_DS].limit = 0xf;
+ }
+ loadsegment(ds, __KERNEL_DS);
+ loadsegment(es, __KERNEL_DS);
+ loadsegment(ss, __KERNEL_DS);
+#else
+ memcpy(pax_enter_kernel_user, (unsigned char []){0xc3}, 1);
+ memcpy(pax_exit_kernel_user, (unsigned char []){0xc3}, 1);
+ clone_pgd_mask = ~(pgdval_t)0UL;
+ pax_user_shadow_base = 0UL;
+ setup_clear_cpu_cap(X86_FEATURE_PCID);
+ setup_clear_cpu_cap(X86_FEATURE_INVPCID);
+#endif
+
+ return 0;
+}
+early_param("pax_nouderef", setup_pax_nouderef);
+
+#ifdef CONFIG_X86_64
+static int __init setup_pax_weakuderef(char *str)
+{
+ if (clone_pgd_mask != ~(pgdval_t)0UL)
+ pax_user_shadow_base = 1UL << TASK_SIZE_MAX_SHIFT;
+ return 1;
+}
+__setup("pax_weakuderef", setup_pax_weakuderef);
+#endif
+#endif
+
+#ifdef CONFIG_PAX_SOFTMODE
+int pax_softmode;
+
+static int __init setup_pax_softmode(char *str)
+{
+ get_option(&str, &pax_softmode);
+ return 1;
+}
+__setup("pax_softmode=", setup_pax_softmode);
+#endif
+
static const char *argv_init[MAX_INIT_ARGS+2] = { "init", NULL, };
const char *envp_init[MAX_INIT_ENVS+2] = { "HOME=/", "TERM=linux", NULL, };
static const char *panic_later, *panic_param;
struct blacklist_entry *entry;
char *fn_name;
- fn_name = kasprintf(GFP_KERNEL, "%pf", fn);
+ fn_name = kasprintf(GFP_KERNEL, "%pX", fn);
if (!fn_name)
return false;
{
int count = preempt_count();
int ret;
- char msgbuf[64];
+ const char *msg1 = "", *msg2 = "";
if (initcall_blacklisted(fn))
return -EPERM;
else
ret = fn();
- msgbuf[0] = 0;
-
if (preempt_count() != count) {
- sprintf(msgbuf, "preemption imbalance ");
+ msg1 = " preemption imbalance";
preempt_count_set(count);
}
if (irqs_disabled()) {
- strlcat(msgbuf, "disabled interrupts ", sizeof(msgbuf));
+ msg2 = " disabled interrupts";
local_irq_enable();
}
- WARN(msgbuf[0], "initcall %pF returned with %s\n", fn, msgbuf);
+ WARN(*msg1 || *msg2, "initcall %pF returned with%s%s\n", fn, msg1, msg2);
+ add_latent_entropy();
return ret;
}
{
argv_init[0] = init_filename;
return do_execve(getname_kernel(init_filename),
- (const char __user *const __user *)argv_init,
- (const char __user *const __user *)envp_init);
+ (const char __user *const __force_user *)argv_init,
+ (const char __user *const __force_user *)envp_init);
}
static int try_to_run_init_process(const char *init_filename)
return ret;
}
+#ifdef CONFIG_GRKERNSEC_CHROOT_INITRD
+extern int gr_init_ran;
+#endif
+
static noinline void __init kernel_init_freeable(void);
static int __ref kernel_init(void *unused)
ramdisk_execute_command, ret);
}
+#ifdef CONFIG_GRKERNSEC_CHROOT_INITRD
+ /* if no initrd was used, be extra sure we enforce chroot restrictions */
+ gr_init_ran = 1;
+#endif
+
/*
* We try each of these until one succeeds.
*
do_basic_setup();
/* Open the /dev/console on the rootfs, this should never fail */
- if (sys_open((const char __user *) "/dev/console", O_RDWR, 0) < 0)
+ if (sys_open((const char __force_user *) "/dev/console", O_RDWR, 0) < 0)
pr_err("Warning: unable to open an initial console.\n");
(void) sys_dup(0);
if (!ramdisk_execute_command)
ramdisk_execute_command = "/init";
- if (sys_access((const char __user *) ramdisk_execute_command, 0) != 0) {
+ if (sys_access((const char __force_user *) ramdisk_execute_command, 0) != 0) {
ramdisk_execute_command = NULL;
prepare_namespace();
}
+ grsecurity_init();
+
/*
* Ok, we have completed the initial bootup, and
* we're essentially up and running. Get rid of the
COMPAT_SHMLBA);
if (err < 0)
return err;
- return put_user(raddr, (compat_ulong_t *)compat_ptr(third));
+ return put_user(raddr, (compat_ulong_t __user *)compat_ptr(third));
}
case SHMDT:
return sys_shmdt(compat_ptr(ptr));
static int proc_ipc_dointvec(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp, loff_t *ppos)
{
- struct ctl_table ipc_table;
+ ctl_table_no_const ipc_table;
memcpy(&ipc_table, table, sizeof(ipc_table));
ipc_table.data = get_ipc(table);
static int proc_ipc_dointvec_minmax(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp, loff_t *ppos)
{
- struct ctl_table ipc_table;
+ ctl_table_no_const ipc_table;
memcpy(&ipc_table, table, sizeof(ipc_table));
ipc_table.data = get_ipc(table);
static int proc_ipc_doulongvec_minmax(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp, loff_t *ppos)
{
- struct ctl_table ipc_table;
+ ctl_table_no_const ipc_table;
memcpy(&ipc_table, table, sizeof(ipc_table));
ipc_table.data = get_ipc(table);
static int proc_ipc_auto_msgmni(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp, loff_t *ppos)
{
- struct ctl_table ipc_table;
+ ctl_table_no_const ipc_table;
int dummy = 0;
memcpy(&ipc_table, table, sizeof(ipc_table));
static int proc_mq_dointvec(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp, loff_t *ppos)
{
- struct ctl_table mq_table;
+ ctl_table_no_const mq_table;
memcpy(&mq_table, table, sizeof(mq_table));
mq_table.data = get_mq(table);
static int proc_mq_dointvec_minmax(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp, loff_t *ppos)
{
- struct ctl_table mq_table;
+ ctl_table_no_const mq_table;
memcpy(&mq_table, table, sizeof(mq_table));
mq_table.data = get_mq(table);
mq_bytes = mq_treesize + (info->attr.mq_maxmsg *
info->attr.mq_msgsize);
+ gr_learn_resource(current, RLIMIT_MSGQUEUE, u->mq_bytes + mq_bytes, 1);
spin_lock(&mq_lock);
if (u->mq_bytes + mq_bytes < u->mq_bytes ||
u->mq_bytes + mq_bytes > rlimit(RLIMIT_MSGQUEUE)) {
static int sysvipc_shm_proc_show(struct seq_file *s, void *it);
#endif
+#ifdef CONFIG_GRKERNSEC
+extern int gr_handle_shmat(const pid_t shm_cprid, const pid_t shm_lapid,
+ const u64 shm_createtime, const kuid_t cuid,
+ const int shmid);
+extern int gr_chroot_shmat(const pid_t shm_cprid, const pid_t shm_lapid,
+ const u64 shm_createtime);
+#endif
+
void shm_init_ns(struct ipc_namespace *ns)
{
ns->shm_ctlmax = SHMMAX;
shp->shm_lprid = 0;
shp->shm_atim = shp->shm_dtim = 0;
shp->shm_ctim = get_seconds();
+#ifdef CONFIG_GRKERNSEC
+ shp->shm_createtime = ktime_get_ns();
+#endif
shp->shm_segsz = size;
shp->shm_nattch = 0;
shp->shm_file = file;
f_mode = FMODE_READ | FMODE_WRITE;
}
if (shmflg & SHM_EXEC) {
+
+#ifdef CONFIG_PAX_MPROTECT
+ if (current->mm->pax_flags & MF_PAX_MPROTECT)
+ goto out;
+#endif
+
prot |= PROT_EXEC;
acc_mode |= S_IXUGO;
}
if (err)
goto out_unlock;
+#ifdef CONFIG_GRKERNSEC
+ if (!gr_handle_shmat(shp->shm_cprid, shp->shm_lapid, shp->shm_createtime,
+ shp->shm_perm.cuid, shmid) ||
+ !gr_chroot_shmat(shp->shm_cprid, shp->shm_lapid, shp->shm_createtime)) {
+ err = -EACCES;
+ goto out_unlock;
+ }
+#endif
+
ipc_lock_object(&shp->shm_perm);
/* check if shm_destroy() is tearing down shp */
path = shp->shm_file->f_path;
path_get(&path);
shp->shm_nattch++;
+#ifdef CONFIG_GRKERNSEC
+ shp->shm_lapid = current->pid;
+#endif
size = i_size_read(path.dentry->d_inode);
ipc_unlock_object(&shp->shm_perm);
rcu_read_unlock();
int (*show)(struct seq_file *, void *);
};
+extern int gr_ipc_permitted(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp, int requested_mode, int granted_mode);
+
/**
* ipc_init - initialise ipc subsystem
*
granted_mode >>= 6;
else if (in_group_p(ipcp->cgid) || in_group_p(ipcp->gid))
granted_mode >>= 3;
+
+ if (!gr_ipc_permitted(ns, ipcp, requested_mode, granted_mode))
+ return -1;
+
/* is there some bit set in requested_mode but not in granted_mode? */
if ((requested_mode & ~granted_mode & 0007) &&
!ns_capable(ns->user_ns, CAP_IPC_OWNER))
3) suppressed due to audit_rate_limit
4) suppressed due to audit_backlog_limit
*/
-static atomic_t audit_lost = ATOMIC_INIT(0);
+static atomic_unchecked_t audit_lost = ATOMIC_INIT(0);
/* The netlink socket. */
static struct sock *audit_sock;
unsigned long now;
int print;
- atomic_inc(&audit_lost);
+ atomic_inc_unchecked(&audit_lost);
print = (audit_failure == AUDIT_FAIL_PANIC || !audit_rate_limit);
if (print) {
if (printk_ratelimit())
pr_warn("audit_lost=%u audit_rate_limit=%u audit_backlog_limit=%u\n",
- atomic_read(&audit_lost),
+ atomic_read_unchecked(&audit_lost),
audit_rate_limit,
audit_backlog_limit);
audit_panic(message);
s.pid = audit_pid;
s.rate_limit = audit_rate_limit;
s.backlog_limit = audit_backlog_limit;
- s.lost = atomic_read(&audit_lost);
+ s.lost = atomic_read_unchecked(&audit_lost);
s.backlog = skb_queue_len(&audit_skb_queue);
s.feature_bitmap = AUDIT_FEATURE_BITMAP_ALL;
s.backlog_wait_time = audit_backlog_wait_time;
}
/* global counter which is incremented every time something logs in */
-static atomic_t session_id = ATOMIC_INIT(0);
+static atomic_unchecked_t session_id = ATOMIC_INIT(0);
static int audit_set_loginuid_perm(kuid_t loginuid)
{
/* are we setting or clearing? */
if (uid_valid(loginuid))
- sessionid = (unsigned int)atomic_inc_return(&session_id);
+ sessionid = (unsigned int)atomic_inc_return_unchecked(&session_id);
task->sessionid = sessionid;
task->loginuid = loginuid;
* random section of illegal instructions.
*/
size = round_up(proglen + sizeof(*hdr) + 128, PAGE_SIZE);
- hdr = module_alloc(size);
+ hdr = module_alloc_exec(size);
if (hdr == NULL)
return NULL;
/* Fill space with illegal/arch-dep instructions. */
bpf_fill_ill_insns(hdr, size);
+ pax_open_kernel();
hdr->pages = size / PAGE_SIZE;
+ pax_close_kernel();
+
hole = min_t(unsigned int, size - (proglen + sizeof(*hdr)),
PAGE_SIZE - sizeof(*hdr));
start = (prandom_u32() % hole) & ~(alignment - 1);
void bpf_jit_binary_free(struct bpf_binary_header *hdr)
{
- module_memfree(hdr);
+ module_memfree_exec(hdr);
}
#endif /* CONFIG_BPF_JIT */
int err;
/* the syscall is limited to root temporarily. This restriction will be
- * lifted when security audit is clean. Note that eBPF+tracing must have
- * this restriction, since it may pass kernel data to user space
+ * lifted by upstream when a half-assed security audit is clean. Note
+ * that eBPF+tracing must have this restriction, since it may pass
+ * kernel data to user space
*/
if (!capable(CAP_SYS_ADMIN))
return -EPERM;
+#ifdef CONFIG_GRKERNSEC
+ return -EPERM;
+#endif
if (!access_ok(VERIFY_READ, uattr, 1))
return -EFAULT;
* before modification is attempted and the application
* fails.
*/
+ if (tocopy > ARRAY_SIZE(kdata))
+ return -EFAULT;
+
if (copy_to_user(dataptr, kdata, tocopy
* sizeof(struct __user_cap_data_struct))) {
return -EFAULT;
int ret;
rcu_read_lock();
- ret = security_capable(__task_cred(t), ns, cap);
+ ret = security_capable(__task_cred(t), ns, cap) == 0 &&
+ gr_task_is_capable(t, __task_cred(t), cap);
rcu_read_unlock();
- return (ret == 0);
+ return ret;
}
/**
int ret;
rcu_read_lock();
- ret = security_capable_noaudit(__task_cred(t), ns, cap);
+ ret = security_capable_noaudit(__task_cred(t), ns, cap) == 0 && gr_task_is_capable_nolog(t, cap);
rcu_read_unlock();
- return (ret == 0);
+ return ret;
}
/**
BUG();
}
- if (security_capable(current_cred(), ns, cap) == 0) {
+ if (security_capable(current_cred(), ns, cap) == 0 && gr_is_capable(cap)) {
current->flags |= PF_SUPERPRIV;
return true;
}
}
EXPORT_SYMBOL(ns_capable);
+bool ns_capable_nolog(struct user_namespace *ns, int cap)
+{
+ if (unlikely(!cap_valid(cap))) {
+ printk(KERN_CRIT "capable_nolog() called with invalid cap=%u\n", cap);
+ BUG();
+ }
+
+ if (security_capable_noaudit(current_cred(), ns, cap) == 0 && gr_is_capable_nolog(cap)) {
+ current->flags |= PF_SUPERPRIV;
+ return true;
+ }
+ return false;
+}
+EXPORT_SYMBOL(ns_capable_nolog);
+
/**
* file_ns_capable - Determine if the file's opener had a capability in effect
* @file: The file we want to check
}
EXPORT_SYMBOL(capable);
+bool capable_nolog(int cap)
+{
+ return ns_capable_nolog(&init_user_ns, cap);
+}
+EXPORT_SYMBOL(capable_nolog);
+
/**
* capable_wrt_inode_uidgid - Check nsown_capable and uid and gid mapped
* @inode: The inode in question
kgid_has_mapping(ns, inode->i_gid);
}
EXPORT_SYMBOL(capable_wrt_inode_uidgid);
+
+bool capable_wrt_inode_uidgid_nolog(const struct inode *inode, int cap)
+{
+ struct user_namespace *ns = current_user_ns();
+
+ return ns_capable_nolog(ns, cap) && kuid_has_mapping(ns, inode->i_uid) &&
+ kgid_has_mapping(ns, inode->i_gid);
+}
+EXPORT_SYMBOL(capable_wrt_inode_uidgid_nolog);
if (!pathbuf || !agentbuf)
goto out;
+ if (agentbuf[0] == '\0')
+ goto out;
+
path = cgroup_path(cgrp, pathbuf, PATH_MAX);
if (!path)
goto out;
struct task_struct *task;
int count = 0;
- seq_printf(seq, "css_set %p\n", cset);
+ seq_printf(seq, "css_set %pK\n", cset);
list_for_each_entry(task, &cset->tasks, cg_list) {
if (count++ > MAX_TASKS_SHOWN_PER_CSS)
#include <linux/linkage.h>
#include <linux/compat.h>
+#include <linux/module.h>
#include <linux/errno.h>
#include <linux/time.h>
#include <linux/signal.h>
mm_segment_t oldfs;
long ret;
- restart->nanosleep.rmtp = (struct timespec __user *) &rmt;
+ restart->nanosleep.rmtp = (struct timespec __force_user *) &rmt;
oldfs = get_fs();
set_fs(KERNEL_DS);
ret = hrtimer_nanosleep_restart(restart);
oldfs = get_fs();
set_fs(KERNEL_DS);
ret = hrtimer_nanosleep(&tu,
- rmtp ? (struct timespec __user *)&rmt : NULL,
+ rmtp ? (struct timespec __force_user *)&rmt : NULL,
HRTIMER_MODE_REL, CLOCK_MONOTONIC);
set_fs(oldfs);
mm_segment_t old_fs = get_fs();
set_fs(KERNEL_DS);
- ret = sys_sigpending((old_sigset_t __user *) &s);
+ ret = sys_sigpending((old_sigset_t __force_user *) &s);
set_fs(old_fs);
if (ret == 0)
ret = put_user(s, set);
mm_segment_t old_fs = get_fs();
set_fs(KERNEL_DS);
- ret = sys_old_getrlimit(resource, (struct rlimit __user *)&r);
+ ret = sys_old_getrlimit(resource, (struct rlimit __force_user *)&r);
set_fs(old_fs);
if (!ret) {
set_fs (KERNEL_DS);
ret = sys_wait4(pid,
(stat_addr ?
- (unsigned int __user *) &status : NULL),
- options, (struct rusage __user *) &r);
+ (unsigned int __force_user *) &status : NULL),
+ options, (struct rusage __force_user *) &r);
set_fs (old_fs);
if (ret > 0) {
memset(&info, 0, sizeof(info));
set_fs(KERNEL_DS);
- ret = sys_waitid(which, pid, (siginfo_t __user *)&info, options,
- uru ? (struct rusage __user *)&ru : NULL);
+ ret = sys_waitid(which, pid, (siginfo_t __force_user *)&info, options,
+ uru ? (struct rusage __force_user *)&ru : NULL);
set_fs(old_fs);
if ((ret < 0) || (info.si_signo == 0))
oldfs = get_fs();
set_fs(KERNEL_DS);
err = sys_timer_settime(timer_id, flags,
- (struct itimerspec __user *) &newts,
- (struct itimerspec __user *) &oldts);
+ (struct itimerspec __force_user *) &newts,
+ (struct itimerspec __force_user *) &oldts);
set_fs(oldfs);
if (!err && old && put_compat_itimerspec(old, &oldts))
return -EFAULT;
oldfs = get_fs();
set_fs(KERNEL_DS);
err = sys_timer_gettime(timer_id,
- (struct itimerspec __user *) &ts);
+ (struct itimerspec __force_user *) &ts);
set_fs(oldfs);
if (!err && put_compat_itimerspec(setting, &ts))
return -EFAULT;
oldfs = get_fs();
set_fs(KERNEL_DS);
err = sys_clock_settime(which_clock,
- (struct timespec __user *) &ts);
+ (struct timespec __force_user *) &ts);
set_fs(oldfs);
return err;
}
oldfs = get_fs();
set_fs(KERNEL_DS);
err = sys_clock_gettime(which_clock,
- (struct timespec __user *) &ts);
+ (struct timespec __force_user *) &ts);
set_fs(oldfs);
if (!err && compat_put_timespec(&ts, tp))
return -EFAULT;
oldfs = get_fs();
set_fs(KERNEL_DS);
- ret = sys_clock_adjtime(which_clock, (struct timex __user *) &txc);
+ ret = sys_clock_adjtime(which_clock, (struct timex __force_user *) &txc);
set_fs(oldfs);
err = compat_put_timex(utp, &txc);
oldfs = get_fs();
set_fs(KERNEL_DS);
err = sys_clock_getres(which_clock,
- (struct timespec __user *) &ts);
+ (struct timespec __force_user *) &ts);
set_fs(oldfs);
if (!err && tp && compat_put_timespec(&ts, tp))
return -EFAULT;
struct timespec tu;
struct compat_timespec __user *rmtp = restart->nanosleep.compat_rmtp;
- restart->nanosleep.rmtp = (struct timespec __user *) &tu;
+ restart->nanosleep.rmtp = (struct timespec __force_user *) &tu;
oldfs = get_fs();
set_fs(KERNEL_DS);
err = clock_nanosleep_restart(restart);
oldfs = get_fs();
set_fs(KERNEL_DS);
err = sys_clock_nanosleep(which_clock, flags,
- (struct timespec __user *) &in,
- (struct timespec __user *) &out);
+ (struct timespec __force_user *) &in,
+ (struct timespec __force_user *) &out);
set_fs(oldfs);
if ((err == -ERESTART_RESTARTBLOCK) && rmtp &&
mm_segment_t old_fs = get_fs();
set_fs(KERNEL_DS);
- ret = sys_sched_rr_get_interval(pid, (struct timespec __user *)&t);
+ ret = sys_sched_rr_get_interval(pid, (struct timespec __force_user *)&t);
set_fs(old_fs);
if (compat_put_timespec(&t, interval))
return -EFAULT;
struct proc_dir_entry *entry;
/* create the current config file */
+#if defined(CONFIG_GRKERNSEC_PROC_ADD) || defined(CONFIG_GRKERNSEC_HIDESYM)
+#if defined(CONFIG_GRKERNSEC_PROC_USER) || defined(CONFIG_GRKERNSEC_HIDESYM)
+ entry = proc_create("config.gz", S_IFREG | S_IRUSR, NULL,
+ &ikconfig_file_ops);
+#elif defined(CONFIG_GRKERNSEC_PROC_USERGROUP)
+ entry = proc_create("config.gz", S_IFREG | S_IRUSR | S_IRGRP, NULL,
+ &ikconfig_file_ops);
+#endif
+#else
entry = proc_create("config.gz", S_IFREG | S_IRUGO, NULL,
&ikconfig_file_ops);
+#endif
+
if (!entry)
return -ENOMEM;
validate_creds(cred);
alter_cred_subscribers(cred, -1);
put_cred(cred);
+
+#ifdef CONFIG_GRKERNSEC_SETXID
+ cred = (struct cred *) tsk->delayed_cred;
+ if (cred != NULL) {
+ tsk->delayed_cred = NULL;
+ validate_creds(cred);
+ alter_cred_subscribers(cred, -1);
+ put_cred(cred);
+ }
+#endif
}
/**
* Always returns 0 thus allowing this function to be tail-called at the end
* of, say, sys_setgid().
*/
-int commit_creds(struct cred *new)
+static int __commit_creds(struct cred *new)
{
struct task_struct *task = current;
const struct cred *old = task->real_cred;
get_cred(new); /* we will require a ref for the subj creds too */
+ gr_set_role_label(task, new->uid, new->gid);
+
/* dumpability changes */
if (!uid_eq(old->euid, new->euid) ||
!gid_eq(old->egid, new->egid) ||
put_cred(old);
return 0;
}
+#ifdef CONFIG_GRKERNSEC_SETXID
+extern int set_user(struct cred *new);
+
+void gr_delayed_cred_worker(void)
+{
+ const struct cred *new = current->delayed_cred;
+ struct cred *ncred;
+
+ current->delayed_cred = NULL;
+
+ if (!uid_eq(current_uid(), GLOBAL_ROOT_UID) && new != NULL) {
+ // from doing get_cred on it when queueing this
+ put_cred(new);
+ return;
+ } else if (new == NULL)
+ return;
+
+ ncred = prepare_creds();
+ if (!ncred)
+ goto die;
+ // uids
+ ncred->uid = new->uid;
+ ncred->euid = new->euid;
+ ncred->suid = new->suid;
+ ncred->fsuid = new->fsuid;
+ // gids
+ ncred->gid = new->gid;
+ ncred->egid = new->egid;
+ ncred->sgid = new->sgid;
+ ncred->fsgid = new->fsgid;
+ // groups
+ set_groups(ncred, new->group_info);
+ // caps
+ ncred->securebits = new->securebits;
+ ncred->cap_inheritable = new->cap_inheritable;
+ ncred->cap_permitted = new->cap_permitted;
+ ncred->cap_effective = new->cap_effective;
+ ncred->cap_bset = new->cap_bset;
+
+ if (set_user(ncred)) {
+ abort_creds(ncred);
+ goto die;
+ }
+
+ // from doing get_cred on it when queueing this
+ put_cred(new);
+
+ __commit_creds(ncred);
+ return;
+die:
+ // from doing get_cred on it when queueing this
+ put_cred(new);
+ do_group_exit(SIGKILL);
+}
+#endif
+
+int commit_creds(struct cred *new)
+{
+#ifdef CONFIG_GRKERNSEC_SETXID
+ int ret;
+ int schedule_it = 0;
+ struct task_struct *t;
+ unsigned oldsecurebits = current_cred()->securebits;
+
+ /* we won't get called with tasklist_lock held for writing
+ and interrupts disabled as the cred struct in that case is
+ init_cred
+ */
+ if (grsec_enable_setxid && !current_is_single_threaded() &&
+ uid_eq(current_uid(), GLOBAL_ROOT_UID) &&
+ !uid_eq(new->uid, GLOBAL_ROOT_UID)) {
+ schedule_it = 1;
+ }
+ ret = __commit_creds(new);
+ if (schedule_it) {
+ rcu_read_lock();
+ read_lock(&tasklist_lock);
+ for (t = next_thread(current); t != current;
+ t = next_thread(t)) {
+ /* we'll check if the thread has uid 0 in
+ * the delayed worker routine
+ */
+ if (task_securebits(t) == oldsecurebits &&
+ t->delayed_cred == NULL) {
+ t->delayed_cred = get_cred(new);
+ set_tsk_thread_flag(t, TIF_GRSEC_SETXID);
+ set_tsk_need_resched(t);
+ }
+ }
+ read_unlock(&tasklist_lock);
+ rcu_read_unlock();
+ }
+
+ return ret;
+#else
+ return __commit_creds(new);
+#endif
+}
+
EXPORT_SYMBOL(commit_creds);
/**
*/
static atomic_t masters_in_kgdb;
static atomic_t slaves_in_kgdb;
-static atomic_t kgdb_break_tasklet_var;
+static atomic_unchecked_t kgdb_break_tasklet_var;
atomic_t kgdb_setting_breakpoint;
struct task_struct *kgdb_usethread;
static pid_t kgdb_sstep_pid;
/* to keep track of the CPU which is doing the single stepping*/
-atomic_t kgdb_cpu_doing_single_step = ATOMIC_INIT(-1);
+atomic_unchecked_t kgdb_cpu_doing_single_step = ATOMIC_INIT(-1);
/*
* If you are debugging a problem where roundup (the collection of
* kernel will only try for the value of sstep_tries before
* giving up and continuing on.
*/
- if (atomic_read(&kgdb_cpu_doing_single_step) != -1 &&
+ if (atomic_read_unchecked(&kgdb_cpu_doing_single_step) != -1 &&
(kgdb_info[cpu].task &&
kgdb_info[cpu].task->pid != kgdb_sstep_pid) && --sstep_tries) {
atomic_set(&kgdb_active, -1);
}
kgdb_restore:
- if (atomic_read(&kgdb_cpu_doing_single_step) != -1) {
- int sstep_cpu = atomic_read(&kgdb_cpu_doing_single_step);
+ if (atomic_read_unchecked(&kgdb_cpu_doing_single_step) != -1) {
+ int sstep_cpu = atomic_read_unchecked(&kgdb_cpu_doing_single_step);
if (kgdb_info[sstep_cpu].task)
kgdb_sstep_pid = kgdb_info[sstep_cpu].task->pid;
else
static void kgdb_tasklet_bpt(unsigned long ing)
{
kgdb_breakpoint();
- atomic_set(&kgdb_break_tasklet_var, 0);
+ atomic_set_unchecked(&kgdb_break_tasklet_var, 0);
}
static DECLARE_TASKLET(kgdb_tasklet_breakpoint, kgdb_tasklet_bpt, 0);
void kgdb_schedule_breakpoint(void)
{
- if (atomic_read(&kgdb_break_tasklet_var) ||
+ if (atomic_read_unchecked(&kgdb_break_tasklet_var) ||
atomic_read(&kgdb_active) != -1 ||
atomic_read(&kgdb_setting_breakpoint))
return;
- atomic_inc(&kgdb_break_tasklet_var);
+ atomic_inc_unchecked(&kgdb_break_tasklet_var);
tasklet_schedule(&kgdb_tasklet_breakpoint);
}
EXPORT_SYMBOL_GPL(kgdb_schedule_breakpoint);
continue;
kdb_printf("%-20s%8u 0x%p ", mod->name,
- mod->core_size, (void *)mod);
+ mod->core_size_rx + mod->core_size_rw, (void *)mod);
#ifdef CONFIG_MODULE_UNLOAD
kdb_printf("%4d ", module_refcount(mod));
#endif
kdb_printf(" (Loading)");
else
kdb_printf(" (Live)");
- kdb_printf(" 0x%p", mod->module_core);
+ kdb_printf(" 0x%p 0x%p", mod->module_core_rx, mod->module_core_rw);
#ifdef CONFIG_MODULE_UNLOAD
{
* 0 - disallow raw tracepoint access for unpriv
* 1 - disallow cpu events for unpriv
* 2 - disallow kernel profiling for unpriv
+ * 3 - disallow all unpriv perf event use
*/
-int sysctl_perf_event_paranoid __read_mostly = 1;
+#ifdef CONFIG_GRKERNSEC_PERF_HARDEN
+int sysctl_perf_event_legitimately_concerned __read_mostly = 3;
+#elif defined(CONFIG_GRKERNSEC_HIDESYM)
+int sysctl_perf_event_legitimately_concerned __read_mostly = 2;
+#else
+int sysctl_perf_event_legitimately_concerned __read_mostly = 1;
+#endif
/* Minimum for 512 kiB + 1 user control page */
int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */
tmp *= sysctl_perf_cpu_time_max_percent;
do_div(tmp, 100);
- ACCESS_ONCE(perf_sample_allowed_ns) = tmp;
+ ACCESS_ONCE_RW(perf_sample_allowed_ns) = tmp;
}
static int perf_rotate_context(struct perf_cpu_context *cpuctx);
}
}
-static atomic64_t perf_event_id;
+static atomic64_unchecked_t perf_event_id;
static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
enum event_type_t event_type);
static inline u64 perf_event_count(struct perf_event *event)
{
- return local64_read(&event->count) + atomic64_read(&event->child_count);
+ return local64_read(&event->count) + atomic64_read_unchecked(&event->child_count);
}
static u64 perf_event_read(struct perf_event *event)
mutex_lock(&event->child_mutex);
total += perf_event_read(event);
*enabled += event->total_time_enabled +
- atomic64_read(&event->child_total_time_enabled);
+ atomic64_read_unchecked(&event->child_total_time_enabled);
*running += event->total_time_running +
- atomic64_read(&event->child_total_time_running);
+ atomic64_read_unchecked(&event->child_total_time_running);
list_for_each_entry(child, &event->child_list, child_list) {
total += perf_event_read(child);
userpg->offset -= local64_read(&event->hw.prev_count);
userpg->time_enabled = enabled +
- atomic64_read(&event->child_total_time_enabled);
+ atomic64_read_unchecked(&event->child_total_time_enabled);
userpg->time_running = running +
- atomic64_read(&event->child_total_time_running);
+ atomic64_read_unchecked(&event->child_total_time_running);
arch_perf_update_userpage(userpg, now);
/* Data. */
sp = perf_user_stack_pointer(regs);
- rem = __output_copy_user(handle, (void *) sp, dump_size);
+ rem = __output_copy_user(handle, (void __user *) sp, dump_size);
dyn_size = dump_size - rem;
perf_output_skip(handle, rem);
values[n++] = perf_event_count(event);
if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
values[n++] = enabled +
- atomic64_read(&event->child_total_time_enabled);
+ atomic64_read_unchecked(&event->child_total_time_enabled);
}
if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
values[n++] = running +
- atomic64_read(&event->child_total_time_running);
+ atomic64_read_unchecked(&event->child_total_time_running);
}
if (read_format & PERF_FORMAT_ID)
values[n++] = primary_event_id(event);
event->parent = parent_event;
event->ns = get_pid_ns(task_active_pid_ns(current));
- event->id = atomic64_inc_return(&perf_event_id);
+ event->id = atomic64_inc_return_unchecked(&perf_event_id);
event->state = PERF_EVENT_STATE_INACTIVE;
if (flags & ~PERF_FLAG_ALL)
return -EINVAL;
+#ifdef CONFIG_GRKERNSEC_PERF_HARDEN
+ if (perf_paranoid_any() && !capable(CAP_SYS_ADMIN))
+ return -EACCES;
+#endif
+
err = perf_copy_attr(attr_uptr, &attr);
if (err)
return err;
/*
* Add back the child's count to the parent's count:
*/
- atomic64_add(child_val, &parent_event->child_count);
- atomic64_add(child_event->total_time_enabled,
+ atomic64_add_unchecked(child_val, &parent_event->child_count);
+ atomic64_add_unchecked(child_event->total_time_enabled,
&parent_event->child_total_time_enabled);
- atomic64_add(child_event->total_time_running,
+ atomic64_add_unchecked(child_event->total_time_running,
&parent_event->child_total_time_running);
/*
return rb->nr_pages << (PAGE_SHIFT + page_order(rb));
}
-#define DEFINE_OUTPUT_COPY(func_name, memcpy_func) \
+#define DEFINE_OUTPUT_COPY(func_name, memcpy_func, user) \
static inline unsigned long \
func_name(struct perf_output_handle *handle, \
- const void *buf, unsigned long len) \
+ const void user *buf, unsigned long len) \
{ \
unsigned long size, written; \
\
return 0;
}
-DEFINE_OUTPUT_COPY(__output_copy, memcpy_common)
+DEFINE_OUTPUT_COPY(__output_copy, memcpy_common, )
static inline unsigned long
memcpy_skip(void *dst, const void *src, unsigned long n)
return 0;
}
-DEFINE_OUTPUT_COPY(__output_skip, memcpy_skip)
+DEFINE_OUTPUT_COPY(__output_skip, memcpy_skip, )
#ifndef arch_perf_out_copy_user
#define arch_perf_out_copy_user arch_perf_out_copy_user
}
#endif
-DEFINE_OUTPUT_COPY(__output_copy_user, arch_perf_out_copy_user)
+DEFINE_OUTPUT_COPY(__output_copy_user, arch_perf_out_copy_user, __user)
/* Callchain handling */
extern struct perf_callchain_entry *
{
struct page *page;
uprobe_opcode_t opcode;
- int result;
+ long result;
pagefault_disable();
result = __copy_from_user_inatomic(&opcode, (void __user*)vaddr,
struct task_struct *leader;
int zap_leader;
repeat:
+#ifdef CONFIG_NET
+ gr_del_task_from_ip_table(p);
+#endif
+
/* don't need to get the RCU readlock here - the process is dead and
* can't be modifying its own credentials. But shut RCU-lockdep up */
rcu_read_lock();
int group_dead;
TASKS_RCU(int tasks_rcu_i);
+ set_fs(USER_DS);
+
profile_task_exit(tsk);
WARN_ON(blk_needs_flush_plug(tsk));
* mm_release()->clear_child_tid() from writing to a user-controlled
* kernel address.
*/
- set_fs(USER_DS);
ptrace_event(PTRACE_EVENT_EXIT, code);
tsk->exit_code = code;
taskstats_exit(tsk, group_dead);
+ gr_acl_handle_psacct(tsk, code);
+ gr_acl_handle_exit();
+
exit_mm(tsk);
if (group_dead)
* Take down every thread in the group. This is called by fatal signals
* as well as by sys_exit_group (below).
*/
-void
+__noreturn void
do_group_exit(int exit_code)
{
struct signal_struct *sig = current->signal;
void thread_info_cache_init(void)
{
thread_info_cache = kmem_cache_create("thread_info", THREAD_SIZE,
- THREAD_SIZE, 0, NULL);
+ THREAD_SIZE, SLAB_USERCOPY, NULL);
BUG_ON(thread_info_cache == NULL);
}
# endif
#endif
+#ifdef CONFIG_GRKERNSEC_KSTACKOVERFLOW
+static inline struct thread_info *gr_alloc_thread_info_node(struct task_struct *tsk,
+ int node, void **lowmem_stack)
+{
+ struct page *pages[THREAD_SIZE / PAGE_SIZE];
+ void *ret = NULL;
+ unsigned int i;
+
+ *lowmem_stack = alloc_thread_info_node(tsk, node);
+ if (*lowmem_stack == NULL)
+ goto out;
+
+ for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++)
+ pages[i] = virt_to_page(*lowmem_stack + (i * PAGE_SIZE));
+
+ /* use VM_IOREMAP to gain THREAD_SIZE alignment */
+ ret = vmap(pages, THREAD_SIZE / PAGE_SIZE, VM_IOREMAP, PAGE_KERNEL);
+ if (ret == NULL) {
+ free_thread_info(*lowmem_stack);
+ *lowmem_stack = NULL;
+ }
+
+out:
+ return ret;
+}
+
+static inline void gr_free_thread_info(struct task_struct *tsk, struct thread_info *ti)
+{
+ unmap_process_stacks(tsk);
+}
+#else
+static inline struct thread_info *gr_alloc_thread_info_node(struct task_struct *tsk,
+ int node, void **lowmem_stack)
+{
+ return alloc_thread_info_node(tsk, node);
+}
+static inline void gr_free_thread_info(struct task_struct *tsk, struct thread_info *ti)
+{
+ free_thread_info(ti);
+}
+#endif
+
/* SLAB cache for signal_struct structures (tsk->signal) */
static struct kmem_cache *signal_cachep;
/* SLAB cache for mm_struct structures (tsk->mm) */
static struct kmem_cache *mm_cachep;
-static void account_kernel_stack(struct thread_info *ti, int account)
+static void account_kernel_stack(struct task_struct *tsk, struct thread_info *ti, int account)
{
+#ifdef CONFIG_GRKERNSEC_KSTACKOVERFLOW
+ struct zone *zone = page_zone(virt_to_page(tsk->lowmem_stack));
+#else
struct zone *zone = page_zone(virt_to_page(ti));
+#endif
mod_zone_page_state(zone, NR_KERNEL_STACK, account);
}
void free_task(struct task_struct *tsk)
{
- account_kernel_stack(tsk->stack, -1);
+ account_kernel_stack(tsk, tsk->stack, -1);
arch_release_thread_info(tsk->stack);
- free_thread_info(tsk->stack);
+ gr_free_thread_info(tsk, tsk->stack);
rt_mutex_debug_task_free(tsk);
ftrace_graph_exit_task(tsk);
put_seccomp_filter(tsk);
{
struct task_struct *tsk;
struct thread_info *ti;
+ void *lowmem_stack;
int node = tsk_fork_get_node(orig);
int err;
if (!tsk)
return NULL;
- ti = alloc_thread_info_node(tsk, node);
+ ti = gr_alloc_thread_info_node(tsk, node, &lowmem_stack);
if (!ti)
goto free_tsk;
goto free_ti;
tsk->stack = ti;
+#ifdef CONFIG_GRKERNSEC_KSTACKOVERFLOW
+ tsk->lowmem_stack = lowmem_stack;
+#endif
#ifdef CONFIG_SECCOMP
/*
* We must handle setting up seccomp filters once we're under
set_task_stack_end_magic(tsk);
#ifdef CONFIG_CC_STACKPROTECTOR
- tsk->stack_canary = get_random_int();
+ tsk->stack_canary = pax_get_random_long();
#endif
/*
tsk->splice_pipe = NULL;
tsk->task_frag.page = NULL;
- account_kernel_stack(ti, 1);
+ account_kernel_stack(tsk, ti, 1);
return tsk;
free_ti:
- free_thread_info(ti);
+ gr_free_thread_info(tsk, ti);
free_tsk:
free_task_struct(tsk);
return NULL;
}
#ifdef CONFIG_MMU
-static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
+static struct vm_area_struct *dup_vma(struct mm_struct *mm, struct mm_struct *oldmm, struct vm_area_struct *mpnt)
+{
+ struct vm_area_struct *tmp;
+ unsigned long charge;
+ struct file *file;
+ int retval;
+
+ charge = 0;
+ if (mpnt->vm_flags & VM_ACCOUNT) {
+ unsigned long len = vma_pages(mpnt);
+
+ if (security_vm_enough_memory_mm(oldmm, len)) /* sic */
+ goto fail_nomem;
+ charge = len;
+ }
+ tmp = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
+ if (!tmp)
+ goto fail_nomem;
+ *tmp = *mpnt;
+ tmp->vm_mm = mm;
+ INIT_LIST_HEAD(&tmp->anon_vma_chain);
+ retval = vma_dup_policy(mpnt, tmp);
+ if (retval)
+ goto fail_nomem_policy;
+ if (anon_vma_fork(tmp, mpnt))
+ goto fail_nomem_anon_vma_fork;
+ tmp->vm_flags &= ~VM_LOCKED;
+ tmp->vm_next = tmp->vm_prev = NULL;
+ tmp->vm_mirror = NULL;
+ file = tmp->vm_file;
+ if (file) {
+ struct inode *inode = file_inode(file);
+ struct address_space *mapping = file->f_mapping;
+
+ get_file(file);
+ if (tmp->vm_flags & VM_DENYWRITE)
+ atomic_dec(&inode->i_writecount);
+ i_mmap_lock_write(mapping);
+ if (tmp->vm_flags & VM_SHARED)
+ atomic_inc(&mapping->i_mmap_writable);
+ flush_dcache_mmap_lock(mapping);
+ /* insert tmp into the share list, just after mpnt */
+ if (unlikely(tmp->vm_flags & VM_NONLINEAR))
+ vma_nonlinear_insert(tmp, &mapping->i_mmap_nonlinear);
+ else
+ vma_interval_tree_insert_after(tmp, mpnt, &mapping->i_mmap);
+ flush_dcache_mmap_unlock(mapping);
+ i_mmap_unlock_write(mapping);
+ }
+
+ /*
+ * Clear hugetlb-related page reserves for children. This only
+ * affects MAP_PRIVATE mappings. Faults generated by the child
+ * are not guaranteed to succeed, even if read-only
+ */
+ if (is_vm_hugetlb_page(tmp))
+ reset_vma_resv_huge_pages(tmp);
+
+ return tmp;
+
+fail_nomem_anon_vma_fork:
+ mpol_put(vma_policy(tmp));
+fail_nomem_policy:
+ kmem_cache_free(vm_area_cachep, tmp);
+fail_nomem:
+ vm_unacct_memory(charge);
+ return NULL;
+}
+
+static __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
{
struct vm_area_struct *mpnt, *tmp, *prev, **pprev;
struct rb_node **rb_link, *rb_parent;
int retval;
- unsigned long charge;
uprobe_start_dup_mmap();
down_write(&oldmm->mmap_sem);
prev = NULL;
for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) {
- struct file *file;
-
if (mpnt->vm_flags & VM_DONTCOPY) {
vm_stat_account(mm, mpnt->vm_flags, mpnt->vm_file,
-vma_pages(mpnt));
continue;
}
- charge = 0;
- if (mpnt->vm_flags & VM_ACCOUNT) {
- unsigned long len = vma_pages(mpnt);
-
- if (security_vm_enough_memory_mm(oldmm, len)) /* sic */
- goto fail_nomem;
- charge = len;
- }
- tmp = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
- if (!tmp)
- goto fail_nomem;
- *tmp = *mpnt;
- INIT_LIST_HEAD(&tmp->anon_vma_chain);
- retval = vma_dup_policy(mpnt, tmp);
- if (retval)
- goto fail_nomem_policy;
- tmp->vm_mm = mm;
- if (anon_vma_fork(tmp, mpnt))
- goto fail_nomem_anon_vma_fork;
- tmp->vm_flags &= ~VM_LOCKED;
- tmp->vm_next = tmp->vm_prev = NULL;
- file = tmp->vm_file;
- if (file) {
- struct inode *inode = file_inode(file);
- struct address_space *mapping = file->f_mapping;
-
- get_file(file);
- if (tmp->vm_flags & VM_DENYWRITE)
- atomic_dec(&inode->i_writecount);
- i_mmap_lock_write(mapping);
- if (tmp->vm_flags & VM_SHARED)
- atomic_inc(&mapping->i_mmap_writable);
- flush_dcache_mmap_lock(mapping);
- /* insert tmp into the share list, just after mpnt */
- if (unlikely(tmp->vm_flags & VM_NONLINEAR))
- vma_nonlinear_insert(tmp,
- &mapping->i_mmap_nonlinear);
- else
- vma_interval_tree_insert_after(tmp, mpnt,
- &mapping->i_mmap);
- flush_dcache_mmap_unlock(mapping);
- i_mmap_unlock_write(mapping);
+ tmp = dup_vma(mm, oldmm, mpnt);
+ if (!tmp) {
+ retval = -ENOMEM;
+ goto out;
}
/*
if (retval)
goto out;
}
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (oldmm->pax_flags & MF_PAX_SEGMEXEC) {
+ struct vm_area_struct *mpnt_m;
+
+ for (mpnt = oldmm->mmap, mpnt_m = mm->mmap; mpnt; mpnt = mpnt->vm_next, mpnt_m = mpnt_m->vm_next) {
+ BUG_ON(!mpnt_m || mpnt_m->vm_mirror || mpnt->vm_mm != oldmm || mpnt_m->vm_mm != mm);
+
+ if (!mpnt->vm_mirror)
+ continue;
+
+ if (mpnt->vm_end <= SEGMEXEC_TASK_SIZE) {
+ BUG_ON(mpnt->vm_mirror->vm_mirror != mpnt);
+ mpnt->vm_mirror = mpnt_m;
+ } else {
+ BUG_ON(mpnt->vm_mirror->vm_mirror == mpnt || mpnt->vm_mirror->vm_mirror->vm_mm != mm);
+ mpnt_m->vm_mirror = mpnt->vm_mirror->vm_mirror;
+ mpnt_m->vm_mirror->vm_mirror = mpnt_m;
+ mpnt->vm_mirror->vm_mirror = mpnt;
+ }
+ }
+ BUG_ON(mpnt_m);
+ }
+#endif
+
/* a new mm has just been created */
arch_dup_mmap(oldmm, mm);
retval = 0;
up_write(&oldmm->mmap_sem);
uprobe_end_dup_mmap();
return retval;
-fail_nomem_anon_vma_fork:
- mpol_put(vma_policy(tmp));
-fail_nomem_policy:
- kmem_cache_free(vm_area_cachep, tmp);
-fail_nomem:
- retval = -ENOMEM;
- vm_unacct_memory(charge);
- goto out;
}
static inline int mm_alloc_pgd(struct mm_struct *mm)
return ERR_PTR(err);
mm = get_task_mm(task);
- if (mm && mm != current->mm &&
- !ptrace_may_access(task, mode)) {
+ if (mm && ((mm != current->mm && !ptrace_may_access(task, mode)) ||
+ (mode == PTRACE_MODE_ATTACH && (gr_handle_proc_ptrace(task) || gr_acl_handle_procpidmem(task))))) {
mmput(mm);
mm = ERR_PTR(-EACCES);
}
spin_unlock(&fs->lock);
return -EAGAIN;
}
- fs->users++;
+ atomic_inc(&fs->users);
spin_unlock(&fs->lock);
return 0;
}
tsk->fs = copy_fs_struct(fs);
if (!tsk->fs)
return -ENOMEM;
+ /* Carry through gr_chroot_dentry and is_chrooted instead
+ of recomputing it here. Already copied when the task struct
+ is duplicated. This allows pivot_root to not be treated as
+ a chroot
+ */
+ //gr_set_chroot_entries(tsk, &tsk->fs->root);
+
return 0;
}
* parts of the process environment (as per the clone
* flags). The actual kick-off is left to the caller.
*/
-static struct task_struct *copy_process(unsigned long clone_flags,
+static __latent_entropy struct task_struct *copy_process(unsigned long clone_flags,
unsigned long stack_start,
unsigned long stack_size,
int __user *child_tidptr,
DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled);
#endif
retval = -EAGAIN;
+
+ gr_learn_resource(p, RLIMIT_NPROC, atomic_read(&p->real_cred->user->processes), 0);
+
if (atomic_read(&p->real_cred->user->processes) >=
task_rlimit(p, RLIMIT_NPROC)) {
if (p->real_cred->user != INIT_USER &&
goto bad_fork_free_pid;
}
+ /* synchronizes with gr_set_acls()
+ we need to call this past the point of no return for fork()
+ */
+ gr_copy_label(p);
+
if (likely(p->pid)) {
ptrace_init_task(p, (clone_flags & CLONE_PTRACE) || trace);
bad_fork_free:
free_task(p);
fork_out:
+ gr_log_forkfail(retval);
+
return ERR_PTR(retval);
}
p = copy_process(clone_flags, stack_start, stack_size,
child_tidptr, NULL, trace);
+ add_latent_entropy();
/*
* Do this prior waking up the new thread - the thread pointer
* might get invalid after that point, if the thread exits quickly.
if (clone_flags & CLONE_PARENT_SETTID)
put_user(nr, parent_tidptr);
+ gr_handle_brute_check();
+
if (clone_flags & CLONE_VFORK) {
p->vfork_done = &vfork;
init_completion(&vfork);
mm_cachep = kmem_cache_create("mm_struct",
sizeof(struct mm_struct), ARCH_MIN_MMSTRUCT_ALIGN,
SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL);
- vm_area_cachep = KMEM_CACHE(vm_area_struct, SLAB_PANIC);
+ vm_area_cachep = KMEM_CACHE(vm_area_struct, SLAB_PANIC | SLAB_NO_SANITIZE);
mmap_init();
nsproxy_cache_init();
}
return 0;
/* don't need lock here; in the worst case we'll do useless copy */
- if (fs->users == 1)
+ if (atomic_read(&fs->users) == 1)
return 0;
*new_fsp = copy_fs_struct(fs);
fs = current->fs;
spin_lock(&fs->lock);
current->fs = new_fs;
- if (--fs->users)
+ gr_set_chroot_entries(current, ¤t->fs->root);
+ if (atomic_dec_return(&fs->users))
new_fs = NULL;
else
new_fs = fs;
atomic_t refcount;
union futex_key key;
-};
+} __randomize_layout;
/**
* struct futex_q - The hashed futex queue entry, one per waiting task
struct rt_mutex_waiter *rt_waiter;
union futex_key *requeue_pi_key;
u32 bitset;
-};
+} __randomize_layout;
static const struct futex_q futex_q_init = {
/* list gets initialized in queue_me()*/
struct page *page, *page_head;
int err, ro = 0;
+#ifdef CONFIG_PAX_SEGMEXEC
+ if ((mm->pax_flags & MF_PAX_SEGMEXEC) && address >= SEGMEXEC_TASK_SIZE)
+ return -EFAULT;
+#endif
+
/*
* The futex address must be "naturally" aligned.
*/
static int get_futex_value_locked(u32 *dest, u32 __user *from)
{
- int ret;
+ unsigned long ret;
pagefault_disable();
ret = __copy_from_user_inatomic(dest, from, sizeof(u32));
{
#ifndef CONFIG_HAVE_FUTEX_CMPXCHG
u32 curval;
+ mm_segment_t oldfs;
/*
* This will fail and we want it. Some arch implementations do
* implementation, the non-functional ones will return
* -ENOSYS.
*/
+ oldfs = get_fs();
+ set_fs(USER_DS);
if (cmpxchg_futex_value_locked(&curval, NULL, 0, 0) == -EFAULT)
futex_cmpxchg_enabled = 1;
+ set_fs(oldfs);
#endif
}
return 0;
}
-static void __user *futex_uaddr(struct robust_list __user *entry,
+static void __user __intentional_overflow(-1) *futex_uaddr(struct robust_list __user *entry,
compat_long_t futex_offset)
{
compat_uptr_t base = ptr_to_compat(entry);
}
#ifdef CONFIG_MODULES
-static inline int within(void *addr, void *start, unsigned long size)
-{
- return ((addr >= start) && (addr < start + size));
-}
-
/* Update list and generate events when modules are unloaded. */
static int gcov_module_notifier(struct notifier_block *nb, unsigned long event,
void *data)
/* Remove entries located in module from linked list. */
while ((info = gcov_info_next(info))) {
- if (within(info, mod->module_core, mod->core_size)) {
+ if (within_module_core_rw((unsigned long)info, mod)) {
gcov_info_unlink(prev, info);
if (gcov_events_enabled)
gcov_event(GCOV_REMOVE, info);
action_ret = handler_fn(desc, action);
if (action_ret == IRQ_HANDLED)
- atomic_inc(&desc->threads_handled);
+ atomic_inc_unchecked(&desc->threads_handled);
wake_threads_waitq(desc);
}
* count. We just care about the count being
* different than the one we saw before.
*/
- handled = atomic_read(&desc->threads_handled);
+ handled = atomic_read_unchecked(&desc->threads_handled);
handled |= SPURIOUS_DEFERRED;
if (handled != desc->threads_handled_last) {
action_ret = IRQ_HANDLED;
#include <linux/err.h>
#include <linux/static_key.h>
#include <linux/jump_label_ratelimit.h>
+#include <linux/mm.h>
#ifdef HAVE_JUMP_LABEL
size = (((unsigned long)stop - (unsigned long)start)
/ sizeof(struct jump_entry));
+ pax_open_kernel();
sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL);
+ pax_close_kernel();
}
static void jump_label_update(struct static_key *key, int enable);
struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
struct jump_entry *iter;
+ pax_open_kernel();
for (iter = iter_start; iter < iter_stop; iter++) {
if (within_module_init(iter->code, mod))
iter->code = 0;
}
+ pax_close_kernel();
}
static int
* Changed the compression method from stem compression to "table lookup"
* compression (see scripts/kallsyms.c for a more complete description)
*/
+#ifdef CONFIG_GRKERNSEC_HIDESYM
+#define __INCLUDED_BY_HIDESYM 1
+#endif
#include <linux/kallsyms.h>
#include <linux/module.h>
#include <linux/init.h>
static inline int is_kernel_inittext(unsigned long addr)
{
+ if (system_state != SYSTEM_BOOTING)
+ return 0;
+
if (addr >= (unsigned long)_sinittext
&& addr <= (unsigned long)_einittext)
return 1;
return 0;
}
+#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_KERNEXEC)
+#ifdef CONFIG_MODULES
+static inline int is_module_text(unsigned long addr)
+{
+ if ((unsigned long)MODULES_EXEC_VADDR <= addr && addr <= (unsigned long)MODULES_EXEC_END)
+ return 1;
+
+ addr = ktla_ktva(addr);
+ return (unsigned long)MODULES_EXEC_VADDR <= addr && addr <= (unsigned long)MODULES_EXEC_END;
+}
+#else
+static inline int is_module_text(unsigned long addr)
+{
+ return 0;
+}
+#endif
+#endif
+
static inline int is_kernel_text(unsigned long addr)
{
if ((addr >= (unsigned long)_stext && addr <= (unsigned long)_etext) ||
static inline int is_kernel(unsigned long addr)
{
+
+#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_KERNEXEC)
+ if (is_kernel_text(addr) || is_kernel_inittext(addr))
+ return 1;
+
+ if (ktla_ktva((unsigned long)_text) <= addr && addr < (unsigned long)_end)
+#else
if (addr >= (unsigned long)_stext && addr <= (unsigned long)_end)
+#endif
+
return 1;
return in_gate_area_no_mm(addr);
}
static int is_ksym_addr(unsigned long addr)
{
+
+#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_KERNEXEC)
+ if (is_module_text(addr))
+ return 0;
+#endif
+
if (all_var)
return is_kernel(addr);
static void reset_iter(struct kallsym_iter *iter, loff_t new_pos)
{
- iter->name[0] = '\0';
iter->nameoff = get_symbol_offset(new_pos);
iter->pos = new_pos;
}
{
struct kallsym_iter *iter = m->private;
+#ifdef CONFIG_GRKERNSEC_HIDESYM
+ if (!uid_eq(current_uid(), GLOBAL_ROOT_UID))
+ return 0;
+#endif
+
/* Some debugging symbols have no name. Ignore them. */
if (!iter->name[0])
return 0;
*/
type = iter->exported ? toupper(iter->type) :
tolower(iter->type);
+
seq_printf(m, "%pK %c %s\t[%s]\n", (void *)iter->value,
type, iter->name, iter->module_name);
} else
struct task_struct *task1, *task2;
int ret;
+#ifdef CONFIG_GRKERNSEC
+ return -ENOSYS;
+#endif
+
rcu_read_lock();
/*
compat_ulong_t, flags)
{
struct compat_kexec_segment in;
- struct kexec_segment out, __user *ksegments;
+ struct kexec_segment out;
+ struct kexec_segment __user *ksegments;
unsigned long i, result;
/* Don't allow clients that don't understand the native
kfree(info->argv);
}
-static int call_modprobe(char *module_name, int wait)
+static int call_modprobe(char *module_name, char *module_param, int wait)
{
struct subprocess_info *info;
static char *envp[] = {
NULL
};
- char **argv = kmalloc(sizeof(char *[5]), GFP_KERNEL);
+ char **argv = kmalloc(sizeof(char *[6]), GFP_KERNEL);
if (!argv)
goto out;
argv[1] = "-q";
argv[2] = "--";
argv[3] = module_name; /* check free_modprobe_argv() */
- argv[4] = NULL;
+ argv[4] = module_param;
+ argv[5] = NULL;
info = call_usermodehelper_setup(modprobe_path, argv, envp, GFP_KERNEL,
NULL, free_modprobe_argv, NULL);
* If module auto-loading support is disabled then this function
* becomes a no-operation.
*/
-int __request_module(bool wait, const char *fmt, ...)
+static int ____request_module(bool wait, char *module_param, const char *fmt, va_list ap)
{
- va_list args;
char module_name[MODULE_NAME_LEN];
unsigned int max_modprobes;
int ret;
if (!modprobe_path[0])
return 0;
- va_start(args, fmt);
- ret = vsnprintf(module_name, MODULE_NAME_LEN, fmt, args);
- va_end(args);
+ ret = vsnprintf(module_name, MODULE_NAME_LEN, fmt, ap);
if (ret >= MODULE_NAME_LEN)
return -ENAMETOOLONG;
if (ret)
return ret;
+#ifdef CONFIG_GRKERNSEC_MODHARDEN
+ if (uid_eq(current_uid(), GLOBAL_ROOT_UID)) {
+ /* hack to workaround consolekit/udisks stupidity */
+ read_lock(&tasklist_lock);
+ if (!strcmp(current->comm, "mount") &&
+ current->real_parent && !strncmp(current->real_parent->comm, "udisk", 5)) {
+ read_unlock(&tasklist_lock);
+ printk(KERN_ALERT "grsec: denied attempt to auto-load fs module %.64s by udisks\n", module_name);
+ return -EPERM;
+ }
+ read_unlock(&tasklist_lock);
+ }
+#endif
+
/* If modprobe needs a service that is in a module, we get a recursive
* loop. Limit the number of running kmod threads to max_threads/2 or
* MAX_KMOD_CONCURRENT, whichever is the smaller. A cleaner method
trace_module_request(module_name, wait, _RET_IP_);
- ret = call_modprobe(module_name, wait ? UMH_WAIT_PROC : UMH_WAIT_EXEC);
+ ret = call_modprobe(module_name, module_param, wait ? UMH_WAIT_PROC : UMH_WAIT_EXEC);
atomic_dec(&kmod_concurrent);
return ret;
}
+
+int ___request_module(bool wait, char *module_param, const char *fmt, ...)
+{
+ va_list args;
+ int ret;
+
+ va_start(args, fmt);
+ ret = ____request_module(wait, module_param, fmt, args);
+ va_end(args);
+
+ return ret;
+}
+
+int __request_module(bool wait, const char *fmt, ...)
+{
+ va_list args;
+ int ret;
+
+#ifdef CONFIG_GRKERNSEC_MODHARDEN
+ if (!uid_eq(current_uid(), GLOBAL_ROOT_UID)) {
+ char module_param[MODULE_NAME_LEN];
+
+ memset(module_param, 0, sizeof(module_param));
+
+ snprintf(module_param, sizeof(module_param) - 1, "grsec_modharden_normal%u_", GR_GLOBAL_UID(current_uid()));
+
+ va_start(args, fmt);
+ ret = ____request_module(wait, module_param, fmt, args);
+ va_end(args);
+
+ return ret;
+ }
+#endif
+
+ va_start(args, fmt);
+ ret = ____request_module(wait, NULL, fmt, args);
+ va_end(args);
+
+ return ret;
+}
+
EXPORT_SYMBOL(__request_module);
#endif /* CONFIG_MODULES */
static void call_usermodehelper_freeinfo(struct subprocess_info *info)
{
+#ifdef CONFIG_GRKERNSEC
+ kfree(info->path);
+ info->path = info->origpath;
+#endif
if (info->cleanup)
(*info->cleanup)(info);
kfree(info);
*/
set_user_nice(current, 0);
+#ifdef CONFIG_GRKERNSEC
+ /* this is race-free as far as userland is concerned as we copied
+ out the path to be used prior to this point and are now operating
+ on that copy
+ */
+ if ((strncmp(sub_info->path, "/sbin/", 6) && strncmp(sub_info->path, "/usr/lib/", 9) &&
+ strncmp(sub_info->path, "/lib/", 5) && strncmp(sub_info->path, "/lib64/", 7) &&
+ strcmp(sub_info->path, "/usr/share/apport/apport")) || strstr(sub_info->path, "..")) {
+ printk(KERN_ALERT "grsec: denied exec of usermode helper binary %.950s located outside of /sbin and system library paths\n", sub_info->path);
+ retval = -EPERM;
+ goto out;
+ }
+#endif
+
retval = -ENOMEM;
new = prepare_kernel_cred(current);
if (!new)
commit_creds(new);
retval = do_execve(getname_kernel(sub_info->path),
- (const char __user *const __user *)sub_info->argv,
- (const char __user *const __user *)sub_info->envp);
+ (const char __user *const __force_user *)sub_info->argv,
+ (const char __user *const __force_user *)sub_info->envp);
out:
sub_info->retval = retval;
/* wait_for_helper() will call umh_complete if UHM_WAIT_PROC. */
*
* Thus the __user pointer cast is valid here.
*/
- sys_wait4(pid, (int __user *)&ret, 0, NULL);
+ sys_wait4(pid, (int __force_user *)&ret, 0, NULL);
/*
* If ret is 0, either ____call_usermodehelper failed and the
goto out;
INIT_WORK(&sub_info->work, __call_usermodehelper);
+#ifdef CONFIG_GRKERNSEC
+ sub_info->origpath = path;
+ sub_info->path = kstrdup(path, gfp_mask);
+#else
sub_info->path = path;
+#endif
sub_info->argv = argv;
sub_info->envp = envp;
static int proc_cap_handler(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp, loff_t *ppos)
{
- struct ctl_table t;
+ ctl_table_no_const t;
unsigned long cap_array[_KERNEL_CAPABILITY_U32S];
kernel_cap_t new_cap;
int err, i;
* <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
* <prasanna@in.ibm.com> added function-return probes.
*/
+#ifdef CONFIG_GRKERNSEC_HIDESYM
+#define __INCLUDED_BY_HIDESYM 1
+#endif
#include <linux/kprobes.h>
#include <linux/hash.h>
#include <linux/init.h>
static void *alloc_insn_page(void)
{
- return module_alloc(PAGE_SIZE);
+ return module_alloc_exec(PAGE_SIZE);
}
static void free_insn_page(void *page)
{
- module_memfree(page);
+ module_memfree_exec(page);
}
struct kprobe_insn_cache kprobe_insn_slots = {
kprobe_type = "k";
if (sym)
- seq_printf(pi, "%p %s %s+0x%x %s ",
+ seq_printf(pi, "%pK %s %s+0x%x %s ",
p->addr, kprobe_type, sym, offset,
(modname ? modname : " "));
else
- seq_printf(pi, "%p %s %p ",
+ seq_printf(pi, "%pK %s %pK ",
p->addr, kprobe_type, p->addr);
if (!pp)
{
if (count+1 > UEVENT_HELPER_PATH_LEN)
return -ENOENT;
+ if (!capable(CAP_SYS_ADMIN))
+ return -EPERM;
memcpy(uevent_helper, buf, count);
uevent_helper[count] = '\0';
if (count && uevent_helper[count-1] == '\n')
return count;
}
-static struct bin_attribute notes_attr = {
+static bin_attribute_no_const notes_attr __read_only = {
.attr = {
.name = "notes",
.mode = S_IRUGO,
end = (unsigned long) &_end,
addr = (unsigned long) obj;
+#ifdef CONFIG_PAX_KERNEXEC
+ start = ktla_ktva(start);
+#endif
+
/*
* static variable?
*/
if (!static_obj(lock->key)) {
debug_locks_off();
printk("INFO: trying to register non-static key.\n");
+ printk("lock:%pS key:%pS.\n", lock, lock->key);
printk("the code is fine but needs lockdep annotation.\n");
printk("turning off the locking correctness validator.\n");
dump_stack();
if (!class)
return 0;
}
- atomic_inc((atomic_t *)&class->ops);
+ atomic_long_inc_unchecked((atomic_long_unchecked_t *)&class->ops);
if (very_verbose(class)) {
printk("\nacquire class [%p] %s", class->key, class->name);
if (class->name_version > 1)
return 0;
}
- seq_printf(m, "%p", class->key);
+ seq_printf(m, "%pK", class->key);
#ifdef CONFIG_DEBUG_LOCKDEP
seq_printf(m, " OPS:%8ld", class->ops);
#endif
list_for_each_entry(entry, &class->locks_after, entry) {
if (entry->distance == 1) {
- seq_printf(m, " -> [%p] ", entry->class->key);
+ seq_printf(m, " -> [%pK] ", entry->class->key);
print_name(m, entry->class);
seq_puts(m, "\n");
}
if (!class->key)
continue;
- seq_printf(m, "[%p] ", class->key);
+ seq_printf(m, "[%pK] ", class->key);
print_name(m, class);
seq_puts(m, "\n");
}
if (!i)
seq_line(m, '-', 40-namelen, namelen);
- snprintf(ip, sizeof(ip), "[<%p>]",
+ snprintf(ip, sizeof(ip), "[<%pK>]",
(void *)class->contention_point[i]);
seq_printf(m, "%40s %14lu %29s %pS\n",
name, stats->contention_point[i],
if (!i)
seq_line(m, '-', 40-namelen, namelen);
- snprintf(ip, sizeof(ip), "[<%p>]",
+ snprintf(ip, sizeof(ip), "[<%pK>]",
(void *)class->contending_point[i]);
seq_printf(m, "%40s %14lu %29s %pS\n",
name, stats->contending_point[i],
prev = decode_cpu(old);
node->prev = prev;
- ACCESS_ONCE(prev->next) = node;
+ ACCESS_ONCE_RW(prev->next) = node;
/*
* Normally @prev is untouchable after the above store; because at that
* it will wait in Step-A.
*/
- ACCESS_ONCE(next->prev) = prev;
- ACCESS_ONCE(prev->next) = next;
+ ACCESS_ONCE_RW(next->prev) = prev;
+ ACCESS_ONCE_RW(prev->next) = next;
return false;
}
node = this_cpu_ptr(&osq_node);
next = xchg(&node->next, NULL);
if (next) {
- ACCESS_ONCE(next->locked) = 1;
+ ACCESS_ONCE_RW(next->locked) = 1;
return;
}
next = osq_wait_next(lock, node, NULL);
if (next)
- ACCESS_ONCE(next->locked) = 1;
+ ACCESS_ONCE_RW(next->locked) = 1;
}
#endif
*/
return;
}
- ACCESS_ONCE(prev->next) = node;
+ ACCESS_ONCE_RW(prev->next) = node;
/* Wait until the lock holder passes the lock down. */
arch_mcs_spin_lock_contended(&node->locked);
}
void debug_mutex_add_waiter(struct mutex *lock, struct mutex_waiter *waiter,
- struct thread_info *ti)
+ struct task_struct *task)
{
SMP_DEBUG_LOCKS_WARN_ON(!spin_is_locked(&lock->wait_lock));
/* Mark the current thread as blocked on the lock: */
- ti->task->blocked_on = waiter;
+ task->blocked_on = waiter;
}
void mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter,
- struct thread_info *ti)
+ struct task_struct *task)
{
DEBUG_LOCKS_WARN_ON(list_empty(&waiter->list));
- DEBUG_LOCKS_WARN_ON(waiter->task != ti->task);
- DEBUG_LOCKS_WARN_ON(ti->task->blocked_on != waiter);
- ti->task->blocked_on = NULL;
+ DEBUG_LOCKS_WARN_ON(waiter->task != task);
+ DEBUG_LOCKS_WARN_ON(task->blocked_on != waiter);
+ task->blocked_on = NULL;
list_del_init(&waiter->list);
waiter->task = NULL;
extern void debug_mutex_free_waiter(struct mutex_waiter *waiter);
extern void debug_mutex_add_waiter(struct mutex *lock,
struct mutex_waiter *waiter,
- struct thread_info *ti);
+ struct task_struct *task);
extern void mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter,
- struct thread_info *ti);
+ struct task_struct *task);
extern void debug_mutex_unlock(struct mutex *lock);
extern void debug_mutex_init(struct mutex *lock, const char *name,
struct lock_class_key *key);
goto skip_wait;
debug_mutex_lock_common(lock, &waiter);
- debug_mutex_add_waiter(lock, &waiter, task_thread_info(task));
+ debug_mutex_add_waiter(lock, &waiter, task);
/* add waiting tasks to the end of the waitqueue (FIFO): */
list_add_tail(&waiter.list, &lock->wait_list);
schedule_preempt_disabled();
spin_lock_mutex(&lock->wait_lock, flags);
}
- mutex_remove_waiter(lock, &waiter, current_thread_info());
+ mutex_remove_waiter(lock, &waiter, task);
/* set it to 0 if there are no waiters left: */
if (likely(list_empty(&lock->wait_list)))
atomic_set(&lock->count, 0);
return 0;
err:
- mutex_remove_waiter(lock, &waiter, task_thread_info(task));
+ mutex_remove_waiter(lock, &waiter, task);
spin_unlock_mutex(&lock->wait_lock, flags);
debug_mutex_free_waiter(&waiter);
mutex_release(&lock->dep_map, 1, ip);
#define MAX_RT_TEST_MUTEXES 8
static spinlock_t rttest_lock;
-static atomic_t rttest_event;
+static atomic_unchecked_t rttest_event;
struct test_thread_data {
int opcode;
case RTTEST_LOCKCONT:
td->mutexes[td->opdata] = 1;
- td->event = atomic_add_return(1, &rttest_event);
+ td->event = atomic_add_return_unchecked(1, &rttest_event);
return 0;
case RTTEST_RESET:
return 0;
case RTTEST_RESETEVENT:
- atomic_set(&rttest_event, 0);
+ atomic_set_unchecked(&rttest_event, 0);
return 0;
default:
return ret;
td->mutexes[id] = 1;
- td->event = atomic_add_return(1, &rttest_event);
+ td->event = atomic_add_return_unchecked(1, &rttest_event);
rt_mutex_lock(&mutexes[id]);
- td->event = atomic_add_return(1, &rttest_event);
+ td->event = atomic_add_return_unchecked(1, &rttest_event);
td->mutexes[id] = 4;
return 0;
return ret;
td->mutexes[id] = 1;
- td->event = atomic_add_return(1, &rttest_event);
+ td->event = atomic_add_return_unchecked(1, &rttest_event);
ret = rt_mutex_lock_interruptible(&mutexes[id], 0);
- td->event = atomic_add_return(1, &rttest_event);
+ td->event = atomic_add_return_unchecked(1, &rttest_event);
td->mutexes[id] = ret ? 0 : 4;
return ret ? -EINTR : 0;
if (id < 0 || id >= MAX_RT_TEST_MUTEXES || td->mutexes[id] != 4)
return ret;
- td->event = atomic_add_return(1, &rttest_event);
+ td->event = atomic_add_return_unchecked(1, &rttest_event);
rt_mutex_unlock(&mutexes[id]);
- td->event = atomic_add_return(1, &rttest_event);
+ td->event = atomic_add_return_unchecked(1, &rttest_event);
td->mutexes[id] = 0;
return 0;
break;
td->mutexes[dat] = 2;
- td->event = atomic_add_return(1, &rttest_event);
+ td->event = atomic_add_return_unchecked(1, &rttest_event);
break;
default:
return;
td->mutexes[dat] = 3;
- td->event = atomic_add_return(1, &rttest_event);
+ td->event = atomic_add_return_unchecked(1, &rttest_event);
break;
case RTTEST_LOCKNOWAIT:
return;
td->mutexes[dat] = 1;
- td->event = atomic_add_return(1, &rttest_event);
+ td->event = atomic_add_return_unchecked(1, &rttest_event);
return;
default:
#include <linux/jump_label.h>
#include <linux/pfn.h>
#include <linux/bsearch.h>
+#include <linux/grsecurity.h>
#include <uapi/linux/module.h>
#include "module-internal.h"
/* Bounds of module allocation, for speeding __module_address.
* Protected by module_mutex. */
-static unsigned long module_addr_min = -1UL, module_addr_max = 0;
+static unsigned long module_addr_min_rw = -1UL, module_addr_max_rw = 0;
+static unsigned long module_addr_min_rx = -1UL, module_addr_max_rx = 0;
int register_module_notifier(struct notifier_block *nb)
{
return true;
list_for_each_entry_rcu(mod, &modules, list) {
- struct symsearch arr[] = {
+ struct symsearch modarr[] = {
{ mod->syms, mod->syms + mod->num_syms, mod->crcs,
NOT_GPL_ONLY, false },
{ mod->gpl_syms, mod->gpl_syms + mod->num_gpl_syms,
if (mod->state == MODULE_STATE_UNFORMED)
continue;
- if (each_symbol_in_section(arr, ARRAY_SIZE(arr), mod, fn, data))
+ if (each_symbol_in_section(modarr, ARRAY_SIZE(modarr), mod, fn, data))
return true;
}
return false;
if (!pcpusec->sh_size)
return 0;
- if (align > PAGE_SIZE) {
+ if (align-1 >= PAGE_SIZE) {
pr_warn("%s: per-cpu alignment %li > %li\n",
mod->name, align, PAGE_SIZE);
align = PAGE_SIZE;
static ssize_t show_coresize(struct module_attribute *mattr,
struct module_kobject *mk, char *buffer)
{
- return sprintf(buffer, "%u\n", mk->mod->core_size);
+ return sprintf(buffer, "%u\n", mk->mod->core_size_rx + mk->mod->core_size_rw);
}
static struct module_attribute modinfo_coresize =
static ssize_t show_initsize(struct module_attribute *mattr,
struct module_kobject *mk, char *buffer)
{
- return sprintf(buffer, "%u\n", mk->mod->init_size);
+ return sprintf(buffer, "%u\n", mk->mod->init_size_rx + mk->mod->init_size_rw);
}
static struct module_attribute modinfo_initsize =
goto bad_version;
}
+#ifdef CONFIG_GRKERNSEC_RANDSTRUCT
+ /*
+ * avoid potentially printing jibberish on attempted load
+ * of a module randomized with a different seed
+ */
+ pr_warn("no symbol version for %s\n", symname);
+#else
pr_warn("%s: no symbol version for %s\n", mod->name, symname);
+#endif
return 0;
bad_version:
+#ifdef CONFIG_GRKERNSEC_RANDSTRUCT
+ /*
+ * avoid potentially printing jibberish on attempted load
+ * of a module randomized with a different seed
+ */
+ pr_warn("attempted module disagrees about version of symbol %s\n",
+ symname);
+#else
pr_warn("%s: disagrees about version of symbol %s\n",
mod->name, symname);
+#endif
return 0;
}
*/
#ifdef CONFIG_SYSFS
-#ifdef CONFIG_KALLSYMS
+#if defined(CONFIG_KALLSYMS) && !defined(CONFIG_GRKERNSEC_HIDESYM)
static inline bool sect_empty(const Elf_Shdr *sect)
{
return !(sect->sh_flags & SHF_ALLOC) || sect->sh_size == 0;
{
unsigned int notes, loaded, i;
struct module_notes_attrs *notes_attrs;
- struct bin_attribute *nattr;
+ bin_attribute_no_const *nattr;
/* failed to create section attributes, so can't create notes */
if (!mod->sect_attrs)
static int module_add_modinfo_attrs(struct module *mod)
{
struct module_attribute *attr;
- struct module_attribute *temp_attr;
+ module_attribute_no_const *temp_attr;
int error = 0;
int i;
static void unset_module_core_ro_nx(struct module *mod)
{
- set_page_attributes(mod->module_core + mod->core_text_size,
- mod->module_core + mod->core_size,
+ set_page_attributes(mod->module_core_rw,
+ mod->module_core_rw + mod->core_size_rw,
set_memory_x);
- set_page_attributes(mod->module_core,
- mod->module_core + mod->core_ro_size,
+ set_page_attributes(mod->module_core_rx,
+ mod->module_core_rx + mod->core_size_rx,
set_memory_rw);
}
static void unset_module_init_ro_nx(struct module *mod)
{
- set_page_attributes(mod->module_init + mod->init_text_size,
- mod->module_init + mod->init_size,
+ set_page_attributes(mod->module_init_rw,
+ mod->module_init_rw + mod->init_size_rw,
set_memory_x);
- set_page_attributes(mod->module_init,
- mod->module_init + mod->init_ro_size,
+ set_page_attributes(mod->module_init_rx,
+ mod->module_init_rx + mod->init_size_rx,
set_memory_rw);
}
list_for_each_entry_rcu(mod, &modules, list) {
if (mod->state == MODULE_STATE_UNFORMED)
continue;
- if ((mod->module_core) && (mod->core_text_size)) {
- set_page_attributes(mod->module_core,
- mod->module_core + mod->core_text_size,
+ if ((mod->module_core_rx) && (mod->core_size_rx)) {
+ set_page_attributes(mod->module_core_rx,
+ mod->module_core_rx + mod->core_size_rx,
set_memory_rw);
}
- if ((mod->module_init) && (mod->init_text_size)) {
- set_page_attributes(mod->module_init,
- mod->module_init + mod->init_text_size,
+ if ((mod->module_init_rx) && (mod->init_size_rx)) {
+ set_page_attributes(mod->module_init_rx,
+ mod->module_init_rx + mod->init_size_rx,
set_memory_rw);
}
}
list_for_each_entry_rcu(mod, &modules, list) {
if (mod->state == MODULE_STATE_UNFORMED)
continue;
- if ((mod->module_core) && (mod->core_text_size)) {
- set_page_attributes(mod->module_core,
- mod->module_core + mod->core_text_size,
+ if ((mod->module_core_rx) && (mod->core_size_rx)) {
+ set_page_attributes(mod->module_core_rx,
+ mod->module_core_rx + mod->core_size_rx,
set_memory_ro);
}
- if ((mod->module_init) && (mod->init_text_size)) {
- set_page_attributes(mod->module_init,
- mod->module_init + mod->init_text_size,
+ if ((mod->module_init_rx) && (mod->init_size_rx)) {
+ set_page_attributes(mod->module_init_rx,
+ mod->module_init_rx + mod->init_size_rx,
set_memory_ro);
}
}
#else
static inline void set_section_ro_nx(void *base, unsigned long text_size, unsigned long ro_size, unsigned long total_size) { }
static void unset_module_core_ro_nx(struct module *mod) { }
-static void unset_module_init_ro_nx(struct module *mod) { }
+static void unset_module_init_ro_nx(struct module *mod)
+{
+
+#ifdef CONFIG_PAX_KERNEXEC
+ set_memory_nx((unsigned long)mod->module_init_rx, PFN_UP(mod->init_size_rx));
+ set_memory_rw((unsigned long)mod->module_init_rx, PFN_UP(mod->init_size_rx));
+#endif
+
+}
#endif
void __weak module_memfree(void *module_region)
/* This may be NULL, but that's OK */
unset_module_init_ro_nx(mod);
module_arch_freeing_init(mod);
- module_memfree(mod->module_init);
+ module_memfree(mod->module_init_rw);
+ module_memfree_exec(mod->module_init_rx);
kfree(mod->args);
percpu_modfree(mod);
/* Free lock-classes: */
- lockdep_free_key_range(mod->module_core, mod->core_size);
+ lockdep_free_key_range(mod->module_core_rx, mod->core_size_rx);
+ lockdep_free_key_range(mod->module_core_rw, mod->core_size_rw);
/* Finally, free the core (containing the module structure) */
unset_module_core_ro_nx(mod);
- module_memfree(mod->module_core);
+ module_memfree_exec(mod->module_core_rx);
+ module_memfree(mod->module_core_rw);
#ifdef CONFIG_MPU
update_protections(current->mm);
int ret = 0;
const struct kernel_symbol *ksym;
+#ifdef CONFIG_GRKERNSEC_MODHARDEN
+ int is_fs_load = 0;
+ int register_filesystem_found = 0;
+ char *p;
+
+ p = strstr(mod->args, "grsec_modharden_fs");
+ if (p) {
+ char *endptr = p + sizeof("grsec_modharden_fs") - 1;
+ /* copy \0 as well */
+ memmove(p, endptr, strlen(mod->args) - (unsigned int)(endptr - mod->args) + 1);
+ is_fs_load = 1;
+ }
+#endif
+
for (i = 1; i < symsec->sh_size / sizeof(Elf_Sym); i++) {
const char *name = info->strtab + sym[i].st_name;
+#ifdef CONFIG_GRKERNSEC_MODHARDEN
+ /* it's a real shame this will never get ripped and copied
+ upstream! ;(
+ */
+ if (is_fs_load && !strcmp(name, "register_filesystem"))
+ register_filesystem_found = 1;
+#endif
+
switch (sym[i].st_shndx) {
case SHN_COMMON:
/* Ignore common symbols */
ksym = resolve_symbol_wait(mod, info, name);
/* Ok if resolved. */
if (ksym && !IS_ERR(ksym)) {
+ pax_open_kernel();
sym[i].st_value = ksym->value;
+ pax_close_kernel();
break;
}
secbase = (unsigned long)mod_percpu(mod);
else
secbase = info->sechdrs[sym[i].st_shndx].sh_addr;
+ pax_open_kernel();
sym[i].st_value += secbase;
+ pax_close_kernel();
break;
}
}
+#ifdef CONFIG_GRKERNSEC_MODHARDEN
+ if (is_fs_load && !register_filesystem_found) {
+ printk(KERN_ALERT "grsec: Denied attempt to load non-fs module %.64s through mount\n", mod->name);
+ ret = -EPERM;
+ }
+#endif
+
return ret;
}
|| s->sh_entsize != ~0UL
|| strstarts(sname, ".init"))
continue;
- s->sh_entsize = get_offset(mod, &mod->core_size, s, i);
+ if ((s->sh_flags & SHF_WRITE) || !(s->sh_flags & SHF_ALLOC))
+ s->sh_entsize = get_offset(mod, &mod->core_size_rw, s, i);
+ else
+ s->sh_entsize = get_offset(mod, &mod->core_size_rx, s, i);
pr_debug("\t%s\n", sname);
}
- switch (m) {
- case 0: /* executable */
- mod->core_size = debug_align(mod->core_size);
- mod->core_text_size = mod->core_size;
- break;
- case 1: /* RO: text and ro-data */
- mod->core_size = debug_align(mod->core_size);
- mod->core_ro_size = mod->core_size;
- break;
- case 3: /* whole core */
- mod->core_size = debug_align(mod->core_size);
- break;
- }
}
pr_debug("Init section allocation order:\n");
|| s->sh_entsize != ~0UL
|| !strstarts(sname, ".init"))
continue;
- s->sh_entsize = (get_offset(mod, &mod->init_size, s, i)
- | INIT_OFFSET_MASK);
+ if ((s->sh_flags & SHF_WRITE) || !(s->sh_flags & SHF_ALLOC))
+ s->sh_entsize = get_offset(mod, &mod->init_size_rw, s, i);
+ else
+ s->sh_entsize = get_offset(mod, &mod->init_size_rx, s, i);
+ s->sh_entsize |= INIT_OFFSET_MASK;
pr_debug("\t%s\n", sname);
}
- switch (m) {
- case 0: /* executable */
- mod->init_size = debug_align(mod->init_size);
- mod->init_text_size = mod->init_size;
- break;
- case 1: /* RO: text and ro-data */
- mod->init_size = debug_align(mod->init_size);
- mod->init_ro_size = mod->init_size;
- break;
- case 3: /* whole init */
- mod->init_size = debug_align(mod->init_size);
- break;
- }
}
}
/* Put symbol section at end of init part of module. */
symsect->sh_flags |= SHF_ALLOC;
- symsect->sh_entsize = get_offset(mod, &mod->init_size, symsect,
+ symsect->sh_entsize = get_offset(mod, &mod->init_size_rx, symsect,
info->index.sym) | INIT_OFFSET_MASK;
pr_debug("\t%s\n", info->secstrings + symsect->sh_name);
}
/* Append room for core symbols at end of core part. */
- info->symoffs = ALIGN(mod->core_size, symsect->sh_addralign ?: 1);
- info->stroffs = mod->core_size = info->symoffs + ndst * sizeof(Elf_Sym);
- mod->core_size += strtab_size;
+ info->symoffs = ALIGN(mod->core_size_rx, symsect->sh_addralign ?: 1);
+ info->stroffs = mod->core_size_rx = info->symoffs + ndst * sizeof(Elf_Sym);
+ mod->core_size_rx += strtab_size;
/* Put string table section at end of init part of module. */
strsect->sh_flags |= SHF_ALLOC;
- strsect->sh_entsize = get_offset(mod, &mod->init_size, strsect,
+ strsect->sh_entsize = get_offset(mod, &mod->init_size_rx, strsect,
info->index.str) | INIT_OFFSET_MASK;
pr_debug("\t%s\n", info->secstrings + strsect->sh_name);
}
/* Make sure we get permanent strtab: don't use info->strtab. */
mod->strtab = (void *)info->sechdrs[info->index.str].sh_addr;
+ pax_open_kernel();
+
/* Set types up while we still have access to sections. */
for (i = 0; i < mod->num_symtab; i++)
mod->symtab[i].st_info = elf_type(&mod->symtab[i], info);
- mod->core_symtab = dst = mod->module_core + info->symoffs;
- mod->core_strtab = s = mod->module_core + info->stroffs;
+ mod->core_symtab = dst = mod->module_core_rx + info->symoffs;
+ mod->core_strtab = s = mod->module_core_rx + info->stroffs;
src = mod->symtab;
for (ndst = i = 0; i < mod->num_symtab; i++) {
if (i == 0 ||
}
}
mod->core_num_syms = ndst;
+
+ pax_close_kernel();
}
#else
static inline void layout_symtab(struct module *mod, struct load_info *info)
return vmalloc_exec(size);
}
-static void *module_alloc_update_bounds(unsigned long size)
+static void *module_alloc_update_bounds_rw(unsigned long size)
{
void *ret = module_alloc(size);
if (ret) {
mutex_lock(&module_mutex);
/* Update module bounds. */
- if ((unsigned long)ret < module_addr_min)
- module_addr_min = (unsigned long)ret;
- if ((unsigned long)ret + size > module_addr_max)
- module_addr_max = (unsigned long)ret + size;
+ if ((unsigned long)ret < module_addr_min_rw)
+ module_addr_min_rw = (unsigned long)ret;
+ if ((unsigned long)ret + size > module_addr_max_rw)
+ module_addr_max_rw = (unsigned long)ret + size;
+ mutex_unlock(&module_mutex);
+ }
+ return ret;
+}
+
+static void *module_alloc_update_bounds_rx(unsigned long size)
+{
+ void *ret = module_alloc_exec(size);
+
+ if (ret) {
+ mutex_lock(&module_mutex);
+ /* Update module bounds. */
+ if ((unsigned long)ret < module_addr_min_rx)
+ module_addr_min_rx = (unsigned long)ret;
+ if ((unsigned long)ret + size > module_addr_max_rx)
+ module_addr_max_rx = (unsigned long)ret + size;
mutex_unlock(&module_mutex);
}
return ret;
mod = (void *)info->sechdrs[info->index.mod].sh_addr;
if (info->index.sym == 0) {
+#ifdef CONFIG_GRKERNSEC_RANDSTRUCT
+ /*
+ * avoid potentially printing jibberish on attempted load
+ * of a module randomized with a different seed
+ */
+ pr_warn("module has no symbols (stripped?)\n");
+#else
pr_warn("%s: module has no symbols (stripped?)\n", mod->name);
+#endif
return ERR_PTR(-ENOEXEC);
}
static int check_modinfo(struct module *mod, struct load_info *info, int flags)
{
const char *modmagic = get_modinfo(info, "vermagic");
+ const char *license = get_modinfo(info, "license");
int err;
+#ifdef CONFIG_PAX_KERNEXEC_PLUGIN_METHOD_OR
+ if (!license || !license_is_gpl_compatible(license))
+ return -ENOEXEC;
+#endif
+
if (flags & MODULE_INIT_IGNORE_VERMAGIC)
modmagic = NULL;
}
/* Set up license info based on the info section */
- set_license(mod, get_modinfo(info, "license"));
+ set_license(mod, license);
return 0;
}
void *ptr;
/* Do the allocs. */
- ptr = module_alloc_update_bounds(mod->core_size);
+ ptr = module_alloc_update_bounds_rw(mod->core_size_rw);
/*
* The pointer to this block is stored in the module structure
* which is inside the block. Just mark it as not being a
if (!ptr)
return -ENOMEM;
- memset(ptr, 0, mod->core_size);
- mod->module_core = ptr;
+ memset(ptr, 0, mod->core_size_rw);
+ mod->module_core_rw = ptr;
- if (mod->init_size) {
- ptr = module_alloc_update_bounds(mod->init_size);
+ if (mod->init_size_rw) {
+ ptr = module_alloc_update_bounds_rw(mod->init_size_rw);
/*
* The pointer to this block is stored in the module structure
* which is inside the block. This block doesn't need to be
*/
kmemleak_ignore(ptr);
if (!ptr) {
- module_memfree(mod->module_core);
+ module_memfree(mod->module_core_rw);
+ return -ENOMEM;
+ }
+ memset(ptr, 0, mod->init_size_rw);
+ mod->module_init_rw = ptr;
+ } else
+ mod->module_init_rw = NULL;
+
+ ptr = module_alloc_update_bounds_rx(mod->core_size_rx);
+ kmemleak_not_leak(ptr);
+ if (!ptr) {
+ if (mod->module_init_rw)
+ module_memfree(mod->module_init_rw);
+ module_memfree(mod->module_core_rw);
+ return -ENOMEM;
+ }
+
+ pax_open_kernel();
+ memset(ptr, 0, mod->core_size_rx);
+ pax_close_kernel();
+ mod->module_core_rx = ptr;
+
+ if (mod->init_size_rx) {
+ ptr = module_alloc_update_bounds_rx(mod->init_size_rx);
+ kmemleak_ignore(ptr);
+ if (!ptr && mod->init_size_rx) {
+ module_memfree_exec(mod->module_core_rx);
+ if (mod->module_init_rw)
+ module_memfree(mod->module_init_rw);
+ module_memfree(mod->module_core_rw);
return -ENOMEM;
}
- memset(ptr, 0, mod->init_size);
- mod->module_init = ptr;
+
+ pax_open_kernel();
+ memset(ptr, 0, mod->init_size_rx);
+ pax_close_kernel();
+ mod->module_init_rx = ptr;
} else
- mod->module_init = NULL;
+ mod->module_init_rx = NULL;
/* Transfer each section which specifies SHF_ALLOC */
pr_debug("final section addresses:\n");
if (!(shdr->sh_flags & SHF_ALLOC))
continue;
- if (shdr->sh_entsize & INIT_OFFSET_MASK)
- dest = mod->module_init
- + (shdr->sh_entsize & ~INIT_OFFSET_MASK);
- else
- dest = mod->module_core + shdr->sh_entsize;
+ if (shdr->sh_entsize & INIT_OFFSET_MASK) {
+ if ((shdr->sh_flags & SHF_WRITE) || !(shdr->sh_flags & SHF_ALLOC))
+ dest = mod->module_init_rw
+ + (shdr->sh_entsize & ~INIT_OFFSET_MASK);
+ else
+ dest = mod->module_init_rx
+ + (shdr->sh_entsize & ~INIT_OFFSET_MASK);
+ } else {
+ if ((shdr->sh_flags & SHF_WRITE) || !(shdr->sh_flags & SHF_ALLOC))
+ dest = mod->module_core_rw + shdr->sh_entsize;
+ else
+ dest = mod->module_core_rx + shdr->sh_entsize;
+ }
+
+ if (shdr->sh_type != SHT_NOBITS) {
+
+#ifdef CONFIG_PAX_KERNEXEC
+#ifdef CONFIG_X86_64
+ if ((shdr->sh_flags & SHF_WRITE) && (shdr->sh_flags & SHF_EXECINSTR))
+ set_memory_x((unsigned long)dest, (shdr->sh_size + PAGE_SIZE) >> PAGE_SHIFT);
+#endif
+ if (!(shdr->sh_flags & SHF_WRITE) && (shdr->sh_flags & SHF_ALLOC)) {
+ pax_open_kernel();
+ memcpy(dest, (void *)shdr->sh_addr, shdr->sh_size);
+ pax_close_kernel();
+ } else
+#endif
- if (shdr->sh_type != SHT_NOBITS)
memcpy(dest, (void *)shdr->sh_addr, shdr->sh_size);
+ }
/* Update sh_addr to point to copy in image. */
- shdr->sh_addr = (unsigned long)dest;
+
+#ifdef CONFIG_PAX_KERNEXEC
+ if (shdr->sh_flags & SHF_EXECINSTR)
+ shdr->sh_addr = ktva_ktla((unsigned long)dest);
+ else
+#endif
+
+ shdr->sh_addr = (unsigned long)dest;
pr_debug("\t0x%lx %s\n",
(long)shdr->sh_addr, info->secstrings + shdr->sh_name);
}
* Do it before processing of module parameters, so the module
* can provide parameter accessor functions of its own.
*/
- if (mod->module_init)
- flush_icache_range((unsigned long)mod->module_init,
- (unsigned long)mod->module_init
- + mod->init_size);
- flush_icache_range((unsigned long)mod->module_core,
- (unsigned long)mod->module_core + mod->core_size);
+ if (mod->module_init_rx)
+ flush_icache_range((unsigned long)mod->module_init_rx,
+ (unsigned long)mod->module_init_rx
+ + mod->init_size_rx);
+ flush_icache_range((unsigned long)mod->module_core_rx,
+ (unsigned long)mod->module_core_rx + mod->core_size_rx);
set_fs(old_fs);
}
{
percpu_modfree(mod);
module_arch_freeing_init(mod);
- module_memfree(mod->module_init);
- module_memfree(mod->module_core);
+ module_memfree_exec(mod->module_init_rx);
+ module_memfree_exec(mod->module_core_rx);
+ module_memfree(mod->module_init_rw);
+ module_memfree(mod->module_core_rw);
}
int __weak module_finalize(const Elf_Ehdr *hdr,
static int post_relocation(struct module *mod, const struct load_info *info)
{
/* Sort exception table now relocations are done. */
+ pax_open_kernel();
sort_extable(mod->extable, mod->extable + mod->num_exentries);
+ pax_close_kernel();
/* Copy relocated percpu area over. */
percpu_modcopy(mod, (void *)info->sechdrs[info->index.pcpu].sh_addr,
/* For freeing module_init on success, in case kallsyms traversing */
struct mod_initfree {
struct rcu_head rcu;
- void *module_init;
+ void *module_init_rw;
+ void *module_init_rx;
};
static void do_free_init(struct rcu_head *head)
{
struct mod_initfree *m = container_of(head, struct mod_initfree, rcu);
- module_memfree(m->module_init);
+ module_memfree(m->module_init_rw);
+ module_memfree_exec(m->module_init_rx);
kfree(m);
}
ret = -ENOMEM;
goto fail;
}
- freeinit->module_init = mod->module_init;
+ freeinit->module_init_rw = mod->module_init_rw;
+ freeinit->module_init_rx = mod->module_init_rx;
/*
* We want to find out whether @mod uses async during init. Clear
#endif
unset_module_init_ro_nx(mod);
module_arch_freeing_init(mod);
- mod->module_init = NULL;
- mod->init_size = 0;
- mod->init_ro_size = 0;
- mod->init_text_size = 0;
+ mod->module_init_rw = NULL;
+ mod->module_init_rx = NULL;
+ mod->init_size_rw = 0;
+ mod->init_size_rx = 0;
/*
* We want to free module_init, but be aware that kallsyms may be
* walking this with preempt disabled. In all the failure paths,
module_bug_finalize(info->hdr, info->sechdrs, mod);
/* Set RO and NX regions for core */
- set_section_ro_nx(mod->module_core,
- mod->core_text_size,
- mod->core_ro_size,
- mod->core_size);
+ set_section_ro_nx(mod->module_core_rx,
+ mod->core_size_rx,
+ mod->core_size_rx,
+ mod->core_size_rx);
/* Set RO and NX regions for init */
- set_section_ro_nx(mod->module_init,
- mod->init_text_size,
- mod->init_ro_size,
- mod->init_size);
+ set_section_ro_nx(mod->module_init_rx,
+ mod->init_size_rx,
+ mod->init_size_rx,
+ mod->init_size_rx);
/* Mark state as coming so strong_try_module_get() ignores us,
* but kallsyms etc. can see us. */
if (err)
goto free_unload;
+ /* Now copy in args */
+ mod->args = strndup_user(uargs, ~0UL >> 1);
+ if (IS_ERR(mod->args)) {
+ err = PTR_ERR(mod->args);
+ goto free_unload;
+ }
+
/* Set up MODINFO_ATTR fields */
setup_modinfo(mod, info);
+#ifdef CONFIG_GRKERNSEC_MODHARDEN
+ {
+ char *p, *p2;
+
+ if (strstr(mod->args, "grsec_modharden_netdev")) {
+ printk(KERN_ALERT "grsec: denied auto-loading kernel module for a network device with CAP_SYS_MODULE (deprecated). Use CAP_NET_ADMIN and alias netdev-%.64s instead.", mod->name);
+ err = -EPERM;
+ goto free_modinfo;
+ } else if ((p = strstr(mod->args, "grsec_modharden_normal"))) {
+ p += sizeof("grsec_modharden_normal") - 1;
+ p2 = strstr(p, "_");
+ if (p2) {
+ *p2 = '\0';
+ printk(KERN_ALERT "grsec: denied kernel module auto-load of %.64s by uid %.9s\n", mod->name, p);
+ *p2 = '_';
+ }
+ err = -EPERM;
+ goto free_modinfo;
+ }
+ }
+#endif
+
/* Fix up syms, so that st_value is a pointer to location. */
err = simplify_symbols(mod, info);
if (err < 0)
flush_module_icache(mod);
- /* Now copy in args */
- mod->args = strndup_user(uargs, ~0UL >> 1);
- if (IS_ERR(mod->args)) {
- err = PTR_ERR(mod->args);
- goto free_arch_cleanup;
- }
-
dynamic_debug_setup(info->debug, info->num_debug);
/* Ftrace init must be called in the MODULE_STATE_UNFORMED state */
ddebug_cleanup:
dynamic_debug_remove(info->debug);
synchronize_sched();
- kfree(mod->args);
- free_arch_cleanup:
module_arch_cleanup(mod);
free_modinfo:
free_modinfo(mod);
+ kfree(mod->args);
free_unload:
module_unload_free(mod);
unlink_mod:
unsigned long nextval;
/* At worse, next value is at end of module */
- if (within_module_init(addr, mod))
- nextval = (unsigned long)mod->module_init+mod->init_text_size;
+ if (within_module_init_rx(addr, mod))
+ nextval = (unsigned long)mod->module_init_rx+mod->init_size_rx;
+ else if (within_module_init_rw(addr, mod))
+ nextval = (unsigned long)mod->module_init_rw+mod->init_size_rw;
+ else if (within_module_core_rx(addr, mod))
+ nextval = (unsigned long)mod->module_core_rx+mod->core_size_rx;
+ else if (within_module_core_rw(addr, mod))
+ nextval = (unsigned long)mod->module_core_rw+mod->core_size_rw;
else
- nextval = (unsigned long)mod->module_core+mod->core_text_size;
+ return NULL;
/* Scan for closest preceding symbol, and next symbol. (ELF
starts real symbols at 1). */
return 0;
seq_printf(m, "%s %u",
- mod->name, mod->init_size + mod->core_size);
+ mod->name, mod->init_size_rx + mod->init_size_rw + mod->core_size_rx + mod->core_size_rw);
print_unload_info(m, mod);
/* Informative for users. */
mod->state == MODULE_STATE_COMING ? "Loading" :
"Live");
/* Used by oprofile and other similar tools. */
- seq_printf(m, " 0x%pK", mod->module_core);
+ seq_printf(m, " 0x%pK 0x%pK", mod->module_core_rx, mod->module_core_rw);
/* Taints info */
if (mod->taints)
static int __init proc_modules_init(void)
{
+#ifndef CONFIG_GRKERNSEC_HIDESYM
+#ifdef CONFIG_GRKERNSEC_PROC_USER
+ proc_create("modules", S_IRUSR, NULL, &proc_modules_operations);
+#elif defined(CONFIG_GRKERNSEC_PROC_USERGROUP)
+ proc_create("modules", S_IRUSR | S_IRGRP, NULL, &proc_modules_operations);
+#else
proc_create("modules", 0, NULL, &proc_modules_operations);
+#endif
+#else
+ proc_create("modules", S_IRUSR, NULL, &proc_modules_operations);
+#endif
return 0;
}
module_init(proc_modules_init);
{
struct module *mod;
- if (addr < module_addr_min || addr > module_addr_max)
+ if ((addr < module_addr_min_rx || addr > module_addr_max_rx) &&
+ (addr < module_addr_min_rw || addr > module_addr_max_rw))
return NULL;
list_for_each_entry_rcu(mod, &modules, list) {
*/
struct module *__module_text_address(unsigned long addr)
{
- struct module *mod = __module_address(addr);
+ struct module *mod;
+
+#ifdef CONFIG_X86_32
+ addr = ktla_ktva(addr);
+#endif
+
+ if (addr < module_addr_min_rx || addr > module_addr_max_rx)
+ return NULL;
+
+ mod = __module_address(addr);
+
if (mod) {
/* Make sure it's within the text section. */
- if (!within(addr, mod->module_init, mod->init_text_size)
- && !within(addr, mod->module_core, mod->core_text_size))
+ if (!within_module_init_rx(addr, mod) && !within_module_core_rx(addr, mod))
mod = NULL;
}
return mod;
#include <linux/rcupdate.h>
#include <linux/vmalloc.h>
#include <linux/reboot.h>
+#include <linux/mm.h>
/*
* Notifier list for kernel code which wants to be called
while ((*nl) != NULL) {
if (n->priority > (*nl)->priority)
break;
- nl = &((*nl)->next);
+ nl = (struct notifier_block **)&((*nl)->next);
}
- n->next = *nl;
+ pax_open_kernel();
+ *(const void **)&n->next = *nl;
rcu_assign_pointer(*nl, n);
+ pax_close_kernel();
return 0;
}
return 0;
if (n->priority > (*nl)->priority)
break;
- nl = &((*nl)->next);
+ nl = (struct notifier_block **)&((*nl)->next);
}
- n->next = *nl;
+ pax_open_kernel();
+ *(const void **)&n->next = *nl;
rcu_assign_pointer(*nl, n);
+ pax_close_kernel();
return 0;
}
{
while ((*nl) != NULL) {
if ((*nl) == n) {
+ pax_open_kernel();
rcu_assign_pointer(*nl, n->next);
+ pax_close_kernel();
return 0;
}
- nl = &((*nl)->next);
+ nl = (struct notifier_block **)&((*nl)->next);
}
return -ENOENT;
}
* seq_nr mod. number of cpus in use.
*/
- seq_nr = atomic_inc_return(&pd->seq_nr);
+ seq_nr = atomic_inc_return_unchecked(&pd->seq_nr);
cpu_index = seq_nr % cpumask_weight(pd->cpumask.pcpu);
return padata_index_to_cpu(pd, cpu_index);
padata_init_pqueues(pd);
padata_init_squeues(pd);
setup_timer(&pd->timer, padata_reorder_timer, (unsigned long)pd);
- atomic_set(&pd->seq_nr, -1);
+ atomic_set_unchecked(&pd->seq_nr, -1);
atomic_set(&pd->reorder_objects, 0);
atomic_set(&pd->refcnt, 0);
pd->pinst = pinst;
/*
* Stop ourself in panic -- architecture code may override this
*/
-void __weak panic_smp_self_stop(void)
+void __weak __noreturn panic_smp_self_stop(void)
{
while (1)
cpu_relax();
disable_trace_on_warning();
pr_warn("------------[ cut here ]------------\n");
- pr_warn("WARNING: CPU: %d PID: %d at %s:%d %pS()\n",
+ pr_warn("WARNING: CPU: %d PID: %d at %s:%d %pA()\n",
raw_smp_processor_id(), current->pid, file, line, caller);
if (args)
*/
__visible void __stack_chk_fail(void)
{
- panic("stack-protector: Kernel stack is corrupted in: %p\n",
+ dump_stack();
+ panic("stack-protector: Kernel stack is corrupted in: %pA\n",
__builtin_return_address(0));
}
EXPORT_SYMBOL(__stack_chk_fail);
#include <linux/rculist.h>
#include <linux/bootmem.h>
#include <linux/hash.h>
+#include <linux/security.h>
#include <linux/pid_namespace.h>
#include <linux/init_task.h>
#include <linux/syscalls.h>
int pid_max = PID_MAX_DEFAULT;
-#define RESERVED_PIDS 300
+#define RESERVED_PIDS 500
int pid_max_min = RESERVED_PIDS + 1;
int pid_max_max = PID_MAX_LIMIT;
*/
struct task_struct *find_task_by_pid_ns(pid_t nr, struct pid_namespace *ns)
{
+ struct task_struct *task;
+
rcu_lockdep_assert(rcu_read_lock_held(),
"find_task_by_pid_ns() needs rcu_read_lock()"
" protection");
- return pid_task(find_pid_ns(nr, ns), PIDTYPE_PID);
+
+ task = pid_task(find_pid_ns(nr, ns), PIDTYPE_PID);
+
+ if (gr_pid_is_chrooted(task))
+ return NULL;
+
+ return task;
}
struct task_struct *find_task_by_vpid(pid_t vnr)
return find_task_by_pid_ns(vnr, task_active_pid_ns(current));
}
+struct task_struct *find_task_by_vpid_unrestricted(pid_t vnr)
+{
+ rcu_lockdep_assert(rcu_read_lock_held(),
+ "find_task_by_pid_ns() needs rcu_read_lock()"
+ " protection");
+ return pid_task(find_pid_ns(vnr, task_active_pid_ns(current)), PIDTYPE_PID);
+}
+
struct pid *get_task_pid(struct task_struct *task, enum pid_type type)
{
struct pid *pid;
void __user *buffer, size_t *lenp, loff_t *ppos)
{
struct pid_namespace *pid_ns = task_active_pid_ns(current);
- struct ctl_table tmp = *table;
+ ctl_table_no_const tmp = *table;
if (write && !ns_capable(pid_ns->user_ns, CAP_SYS_ADMIN))
return -EPERM;
config HIBERNATION
bool "Hibernation (aka 'suspend to disk')"
depends on SWAP && ARCH_HIBERNATION_POSSIBLE
+ depends on !GRKERNSEC_KMEM
+ depends on !PAX_MEMORY_SANITIZE
select HIBERNATE_CALLBACKS
select LZO_COMPRESS
select LZO_DECOMPRESS
unsigned int elapsed_msecs;
bool wakeup = false;
int sleep_usecs = USEC_PER_MSEC;
+ bool timedout = false;
do_gettimeofday(&start);
while (true) {
todo = 0;
+ if (time_after(jiffies, end_time))
+ timedout = true;
read_lock(&tasklist_lock);
for_each_process_thread(g, p) {
if (p == current || !freeze_task(p))
continue;
- if (!freezer_should_skip(p))
+ if (!freezer_should_skip(p)) {
todo++;
+ if (timedout) {
+ printk(KERN_ERR "Task refusing to freeze:\n");
+ sched_show_task(p);
+ }
+ }
}
read_unlock(&tasklist_lock);
todo += wq_busy;
}
- if (!todo || time_after(jiffies, end_time))
+ if (!todo || timedout)
break;
if (pm_wakeup_pending()) {
struct console_cmdline
{
- char name[8]; /* Name of the driver */
+ char name[16]; /* Name of the driver */
int index; /* Minor dev. to use */
char *options; /* Options for the driver */
#ifdef CONFIG_A11Y_BRAILLE_CONSOLE
if (from_file && type != SYSLOG_ACTION_OPEN)
return 0;
+#ifdef CONFIG_GRKERNSEC_DMESG
+ if (grsec_enable_dmesg && !capable(CAP_SYSLOG) && !capable_nolog(CAP_SYS_ADMIN))
+ return -EPERM;
+#endif
+
if (syslog_action_restricted(type)) {
if (capable(CAP_SYSLOG))
return 0;
for (i = 0, c = console_cmdline;
i < MAX_CMDLINECONSOLES && c->name[0];
i++, c++) {
+ BUILD_BUG_ON(sizeof(c->name) != sizeof(newcon->name));
if (strcmp(c->name, newcon->name) != 0)
continue;
if (newcon->index >= 0 &&
#define NR_PROFILE_HIT (PAGE_SIZE/sizeof(struct profile_hit))
#define NR_PROFILE_GRP (NR_PROFILE_HIT/PROFILE_GRPSZ)
-static atomic_t *prof_buffer;
+static atomic_unchecked_t *prof_buffer;
static unsigned long prof_len, prof_shift;
int prof_on __read_mostly;
hits[i].pc = 0;
continue;
}
- atomic_add(hits[i].hits, &prof_buffer[hits[i].pc]);
+ atomic_add_unchecked(hits[i].hits, &prof_buffer[hits[i].pc]);
hits[i].hits = hits[i].pc = 0;
}
}
* Add the current hit(s) and flush the write-queue out
* to the global buffer:
*/
- atomic_add(nr_hits, &prof_buffer[pc]);
+ atomic_add_unchecked(nr_hits, &prof_buffer[pc]);
for (i = 0; i < NR_PROFILE_HIT; ++i) {
- atomic_add(hits[i].hits, &prof_buffer[hits[i].pc]);
+ atomic_add_unchecked(hits[i].hits, &prof_buffer[hits[i].pc]);
hits[i].pc = hits[i].hits = 0;
}
out:
{
unsigned long pc;
pc = ((unsigned long)__pc - (unsigned long)_stext) >> prof_shift;
- atomic_add(nr_hits, &prof_buffer[min(pc, prof_len - 1)]);
+ atomic_add_unchecked(nr_hits, &prof_buffer[min(pc, prof_len - 1)]);
}
#endif /* !CONFIG_SMP */
return -EFAULT;
buf++; p++; count--; read++;
}
- pnt = (char *)prof_buffer + p - sizeof(atomic_t);
+ pnt = (char *)prof_buffer + p - sizeof(atomic_unchecked_t);
if (copy_to_user(buf, (void *)pnt, count))
return -EFAULT;
read += count;
}
#endif
profile_discard_flip_buffers();
- memset(prof_buffer, 0, prof_len * sizeof(atomic_t));
+ memset(prof_buffer, 0, prof_len * sizeof(atomic_unchecked_t));
return count;
}
if (seize)
flags |= PT_SEIZED;
rcu_read_lock();
- if (ns_capable(__task_cred(task)->user_ns, CAP_SYS_PTRACE))
+ if (ns_capable_nolog(__task_cred(task)->user_ns, CAP_SYS_PTRACE))
flags |= PT_PTRACE_CAP;
rcu_read_unlock();
task->ptrace = flags;
break;
return -EIO;
}
- if (copy_to_user(dst, buf, retval))
+ if (retval > sizeof(buf) || copy_to_user(dst, buf, retval))
return -EFAULT;
copied += retval;
src += retval;
bool seized = child->ptrace & PT_SEIZED;
int ret = -EIO;
siginfo_t siginfo, *si;
- void __user *datavp = (void __user *) data;
+ void __user *datavp = (__force void __user *) data;
unsigned long __user *datalp = datavp;
unsigned long flags;
goto out;
}
+ if (gr_handle_ptrace(child, request)) {
+ ret = -EPERM;
+ goto out_put_task_struct;
+ }
+
if (request == PTRACE_ATTACH || request == PTRACE_SEIZE) {
ret = ptrace_attach(child, request, addr, data);
/*
* Some architectures need to do book-keeping after
* a ptrace attach.
*/
- if (!ret)
+ if (!ret) {
arch_ptrace_attach(child);
+ gr_audit_ptrace(child);
+ }
goto out_put_task_struct;
}
copied = access_process_vm(tsk, addr, &tmp, sizeof(tmp), 0);
if (copied != sizeof(tmp))
return -EIO;
- return put_user(tmp, (unsigned long __user *)data);
+ return put_user(tmp, (__force unsigned long __user *)data);
}
int generic_ptrace_pokedata(struct task_struct *tsk, unsigned long addr,
}
COMPAT_SYSCALL_DEFINE4(ptrace, compat_long_t, request, compat_long_t, pid,
- compat_long_t, addr, compat_long_t, data)
+ compat_ulong_t, addr, compat_ulong_t, data)
{
struct task_struct *child;
long ret;
goto out;
}
+ if (gr_handle_ptrace(child, request)) {
+ ret = -EPERM;
+ goto out_put_task_struct;
+ }
+
if (request == PTRACE_ATTACH || request == PTRACE_SEIZE) {
ret = ptrace_attach(child, request, addr, data);
/*
* Some architectures need to do book-keeping after
* a ptrace attach.
*/
- if (!ret)
+ if (!ret) {
arch_ptrace_attach(child);
+ gr_audit_ptrace(child);
+ }
goto out_put_task_struct;
}
rcu_torture_count) = { 0 };
static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1],
rcu_torture_batch) = { 0 };
-static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1];
-static atomic_t n_rcu_torture_alloc;
-static atomic_t n_rcu_torture_alloc_fail;
-static atomic_t n_rcu_torture_free;
-static atomic_t n_rcu_torture_mberror;
-static atomic_t n_rcu_torture_error;
+static atomic_unchecked_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1];
+static atomic_unchecked_t n_rcu_torture_alloc;
+static atomic_unchecked_t n_rcu_torture_alloc_fail;
+static atomic_unchecked_t n_rcu_torture_free;
+static atomic_unchecked_t n_rcu_torture_mberror;
+static atomic_unchecked_t n_rcu_torture_error;
static long n_rcu_torture_barrier_error;
static long n_rcu_torture_boost_ktrerror;
static long n_rcu_torture_boost_rterror;
static long n_rcu_torture_timers;
static long n_barrier_attempts;
static long n_barrier_successes;
-static atomic_long_t n_cbfloods;
+static atomic_long_unchecked_t n_cbfloods;
static struct list_head rcu_torture_removed;
static int rcu_torture_writer_state;
spin_lock_bh(&rcu_torture_lock);
if (list_empty(&rcu_torture_freelist)) {
- atomic_inc(&n_rcu_torture_alloc_fail);
+ atomic_inc_unchecked(&n_rcu_torture_alloc_fail);
spin_unlock_bh(&rcu_torture_lock);
return NULL;
}
- atomic_inc(&n_rcu_torture_alloc);
+ atomic_inc_unchecked(&n_rcu_torture_alloc);
p = rcu_torture_freelist.next;
list_del_init(p);
spin_unlock_bh(&rcu_torture_lock);
static void
rcu_torture_free(struct rcu_torture *p)
{
- atomic_inc(&n_rcu_torture_free);
+ atomic_inc_unchecked(&n_rcu_torture_free);
spin_lock_bh(&rcu_torture_lock);
list_add_tail(&p->rtort_free, &rcu_torture_freelist);
spin_unlock_bh(&rcu_torture_lock);
i = rp->rtort_pipe_count;
if (i > RCU_TORTURE_PIPE_LEN)
i = RCU_TORTURE_PIPE_LEN;
- atomic_inc(&rcu_torture_wcount[i]);
+ atomic_inc_unchecked(&rcu_torture_wcount[i]);
if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
rp->rtort_mbtest = 0;
return true;
VERBOSE_TOROUT_STRING("rcu_torture_cbflood task started");
do {
schedule_timeout_interruptible(cbflood_inter_holdoff);
- atomic_long_inc(&n_cbfloods);
+ atomic_long_inc_unchecked(&n_cbfloods);
WARN_ON(signal_pending(current));
for (i = 0; i < cbflood_n_burst; i++) {
for (j = 0; j < cbflood_n_per_burst; j++) {
i = old_rp->rtort_pipe_count;
if (i > RCU_TORTURE_PIPE_LEN)
i = RCU_TORTURE_PIPE_LEN;
- atomic_inc(&rcu_torture_wcount[i]);
+ atomic_inc_unchecked(&rcu_torture_wcount[i]);
old_rp->rtort_pipe_count++;
switch (synctype[torture_random(&rand) % nsynctypes]) {
case RTWS_DEF_FREE:
return;
}
if (p->rtort_mbtest == 0)
- atomic_inc(&n_rcu_torture_mberror);
+ atomic_inc_unchecked(&n_rcu_torture_mberror);
spin_lock(&rand_lock);
cur_ops->read_delay(&rand);
n_rcu_torture_timers++;
continue;
}
if (p->rtort_mbtest == 0)
- atomic_inc(&n_rcu_torture_mberror);
+ atomic_inc_unchecked(&n_rcu_torture_mberror);
cur_ops->read_delay(&rand);
preempt_disable();
pipe_count = p->rtort_pipe_count;
rcu_torture_current,
rcu_torture_current_version,
list_empty(&rcu_torture_freelist),
- atomic_read(&n_rcu_torture_alloc),
- atomic_read(&n_rcu_torture_alloc_fail),
- atomic_read(&n_rcu_torture_free));
+ atomic_read_unchecked(&n_rcu_torture_alloc),
+ atomic_read_unchecked(&n_rcu_torture_alloc_fail),
+ atomic_read_unchecked(&n_rcu_torture_free));
pr_cont("rtmbe: %d rtbke: %ld rtbre: %ld ",
- atomic_read(&n_rcu_torture_mberror),
+ atomic_read_unchecked(&n_rcu_torture_mberror),
n_rcu_torture_boost_ktrerror,
n_rcu_torture_boost_rterror);
pr_cont("rtbf: %ld rtb: %ld nt: %ld ",
n_barrier_successes,
n_barrier_attempts,
n_rcu_torture_barrier_error);
- pr_cont("cbflood: %ld\n", atomic_long_read(&n_cbfloods));
+ pr_cont("cbflood: %ld\n", atomic_long_read_unchecked(&n_cbfloods));
pr_alert("%s%s ", torture_type, TORTURE_FLAG);
- if (atomic_read(&n_rcu_torture_mberror) != 0 ||
+ if (atomic_read_unchecked(&n_rcu_torture_mberror) != 0 ||
n_rcu_torture_barrier_error != 0 ||
n_rcu_torture_boost_ktrerror != 0 ||
n_rcu_torture_boost_rterror != 0 ||
n_rcu_torture_boost_failure != 0 ||
i > 1) {
pr_cont("%s", "!!! ");
- atomic_inc(&n_rcu_torture_error);
+ atomic_inc_unchecked(&n_rcu_torture_error);
WARN_ON_ONCE(1);
}
pr_cont("Reader Pipe: ");
pr_alert("%s%s ", torture_type, TORTURE_FLAG);
pr_cont("Free-Block Circulation: ");
for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
- pr_cont(" %d", atomic_read(&rcu_torture_wcount[i]));
+ pr_cont(" %d", atomic_read_unchecked(&rcu_torture_wcount[i]));
}
pr_cont("\n");
rcu_torture_stats_print(); /* -After- the stats thread is stopped! */
- if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error)
+ if (atomic_read_unchecked(&n_rcu_torture_error) || n_rcu_torture_barrier_error)
rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE");
else if (torture_onoff_failures())
rcu_torture_print_module_parms(cur_ops,
rcu_torture_current = NULL;
rcu_torture_current_version = 0;
- atomic_set(&n_rcu_torture_alloc, 0);
- atomic_set(&n_rcu_torture_alloc_fail, 0);
- atomic_set(&n_rcu_torture_free, 0);
- atomic_set(&n_rcu_torture_mberror, 0);
- atomic_set(&n_rcu_torture_error, 0);
+ atomic_set_unchecked(&n_rcu_torture_alloc, 0);
+ atomic_set_unchecked(&n_rcu_torture_alloc_fail, 0);
+ atomic_set_unchecked(&n_rcu_torture_free, 0);
+ atomic_set_unchecked(&n_rcu_torture_mberror, 0);
+ atomic_set_unchecked(&n_rcu_torture_error, 0);
n_rcu_torture_barrier_error = 0;
n_rcu_torture_boost_ktrerror = 0;
n_rcu_torture_boost_rterror = 0;
n_rcu_torture_boost_failure = 0;
n_rcu_torture_boosts = 0;
for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
- atomic_set(&rcu_torture_wcount[i], 0);
+ atomic_set_unchecked(&rcu_torture_wcount[i], 0);
for_each_possible_cpu(cpu) {
for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
per_cpu(rcu_torture_count, cpu)[i] = 0;
/* Forward declarations for tiny_plugin.h. */
struct rcu_ctrlblk;
static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp);
-static void rcu_process_callbacks(struct softirq_action *unused);
+static void rcu_process_callbacks(void);
static void __call_rcu(struct rcu_head *head,
void (*func)(struct rcu_head *rcu),
struct rcu_ctrlblk *rcp);
false));
}
-static void rcu_process_callbacks(struct softirq_action *unused)
+static __latent_entropy void rcu_process_callbacks(void)
{
__rcu_process_callbacks(&rcu_sched_ctrlblk);
__rcu_process_callbacks(&rcu_bh_ctrlblk);
dump_stack();
}
if (*rcp->curtail && ULONG_CMP_GE(j, js))
- ACCESS_ONCE(rcp->jiffies_stall) = jiffies +
+ ACCESS_ONCE_RW(rcp->jiffies_stall) = jiffies +
3 * rcu_jiffies_till_stall_check() + 3;
else if (ULONG_CMP_GE(j, js))
- ACCESS_ONCE(rcp->jiffies_stall) = jiffies + rcu_jiffies_till_stall_check();
+ ACCESS_ONCE_RW(rcp->jiffies_stall) = jiffies + rcu_jiffies_till_stall_check();
}
static void reset_cpu_stall_ticks(struct rcu_ctrlblk *rcp)
{
rcp->ticks_this_gp = 0;
rcp->gp_start = jiffies;
- ACCESS_ONCE(rcp->jiffies_stall) = jiffies + rcu_jiffies_till_stall_check();
+ ACCESS_ONCE_RW(rcp->jiffies_stall) = jiffies + rcu_jiffies_till_stall_check();
}
static void check_cpu_stalls(void)
*/
rdtp = this_cpu_ptr(&rcu_dynticks);
smp_mb__before_atomic(); /* Earlier stuff before QS. */
- atomic_add(2, &rdtp->dynticks); /* QS. */
+ atomic_add_unchecked(2, &rdtp->dynticks); /* QS. */
smp_mb__after_atomic(); /* Later stuff after QS. */
break;
}
rcu_prepare_for_idle();
/* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
smp_mb__before_atomic(); /* See above. */
- atomic_inc(&rdtp->dynticks);
+ atomic_inc_unchecked(&rdtp->dynticks);
smp_mb__after_atomic(); /* Force ordering with next sojourn. */
- WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);
+ WARN_ON_ONCE(atomic_read_unchecked(&rdtp->dynticks) & 0x1);
rcu_dynticks_task_enter();
/*
rcu_dynticks_task_exit();
smp_mb__before_atomic(); /* Force ordering w/previous sojourn. */
- atomic_inc(&rdtp->dynticks);
+ atomic_inc_unchecked(&rdtp->dynticks);
/* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
smp_mb__after_atomic(); /* See above. */
- WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
+ WARN_ON_ONCE(!(atomic_read_unchecked(&rdtp->dynticks) & 0x1));
rcu_cleanup_after_idle();
trace_rcu_dyntick(TPS("End"), oldval, rdtp->dynticks_nesting);
if (!user && !is_idle_task(current)) {
struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
if (rdtp->dynticks_nmi_nesting == 0 &&
- (atomic_read(&rdtp->dynticks) & 0x1))
+ (atomic_read_unchecked(&rdtp->dynticks) & 0x1))
return;
rdtp->dynticks_nmi_nesting++;
smp_mb__before_atomic(); /* Force delay from prior write. */
- atomic_inc(&rdtp->dynticks);
+ atomic_inc_unchecked(&rdtp->dynticks);
/* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
smp_mb__after_atomic(); /* See above. */
- WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
+ WARN_ON_ONCE(!(atomic_read_unchecked(&rdtp->dynticks) & 0x1));
}
/**
return;
/* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
smp_mb__before_atomic(); /* See above. */
- atomic_inc(&rdtp->dynticks);
+ atomic_inc_unchecked(&rdtp->dynticks);
smp_mb__after_atomic(); /* Force delay to next write. */
- WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);
+ WARN_ON_ONCE(atomic_read_unchecked(&rdtp->dynticks) & 0x1);
}
/**
*/
bool notrace __rcu_is_watching(void)
{
- return atomic_read(this_cpu_ptr(&rcu_dynticks.dynticks)) & 0x1;
+ return atomic_read_unchecked(this_cpu_ptr(&rcu_dynticks.dynticks)) & 0x1;
}
/**
static int dyntick_save_progress_counter(struct rcu_data *rdp,
bool *isidle, unsigned long *maxj)
{
- rdp->dynticks_snap = atomic_add_return(0, &rdp->dynticks->dynticks);
+ rdp->dynticks_snap = atomic_add_return_unchecked(0, &rdp->dynticks->dynticks);
rcu_sysidle_check_cpu(rdp, isidle, maxj);
if ((rdp->dynticks_snap & 0x1) == 0) {
trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("dti"));
int *rcrmp;
unsigned int snap;
- curr = (unsigned int)atomic_add_return(0, &rdp->dynticks->dynticks);
+ curr = (unsigned int)atomic_add_return_unchecked(0, &rdp->dynticks->dynticks);
snap = (unsigned int)rdp->dynticks_snap;
/*
rdp->rsp->gp_start + jiffies_till_sched_qs) ||
ULONG_CMP_GE(jiffies, rdp->rsp->jiffies_resched)) {
if (!(ACCESS_ONCE(*rcrmp) & rdp->rsp->flavor_mask)) {
- ACCESS_ONCE(rdp->cond_resched_completed) =
+ ACCESS_ONCE_RW(rdp->cond_resched_completed) =
ACCESS_ONCE(rdp->mynode->completed);
smp_mb(); /* ->cond_resched_completed before *rcrmp. */
- ACCESS_ONCE(*rcrmp) =
+ ACCESS_ONCE_RW(*rcrmp) =
ACCESS_ONCE(*rcrmp) + rdp->rsp->flavor_mask;
resched_cpu(rdp->cpu); /* Force CPU into scheduler. */
rdp->rsp->jiffies_resched += 5; /* Enable beating. */
rsp->gp_start = j;
smp_wmb(); /* Record start time before stall time. */
j1 = rcu_jiffies_till_stall_check();
- ACCESS_ONCE(rsp->jiffies_stall) = j + j1;
+ ACCESS_ONCE_RW(rsp->jiffies_stall) = j + j1;
rsp->jiffies_resched = j + j1 / 2;
}
raw_spin_unlock_irqrestore(&rnp->lock, flags);
return;
}
- ACCESS_ONCE(rsp->jiffies_stall) = jiffies + 3 * rcu_jiffies_till_stall_check() + 3;
+ ACCESS_ONCE_RW(rsp->jiffies_stall) = jiffies + 3 * rcu_jiffies_till_stall_check() + 3;
raw_spin_unlock_irqrestore(&rnp->lock, flags);
/*
raw_spin_lock_irqsave(&rnp->lock, flags);
if (ULONG_CMP_GE(jiffies, ACCESS_ONCE(rsp->jiffies_stall)))
- ACCESS_ONCE(rsp->jiffies_stall) = jiffies +
+ ACCESS_ONCE_RW(rsp->jiffies_stall) = jiffies +
3 * rcu_jiffies_till_stall_check() + 3;
raw_spin_unlock_irqrestore(&rnp->lock, flags);
struct rcu_state *rsp;
for_each_rcu_flavor(rsp)
- ACCESS_ONCE(rsp->jiffies_stall) = jiffies + ULONG_MAX / 2;
+ ACCESS_ONCE_RW(rsp->jiffies_stall) = jiffies + ULONG_MAX / 2;
}
/*
raw_spin_unlock_irq(&rnp->lock);
return 0;
}
- ACCESS_ONCE(rsp->gp_flags) = 0; /* Clear all flags: New grace period. */
+ ACCESS_ONCE_RW(rsp->gp_flags) = 0; /* Clear all flags: New grace period. */
if (WARN_ON_ONCE(rcu_gp_in_progress(rsp))) {
/*
rdp = this_cpu_ptr(rsp->rda);
rcu_preempt_check_blocked_tasks(rnp);
rnp->qsmask = rnp->qsmaskinit;
- ACCESS_ONCE(rnp->gpnum) = rsp->gpnum;
+ ACCESS_ONCE_RW(rnp->gpnum) = rsp->gpnum;
WARN_ON_ONCE(rnp->completed != rsp->completed);
- ACCESS_ONCE(rnp->completed) = rsp->completed;
+ ACCESS_ONCE_RW(rnp->completed) = rsp->completed;
if (rnp == rdp->mynode)
(void)__note_gp_changes(rsp, rnp, rdp);
rcu_preempt_boost_start_gp(rnp);
if (ACCESS_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) {
raw_spin_lock_irq(&rnp->lock);
smp_mb__after_unlock_lock();
- ACCESS_ONCE(rsp->gp_flags) =
+ ACCESS_ONCE_RW(rsp->gp_flags) =
ACCESS_ONCE(rsp->gp_flags) & ~RCU_GP_FLAG_FQS;
raw_spin_unlock_irq(&rnp->lock);
}
rcu_for_each_node_breadth_first(rsp, rnp) {
raw_spin_lock_irq(&rnp->lock);
smp_mb__after_unlock_lock();
- ACCESS_ONCE(rnp->completed) = rsp->gpnum;
+ ACCESS_ONCE_RW(rnp->completed) = rsp->gpnum;
rdp = this_cpu_ptr(rsp->rda);
if (rnp == rdp->mynode)
needgp = __note_gp_changes(rsp, rnp, rdp) || needgp;
rcu_nocb_gp_set(rnp, nocb);
/* Declare grace period done. */
- ACCESS_ONCE(rsp->completed) = rsp->gpnum;
+ ACCESS_ONCE_RW(rsp->completed) = rsp->gpnum;
trace_rcu_grace_period(rsp->name, rsp->completed, TPS("end"));
rsp->fqs_state = RCU_GP_IDLE;
rdp = this_cpu_ptr(rsp->rda);
/* Advance CBs to reduce false positives below. */
needgp = rcu_advance_cbs(rsp, rnp, rdp) || needgp;
if (needgp || cpu_needs_another_gp(rsp, rdp)) {
- ACCESS_ONCE(rsp->gp_flags) = RCU_GP_FLAG_INIT;
+ ACCESS_ONCE_RW(rsp->gp_flags) = RCU_GP_FLAG_INIT;
trace_rcu_grace_period(rsp->name,
ACCESS_ONCE(rsp->gpnum),
TPS("newreq"));
*/
return false;
}
- ACCESS_ONCE(rsp->gp_flags) = RCU_GP_FLAG_INIT;
+ ACCESS_ONCE_RW(rsp->gp_flags) = RCU_GP_FLAG_INIT;
trace_rcu_grace_period(rsp->name, ACCESS_ONCE(rsp->gpnum),
TPS("newreq"));
rsp->qlen += rdp->qlen;
rdp->n_cbs_orphaned += rdp->qlen;
rdp->qlen_lazy = 0;
- ACCESS_ONCE(rdp->qlen) = 0;
+ ACCESS_ONCE_RW(rdp->qlen) = 0;
}
/*
}
smp_mb(); /* List handling before counting for rcu_barrier(). */
rdp->qlen_lazy -= count_lazy;
- ACCESS_ONCE(rdp->qlen) = rdp->qlen - count;
+ ACCESS_ONCE_RW(rdp->qlen) = rdp->qlen - count;
rdp->n_cbs_invoked += count;
/* Reinstate batch limit if we have worked down the excess. */
raw_spin_unlock_irqrestore(&rnp_old->lock, flags);
return; /* Someone beat us to it. */
}
- ACCESS_ONCE(rsp->gp_flags) =
+ ACCESS_ONCE_RW(rsp->gp_flags) =
ACCESS_ONCE(rsp->gp_flags) | RCU_GP_FLAG_FQS;
raw_spin_unlock_irqrestore(&rnp_old->lock, flags);
rcu_gp_kthread_wake(rsp);
/*
* Do RCU core processing for the current CPU.
*/
-static void rcu_process_callbacks(struct softirq_action *unused)
+static void rcu_process_callbacks(void)
{
struct rcu_state *rsp;
WARN_ON_ONCE((unsigned long)head & 0x1); /* Misaligned rcu_head! */
if (debug_rcu_head_queue(head)) {
/* Probable double call_rcu(), so leak the callback. */
- ACCESS_ONCE(head->func) = rcu_leak_callback;
+ ACCESS_ONCE_RW(head->func) = rcu_leak_callback;
WARN_ONCE(1, "__call_rcu(): Leaked duplicate callback\n");
return;
}
local_irq_restore(flags);
return;
}
- ACCESS_ONCE(rdp->qlen) = rdp->qlen + 1;
+ ACCESS_ONCE_RW(rdp->qlen) = rdp->qlen + 1;
if (lazy)
rdp->qlen_lazy++;
else
* counter wrap on a 32-bit system. Quite a few more CPUs would of
* course be required on a 64-bit system.
*/
- if (ULONG_CMP_GE((ulong)atomic_long_read(&rsp->expedited_start),
+ if (ULONG_CMP_GE((ulong)atomic_long_read_unchecked(&rsp->expedited_start),
(ulong)atomic_long_read(&rsp->expedited_done) +
ULONG_MAX / 8)) {
synchronize_sched();
- atomic_long_inc(&rsp->expedited_wrap);
+ atomic_long_inc_unchecked(&rsp->expedited_wrap);
return;
}
* Take a ticket. Note that atomic_inc_return() implies a
* full memory barrier.
*/
- snap = atomic_long_inc_return(&rsp->expedited_start);
+ snap = atomic_long_inc_return_unchecked(&rsp->expedited_start);
firstsnap = snap;
if (!try_get_online_cpus()) {
/* CPU hotplug operation in flight, fall back to normal GP. */
wait_rcu_gp(call_rcu_sched);
- atomic_long_inc(&rsp->expedited_normal);
+ atomic_long_inc_unchecked(&rsp->expedited_normal);
return;
}
WARN_ON_ONCE(cpu_is_offline(raw_smp_processor_id()));
for_each_cpu(cpu, cm) {
struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
- if (!(atomic_add_return(0, &rdtp->dynticks) & 0x1))
+ if (!(atomic_add_return_unchecked(0, &rdtp->dynticks) & 0x1))
cpumask_clear_cpu(cpu, cm);
}
if (cpumask_weight(cm) == 0)
synchronize_sched_expedited_cpu_stop,
NULL) == -EAGAIN) {
put_online_cpus();
- atomic_long_inc(&rsp->expedited_tryfail);
+ atomic_long_inc_unchecked(&rsp->expedited_tryfail);
/* Check to see if someone else did our work for us. */
s = atomic_long_read(&rsp->expedited_done);
if (ULONG_CMP_GE((ulong)s, (ulong)firstsnap)) {
/* ensure test happens before caller kfree */
smp_mb__before_atomic(); /* ^^^ */
- atomic_long_inc(&rsp->expedited_workdone1);
+ atomic_long_inc_unchecked(&rsp->expedited_workdone1);
free_cpumask_var(cm);
return;
}
udelay(trycount * num_online_cpus());
} else {
wait_rcu_gp(call_rcu_sched);
- atomic_long_inc(&rsp->expedited_normal);
+ atomic_long_inc_unchecked(&rsp->expedited_normal);
free_cpumask_var(cm);
return;
}
if (ULONG_CMP_GE((ulong)s, (ulong)firstsnap)) {
/* ensure test happens before caller kfree */
smp_mb__before_atomic(); /* ^^^ */
- atomic_long_inc(&rsp->expedited_workdone2);
+ atomic_long_inc_unchecked(&rsp->expedited_workdone2);
free_cpumask_var(cm);
return;
}
if (!try_get_online_cpus()) {
/* CPU hotplug operation in flight, use normal GP. */
wait_rcu_gp(call_rcu_sched);
- atomic_long_inc(&rsp->expedited_normal);
+ atomic_long_inc_unchecked(&rsp->expedited_normal);
free_cpumask_var(cm);
return;
}
- snap = atomic_long_read(&rsp->expedited_start);
+ snap = atomic_long_read_unchecked(&rsp->expedited_start);
smp_mb(); /* ensure read is before try_stop_cpus(). */
}
- atomic_long_inc(&rsp->expedited_stoppedcpus);
+ atomic_long_inc_unchecked(&rsp->expedited_stoppedcpus);
all_cpus_idle:
free_cpumask_var(cm);
* than we did already did their update.
*/
do {
- atomic_long_inc(&rsp->expedited_done_tries);
+ atomic_long_inc_unchecked(&rsp->expedited_done_tries);
s = atomic_long_read(&rsp->expedited_done);
if (ULONG_CMP_GE((ulong)s, (ulong)snap)) {
/* ensure test happens before caller kfree */
smp_mb__before_atomic(); /* ^^^ */
- atomic_long_inc(&rsp->expedited_done_lost);
+ atomic_long_inc_unchecked(&rsp->expedited_done_lost);
break;
}
} while (atomic_long_cmpxchg(&rsp->expedited_done, s, snap) != s);
- atomic_long_inc(&rsp->expedited_done_exit);
+ atomic_long_inc_unchecked(&rsp->expedited_done_exit);
put_online_cpus();
}
* ACCESS_ONCE() to prevent the compiler from speculating
* the increment to precede the early-exit check.
*/
- ACCESS_ONCE(rsp->n_barrier_done) = rsp->n_barrier_done + 1;
+ ACCESS_ONCE_RW(rsp->n_barrier_done) = rsp->n_barrier_done + 1;
WARN_ON_ONCE((rsp->n_barrier_done & 0x1) != 1);
_rcu_barrier_trace(rsp, "Inc1", -1, rsp->n_barrier_done);
smp_mb(); /* Order ->n_barrier_done increment with below mechanism. */
/* Increment ->n_barrier_done to prevent duplicate work. */
smp_mb(); /* Keep increment after above mechanism. */
- ACCESS_ONCE(rsp->n_barrier_done) = rsp->n_barrier_done + 1;
+ ACCESS_ONCE_RW(rsp->n_barrier_done) = rsp->n_barrier_done + 1;
WARN_ON_ONCE((rsp->n_barrier_done & 0x1) != 0);
_rcu_barrier_trace(rsp, "Inc2", -1, rsp->n_barrier_done);
smp_mb(); /* Keep increment before caller's subsequent code. */
rdp->grpmask = 1UL << (cpu - rdp->mynode->grplo);
init_callback_list(rdp);
rdp->qlen_lazy = 0;
- ACCESS_ONCE(rdp->qlen) = 0;
+ ACCESS_ONCE_RW(rdp->qlen) = 0;
rdp->dynticks = &per_cpu(rcu_dynticks, cpu);
WARN_ON_ONCE(rdp->dynticks->dynticks_nesting != DYNTICK_TASK_EXIT_IDLE);
- WARN_ON_ONCE(atomic_read(&rdp->dynticks->dynticks) != 1);
+ WARN_ON_ONCE(atomic_read_unchecked(&rdp->dynticks->dynticks) != 1);
rdp->cpu = cpu;
rdp->rsp = rsp;
rcu_boot_init_nocb_percpu_data(rdp);
init_callback_list(rdp); /* Re-enable callbacks on this CPU. */
rdp->dynticks->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;
rcu_sysidle_init_percpu_data(rdp->dynticks);
- atomic_set(&rdp->dynticks->dynticks,
- (atomic_read(&rdp->dynticks->dynticks) & ~0x1) + 1);
+ atomic_set_unchecked(&rdp->dynticks->dynticks,
+ (atomic_read_unchecked(&rdp->dynticks->dynticks) & ~0x1) + 1);
raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
/* Add CPU to rcu_node bitmasks. */
long long dynticks_nesting; /* Track irq/process nesting level. */
/* Process level is worth LLONG_MAX/2. */
int dynticks_nmi_nesting; /* Track NMI nesting level. */
- atomic_t dynticks; /* Even value for idle, else odd. */
+ atomic_unchecked_t dynticks;/* Even value for idle, else odd. */
#ifdef CONFIG_NO_HZ_FULL_SYSIDLE
long long dynticks_idle_nesting;
/* irq/process nesting level from idle. */
- atomic_t dynticks_idle; /* Even value for idle, else odd. */
+ atomic_unchecked_t dynticks_idle;/* Even value for idle, else odd. */
/* "Idle" excludes userspace execution. */
unsigned long dynticks_idle_jiffies;
/* End of last non-NMI non-idle period. */
/* _rcu_barrier(). */
/* End of fields guarded by barrier_mutex. */
- atomic_long_t expedited_start; /* Starting ticket. */
- atomic_long_t expedited_done; /* Done ticket. */
- atomic_long_t expedited_wrap; /* # near-wrap incidents. */
- atomic_long_t expedited_tryfail; /* # acquisition failures. */
- atomic_long_t expedited_workdone1; /* # done by others #1. */
- atomic_long_t expedited_workdone2; /* # done by others #2. */
- atomic_long_t expedited_normal; /* # fallbacks to normal. */
- atomic_long_t expedited_stoppedcpus; /* # successful stop_cpus. */
- atomic_long_t expedited_done_tries; /* # tries to update _done. */
- atomic_long_t expedited_done_lost; /* # times beaten to _done. */
- atomic_long_t expedited_done_exit; /* # times exited _done loop. */
+ atomic_long_unchecked_t expedited_start; /* Starting ticket. */
+ atomic_long_t expedited_done; /* Done ticket. */
+ atomic_long_unchecked_t expedited_wrap; /* # near-wrap incidents. */
+ atomic_long_unchecked_t expedited_tryfail; /* # acquisition failures. */
+ atomic_long_unchecked_t expedited_workdone1; /* # done by others #1. */
+ atomic_long_unchecked_t expedited_workdone2; /* # done by others #2. */
+ atomic_long_unchecked_t expedited_normal; /* # fallbacks to normal. */
+ atomic_long_unchecked_t expedited_stoppedcpus; /* # successful stop_cpus. */
+ atomic_long_unchecked_t expedited_done_tries; /* # tries to update _done. */
+ atomic_long_unchecked_t expedited_done_lost; /* # times beaten to _done. */
+ atomic_long_unchecked_t expedited_done_exit; /* # times exited _done loop. */
unsigned long jiffies_force_qs; /* Time at which to invoke */
/* force_quiescent_state(). */
static int sync_rcu_preempt_exp_done(struct rcu_node *rnp)
{
return !rcu_preempted_readers_exp(rnp) &&
- ACCESS_ONCE(rnp->expmask) == 0;
+ ACCESS_ONCE_RW(rnp->expmask) == 0;
}
/*
/* Clean up and exit. */
smp_mb(); /* ensure expedited GP seen before counter increment. */
- ACCESS_ONCE(sync_rcu_preempt_exp_count) =
+ ACCESS_ONCE_RW(sync_rcu_preempt_exp_count) =
sync_rcu_preempt_exp_count + 1;
unlock_mb_ret:
mutex_unlock(&sync_rcu_preempt_exp_mutex);
free_cpumask_var(cm);
}
-static struct smp_hotplug_thread rcu_cpu_thread_spec = {
+static struct smp_hotplug_thread rcu_cpu_thread_spec __read_only = {
.store = &rcu_cpu_kthread_task,
.thread_should_run = rcu_cpu_kthread_should_run,
.thread_fn = rcu_cpu_kthread,
print_cpu_stall_fast_no_hz(fast_no_hz, cpu);
pr_err("\t%d: (%lu %s) idle=%03x/%llx/%d softirq=%u/%u %s\n",
cpu, ticks_value, ticks_title,
- atomic_read(&rdtp->dynticks) & 0xfff,
+ atomic_read_unchecked(&rdtp->dynticks) & 0xfff,
rdtp->dynticks_nesting, rdtp->dynticks_nmi_nesting,
rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu),
fast_no_hz);
return;
if (ACCESS_ONCE(rdp_leader->nocb_leader_sleep) || force) {
/* Prior smp_mb__after_atomic() orders against prior enqueue. */
- ACCESS_ONCE(rdp_leader->nocb_leader_sleep) = false;
+ ACCESS_ONCE_RW(rdp_leader->nocb_leader_sleep) = false;
wake_up(&rdp_leader->nocb_wq);
}
}
/* Enqueue the callback on the nocb list and update counts. */
old_rhpp = xchg(&rdp->nocb_tail, rhtp);
- ACCESS_ONCE(*old_rhpp) = rhp;
+ ACCESS_ONCE_RW(*old_rhpp) = rhp;
atomic_long_add(rhcount, &rdp->nocb_q_count);
atomic_long_add(rhcount_lazy, &rdp->nocb_q_count_lazy);
smp_mb__after_atomic(); /* Store *old_rhpp before _wake test. */
continue; /* No CBs here, try next follower. */
/* Move callbacks to wait-for-GP list, which is empty. */
- ACCESS_ONCE(rdp->nocb_head) = NULL;
+ ACCESS_ONCE_RW(rdp->nocb_head) = NULL;
rdp->nocb_gp_tail = xchg(&rdp->nocb_tail, &rdp->nocb_head);
rdp->nocb_gp_count = atomic_long_xchg(&rdp->nocb_q_count, 0);
rdp->nocb_gp_count_lazy =
list = ACCESS_ONCE(rdp->nocb_follower_head);
BUG_ON(!list);
trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, "WokeNonEmpty");
- ACCESS_ONCE(rdp->nocb_follower_head) = NULL;
+ ACCESS_ONCE_RW(rdp->nocb_follower_head) = NULL;
tail = xchg(&rdp->nocb_follower_tail, &rdp->nocb_follower_head);
c = atomic_long_xchg(&rdp->nocb_follower_count, 0);
cl = atomic_long_xchg(&rdp->nocb_follower_count_lazy, 0);
list = next;
}
trace_rcu_batch_end(rdp->rsp->name, c, !!list, 0, 0, 1);
- ACCESS_ONCE(rdp->nocb_p_count) = rdp->nocb_p_count - c;
- ACCESS_ONCE(rdp->nocb_p_count_lazy) =
+ ACCESS_ONCE_RW(rdp->nocb_p_count) = rdp->nocb_p_count - c;
+ ACCESS_ONCE_RW(rdp->nocb_p_count_lazy) =
rdp->nocb_p_count_lazy - cl;
rdp->n_nocbs_invoked += c;
}
if (!rcu_nocb_need_deferred_wakeup(rdp))
return;
ndw = ACCESS_ONCE(rdp->nocb_defer_wakeup);
- ACCESS_ONCE(rdp->nocb_defer_wakeup) = RCU_NOGP_WAKE_NOT;
+ ACCESS_ONCE_RW(rdp->nocb_defer_wakeup) = RCU_NOGP_WAKE_NOT;
wake_nocb_leader(rdp, ndw == RCU_NOGP_WAKE_FORCE);
trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("DeferredWake"));
}
t = kthread_run(rcu_nocb_kthread, rdp_spawn,
"rcuo%c/%d", rsp->abbr, cpu);
BUG_ON(IS_ERR(t));
- ACCESS_ONCE(rdp_spawn->nocb_kthread) = t;
+ ACCESS_ONCE_RW(rdp_spawn->nocb_kthread) = t;
}
/*
/* Record start of fully idle period. */
j = jiffies;
- ACCESS_ONCE(rdtp->dynticks_idle_jiffies) = j;
+ ACCESS_ONCE_RW(rdtp->dynticks_idle_jiffies) = j;
smp_mb__before_atomic();
- atomic_inc(&rdtp->dynticks_idle);
+ atomic_inc_unchecked(&rdtp->dynticks_idle);
smp_mb__after_atomic();
- WARN_ON_ONCE(atomic_read(&rdtp->dynticks_idle) & 0x1);
+ WARN_ON_ONCE(atomic_read_unchecked(&rdtp->dynticks_idle) & 0x1);
}
/*
/* Record end of idle period. */
smp_mb__before_atomic();
- atomic_inc(&rdtp->dynticks_idle);
+ atomic_inc_unchecked(&rdtp->dynticks_idle);
smp_mb__after_atomic();
- WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks_idle) & 0x1));
+ WARN_ON_ONCE(!(atomic_read_unchecked(&rdtp->dynticks_idle) & 0x1));
/*
* If we are the timekeeping CPU, we are permitted to be non-idle
WARN_ON_ONCE(smp_processor_id() != tick_do_timer_cpu);
/* Pick up current idle and NMI-nesting counter and check. */
- cur = atomic_read(&rdtp->dynticks_idle);
+ cur = atomic_read_unchecked(&rdtp->dynticks_idle);
if (cur & 0x1) {
*isidle = false; /* We are not idle! */
return;
case RCU_SYSIDLE_NOT:
/* First time all are idle, so note a short idle period. */
- ACCESS_ONCE(full_sysidle_state) = RCU_SYSIDLE_SHORT;
+ ACCESS_ONCE_RW(full_sysidle_state) = RCU_SYSIDLE_SHORT;
break;
case RCU_SYSIDLE_SHORT:
{
smp_mb();
if (full_sysidle_state > RCU_SYSIDLE_SHORT)
- ACCESS_ONCE(full_sysidle_state) = RCU_SYSIDLE_NOT;
+ ACCESS_ONCE_RW(full_sysidle_state) = RCU_SYSIDLE_NOT;
}
/*
smp_mb(); /* grace period precedes setting inuse. */
rshp = container_of(rhp, struct rcu_sysidle_head, rh);
- ACCESS_ONCE(rshp->inuse) = 0;
+ ACCESS_ONCE_RW(rshp->inuse) = 0;
}
/*
static void rcu_dynticks_task_enter(void)
{
#if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL)
- ACCESS_ONCE(current->rcu_tasks_idle_cpu) = smp_processor_id();
+ ACCESS_ONCE_RW(current->rcu_tasks_idle_cpu) = smp_processor_id();
#endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */
}
static void rcu_dynticks_task_exit(void)
{
#if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL)
- ACCESS_ONCE(current->rcu_tasks_idle_cpu) = -1;
+ ACCESS_ONCE_RW(current->rcu_tasks_idle_cpu) = -1;
#endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */
}
ulong2long(rdp->completed), ulong2long(rdp->gpnum),
rdp->passed_quiesce, rdp->qs_pending);
seq_printf(m, " dt=%d/%llx/%d df=%lu",
- atomic_read(&rdp->dynticks->dynticks),
+ atomic_read_unchecked(&rdp->dynticks->dynticks),
rdp->dynticks->dynticks_nesting,
rdp->dynticks->dynticks_nmi_nesting,
rdp->dynticks_fqs);
struct rcu_state *rsp = (struct rcu_state *)m->private;
seq_printf(m, "s=%lu d=%lu w=%lu tf=%lu wd1=%lu wd2=%lu n=%lu sc=%lu dt=%lu dl=%lu dx=%lu\n",
- atomic_long_read(&rsp->expedited_start),
+ atomic_long_read_unchecked(&rsp->expedited_start),
atomic_long_read(&rsp->expedited_done),
- atomic_long_read(&rsp->expedited_wrap),
- atomic_long_read(&rsp->expedited_tryfail),
- atomic_long_read(&rsp->expedited_workdone1),
- atomic_long_read(&rsp->expedited_workdone2),
- atomic_long_read(&rsp->expedited_normal),
- atomic_long_read(&rsp->expedited_stoppedcpus),
- atomic_long_read(&rsp->expedited_done_tries),
- atomic_long_read(&rsp->expedited_done_lost),
- atomic_long_read(&rsp->expedited_done_exit));
+ atomic_long_read_unchecked(&rsp->expedited_wrap),
+ atomic_long_read_unchecked(&rsp->expedited_tryfail),
+ atomic_long_read_unchecked(&rsp->expedited_workdone1),
+ atomic_long_read_unchecked(&rsp->expedited_workdone2),
+ atomic_long_read_unchecked(&rsp->expedited_normal),
+ atomic_long_read_unchecked(&rsp->expedited_stoppedcpus),
+ atomic_long_read_unchecked(&rsp->expedited_done_tries),
+ atomic_long_read_unchecked(&rsp->expedited_done_lost),
+ atomic_long_read_unchecked(&rsp->expedited_done_exit));
return 0;
}
* for CONFIG_RCU_CPU_STALL_TIMEOUT.
*/
if (till_stall_check < 3) {
- ACCESS_ONCE(rcu_cpu_stall_timeout) = 3;
+ ACCESS_ONCE_RW(rcu_cpu_stall_timeout) = 3;
till_stall_check = 3;
} else if (till_stall_check > 300) {
- ACCESS_ONCE(rcu_cpu_stall_timeout) = 300;
+ ACCESS_ONCE_RW(rcu_cpu_stall_timeout) = 300;
till_stall_check = 300;
}
return till_stall_check * HZ + RCU_STALL_DELAY_DELTA;
!ACCESS_ONCE(t->on_rq) ||
(IS_ENABLED(CONFIG_NO_HZ_FULL) &&
!is_idle_task(t) && t->rcu_tasks_idle_cpu >= 0)) {
- ACCESS_ONCE(t->rcu_tasks_holdout) = false;
+ ACCESS_ONCE_RW(t->rcu_tasks_holdout) = false;
list_del_init(&t->rcu_tasks_holdout_list);
put_task_struct(t);
return;
!is_idle_task(t)) {
get_task_struct(t);
t->rcu_tasks_nvcsw = ACCESS_ONCE(t->nvcsw);
- ACCESS_ONCE(t->rcu_tasks_holdout) = true;
+ ACCESS_ONCE_RW(t->rcu_tasks_holdout) = true;
list_add(&t->rcu_tasks_holdout_list,
&rcu_tasks_holdouts);
}
t = kthread_run(rcu_tasks_kthread, NULL, "rcu_tasks_kthread");
BUG_ON(IS_ERR(t));
smp_mb(); /* Ensure others see full kthread. */
- ACCESS_ONCE(rcu_tasks_kthread_ptr) = t;
+ ACCESS_ONCE_RW(rcu_tasks_kthread_ptr) = t;
mutex_unlock(&rcu_tasks_kthread_mutex);
}
static int __init ioresources_init(void)
{
+#ifdef CONFIG_GRKERNSEC_PROC_ADD
+#ifdef CONFIG_GRKERNSEC_PROC_USER
+ proc_create("ioports", S_IRUSR, NULL, &proc_ioports_operations);
+ proc_create("iomem", S_IRUSR, NULL, &proc_iomem_operations);
+#elif defined(CONFIG_GRKERNSEC_PROC_USERGROUP)
+ proc_create("ioports", S_IRUSR | S_IRGRP, NULL, &proc_ioports_operations);
+ proc_create("iomem", S_IRUSR | S_IRGRP, NULL, &proc_iomem_operations);
+#endif
+#else
proc_create("ioports", 0, NULL, &proc_ioports_operations);
proc_create("iomem", 0, NULL, &proc_iomem_operations);
+#endif
return 0;
}
__initcall(ioresources_init);
unsigned int __read_mostly sysctl_sched_autogroup_enabled = 1;
static struct autogroup autogroup_default;
-static atomic_t autogroup_seq_nr;
+static atomic_unchecked_t autogroup_seq_nr;
void __init autogroup_init(struct task_struct *init_task)
{
kref_init(&ag->kref);
init_rwsem(&ag->lock);
- ag->id = atomic_inc_return(&autogroup_seq_nr);
+ ag->id = atomic_inc_return_unchecked(&autogroup_seq_nr);
ag->tg = tg;
#ifdef CONFIG_RT_GROUP_SCHED
/*
* Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
* or number of jiffies left till timeout) if completed.
*/
-long __sched
+long __sched __intentional_overflow(-1)
wait_for_completion_interruptible_timeout(struct completion *x,
unsigned long timeout)
{
*
* Return: -ERESTARTSYS if interrupted, 0 if completed.
*/
-int __sched wait_for_completion_killable(struct completion *x)
+int __sched __intentional_overflow(-1) wait_for_completion_killable(struct completion *x)
{
long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
if (t == -ERESTARTSYS)
* Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
* or number of jiffies left till timeout) if completed.
*/
-long __sched
+long __sched __intentional_overflow(-1)
wait_for_completion_killable_timeout(struct completion *x,
unsigned long timeout)
{
int sysctl_numa_balancing(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp, loff_t *ppos)
{
- struct ctl_table t;
+ ctl_table_no_const t;
int err;
int state = numabalancing_enabled;
next->active_mm = oldmm;
atomic_inc(&oldmm->mm_count);
enter_lazy_tlb(oldmm, next);
- } else
+ } else {
switch_mm(oldmm, mm, next);
+ populate_stack();
+ }
if (!prev->mm) {
prev->active_mm = NULL;
/* convert nice value [19,-20] to rlimit style value [1,40] */
int nice_rlim = nice_to_rlimit(nice);
+ gr_learn_resource(p, RLIMIT_NICE, nice_rlim, 1);
+
return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) ||
capable(CAP_SYS_NICE));
}
nice = task_nice(current) + increment;
nice = clamp_val(nice, MIN_NICE, MAX_NICE);
- if (increment < 0 && !can_nice(current, nice))
+ if (increment < 0 && (!can_nice(current, nice) ||
+ gr_handle_chroot_nice()))
return -EPERM;
retval = security_task_setnice(current, nice);
if (policy != p->policy && !rlim_rtprio)
return -EPERM;
+ gr_learn_resource(p, RLIMIT_RTPRIO, attr->sched_priority, 1);
/* can't increase priority */
if (attr->sched_priority > p->rt_priority &&
attr->sched_priority > rlim_rtprio)
if (mm != &init_mm) {
switch_mm(mm, &init_mm, current);
+ populate_stack();
finish_arch_post_lock_switch();
}
mmdrop(mm);
#if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL)
-static struct ctl_table sd_ctl_dir[] = {
+static ctl_table_no_const sd_ctl_dir[] __read_only = {
{
.procname = "sched_domain",
.mode = 0555,
{}
};
-static struct ctl_table *sd_alloc_ctl_entry(int n)
+static ctl_table_no_const *sd_alloc_ctl_entry(int n)
{
- struct ctl_table *entry =
+ ctl_table_no_const *entry =
kcalloc(n, sizeof(struct ctl_table), GFP_KERNEL);
return entry;
}
-static void sd_free_ctl_entry(struct ctl_table **tablep)
+static void sd_free_ctl_entry(ctl_table_no_const *tablep)
{
- struct ctl_table *entry;
+ ctl_table_no_const *entry;
/*
* In the intermediate directories, both the child directory and
* will always be set. In the lowest directory the names are
* static strings and all have proc handlers.
*/
- for (entry = *tablep; entry->mode; entry++) {
- if (entry->child)
- sd_free_ctl_entry(&entry->child);
+ for (entry = tablep; entry->mode; entry++) {
+ if (entry->child) {
+ sd_free_ctl_entry(entry->child);
+ pax_open_kernel();
+ entry->child = NULL;
+ pax_close_kernel();
+ }
if (entry->proc_handler == NULL)
kfree(entry->procname);
}
- kfree(*tablep);
- *tablep = NULL;
+ kfree(tablep);
}
static int min_load_idx = 0;
static int max_load_idx = CPU_LOAD_IDX_MAX-1;
static void
-set_table_entry(struct ctl_table *entry,
+set_table_entry(ctl_table_no_const *entry,
const char *procname, void *data, int maxlen,
umode_t mode, proc_handler *proc_handler,
bool load_idx)
static struct ctl_table *
sd_alloc_ctl_domain_table(struct sched_domain *sd)
{
- struct ctl_table *table = sd_alloc_ctl_entry(14);
+ ctl_table_no_const *table = sd_alloc_ctl_entry(14);
if (table == NULL)
return NULL;
return table;
}
-static struct ctl_table *sd_alloc_ctl_cpu_table(int cpu)
+static ctl_table_no_const *sd_alloc_ctl_cpu_table(int cpu)
{
- struct ctl_table *entry, *table;
+ ctl_table_no_const *entry, *table;
struct sched_domain *sd;
int domain_num = 0, i;
char buf[32];
static void register_sched_domain_sysctl(void)
{
int i, cpu_num = num_possible_cpus();
- struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1);
+ ctl_table_no_const *entry = sd_alloc_ctl_entry(cpu_num + 1);
char buf[32];
WARN_ON(sd_ctl_dir[0].child);
+ pax_open_kernel();
sd_ctl_dir[0].child = entry;
+ pax_close_kernel();
if (entry == NULL)
return;
if (sd_sysctl_header)
unregister_sysctl_table(sd_sysctl_header);
sd_sysctl_header = NULL;
- if (sd_ctl_dir[0].child)
- sd_free_ctl_entry(&sd_ctl_dir[0].child);
+ if (sd_ctl_dir[0].child) {
+ sd_free_ctl_entry(sd_ctl_dir[0].child);
+ pax_open_kernel();
+ sd_ctl_dir[0].child = NULL;
+ pax_close_kernel();
+ }
}
#else
static void register_sched_domain_sysctl(void)
static void reset_ptenuma_scan(struct task_struct *p)
{
- ACCESS_ONCE(p->mm->numa_scan_seq)++;
+ ACCESS_ONCE_RW(p->mm->numa_scan_seq)++;
p->mm->numa_scan_offset = 0;
}
* run_rebalance_domains is triggered when needed from the scheduler tick.
* Also triggered for nohz idle balancing (with nohz_balancing_kick set).
*/
-static void run_rebalance_domains(struct softirq_action *h)
+static __latent_entropy void run_rebalance_domains(void)
{
struct rq *this_rq = this_rq();
enum cpu_idle_type idle = this_rq->idle_balance ?
#ifdef CONFIG_FAIR_GROUP_SCHED
void (*task_move_group) (struct task_struct *p, int on_rq);
#endif
-};
+} __do_const;
static inline void put_prev_task(struct rq *rq, struct task_struct *prev)
{
switch (action) {
case SECCOMP_RET_ERRNO:
- /* Set the low-order 16-bits as a errno. */
+ /* Set low-order bits as an errno, capped at MAX_ERRNO. */
+ if (data > MAX_ERRNO)
+ data = MAX_ERRNO;
syscall_set_return_value(current, task_pt_regs(current),
-data, 0);
goto skip;
int print_fatal_signals __read_mostly;
-static void __user *sig_handler(struct task_struct *t, int sig)
+static __sighandler_t sig_handler(struct task_struct *t, int sig)
{
return t->sighand->action[sig - 1].sa.sa_handler;
}
-static int sig_handler_ignored(void __user *handler, int sig)
+static int sig_handler_ignored(__sighandler_t handler, int sig)
{
/* Is it explicitly or implicitly ignored? */
return handler == SIG_IGN ||
static int sig_task_ignored(struct task_struct *t, int sig, bool force)
{
- void __user *handler;
+ __sighandler_t handler;
handler = sig_handler(t, sig);
atomic_inc(&user->sigpending);
rcu_read_unlock();
+ if (!override_rlimit)
+ gr_learn_resource(t, RLIMIT_SIGPENDING, atomic_read(&user->sigpending), 1);
+
if (override_rlimit ||
atomic_read(&user->sigpending) <=
task_rlimit(t, RLIMIT_SIGPENDING)) {
int unhandled_signal(struct task_struct *tsk, int sig)
{
- void __user *handler = tsk->sighand->action[sig-1].sa.sa_handler;
+ __sighandler_t handler = tsk->sighand->action[sig-1].sa.sa_handler;
if (is_global_init(tsk))
return 1;
if (handler != SIG_IGN && handler != SIG_DFL)
}
}
+ /* allow glibc communication via tgkill to other threads in our
+ thread group */
+ if ((info == SEND_SIG_NOINFO || info->si_code != SI_TKILL ||
+ sig != (SIGRTMIN+1) || task_tgid_vnr(t) != info->si_pid)
+ && gr_handle_signal(t, sig))
+ return -EPERM;
+
return security_task_kill(t, info, sig, 0);
}
return send_signal(sig, info, p, 1);
}
-static int
+int
specific_send_sig_info(int sig, struct siginfo *info, struct task_struct *t)
{
return send_signal(sig, info, t, 0);
unsigned long int flags;
int ret, blocked, ignored;
struct k_sigaction *action;
+ int is_unhandled = 0;
spin_lock_irqsave(&t->sighand->siglock, flags);
action = &t->sighand->action[sig-1];
}
if (action->sa.sa_handler == SIG_DFL)
t->signal->flags &= ~SIGNAL_UNKILLABLE;
+ if (action->sa.sa_handler == SIG_IGN || action->sa.sa_handler == SIG_DFL)
+ is_unhandled = 1;
ret = specific_send_sig_info(sig, info, t);
spin_unlock_irqrestore(&t->sighand->siglock, flags);
+ /* only deal with unhandled signals, java etc trigger SIGSEGV during
+ normal operation */
+ if (is_unhandled) {
+ gr_log_signal(sig, !is_si_special(info) ? info->si_addr : NULL, t);
+ gr_handle_crash(t, sig);
+ }
+
return ret;
}
ret = check_kill_permission(sig, info, p);
rcu_read_unlock();
- if (!ret && sig)
+ if (!ret && sig) {
ret = do_send_sig_info(sig, info, p, true);
+ if (!ret)
+ gr_log_signal(sig, !is_si_special(info) ? info->si_addr : NULL, p);
+ }
return ret;
}
int error = -ESRCH;
rcu_read_lock();
- p = find_task_by_vpid(pid);
+#ifdef CONFIG_GRKERNSEC_CHROOT_FINDTASK
+ /* allow glibc communication via tgkill to other threads in our
+ thread group */
+ if (grsec_enable_chroot_findtask && info->si_code == SI_TKILL &&
+ sig == (SIGRTMIN+1) && tgid == info->si_pid)
+ p = find_task_by_vpid_unrestricted(pid);
+ else
+#endif
+ p = find_task_by_vpid(pid);
if (p && (tgid <= 0 || task_tgid_vnr(p) == tgid)) {
error = check_kill_permission(sig, info, p);
/*
}
seg = get_fs();
set_fs(KERNEL_DS);
- ret = do_sigaltstack((stack_t __force __user *) (uss_ptr ? &uss : NULL),
- (stack_t __force __user *) &uoss,
+ ret = do_sigaltstack((stack_t __force_user *) (uss_ptr ? &uss : NULL),
+ (stack_t __force_user *) &uoss,
compat_user_stack_pointer());
set_fs(seg);
if (ret >= 0 && uoss_ptr) {
}
smpboot_unpark_thread(plug_thread, cpu);
}
- list_add(&plug_thread->list, &hotplug_threads);
+ pax_list_add(&plug_thread->list, &hotplug_threads);
out:
mutex_unlock(&smpboot_threads_lock);
put_online_cpus();
{
get_online_cpus();
mutex_lock(&smpboot_threads_lock);
- list_del(&plug_thread->list);
+ pax_list_del(&plug_thread->list);
smpboot_destroy_threads(plug_thread);
mutex_unlock(&smpboot_threads_lock);
put_online_cpus();
EXPORT_SYMBOL(irq_stat);
#endif
-static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
+static struct softirq_action softirq_vec[NR_SOFTIRQS] __read_only __aligned(PAGE_SIZE);
DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
kstat_incr_softirqs_this_cpu(vec_nr);
trace_softirq_entry(vec_nr);
- h->action(h);
+ h->action();
trace_softirq_exit(vec_nr);
if (unlikely(prev_count != preempt_count())) {
pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n",
or_softirq_pending(1UL << nr);
}
-void open_softirq(int nr, void (*action)(struct softirq_action *))
+void __init open_softirq(int nr, void (*action)(void))
{
softirq_vec[nr].action = action;
}
}
EXPORT_SYMBOL(__tasklet_hi_schedule_first);
-static void tasklet_action(struct softirq_action *a)
+static void tasklet_action(void)
{
struct tasklet_struct *list;
}
}
-static void tasklet_hi_action(struct softirq_action *a)
+static __latent_entropy void tasklet_hi_action(void)
{
struct tasklet_struct *list;
.notifier_call = cpu_callback
};
-static struct smp_hotplug_thread softirq_threads = {
+static struct smp_hotplug_thread softirq_threads __read_only = {
.store = &ksoftirqd,
.thread_should_run = ksoftirqd_should_run,
.thread_fn = run_ksoftirqd,
error = -EACCES;
goto out;
}
+
+ if (gr_handle_chroot_setpriority(p, niceval)) {
+ error = -EACCES;
+ goto out;
+ }
+
no_nice = security_task_setnice(p, niceval);
if (no_nice) {
error = no_nice;
goto error;
}
+ if (gr_check_group_change(new->gid, new->egid, INVALID_GID))
+ goto error;
+
+ if (!gid_eq(new->gid, old->gid)) {
+ /* make sure we generate a learn log for what will
+ end up being a role transition after a full-learning
+ policy is generated
+ CAP_SETGID is required to perform a transition
+ we may not log a CAP_SETGID check above, e.g.
+ in the case where new rgid = old egid
+ */
+ gr_learn_cap(current, new, CAP_SETGID);
+ }
+
if (rgid != (gid_t) -1 ||
(egid != (gid_t) -1 && !gid_eq(kegid, old->gid)))
new->sgid = new->egid;
old = current_cred();
retval = -EPERM;
+
+ if (gr_check_group_change(kgid, kgid, kgid))
+ goto error;
+
if (ns_capable(old->user_ns, CAP_SETGID))
new->gid = new->egid = new->sgid = new->fsgid = kgid;
else if (gid_eq(kgid, old->gid) || gid_eq(kgid, old->sgid))
/*
* change the user struct in a credentials set to match the new UID
*/
-static int set_user(struct cred *new)
+int set_user(struct cred *new)
{
struct user_struct *new_user;
goto error;
}
+ if (gr_check_user_change(new->uid, new->euid, INVALID_UID))
+ goto error;
+
if (!uid_eq(new->uid, old->uid)) {
+ /* make sure we generate a learn log for what will
+ end up being a role transition after a full-learning
+ policy is generated
+ CAP_SETUID is required to perform a transition
+ we may not log a CAP_SETUID check above, e.g.
+ in the case where new ruid = old euid
+ */
+ gr_learn_cap(current, new, CAP_SETUID);
retval = set_user(new);
if (retval < 0)
goto error;
old = current_cred();
retval = -EPERM;
+
+ if (gr_check_crash_uid(kuid))
+ goto error;
+ if (gr_check_user_change(kuid, kuid, kuid))
+ goto error;
+
if (ns_capable(old->user_ns, CAP_SETUID)) {
new->suid = new->uid = kuid;
if (!uid_eq(kuid, old->uid)) {
goto error;
}
+ if (gr_check_user_change(kruid, keuid, INVALID_UID))
+ goto error;
+
if (ruid != (uid_t) -1) {
new->uid = kruid;
if (!uid_eq(kruid, old->uid)) {
goto error;
}
+ if (gr_check_group_change(krgid, kegid, INVALID_GID))
+ goto error;
+
if (rgid != (gid_t) -1)
new->gid = krgid;
if (egid != (gid_t) -1)
uid_eq(kuid, old->suid) || uid_eq(kuid, old->fsuid) ||
ns_capable(old->user_ns, CAP_SETUID)) {
if (!uid_eq(kuid, old->fsuid)) {
+ if (gr_check_user_change(INVALID_UID, INVALID_UID, kuid))
+ goto error;
+
new->fsuid = kuid;
if (security_task_fix_setuid(new, old, LSM_SETID_FS) == 0)
goto change_okay;
}
}
+error:
abort_creds(new);
return old_fsuid;
if (gid_eq(kgid, old->gid) || gid_eq(kgid, old->egid) ||
gid_eq(kgid, old->sgid) || gid_eq(kgid, old->fsgid) ||
ns_capable(old->user_ns, CAP_SETGID)) {
+ if (gr_check_group_change(INVALID_GID, INVALID_GID, kgid))
+ goto error;
+
if (!gid_eq(kgid, old->fsgid)) {
new->fsgid = kgid;
goto change_okay;
}
}
+error:
abort_creds(new);
return old_fsgid;
return -EFAULT;
down_read(&uts_sem);
- error = __copy_to_user(&name->sysname, &utsname()->sysname,
+ error = __copy_to_user(name->sysname, &utsname()->sysname,
__OLD_UTS_LEN);
error |= __put_user(0, name->sysname + __OLD_UTS_LEN);
- error |= __copy_to_user(&name->nodename, &utsname()->nodename,
+ error |= __copy_to_user(name->nodename, &utsname()->nodename,
__OLD_UTS_LEN);
error |= __put_user(0, name->nodename + __OLD_UTS_LEN);
- error |= __copy_to_user(&name->release, &utsname()->release,
+ error |= __copy_to_user(name->release, &utsname()->release,
__OLD_UTS_LEN);
error |= __put_user(0, name->release + __OLD_UTS_LEN);
- error |= __copy_to_user(&name->version, &utsname()->version,
+ error |= __copy_to_user(name->version, &utsname()->version,
__OLD_UTS_LEN);
error |= __put_user(0, name->version + __OLD_UTS_LEN);
- error |= __copy_to_user(&name->machine, &utsname()->machine,
+ error |= __copy_to_user(name->machine, &utsname()->machine,
__OLD_UTS_LEN);
error |= __put_user(0, name->machine + __OLD_UTS_LEN);
up_read(&uts_sem);
*/
new_rlim->rlim_cur = 1;
}
+ /* Handle the case where a fork and setuid occur and then RLIMIT_NPROC
+ is changed to a lower value. Since tasks can be created by the same
+ user in between this limit change and an execve by this task, force
+ a recheck only for this task by setting PF_NPROC_EXCEEDED
+ */
+ if (resource == RLIMIT_NPROC && tsk->real_cred->user != INIT_USER)
+ tsk->flags |= PF_NPROC_EXCEEDED;
}
if (!retval) {
if (old_rlim)
#if defined(CONFIG_SYSCTL)
-
/* External variables not in a header file. */
extern int max_threads;
extern int suid_dumpable;
/* Constants used for minimum and maximum */
#ifdef CONFIG_LOCKUP_DETECTOR
-static int sixty = 60;
+static int sixty __read_only = 60;
#endif
-static int __maybe_unused neg_one = -1;
+static int __maybe_unused neg_one __read_only = -1;
-static int zero;
-static int __maybe_unused one = 1;
-static int __maybe_unused two = 2;
-static int __maybe_unused four = 4;
-static unsigned long one_ul = 1;
-static int one_hundred = 100;
+static int zero __read_only = 0;
+static int __maybe_unused one __read_only = 1;
+static int __maybe_unused two __read_only = 2;
+static int __maybe_unused three __read_only = 3;
+static int __maybe_unused four __read_only = 4;
+static unsigned long one_ul __read_only = 1;
+static int one_hundred __read_only = 100;
#ifdef CONFIG_PRINTK
-static int ten_thousand = 10000;
+static int ten_thousand __read_only = 10000;
#endif
/* this is needed for the proc_doulongvec_minmax of vm_dirty_bytes */
void __user *buffer, size_t *lenp, loff_t *ppos);
#endif
-#ifdef CONFIG_PRINTK
static int proc_dointvec_minmax_sysadmin(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp, loff_t *ppos);
-#endif
static int proc_dointvec_minmax_coredump(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp, loff_t *ppos);
#endif
+extern struct ctl_table grsecurity_table[];
+
static struct ctl_table kern_table[];
static struct ctl_table vm_table[];
static struct ctl_table fs_table[];
int sysctl_legacy_va_layout;
#endif
+#ifdef CONFIG_PAX_SOFTMODE
+static struct ctl_table pax_table[] = {
+ {
+ .procname = "softmode",
+ .data = &pax_softmode,
+ .maxlen = sizeof(unsigned int),
+ .mode = 0600,
+ .proc_handler = &proc_dointvec,
+ },
+
+ { }
+};
+#endif
+
/* The default sysctl tables: */
static struct ctl_table sysctl_base_table[] = {
#endif
static struct ctl_table kern_table[] = {
+#if defined(CONFIG_GRKERNSEC_SYSCTL) || defined(CONFIG_GRKERNSEC_ROFS)
+ {
+ .procname = "grsecurity",
+ .mode = 0500,
+ .child = grsecurity_table,
+ },
+#endif
+
+#ifdef CONFIG_PAX_SOFTMODE
+ {
+ .procname = "pax",
+ .mode = 0500,
+ .child = pax_table,
+ },
+#endif
+
{
.procname = "sched_child_runs_first",
.data = &sysctl_sched_child_runs_first,
.data = &modprobe_path,
.maxlen = KMOD_PATH_LEN,
.mode = 0644,
- .proc_handler = proc_dostring,
+ .proc_handler = proc_dostring_modpriv,
},
{
.procname = "modules_disabled",
.extra1 = &zero,
.extra2 = &one,
},
+#endif
{
.procname = "kptr_restrict",
.data = &kptr_restrict,
.maxlen = sizeof(int),
.mode = 0644,
.proc_handler = proc_dointvec_minmax_sysadmin,
+#ifdef CONFIG_GRKERNSEC_HIDESYM
+ .extra1 = &two,
+#else
.extra1 = &zero,
+#endif
.extra2 = &two,
},
-#endif
{
.procname = "ngroups_max",
.data = &ngroups_max,
*/
{
.procname = "perf_event_paranoid",
- .data = &sysctl_perf_event_paranoid,
- .maxlen = sizeof(sysctl_perf_event_paranoid),
+ .data = &sysctl_perf_event_legitimately_concerned,
+ .maxlen = sizeof(sysctl_perf_event_legitimately_concerned),
.mode = 0644,
- .proc_handler = proc_dointvec,
+ /* go ahead, be a hero */
+ .proc_handler = proc_dointvec_minmax_sysadmin,
+ .extra1 = &neg_one,
+#ifdef CONFIG_GRKERNSEC_PERF_HARDEN
+ .extra2 = &three,
+#else
+ .extra2 = &two,
+#endif
},
{
.procname = "perf_event_mlock_kb",
.proc_handler = proc_dointvec_minmax,
.extra1 = &zero,
},
+ {
+ .procname = "heap_stack_gap",
+ .data = &sysctl_heap_stack_gap,
+ .maxlen = sizeof(sysctl_heap_stack_gap),
+ .mode = 0644,
+ .proc_handler = proc_doulongvec_minmax,
+ },
#else
{
.procname = "nr_trim_pages",
(char __user *)buffer, lenp, ppos);
}
+int proc_dostring_modpriv(struct ctl_table *table, int write,
+ void __user *buffer, size_t *lenp, loff_t *ppos)
+{
+ if (write && !capable(CAP_SYS_MODULE))
+ return -EPERM;
+
+ return _proc_do_string(table->data, table->maxlen, write,
+ buffer, lenp, ppos);
+}
+
static size_t proc_skip_spaces(char **buf)
{
size_t ret;
len = strlen(tmp);
if (len > *size)
len = *size;
+ if (len > sizeof(tmp))
+ len = sizeof(tmp);
if (copy_to_user(*buf, tmp, len))
return -EFAULT;
*size -= len;
static int proc_taint(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp, loff_t *ppos)
{
- struct ctl_table t;
+ ctl_table_no_const t;
unsigned long tmptaint = get_taint();
int err;
return err;
}
-#ifdef CONFIG_PRINTK
static int proc_dointvec_minmax_sysadmin(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp, loff_t *ppos)
{
return proc_dointvec_minmax(table, write, buffer, lenp, ppos);
}
-#endif
struct do_proc_dointvec_minmax_conv_param {
int *min;
return -ENOSYS;
}
+int proc_dostring_modpriv(struct ctl_table *table, int write,
+ void __user *buffer, size_t *lenp, loff_t *ppos)
+{
+ return -ENOSYS;
+}
+
int proc_dointvec(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp, loff_t *ppos)
{
EXPORT_SYMBOL(proc_dointvec_userhz_jiffies);
EXPORT_SYMBOL(proc_dointvec_ms_jiffies);
EXPORT_SYMBOL(proc_dostring);
+EXPORT_SYMBOL(proc_dostring_modpriv);
EXPORT_SYMBOL(proc_doulongvec_minmax);
EXPORT_SYMBOL(proc_doulongvec_ms_jiffies_minmax);
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/pid_namespace.h>
+#include <linux/grsecurity.h>
#include <net/genetlink.h>
#include <linux/atomic.h>
+extern int gr_is_taskstats_denied(int pid);
+
/*
* Maximum length of a cpumask that can be specified in
* the TASKSTATS_CMD_ATTR_REGISTER/DEREGISTER_CPUMASK attribute
static int taskstats_user_cmd(struct sk_buff *skb, struct genl_info *info)
{
+ if (gr_is_taskstats_denied(current->pid))
+ return -EACCES;
+
if (info->attrs[TASKSTATS_CMD_ATTR_REGISTER_CPUMASK])
return cmd_attr_register_cpumask(info);
else if (info->attrs[TASKSTATS_CMD_ATTR_DEREGISTER_CPUMASK])
struct platform_device *pdev;
int error = 0;
int i;
- struct k_clock alarm_clock = {
+ static struct k_clock alarm_clock = {
.clock_getres = alarm_clock_getres,
.clock_get = alarm_clock_get,
.timer_create = alarm_timer_create,
local_irq_restore(flags);
}
-static void run_hrtimer_softirq(struct softirq_action *h)
+static __latent_entropy void run_hrtimer_softirq(void)
{
hrtimer_peek_ahead_timers();
}
static __init int init_posix_cpu_timers(void)
{
- struct k_clock process = {
+ static struct k_clock process = {
.clock_getres = process_cpu_clock_getres,
.clock_get = process_cpu_clock_get,
.timer_create = process_cpu_timer_create,
.nsleep = process_cpu_nsleep,
.nsleep_restart = process_cpu_nsleep_restart,
};
- struct k_clock thread = {
+ static struct k_clock thread = {
.clock_getres = thread_cpu_clock_getres,
.clock_get = thread_cpu_clock_get,
.timer_create = thread_cpu_timer_create,
#include <linux/hash.h>
#include <linux/posix-clock.h>
#include <linux/posix-timers.h>
+#include <linux/grsecurity.h>
#include <linux/syscalls.h>
#include <linux/wait.h>
#include <linux/workqueue.h>
* which we beg off on and pass to do_sys_settimeofday().
*/
-static struct k_clock posix_clocks[MAX_CLOCKS];
+static struct k_clock *posix_clocks[MAX_CLOCKS];
/*
* These ones are defined below.
*/
static __init int init_posix_timers(void)
{
- struct k_clock clock_realtime = {
+ static struct k_clock clock_realtime = {
.clock_getres = hrtimer_get_res,
.clock_get = posix_clock_realtime_get,
.clock_set = posix_clock_realtime_set,
.timer_get = common_timer_get,
.timer_del = common_timer_del,
};
- struct k_clock clock_monotonic = {
+ static struct k_clock clock_monotonic = {
.clock_getres = hrtimer_get_res,
.clock_get = posix_ktime_get_ts,
.nsleep = common_nsleep,
.timer_get = common_timer_get,
.timer_del = common_timer_del,
};
- struct k_clock clock_monotonic_raw = {
+ static struct k_clock clock_monotonic_raw = {
.clock_getres = hrtimer_get_res,
.clock_get = posix_get_monotonic_raw,
};
- struct k_clock clock_realtime_coarse = {
+ static struct k_clock clock_realtime_coarse = {
.clock_getres = posix_get_coarse_res,
.clock_get = posix_get_realtime_coarse,
};
- struct k_clock clock_monotonic_coarse = {
+ static struct k_clock clock_monotonic_coarse = {
.clock_getres = posix_get_coarse_res,
.clock_get = posix_get_monotonic_coarse,
};
- struct k_clock clock_tai = {
+ static struct k_clock clock_tai = {
.clock_getres = hrtimer_get_res,
.clock_get = posix_get_tai,
.nsleep = common_nsleep,
.timer_get = common_timer_get,
.timer_del = common_timer_del,
};
- struct k_clock clock_boottime = {
+ static struct k_clock clock_boottime = {
.clock_getres = hrtimer_get_res,
.clock_get = posix_get_boottime,
.nsleep = common_nsleep,
return;
}
- posix_clocks[clock_id] = *new_clock;
+ posix_clocks[clock_id] = new_clock;
}
EXPORT_SYMBOL_GPL(posix_timers_register_clock);
return (id & CLOCKFD_MASK) == CLOCKFD ?
&clock_posix_dynamic : &clock_posix_cpu;
- if (id >= MAX_CLOCKS || !posix_clocks[id].clock_getres)
+ if (id >= MAX_CLOCKS || !posix_clocks[id] || !posix_clocks[id]->clock_getres)
return NULL;
- return &posix_clocks[id];
+ return posix_clocks[id];
}
static int common_timer_create(struct k_itimer *new_timer)
struct k_clock *kc = clockid_to_kclock(which_clock);
struct k_itimer *new_timer;
int error, new_timer_id;
- sigevent_t event;
+ sigevent_t event = { };
int it_id_set = IT_ID_NOT_SET;
if (!kc)
if (copy_from_user(&new_tp, tp, sizeof (*tp)))
return -EFAULT;
+ /* only the CLOCK_REALTIME clock can be set, all other clocks
+ have their clock_set fptr set to a nosettime dummy function
+ CLOCK_REALTIME has a NULL clock_set fptr which causes it to
+ call common_clock_set, which calls do_sys_settimeofday, which
+ we hook
+ */
+
return kc->clock_set(which_clock, &new_tp);
}
return error;
if (tz) {
+ /* we log in do_settimeofday called below, so don't log twice
+ */
+ if (!tv)
+ gr_log_timechange();
+
sys_tz = *tz;
update_vsyscall_tz();
if (firsttime) {
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/sched.h>
+#include <linux/grsecurity.h>
#include <linux/syscore_ops.h>
#include <linux/clocksource.h>
#include <linux/jiffies.h>
if (!timespec64_valid_strict(ts))
return -EINVAL;
+ gr_log_timechange();
+
raw_spin_lock_irqsave(&timekeeper_lock, flags);
write_seqcount_begin(&tk_core.seq);
/*
* This function runs timers and the timer-tq in bottom half context.
*/
-static void run_timer_softirq(struct softirq_action *h)
+static __latent_entropy void run_timer_softirq(void)
{
struct tvec_base *base = __this_cpu_read(tvec_bases);
*
* In all cases the return value is guaranteed to be non-negative.
*/
-signed long __sched schedule_timeout(signed long timeout)
+signed long __sched __intentional_overflow(-1) schedule_timeout(signed long timeout)
{
struct timer_list timer;
unsigned long expire;
static void print_name_offset(struct seq_file *m, void *sym)
{
+#ifdef CONFIG_GRKERNSEC_HIDESYM
+ SEQ_printf(m, "<%p>", NULL);
+#else
char symname[KSYM_NAME_LEN];
if (lookup_symbol_name((unsigned long)sym, symname) < 0)
SEQ_printf(m, "<%pK>", sym);
else
SEQ_printf(m, "%s", symname);
+#endif
}
static void
static void
print_base(struct seq_file *m, struct hrtimer_clock_base *base, u64 now)
{
+#ifdef CONFIG_GRKERNSEC_HIDESYM
+ SEQ_printf(m, " .base: %p\n", NULL);
+#else
SEQ_printf(m, " .base: %pK\n", base);
+#endif
SEQ_printf(m, " .index: %d\n",
base->index);
SEQ_printf(m, " .resolution: %Lu nsecs\n",
{
struct proc_dir_entry *pe;
+#ifdef CONFIG_GRKERNSEC_PROC_ADD
+ pe = proc_create("timer_list", 0400, NULL, &timer_list_fops);
+#else
pe = proc_create("timer_list", 0444, NULL, &timer_list_fops);
+#endif
if (!pe)
return -ENOMEM;
return 0;
static unsigned long nr_entries;
static struct entry entries[MAX_ENTRIES];
-static atomic_t overflow_count;
+static atomic_unchecked_t overflow_count;
/*
* The entries are in a hash-table, for fast lookup:
nr_entries = 0;
memset(entries, 0, sizeof(entries));
memset(tstat_hash_table, 0, sizeof(tstat_hash_table));
- atomic_set(&overflow_count, 0);
+ atomic_set_unchecked(&overflow_count, 0);
}
static struct entry *alloc_entry(void)
if (likely(entry))
entry->count++;
else
- atomic_inc(&overflow_count);
+ atomic_inc_unchecked(&overflow_count);
out_unlock:
raw_spin_unlock_irqrestore(lock, flags);
static void print_name_offset(struct seq_file *m, unsigned long addr)
{
+#ifdef CONFIG_GRKERNSEC_HIDESYM
+ seq_printf(m, "<%p>", NULL);
+#else
char symname[KSYM_NAME_LEN];
if (lookup_symbol_name(addr, symname) < 0)
- seq_printf(m, "<%p>", (void *)addr);
+ seq_printf(m, "<%pK>", (void *)addr);
else
seq_printf(m, "%s", symname);
+#endif
}
static int tstats_show(struct seq_file *m, void *v)
seq_puts(m, "Timer Stats Version: v0.3\n");
seq_printf(m, "Sample period: %ld.%03ld s\n", period.tv_sec, ms);
- if (atomic_read(&overflow_count))
- seq_printf(m, "Overflow: %d entries\n", atomic_read(&overflow_count));
+ if (atomic_read_unchecked(&overflow_count))
+ seq_printf(m, "Overflow: %d entries\n", atomic_read_unchecked(&overflow_count));
seq_printf(m, "Collection: %s\n", timer_stats_active ? "active" : "inactive");
for (i = 0; i < nr_entries; i++) {
{
struct proc_dir_entry *pe;
+#ifdef CONFIG_GRKERNSEC_PROC_ADD
+ pe = proc_create("timer_stats", 0600, NULL, &tstats_fops);
+#else
pe = proc_create("timer_stats", 0644, NULL, &tstats_fops);
+#endif
if (!pe)
return -ENOMEM;
return 0;
mutex_lock(&fullstop_mutex);
if (ACCESS_ONCE(fullstop) == FULLSTOP_DONTSTOP) {
VERBOSE_TOROUT_STRING("Unscheduled system shutdown detected");
- ACCESS_ONCE(fullstop) = FULLSTOP_SHUTDOWN;
+ ACCESS_ONCE_RW(fullstop) = FULLSTOP_SHUTDOWN;
} else {
pr_warn("Concurrent rmmod and shutdown illegal!\n");
}
if (!torture_must_stop()) {
if (stutter > 1) {
schedule_timeout_interruptible(stutter - 1);
- ACCESS_ONCE(stutter_pause_test) = 2;
+ ACCESS_ONCE_RW(stutter_pause_test) = 2;
}
schedule_timeout_interruptible(1);
- ACCESS_ONCE(stutter_pause_test) = 1;
+ ACCESS_ONCE_RW(stutter_pause_test) = 1;
}
if (!torture_must_stop())
schedule_timeout_interruptible(stutter);
- ACCESS_ONCE(stutter_pause_test) = 0;
+ ACCESS_ONCE_RW(stutter_pause_test) = 0;
torture_shutdown_absorb("torture_stutter");
} while (!torture_must_stop());
torture_kthread_stopping("torture_stutter");
schedule_timeout_uninterruptible(10);
return true;
}
- ACCESS_ONCE(fullstop) = FULLSTOP_RMMOD;
+ ACCESS_ONCE_RW(fullstop) = FULLSTOP_RMMOD;
mutex_unlock(&fullstop_mutex);
torture_shutdown_cleanup();
torture_shuffle_cleanup();
struct blk_trace *bt = filp->private_data;
char buf[16];
- snprintf(buf, sizeof(buf), "%u\n", atomic_read(&bt->dropped));
+ snprintf(buf, sizeof(buf), "%u\n", atomic_read_unchecked(&bt->dropped));
return simple_read_from_buffer(buffer, count, ppos, buf, strlen(buf));
}
return 1;
bt = buf->chan->private_data;
- atomic_inc(&bt->dropped);
+ atomic_inc_unchecked(&bt->dropped);
return 0;
}
bt->dir = dir;
bt->dev = dev;
- atomic_set(&bt->dropped, 0);
+ atomic_set_unchecked(&bt->dropped, 0);
INIT_LIST_HEAD(&bt->running_list);
ret = -EIO;
if (unlikely(ftrace_disabled))
return 0;
+ ret = ftrace_arch_code_modify_prepare();
+ FTRACE_WARN_ON(ret);
+ if (ret)
+ return 0;
+
ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
+ FTRACE_WARN_ON(ftrace_arch_code_modify_post_process());
if (ret) {
ftrace_bug(ret, rec);
- return 0;
}
- return 1;
+ return ret ? 0 : 1;
}
/*
if (!count)
return 0;
+ pax_open_kernel();
sort(start, count, sizeof(*start),
ftrace_cmp_ips, ftrace_swap_ips);
+ pax_close_kernel();
start_pg = ftrace_allocate_pages(count);
if (!start_pg)
if (t->ret_stack == NULL) {
atomic_set(&t->tracing_graph_pause, 0);
- atomic_set(&t->trace_overrun, 0);
+ atomic_set_unchecked(&t->trace_overrun, 0);
t->curr_ret_stack = -1;
/* Make sure the tasks see the -1 first: */
smp_wmb();
graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack)
{
atomic_set(&t->tracing_graph_pause, 0);
- atomic_set(&t->trace_overrun, 0);
+ atomic_set_unchecked(&t->trace_overrun, 0);
t->ftrace_timestamp = 0;
/* make curr_ret_stack visible before we add the ret_stack */
smp_wmb();
*/
struct buffer_page {
struct list_head list; /* list of buffer pages */
- local_t write; /* index for next write */
+ local_unchecked_t write; /* index for next write */
unsigned read; /* index for next read */
- local_t entries; /* entries on this page */
+ local_unchecked_t entries; /* entries on this page */
unsigned long real_end; /* real end of data */
struct buffer_data_page *page; /* Actual data page */
};
unsigned long last_overrun;
local_t entries_bytes;
local_t entries;
- local_t overrun;
- local_t commit_overrun;
+ local_unchecked_t overrun;
+ local_unchecked_t commit_overrun;
local_t dropped_events;
local_t committing;
local_t commits;
*
* We add a counter to the write field to denote this.
*/
- old_write = local_add_return(RB_WRITE_INTCNT, &next_page->write);
- old_entries = local_add_return(RB_WRITE_INTCNT, &next_page->entries);
+ old_write = local_add_return_unchecked(RB_WRITE_INTCNT, &next_page->write);
+ old_entries = local_add_return_unchecked(RB_WRITE_INTCNT, &next_page->entries);
/*
* Just make sure we have seen our old_write and synchronize
* cmpxchg to only update if an interrupt did not already
* do it for us. If the cmpxchg fails, we don't care.
*/
- (void)local_cmpxchg(&next_page->write, old_write, val);
- (void)local_cmpxchg(&next_page->entries, old_entries, eval);
+ (void)local_cmpxchg_unchecked(&next_page->write, old_write, val);
+ (void)local_cmpxchg_unchecked(&next_page->entries, old_entries, eval);
/*
* No need to worry about races with clearing out the commit.
static inline unsigned long rb_page_entries(struct buffer_page *bpage)
{
- return local_read(&bpage->entries) & RB_WRITE_MASK;
+ return local_read_unchecked(&bpage->entries) & RB_WRITE_MASK;
}
static inline unsigned long rb_page_write(struct buffer_page *bpage)
{
- return local_read(&bpage->write) & RB_WRITE_MASK;
+ return local_read_unchecked(&bpage->write) & RB_WRITE_MASK;
}
static int
* bytes consumed in ring buffer from here.
* Increment overrun to account for the lost events.
*/
- local_add(page_entries, &cpu_buffer->overrun);
+ local_add_unchecked(page_entries, &cpu_buffer->overrun);
local_sub(BUF_PAGE_SIZE, &cpu_buffer->entries_bytes);
}
* it is our responsibility to update
* the counters.
*/
- local_add(entries, &cpu_buffer->overrun);
+ local_add_unchecked(entries, &cpu_buffer->overrun);
local_sub(BUF_PAGE_SIZE, &cpu_buffer->entries_bytes);
/*
if (tail == BUF_PAGE_SIZE)
tail_page->real_end = 0;
- local_sub(length, &tail_page->write);
+ local_sub_unchecked(length, &tail_page->write);
return;
}
rb_event_set_padding(event);
/* Set the write back to the previous setting */
- local_sub(length, &tail_page->write);
+ local_sub_unchecked(length, &tail_page->write);
return;
}
/* Set write to end of buffer */
length = (tail + length) - BUF_PAGE_SIZE;
- local_sub(length, &tail_page->write);
+ local_sub_unchecked(length, &tail_page->write);
}
/*
* about it.
*/
if (unlikely(next_page == commit_page)) {
- local_inc(&cpu_buffer->commit_overrun);
+ local_inc_unchecked(&cpu_buffer->commit_overrun);
goto out_reset;
}
cpu_buffer->tail_page) &&
(cpu_buffer->commit_page ==
cpu_buffer->reader_page))) {
- local_inc(&cpu_buffer->commit_overrun);
+ local_inc_unchecked(&cpu_buffer->commit_overrun);
goto out_reset;
}
}
length += RB_LEN_TIME_EXTEND;
tail_page = cpu_buffer->tail_page;
- write = local_add_return(length, &tail_page->write);
+ write = local_add_return_unchecked(length, &tail_page->write);
/* set write to only the index of the write */
write &= RB_WRITE_MASK;
kmemcheck_annotate_bitfield(event, bitfield);
rb_update_event(cpu_buffer, event, length, add_timestamp, delta);
- local_inc(&tail_page->entries);
+ local_inc_unchecked(&tail_page->entries);
/*
* If this is the first commit on the page, then update
if (bpage->page == (void *)addr && rb_page_write(bpage) == old_index) {
unsigned long write_mask =
- local_read(&bpage->write) & ~RB_WRITE_MASK;
+ local_read_unchecked(&bpage->write) & ~RB_WRITE_MASK;
unsigned long event_length = rb_event_length(event);
/*
* This is on the tail page. It is possible that
*/
old_index += write_mask;
new_index += write_mask;
- index = local_cmpxchg(&bpage->write, old_index, new_index);
+ index = local_cmpxchg_unchecked(&bpage->write, old_index, new_index);
if (index == old_index) {
/* update counters */
local_sub(event_length, &cpu_buffer->entries_bytes);
/* Do the likely case first */
if (likely(bpage->page == (void *)addr)) {
- local_dec(&bpage->entries);
+ local_dec_unchecked(&bpage->entries);
return;
}
start = bpage;
do {
if (bpage->page == (void *)addr) {
- local_dec(&bpage->entries);
+ local_dec_unchecked(&bpage->entries);
return;
}
rb_inc_page(cpu_buffer, &bpage);
rb_num_of_entries(struct ring_buffer_per_cpu *cpu_buffer)
{
return local_read(&cpu_buffer->entries) -
- (local_read(&cpu_buffer->overrun) + cpu_buffer->read);
+ (local_read_unchecked(&cpu_buffer->overrun) + cpu_buffer->read);
}
/**
return 0;
cpu_buffer = buffer->buffers[cpu];
- ret = local_read(&cpu_buffer->overrun);
+ ret = local_read_unchecked(&cpu_buffer->overrun);
return ret;
}
return 0;
cpu_buffer = buffer->buffers[cpu];
- ret = local_read(&cpu_buffer->commit_overrun);
+ ret = local_read_unchecked(&cpu_buffer->commit_overrun);
return ret;
}
/* if you care about this being correct, lock the buffer */
for_each_buffer_cpu(buffer, cpu) {
cpu_buffer = buffer->buffers[cpu];
- overruns += local_read(&cpu_buffer->overrun);
+ overruns += local_read_unchecked(&cpu_buffer->overrun);
}
return overruns;
/*
* Reset the reader page to size zero.
*/
- local_set(&cpu_buffer->reader_page->write, 0);
- local_set(&cpu_buffer->reader_page->entries, 0);
+ local_set_unchecked(&cpu_buffer->reader_page->write, 0);
+ local_set_unchecked(&cpu_buffer->reader_page->entries, 0);
local_set(&cpu_buffer->reader_page->page->commit, 0);
cpu_buffer->reader_page->real_end = 0;
* want to compare with the last_overrun.
*/
smp_mb();
- overwrite = local_read(&(cpu_buffer->overrun));
+ overwrite = local_read_unchecked(&(cpu_buffer->overrun));
/*
* Here's the tricky part.
cpu_buffer->head_page
= list_entry(cpu_buffer->pages, struct buffer_page, list);
- local_set(&cpu_buffer->head_page->write, 0);
- local_set(&cpu_buffer->head_page->entries, 0);
+ local_set_unchecked(&cpu_buffer->head_page->write, 0);
+ local_set_unchecked(&cpu_buffer->head_page->entries, 0);
local_set(&cpu_buffer->head_page->page->commit, 0);
cpu_buffer->head_page->read = 0;
INIT_LIST_HEAD(&cpu_buffer->reader_page->list);
INIT_LIST_HEAD(&cpu_buffer->new_pages);
- local_set(&cpu_buffer->reader_page->write, 0);
- local_set(&cpu_buffer->reader_page->entries, 0);
+ local_set_unchecked(&cpu_buffer->reader_page->write, 0);
+ local_set_unchecked(&cpu_buffer->reader_page->entries, 0);
local_set(&cpu_buffer->reader_page->page->commit, 0);
cpu_buffer->reader_page->read = 0;
local_set(&cpu_buffer->entries_bytes, 0);
- local_set(&cpu_buffer->overrun, 0);
- local_set(&cpu_buffer->commit_overrun, 0);
+ local_set_unchecked(&cpu_buffer->overrun, 0);
+ local_set_unchecked(&cpu_buffer->commit_overrun, 0);
local_set(&cpu_buffer->dropped_events, 0);
local_set(&cpu_buffer->entries, 0);
local_set(&cpu_buffer->committing, 0);
rb_init_page(bpage);
bpage = reader->page;
reader->page = *data_page;
- local_set(&reader->write, 0);
- local_set(&reader->entries, 0);
+ local_set_unchecked(&reader->write, 0);
+ local_set_unchecked(&reader->entries, 0);
reader->read = 0;
*data_page = bpage;
return 0;
}
-int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
+int set_tracer_flag(struct trace_array *tr, unsigned long mask, int enabled)
{
/* do nothing if flag is already set */
if (!!(trace_flags & mask) == !!enabled)
void trace_printk_init_buffers(void);
void trace_printk_start_comm(void);
int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set);
-int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled);
+int set_tracer_flag(struct trace_array *tr, unsigned long mask, int enabled);
/*
* Normal trace_printk() and friends allocates special buffers
return now;
}
-static atomic64_t trace_counter;
+static atomic64_unchecked_t trace_counter;
/*
* trace_clock_counter(): simply an atomic counter.
*/
u64 notrace trace_clock_counter(void)
{
- return atomic64_add_return(1, &trace_counter);
+ return atomic64_inc_return_unchecked(&trace_counter);
}
return 0;
}
-struct ftrace_module_file_ops;
static void __add_event_to_tracers(struct ftrace_event_call *call);
/* Add an additional event_call dynamically */
/* The return trace stack is full */
if (current->curr_ret_stack == FTRACE_RETFUNC_DEPTH - 1) {
- atomic_inc(¤t->trace_overrun);
+ atomic_inc_unchecked(¤t->trace_overrun);
return -EBUSY;
}
*ret = current->ret_stack[index].ret;
trace->func = current->ret_stack[index].func;
trace->calltime = current->ret_stack[index].calltime;
- trace->overrun = atomic_read(¤t->trace_overrun);
+ trace->overrun = atomic_read_unchecked(¤t->trace_overrun);
trace->depth = index;
}
static struct trace_array *mmio_trace_array;
static bool overrun_detected;
static unsigned long prev_overruns;
-static atomic_t dropped_count;
+static atomic_unchecked_t dropped_count;
static void mmio_reset_data(struct trace_array *tr)
{
static unsigned long count_overruns(struct trace_iterator *iter)
{
- unsigned long cnt = atomic_xchg(&dropped_count, 0);
+ unsigned long cnt = atomic_xchg_unchecked(&dropped_count, 0);
unsigned long over = ring_buffer_overruns(iter->trace_buffer->buffer);
if (over > prev_overruns)
event = trace_buffer_lock_reserve(buffer, TRACE_MMIO_RW,
sizeof(*entry), 0, pc);
if (!event) {
- atomic_inc(&dropped_count);
+ atomic_inc_unchecked(&dropped_count);
return;
}
entry = ring_buffer_event_data(event);
event = trace_buffer_lock_reserve(buffer, TRACE_MMIO_MAP,
sizeof(*entry), 0, pc);
if (!event) {
- atomic_inc(&dropped_count);
+ atomic_inc_unchecked(&dropped_count);
return;
}
entry = ring_buffer_event_data(event);
goto out;
}
+ pax_open_kernel();
if (event->funcs->trace == NULL)
- event->funcs->trace = trace_nop_print;
+ *(void **)&event->funcs->trace = trace_nop_print;
if (event->funcs->raw == NULL)
- event->funcs->raw = trace_nop_print;
+ *(void **)&event->funcs->raw = trace_nop_print;
if (event->funcs->hex == NULL)
- event->funcs->hex = trace_nop_print;
+ *(void **)&event->funcs->hex = trace_nop_print;
if (event->funcs->binary == NULL)
- event->funcs->binary = trace_nop_print;
+ *(void **)&event->funcs->binary = trace_nop_print;
+ pax_close_kernel();
key = event->type & (EVENT_HASHSIZE - 1);
return 0;
}
- seq_buf_path(&s->seq, path, "\n");
+ seq_buf_path(&s->seq, path, "\n\\");
if (unlikely(seq_buf_has_overflowed(&s->seq))) {
s->seq.len = save_len;
return;
/* we do not handle interrupt stacks yet */
- if (!object_is_on_stack(stack))
+ if (!object_starts_on_stack(stack))
return;
local_irq_save(flags);
int num;
num = ((struct syscall_metadata *)call->data)->syscall_nr;
+ if (WARN_ON_ONCE(num < 0 || num >= NR_syscalls))
+ return -EINVAL;
mutex_lock(&syscall_trace_lock);
if (!sys_perf_refcount_enter)
int num;
num = ((struct syscall_metadata *)call->data)->syscall_nr;
+ if (WARN_ON_ONCE(num < 0 || num >= NR_syscalls))
+ return;
mutex_lock(&syscall_trace_lock);
sys_perf_refcount_enter--;
int num;
num = ((struct syscall_metadata *)call->data)->syscall_nr;
+ if (WARN_ON_ONCE(num < 0 || num >= NR_syscalls))
+ return -EINVAL;
mutex_lock(&syscall_trace_lock);
if (!sys_perf_refcount_exit)
int num;
num = ((struct syscall_metadata *)call->data)->syscall_nr;
+ if (WARN_ON_ONCE(num < 0 || num >= NR_syscalls))
+ return;
mutex_lock(&syscall_trace_lock);
sys_perf_refcount_exit--;
!kgid_has_mapping(parent_ns, group))
return -EPERM;
+#ifdef CONFIG_GRKERNSEC
+ /*
+ * This doesn't really inspire confidence:
+ * http://marc.info/?l=linux-kernel&m=135543612731939&w=2
+ * http://marc.info/?l=linux-kernel&m=135545831607095&w=2
+ * Increases kernel attack surface in areas developers
+ * previously cared little about ("low importance due
+ * to requiring "root" capability")
+ * To be removed when this code receives *proper* review
+ */
+ if (!capable(CAP_SYS_ADMIN) || !capable(CAP_SETUID) ||
+ !capable(CAP_SETGID))
+ return -EPERM;
+#endif
+
ns = kmem_cache_zalloc(user_ns_cachep, GFP_KERNEL);
if (!ns)
return -ENOMEM;
if (atomic_read(¤t->mm->mm_users) > 1)
return -EINVAL;
- if (current->fs->users != 1)
+ if (atomic_read(¤t->fs->users) != 1)
return -EINVAL;
if (!ns_capable(user_ns, CAP_SYS_ADMIN))
static int proc_do_uts_string(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp, loff_t *ppos)
{
- struct ctl_table uts_table;
+ ctl_table_no_const uts_table;
int r;
memcpy(&uts_table, table, sizeof(uts_table));
uts_table.data = get_uts(table, write);
static void watchdog_nmi_disable(unsigned int cpu) { return; }
#endif /* CONFIG_HARDLOCKUP_DETECTOR */
-static struct smp_hotplug_thread watchdog_threads = {
+static struct smp_hotplug_thread watchdog_threads __read_only = {
.store = &softlockup_watchdog,
.thread_should_run = watchdog_should_run,
.thread_fn = watchdog,
WARN_ON_ONCE(!(worker_flags & WORKER_UNBOUND));
worker_flags |= WORKER_REBOUND;
worker_flags &= ~WORKER_UNBOUND;
- ACCESS_ONCE(worker->flags) = worker_flags;
+ ACCESS_ONCE_RW(worker->flags) = worker_flags;
}
spin_unlock_irq(&pool->lock);
config DEBUG_WW_MUTEX_SLOWPATH
bool "Wait/wound mutex debugging: Slowpath testing"
- depends on DEBUG_KERNEL && TRACE_IRQFLAGS_SUPPORT && STACKTRACE_SUPPORT && LOCKDEP_SUPPORT
+ depends on DEBUG_KERNEL && TRACE_IRQFLAGS_SUPPORT && STACKTRACE_SUPPORT && LOCKDEP_SUPPORT && !PAX_CONSTIFY_PLUGIN
select DEBUG_LOCK_ALLOC
select DEBUG_SPINLOCK
select DEBUG_MUTEXES
config DEBUG_LOCK_ALLOC
bool "Lock debugging: detect incorrect freeing of live locks"
- depends on DEBUG_KERNEL && TRACE_IRQFLAGS_SUPPORT && STACKTRACE_SUPPORT && LOCKDEP_SUPPORT
+ depends on DEBUG_KERNEL && TRACE_IRQFLAGS_SUPPORT && STACKTRACE_SUPPORT && LOCKDEP_SUPPORT && !PAX_CONSTIFY_PLUGIN
select DEBUG_SPINLOCK
select DEBUG_MUTEXES
select LOCKDEP
config PROVE_LOCKING
bool "Lock debugging: prove locking correctness"
- depends on DEBUG_KERNEL && TRACE_IRQFLAGS_SUPPORT && STACKTRACE_SUPPORT && LOCKDEP_SUPPORT
+ depends on DEBUG_KERNEL && TRACE_IRQFLAGS_SUPPORT && STACKTRACE_SUPPORT && LOCKDEP_SUPPORT && !PAX_CONSTIFY_PLUGIN
select LOCKDEP
select DEBUG_SPINLOCK
select DEBUG_MUTEXES
config LOCK_STAT
bool "Lock usage statistics"
- depends on DEBUG_KERNEL && TRACE_IRQFLAGS_SUPPORT && STACKTRACE_SUPPORT && LOCKDEP_SUPPORT
+ depends on DEBUG_KERNEL && TRACE_IRQFLAGS_SUPPORT && STACKTRACE_SUPPORT && LOCKDEP_SUPPORT && !PAX_CONSTIFY_PLUGIN
select LOCKDEP
select DEBUG_SPINLOCK
select DEBUG_MUTEXES
depends on DEBUG_KERNEL
depends on STACKTRACE_SUPPORT
depends on PROC_FS
+ depends on !GRKERNSEC_HIDESYM
select FRAME_POINTER if !MIPS && !PPC && !S390 && !MICROBLAZE && !ARM_UNWIND && !ARC
select KALLSYMS
select KALLSYMS_ALL
config DEBUG_STRICT_USER_COPY_CHECKS
bool "Strict user copy size checks"
depends on ARCH_HAS_DEBUG_STRICT_USER_COPY_CHECKS
- depends on DEBUG_KERNEL && !TRACE_BRANCH_PROFILING
+ depends on DEBUG_KERNEL && !TRACE_BRANCH_PROFILING && !PAX_SIZE_OVERFLOW
help
Enabling this option turns a certain set of sanity checks for user
copy operations into compile time failures.
config PROVIDE_OHCI1394_DMA_INIT
bool "Remote debugging over FireWire early on boot"
- depends on PCI && X86
+ depends on PCI && X86 && !GRKERNSEC
help
If you want to debug problems which hang or crash the kernel early
on boot and the crashing machine has a FireWire port, you can use
obj-$(CONFIG_INTERVAL_TREE) += interval_tree.o
obj-$(CONFIG_ASSOCIATIVE_ARRAY) += assoc_array.o
obj-$(CONFIG_DEBUG_PREEMPT) += smp_processor_id.o
-obj-$(CONFIG_DEBUG_LIST) += list_debug.o
+obj-y += list_debug.o
obj-$(CONFIG_DEBUG_OBJECTS) += debugobjects.o
ifneq ($(CONFIG_HAVE_DEC_LOCK),y)
{
unsigned long internal = ACCESS_ONCE(avg->internal);
- ACCESS_ONCE(avg->internal) = internal ?
+ ACCESS_ONCE_RW(avg->internal) = internal ?
(((internal << avg->weight) - internal) +
(val << avg->factor)) >> avg->weight :
(val << avg->factor);
}
EXPORT_SYMBOL(__bitmap_subset);
-int __bitmap_weight(const unsigned long *bitmap, unsigned int bits)
+int __intentional_overflow(-1) __bitmap_weight(const unsigned long *bitmap, unsigned int bits)
{
unsigned int k, lim = bits/BITS_PER_LONG;
int w = 0;
{
int c, old_c, totaldigits, ndigits, nchunks, nbits;
u32 chunk;
- const char __user __force *ubuf = (const char __user __force *)buf;
+ const char __user *ubuf = (const char __force_user *)buf;
bitmap_zero(maskp, nmaskbits);
{
if (!access_ok(VERIFY_READ, ubuf, ulen))
return -EFAULT;
- return __bitmap_parse((const char __force *)ubuf,
+ return __bitmap_parse((const char __force_kernel *)ubuf,
ulen, 1, maskp, nmaskbits);
}
{
unsigned a, b;
int c, old_c, totaldigits;
- const char __user __force *ubuf = (const char __user __force *)buf;
+ const char __user *ubuf = (const char __force_user *)buf;
int exp_digit, in_range;
totaldigits = c = 0;
{
if (!access_ok(VERIFY_READ, ubuf, ulen))
return -EFAULT;
- return __bitmap_parselist((const char __force *)ubuf,
+ return __bitmap_parselist((const char __force_kernel *)ubuf,
ulen, 1, maskp, nmaskbits);
}
EXPORT_SYMBOL(bitmap_parselist_user);
return BUG_TRAP_TYPE_NONE;
bug = find_bug(bugaddr);
+ if (!bug)
+ return BUG_TRAP_TYPE_NONE;
file = NULL;
line = 0;
if (limit > 4)
return;
- is_on_stack = object_is_on_stack(addr);
+ is_on_stack = object_starts_on_stack(addr);
if (is_on_stack == onstack)
return;
EXPORT_SYMBOL(__div64_32);
#ifndef div_s64_rem
-s64 div_s64_rem(s64 dividend, s32 divisor, s32 *remainder)
+s64 __intentional_overflow(-1) div_s64_rem(s64 dividend, s32 divisor, s32 *remainder)
{
u64 quotient;
* 'http://www.hackersdelight.org/HDcode/newCode/divDouble.c.txt'
*/
#ifndef div64_u64
-u64 div64_u64(u64 dividend, u64 divisor)
+u64 __intentional_overflow(-1) div64_u64(u64 dividend, u64 divisor)
{
u32 high = divisor >> 32;
u64 quot;
void dma_debug_add_bus(struct bus_type *bus)
{
- struct notifier_block *nb;
+ notifier_block_no_const *nb;
if (dma_debug_disabled())
return;
static void check_for_stack(struct device *dev, void *addr)
{
- if (object_is_on_stack(addr))
+ if (object_starts_on_stack(addr))
err_printk(dev, NULL, "DMA-API: device driver maps memory from "
"stack [addr=%p]\n", addr);
}
malloc_ptr = free_mem_ptr;
}
#else
-#define malloc(a) kmalloc(a, GFP_KERNEL)
+#define malloc(a) kmalloc((a), GFP_KERNEL)
#define free(a) kfree(a)
#endif
unsigned long next;
phys_addr -= addr;
- pmd = pmd_alloc(&init_mm, pud, addr);
+ pmd = pmd_alloc_kernel(&init_mm, pud, addr);
if (!pmd)
return -ENOMEM;
do {
unsigned long next;
phys_addr -= addr;
- pud = pud_alloc(&init_mm, pgd, addr);
+ pud = pud_alloc_kernel(&init_mm, pgd, addr);
if (!pud)
return -ENOMEM;
do {
struct task_struct *p, *t;
bool ret;
+ if (!mm)
+ return true;
+
if (atomic_read(&task->signal->live) != 1)
return false;
static DEFINE_SPINLOCK(kobj_ns_type_lock);
-static const struct kobj_ns_type_operations *kobj_ns_ops_tbl[KOBJ_NS_TYPES];
+static const struct kobj_ns_type_operations *kobj_ns_ops_tbl[KOBJ_NS_TYPES] __read_only;
-int kobj_ns_type_register(const struct kobj_ns_type_operations *ops)
+int __init kobj_ns_type_register(const struct kobj_ns_type_operations *ops)
{
enum kobj_ns_type type = ops->type;
int error;
#include <linux/bug.h>
#include <linux/kernel.h>
#include <linux/rculist.h>
+#include <linux/mm.h>
+#ifdef CONFIG_DEBUG_LIST
/*
* Insert a new entry between two known consecutive entries.
*
* the prev/next entries already!
*/
+static bool __list_add_debug(struct list_head *new,
+ struct list_head *prev,
+ struct list_head *next)
+{
+ if (unlikely(next->prev != prev)) {
+ printk(KERN_ERR "list_add corruption. next->prev should be "
+ "prev (%p), but was %p. (next=%p).\n",
+ prev, next->prev, next);
+ BUG();
+ return false;
+ }
+ if (unlikely(prev->next != next)) {
+ printk(KERN_ERR "list_add corruption. prev->next should be "
+ "next (%p), but was %p. (prev=%p).\n",
+ next, prev->next, prev);
+ BUG();
+ return false;
+ }
+ if (unlikely(new == prev || new == next)) {
+ printk(KERN_ERR "list_add double add: new=%p, prev=%p, next=%p.\n",
+ new, prev, next);
+ BUG();
+ return false;
+ }
+ return true;
+}
+
void __list_add(struct list_head *new,
- struct list_head *prev,
- struct list_head *next)
+ struct list_head *prev,
+ struct list_head *next)
{
- WARN(next->prev != prev,
- "list_add corruption. next->prev should be "
- "prev (%p), but was %p. (next=%p).\n",
- prev, next->prev, next);
- WARN(prev->next != next,
- "list_add corruption. prev->next should be "
- "next (%p), but was %p. (prev=%p).\n",
- next, prev->next, prev);
- WARN(new == prev || new == next,
- "list_add double add: new=%p, prev=%p, next=%p.\n",
- new, prev, next);
+ if (!__list_add_debug(new, prev, next))
+ return;
+
next->prev = new;
new->next = next;
new->prev = prev;
}
EXPORT_SYMBOL(__list_add);
-void __list_del_entry(struct list_head *entry)
+static bool __list_del_entry_debug(struct list_head *entry)
{
struct list_head *prev, *next;
prev = entry->prev;
next = entry->next;
- if (WARN(next == LIST_POISON1,
- "list_del corruption, %p->next is LIST_POISON1 (%p)\n",
- entry, LIST_POISON1) ||
- WARN(prev == LIST_POISON2,
- "list_del corruption, %p->prev is LIST_POISON2 (%p)\n",
- entry, LIST_POISON2) ||
- WARN(prev->next != entry,
- "list_del corruption. prev->next should be %p, "
- "but was %p\n", entry, prev->next) ||
- WARN(next->prev != entry,
- "list_del corruption. next->prev should be %p, "
- "but was %p\n", entry, next->prev))
+ if (unlikely(next == LIST_POISON1)) {
+ printk(KERN_ERR "list_del corruption, %p->next is LIST_POISON1 (%p)\n",
+ entry, LIST_POISON1);
+ BUG();
+ return false;
+ }
+ if (unlikely(prev == LIST_POISON2)) {
+ printk(KERN_ERR "list_del corruption, %p->prev is LIST_POISON2 (%p)\n",
+ entry, LIST_POISON2);
+ BUG();
+ return false;
+ }
+ if (unlikely(entry->prev->next != entry)) {
+ printk(KERN_ERR "list_del corruption. prev->next should be %p, "
+ "but was %p\n", entry, prev->next);
+ BUG();
+ return false;
+ }
+ if (unlikely(entry->next->prev != entry)) {
+ printk(KERN_ERR "list_del corruption. next->prev should be %p, "
+ "but was %p\n", entry, next->prev);
+ BUG();
+ return false;
+ }
+ return true;
+}
+
+void __list_del_entry(struct list_head *entry)
+{
+ if (!__list_del_entry_debug(entry))
return;
- __list_del(prev, next);
+ __list_del(entry->prev, entry->next);
}
EXPORT_SYMBOL(__list_del_entry);
void __list_add_rcu(struct list_head *new,
struct list_head *prev, struct list_head *next)
{
- WARN(next->prev != prev,
- "list_add_rcu corruption. next->prev should be prev (%p), but was %p. (next=%p).\n",
- prev, next->prev, next);
- WARN(prev->next != next,
- "list_add_rcu corruption. prev->next should be next (%p), but was %p. (prev=%p).\n",
- next, prev->next, prev);
+ if (!__list_add_debug(new, prev, next))
+ return;
+
new->next = next;
new->prev = prev;
rcu_assign_pointer(list_next_rcu(prev), new);
next->prev = new;
}
EXPORT_SYMBOL(__list_add_rcu);
+#endif
+
+void __pax_list_add(struct list_head *new, struct list_head *prev, struct list_head *next)
+{
+#ifdef CONFIG_DEBUG_LIST
+ if (!__list_add_debug(new, prev, next))
+ return;
+#endif
+
+ pax_open_kernel();
+ next->prev = new;
+ new->next = next;
+ new->prev = prev;
+ prev->next = new;
+ pax_close_kernel();
+}
+EXPORT_SYMBOL(__pax_list_add);
+
+void pax_list_del(struct list_head *entry)
+{
+#ifdef CONFIG_DEBUG_LIST
+ if (!__list_del_entry_debug(entry))
+ return;
+#endif
+
+ pax_open_kernel();
+ __list_del(entry->prev, entry->next);
+ entry->next = LIST_POISON1;
+ entry->prev = LIST_POISON2;
+ pax_close_kernel();
+}
+EXPORT_SYMBOL(pax_list_del);
+
+void pax_list_del_init(struct list_head *entry)
+{
+ pax_open_kernel();
+ __list_del(entry->prev, entry->next);
+ INIT_LIST_HEAD(entry);
+ pax_close_kernel();
+}
+EXPORT_SYMBOL(pax_list_del_init);
+
+void __pax_list_add_rcu(struct list_head *new,
+ struct list_head *prev, struct list_head *next)
+{
+#ifdef CONFIG_DEBUG_LIST
+ if (!__list_add_debug(new, prev, next))
+ return;
+#endif
+
+ pax_open_kernel();
+ new->next = next;
+ new->prev = prev;
+ rcu_assign_pointer(list_next_rcu(prev), new);
+ next->prev = new;
+ pax_close_kernel();
+}
+EXPORT_SYMBOL(__pax_list_add_rcu);
+
+void pax_list_del_rcu(struct list_head *entry)
+{
+#ifdef CONFIG_DEBUG_LIST
+ if (!__list_del_entry_debug(entry))
+ return;
+#endif
+
+ pax_open_kernel();
+ __list_del(entry->prev, entry->next);
+ entry->next = LIST_POISON1;
+ entry->prev = LIST_POISON2;
+ pax_close_kernel();
+}
+EXPORT_SYMBOL(pax_list_del_rcu);
void lockref_get(struct lockref *lockref)
{
CMPXCHG_LOOP(
- new.count++;
+ __lockref_inc(&new);
,
return;
);
spin_lock(&lockref->lock);
- lockref->count++;
+ __lockref_inc(lockref);
spin_unlock(&lockref->lock);
}
EXPORT_SYMBOL(lockref_get);
int retval;
CMPXCHG_LOOP(
- new.count++;
+ __lockref_inc(&new);
if (!old.count)
return 0;
,
spin_lock(&lockref->lock);
retval = 0;
if (lockref->count) {
- lockref->count++;
+ __lockref_inc(lockref);
retval = 1;
}
spin_unlock(&lockref->lock);
int lockref_get_or_lock(struct lockref *lockref)
{
CMPXCHG_LOOP(
- new.count++;
+ __lockref_inc(&new);
if (!old.count)
break;
,
spin_lock(&lockref->lock);
if (!lockref->count)
return 0;
- lockref->count++;
+ __lockref_inc(lockref);
spin_unlock(&lockref->lock);
return 1;
}
int lockref_put_or_lock(struct lockref *lockref)
{
CMPXCHG_LOOP(
- new.count--;
+ __lockref_dec(&new);
if (old.count <= 1)
break;
,
spin_lock(&lockref->lock);
if (lockref->count <= 1)
return 0;
- lockref->count--;
+ __lockref_dec(lockref);
spin_unlock(&lockref->lock);
return 1;
}
int retval;
CMPXCHG_LOOP(
- new.count++;
+ __lockref_inc(&new);
if ((int)old.count < 0)
return 0;
,
spin_lock(&lockref->lock);
retval = 0;
if ((int) lockref->count >= 0) {
- lockref->count++;
+ __lockref_inc(lockref);
retval = 1;
}
spin_unlock(&lockref->lock);
* atomic_long_t can't hit 0 before we've added up all the percpu refs.
*/
-#define PERCPU_COUNT_BIAS (1LU << (BITS_PER_LONG - 1))
+#define PERCPU_COUNT_BIAS (1LU << (BITS_PER_LONG - 2))
static DECLARE_WAIT_QUEUE_HEAD(percpu_ref_switch_waitq);
int nr;
struct radix_tree_node *nodes[RADIX_TREE_PRELOAD_SIZE];
};
-static DEFINE_PER_CPU(struct radix_tree_preload, radix_tree_preloads) = { 0, };
+static DEFINE_PER_CPU(struct radix_tree_preload, radix_tree_preloads);
static inline void *ptr_to_indirect(void *ptr)
{
}
#endif
-static DEFINE_PER_CPU(struct rnd_state, net_rand_state);
+static DEFINE_PER_CPU(struct rnd_state, net_rand_state) __latent_entropy;
/**
* prandom_u32_state - seeded pseudo-random number generator.
static inline void dummy_rotate(struct rb_node *old, struct rb_node *new) {}
static const struct rb_augment_callbacks dummy_callbacks = {
- dummy_propagate, dummy_copy, dummy_rotate
+ .propagate = dummy_propagate,
+ .copy = dummy_copy,
+ .rotate = dummy_rotate
};
void rb_insert_color(struct rb_node *node, struct rb_root *root)
quicklist_total_size());
#endif
#ifdef CONFIG_MEMORY_FAILURE
- printk("%lu pages hwpoisoned\n", atomic_long_read(&num_poisoned_pages));
+ printk("%lu pages hwpoisoned\n", atomic_long_read_unchecked(&num_poisoned_pages));
#endif
}
*/
static inline long do_strncpy_from_user(char *dst, const char __user *src, long count, unsigned long max)
{
- const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS;
+ static const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS;
long res = 0;
/*
*/
static inline long do_strnlen_user(const char __user *src, unsigned long count, unsigned long max)
{
- const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS;
+ static const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS;
long align, res = 0;
unsigned long c;
void
swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr,
- dma_addr_t dev_addr)
+ dma_addr_t dev_addr, struct dma_attrs *attrs)
{
phys_addr_t paddr = dma_to_phys(hwdev, dev_addr);
WARN(1, "Buffer overflow detected!\n");
}
EXPORT_SYMBOL(copy_from_user_overflow);
+
+void copy_to_user_overflow(void)
+{
+ WARN(1, "Buffer overflow detected!\n");
+}
+EXPORT_SYMBOL(copy_to_user_overflow);
* - scnprintf and vscnprintf
*/
+#ifdef CONFIG_GRKERNSEC_HIDESYM
+#define __INCLUDED_BY_HIDESYM 1
+#endif
#include <stdarg.h>
#include <linux/module.h> /* for KSYM_SYMBOL_LEN */
#include <linux/types.h>
#ifdef CONFIG_KALLSYMS
if (*fmt == 'B')
sprint_backtrace(sym, value);
- else if (*fmt != 'f' && *fmt != 's')
+ else if (*fmt != 'f' && *fmt != 's' && *fmt != 'X')
sprint_symbol(sym, value);
else
sprint_symbol_no_offset(sym, value);
return number(buf, end, num, spec);
}
+#ifdef CONFIG_GRKERNSEC_HIDESYM
+int kptr_restrict __read_mostly = 2;
+#else
int kptr_restrict __read_mostly;
+#endif
/*
* Show a '%p' thing. A kernel extension is that the '%p' is followed
*
* - 'F' For symbolic function descriptor pointers with offset
* - 'f' For simple symbolic function names without offset
+ * - 'X' For simple symbolic function names without offset approved for use with GRKERNSEC_HIDESYM
* - 'S' For symbolic direct pointers with offset
* - 's' For symbolic direct pointers without offset
+ * - 'A' For symbolic direct pointers with offset approved for use with GRKERNSEC_HIDESYM
* - '[FfSs]R' as above with __builtin_extract_return_addr() translation
* - 'B' For backtraced symbolic direct pointers with offset
* - 'R' For decoded struct resource, e.g., [mem 0x0-0x1f 64bit pref]
if (!ptr && *fmt != 'K') {
/*
- * Print (null) with the same width as a pointer so it makes
+ * Print (nil) with the same width as a pointer so it makes
* tabular output look nice.
*/
if (spec.field_width == -1)
spec.field_width = default_width;
- return string(buf, end, "(null)", spec);
+ return string(buf, end, "(nil)", spec);
}
switch (*fmt) {
/* Fallthrough */
case 'S':
case 's':
+#ifdef CONFIG_GRKERNSEC_HIDESYM
+ break;
+#else
+ return symbol_string(buf, end, ptr, spec, fmt);
+#endif
+ case 'X':
+ ptr = dereference_function_descriptor(ptr);
+ case 'A':
case 'B':
return symbol_string(buf, end, ptr, spec, fmt);
case 'R':
va_end(va);
return buf;
}
+ case 'P':
+ break;
case 'K':
/*
* %pK cannot be used in IRQ context because its test
((const struct file *)ptr)->f_path.dentry,
spec, fmt);
}
+
+#ifdef CONFIG_GRKERNSEC_HIDESYM
+ /* 'P' = approved pointers to copy to userland,
+ as in the /proc/kallsyms case, as we make it display nothing
+ for non-root users, and the real contents for root users
+ 'X' = approved simple symbols
+ Also ignore 'K' pointers, since we force their NULLing for non-root users
+ above
+ */
+ if ((unsigned long)ptr > TASK_SIZE && *fmt != 'P' && *fmt != 'X' && *fmt != 'K' && is_usercopy_object(buf)) {
+ printk(KERN_ALERT "grsec: kernel infoleak detected! Please report this log to spender@grsecurity.net.\n");
+ dump_stack();
+ ptr = NULL;
+ }
+#endif
+
spec.flags |= SMALL;
if (spec.field_width == -1) {
spec.field_width = default_width;
typeof(type) value; \
if (sizeof(type) == 8) { \
args = PTR_ALIGN(args, sizeof(u32)); \
- *(u32 *)&value = *(u32 *)args; \
- *((u32 *)&value + 1) = *(u32 *)(args + 4); \
+ *(u32 *)&value = *(const u32 *)args; \
+ *((u32 *)&value + 1) = *(const u32 *)(args + 4); \
} else { \
args = PTR_ALIGN(args, sizeof(type)); \
- value = *(typeof(type) *)args; \
+ value = *(const typeof(type) *)args; \
} \
args += sizeof(type); \
value; \
case FORMAT_TYPE_STR: {
const char *str_arg = args;
args += strlen(str_arg) + 1;
- str = string(str, end, (char *)str_arg, spec);
+ str = string(str, end, str_arg, spec);
break;
}
root has set /sys/kernel/mm/ksm/run to 1 (if CONFIG_SYSFS is set).
config DEFAULT_MMAP_MIN_ADDR
- int "Low address space to protect from user allocation"
+ int "Low address space to protect from user allocation"
depends on MMU
- default 4096
- help
+ default 32768 if ALPHA || ARM || PARISC || SPARC32
+ default 65536
+ help
This is the portion of low virtual memory which should be protected
from userspace allocation. Keeping a user from writing to low pages
can help reduce the impact of kernel NULL pointer bugs.
config HWPOISON_INJECT
tristate "HWPoison pages injector"
- depends on MEMORY_FAILURE && DEBUG_KERNEL && PROC_FS
+ depends on MEMORY_FAILURE && DEBUG_KERNEL && PROC_FS && !GRKERNSEC
select PROC_PAGE_MONITOR
config NOMMU_INITIAL_TRIM_EXCESS
#include <linux/device.h>
#include <trace/events/writeback.h>
-static atomic_long_t bdi_seq = ATOMIC_LONG_INIT(0);
+static atomic_long_unchecked_t bdi_seq = ATOMIC_LONG_INIT(0);
struct backing_dev_info default_backing_dev_info = {
.name = "default",
return err;
err = bdi_register(bdi, NULL, "%.28s-%ld", name,
- atomic_long_inc_return(&bdi_seq));
+ atomic_long_inc_return_unchecked(&bdi_seq));
if (err) {
bdi_destroy(bdi);
return err;
struct address_space *mapping = file->f_mapping;
if (!mapping->a_ops->readpage)
- return -ENOEXEC;
+ return -ENODEV;
file_accessed(file);
vma->vm_ops = &generic_file_vm_ops;
return 0;
*pos = i_size_read(inode);
if (limit != RLIM_INFINITY) {
+ gr_learn_resource(current, RLIMIT_FSIZE,*pos, 0);
if (*pos >= limit) {
send_sig(SIGXFSZ, current, 0);
return -EFBIG;
retry:
vma = find_vma(mm, start);
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (vma && (mm->pax_flags & MF_PAX_SEGMEXEC) && (vma->vm_flags & VM_MAYEXEC))
+ goto out;
+#endif
+
/*
* Make sure the vma is shared, that it supports prefaulting,
* and that the remapped range is valid and fully within
unsigned int fault_flags = 0;
int ret;
- /* For mlock, just skip the stack guard page. */
- if ((*flags & FOLL_MLOCK) &&
- (stack_guard_page_start(vma, address) ||
- stack_guard_page_end(vma, address + PAGE_SIZE)))
- return -ENOENT;
if (*flags & FOLL_WRITE)
fault_flags |= FAULT_FLAG_WRITE;
if (nonblocking)
if (!(gup_flags & FOLL_FORCE))
gup_flags |= FOLL_NUMA;
- do {
+ while (nr_pages) {
struct page *page;
unsigned int foll_flags = gup_flags;
unsigned int page_increm;
/* first iteration or cross vma bound */
if (!vma || start >= vma->vm_end) {
- vma = find_extend_vma(mm, start);
+ vma = find_vma(mm, start);
if (!vma && in_gate_area(mm, start)) {
int ret;
ret = get_gate_page(mm, start & PAGE_MASK,
goto next_page;
}
- if (!vma || check_vma_flags(vma, gup_flags))
+ if (!vma || start < vma->vm_start || check_vma_flags(vma, gup_flags))
return i ? : -EFAULT;
if (is_vm_hugetlb_page(vma)) {
i = follow_hugetlb_page(mm, vma, pages, vmas,
i += page_increm;
start += page_increm * PAGE_SIZE;
nr_pages -= page_increm;
- } while (nr_pages);
+ }
return i;
}
EXPORT_SYMBOL(__get_user_pages);
* So no dangers, even with speculative execution.
*/
page = pte_page(pkmap_page_table[i]);
+ pax_open_kernel();
pte_clear(&init_mm, PKMAP_ADDR(i), &pkmap_page_table[i]);
-
+ pax_close_kernel();
set_page_address(page, NULL);
need_flush = 1;
}
}
}
vaddr = PKMAP_ADDR(last_pkmap_nr);
+
+ pax_open_kernel();
set_pte_at(&init_mm, vaddr,
&(pkmap_page_table[last_pkmap_nr]), mk_pte(page, kmap_prot));
-
+ pax_close_kernel();
pkmap_count[last_pkmap_nr] = 1;
set_page_address(page, (void *)vaddr);
struct ctl_table *table, int write,
void __user *buffer, size_t *length, loff_t *ppos)
{
+ ctl_table_no_const t;
struct hstate *h = &default_hstate;
unsigned long tmp = h->max_huge_pages;
int ret;
if (!hugepages_supported())
return -ENOTSUPP;
- table->data = &tmp;
- table->maxlen = sizeof(unsigned long);
- ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
+ t = *table;
+ t.data = &tmp;
+ t.maxlen = sizeof(unsigned long);
+ ret = proc_doulongvec_minmax(&t, write, buffer, length, ppos);
if (ret)
goto out;
struct hstate *h = &default_hstate;
unsigned long tmp;
int ret;
+ ctl_table_no_const hugetlb_table;
if (!hugepages_supported())
return -ENOTSUPP;
if (write && hstate_is_gigantic(h))
return -EINVAL;
- table->data = &tmp;
- table->maxlen = sizeof(unsigned long);
- ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
+ hugetlb_table = *table;
+ hugetlb_table.data = &tmp;
+ hugetlb_table.maxlen = sizeof(unsigned long);
+ ret = proc_doulongvec_minmax(&hugetlb_table, write, buffer, length, ppos);
if (ret)
goto out;
i_mmap_unlock_write(mapping);
}
+#ifdef CONFIG_PAX_SEGMEXEC
+static void pax_mirror_huge_pte(struct vm_area_struct *vma, unsigned long address, struct page *page_m)
+{
+ struct mm_struct *mm = vma->vm_mm;
+ struct vm_area_struct *vma_m;
+ unsigned long address_m;
+ pte_t *ptep_m;
+
+ vma_m = pax_find_mirror_vma(vma);
+ if (!vma_m)
+ return;
+
+ BUG_ON(address >= SEGMEXEC_TASK_SIZE);
+ address_m = address + SEGMEXEC_TASK_SIZE;
+ ptep_m = huge_pte_offset(mm, address_m & HPAGE_MASK);
+ get_page(page_m);
+ hugepage_add_anon_rmap(page_m, vma_m, address_m);
+ set_huge_pte_at(mm, address_m, ptep_m, make_huge_pte(vma_m, page_m, 0));
+}
+#endif
+
/*
* Hugetlb_cow() should be called with page lock of the original hugepage held.
* Called with hugetlb_instantiation_mutex held and pte_page locked so we
make_huge_pte(vma, new_page, 1));
page_remove_rmap(old_page);
hugepage_add_new_anon_rmap(new_page, vma, address);
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ pax_mirror_huge_pte(vma, address, new_page);
+#endif
+
/* Make the old page be freed below */
new_page = old_page;
}
&& (vma->vm_flags & VM_SHARED)));
set_huge_pte_at(mm, address, ptep, new_pte);
+#ifdef CONFIG_PAX_SEGMEXEC
+ pax_mirror_huge_pte(vma, address, page);
+#endif
+
if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
/* Optimization, do the COW without a second fault */
ret = hugetlb_cow(mm, vma, address, ptep, new_pte, page, ptl);
struct address_space *mapping;
int need_wait_lock = 0;
+#ifdef CONFIG_PAX_SEGMEXEC
+ struct vm_area_struct *vma_m;
+#endif
+
address &= huge_page_mask(h);
ptep = huge_pte_offset(mm, address);
VM_FAULT_SET_HINDEX(hstate_index(h));
}
+#ifdef CONFIG_PAX_SEGMEXEC
+ vma_m = pax_find_mirror_vma(vma);
+ if (vma_m) {
+ unsigned long address_m;
+
+ if (vma->vm_start > vma_m->vm_start) {
+ address_m = address;
+ address -= SEGMEXEC_TASK_SIZE;
+ vma = vma_m;
+ h = hstate_vma(vma);
+ } else
+ address_m = address + SEGMEXEC_TASK_SIZE;
+
+ if (!huge_pte_alloc(mm, address_m, huge_page_size(h)))
+ return VM_FAULT_OOM;
+ address_m &= HPAGE_MASK;
+ unmap_hugepage_range(vma, address_m, address_m + HPAGE_SIZE, NULL);
+ }
+#endif
+
ptep = huge_pte_alloc(mm, address, huge_page_size(h));
if (!ptep)
return VM_FAULT_OOM;
extern int __isolate_free_page(struct page *page, unsigned int order);
extern void __free_pages_bootmem(struct page *page, unsigned int order);
+extern void free_compound_page(struct page *page);
extern void prep_compound_page(struct page *page, unsigned long order);
#ifdef CONFIG_MEMORY_FAILURE
extern bool is_free_buddy_page(struct page *page);
extern unsigned long vm_mmap_pgoff(struct file *, unsigned long,
unsigned long, unsigned long,
- unsigned long, unsigned long);
+ unsigned long, unsigned long) __intentional_overflow(-1);
extern void set_pageblock_order(void);
unsigned long reclaim_clean_pages_from_list(struct zone *zone,
for (i = 0; i < object->trace_len; i++) {
void *ptr = (void *)object->trace[i];
- seq_printf(seq, " [<%p>] %pS\n", ptr, ptr);
+ seq_printf(seq, " [<%pP>] %pA\n", ptr, ptr);
}
}
return -ENOMEM;
}
- dentry = debugfs_create_file("kmemleak", S_IRUGO, NULL, NULL,
+ dentry = debugfs_create_file("kmemleak", S_IRUSR, NULL, NULL,
&kmemleak_fops);
if (!dentry)
pr_warning("Failed to create the debugfs kmemleak file\n");
set_fs(KERNEL_DS);
pagefault_disable();
ret = __copy_from_user_inatomic(dst,
- (__force const void __user *)src, size);
+ (const void __force_user *)src, size);
pagefault_enable();
set_fs(old_fs);
set_fs(KERNEL_DS);
pagefault_disable();
- ret = __copy_to_user_inatomic((__force void __user *)dst, src, size);
+ ret = __copy_to_user_inatomic((void __force_user *)dst, src, size);
pagefault_enable();
set_fs(old_fs);
pgoff_t pgoff;
unsigned long new_flags = vma->vm_flags;
+#ifdef CONFIG_PAX_SEGMEXEC
+ struct vm_area_struct *vma_m;
+#endif
+
switch (behavior) {
case MADV_NORMAL:
new_flags = new_flags & ~VM_RAND_READ & ~VM_SEQ_READ;
/*
* vm_flags is protected by the mmap_sem held in write mode.
*/
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ vma_m = pax_find_mirror_vma(vma);
+ if (vma_m)
+ vma_m->vm_flags = new_flags & ~(VM_WRITE | VM_MAYWRITE | VM_ACCOUNT);
+#endif
+
vma->vm_flags = new_flags;
out:
struct vm_area_struct **prev,
unsigned long start, unsigned long end)
{
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ struct vm_area_struct *vma_m;
+#endif
+
*prev = vma;
if (vma->vm_flags & (VM_LOCKED|VM_HUGETLB|VM_PFNMAP))
return -EINVAL;
zap_page_range(vma, start, end - start, &details);
} else
zap_page_range(vma, start, end - start, NULL);
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ vma_m = pax_find_mirror_vma(vma);
+ if (vma_m) {
+ if (unlikely(vma->vm_flags & VM_NONLINEAR)) {
+ struct zap_details details = {
+ .nonlinear_vma = vma_m,
+ .last_index = ULONG_MAX,
+ };
+ zap_page_range(vma, start + SEGMEXEC_TASK_SIZE, end - start, &details);
+ } else
+ zap_page_range(vma, start + SEGMEXEC_TASK_SIZE, end - start, NULL);
+ }
+#endif
+
return 0;
}
if (end < start)
return error;
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (current->mm->pax_flags & MF_PAX_SEGMEXEC) {
+ if (end > SEGMEXEC_TASK_SIZE)
+ return error;
+ } else
+#endif
+
+ if (end > TASK_SIZE)
+ return error;
+
error = 0;
if (end == start)
return error;
int sysctl_memory_failure_recovery __read_mostly = 1;
-atomic_long_t num_poisoned_pages __read_mostly = ATOMIC_LONG_INIT(0);
+atomic_long_unchecked_t num_poisoned_pages __read_mostly = ATOMIC_LONG_INIT(0);
#if defined(CONFIG_HWPOISON_INJECT) || defined(CONFIG_HWPOISON_INJECT_MODULE)
pfn, t->comm, t->pid);
si.si_signo = SIGBUS;
si.si_errno = 0;
- si.si_addr = (void *)addr;
+ si.si_addr = (void __user *)addr;
#ifdef __ARCH_SI_TRAPNO
si.si_trapno = trapno;
#endif
unsigned long res;
char *msg;
int (*action)(struct page *p, unsigned long pfn);
-} error_states[] = {
+} __do_const error_states[] = {
{ reserved, reserved, "reserved kernel", me_kernel },
/*
* free pages are specially detected outside this table:
nr_pages = 1 << compound_order(hpage);
else /* normal page or thp */
nr_pages = 1;
- atomic_long_add(nr_pages, &num_poisoned_pages);
+ atomic_long_add_unchecked(nr_pages, &num_poisoned_pages);
/*
* We need/can do nothing about count=0 pages.
if (PageHWPoison(hpage)) {
if ((hwpoison_filter(p) && TestClearPageHWPoison(p))
|| (p != hpage && TestSetPageHWPoison(hpage))) {
- atomic_long_sub(nr_pages, &num_poisoned_pages);
+ atomic_long_sub_unchecked(nr_pages, &num_poisoned_pages);
unlock_page(hpage);
return 0;
}
*/
if (!PageHWPoison(p)) {
printk(KERN_ERR "MCE %#lx: just unpoisoned\n", pfn);
- atomic_long_sub(nr_pages, &num_poisoned_pages);
+ atomic_long_sub_unchecked(nr_pages, &num_poisoned_pages);
put_page(hpage);
res = 0;
goto out;
}
if (hwpoison_filter(p)) {
if (TestClearPageHWPoison(p))
- atomic_long_sub(nr_pages, &num_poisoned_pages);
+ atomic_long_sub_unchecked(nr_pages, &num_poisoned_pages);
unlock_page(hpage);
put_page(hpage);
return 0;
return 0;
}
if (TestClearPageHWPoison(p))
- atomic_long_dec(&num_poisoned_pages);
+ atomic_long_dec_unchecked(&num_poisoned_pages);
pr_info("MCE: Software-unpoisoned free page %#lx\n", pfn);
return 0;
}
*/
if (TestClearPageHWPoison(page)) {
pr_info("MCE: Software-unpoisoned page %#lx\n", pfn);
- atomic_long_sub(nr_pages, &num_poisoned_pages);
+ atomic_long_sub_unchecked(nr_pages, &num_poisoned_pages);
freeit = 1;
if (PageHuge(page))
clear_page_hwpoison_huge_page(page);
if (PageHuge(page)) {
set_page_hwpoison_huge_page(hpage);
dequeue_hwpoisoned_huge_page(hpage);
- atomic_long_add(1 << compound_order(hpage),
+ atomic_long_add_unchecked(1 << compound_order(hpage),
&num_poisoned_pages);
} else {
SetPageHWPoison(page);
- atomic_long_inc(&num_poisoned_pages);
+ atomic_long_inc_unchecked(&num_poisoned_pages);
}
}
return ret;
put_page(page);
pr_info("soft_offline: %#lx: invalidated\n", pfn);
SetPageHWPoison(page);
- atomic_long_inc(&num_poisoned_pages);
+ atomic_long_inc_unchecked(&num_poisoned_pages);
return 0;
}
if (!is_free_buddy_page(page))
pr_info("soft offline: %#lx: page leaked\n",
pfn);
- atomic_long_inc(&num_poisoned_pages);
+ atomic_long_inc_unchecked(&num_poisoned_pages);
}
} else {
pr_info("soft offline: %#lx: isolation failed: %d, page count %d, type %lx\n",
if (PageHuge(page)) {
set_page_hwpoison_huge_page(hpage);
dequeue_hwpoisoned_huge_page(hpage);
- atomic_long_add(1 << compound_order(hpage),
+ atomic_long_add_unchecked(1 << compound_order(hpage),
&num_poisoned_pages);
} else {
SetPageHWPoison(page);
- atomic_long_inc(&num_poisoned_pages);
+ atomic_long_inc_unchecked(&num_poisoned_pages);
}
}
unset_migratetype_isolate(page, MIGRATE_MOVABLE);
free_pte_range(tlb, pmd, addr);
} while (pmd++, addr = next, addr != end);
+#if !defined(CONFIG_X86_32) || !defined(CONFIG_PAX_PER_CPU_PGD)
start &= PUD_MASK;
if (start < floor)
return;
pmd = pmd_offset(pud, start);
pud_clear(pud);
pmd_free_tlb(tlb, pmd, start);
+#endif
+
}
static inline void free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
free_pmd_range(tlb, pud, addr, next, floor, ceiling);
} while (pud++, addr = next, addr != end);
+#if !defined(CONFIG_X86_64) || !defined(CONFIG_PAX_PER_CPU_PGD)
start &= PGDIR_MASK;
if (start < floor)
return;
pud = pud_offset(pgd, start);
pgd_clear(pgd);
pud_free_tlb(tlb, pud, start);
+#endif
+
}
/*
* Choose text because data symbols depend on CONFIG_KALLSYMS_ALL=y
*/
if (vma->vm_ops)
- printk(KERN_ALERT "vma->vm_ops->fault: %pSR\n",
+ printk(KERN_ALERT "vma->vm_ops->fault: %pAR\n",
vma->vm_ops->fault);
if (vma->vm_file)
- printk(KERN_ALERT "vma->vm_file->f_op->mmap: %pSR\n",
+ printk(KERN_ALERT "vma->vm_file->f_op->mmap: %pAR\n",
vma->vm_file->f_op->mmap);
dump_stack();
add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
page_add_file_rmap(page);
set_pte_at(mm, addr, pte, mk_pte(page, prot));
+#ifdef CONFIG_PAX_SEGMEXEC
+ pax_mirror_file_pte(vma, addr, page, ptl);
+#endif
+
retval = 0;
pte_unmap_unlock(pte, ptl);
return retval;
if (!page_count(page))
return -EINVAL;
if (!(vma->vm_flags & VM_MIXEDMAP)) {
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ struct vm_area_struct *vma_m;
+#endif
+
BUG_ON(down_read_trylock(&vma->vm_mm->mmap_sem));
BUG_ON(vma->vm_flags & VM_PFNMAP);
vma->vm_flags |= VM_MIXEDMAP;
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ vma_m = pax_find_mirror_vma(vma);
+ if (vma_m)
+ vma_m->vm_flags |= VM_MIXEDMAP;
+#endif
+
}
return insert_page(vma, addr, page, vma->vm_page_prot);
}
unsigned long pfn)
{
BUG_ON(!(vma->vm_flags & VM_MIXEDMAP));
+ BUG_ON(vma->vm_mirror);
if (addr < vma->vm_start || addr >= vma->vm_end)
return -EFAULT;
BUG_ON(pud_huge(*pud));
- pmd = pmd_alloc(mm, pud, addr);
+ pmd = (mm == &init_mm) ?
+ pmd_alloc_kernel(mm, pud, addr) :
+ pmd_alloc(mm, pud, addr);
if (!pmd)
return -ENOMEM;
do {
unsigned long next;
int err;
- pud = pud_alloc(mm, pgd, addr);
+ pud = (mm == &init_mm) ?
+ pud_alloc_kernel(mm, pgd, addr) :
+ pud_alloc(mm, pgd, addr);
if (!pud)
return -ENOMEM;
do {
return ret;
}
+#ifdef CONFIG_PAX_SEGMEXEC
+static void pax_unmap_mirror_pte(struct vm_area_struct *vma, unsigned long address, pmd_t *pmd)
+{
+ struct mm_struct *mm = vma->vm_mm;
+ spinlock_t *ptl;
+ pte_t *pte, entry;
+
+ pte = pte_offset_map_lock(mm, pmd, address, &ptl);
+ entry = *pte;
+ if (!pte_present(entry)) {
+ if (!pte_none(entry)) {
+ BUG_ON(pte_file(entry));
+ free_swap_and_cache(pte_to_swp_entry(entry));
+ pte_clear_not_present_full(mm, address, pte, 0);
+ }
+ } else {
+ struct page *page;
+
+ flush_cache_page(vma, address, pte_pfn(entry));
+ entry = ptep_clear_flush(vma, address, pte);
+ BUG_ON(pte_dirty(entry));
+ page = vm_normal_page(vma, address, entry);
+ if (page) {
+ update_hiwater_rss(mm);
+ if (PageAnon(page))
+ dec_mm_counter_fast(mm, MM_ANONPAGES);
+ else
+ dec_mm_counter_fast(mm, MM_FILEPAGES);
+ page_remove_rmap(page);
+ page_cache_release(page);
+ }
+ }
+ pte_unmap_unlock(pte, ptl);
+}
+
+/* PaX: if vma is mirrored, synchronize the mirror's PTE
+ *
+ * the ptl of the lower mapped page is held on entry and is not released on exit
+ * or inside to ensure atomic changes to the PTE states (swapout, mremap, munmap, etc)
+ */
+static void pax_mirror_anon_pte(struct vm_area_struct *vma, unsigned long address, struct page *page_m, spinlock_t *ptl)
+{
+ struct mm_struct *mm = vma->vm_mm;
+ unsigned long address_m;
+ spinlock_t *ptl_m;
+ struct vm_area_struct *vma_m;
+ pmd_t *pmd_m;
+ pte_t *pte_m, entry_m;
+
+ BUG_ON(!page_m || !PageAnon(page_m));
+
+ vma_m = pax_find_mirror_vma(vma);
+ if (!vma_m)
+ return;
+
+ BUG_ON(!PageLocked(page_m));
+ BUG_ON(address >= SEGMEXEC_TASK_SIZE);
+ address_m = address + SEGMEXEC_TASK_SIZE;
+ pmd_m = pmd_offset(pud_offset(pgd_offset(mm, address_m), address_m), address_m);
+ pte_m = pte_offset_map(pmd_m, address_m);
+ ptl_m = pte_lockptr(mm, pmd_m);
+ if (ptl != ptl_m) {
+ spin_lock_nested(ptl_m, SINGLE_DEPTH_NESTING);
+ if (!pte_none(*pte_m))
+ goto out;
+ }
+
+ entry_m = pfn_pte(page_to_pfn(page_m), vma_m->vm_page_prot);
+ page_cache_get(page_m);
+ page_add_anon_rmap(page_m, vma_m, address_m);
+ inc_mm_counter_fast(mm, MM_ANONPAGES);
+ set_pte_at(mm, address_m, pte_m, entry_m);
+ update_mmu_cache(vma_m, address_m, pte_m);
+out:
+ if (ptl != ptl_m)
+ spin_unlock(ptl_m);
+ pte_unmap(pte_m);
+ unlock_page(page_m);
+}
+
+void pax_mirror_file_pte(struct vm_area_struct *vma, unsigned long address, struct page *page_m, spinlock_t *ptl)
+{
+ struct mm_struct *mm = vma->vm_mm;
+ unsigned long address_m;
+ spinlock_t *ptl_m;
+ struct vm_area_struct *vma_m;
+ pmd_t *pmd_m;
+ pte_t *pte_m, entry_m;
+
+ BUG_ON(!page_m || PageAnon(page_m));
+
+ vma_m = pax_find_mirror_vma(vma);
+ if (!vma_m)
+ return;
+
+ BUG_ON(address >= SEGMEXEC_TASK_SIZE);
+ address_m = address + SEGMEXEC_TASK_SIZE;
+ pmd_m = pmd_offset(pud_offset(pgd_offset(mm, address_m), address_m), address_m);
+ pte_m = pte_offset_map(pmd_m, address_m);
+ ptl_m = pte_lockptr(mm, pmd_m);
+ if (ptl != ptl_m) {
+ spin_lock_nested(ptl_m, SINGLE_DEPTH_NESTING);
+ if (!pte_none(*pte_m))
+ goto out;
+ }
+
+ entry_m = pfn_pte(page_to_pfn(page_m), vma_m->vm_page_prot);
+ page_cache_get(page_m);
+ page_add_file_rmap(page_m);
+ inc_mm_counter_fast(mm, MM_FILEPAGES);
+ set_pte_at(mm, address_m, pte_m, entry_m);
+ update_mmu_cache(vma_m, address_m, pte_m);
+out:
+ if (ptl != ptl_m)
+ spin_unlock(ptl_m);
+ pte_unmap(pte_m);
+}
+
+static void pax_mirror_pfn_pte(struct vm_area_struct *vma, unsigned long address, unsigned long pfn_m, spinlock_t *ptl)
+{
+ struct mm_struct *mm = vma->vm_mm;
+ unsigned long address_m;
+ spinlock_t *ptl_m;
+ struct vm_area_struct *vma_m;
+ pmd_t *pmd_m;
+ pte_t *pte_m, entry_m;
+
+ vma_m = pax_find_mirror_vma(vma);
+ if (!vma_m)
+ return;
+
+ BUG_ON(address >= SEGMEXEC_TASK_SIZE);
+ address_m = address + SEGMEXEC_TASK_SIZE;
+ pmd_m = pmd_offset(pud_offset(pgd_offset(mm, address_m), address_m), address_m);
+ pte_m = pte_offset_map(pmd_m, address_m);
+ ptl_m = pte_lockptr(mm, pmd_m);
+ if (ptl != ptl_m) {
+ spin_lock_nested(ptl_m, SINGLE_DEPTH_NESTING);
+ if (!pte_none(*pte_m))
+ goto out;
+ }
+
+ entry_m = pfn_pte(pfn_m, vma_m->vm_page_prot);
+ set_pte_at(mm, address_m, pte_m, entry_m);
+out:
+ if (ptl != ptl_m)
+ spin_unlock(ptl_m);
+ pte_unmap(pte_m);
+}
+
+static void pax_mirror_pte(struct vm_area_struct *vma, unsigned long address, pte_t *pte, pmd_t *pmd, spinlock_t *ptl)
+{
+ struct page *page_m;
+ pte_t entry;
+
+ if (!(vma->vm_mm->pax_flags & MF_PAX_SEGMEXEC))
+ goto out;
+
+ entry = *pte;
+ page_m = vm_normal_page(vma, address, entry);
+ if (!page_m)
+ pax_mirror_pfn_pte(vma, address, pte_pfn(entry), ptl);
+ else if (PageAnon(page_m)) {
+ if (pax_find_mirror_vma(vma)) {
+ pte_unmap_unlock(pte, ptl);
+ lock_page(page_m);
+ pte = pte_offset_map_lock(vma->vm_mm, pmd, address, &ptl);
+ if (pte_same(entry, *pte))
+ pax_mirror_anon_pte(vma, address, page_m, ptl);
+ else
+ unlock_page(page_m);
+ }
+ } else
+ pax_mirror_file_pte(vma, address, page_m, ptl);
+
+out:
+ pte_unmap_unlock(pte, ptl);
+}
+#endif
+
/*
* This routine handles present pages, when users try to write
* to a shared page. It is done by copying the page to a new address
*/
page_table = pte_offset_map_lock(mm, pmd, address, &ptl);
if (likely(pte_same(*page_table, orig_pte))) {
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (pax_find_mirror_vma(vma))
+ BUG_ON(!trylock_page(new_page));
+#endif
+
if (old_page) {
if (!PageAnon(old_page)) {
dec_mm_counter_fast(mm, MM_FILEPAGES);
page_remove_rmap(old_page);
}
+#ifdef CONFIG_PAX_SEGMEXEC
+ pax_mirror_anon_pte(vma, address, new_page, ptl);
+#endif
+
/* Free the old page.. */
new_page = old_page;
ret |= VM_FAULT_WRITE;
swap_free(entry);
if (vm_swap_full() || (vma->vm_flags & VM_LOCKED) || PageMlocked(page))
try_to_free_swap(page);
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if ((flags & FAULT_FLAG_WRITE) || !pax_find_mirror_vma(vma))
+#endif
+
unlock_page(page);
if (page != swapcache) {
/*
/* No need to invalidate - it was non-present before */
update_mmu_cache(vma, address, page_table);
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ pax_mirror_anon_pte(vma, address, page, ptl);
+#endif
+
unlock:
pte_unmap_unlock(page_table, ptl);
out:
return ret;
}
-/*
- * This is like a special single-page "expand_{down|up}wards()",
- * except we must first make sure that 'address{-|+}PAGE_SIZE'
- * doesn't hit another vma.
- */
-static inline int check_stack_guard_page(struct vm_area_struct *vma, unsigned long address)
-{
- address &= PAGE_MASK;
- if ((vma->vm_flags & VM_GROWSDOWN) && address == vma->vm_start) {
- struct vm_area_struct *prev = vma->vm_prev;
-
- /*
- * Is there a mapping abutting this one below?
- *
- * That's only ok if it's the same stack mapping
- * that has gotten split..
- */
- if (prev && prev->vm_end == address)
- return prev->vm_flags & VM_GROWSDOWN ? 0 : -ENOMEM;
-
- return expand_downwards(vma, address - PAGE_SIZE);
- }
- if ((vma->vm_flags & VM_GROWSUP) && address + PAGE_SIZE == vma->vm_end) {
- struct vm_area_struct *next = vma->vm_next;
-
- /* As VM_GROWSDOWN but s/below/above/ */
- if (next && next->vm_start == address + PAGE_SIZE)
- return next->vm_flags & VM_GROWSUP ? 0 : -ENOMEM;
-
- return expand_upwards(vma, address + PAGE_SIZE);
- }
- return 0;
-}
-
/*
* We enter with non-exclusive mmap_sem (to exclude vma changes,
* but allow concurrent faults), and pte mapped but not yet locked.
unsigned int flags)
{
struct mem_cgroup *memcg;
- struct page *page;
+ struct page *page = NULL;
spinlock_t *ptl;
pte_t entry;
- pte_unmap(page_table);
-
- /* Check if we need to add a guard page to the stack */
- if (check_stack_guard_page(vma, address) < 0)
- return VM_FAULT_SIGSEGV;
-
- /* Use the zero-page for reads */
if (!(flags & FAULT_FLAG_WRITE) && !mm_forbids_zeropage(mm)) {
entry = pte_mkspecial(pfn_pte(my_zero_pfn(address),
vma->vm_page_prot));
- page_table = pte_offset_map_lock(mm, pmd, address, &ptl);
+ ptl = pte_lockptr(mm, pmd);
+ spin_lock(ptl);
if (!pte_none(*page_table))
goto unlock;
goto setpte;
}
/* Allocate our own private page. */
+ pte_unmap(page_table);
+
if (unlikely(anon_vma_prepare(vma)))
goto oom;
page = alloc_zeroed_user_highpage_movable(vma, address);
if (!pte_none(*page_table))
goto release;
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (pax_find_mirror_vma(vma))
+ BUG_ON(!trylock_page(page));
+#endif
+
inc_mm_counter_fast(mm, MM_ANONPAGES);
page_add_new_anon_rmap(page, vma, address);
mem_cgroup_commit_charge(page, memcg, false);
/* No need to invalidate - it was non-present before */
update_mmu_cache(vma, address, page_table);
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (page)
+ pax_mirror_anon_pte(vma, address, page, ptl);
+#endif
+
unlock:
pte_unmap_unlock(page_table, ptl);
return 0;
return ret;
}
do_set_pte(vma, address, fault_page, pte, false, false);
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ pax_mirror_file_pte(vma, address, fault_page, ptl);
+#endif
+
unlock_page(fault_page);
unlock_out:
pte_unmap_unlock(pte, ptl);
page_cache_release(fault_page);
goto uncharge_out;
}
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (pax_find_mirror_vma(vma))
+ BUG_ON(!trylock_page(new_page));
+#endif
+
do_set_pte(vma, address, new_page, pte, true, true);
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ pax_mirror_anon_pte(vma, address, new_page, ptl);
+#endif
+
mem_cgroup_commit_charge(new_page, memcg, false);
lru_cache_add_active_or_unevictable(new_page, vma);
pte_unmap_unlock(pte, ptl);
return ret;
}
do_set_pte(vma, address, fault_page, pte, true, false);
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ pax_mirror_file_pte(vma, address, fault_page, ptl);
+#endif
+
pte_unmap_unlock(pte, ptl);
if (set_page_dirty(fault_page))
if (flags & FAULT_FLAG_WRITE)
flush_tlb_fix_spurious_fault(vma, address);
}
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ pax_mirror_pte(vma, address, pte, pmd, ptl);
+ return 0;
+#endif
+
unlock:
pte_unmap_unlock(pte, ptl);
return 0;
pmd_t *pmd;
pte_t *pte;
+#ifdef CONFIG_PAX_SEGMEXEC
+ struct vm_area_struct *vma_m;
+#endif
+
if (unlikely(is_vm_hugetlb_page(vma)))
return hugetlb_fault(mm, vma, address, flags);
+#ifdef CONFIG_PAX_SEGMEXEC
+ vma_m = pax_find_mirror_vma(vma);
+ if (vma_m) {
+ unsigned long address_m;
+ pgd_t *pgd_m;
+ pud_t *pud_m;
+ pmd_t *pmd_m;
+
+ if (vma->vm_start > vma_m->vm_start) {
+ address_m = address;
+ address -= SEGMEXEC_TASK_SIZE;
+ vma = vma_m;
+ } else
+ address_m = address + SEGMEXEC_TASK_SIZE;
+
+ pgd_m = pgd_offset(mm, address_m);
+ pud_m = pud_alloc(mm, pgd_m, address_m);
+ if (!pud_m)
+ return VM_FAULT_OOM;
+ pmd_m = pmd_alloc(mm, pud_m, address_m);
+ if (!pmd_m)
+ return VM_FAULT_OOM;
+ if (!pmd_present(*pmd_m) && __pte_alloc(mm, vma_m, pmd_m, address_m))
+ return VM_FAULT_OOM;
+ pax_unmap_mirror_pte(vma_m, address_m, pmd_m);
+ }
+#endif
+
pgd = pgd_offset(mm, address);
pud = pud_alloc(mm, pgd, address);
if (!pud)
spin_unlock(&mm->page_table_lock);
return 0;
}
+
+int __pud_alloc_kernel(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
+{
+ pud_t *new = pud_alloc_one(mm, address);
+ if (!new)
+ return -ENOMEM;
+
+ smp_wmb(); /* See comment in __pte_alloc */
+
+ spin_lock(&mm->page_table_lock);
+ if (pgd_present(*pgd)) /* Another has populated it */
+ pud_free(mm, new);
+ else
+ pgd_populate_kernel(mm, pgd, new);
+ spin_unlock(&mm->page_table_lock);
+ return 0;
+}
#endif /* __PAGETABLE_PUD_FOLDED */
#ifndef __PAGETABLE_PMD_FOLDED
spin_unlock(&mm->page_table_lock);
return 0;
}
+
+int __pmd_alloc_kernel(struct mm_struct *mm, pud_t *pud, unsigned long address)
+{
+ pmd_t *new = pmd_alloc_one(mm, address);
+ if (!new)
+ return -ENOMEM;
+
+ smp_wmb(); /* See comment in __pte_alloc */
+
+ spin_lock(&mm->page_table_lock);
+#ifndef __ARCH_HAS_4LEVEL_HACK
+ if (pud_present(*pud)) /* Another has populated it */
+ pmd_free(mm, new);
+ else
+ pud_populate_kernel(mm, pud, new);
+#else
+ if (pgd_present(*pud)) /* Another has populated it */
+ pmd_free(mm, new);
+ else
+ pgd_populate_kernel(mm, pud, new);
+#endif /* __ARCH_HAS_4LEVEL_HACK */
+ spin_unlock(&mm->page_table_lock);
+ return 0;
+}
#endif /* __PAGETABLE_PMD_FOLDED */
static int __follow_pte(struct mm_struct *mm, unsigned long address,
return ret;
}
-int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
- void *buf, int len, int write)
+ssize_t generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
+ void *buf, size_t len, int write)
{
resource_size_t phys_addr;
unsigned long prot = 0;
* Access another process' address space as given in mm. If non-NULL, use the
* given task for page fault accounting.
*/
-static int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
- unsigned long addr, void *buf, int len, int write)
+static ssize_t __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
+ unsigned long addr, void *buf, size_t len, int write)
{
struct vm_area_struct *vma;
void *old_buf = buf;
down_read(&mm->mmap_sem);
/* ignore errors, just check how much was successfully transferred */
while (len) {
- int bytes, ret, offset;
+ ssize_t bytes, ret, offset;
void *maddr;
struct page *page = NULL;
*
* The caller must hold a reference on @mm.
*/
-int access_remote_vm(struct mm_struct *mm, unsigned long addr,
- void *buf, int len, int write)
+ssize_t access_remote_vm(struct mm_struct *mm, unsigned long addr,
+ void *buf, size_t len, int write)
{
return __access_remote_vm(NULL, mm, addr, buf, len, write);
}
* Source/target buffer must be kernel space,
* Do not walk the page table directly, use get_user_pages
*/
-int access_process_vm(struct task_struct *tsk, unsigned long addr,
- void *buf, int len, int write)
+ssize_t access_process_vm(struct task_struct *tsk, unsigned long addr,
+ void *buf, size_t len, int write)
{
struct mm_struct *mm;
- int ret;
+ ssize_t ret;
mm = get_task_mm(tsk);
if (!mm)
unsigned long vmstart;
unsigned long vmend;
+#ifdef CONFIG_PAX_SEGMEXEC
+ struct vm_area_struct *vma_m;
+#endif
+
vma = find_vma(mm, start);
if (!vma || vma->vm_start > start)
return -EFAULT;
err = vma_replace_policy(vma, new_pol);
if (err)
goto out;
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ vma_m = pax_find_mirror_vma(vma);
+ if (vma_m) {
+ err = vma_replace_policy(vma_m, new_pol);
+ if (err)
+ goto out;
+ }
+#endif
+
}
out:
if (end < start)
return -EINVAL;
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (mm->pax_flags & MF_PAX_SEGMEXEC) {
+ if (end > SEGMEXEC_TASK_SIZE)
+ return -EINVAL;
+ } else
+#endif
+
+ if (end > TASK_SIZE)
+ return -EINVAL;
+
if (end == start)
return 0;
*/
tcred = __task_cred(task);
if (!uid_eq(cred->euid, tcred->suid) && !uid_eq(cred->euid, tcred->uid) &&
- !uid_eq(cred->uid, tcred->suid) && !uid_eq(cred->uid, tcred->uid) &&
- !capable(CAP_SYS_NICE)) {
+ !uid_eq(cred->uid, tcred->suid) && !capable(CAP_SYS_NICE)) {
rcu_read_unlock();
err = -EPERM;
goto out_put;
goto out;
}
+#ifdef CONFIG_GRKERNSEC_PROC_MEMMAP
+ if (mm != current->mm &&
+ (mm->pax_flags & MF_PAX_RANDMMAP || mm->pax_flags & MF_PAX_SEGMEXEC)) {
+ mmput(mm);
+ err = -EPERM;
+ goto out;
+ }
+#endif
+
err = do_migrate_pages(mm, old, new,
capable(CAP_SYS_NICE) ? MPOL_MF_MOVE_ALL : MPOL_MF_MOVE);
*/
tcred = __task_cred(task);
if (!uid_eq(cred->euid, tcred->suid) && !uid_eq(cred->euid, tcred->uid) &&
- !uid_eq(cred->uid, tcred->suid) && !uid_eq(cred->uid, tcred->uid) &&
- !capable(CAP_SYS_NICE)) {
+ !uid_eq(cred->uid, tcred->suid) && !capable(CAP_SYS_NICE)) {
rcu_read_unlock();
err = -EPERM;
goto out;
#include <linux/pagevec.h>
#include <linux/mempolicy.h>
#include <linux/syscalls.h>
+#include <linux/security.h>
#include <linux/sched.h>
#include <linux/export.h>
#include <linux/rmap.h>
{
unsigned long nstart, end, tmp;
struct vm_area_struct * vma, * prev;
- int error;
+ int error = 0;
VM_BUG_ON(start & ~PAGE_MASK);
VM_BUG_ON(len != PAGE_ALIGN(len));
return -EINVAL;
if (end == start)
return 0;
+ if (end > TASK_SIZE)
+ return -EINVAL;
+
vma = find_vma(current->mm, start);
if (!vma || vma->vm_start > start)
return -ENOMEM;
for (nstart = start ; ; ) {
vm_flags_t newflags;
+#ifdef CONFIG_PAX_SEGMEXEC
+ if ((current->mm->pax_flags & MF_PAX_SEGMEXEC) && (vma->vm_start >= SEGMEXEC_TASK_SIZE))
+ break;
+#endif
+
/* Here we know that vma->vm_start <= nstart < vma->vm_end. */
newflags = vma->vm_flags & ~VM_LOCKED;
locked += current->mm->locked_vm;
/* check against resource limits */
+ gr_learn_resource(current, RLIMIT_MEMLOCK, (current->mm->locked_vm << PAGE_SHIFT) + len, 1);
if ((locked <= lock_limit) || capable(CAP_IPC_LOCK))
error = do_mlock(start, len, 1);
for (vma = current->mm->mmap; vma ; vma = prev->vm_next) {
vm_flags_t newflags;
+#ifdef CONFIG_PAX_SEGMEXEC
+ if ((current->mm->pax_flags & MF_PAX_SEGMEXEC) && (vma->vm_start >= SEGMEXEC_TASK_SIZE))
+ break;
+#endif
+
newflags = vma->vm_flags & ~VM_LOCKED;
if (flags & MCL_CURRENT)
newflags |= VM_LOCKED;
lock_limit >>= PAGE_SHIFT;
ret = -ENOMEM;
- down_write(¤t->mm->mmap_sem);
+ gr_learn_resource(current, RLIMIT_MEMLOCK, current->mm->total_vm << PAGE_SHIFT, 1);
+
+ down_write(¤t->mm->mmap_sem);
if (!(flags & MCL_CURRENT) || (current->mm->total_vm <= lock_limit) ||
capable(CAP_IPC_LOCK))
ret = do_mlockall(flags);
#include <linux/notifier.h>
#include <linux/memory.h>
#include <linux/printk.h>
+#include <linux/random.h>
#include <asm/uaccess.h>
#include <asm/cacheflush.h>
#define arch_rebalance_pgtables(addr, len) (addr)
#endif
+static inline void verify_mm_writelocked(struct mm_struct *mm)
+{
+#if defined(CONFIG_DEBUG_VM) || defined(CONFIG_PAX)
+ if (unlikely(down_read_trylock(&mm->mmap_sem))) {
+ up_read(&mm->mmap_sem);
+ BUG();
+ }
+#endif
+}
+
static void unmap_region(struct mm_struct *mm,
struct vm_area_struct *vma, struct vm_area_struct *prev,
unsigned long start, unsigned long end);
* x: (no) no x: (no) yes x: (no) yes x: (yes) yes
*
*/
-pgprot_t protection_map[16] = {
+pgprot_t protection_map[16] __read_only = {
__P000, __P001, __P010, __P011, __P100, __P101, __P110, __P111,
__S000, __S001, __S010, __S011, __S100, __S101, __S110, __S111
};
-pgprot_t vm_get_page_prot(unsigned long vm_flags)
+pgprot_t vm_get_page_prot(vm_flags_t vm_flags)
{
- return __pgprot(pgprot_val(protection_map[vm_flags &
+ pgprot_t prot = __pgprot(pgprot_val(protection_map[vm_flags &
(VM_READ|VM_WRITE|VM_EXEC|VM_SHARED)]) |
pgprot_val(arch_vm_get_page_prot(vm_flags)));
+
+#if defined(CONFIG_PAX_PAGEEXEC) && defined(CONFIG_X86_32)
+ if (!(__supported_pte_mask & _PAGE_NX) &&
+ (vm_flags & (VM_PAGEEXEC | VM_EXEC)) == VM_PAGEEXEC &&
+ (vm_flags & (VM_READ | VM_WRITE)))
+ prot = __pgprot(pte_val(pte_exprotect(__pte(pgprot_val(prot)))));
+#endif
+
+ return prot;
}
EXPORT_SYMBOL(vm_get_page_prot);
int sysctl_max_map_count __read_mostly = DEFAULT_MAX_MAP_COUNT;
unsigned long sysctl_user_reserve_kbytes __read_mostly = 1UL << 17; /* 128MB */
unsigned long sysctl_admin_reserve_kbytes __read_mostly = 1UL << 13; /* 8MB */
+unsigned long sysctl_heap_stack_gap __read_mostly = 64*1024;
/*
* Make sure vm_committed_as in one cacheline and not cacheline shared with
* other variables. It can be updated by several CPUs frequently.
struct vm_area_struct *next = vma->vm_next;
might_sleep();
+ BUG_ON(vma->vm_mirror);
if (vma->vm_ops && vma->vm_ops->close)
vma->vm_ops->close(vma);
if (vma->vm_file)
SYSCALL_DEFINE1(brk, unsigned long, brk)
{
+ unsigned long rlim;
unsigned long retval;
unsigned long newbrk, oldbrk;
struct mm_struct *mm = current->mm;
* segment grow beyond its set limit the in case where the limit is
* not page aligned -Ram Gupta
*/
- if (check_data_rlimit(rlimit(RLIMIT_DATA), brk, mm->start_brk,
+ rlim = rlimit(RLIMIT_DATA);
+#ifdef CONFIG_GRKERNSEC_PROC_MEMMAP
+ /* force a minimum 16MB brk heap on setuid/setgid binaries */
+ if (rlim < PAGE_SIZE && (get_dumpable(mm) != SUID_DUMP_USER) && gr_is_global_nonroot(current_uid()))
+ rlim = 4096 * PAGE_SIZE;
+#endif
+ if (check_data_rlimit(rlim, brk, mm->start_brk,
mm->end_data, mm->start_data))
goto out;
can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,
struct anon_vma *anon_vma, struct file *file, pgoff_t vm_pgoff)
{
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if ((vma->vm_mm->pax_flags & MF_PAX_SEGMEXEC) && vma->vm_start == SEGMEXEC_TASK_SIZE)
+ return 0;
+#endif
+
if (is_mergeable_vma(vma, file, vm_flags) &&
is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
if (vma->vm_pgoff == vm_pgoff)
can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
struct anon_vma *anon_vma, struct file *file, pgoff_t vm_pgoff)
{
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if ((vma->vm_mm->pax_flags & MF_PAX_SEGMEXEC) && vma->vm_end == SEGMEXEC_TASK_SIZE)
+ return 0;
+#endif
+
if (is_mergeable_vma(vma, file, vm_flags) &&
is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
pgoff_t vm_pglen;
struct vm_area_struct *area, *next;
int err;
+#ifdef CONFIG_PAX_SEGMEXEC
+ unsigned long addr_m = addr + SEGMEXEC_TASK_SIZE, end_m = end + SEGMEXEC_TASK_SIZE;
+ struct vm_area_struct *area_m = NULL, *next_m = NULL, *prev_m = NULL;
+
+ BUG_ON((mm->pax_flags & MF_PAX_SEGMEXEC) && SEGMEXEC_TASK_SIZE < end);
+#endif
+
/*
* We later require that vma->vm_flags == vm_flags,
* so this tests vma->vm_flags & VM_SPECIAL, too.
if (next && next->vm_end == end) /* cases 6, 7, 8 */
next = next->vm_next;
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (prev)
+ prev_m = pax_find_mirror_vma(prev);
+ if (area)
+ area_m = pax_find_mirror_vma(area);
+ if (next)
+ next_m = pax_find_mirror_vma(next);
+#endif
+
/*
* Can it merge with the predecessor?
*/
/* cases 1, 6 */
err = vma_adjust(prev, prev->vm_start,
next->vm_end, prev->vm_pgoff, NULL);
- } else /* cases 2, 5, 7 */
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (!err && prev_m)
+ err = vma_adjust(prev_m, prev_m->vm_start,
+ next_m->vm_end, prev_m->vm_pgoff, NULL);
+#endif
+
+ } else { /* cases 2, 5, 7 */
err = vma_adjust(prev, prev->vm_start,
end, prev->vm_pgoff, NULL);
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (!err && prev_m)
+ err = vma_adjust(prev_m, prev_m->vm_start,
+ end_m, prev_m->vm_pgoff, NULL);
+#endif
+
+ }
if (err)
return NULL;
khugepaged_enter_vma_merge(prev, vm_flags);
mpol_equal(policy, vma_policy(next)) &&
can_vma_merge_before(next, vm_flags,
anon_vma, file, pgoff+pglen)) {
- if (prev && addr < prev->vm_end) /* case 4 */
+ if (prev && addr < prev->vm_end) { /* case 4 */
err = vma_adjust(prev, prev->vm_start,
addr, prev->vm_pgoff, NULL);
- else /* cases 3, 8 */
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (!err && prev_m)
+ err = vma_adjust(prev_m, prev_m->vm_start,
+ addr_m, prev_m->vm_pgoff, NULL);
+#endif
+
+ } else { /* cases 3, 8 */
err = vma_adjust(area, addr, next->vm_end,
next->vm_pgoff - pglen, NULL);
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (!err && area_m)
+ err = vma_adjust(area_m, addr_m, next_m->vm_end,
+ next_m->vm_pgoff - pglen, NULL);
+#endif
+
+ }
if (err)
return NULL;
khugepaged_enter_vma_merge(area, vm_flags);
void vm_stat_account(struct mm_struct *mm, unsigned long flags,
struct file *file, long pages)
{
- const unsigned long stack_flags
- = VM_STACK_FLAGS & (VM_GROWSUP|VM_GROWSDOWN);
+
+#ifdef CONFIG_PAX_RANDMMAP
+ if (!(mm->pax_flags & MF_PAX_RANDMMAP) || (flags & (VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC)))
+#endif
mm->total_vm += pages;
mm->shared_vm += pages;
if ((flags & (VM_EXEC|VM_WRITE)) == VM_EXEC)
mm->exec_vm += pages;
- } else if (flags & stack_flags)
+ } else if (flags & (VM_GROWSUP|VM_GROWSDOWN))
mm->stack_vm += pages;
}
#endif /* CONFIG_PROC_FS */
locked += mm->locked_vm;
lock_limit = rlimit(RLIMIT_MEMLOCK);
lock_limit >>= PAGE_SHIFT;
+ gr_learn_resource(current, RLIMIT_MEMLOCK, locked << PAGE_SHIFT, 1);
if (locked > lock_limit && !capable(CAP_IPC_LOCK))
return -EAGAIN;
}
* (the exception is when the underlying filesystem is noexec
* mounted, in which case we dont add PROT_EXEC.)
*/
- if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC))
+ if ((prot & (PROT_READ | PROT_WRITE)) && (current->personality & READ_IMPLIES_EXEC))
if (!(file && (file->f_path.mnt->mnt_flags & MNT_NOEXEC)))
prot |= PROT_EXEC;
/* Obtain the address to map to. we verify (or select) it and ensure
* that it represents a valid section of the address space.
*/
- addr = get_unmapped_area(file, addr, len, pgoff, flags);
+ addr = get_unmapped_area(file, addr, len, pgoff, flags | ((prot & PROT_EXEC) ? MAP_EXECUTABLE : 0));
if (addr & ~PAGE_MASK)
return addr;
vm_flags = calc_vm_prot_bits(prot) | calc_vm_flag_bits(flags) |
mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
+#ifdef CONFIG_PAX_MPROTECT
+ if (mm->pax_flags & MF_PAX_MPROTECT) {
+
+#ifdef CONFIG_GRKERNSEC_RWXMAP_LOG
+ if (file && !pgoff && (vm_flags & VM_EXEC) && mm->binfmt &&
+ mm->binfmt->handle_mmap)
+ mm->binfmt->handle_mmap(file);
+#endif
+
+#ifndef CONFIG_PAX_MPROTECT_COMPAT
+ if ((vm_flags & (VM_WRITE | VM_EXEC)) == (VM_WRITE | VM_EXEC)) {
+ gr_log_rwxmmap(file);
+
+#ifdef CONFIG_PAX_EMUPLT
+ vm_flags &= ~VM_EXEC;
+#else
+ return -EPERM;
+#endif
+
+ }
+
+ if (!(vm_flags & VM_EXEC))
+ vm_flags &= ~VM_MAYEXEC;
+#else
+ if ((vm_flags & (VM_WRITE | VM_EXEC)) != VM_EXEC)
+ vm_flags &= ~(VM_EXEC | VM_MAYEXEC);
+#endif
+ else
+ vm_flags &= ~VM_MAYWRITE;
+ }
+#endif
+
+#if defined(CONFIG_PAX_PAGEEXEC) && defined(CONFIG_X86_32)
+ if ((mm->pax_flags & MF_PAX_PAGEEXEC) && file)
+ vm_flags &= ~VM_PAGEEXEC;
+#endif
+
if (flags & MAP_LOCKED)
if (!can_do_mlock())
return -EPERM;
vm_flags |= VM_NORESERVE;
}
+ if (!gr_acl_handle_mmap(file, prot))
+ return -EACCES;
+
addr = mmap_region(file, addr, len, vm_flags, pgoff);
if (!IS_ERR_VALUE(addr) &&
((vm_flags & VM_LOCKED) ||
vm_flags_t vm_flags = vma->vm_flags;
/* If it was private or non-writable, the write bit is already clear */
- if ((vm_flags & (VM_WRITE|VM_SHARED)) != ((VM_WRITE|VM_SHARED)))
+ if ((vm_flags & (VM_WRITE|VM_SHARED)) != (VM_WRITE|VM_SHARED))
return 0;
/* The backer wishes to know when pages are first written to? */
struct rb_node **rb_link, *rb_parent;
unsigned long charged = 0;
+#ifdef CONFIG_PAX_SEGMEXEC
+ struct vm_area_struct *vma_m = NULL;
+#endif
+
+ /*
+ * mm->mmap_sem is required to protect against another thread
+ * changing the mappings in case we sleep.
+ */
+ verify_mm_writelocked(mm);
+
/* Check against address space limit. */
+
+#ifdef CONFIG_PAX_RANDMMAP
+ if (!(mm->pax_flags & MF_PAX_RANDMMAP) || (vm_flags & (VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC)))
+#endif
+
if (!may_expand_vm(mm, len >> PAGE_SHIFT)) {
unsigned long nr_pages;
/* Clear old maps */
error = -ENOMEM;
-munmap_back:
if (find_vma_links(mm, addr, addr + len, &prev, &rb_link, &rb_parent)) {
if (do_munmap(mm, addr, len))
return -ENOMEM;
- goto munmap_back;
+ BUG_ON(find_vma_links(mm, addr, addr + len, &prev, &rb_link, &rb_parent));
}
/*
goto unacct_error;
}
+#ifdef CONFIG_PAX_SEGMEXEC
+ if ((mm->pax_flags & MF_PAX_SEGMEXEC) && (vm_flags & VM_EXEC)) {
+ vma_m = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
+ if (!vma_m) {
+ error = -ENOMEM;
+ goto free_vma;
+ }
+ }
+#endif
+
vma->vm_mm = mm;
vma->vm_start = addr;
vma->vm_end = addr + len;
if (error)
goto unmap_and_free_vma;
+#if defined(CONFIG_PAX_PAGEEXEC) && defined(CONFIG_X86_32)
+ if ((mm->pax_flags & MF_PAX_PAGEEXEC) && !(vma->vm_flags & VM_SPECIAL)) {
+ vma->vm_flags |= VM_PAGEEXEC;
+ vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
+ }
+#endif
+
/* Can addr have changed??
*
* Answer: Yes, several device drivers can do it in their
}
vma_link(mm, vma, prev, rb_link, rb_parent);
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (vma_m)
+ BUG_ON(pax_mirror_vma(vma_m, vma));
+#endif
+
/* Once vma denies write, undo our temporary denial count */
if (file) {
if (vm_flags & VM_SHARED)
perf_event_mmap(vma);
vm_stat_account(mm, vm_flags, file, len >> PAGE_SHIFT);
+ track_exec_limit(mm, addr, addr + len, vm_flags);
if (vm_flags & VM_LOCKED) {
if (!((vm_flags & VM_SPECIAL) || is_vm_hugetlb_page(vma) ||
vma == get_gate_vma(current->mm)))
if (vm_flags & VM_DENYWRITE)
allow_write_access(file);
free_vma:
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (vma_m)
+ kmem_cache_free(vm_area_cachep, vma_m);
+#endif
+
kmem_cache_free(vm_area_cachep, vma);
unacct_error:
if (charged)
return error;
}
-unsigned long unmapped_area(struct vm_unmapped_area_info *info)
+#ifdef CONFIG_GRKERNSEC_RAND_THREADSTACK
+unsigned long gr_rand_threadstack_offset(const struct mm_struct *mm, const struct file *filp, unsigned long flags)
+{
+ if ((mm->pax_flags & MF_PAX_RANDMMAP) && !filp && (flags & MAP_STACK))
+ return ((prandom_u32() & 0xFF) + 1) << PAGE_SHIFT;
+
+ return 0;
+}
+#endif
+
+bool check_heap_stack_gap(const struct vm_area_struct *vma, unsigned long addr, unsigned long len, unsigned long offset)
+{
+ if (!vma) {
+#ifdef CONFIG_STACK_GROWSUP
+ if (addr > sysctl_heap_stack_gap)
+ vma = find_vma(current->mm, addr - sysctl_heap_stack_gap);
+ else
+ vma = find_vma(current->mm, 0);
+ if (vma && (vma->vm_flags & VM_GROWSUP))
+ return false;
+#endif
+ return true;
+ }
+
+ if (addr + len > vma->vm_start)
+ return false;
+
+ if (vma->vm_flags & VM_GROWSDOWN)
+ return sysctl_heap_stack_gap <= vma->vm_start - addr - len;
+#ifdef CONFIG_STACK_GROWSUP
+ else if (vma->vm_prev && (vma->vm_prev->vm_flags & VM_GROWSUP))
+ return addr - vma->vm_prev->vm_end >= sysctl_heap_stack_gap;
+#endif
+ else if (offset)
+ return offset <= vma->vm_start - addr - len;
+
+ return true;
+}
+
+unsigned long skip_heap_stack_gap(const struct vm_area_struct *vma, unsigned long len, unsigned long offset)
+{
+ if (vma->vm_start < len)
+ return -ENOMEM;
+
+ if (!(vma->vm_flags & VM_GROWSDOWN)) {
+ if (offset <= vma->vm_start - len)
+ return vma->vm_start - len - offset;
+ else
+ return -ENOMEM;
+ }
+
+ if (sysctl_heap_stack_gap <= vma->vm_start - len)
+ return vma->vm_start - len - sysctl_heap_stack_gap;
+ return -ENOMEM;
+}
+
+unsigned long unmapped_area(const struct vm_unmapped_area_info *info)
{
/*
* We implement the search by looking for an rbtree node that
}
}
- gap_start = vma->vm_prev ? vma->vm_prev->vm_end : 0;
+ gap_start = vma->vm_prev ? vma->vm_prev->vm_end: 0;
check_current:
/* Check if current node has a suitable gap */
if (gap_start > high_limit)
return -ENOMEM;
+
+ if (gap_end - gap_start > info->threadstack_offset)
+ gap_start += info->threadstack_offset;
+ else
+ gap_start = gap_end;
+
+ if (vma->vm_prev && (vma->vm_prev->vm_flags & VM_GROWSUP)) {
+ if (gap_end - gap_start > sysctl_heap_stack_gap)
+ gap_start += sysctl_heap_stack_gap;
+ else
+ gap_start = gap_end;
+ }
+ if (vma->vm_flags & VM_GROWSDOWN) {
+ if (gap_end - gap_start > sysctl_heap_stack_gap)
+ gap_end -= sysctl_heap_stack_gap;
+ else
+ gap_end = gap_start;
+ }
if (gap_end >= low_limit && gap_end - gap_start >= length)
goto found;
return gap_start;
}
-unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info)
+unsigned long unmapped_area_topdown(const struct vm_unmapped_area_info *info)
{
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
gap_end = vma->vm_start;
if (gap_end < low_limit)
return -ENOMEM;
+
+ if (gap_end - gap_start > info->threadstack_offset)
+ gap_end -= info->threadstack_offset;
+ else
+ gap_end = gap_start;
+
+ if (vma->vm_prev && (vma->vm_prev->vm_flags & VM_GROWSUP)) {
+ if (gap_end - gap_start > sysctl_heap_stack_gap)
+ gap_start += sysctl_heap_stack_gap;
+ else
+ gap_start = gap_end;
+ }
+ if (vma->vm_flags & VM_GROWSDOWN) {
+ if (gap_end - gap_start > sysctl_heap_stack_gap)
+ gap_end -= sysctl_heap_stack_gap;
+ else
+ gap_end = gap_start;
+ }
if (gap_start <= high_limit && gap_end - gap_start >= length)
goto found;
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
struct vm_unmapped_area_info info;
+ unsigned long offset = gr_rand_threadstack_offset(mm, filp, flags);
if (len > TASK_SIZE - mmap_min_addr)
return -ENOMEM;
if (flags & MAP_FIXED)
return addr;
+#ifdef CONFIG_PAX_RANDMMAP
+ if (!(mm->pax_flags & MF_PAX_RANDMMAP))
+#endif
+
if (addr) {
addr = PAGE_ALIGN(addr);
vma = find_vma(mm, addr);
if (TASK_SIZE - len >= addr && addr >= mmap_min_addr &&
- (!vma || addr + len <= vma->vm_start))
+ check_heap_stack_gap(vma, addr, len, offset))
return addr;
}
info.low_limit = mm->mmap_base;
info.high_limit = TASK_SIZE;
info.align_mask = 0;
+ info.threadstack_offset = offset;
return vm_unmapped_area(&info);
}
#endif
struct mm_struct *mm = current->mm;
unsigned long addr = addr0;
struct vm_unmapped_area_info info;
+ unsigned long offset = gr_rand_threadstack_offset(mm, filp, flags);
/* requested length too big for entire address space */
if (len > TASK_SIZE - mmap_min_addr)
if (flags & MAP_FIXED)
return addr;
+#ifdef CONFIG_PAX_RANDMMAP
+ if (!(mm->pax_flags & MF_PAX_RANDMMAP))
+#endif
+
/* requesting a specific address */
if (addr) {
addr = PAGE_ALIGN(addr);
vma = find_vma(mm, addr);
if (TASK_SIZE - len >= addr && addr >= mmap_min_addr &&
- (!vma || addr + len <= vma->vm_start))
+ check_heap_stack_gap(vma, addr, len, offset))
return addr;
}
info.low_limit = max(PAGE_SIZE, mmap_min_addr);
info.high_limit = mm->mmap_base;
info.align_mask = 0;
+ info.threadstack_offset = offset;
addr = vm_unmapped_area(&info);
/*
VM_BUG_ON(addr != -ENOMEM);
info.flags = 0;
info.low_limit = TASK_UNMAPPED_BASE;
+
+#ifdef CONFIG_PAX_RANDMMAP
+ if (mm->pax_flags & MF_PAX_RANDMMAP)
+ info.low_limit += mm->delta_mmap;
+#endif
+
info.high_limit = TASK_SIZE;
addr = vm_unmapped_area(&info);
}
return vma;
}
+#ifdef CONFIG_PAX_SEGMEXEC
+struct vm_area_struct *pax_find_mirror_vma(struct vm_area_struct *vma)
+{
+ struct vm_area_struct *vma_m;
+
+ BUG_ON(!vma || vma->vm_start >= vma->vm_end);
+ if (!(vma->vm_mm->pax_flags & MF_PAX_SEGMEXEC) || !(vma->vm_flags & VM_EXEC)) {
+ BUG_ON(vma->vm_mirror);
+ return NULL;
+ }
+ BUG_ON(vma->vm_start < SEGMEXEC_TASK_SIZE && SEGMEXEC_TASK_SIZE < vma->vm_end);
+ vma_m = vma->vm_mirror;
+ BUG_ON(!vma_m || vma_m->vm_mirror != vma);
+ BUG_ON(vma->vm_file != vma_m->vm_file);
+ BUG_ON(vma->vm_end - vma->vm_start != vma_m->vm_end - vma_m->vm_start);
+ BUG_ON(vma->vm_pgoff != vma_m->vm_pgoff);
+ BUG_ON(vma->anon_vma != vma_m->anon_vma && vma->anon_vma->root != vma_m->anon_vma->root);
+ BUG_ON((vma->vm_flags ^ vma_m->vm_flags) & ~(VM_WRITE | VM_MAYWRITE | VM_ACCOUNT | VM_LOCKED));
+ return vma_m;
+}
+#endif
+
/*
* Verify that the stack growth is acceptable and
* update accounting. This is shared with both the
/* Stack limit test */
actual_size = size;
- if (size && (vma->vm_flags & (VM_GROWSUP | VM_GROWSDOWN)))
- actual_size -= PAGE_SIZE;
+ gr_learn_resource(current, RLIMIT_STACK, actual_size, 1);
if (actual_size > ACCESS_ONCE(rlim[RLIMIT_STACK].rlim_cur))
return -ENOMEM;
locked = mm->locked_vm + grow;
limit = ACCESS_ONCE(rlim[RLIMIT_MEMLOCK].rlim_cur);
limit >>= PAGE_SHIFT;
+ gr_learn_resource(current, RLIMIT_MEMLOCK, locked << PAGE_SHIFT, 1);
if (locked > limit && !capable(CAP_IPC_LOCK))
return -ENOMEM;
}
* PA-RISC uses this for its stack; IA64 for its Register Backing Store.
* vma is the last one with address > vma->vm_end. Have to extend vma.
*/
+#ifndef CONFIG_IA64
+static
+#endif
int expand_upwards(struct vm_area_struct *vma, unsigned long address)
{
int error;
+ bool locknext;
if (!(vma->vm_flags & VM_GROWSUP))
return -EFAULT;
+ /* Also guard against wrapping around to address 0. */
+ if (address < PAGE_ALIGN(address+1))
+ address = PAGE_ALIGN(address+1);
+ else
+ return -ENOMEM;
+
/*
* We must make sure the anon_vma is allocated
* so that the anon_vma locking is not a noop.
*/
if (unlikely(anon_vma_prepare(vma)))
return -ENOMEM;
+ locknext = vma->vm_next && (vma->vm_next->vm_flags & VM_GROWSDOWN);
+ if (locknext && anon_vma_prepare(vma->vm_next))
+ return -ENOMEM;
vma_lock_anon_vma(vma);
+ if (locknext)
+ vma_lock_anon_vma(vma->vm_next);
/*
* vma->vm_start/vm_end cannot change under us because the caller
* is required to hold the mmap_sem in read mode. We need the
- * anon_vma lock to serialize against concurrent expand_stacks.
- * Also guard against wrapping around to address 0.
+ * anon_vma locks to serialize against concurrent expand_stacks
+ * and expand_upwards.
*/
- if (address < PAGE_ALIGN(address+4))
- address = PAGE_ALIGN(address+4);
- else {
- vma_unlock_anon_vma(vma);
- return -ENOMEM;
- }
error = 0;
/* Somebody else might have raced and expanded it already */
- if (address > vma->vm_end) {
+ if (vma->vm_next && (vma->vm_next->vm_flags & (VM_READ | VM_WRITE | VM_EXEC)) && vma->vm_next->vm_start - address < sysctl_heap_stack_gap)
+ error = -ENOMEM;
+ else if (address > vma->vm_end && (!locknext || vma->vm_next->vm_start >= address)) {
unsigned long size, grow;
size = address - vma->vm_start;
}
}
}
+ if (locknext)
+ vma_unlock_anon_vma(vma->vm_next);
vma_unlock_anon_vma(vma);
khugepaged_enter_vma_merge(vma, vma->vm_flags);
validate_mm(vma->vm_mm);
unsigned long address)
{
int error;
+ bool lockprev = false;
+ struct vm_area_struct *prev;
/*
* We must make sure the anon_vma is allocated
if (error)
return error;
+ prev = vma->vm_prev;
+#if defined(CONFIG_STACK_GROWSUP) || defined(CONFIG_IA64)
+ lockprev = prev && (prev->vm_flags & VM_GROWSUP);
+#endif
+ if (lockprev && anon_vma_prepare(prev))
+ return -ENOMEM;
+ if (lockprev)
+ vma_lock_anon_vma(prev);
+
vma_lock_anon_vma(vma);
/*
*/
/* Somebody else might have raced and expanded it already */
- if (address < vma->vm_start) {
+ if (prev && (prev->vm_flags & (VM_READ | VM_WRITE | VM_EXEC)) && address - prev->vm_end < sysctl_heap_stack_gap)
+ error = -ENOMEM;
+ else if (address < vma->vm_start && (!lockprev || prev->vm_end <= address)) {
unsigned long size, grow;
+#ifdef CONFIG_PAX_SEGMEXEC
+ struct vm_area_struct *vma_m;
+
+ vma_m = pax_find_mirror_vma(vma);
+#endif
+
size = vma->vm_end - address;
grow = (vma->vm_start - address) >> PAGE_SHIFT;
vma->vm_pgoff -= grow;
anon_vma_interval_tree_post_update_vma(vma);
vma_gap_update(vma);
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (vma_m) {
+ anon_vma_interval_tree_pre_update_vma(vma_m);
+ vma_m->vm_start -= grow << PAGE_SHIFT;
+ vma_m->vm_pgoff -= grow;
+ anon_vma_interval_tree_post_update_vma(vma_m);
+ vma_gap_update(vma_m);
+ }
+#endif
+
spin_unlock(&vma->vm_mm->page_table_lock);
+ track_exec_limit(vma->vm_mm, vma->vm_start, vma->vm_end, vma->vm_flags);
perf_event_mmap(vma);
}
}
}
vma_unlock_anon_vma(vma);
+ if (lockprev)
+ vma_unlock_anon_vma(prev);
khugepaged_enter_vma_merge(vma, vma->vm_flags);
validate_mm(vma->vm_mm);
return error;
do {
long nrpages = vma_pages(vma);
+#ifdef CONFIG_PAX_SEGMEXEC
+ if ((mm->pax_flags & MF_PAX_SEGMEXEC) && (vma->vm_start >= SEGMEXEC_TASK_SIZE)) {
+ vma = remove_vma(vma);
+ continue;
+ }
+#endif
+
if (vma->vm_flags & VM_ACCOUNT)
nr_accounted += nrpages;
vm_stat_account(mm, vma->vm_flags, vma->vm_file, -nrpages);
insertion_point = (prev ? &prev->vm_next : &mm->mmap);
vma->vm_prev = NULL;
do {
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (vma->vm_mirror) {
+ BUG_ON(!vma->vm_mirror->vm_mirror || vma->vm_mirror->vm_mirror != vma);
+ vma->vm_mirror->vm_mirror = NULL;
+ vma->vm_mirror->vm_flags &= ~VM_EXEC;
+ vma->vm_mirror = NULL;
+ }
+#endif
+
vma_rb_erase(vma, &mm->mm_rb);
mm->map_count--;
tail_vma = vma;
struct vm_area_struct *new;
int err = -ENOMEM;
+#ifdef CONFIG_PAX_SEGMEXEC
+ struct vm_area_struct *vma_m, *new_m = NULL;
+ unsigned long addr_m = addr + SEGMEXEC_TASK_SIZE;
+#endif
+
if (is_vm_hugetlb_page(vma) && (addr &
~(huge_page_mask(hstate_vma(vma)))))
return -EINVAL;
+#ifdef CONFIG_PAX_SEGMEXEC
+ vma_m = pax_find_mirror_vma(vma);
+#endif
+
new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
if (!new)
goto out_err;
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (vma_m) {
+ new_m = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
+ if (!new_m) {
+ kmem_cache_free(vm_area_cachep, new);
+ goto out_err;
+ }
+ }
+#endif
+
/* most fields are the same, copy all, and then fixup */
*new = *vma;
new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT);
}
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (vma_m) {
+ *new_m = *vma_m;
+ INIT_LIST_HEAD(&new_m->anon_vma_chain);
+ new_m->vm_mirror = new;
+ new->vm_mirror = new_m;
+
+ if (new_below)
+ new_m->vm_end = addr_m;
+ else {
+ new_m->vm_start = addr_m;
+ new_m->vm_pgoff += ((addr_m - vma_m->vm_start) >> PAGE_SHIFT);
+ }
+ }
+#endif
+
err = vma_dup_policy(vma, new);
if (err)
goto out_free_vma;
else
err = vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new);
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (!err && vma_m) {
+ struct mempolicy *pol = vma_policy(new);
+
+ if (anon_vma_clone(new_m, vma_m))
+ goto out_free_mpol;
+
+ mpol_get(pol);
+ set_vma_policy(new_m, pol);
+
+ if (new_m->vm_file)
+ get_file(new_m->vm_file);
+
+ if (new_m->vm_ops && new_m->vm_ops->open)
+ new_m->vm_ops->open(new_m);
+
+ if (new_below)
+ err = vma_adjust(vma_m, addr_m, vma_m->vm_end, vma_m->vm_pgoff +
+ ((addr_m - new_m->vm_start) >> PAGE_SHIFT), new_m);
+ else
+ err = vma_adjust(vma_m, vma_m->vm_start, addr_m, vma_m->vm_pgoff, new_m);
+
+ if (err) {
+ if (new_m->vm_ops && new_m->vm_ops->close)
+ new_m->vm_ops->close(new_m);
+ if (new_m->vm_file)
+ fput(new_m->vm_file);
+ mpol_put(pol);
+ }
+ }
+#endif
+
/* Success. */
if (!err)
return 0;
new->vm_ops->close(new);
if (new->vm_file)
fput(new->vm_file);
- unlink_anon_vmas(new);
out_free_mpol:
mpol_put(vma_policy(new));
out_free_vma:
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (new_m) {
+ unlink_anon_vmas(new_m);
+ kmem_cache_free(vm_area_cachep, new_m);
+ }
+#endif
+
+ unlink_anon_vmas(new);
kmem_cache_free(vm_area_cachep, new);
out_err:
return err;
int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
unsigned long addr, int new_below)
{
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (mm->pax_flags & MF_PAX_SEGMEXEC) {
+ BUG_ON(vma->vm_end > SEGMEXEC_TASK_SIZE);
+ if (mm->map_count >= sysctl_max_map_count-1)
+ return -ENOMEM;
+ } else
+#endif
+
if (mm->map_count >= sysctl_max_map_count)
return -ENOMEM;
* work. This now handles partial unmappings.
* Jeremy Fitzhardinge <jeremy@goop.org>
*/
+#ifdef CONFIG_PAX_SEGMEXEC
int do_munmap(struct mm_struct *mm, unsigned long start, size_t len)
+{
+ int ret = __do_munmap(mm, start, len);
+ if (ret || !(mm->pax_flags & MF_PAX_SEGMEXEC))
+ return ret;
+
+ return __do_munmap(mm, start + SEGMEXEC_TASK_SIZE, len);
+}
+
+int __do_munmap(struct mm_struct *mm, unsigned long start, size_t len)
+#else
+int do_munmap(struct mm_struct *mm, unsigned long start, size_t len)
+#endif
{
unsigned long end;
struct vm_area_struct *vma, *prev, *last;
+ /*
+ * mm->mmap_sem is required to protect against another thread
+ * changing the mappings in case we sleep.
+ */
+ verify_mm_writelocked(mm);
+
if ((start & ~PAGE_MASK) || start > TASK_SIZE || len > TASK_SIZE-start)
return -EINVAL;
/* Fix up all other VM information */
remove_vma_list(mm, vma);
+ track_exec_limit(mm, start, end, 0UL);
+
return 0;
}
int ret;
struct mm_struct *mm = current->mm;
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if ((mm->pax_flags & MF_PAX_SEGMEXEC) &&
+ (len > SEGMEXEC_TASK_SIZE || start > SEGMEXEC_TASK_SIZE-len))
+ return -EINVAL;
+#endif
+
down_write(&mm->mmap_sem);
ret = do_munmap(mm, start, len);
up_write(&mm->mmap_sem);
return vm_munmap(addr, len);
}
-static inline void verify_mm_writelocked(struct mm_struct *mm)
-{
-#ifdef CONFIG_DEBUG_VM
- if (unlikely(down_read_trylock(&mm->mmap_sem))) {
- WARN_ON(1);
- up_read(&mm->mmap_sem);
- }
-#endif
-}
-
/*
* this is really a simplified "do_mmap". it only handles
* anonymous maps. eventually we may be able to do some
struct rb_node **rb_link, *rb_parent;
pgoff_t pgoff = addr >> PAGE_SHIFT;
int error;
+ unsigned long charged;
len = PAGE_ALIGN(len);
if (!len)
flags = VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags;
+#if defined(CONFIG_PAX_PAGEEXEC) || defined(CONFIG_PAX_SEGMEXEC)
+ if (mm->pax_flags & (MF_PAX_PAGEEXEC | MF_PAX_SEGMEXEC)) {
+ flags &= ~VM_EXEC;
+
+#ifdef CONFIG_PAX_MPROTECT
+ if (mm->pax_flags & MF_PAX_MPROTECT)
+ flags &= ~VM_MAYEXEC;
+#endif
+
+ }
+#endif
+
error = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED);
if (error & ~PAGE_MASK)
return error;
+ charged = len >> PAGE_SHIFT;
+
error = mlock_future_check(mm, mm->def_flags, len);
if (error)
return error;
/*
* Clear old maps. this also does some error checking for us
*/
- munmap_back:
if (find_vma_links(mm, addr, addr + len, &prev, &rb_link, &rb_parent)) {
if (do_munmap(mm, addr, len))
return -ENOMEM;
- goto munmap_back;
+ BUG_ON(find_vma_links(mm, addr, addr + len, &prev, &rb_link, &rb_parent));
}
/* Check against address space limits *after* clearing old maps... */
- if (!may_expand_vm(mm, len >> PAGE_SHIFT))
+ if (!may_expand_vm(mm, charged))
return -ENOMEM;
if (mm->map_count > sysctl_max_map_count)
return -ENOMEM;
- if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT))
+ if (security_vm_enough_memory_mm(mm, charged))
return -ENOMEM;
/* Can we just expand an old private anonymous mapping? */
*/
vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
if (!vma) {
- vm_unacct_memory(len >> PAGE_SHIFT);
+ vm_unacct_memory(charged);
return -ENOMEM;
}
vma_link(mm, vma, prev, rb_link, rb_parent);
out:
perf_event_mmap(vma);
- mm->total_vm += len >> PAGE_SHIFT;
+ mm->total_vm += charged;
if (flags & VM_LOCKED)
- mm->locked_vm += (len >> PAGE_SHIFT);
+ mm->locked_vm += charged;
vma->vm_flags |= VM_SOFTDIRTY;
+ track_exec_limit(mm, addr, addr + len, flags);
return addr;
}
while (vma) {
if (vma->vm_flags & VM_ACCOUNT)
nr_accounted += vma_pages(vma);
+ vma->vm_mirror = NULL;
vma = remove_vma(vma);
}
vm_unacct_memory(nr_accounted);
struct vm_area_struct *prev;
struct rb_node **rb_link, *rb_parent;
+#ifdef CONFIG_PAX_SEGMEXEC
+ struct vm_area_struct *vma_m = NULL;
+#endif
+
+ if (security_mmap_addr(vma->vm_start))
+ return -EPERM;
+
/*
* The vm_pgoff of a purely anonymous vma should be irrelevant
* until its first write fault, when page's anon_vma and index
security_vm_enough_memory_mm(mm, vma_pages(vma)))
return -ENOMEM;
+#ifdef CONFIG_PAX_SEGMEXEC
+ if ((mm->pax_flags & MF_PAX_SEGMEXEC) && (vma->vm_flags & VM_EXEC)) {
+ vma_m = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
+ if (!vma_m)
+ return -ENOMEM;
+ }
+#endif
+
vma_link(mm, vma, prev, rb_link, rb_parent);
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (vma_m)
+ BUG_ON(pax_mirror_vma(vma_m, vma));
+#endif
+
return 0;
}
struct rb_node **rb_link, *rb_parent;
bool faulted_in_anon_vma = true;
+ BUG_ON(vma->vm_mirror);
+
/*
* If anonymous vma has not yet been faulted, update new pgoff
* to match new location, to increase its chance of merging.
return NULL;
}
+#ifdef CONFIG_PAX_SEGMEXEC
+long pax_mirror_vma(struct vm_area_struct *vma_m, struct vm_area_struct *vma)
+{
+ struct vm_area_struct *prev_m;
+ struct rb_node **rb_link_m, *rb_parent_m;
+ struct mempolicy *pol_m;
+
+ BUG_ON(!(vma->vm_mm->pax_flags & MF_PAX_SEGMEXEC) || !(vma->vm_flags & VM_EXEC));
+ BUG_ON(vma->vm_mirror || vma_m->vm_mirror);
+ BUG_ON(!mpol_equal(vma_policy(vma), vma_policy(vma_m)));
+ *vma_m = *vma;
+ INIT_LIST_HEAD(&vma_m->anon_vma_chain);
+ if (anon_vma_clone(vma_m, vma))
+ return -ENOMEM;
+ pol_m = vma_policy(vma_m);
+ mpol_get(pol_m);
+ set_vma_policy(vma_m, pol_m);
+ vma_m->vm_start += SEGMEXEC_TASK_SIZE;
+ vma_m->vm_end += SEGMEXEC_TASK_SIZE;
+ vma_m->vm_flags &= ~(VM_WRITE | VM_MAYWRITE | VM_ACCOUNT | VM_LOCKED);
+ vma_m->vm_page_prot = vm_get_page_prot(vma_m->vm_flags);
+ if (vma_m->vm_file)
+ get_file(vma_m->vm_file);
+ if (vma_m->vm_ops && vma_m->vm_ops->open)
+ vma_m->vm_ops->open(vma_m);
+ BUG_ON(find_vma_links(vma->vm_mm, vma_m->vm_start, vma_m->vm_end, &prev_m, &rb_link_m, &rb_parent_m));
+ vma_link(vma->vm_mm, vma_m, prev_m, rb_link_m, rb_parent_m);
+ vma_m->vm_mirror = vma;
+ vma->vm_mirror = vma_m;
+ return 0;
+}
+#endif
+
/*
* Return true if the calling process may expand its vm space by the passed
* number of pages
lim = rlimit(RLIMIT_AS) >> PAGE_SHIFT;
+ gr_learn_resource(current, RLIMIT_AS, (cur + npages) << PAGE_SHIFT, 1);
if (cur + npages > lim)
return 0;
return 1;
vma->vm_start = addr;
vma->vm_end = addr + len;
+#ifdef CONFIG_PAX_MPROTECT
+ if (mm->pax_flags & MF_PAX_MPROTECT) {
+#ifndef CONFIG_PAX_MPROTECT_COMPAT
+ if ((vm_flags & (VM_WRITE | VM_EXEC)) == (VM_WRITE | VM_EXEC))
+ return ERR_PTR(-EPERM);
+ if (!(vm_flags & VM_EXEC))
+ vm_flags &= ~VM_MAYEXEC;
+#else
+ if ((vm_flags & (VM_WRITE | VM_EXEC)) != VM_EXEC)
+ vm_flags &= ~(VM_EXEC | VM_MAYEXEC);
+#endif
+ else
+ vm_flags &= ~VM_MAYWRITE;
+ }
+#endif
+
vma->vm_flags = vm_flags | mm->def_flags | VM_DONTEXPAND | VM_SOFTDIRTY;
vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
#include <linux/migrate.h>
#include <linux/perf_event.h>
#include <linux/ksm.h>
+#include <linux/sched/sysctl.h>
+
+#ifdef CONFIG_PAX_MPROTECT
+#include <linux/elf.h>
+#include <linux/binfmts.h>
+#endif
+
#include <asm/uaccess.h>
#include <asm/pgtable.h>
#include <asm/cacheflush.h>
#include <asm/tlbflush.h>
+#include <asm/mmu_context.h>
/*
* For a prot_numa update we only hold mmap_sem for read so there is a
return pages;
}
+#ifdef CONFIG_ARCH_TRACK_EXEC_LIMIT
+/* called while holding the mmap semaphor for writing except stack expansion */
+void track_exec_limit(struct mm_struct *mm, unsigned long start, unsigned long end, unsigned long prot)
+{
+ unsigned long oldlimit, newlimit = 0UL;
+
+ if (!(mm->pax_flags & MF_PAX_PAGEEXEC) || (__supported_pte_mask & _PAGE_NX))
+ return;
+
+ spin_lock(&mm->page_table_lock);
+ oldlimit = mm->context.user_cs_limit;
+ if ((prot & VM_EXEC) && oldlimit < end)
+ /* USER_CS limit moved up */
+ newlimit = end;
+ else if (!(prot & VM_EXEC) && start < oldlimit && oldlimit <= end)
+ /* USER_CS limit moved down */
+ newlimit = start;
+
+ if (newlimit) {
+ mm->context.user_cs_limit = newlimit;
+
+#ifdef CONFIG_SMP
+ wmb();
+ cpus_clear(mm->context.cpu_user_cs_mask);
+ cpu_set(smp_processor_id(), mm->context.cpu_user_cs_mask);
+#endif
+
+ set_user_cs(mm->context.user_cs_base, mm->context.user_cs_limit, smp_processor_id());
+ }
+ spin_unlock(&mm->page_table_lock);
+ if (newlimit == end) {
+ struct vm_area_struct *vma = find_vma(mm, oldlimit);
+
+ for (; vma && vma->vm_start < end; vma = vma->vm_next)
+ if (is_vm_hugetlb_page(vma))
+ hugetlb_change_protection(vma, vma->vm_start, vma->vm_end, vma->vm_page_prot);
+ else
+ change_protection(vma, vma->vm_start, vma->vm_end, vma->vm_page_prot, vma_wants_writenotify(vma), 0);
+ }
+}
+#endif
+
int
mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev,
unsigned long start, unsigned long end, unsigned long newflags)
int error;
int dirty_accountable = 0;
+#ifdef CONFIG_PAX_SEGMEXEC
+ struct vm_area_struct *vma_m = NULL;
+ unsigned long start_m, end_m;
+
+ start_m = start + SEGMEXEC_TASK_SIZE;
+ end_m = end + SEGMEXEC_TASK_SIZE;
+#endif
+
if (newflags == oldflags) {
*pprev = vma;
return 0;
}
+ if (newflags & (VM_READ | VM_WRITE | VM_EXEC)) {
+ struct vm_area_struct *prev = vma->vm_prev, *next = vma->vm_next;
+
+ if (next && (next->vm_flags & VM_GROWSDOWN) && sysctl_heap_stack_gap > next->vm_start - end)
+ return -ENOMEM;
+
+ if (prev && (prev->vm_flags & VM_GROWSUP) && sysctl_heap_stack_gap > start - prev->vm_end)
+ return -ENOMEM;
+ }
+
/*
* If we make a private mapping writable we increase our commit;
* but (without finer accounting) cannot reduce our commit if we
}
}
+#ifdef CONFIG_PAX_SEGMEXEC
+ if ((mm->pax_flags & MF_PAX_SEGMEXEC) && ((oldflags ^ newflags) & VM_EXEC)) {
+ if (start != vma->vm_start) {
+ error = split_vma(mm, vma, start, 1);
+ if (error)
+ goto fail;
+ BUG_ON(!*pprev || (*pprev)->vm_next == vma);
+ *pprev = (*pprev)->vm_next;
+ }
+
+ if (end != vma->vm_end) {
+ error = split_vma(mm, vma, end, 0);
+ if (error)
+ goto fail;
+ }
+
+ if (pax_find_mirror_vma(vma)) {
+ error = __do_munmap(mm, start_m, end_m - start_m);
+ if (error)
+ goto fail;
+ } else {
+ vma_m = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
+ if (!vma_m) {
+ error = -ENOMEM;
+ goto fail;
+ }
+ vma->vm_flags = newflags;
+ error = pax_mirror_vma(vma_m, vma);
+ if (error) {
+ vma->vm_flags = oldflags;
+ goto fail;
+ }
+ }
+ }
+#endif
+
/*
* First try to merge with previous and/or next vma.
*/
* vm_flags and vm_page_prot are protected by the mmap_sem
* held in write mode.
*/
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if ((mm->pax_flags & MF_PAX_SEGMEXEC) && (newflags & VM_EXEC) && ((vma->vm_flags ^ newflags) & VM_READ))
+ pax_find_mirror_vma(vma)->vm_flags ^= VM_READ;
+#endif
+
vma->vm_flags = newflags;
+
+#ifdef CONFIG_PAX_MPROTECT
+ if (mm->binfmt && mm->binfmt->handle_mprotect)
+ mm->binfmt->handle_mprotect(vma, newflags);
+#endif
+
dirty_accountable = vma_wants_writenotify(vma);
vma_set_page_prot(vma);
end = start + len;
if (end <= start)
return -ENOMEM;
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (current->mm->pax_flags & MF_PAX_SEGMEXEC) {
+ if (end > SEGMEXEC_TASK_SIZE)
+ return -EINVAL;
+ } else
+#endif
+
+ if (end > TASK_SIZE)
+ return -EINVAL;
+
if (!arch_validate_prot(prot))
return -EINVAL;
/*
* Does the application expect PROT_READ to imply PROT_EXEC:
*/
- if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC))
+ if ((prot & (PROT_READ | PROT_WRITE)) && (current->personality & READ_IMPLIES_EXEC))
prot |= PROT_EXEC;
vm_flags = calc_vm_prot_bits(prot);
if (start > vma->vm_start)
prev = vma;
+#ifdef CONFIG_PAX_MPROTECT
+ if (current->mm->binfmt && current->mm->binfmt->handle_mprotect)
+ current->mm->binfmt->handle_mprotect(vma, vm_flags);
+#endif
+
for (nstart = start ; ; ) {
unsigned long newflags;
/* newflags >> 4 shift VM_MAY% in place of VM_% */
if ((newflags & ~(newflags >> 4)) & (VM_READ | VM_WRITE | VM_EXEC)) {
+ if (prot & (PROT_WRITE | PROT_EXEC))
+ gr_log_rwxmprotect(vma);
+
+ error = -EACCES;
+ goto out;
+ }
+
+ if (!gr_acl_handle_mprotect(vma->vm_file, prot)) {
error = -EACCES;
goto out;
}
error = mprotect_fixup(vma, &prev, nstart, tmp, newflags);
if (error)
goto out;
+
+ track_exec_limit(current->mm, nstart, tmp, vm_flags);
+
nstart = tmp;
if (nstart < prev->vm_end)
continue;
pte = ptep_get_and_clear(mm, old_addr, old_pte);
pte = move_pte(pte, new_vma->vm_page_prot, old_addr, new_addr);
+
+#ifdef CONFIG_ARCH_TRACK_EXEC_LIMIT
+ if (!(__supported_pte_mask & _PAGE_NX) && pte_present(pte) && (new_vma->vm_flags & (VM_PAGEEXEC | VM_EXEC)) == VM_PAGEEXEC)
+ pte = pte_exprotect(pte);
+#endif
+
pte = move_soft_dirty_pte(pte);
set_pte_at(mm, new_addr, new_pte, pte);
}
if (is_vm_hugetlb_page(vma))
goto Einval;
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (pax_find_mirror_vma(vma))
+ goto Einval;
+#endif
+
/* We can't remap across vm area boundaries */
if (old_len > vma->vm_end - addr)
goto Efault;
unsigned long ret = -EINVAL;
unsigned long charged = 0;
unsigned long map_flags;
+ unsigned long pax_task_size = TASK_SIZE;
if (new_addr & ~PAGE_MASK)
goto out;
- if (new_len > TASK_SIZE || new_addr > TASK_SIZE - new_len)
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (mm->pax_flags & MF_PAX_SEGMEXEC)
+ pax_task_size = SEGMEXEC_TASK_SIZE;
+#endif
+
+ pax_task_size -= PAGE_SIZE;
+
+ if (new_len > TASK_SIZE || new_addr > pax_task_size - new_len)
goto out;
/* Check if the location we're moving into overlaps the
* old location at all, and fail if it does.
*/
- if ((new_addr <= addr) && (new_addr+new_len) > addr)
- goto out;
-
- if ((addr <= new_addr) && (addr+old_len) > new_addr)
+ if (addr + old_len > new_addr && new_addr + new_len > addr)
goto out;
ret = do_munmap(mm, new_addr, new_len);
unsigned long ret = -EINVAL;
unsigned long charged = 0;
bool locked = false;
+ unsigned long pax_task_size = TASK_SIZE;
if (flags & ~(MREMAP_FIXED | MREMAP_MAYMOVE))
return ret;
if (!new_len)
return ret;
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (mm->pax_flags & MF_PAX_SEGMEXEC)
+ pax_task_size = SEGMEXEC_TASK_SIZE;
+#endif
+
+ pax_task_size -= PAGE_SIZE;
+
+ if (new_len > pax_task_size || addr > pax_task_size-new_len ||
+ old_len > pax_task_size || addr > pax_task_size-old_len)
+ return ret;
+
down_write(¤t->mm->mmap_sem);
if (flags & MREMAP_FIXED) {
new_addr = addr;
}
ret = addr;
+ track_exec_limit(vma->vm_mm, vma->vm_start, addr + new_len, vma->vm_flags);
goto out;
}
}
goto out;
}
+ map_flags = vma->vm_flags;
ret = move_vma(vma, addr, old_len, new_len, new_addr, &locked);
+ if (!(ret & ~PAGE_MASK)) {
+ track_exec_limit(current->mm, addr, addr + old_len, 0UL);
+ track_exec_limit(current->mm, new_addr, new_addr + new_len, map_flags);
+ }
}
out:
if (ret & ~PAGE_MASK)
int sysctl_nr_trim_pages = CONFIG_NOMMU_INITIAL_TRIM_EXCESS;
unsigned long sysctl_user_reserve_kbytes __read_mostly = 1UL << 17; /* 128MB */
unsigned long sysctl_admin_reserve_kbytes __read_mostly = 1UL << 13; /* 8MB */
-int heap_stack_gap = 0;
atomic_long_t mmap_pages_allocated;
}
EXPORT_SYMBOL(find_vma);
-/*
- * find a VMA
- * - we don't extend stack VMAs under NOMMU conditions
- */
-struct vm_area_struct *find_extend_vma(struct mm_struct *mm, unsigned long addr)
-{
- return find_vma(mm, addr);
-}
-
/*
* expand a stack to a given address
* - not supported under NOMMU conditions
/* most fields are the same, copy all, and then fixup */
*new = *vma;
+ INIT_LIST_HEAD(&new->anon_vma_chain);
*region = *vma->vm_region;
new->vm_region = region;
}
EXPORT_SYMBOL(generic_file_remap_pages);
-static int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
- unsigned long addr, void *buf, int len, int write)
+static ssize_t __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
+ unsigned long addr, void *buf, size_t len, int write)
{
struct vm_area_struct *vma;
*
* The caller must hold a reference on @mm.
*/
-int access_remote_vm(struct mm_struct *mm, unsigned long addr,
- void *buf, int len, int write)
+ssize_t access_remote_vm(struct mm_struct *mm, unsigned long addr,
+ void *buf, size_t len, int write)
{
return __access_remote_vm(NULL, mm, addr, buf, len, write);
}
* Access another process' address space.
* - source/target buffer must be kernel space
*/
-int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write)
+ssize_t access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, size_t len, int write)
{
struct mm_struct *mm;
* card's bdi_dirty may rush to many times higher than bdi_setpoint.
* - the bdi dirty thresh drops quickly due to change of JBOD workload
*/
-static unsigned long bdi_position_ratio(struct backing_dev_info *bdi,
+static unsigned long __intentional_overflow(-1) bdi_position_ratio(struct backing_dev_info *bdi,
unsigned long thresh,
unsigned long bg_thresh,
unsigned long dirty,
#include <linux/hugetlb.h>
#include <linux/sched/rt.h>
#include <linux/page_owner.h>
+#include <linux/random.h>
#include <asm/sections.h>
#include <asm/tlbflush.h>
* This usage means that zero-order pages may not be compound.
*/
-static void free_compound_page(struct page *page)
+void free_compound_page(struct page *page)
{
__free_pages_ok(page, compound_order(page));
}
__mod_zone_freepage_state(zone, (1 << order), migratetype);
}
#else
-struct page_ext_operations debug_guardpage_ops = { NULL, };
+struct page_ext_operations debug_guardpage_ops = { .need = NULL, .init = NULL };
static inline void set_page_guard(struct zone *zone, struct page *page,
unsigned int order, int migratetype) {}
static inline void clear_page_guard(struct zone *zone, struct page *page,
int i;
int bad = 0;
+#ifdef CONFIG_PAX_MEMORY_SANITIZE
+ unsigned long index = 1UL << order;
+#endif
+
VM_BUG_ON_PAGE(PageTail(page), page);
VM_BUG_ON_PAGE(PageHead(page) && compound_order(page) != order, page);
debug_check_no_obj_freed(page_address(page),
PAGE_SIZE << order);
}
+
+#ifdef CONFIG_PAX_MEMORY_SANITIZE
+ for (; index; --index)
+ sanitize_highpage(page + index - 1);
+#endif
+
arch_free_page(page, order);
kernel_map_pages(page, 1 << order, 0);
local_irq_restore(flags);
}
+#ifdef CONFIG_PAX_LATENT_ENTROPY
+bool __meminitdata extra_latent_entropy;
+
+static int __init setup_pax_extra_latent_entropy(char *str)
+{
+ extra_latent_entropy = true;
+ return 0;
+}
+early_param("pax_extra_latent_entropy", setup_pax_extra_latent_entropy);
+
+volatile u64 latent_entropy __latent_entropy;
+EXPORT_SYMBOL(latent_entropy);
+#endif
+
void __init __free_pages_bootmem(struct page *page, unsigned int order)
{
unsigned int nr_pages = 1 << order;
__ClearPageReserved(p);
set_page_count(p, 0);
+#ifdef CONFIG_PAX_LATENT_ENTROPY
+ if (extra_latent_entropy && !PageHighMem(page) && page_to_pfn(page) < 0x100000) {
+ u64 hash = 0;
+ size_t index, end = PAGE_SIZE * nr_pages / sizeof hash;
+ const u64 *data = lowmem_page_address(page);
+
+ for (index = 0; index < end; index++)
+ hash ^= hash + data[index];
+ latent_entropy ^= hash;
+ add_device_randomness((const void *)&latent_entropy, sizeof(latent_entropy));
+ }
+#endif
+
page_zone(page)->managed_pages += nr_pages;
set_page_refcounted(page);
__free_pages(page, order);
arch_alloc_page(page, order);
kernel_map_pages(page, 1 << order, 1);
+#ifndef CONFIG_PAX_MEMORY_SANITIZE
if (gfp_flags & __GFP_ZERO)
prep_zero_page(page, order, gfp_flags);
+#endif
if (order && (gfp_flags & __GFP_COMP))
prep_compound_page(page, order);
}
__mod_zone_page_state(zone, NR_ALLOC_BATCH, -(1 << order));
- if (atomic_long_read(&zone->vm_stat[NR_ALLOC_BATCH]) <= 0 &&
+ if (atomic_long_read_unchecked(&zone->vm_stat[NR_ALLOC_BATCH]) <= 0 &&
!test_bit(ZONE_FAIR_DEPLETED, &zone->flags))
set_bit(ZONE_FAIR_DEPLETED, &zone->flags);
do {
mod_zone_page_state(zone, NR_ALLOC_BATCH,
high_wmark_pages(zone) - low_wmark_pages(zone) -
- atomic_long_read(&zone->vm_stat[NR_ALLOC_BATCH]));
+ atomic_long_read_unchecked(&zone->vm_stat[NR_ALLOC_BATCH]));
clear_bit(ZONE_FAIR_DEPLETED, &zone->flags);
} while (zone++ != preferred_zone);
}
__mod_zone_page_state(zone, NR_ALLOC_BATCH,
high_wmark_pages(zone) - low_wmark_pages(zone) -
- atomic_long_read(&zone->vm_stat[NR_ALLOC_BATCH]));
+ atomic_long_read_unchecked(&zone->vm_stat[NR_ALLOC_BATCH]));
setup_zone_migrate_reserve(zone);
spin_unlock_irqrestore(&zone->lock, flags);
static unsigned int pcpu_high_unit_cpu __read_mostly;
/* the address of the first chunk which starts with the kernel static area */
-void *pcpu_base_addr __read_mostly;
+void *pcpu_base_addr __read_only;
EXPORT_SYMBOL_GPL(pcpu_base_addr);
static const int *pcpu_unit_map __read_mostly; /* cpu -> unit */
#include <linux/uio.h>
#include <linux/sched.h>
#include <linux/highmem.h>
+#include <linux/security.h>
#include <linux/ptrace.h>
#include <linux/slab.h>
#include <linux/syscalls.h>
ssize_t iov_len;
size_t total_len = iov_iter_count(iter);
+ return -ENOSYS; // PaX: until properly audited
+
/*
* Work out how many pages of struct pages we're going to need
* when eventually calling get_user_pages
*/
for (i = 0; i < riovcnt; i++) {
iov_len = rvec[i].iov_len;
- if (iov_len > 0) {
- nr_pages_iov = ((unsigned long)rvec[i].iov_base
- + iov_len)
- / PAGE_SIZE - (unsigned long)rvec[i].iov_base
- / PAGE_SIZE + 1;
- nr_pages = max(nr_pages, nr_pages_iov);
- }
+ if (iov_len <= 0)
+ continue;
+ nr_pages_iov = ((unsigned long)rvec[i].iov_base + iov_len) / PAGE_SIZE -
+ (unsigned long)rvec[i].iov_base / PAGE_SIZE + 1;
+ nr_pages = max(nr_pages, nr_pages_iov);
}
if (nr_pages == 0)
goto free_proc_pages;
}
+ if (gr_handle_ptrace(task, vm_write ? PTRACE_POKETEXT : PTRACE_ATTACH)) {
+ rc = -EPERM;
+ goto put_task_struct;
+ }
+
mm = mm_access(task, PTRACE_MODE_ATTACH);
if (!mm || IS_ERR(mm)) {
rc = IS_ERR(mm) ? PTR_ERR(mm) : -ESRCH;
struct anon_vma *anon_vma = vma->anon_vma;
struct anon_vma_chain *avc;
+#ifdef CONFIG_PAX_SEGMEXEC
+ struct anon_vma_chain *avc_m = NULL;
+#endif
+
might_sleep();
if (unlikely(!anon_vma)) {
struct mm_struct *mm = vma->vm_mm;
if (!avc)
goto out_enomem;
+#ifdef CONFIG_PAX_SEGMEXEC
+ avc_m = anon_vma_chain_alloc(GFP_KERNEL);
+ if (!avc_m)
+ goto out_enomem_free_avc;
+#endif
+
anon_vma = find_mergeable_anon_vma(vma);
allocated = NULL;
if (!anon_vma) {
/* page_table_lock to protect against threads */
spin_lock(&mm->page_table_lock);
if (likely(!vma->anon_vma)) {
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ struct vm_area_struct *vma_m = pax_find_mirror_vma(vma);
+
+ if (vma_m) {
+ BUG_ON(vma_m->anon_vma);
+ vma_m->anon_vma = anon_vma;
+ anon_vma_chain_link(vma_m, avc_m, anon_vma);
+ anon_vma->degree++;
+ avc_m = NULL;
+ }
+#endif
+
vma->anon_vma = anon_vma;
anon_vma_chain_link(vma, avc, anon_vma);
/* vma reference or self-parent link for new root */
if (unlikely(allocated))
put_anon_vma(allocated);
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (unlikely(avc_m))
+ anon_vma_chain_free(avc_m);
+#endif
+
if (unlikely(avc))
anon_vma_chain_free(avc);
}
return 0;
out_enomem_free_avc:
+
+#ifdef CONFIG_PAX_SEGMEXEC
+ if (avc_m)
+ anon_vma_chain_free(avc_m);
+#endif
+
anon_vma_chain_free(avc);
out_enomem:
return -ENOMEM;
* good chance of avoiding scanning the whole hierarchy when it searches where
* page is mapped.
*/
-int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src)
+int anon_vma_clone(struct vm_area_struct *dst, const struct vm_area_struct *src)
{
struct anon_vma_chain *avc, *pavc;
struct anon_vma *root = NULL;
* the corresponding VMA in the parent process is attached to.
* Returns 0 on success, non-zero on failure.
*/
-int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma)
+int anon_vma_fork(struct vm_area_struct *vma, const struct vm_area_struct *pvma)
{
struct anon_vma_chain *avc;
struct anon_vma *anon_vma;
void __init anon_vma_init(void)
{
anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma),
- 0, SLAB_DESTROY_BY_RCU|SLAB_PANIC, anon_vma_ctor);
- anon_vma_chain_cachep = KMEM_CACHE(anon_vma_chain, SLAB_PANIC);
+ 0, SLAB_DESTROY_BY_RCU|SLAB_PANIC|SLAB_NO_SANITIZE,
+ anon_vma_ctor);
+ anon_vma_chain_cachep = KMEM_CACHE(anon_vma_chain,
+ SLAB_PANIC|SLAB_NO_SANITIZE);
}
/*
#include <linux/swap.h>
#include <linux/aio.h>
-static struct vfsmount *shm_mnt;
+struct vfsmount *shm_mnt;
#ifdef CONFIG_SHMEM
/*
#define BOGO_DIRENT_SIZE 20
/* Symlink up to this size is kmalloc'ed instead of using a swappable page */
-#define SHORT_SYMLINK_LEN 128
+#define SHORT_SYMLINK_LEN 64
/*
* shmem_fallocate communicates with shmem_fault or shmem_writepage via
static int shmem_xattr_validate(const char *name)
{
struct { const char *prefix; size_t len; } arr[] = {
+
+#ifdef CONFIG_PAX_XATTR_PAX_FLAGS
+ { XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN},
+#endif
+
{ XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN },
{ XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN }
};
if (err)
return err;
+#ifdef CONFIG_PAX_XATTR_PAX_FLAGS
+ if (!strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN)) {
+ if (strcmp(name, XATTR_NAME_PAX_FLAGS))
+ return -EOPNOTSUPP;
+ if (size > 8)
+ return -EINVAL;
+ }
+#endif
+
return simple_xattr_set(&info->xattrs, name, value, size, flags);
}
int err = -ENOMEM;
/* Round up to L1_CACHE_BYTES to resist false sharing */
- sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
- L1_CACHE_BYTES), GFP_KERNEL);
+ sbinfo = kzalloc(max(sizeof(struct shmem_sb_info), L1_CACHE_BYTES), GFP_KERNEL);
if (!sbinfo)
return -ENOMEM;
if ((x)->max_freeable < i) \
(x)->max_freeable = i; \
} while (0)
-#define STATS_INC_ALLOCHIT(x) atomic_inc(&(x)->allochit)
-#define STATS_INC_ALLOCMISS(x) atomic_inc(&(x)->allocmiss)
-#define STATS_INC_FREEHIT(x) atomic_inc(&(x)->freehit)
-#define STATS_INC_FREEMISS(x) atomic_inc(&(x)->freemiss)
+#define STATS_INC_ALLOCHIT(x) atomic_inc_unchecked(&(x)->allochit)
+#define STATS_INC_ALLOCMISS(x) atomic_inc_unchecked(&(x)->allocmiss)
+#define STATS_INC_FREEHIT(x) atomic_inc_unchecked(&(x)->freehit)
+#define STATS_INC_FREEMISS(x) atomic_inc_unchecked(&(x)->freemiss)
+#define STATS_INC_SANITIZED(x) atomic_inc_unchecked(&(x)->sanitized)
+#define STATS_INC_NOT_SANITIZED(x) atomic_inc_unchecked(&(x)->not_sanitized)
#else
#define STATS_INC_ACTIVE(x) do { } while (0)
#define STATS_DEC_ACTIVE(x) do { } while (0)
#define STATS_INC_ALLOCMISS(x) do { } while (0)
#define STATS_INC_FREEHIT(x) do { } while (0)
#define STATS_INC_FREEMISS(x) do { } while (0)
+#define STATS_INC_SANITIZED(x) do { } while (0)
+#define STATS_INC_NOT_SANITIZED(x) do { } while (0)
#endif
#if DEBUG
* reciprocal_divide(offset, cache->reciprocal_buffer_size)
*/
static inline unsigned int obj_to_index(const struct kmem_cache *cache,
- const struct page *page, void *obj)
+ const struct page *page, const void *obj)
{
u32 offset = (obj - page->s_mem);
return reciprocal_divide(offset, cache->reciprocal_buffer_size);
* structures first. Without this, further allocations will bug.
*/
kmalloc_caches[INDEX_NODE] = create_kmalloc_cache("kmalloc-node",
- kmalloc_size(INDEX_NODE), ARCH_KMALLOC_FLAGS);
+ kmalloc_size(INDEX_NODE), SLAB_USERCOPY | ARCH_KMALLOC_FLAGS);
slab_state = PARTIAL_NODE;
slab_early_init = 0;
cachep = find_mergeable(size, align, flags, name, ctor);
if (cachep) {
- cachep->refcount++;
+ atomic_inc(&cachep->refcount);
/*
* Adjust the object sizes so that we clear
struct array_cache *ac = cpu_cache_get(cachep);
check_irq_off();
+
+#ifdef CONFIG_PAX_MEMORY_SANITIZE
+ if (cachep->flags & (SLAB_POISON | SLAB_NO_SANITIZE))
+ STATS_INC_NOT_SANITIZED(cachep);
+ else {
+ memset(objp, PAX_MEMORY_SANITIZE_VALUE, cachep->object_size);
+
+ if (cachep->ctor)
+ cachep->ctor(objp);
+
+ STATS_INC_SANITIZED(cachep);
+ }
+#endif
+
kmemleak_free_recursive(objp, cachep->flags);
objp = cache_free_debugcheck(cachep, objp, caller);
return kmem_cache_alloc_node_trace(cachep, flags, node, size);
}
-void *__kmalloc_node(size_t size, gfp_t flags, int node)
+void * __size_overflow(1) __kmalloc_node(size_t size, gfp_t flags, int node)
{
return __do_kmalloc_node(size, flags, node, _RET_IP_);
}
* @flags: the type of memory to allocate (see kmalloc).
* @caller: function caller for debug tracking of the caller
*/
-static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
+static __always_inline void * __size_overflow(1) __do_kmalloc(size_t size, gfp_t flags,
unsigned long caller)
{
struct kmem_cache *cachep;
if (unlikely(ZERO_OR_NULL_PTR(objp)))
return;
+ VM_BUG_ON(!virt_addr_valid(objp));
local_irq_save(flags);
kfree_debugcheck(objp);
c = virt_to_cache(objp);
}
/* cpu stats */
{
- unsigned long allochit = atomic_read(&cachep->allochit);
- unsigned long allocmiss = atomic_read(&cachep->allocmiss);
- unsigned long freehit = atomic_read(&cachep->freehit);
- unsigned long freemiss = atomic_read(&cachep->freemiss);
+ unsigned long allochit = atomic_read_unchecked(&cachep->allochit);
+ unsigned long allocmiss = atomic_read_unchecked(&cachep->allocmiss);
+ unsigned long freehit = atomic_read_unchecked(&cachep->freehit);
+ unsigned long freemiss = atomic_read_unchecked(&cachep->freemiss);
seq_printf(m, " : cpustat %6lu %6lu %6lu %6lu",
allochit, allocmiss, freehit, freemiss);
}
+#ifdef CONFIG_PAX_MEMORY_SANITIZE
+ {
+ unsigned long sanitized = atomic_read_unchecked(&cachep->sanitized);
+ unsigned long not_sanitized = atomic_read_unchecked(&cachep->not_sanitized);
+
+ seq_printf(m, " : pax %6lu %6lu", sanitized, not_sanitized);
+ }
+#endif
#endif
}
static int __init slab_proc_init(void)
{
#ifdef CONFIG_DEBUG_SLAB_LEAK
- proc_create("slab_allocators", 0, NULL, &proc_slabstats_operations);
+ proc_create("slab_allocators", S_IRUSR, NULL, &proc_slabstats_operations);
#endif
return 0;
}
module_init(slab_proc_init);
#endif
+bool is_usercopy_object(const void *ptr)
+{
+ struct page *page;
+ struct kmem_cache *cachep;
+
+ if (ZERO_OR_NULL_PTR(ptr))
+ return false;
+
+ if (!slab_is_available())
+ return false;
+
+ if (!virt_addr_valid(ptr))
+ return false;
+
+ page = virt_to_head_page(ptr);
+
+ if (!PageSlab(page))
+ return false;
+
+ cachep = page->slab_cache;
+ return cachep->flags & SLAB_USERCOPY;
+}
+
+#ifdef CONFIG_PAX_USERCOPY
+const char *check_heap_object(const void *ptr, unsigned long n)
+{
+ struct page *page;
+ struct kmem_cache *cachep;
+ unsigned int objnr;
+ unsigned long offset;
+
+ if (ZERO_OR_NULL_PTR(ptr))
+ return "<null>";
+
+ if (!virt_addr_valid(ptr))
+ return NULL;
+
+ page = virt_to_head_page(ptr);
+
+ if (!PageSlab(page))
+ return NULL;
+
+ cachep = page->slab_cache;
+ if (!(cachep->flags & SLAB_USERCOPY))
+ return cachep->name;
+
+ objnr = obj_to_index(cachep, page, ptr);
+ BUG_ON(objnr >= cachep->num);
+ offset = ptr - index_to_obj(cachep, page, objnr) - obj_offset(cachep);
+ if (offset <= cachep->object_size && n <= cachep->object_size - offset)
+ return NULL;
+
+ return cachep->name;
+}
+#endif
+
/**
* ksize - get the actual amount of memory allocated for a given object
* @objp: Pointer to the object
unsigned int align; /* Alignment as calculated */
unsigned long flags; /* Active flags on the slab */
const char *name; /* Slab name for sysfs */
- int refcount; /* Use counter */
+ atomic_t refcount; /* Use counter */
void (*ctor)(void *); /* Called on object slot creation */
struct list_head list; /* List of all slab caches on the system */
};
/* The slab cache that manages slab cache information */
extern struct kmem_cache *kmem_cache;
+#ifdef CONFIG_PAX_MEMORY_SANITIZE
+#ifdef CONFIG_X86_64
+#define PAX_MEMORY_SANITIZE_VALUE '\xfe'
+#else
+#define PAX_MEMORY_SANITIZE_VALUE '\xff'
+#endif
+enum pax_sanitize_mode {
+ PAX_SANITIZE_SLAB_OFF = 0,
+ PAX_SANITIZE_SLAB_FAST,
+ PAX_SANITIZE_SLAB_FULL,
+};
+extern enum pax_sanitize_mode pax_sanitize_slab;
+#endif
+
unsigned long calculate_alignment(unsigned long flags,
unsigned long align, unsigned long size);
/* Legal flag mask for kmem_cache_create(), for various configurations */
#define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA | SLAB_PANIC | \
- SLAB_DESTROY_BY_RCU | SLAB_DEBUG_OBJECTS )
+ SLAB_DESTROY_BY_RCU | SLAB_DEBUG_OBJECTS | \
+ SLAB_USERCOPY | SLAB_NO_SANITIZE)
#if defined(CONFIG_DEBUG_SLAB)
#define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
return s;
page = virt_to_head_page(x);
+
+ BUG_ON(!PageSlab(page));
+
cachep = page->slab_cache;
if (slab_equal_or_root(cachep, s))
return cachep;
#include "slab.h"
-enum slab_state slab_state;
+enum slab_state slab_state __read_only;
LIST_HEAD(slab_caches);
DEFINE_MUTEX(slab_mutex);
struct kmem_cache *kmem_cache;
+#ifdef CONFIG_PAX_MEMORY_SANITIZE
+enum pax_sanitize_mode pax_sanitize_slab __read_only = PAX_SANITIZE_SLAB_FAST;
+static int __init pax_sanitize_slab_setup(char *str)
+{
+ if (!str)
+ return 0;
+
+ if (!strcmp(str, "0") || !strcmp(str, "off")) {
+ pr_info("PaX slab sanitization: %s\n", "disabled");
+ pax_sanitize_slab = PAX_SANITIZE_SLAB_OFF;
+ } else if (!strcmp(str, "1") || !strcmp(str, "fast")) {
+ pr_info("PaX slab sanitization: %s\n", "fast");
+ pax_sanitize_slab = PAX_SANITIZE_SLAB_FAST;
+ } else if (!strcmp(str, "full")) {
+ pr_info("PaX slab sanitization: %s\n", "full");
+ pax_sanitize_slab = PAX_SANITIZE_SLAB_FULL;
+ } else
+ pr_err("PaX slab sanitization: unsupported option '%s'\n", str);
+
+ return 0;
+}
+early_param("pax_sanitize_slab", pax_sanitize_slab_setup);
+#endif
+
/*
* Set of flags that will prevent slab merging
*/
* Merge control. If this is set then no merging of slab caches will occur.
* (Could be removed. This was introduced to pacify the merge skeptics.)
*/
-static int slab_nomerge;
+static int slab_nomerge = 1;
static int __init setup_slab_nomerge(char *str)
{
/*
* We may have set a slab to be unmergeable during bootstrap.
*/
- if (s->refcount < 0)
+ if (atomic_read(&s->refcount) < 0)
return 1;
return 0;
if (err)
goto out_free_cache;
- s->refcount = 1;
+ atomic_set(&s->refcount, 1);
list_add(&s->list, &slab_caches);
out:
if (err)
*/
flags &= CACHE_CREATE_MASK;
+#ifdef CONFIG_PAX_MEMORY_SANITIZE
+ if (pax_sanitize_slab == PAX_SANITIZE_SLAB_OFF || (flags & SLAB_DESTROY_BY_RCU))
+ flags |= SLAB_NO_SANITIZE;
+ else if (pax_sanitize_slab == PAX_SANITIZE_SLAB_FULL)
+ flags &= ~SLAB_NO_SANITIZE;
+#endif
+
s = __kmem_cache_alias(name, size, align, flags, ctor);
if (s)
goto out_unlock;
mutex_lock(&slab_mutex);
- s->refcount--;
- if (s->refcount)
+ if (!atomic_dec_and_test(&s->refcount))
goto out_unlock;
if (memcg_cleanup_cache_params(s) != 0)
rcu_barrier();
memcg_free_cache_params(s);
-#ifdef SLAB_SUPPORTS_SYSFS
+#if defined(SLAB_SUPPORTS_SYSFS) && !defined(CONFIG_GRKERNSEC_PROC_ADD)
sysfs_slab_remove(s);
#else
slab_kmem_cache_release(s);
panic("Creation of kmalloc slab %s size=%zu failed. Reason %d\n",
name, size, err);
- s->refcount = -1; /* Exempt from merging for now */
+ atomic_set(&s->refcount, -1); /* Exempt from merging for now */
}
struct kmem_cache *__init create_kmalloc_cache(const char *name, size_t size,
create_boot_cache(s, name, size, flags);
list_add(&s->list, &slab_caches);
- s->refcount = 1;
+ atomic_set(&s->refcount, 1);
return s;
}
EXPORT_SYMBOL(kmalloc_dma_caches);
#endif
+#ifdef CONFIG_PAX_USERCOPY_SLABS
+struct kmem_cache *kmalloc_usercopy_caches[KMALLOC_SHIFT_HIGH + 1];
+EXPORT_SYMBOL(kmalloc_usercopy_caches);
+#endif
+
/*
* Conversion table for small slabs sizes / 8 to the index in the
* kmalloc array. This is necessary for slabs < 192 since we have non power
return kmalloc_dma_caches[index];
#endif
+
+#ifdef CONFIG_PAX_USERCOPY_SLABS
+ if (unlikely((flags & GFP_USERCOPY)))
+ return kmalloc_usercopy_caches[index];
+
+#endif
+
return kmalloc_caches[index];
}
for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) {
if (!kmalloc_caches[i]) {
kmalloc_caches[i] = create_kmalloc_cache(NULL,
- 1 << i, flags);
+ 1 << i, SLAB_USERCOPY | flags);
}
/*
* earlier power of two caches
*/
if (KMALLOC_MIN_SIZE <= 32 && !kmalloc_caches[1] && i == 6)
- kmalloc_caches[1] = create_kmalloc_cache(NULL, 96, flags);
+ kmalloc_caches[1] = create_kmalloc_cache(NULL, 96, SLAB_USERCOPY | flags);
if (KMALLOC_MIN_SIZE <= 64 && !kmalloc_caches[2] && i == 7)
- kmalloc_caches[2] = create_kmalloc_cache(NULL, 192, flags);
+ kmalloc_caches[2] = create_kmalloc_cache(NULL, 192, SLAB_USERCOPY | flags);
}
/* Kmalloc array is now usable */
}
}
#endif
+
+#ifdef CONFIG_PAX_USERCOPY_SLABS
+ for (i = 0; i <= KMALLOC_SHIFT_HIGH; i++) {
+ struct kmem_cache *s = kmalloc_caches[i];
+
+ if (s) {
+ int size = kmalloc_size(i);
+ char *n = kasprintf(GFP_NOWAIT,
+ "usercopy-kmalloc-%d", size);
+
+ BUG_ON(!n);
+ kmalloc_usercopy_caches[i] = create_kmalloc_cache(n,
+ size, SLAB_USERCOPY | flags);
+ }
+ }
+#endif
+
}
#endif /* !CONFIG_SLOB */
seq_puts(m, " : globalstat <listallocs> <maxobjs> <grown> <reaped> "
"<error> <maxfreeable> <nodeallocs> <remotefrees> <alienoverflow>");
seq_puts(m, " : cpustat <allochit> <allocmiss> <freehit> <freemiss>");
+#ifdef CONFIG_PAX_MEMORY_SANITIZE
+ seq_puts(m, " : pax <sanitized> <not_sanitized>");
+#endif
#endif
seq_putc(m, '\n');
}
module_init(slab_proc_init);
#endif /* CONFIG_SLABINFO */
-static __always_inline void *__do_krealloc(const void *p, size_t new_size,
+static __always_inline void * __size_overflow(2) __do_krealloc(const void *p, size_t new_size,
gfp_t flags)
{
void *ret;
/*
* Return the size of a slob block.
*/
-static slobidx_t slob_units(slob_t *s)
+static slobidx_t slob_units(const slob_t *s)
{
if (s->units > 0)
return s->units;
/*
* Return the next free slob block pointer after this one.
*/
-static slob_t *slob_next(slob_t *s)
+static slob_t *slob_next(const slob_t *s)
{
slob_t *base = (slob_t *)((unsigned long)s & PAGE_MASK);
slobidx_t next;
/*
* Returns true if s is the last free block in its page.
*/
-static int slob_last(slob_t *s)
+static int slob_last(const slob_t *s)
{
return !((unsigned long)slob_next(s) & ~PAGE_MASK);
}
-static void *slob_new_pages(gfp_t gfp, int order, int node)
+static struct page *slob_new_pages(gfp_t gfp, unsigned int order, int node)
{
- void *page;
+ struct page *page;
#ifdef CONFIG_NUMA
if (node != NUMA_NO_NODE)
if (!page)
return NULL;
- return page_address(page);
+ __SetPageSlab(page);
+ return page;
}
-static void slob_free_pages(void *b, int order)
+static void slob_free_pages(struct page *sp, int order)
{
if (current->reclaim_state)
current->reclaim_state->reclaimed_slab += 1 << order;
- free_pages((unsigned long)b, order);
+ __ClearPageSlab(sp);
+ page_mapcount_reset(sp);
+ sp->private = 0;
+ __free_pages(sp, order);
}
/*
/* Not enough space: must allocate a new page */
if (!b) {
- b = slob_new_pages(gfp & ~__GFP_ZERO, 0, node);
- if (!b)
+ sp = slob_new_pages(gfp & ~__GFP_ZERO, 0, node);
+ if (!sp)
return NULL;
- sp = virt_to_page(b);
- __SetPageSlab(sp);
+ b = page_address(sp);
spin_lock_irqsave(&slob_lock, flags);
sp->units = SLOB_UNITS(PAGE_SIZE);
sp->freelist = b;
+ sp->private = 0;
INIT_LIST_HEAD(&sp->lru);
set_slob(b, SLOB_UNITS(PAGE_SIZE), b + SLOB_UNITS(PAGE_SIZE));
set_slob_page_free(sp, slob_list);
/*
* slob_free: entry point into the slob allocator.
*/
-static void slob_free(void *block, int size)
+static void slob_free(struct kmem_cache *c, void *block, int size)
{
struct page *sp;
slob_t *prev, *next, *b = (slob_t *)block;
if (slob_page_free(sp))
clear_slob_page_free(sp);
spin_unlock_irqrestore(&slob_lock, flags);
- __ClearPageSlab(sp);
- page_mapcount_reset(sp);
- slob_free_pages(b, 0);
+ slob_free_pages(sp, 0);
return;
}
+#ifdef CONFIG_PAX_MEMORY_SANITIZE
+ if (pax_sanitize_slab && !(c && (c->flags & SLAB_NO_SANITIZE)))
+ memset(block, PAX_MEMORY_SANITIZE_VALUE, size);
+#endif
+
if (!slob_page_free(sp)) {
/* This slob page is about to become partially free. Easy! */
sp->units = units;
*/
static __always_inline void *
-__do_kmalloc_node(size_t size, gfp_t gfp, int node, unsigned long caller)
+__do_kmalloc_node_align(size_t size, gfp_t gfp, int node, unsigned long caller, int align)
{
- unsigned int *m;
- int align = max_t(size_t, ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN);
- void *ret;
+ slob_t *m;
+ void *ret = NULL;
gfp &= gfp_allowed_mask;
if (!m)
return NULL;
- *m = size;
+ BUILD_BUG_ON(ARCH_KMALLOC_MINALIGN < 2 * SLOB_UNIT);
+ BUILD_BUG_ON(ARCH_SLAB_MINALIGN < 2 * SLOB_UNIT);
+ m[0].units = size;
+ m[1].units = align;
ret = (void *)m + align;
trace_kmalloc_node(caller, ret,
size, size + align, gfp, node);
} else {
unsigned int order = get_order(size);
+ struct page *page;
if (likely(order))
gfp |= __GFP_COMP;
- ret = slob_new_pages(gfp, order, node);
+ page = slob_new_pages(gfp, order, node);
+ if (page) {
+ ret = page_address(page);
+ page->private = size;
+ }
trace_kmalloc_node(caller, ret,
size, PAGE_SIZE << order, gfp, node);
}
- kmemleak_alloc(ret, size, 1, gfp);
return ret;
}
-void *__kmalloc(size_t size, gfp_t gfp)
+static __always_inline void *
+__do_kmalloc_node(size_t size, gfp_t gfp, int node, unsigned long caller)
+{
+ int align = max(ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN);
+ void *ret = __do_kmalloc_node_align(size, gfp, node, caller, align);
+
+ if (!ZERO_OR_NULL_PTR(ret))
+ kmemleak_alloc(ret, size, 1, gfp);
+ return ret;
+}
+
+void * __size_overflow(1) __kmalloc(size_t size, gfp_t gfp)
{
return __do_kmalloc_node(size, gfp, NUMA_NO_NODE, _RET_IP_);
}
return;
kmemleak_free(block);
+ VM_BUG_ON(!virt_addr_valid(block));
sp = virt_to_page(block);
- if (PageSlab(sp)) {
+ VM_BUG_ON(!PageSlab(sp));
+ if (!sp->private) {
int align = max_t(size_t, ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN);
- unsigned int *m = (unsigned int *)(block - align);
- slob_free(m, *m + align);
- } else
+ slob_t *m = (slob_t *)(block - align);
+ slob_free(NULL, m, m[0].units + align);
+ } else {
+ __ClearPageSlab(sp);
+ page_mapcount_reset(sp);
+ sp->private = 0;
__free_pages(sp, compound_order(sp));
+ }
}
EXPORT_SYMBOL(kfree);
+bool is_usercopy_object(const void *ptr)
+{
+ if (!slab_is_available())
+ return false;
+
+ // PAX: TODO
+
+ return false;
+}
+
+#ifdef CONFIG_PAX_USERCOPY
+const char *check_heap_object(const void *ptr, unsigned long n)
+{
+ struct page *page;
+ const slob_t *free;
+ const void *base;
+ unsigned long flags;
+
+ if (ZERO_OR_NULL_PTR(ptr))
+ return "<null>";
+
+ if (!virt_addr_valid(ptr))
+ return NULL;
+
+ page = virt_to_head_page(ptr);
+ if (!PageSlab(page))
+ return NULL;
+
+ if (page->private) {
+ base = page;
+ if (base <= ptr && n <= page->private - (ptr - base))
+ return NULL;
+ return "<slob>";
+ }
+
+ /* some tricky double walking to find the chunk */
+ spin_lock_irqsave(&slob_lock, flags);
+ base = (void *)((unsigned long)ptr & PAGE_MASK);
+ free = page->freelist;
+
+ while (!slob_last(free) && (void *)free <= ptr) {
+ base = free + slob_units(free);
+ free = slob_next(free);
+ }
+
+ while (base < (void *)free) {
+ slobidx_t m = ((slob_t *)base)[0].units, align = ((slob_t *)base)[1].units;
+ int size = SLOB_UNIT * SLOB_UNITS(m + align);
+ int offset;
+
+ if (ptr < base + align)
+ break;
+
+ offset = ptr - base - align;
+ if (offset >= m) {
+ base += size;
+ continue;
+ }
+
+ if (n > m - offset)
+ break;
+
+ spin_unlock_irqrestore(&slob_lock, flags);
+ return NULL;
+ }
+
+ spin_unlock_irqrestore(&slob_lock, flags);
+ return "<slob>";
+}
+#endif
+
/* can't use ksize for kmem_cache_alloc memory, only kmalloc */
size_t ksize(const void *block)
{
struct page *sp;
int align;
- unsigned int *m;
+ slob_t *m;
BUG_ON(!block);
if (unlikely(block == ZERO_SIZE_PTR))
return 0;
sp = virt_to_page(block);
- if (unlikely(!PageSlab(sp)))
- return PAGE_SIZE << compound_order(sp);
+ VM_BUG_ON(!PageSlab(sp));
+ if (sp->private)
+ return sp->private;
align = max_t(size_t, ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN);
- m = (unsigned int *)(block - align);
- return SLOB_UNITS(*m) * SLOB_UNIT;
+ m = (slob_t *)(block - align);
+ return SLOB_UNITS(m[0].units) * SLOB_UNIT;
}
EXPORT_SYMBOL(ksize);
void *slob_alloc_node(struct kmem_cache *c, gfp_t flags, int node)
{
- void *b;
+ void *b = NULL;
flags &= gfp_allowed_mask;
lockdep_trace_alloc(flags);
+#ifdef CONFIG_PAX_USERCOPY_SLABS
+ b = __do_kmalloc_node_align(c->size, flags, node, _RET_IP_, c->align);
+#else
if (c->size < PAGE_SIZE) {
b = slob_alloc(c->size, flags, c->align, node);
trace_kmem_cache_alloc_node(_RET_IP_, b, c->object_size,
SLOB_UNITS(c->size) * SLOB_UNIT,
flags, node);
} else {
- b = slob_new_pages(flags, get_order(c->size), node);
+ struct page *sp;
+
+ sp = slob_new_pages(flags, get_order(c->size), node);
+ if (sp) {
+ b = page_address(sp);
+ sp->private = c->size;
+ }
trace_kmem_cache_alloc_node(_RET_IP_, b, c->object_size,
PAGE_SIZE << get_order(c->size),
flags, node);
}
+#endif
if (b && c->ctor)
c->ctor(b);
EXPORT_SYMBOL(kmem_cache_alloc);
#ifdef CONFIG_NUMA
-void *__kmalloc_node(size_t size, gfp_t gfp, int node)
+void * __size_overflow(1) __kmalloc_node(size_t size, gfp_t gfp, int node)
{
return __do_kmalloc_node(size, gfp, node, _RET_IP_);
}
EXPORT_SYMBOL(kmem_cache_alloc_node);
#endif
-static void __kmem_cache_free(void *b, int size)
+static void __kmem_cache_free(struct kmem_cache *c, void *b, int size)
{
- if (size < PAGE_SIZE)
- slob_free(b, size);
+ struct page *sp;
+
+ sp = virt_to_page(b);
+ BUG_ON(!PageSlab(sp));
+ if (!sp->private)
+ slob_free(c, b, size);
else
- slob_free_pages(b, get_order(size));
+ slob_free_pages(sp, get_order(size));
}
static void kmem_rcu_free(struct rcu_head *head)
struct slob_rcu *slob_rcu = (struct slob_rcu *)head;
void *b = (void *)slob_rcu - (slob_rcu->size - sizeof(struct slob_rcu));
- __kmem_cache_free(b, slob_rcu->size);
+ __kmem_cache_free(NULL, b, slob_rcu->size);
}
void kmem_cache_free(struct kmem_cache *c, void *b)
{
+ int size = c->size;
+
+#ifdef CONFIG_PAX_USERCOPY_SLABS
+ if (size + c->align < PAGE_SIZE) {
+ size += c->align;
+ b -= c->align;
+ }
+#endif
+
kmemleak_free_recursive(b, c->flags);
if (unlikely(c->flags & SLAB_DESTROY_BY_RCU)) {
struct slob_rcu *slob_rcu;
- slob_rcu = b + (c->size - sizeof(struct slob_rcu));
- slob_rcu->size = c->size;
+ slob_rcu = b + (size - sizeof(struct slob_rcu));
+ slob_rcu->size = size;
call_rcu(&slob_rcu->head, kmem_rcu_free);
} else {
- __kmem_cache_free(b, c->size);
+ __kmem_cache_free(c, b, size);
}
+#ifdef CONFIG_PAX_USERCOPY_SLABS
+ trace_kfree(_RET_IP_, b);
+#else
trace_kmem_cache_free(_RET_IP_, b);
+#endif
+
}
EXPORT_SYMBOL(kmem_cache_free);
enum track_item { TRACK_ALLOC, TRACK_FREE };
-#ifdef CONFIG_SYSFS
+#if defined(CONFIG_SYSFS) && !defined(CONFIG_GRKERNSEC_PROC_ADD)
static int sysfs_slab_add(struct kmem_cache *);
static int sysfs_slab_alias(struct kmem_cache *, const char *);
static void memcg_propagate_slab_attrs(struct kmem_cache *s);
if (!t->addr)
return;
- pr_err("INFO: %s in %pS age=%lu cpu=%u pid=%d\n",
+ pr_err("INFO: %s in %pA age=%lu cpu=%u pid=%d\n",
s, (void *)t->addr, jiffies - t->when, t->cpu, t->pid);
#ifdef CONFIG_STACKTRACE
{
slab_free_hook(s, x);
+#ifdef CONFIG_PAX_MEMORY_SANITIZE
+ if (!(s->flags & SLAB_NO_SANITIZE)) {
+ memset(x, PAX_MEMORY_SANITIZE_VALUE, s->object_size);
+ if (s->ctor)
+ s->ctor(x);
+ }
+#endif
+
redo:
/*
* Determine the currently cpus per cpu slab.
s->inuse = size;
if (((flags & (SLAB_DESTROY_BY_RCU | SLAB_POISON)) ||
+#ifdef CONFIG_PAX_MEMORY_SANITIZE
+ (!(flags & SLAB_NO_SANITIZE)) ||
+#endif
s->ctor)) {
/*
* Relocate free pointer after the object if it is not
__setup("slub_min_objects=", setup_slub_min_objects);
-void *__kmalloc(size_t size, gfp_t flags)
+void * __size_overflow(1) __kmalloc(size_t size, gfp_t flags)
{
struct kmem_cache *s;
void *ret;
return ptr;
}
-void *__kmalloc_node(size_t size, gfp_t flags, int node)
+void * __size_overflow(1) __kmalloc_node(size_t size, gfp_t flags, int node)
{
struct kmem_cache *s;
void *ret;
EXPORT_SYMBOL(__kmalloc_node);
#endif
+bool is_usercopy_object(const void *ptr)
+{
+ struct page *page;
+ struct kmem_cache *s;
+
+ if (ZERO_OR_NULL_PTR(ptr))
+ return false;
+
+ if (!slab_is_available())
+ return false;
+
+ if (!virt_addr_valid(ptr))
+ return false;
+
+ page = virt_to_head_page(ptr);
+
+ if (!PageSlab(page))
+ return false;
+
+ s = page->slab_cache;
+ return s->flags & SLAB_USERCOPY;
+}
+
+#ifdef CONFIG_PAX_USERCOPY
+const char *check_heap_object(const void *ptr, unsigned long n)
+{
+ struct page *page;
+ struct kmem_cache *s;
+ unsigned long offset;
+
+ if (ZERO_OR_NULL_PTR(ptr))
+ return "<null>";
+
+ if (!virt_addr_valid(ptr))
+ return NULL;
+
+ page = virt_to_head_page(ptr);
+
+ if (!PageSlab(page))
+ return NULL;
+
+ s = page->slab_cache;
+ if (!(s->flags & SLAB_USERCOPY))
+ return s->name;
+
+ offset = (ptr - page_address(page)) % s->size;
+ if (offset <= s->object_size && n <= s->object_size - offset)
+ return NULL;
+
+ return s->name;
+}
+#endif
+
size_t ksize(const void *object)
{
struct page *page;
if (unlikely(ZERO_OR_NULL_PTR(x)))
return;
+ VM_BUG_ON(!virt_addr_valid(x));
page = virt_to_head_page(x);
if (unlikely(!PageSlab(page))) {
BUG_ON(!PageCompound(page));
int i;
struct kmem_cache *c;
- s->refcount++;
+ atomic_inc(&s->refcount);
/*
* Adjust the object sizes so that we clear
}
if (sysfs_slab_alias(s, name)) {
- s->refcount--;
+ atomic_dec(&s->refcount);
s = NULL;
}
}
}
#endif
-#ifdef CONFIG_SYSFS
+#if defined(CONFIG_SYSFS) && !defined(CONFIG_GRKERNSEC_PROC_ADD)
static int count_inuse(struct page *page)
{
return page->inuse;
len += sprintf(buf + len, "%7ld ", l->count);
if (l->addr)
+#ifdef CONFIG_GRKERNSEC_HIDESYM
+ len += sprintf(buf + len, "%pS", NULL);
+#else
len += sprintf(buf + len, "%pS", (void *)l->addr);
+#endif
else
len += sprintf(buf + len, "<not-available>");
validate_slab_cache(kmalloc_caches[9]);
}
#else
-#ifdef CONFIG_SYSFS
+#if defined(CONFIG_SYSFS) && !defined(CONFIG_GRKERNSEC_PROC_ADD)
static void resiliency_test(void) {};
#endif
#endif
-#ifdef CONFIG_SYSFS
+#if defined(CONFIG_SYSFS) && !defined(CONFIG_GRKERNSEC_PROC_ADD)
enum slab_stat_type {
SL_ALL, /* All slabs */
SL_PARTIAL, /* Only partially allocated slabs */
{
if (!s->ctor)
return 0;
+#ifdef CONFIG_GRKERNSEC_HIDESYM
+ return sprintf(buf, "%pS\n", NULL);
+#else
return sprintf(buf, "%pS\n", s->ctor);
+#endif
}
SLAB_ATTR_RO(ctor);
static ssize_t aliases_show(struct kmem_cache *s, char *buf)
{
- return sprintf(buf, "%d\n", s->refcount < 0 ? 0 : s->refcount - 1);
+ return sprintf(buf, "%d\n", atomic_read(&s->refcount) < 0 ? 0 : atomic_read(&s->refcount) - 1);
}
SLAB_ATTR_RO(aliases);
SLAB_ATTR_RO(cache_dma);
#endif
+#ifdef CONFIG_PAX_USERCOPY_SLABS
+static ssize_t usercopy_show(struct kmem_cache *s, char *buf)
+{
+ return sprintf(buf, "%d\n", !!(s->flags & SLAB_USERCOPY));
+}
+SLAB_ATTR_RO(usercopy);
+#endif
+
+#ifdef CONFIG_PAX_MEMORY_SANITIZE
+static ssize_t sanitize_show(struct kmem_cache *s, char *buf)
+{
+ return sprintf(buf, "%d\n", !(s->flags & SLAB_NO_SANITIZE));
+}
+SLAB_ATTR_RO(sanitize);
+#endif
+
static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf)
{
return sprintf(buf, "%d\n", !!(s->flags & SLAB_DESTROY_BY_RCU));
* as well as cause other issues like converting a mergeable
* cache into an umergeable one.
*/
- if (s->refcount > 1)
+ if (atomic_read(&s->refcount) > 1)
return -EINVAL;
s->flags &= ~SLAB_TRACE;
static ssize_t failslab_store(struct kmem_cache *s, const char *buf,
size_t length)
{
- if (s->refcount > 1)
+ if (atomic_read(&s->refcount) > 1)
return -EINVAL;
s->flags &= ~SLAB_FAILSLAB;
#ifdef CONFIG_ZONE_DMA
&cache_dma_attr.attr,
#endif
+#ifdef CONFIG_PAX_USERCOPY_SLABS
+ &usercopy_attr.attr,
+#endif
+#ifdef CONFIG_PAX_MEMORY_SANITIZE
+ &sanitize_attr.attr,
+#endif
#ifdef CONFIG_NUMA
&remote_node_defrag_ratio_attr.attr,
#endif
return name;
}
+#if defined(CONFIG_SYSFS) && !defined(CONFIG_GRKERNSEC_PROC_ADD)
static int sysfs_slab_add(struct kmem_cache *s)
{
int err;
kobject_del(&s->kobj);
kobject_put(&s->kobj);
}
+#endif
/*
* Need to buffer aliases during bootup until sysfs becomes
static struct saved_alias *alias_list;
+#if defined(CONFIG_SYSFS) && !defined(CONFIG_GRKERNSEC_PROC_ADD)
static int sysfs_slab_alias(struct kmem_cache *s, const char *name)
{
struct saved_alias *al;
alias_list = al;
return 0;
}
+#endif
static int __init slab_sysfs_init(void)
{
void *p = vmemmap_alloc_block(PAGE_SIZE, node);
if (!p)
return NULL;
- pud_populate(&init_mm, pud, p);
+ pud_populate_kernel(&init_mm, pud, p);
}
return pud;
}
void *p = vmemmap_alloc_block(PAGE_SIZE, node);
if (!p)
return NULL;
- pgd_populate(&init_mm, pgd, p);
+ pgd_populate_kernel(&init_mm, pgd, p);
}
return pgd;
}
for (i = 0; i < PAGES_PER_SECTION; i++) {
if (PageHWPoison(&memmap[i])) {
- atomic_long_sub(1, &num_poisoned_pages);
+ atomic_long_sub_unchecked(1, &num_poisoned_pages);
ClearPageHWPoison(&memmap[i]);
}
}
#include <linux/memcontrol.h>
#include <linux/gfp.h>
#include <linux/uio.h>
+#include <linux/hugetlb.h>
#include "internal.h"
__page_cache_release(page);
dtor = get_compound_page_dtor(page);
+ if (!PageHuge(page))
+ BUG_ON(dtor != free_compound_page);
(*dtor)(page);
}
static DECLARE_WAIT_QUEUE_HEAD(proc_poll_wait);
/* Activity counter to indicate that a swapon or swapoff has occurred */
-static atomic_t proc_poll_event = ATOMIC_INIT(0);
+static atomic_unchecked_t proc_poll_event = ATOMIC_INIT(0);
static inline unsigned char swap_count(unsigned char ent)
{
spin_unlock(&swap_lock);
err = 0;
- atomic_inc(&proc_poll_event);
+ atomic_inc_unchecked(&proc_poll_event);
wake_up_interruptible(&proc_poll_wait);
out_dput:
poll_wait(file, &proc_poll_wait, wait);
- if (seq->poll_event != atomic_read(&proc_poll_event)) {
- seq->poll_event = atomic_read(&proc_poll_event);
+ if (seq->poll_event != atomic_read_unchecked(&proc_poll_event)) {
+ seq->poll_event = atomic_read_unchecked(&proc_poll_event);
return POLLIN | POLLRDNORM | POLLERR | POLLPRI;
}
return ret;
seq = file->private_data;
- seq->poll_event = atomic_read(&proc_poll_event);
+ seq->poll_event = atomic_read_unchecked(&proc_poll_event);
return 0;
}
(frontswap_map) ? "FS" : "");
mutex_unlock(&swapon_mutex);
- atomic_inc(&proc_poll_event);
+ atomic_inc_unchecked(&proc_poll_event);
wake_up_interruptible(&proc_poll_wait);
if (S_ISREG(inode->i_mode))
void arch_pick_mmap_layout(struct mm_struct *mm)
{
mm->mmap_base = TASK_UNMAPPED_BASE;
+
+#ifdef CONFIG_PAX_RANDMMAP
+ if (mm->pax_flags & MF_PAX_RANDMMAP)
+ mm->mmap_base += mm->delta_mmap;
+#endif
+
mm->get_unmapped_area = arch_get_unmapped_area;
}
#endif
if (!mm->arg_end)
goto out_mm; /* Shh! No looking before we're done */
+ if (gr_acl_handle_procpidmem(task))
+ goto out_mm;
+
len = mm->arg_end - mm->arg_start;
if (len > buflen)
struct work_struct wq;
};
static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred);
+static DEFINE_PER_CPU(struct vfree_deferred, vunmap_deferred);
+
+#ifdef CONFIG_GRKERNSEC_KSTACKOVERFLOW
+struct stack_deferred_llist {
+ struct llist_head list;
+ void *stack;
+ void *lowmem_stack;
+};
+
+struct stack_deferred {
+ struct stack_deferred_llist list;
+ struct work_struct wq;
+};
+
+static DEFINE_PER_CPU(struct stack_deferred, stack_deferred);
+#endif
static void __vunmap(const void *, int);
-static void free_work(struct work_struct *w)
+static void vfree_work(struct work_struct *w)
+{
+ struct vfree_deferred *p = container_of(w, struct vfree_deferred, wq);
+ struct llist_node *llnode = llist_del_all(&p->list);
+ while (llnode) {
+ void *x = llnode;
+ llnode = llist_next(llnode);
+ __vunmap(x, 1);
+ }
+}
+
+static void vunmap_work(struct work_struct *w)
{
struct vfree_deferred *p = container_of(w, struct vfree_deferred, wq);
struct llist_node *llnode = llist_del_all(&p->list);
while (llnode) {
void *p = llnode;
llnode = llist_next(llnode);
- __vunmap(p, 1);
+ __vunmap(p, 0);
}
}
+#ifdef CONFIG_GRKERNSEC_KSTACKOVERFLOW
+static void unmap_work(struct work_struct *w)
+{
+ struct stack_deferred *p = container_of(w, struct stack_deferred, wq);
+ struct llist_node *llnode = llist_del_all(&p->list.list);
+ while (llnode) {
+ struct stack_deferred_llist *x =
+ llist_entry((struct llist_head *)llnode,
+ struct stack_deferred_llist, list);
+ void *stack = ACCESS_ONCE(x->stack);
+ void *lowmem_stack = ACCESS_ONCE(x->lowmem_stack);
+ llnode = llist_next(llnode);
+ __vunmap(stack, 0);
+ free_kmem_pages((unsigned long)lowmem_stack, THREAD_SIZE_ORDER);
+ }
+}
+#endif
+
/*** Page table manipulation functions ***/
static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end)
pte = pte_offset_kernel(pmd, addr);
do {
- pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte);
- WARN_ON(!pte_none(ptent) && !pte_present(ptent));
+
+#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_KERNEXEC)
+ if ((unsigned long)MODULES_EXEC_VADDR <= addr && addr < (unsigned long)MODULES_EXEC_END) {
+ BUG_ON(!pte_exec(*pte));
+ set_pte_at(&init_mm, addr, pte, pfn_pte(__pa(addr) >> PAGE_SHIFT, PAGE_KERNEL_EXEC));
+ continue;
+ }
+#endif
+
+ {
+ pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte);
+ WARN_ON(!pte_none(ptent) && !pte_present(ptent));
+ }
} while (pte++, addr += PAGE_SIZE, addr != end);
}
pte = pte_alloc_kernel(pmd, addr);
if (!pte)
return -ENOMEM;
+
+ pax_open_kernel();
do {
struct page *page = pages[*nr];
- if (WARN_ON(!pte_none(*pte)))
+#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_KERNEXEC)
+ if (pgprot_val(prot) & _PAGE_NX)
+#endif
+
+ if (!pte_none(*pte)) {
+ pax_close_kernel();
+ WARN_ON(1);
return -EBUSY;
- if (WARN_ON(!page))
+ }
+ if (!page) {
+ pax_close_kernel();
+ WARN_ON(1);
return -ENOMEM;
+ }
set_pte_at(&init_mm, addr, pte, mk_pte(page, prot));
(*nr)++;
} while (pte++, addr += PAGE_SIZE, addr != end);
+ pax_close_kernel();
return 0;
}
pmd_t *pmd;
unsigned long next;
- pmd = pmd_alloc(&init_mm, pud, addr);
+ pmd = pmd_alloc_kernel(&init_mm, pud, addr);
if (!pmd)
return -ENOMEM;
do {
pud_t *pud;
unsigned long next;
- pud = pud_alloc(&init_mm, pgd, addr);
+ pud = pud_alloc_kernel(&init_mm, pgd, addr);
if (!pud)
return -ENOMEM;
do {
if (addr >= MODULES_VADDR && addr < MODULES_END)
return 1;
#endif
+
+#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_KERNEXEC)
+ if (x >= (const void *)MODULES_EXEC_VADDR && x < (const void *)MODULES_EXEC_END)
+ return 1;
+#endif
+
return is_vmalloc_addr(x);
}
if (!pgd_none(*pgd)) {
pud_t *pud = pud_offset(pgd, addr);
+#ifdef CONFIG_X86
+ if (!pud_large(*pud))
+#endif
if (!pud_none(*pud)) {
pmd_t *pmd = pmd_offset(pud, addr);
+#ifdef CONFIG_X86
+ if (!pmd_large(*pmd))
+#endif
if (!pmd_none(*pmd)) {
pte_t *ptep, pte;
* Allocate a region of KVA of the specified size and alignment, within the
* vstart and vend.
*/
-static struct vmap_area *alloc_vmap_area(unsigned long size,
+static struct vmap_area * __size_overflow(1) alloc_vmap_area(unsigned long size,
unsigned long align,
unsigned long vstart, unsigned long vend,
int node, gfp_t gfp_mask)
for_each_possible_cpu(i) {
struct vmap_block_queue *vbq;
struct vfree_deferred *p;
+#ifdef CONFIG_GRKERNSEC_KSTACKOVERFLOW
+ struct stack_deferred *p2;
+#endif
vbq = &per_cpu(vmap_block_queue, i);
spin_lock_init(&vbq->lock);
INIT_LIST_HEAD(&vbq->free);
+
p = &per_cpu(vfree_deferred, i);
init_llist_head(&p->list);
- INIT_WORK(&p->wq, free_work);
+ INIT_WORK(&p->wq, vfree_work);
+
+ p = &per_cpu(vunmap_deferred, i);
+ init_llist_head(&p->list);
+ INIT_WORK(&p->wq, vunmap_work);
+
+#ifdef CONFIG_GRKERNSEC_KSTACKOVERFLOW
+ p2 = &per_cpu(stack_deferred, i);
+ init_llist_head(&p2->list.list);
+ INIT_WORK(&p2->wq, unmap_work);
+#endif
}
/* Import existing vmlist entries. */
struct vm_struct *area;
BUG_ON(in_interrupt());
+
+#if defined(CONFIG_X86) && defined(CONFIG_PAX_KERNEXEC)
+ if (flags & VM_KERNEXEC) {
+ if (start != VMALLOC_START || end != VMALLOC_END)
+ return NULL;
+ start = (unsigned long)MODULES_EXEC_VADDR;
+ end = (unsigned long)MODULES_EXEC_END;
+ }
+#endif
+
if (flags & VM_IOREMAP)
align = 1ul << clamp(fls(size), PAGE_SHIFT, IOREMAP_MAX_ORDER);
*/
void vunmap(const void *addr)
{
- BUG_ON(in_interrupt());
- might_sleep();
- if (addr)
+ if (!addr)
+ return;
+
+ if (unlikely(in_interrupt())) {
+ struct vfree_deferred *p = this_cpu_ptr(&vunmap_deferred);
+ if (llist_add((struct llist_node *)addr, &p->list))
+ schedule_work(&p->wq);
+ } else {
+ might_sleep();
__vunmap(addr, 0);
+ }
}
EXPORT_SYMBOL(vunmap);
+#ifdef CONFIG_GRKERNSEC_KSTACKOVERFLOW
+void unmap_process_stacks(struct task_struct *task)
+{
+ if (unlikely(in_interrupt())) {
+ struct stack_deferred *p = this_cpu_ptr(&stack_deferred);
+ struct stack_deferred_llist *list = task->stack;
+ list->stack = task->stack;
+ list->lowmem_stack = task->lowmem_stack;
+ if (llist_add((struct llist_node *)&list->list, &p->list.list))
+ schedule_work(&p->wq);
+ } else {
+ __vunmap(task->stack, 0);
+ free_kmem_pages((unsigned long)task->lowmem_stack, THREAD_SIZE_ORDER);
+ }
+}
+#endif
+
/**
* vmap - map an array of pages into virtually contiguous space
* @pages: array of page pointers
if (count > totalram_pages)
return NULL;
+#if defined(CONFIG_X86) && defined(CONFIG_PAX_KERNEXEC)
+ if (!(pgprot_val(prot) & _PAGE_NX))
+ flags |= VM_KERNEXEC;
+#endif
+
area = get_vm_area_caller((count << PAGE_SHIFT), flags,
__builtin_return_address(0));
if (!area)
if (!size || (size >> PAGE_SHIFT) > totalram_pages)
goto fail;
+#if defined(CONFIG_X86) && defined(CONFIG_PAX_KERNEXEC)
+ if (!(pgprot_val(prot) & _PAGE_NX))
+ area = __get_vm_area_node(size, align, VM_ALLOC | VM_UNINITIALIZED | VM_KERNEXEC,
+ VMALLOC_START, VMALLOC_END, node, gfp_mask, caller);
+ else
+#endif
+
area = __get_vm_area_node(size, align, VM_ALLOC | VM_UNINITIALIZED,
start, end, node, gfp_mask, caller);
if (!area)
* For tight control over page level allocator and protection flags
* use __vmalloc() instead.
*/
-
void *vmalloc_exec(unsigned long size)
{
- return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC,
+ return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO, PAGE_KERNEL_EXEC,
NUMA_NO_NODE, __builtin_return_address(0));
}
{
struct vm_struct *area;
+ BUG_ON(vma->vm_mirror);
+
size = PAGE_ALIGN(size);
if (!PAGE_ALIGNED(uaddr) || !PAGE_ALIGNED(kaddr))
v->addr, v->addr + v->size, v->size);
if (v->caller)
+#ifdef CONFIG_GRKERNSEC_HIDESYM
+ seq_printf(m, " %pK", v->caller);
+#else
seq_printf(m, " %pS", v->caller);
+#endif
if (v->nr_pages)
seq_printf(m, " pages=%d", v->nr_pages);
#include <linux/mm_inline.h>
#include <linux/page_ext.h>
#include <linux/page_owner.h>
+#include <linux/grsecurity.h>
#include "internal.h"
*
* vm_stat contains the global counters
*/
-atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS] __cacheline_aligned_in_smp;
+atomic_long_unchecked_t vm_stat[NR_VM_ZONE_STAT_ITEMS] __cacheline_aligned_in_smp;
EXPORT_SYMBOL(vm_stat);
#ifdef CONFIG_SMP
for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
if (diff[i]) {
- atomic_long_add(diff[i], &vm_stat[i]);
+ atomic_long_add_unchecked(diff[i], &vm_stat[i]);
changes++;
}
return changes;
v = this_cpu_xchg(p->vm_stat_diff[i], 0);
if (v) {
- atomic_long_add(v, &zone->vm_stat[i]);
+ atomic_long_add_unchecked(v, &zone->vm_stat[i]);
global_diff[i] += v;
#ifdef CONFIG_NUMA
/* 3 seconds idle till flush */
v = p->vm_stat_diff[i];
p->vm_stat_diff[i] = 0;
- atomic_long_add(v, &zone->vm_stat[i]);
+ atomic_long_add_unchecked(v, &zone->vm_stat[i]);
global_diff[i] += v;
}
}
if (pset->vm_stat_diff[i]) {
int v = pset->vm_stat_diff[i];
pset->vm_stat_diff[i] = 0;
- atomic_long_add(v, &zone->vm_stat[i]);
- atomic_long_add(v, &vm_stat[i]);
+ atomic_long_add_unchecked(v, &zone->vm_stat[i]);
+ atomic_long_add_unchecked(v, &vm_stat[i]);
}
}
#endif
stat_items_size += sizeof(struct vm_event_state);
#endif
- v = kmalloc(stat_items_size, GFP_KERNEL);
+ v = kzalloc(stat_items_size, GFP_KERNEL);
m->private = v;
if (!v)
return ERR_PTR(-ENOMEM);
+
+#ifdef CONFIG_GRKERNSEC_PROC_ADD
+#if defined(CONFIG_GRKERNSEC_PROC_USER) || defined(CONFIG_GRKERNSEC_PROC_USERGROUP)
+ if (!uid_eq(current_uid(), GLOBAL_ROOT_UID)
+#ifdef CONFIG_GRKERNSEC_PROC_USERGROUP
+ && !in_group_p(grsec_proc_gid)
+#endif
+ )
+ return (unsigned long *)m->private + *pos;
+#endif
+#endif
+
for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
v[i] = global_page_state(i);
v += NR_VM_ZONE_STAT_ITEMS;
cpu_notifier_register_done();
#endif
#ifdef CONFIG_PROC_FS
- proc_create("buddyinfo", S_IRUGO, NULL, &fragmentation_file_operations);
- proc_create("pagetypeinfo", S_IRUGO, NULL, &pagetypeinfo_file_ops);
- proc_create("vmstat", S_IRUGO, NULL, &proc_vmstat_file_operations);
- proc_create("zoneinfo", S_IRUGO, NULL, &proc_zoneinfo_file_operations);
+ {
+ mode_t gr_mode = S_IRUGO;
+#ifdef CONFIG_GRKERNSEC_PROC_ADD
+ gr_mode = S_IRUSR;
+#endif
+ proc_create("buddyinfo", gr_mode, NULL, &fragmentation_file_operations);
+ proc_create("pagetypeinfo", gr_mode, NULL, &pagetypeinfo_file_ops);
+ proc_create("vmstat", S_IRUGO, NULL, &proc_vmstat_file_operations);
+ proc_create("zoneinfo", gr_mode, NULL, &proc_zoneinfo_file_operations);
+ }
#endif
return 0;
}
return NOTIFY_DONE;
}
-static struct notifier_block vlan_notifier_block __read_mostly = {
+static struct notifier_block vlan_notifier_block = {
.notifier_call = vlan_device_event,
};
err = -EPERM;
if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
break;
- if ((args.u.name_type >= 0) &&
- (args.u.name_type < VLAN_NAME_TYPE_HIGHEST)) {
+ if (args.u.name_type < VLAN_NAME_TYPE_HIGHEST) {
struct vlan_net *vn;
vn = net_generic(net, vlan_net_id);
return -EMSGSIZE;
}
-struct rtnl_link_ops vlan_link_ops __read_mostly = {
+struct rtnl_link_ops vlan_link_ops = {
.kind = "vlan",
.maxtype = IFLA_VLAN_MAX,
.policy = vlan_policy,
len - inline_len);
} else {
err = copy_from_user(ename + inline_len,
- uidata, len - inline_len);
+ (char __force_user *)uidata, len - inline_len);
if (err) {
err = -EFAULT;
goto out_err;
kernel_buf = 1;
indata = data;
} else
- indata = (__force char *)udata;
+ indata = (__force_kernel char *)udata;
/*
* response header len is 11
* PDU Header(7) + IO Size (4)
kernel_buf = 1;
odata = data;
} else
- odata = (char *)udata;
+ odata = (char __force_kernel *)udata;
req = p9_client_zc_rpc(clnt, P9_TWRITE, NULL, odata, 0, rsize,
P9_ZC_HDR_SZ, kernel_buf, "dqd",
fid->fid, offset, rsize);
void v9fs_register_trans(struct p9_trans_module *m)
{
spin_lock(&v9fs_trans_lock);
- list_add_tail(&m->list, &v9fs_trans_list);
+ pax_list_add_tail((struct list_head *)&m->list, &v9fs_trans_list);
spin_unlock(&v9fs_trans_lock);
}
EXPORT_SYMBOL(v9fs_register_trans);
void v9fs_unregister_trans(struct p9_trans_module *m)
{
spin_lock(&v9fs_trans_lock);
- list_del_init(&m->list);
+ pax_list_del_init((struct list_head *)&m->list);
spin_unlock(&v9fs_trans_lock);
}
EXPORT_SYMBOL(v9fs_unregister_trans);
oldfs = get_fs();
set_fs(get_ds());
/* The cast to a user pointer is valid due to the set_fs() */
- ret = vfs_write(ts->wr, (__force void __user *)v, len, &ts->wr->f_pos);
+ ret = vfs_write(ts->wr, (void __force_user *)v, len, &ts->wr->f_pos);
set_fs(oldfs);
if (ret <= 0 && ret != -ERESTARTSYS && ret != -EAGAIN)
struct proc_dir_entry *p;
int rc = -ENOMEM;
- atalk_proc_dir = proc_mkdir("atalk", init_net.proc_net);
+ atalk_proc_dir = proc_mkdir_restrict("atalk", init_net.proc_net);
if (!atalk_proc_dir)
goto out;
if (atomic_read(&sk_atm(vcc)->sk_rmem_alloc) <= sk_atm(vcc)->sk_rcvbuf)
return 1;
atm_return(vcc, truesize);
- atomic_inc(&vcc->stats->rx_drop);
+ atomic_inc_unchecked(&vcc->stats->rx_drop);
return 0;
}
EXPORT_SYMBOL(atm_charge);
}
}
atm_return(vcc, guess);
- atomic_inc(&vcc->stats->rx_drop);
+ atomic_inc_unchecked(&vcc->stats->rx_drop);
return NULL;
}
EXPORT_SYMBOL(atm_alloc_charge);
void sonet_copy_stats(struct k_sonet_stats *from, struct sonet_stats *to)
{
-#define __HANDLE_ITEM(i) to->i = atomic_read(&from->i)
+#define __HANDLE_ITEM(i) to->i = atomic_read_unchecked(&from->i)
__SONET_ITEMS
#undef __HANDLE_ITEM
}
void sonet_subtract_stats(struct k_sonet_stats *from, struct sonet_stats *to)
{
-#define __HANDLE_ITEM(i) atomic_sub(to->i, &from->i)
+#define __HANDLE_ITEM(i) atomic_sub_unchecked(to->i,&from->i)
__SONET_ITEMS
#undef __HANDLE_ITEM
}
}
static struct lane2_ops lane2_ops = {
- lane2_resolve, /* resolve, spec 3.1.3 */
- lane2_associate_req, /* associate_req, spec 3.1.4 */
- NULL /* associate indicator, spec 3.1.5 */
+ .resolve = lane2_resolve,
+ .associate_req = lane2_associate_req,
+ .associate_indicator = NULL
};
static unsigned char bus_mac[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
const u8 *tlvs, u32 sizeoftlvs);
void (*associate_indicator) (struct net_device *dev, const u8 *mac_addr,
const u8 *tlvs, u32 sizeoftlvs);
-};
+} __no_const;
/*
* ATM LAN Emulation supports both LLC & Dix Ethernet EtherType
static struct in_cache_ops ingress_ops = {
- in_cache_add_entry, /* add_entry */
- in_cache_get, /* get */
- in_cache_get_with_mask, /* get_with_mask */
- in_cache_get_by_vcc, /* get_by_vcc */
- in_cache_put, /* put */
- in_cache_remove_entry, /* remove_entry */
- cache_hit, /* cache_hit */
- clear_count_and_expired, /* clear_count */
- check_resolving_entries, /* check_resolving */
- refresh_entries, /* refresh */
- in_destroy_cache /* destroy_cache */
+ .add_entry = in_cache_add_entry,
+ .get = in_cache_get,
+ .get_with_mask = in_cache_get_with_mask,
+ .get_by_vcc = in_cache_get_by_vcc,
+ .put = in_cache_put,
+ .remove_entry = in_cache_remove_entry,
+ .cache_hit = cache_hit,
+ .clear_count = clear_count_and_expired,
+ .check_resolving = check_resolving_entries,
+ .refresh = refresh_entries,
+ .destroy_cache = in_destroy_cache
};
static struct eg_cache_ops egress_ops = {
- eg_cache_add_entry, /* add_entry */
- eg_cache_get_by_cache_id, /* get_by_cache_id */
- eg_cache_get_by_tag, /* get_by_tag */
- eg_cache_get_by_vcc, /* get_by_vcc */
- eg_cache_get_by_src_ip, /* get_by_src_ip */
- eg_cache_put, /* put */
- eg_cache_remove_entry, /* remove_entry */
- update_eg_cache_entry, /* update */
- clear_expired, /* clear_expired */
- eg_destroy_cache /* destroy_cache */
+ .add_entry = eg_cache_add_entry,
+ .get_by_cache_id = eg_cache_get_by_cache_id,
+ .get_by_tag = eg_cache_get_by_tag,
+ .get_by_vcc = eg_cache_get_by_vcc,
+ .get_by_src_ip = eg_cache_get_by_src_ip,
+ .put = eg_cache_put,
+ .remove_entry = eg_cache_remove_entry,
+ .update = update_eg_cache_entry,
+ .clear_expired = clear_expired,
+ .destroy_cache = eg_destroy_cache
};
const struct k_atm_aal_stats *stats)
{
seq_printf(seq, "%s ( %d %d %d %d %d )", aal,
- atomic_read(&stats->tx), atomic_read(&stats->tx_err),
- atomic_read(&stats->rx), atomic_read(&stats->rx_err),
- atomic_read(&stats->rx_drop));
+ atomic_read_unchecked(&stats->tx),atomic_read_unchecked(&stats->tx_err),
+ atomic_read_unchecked(&stats->rx),atomic_read_unchecked(&stats->rx_err),
+ atomic_read_unchecked(&stats->rx_drop));
}
static void atm_dev_info(struct seq_file *seq, const struct atm_dev *dev)
static void copy_aal_stats(struct k_atm_aal_stats *from,
struct atm_aal_stats *to)
{
-#define __HANDLE_ITEM(i) to->i = atomic_read(&from->i)
+#define __HANDLE_ITEM(i) to->i = atomic_read_unchecked(&from->i)
__AAL_STAT_ITEMS
#undef __HANDLE_ITEM
}
static void subtract_aal_stats(struct k_atm_aal_stats *from,
struct atm_aal_stats *to)
{
-#define __HANDLE_ITEM(i) atomic_sub(to->i, &from->i)
+#define __HANDLE_ITEM(i) atomic_sub_unchecked(to->i, &from->i)
__AAL_STAT_ITEMS
#undef __HANDLE_ITEM
}
{
char path[sizeof("net/ax25/") + IFNAMSIZ];
int k;
- struct ctl_table *table;
+ ctl_table_no_const *table;
table = kmemdup(ax25_param_table, sizeof(ax25_param_table), GFP_KERNEL);
if (!table)
/* randomize initial seqno to avoid collision */
get_random_bytes(&random_seqno, sizeof(random_seqno));
- atomic_set(&hard_iface->bat_iv.ogm_seqno, random_seqno);
+ atomic_set_unchecked(&hard_iface->bat_iv.ogm_seqno, random_seqno);
hard_iface->bat_iv.ogm_buff_len = BATADV_OGM_HLEN;
ogm_buff = kmalloc(hard_iface->bat_iv.ogm_buff_len, GFP_ATOMIC);
batadv_ogm_packet->tvlv_len = htons(tvlv_len);
/* change sequence number to network order */
- seqno = (uint32_t)atomic_read(&hard_iface->bat_iv.ogm_seqno);
+ seqno = (uint32_t)atomic_read_unchecked(&hard_iface->bat_iv.ogm_seqno);
batadv_ogm_packet->seqno = htonl(seqno);
- atomic_inc(&hard_iface->bat_iv.ogm_seqno);
+ atomic_inc_unchecked(&hard_iface->bat_iv.ogm_seqno);
batadv_iv_ogm_slide_own_bcast_window(hard_iface);
return;
/* could be changed by schedule_own_packet() */
- if_incoming_seqno = atomic_read(&if_incoming->bat_iv.ogm_seqno);
+ if_incoming_seqno = atomic_read_unchecked(&if_incoming->bat_iv.ogm_seqno);
if (ogm_packet->flags & BATADV_DIRECTLINK)
has_directlink_flag = true;
frag_header.packet_type = BATADV_UNICAST_FRAG;
frag_header.version = BATADV_COMPAT_VERSION;
frag_header.ttl = BATADV_TTL;
- frag_header.seqno = htons(atomic_inc_return(&bat_priv->frag_seqno));
+ frag_header.seqno = htons(atomic_inc_return_unchecked(&bat_priv->frag_seqno));
frag_header.reserved = 0;
frag_header.no = 0;
frag_header.total_size = htons(skb->len);
primary_if->net_dev->dev_addr);
/* set broadcast sequence number */
- seqno = atomic_inc_return(&bat_priv->bcast_seqno);
+ seqno = atomic_inc_return_unchecked(&bat_priv->bcast_seqno);
bcast_packet->seqno = htonl(seqno);
batadv_add_bcast_packet_to_list(bat_priv, skb, brd_delay);
atomic_set(&bat_priv->batman_queue_left, BATADV_BATMAN_QUEUE_LEN);
atomic_set(&bat_priv->mesh_state, BATADV_MESH_INACTIVE);
- atomic_set(&bat_priv->bcast_seqno, 1);
+ atomic_set_unchecked(&bat_priv->bcast_seqno, 1);
atomic_set(&bat_priv->tt.vn, 0);
atomic_set(&bat_priv->tt.local_changes, 0);
atomic_set(&bat_priv->tt.ogm_append_cnt, 0);
/* randomize initial seqno to avoid collision */
get_random_bytes(&random_seqno, sizeof(random_seqno));
- atomic_set(&bat_priv->frag_seqno, random_seqno);
+ atomic_set_unchecked(&bat_priv->frag_seqno, random_seqno);
bat_priv->primary_if = NULL;
bat_priv->num_ifaces = 0;
return 0;
}
-struct rtnl_link_ops batadv_link_ops __read_mostly = {
+struct rtnl_link_ops batadv_link_ops = {
.kind = "batadv",
.priv_size = sizeof(struct batadv_priv),
.setup = batadv_softif_init_early,
struct batadv_hard_iface_bat_iv {
unsigned char *ogm_buff;
int ogm_buff_len;
- atomic_t ogm_seqno;
+ atomic_unchecked_t ogm_seqno;
};
/**
atomic_t bonding;
atomic_t fragmentation;
atomic_t packet_size_max;
- atomic_t frag_seqno;
+ atomic_unchecked_t frag_seqno;
#ifdef CONFIG_BATMAN_ADV_BLA
atomic_t bridge_loop_avoidance;
#endif
#endif
uint32_t isolation_mark;
uint32_t isolation_mark_mask;
- atomic_t bcast_seqno;
+ atomic_unchecked_t bcast_seqno;
atomic_t bcast_queue_left;
atomic_t batman_queue_left;
char num_ifaces;
uf.event_mask[1] = *((u32 *) f->event_mask + 1);
}
- len = min_t(unsigned int, len, sizeof(uf));
+ len = min((size_t)len, sizeof(uf));
if (copy_from_user(&uf, optval, len)) {
err = -EFAULT;
break;
break;
case L2CAP_CONF_RFC:
- if (olen == sizeof(rfc))
- memcpy(&rfc, (void *)val, olen);
+ if (olen != sizeof(rfc))
+ break;
+
+ memcpy(&rfc, (void *)val, olen);
if (test_bit(CONF_STATE2_DEVICE, &chan->conf_state) &&
rfc.mode != chan->mode)
struct sock *sk = sock->sk;
struct l2cap_chan *chan = l2cap_pi(sk)->chan;
struct l2cap_options opts;
- int len, err = 0;
+ int err = 0;
+ size_t len = optlen;
u32 opt;
BT_DBG("sk %p", sk);
opts.max_tx = chan->max_tx;
opts.txwin_size = chan->tx_win;
- len = min_t(unsigned int, sizeof(opts), optlen);
+ len = min(sizeof(opts), len);
if (copy_from_user((char *) &opts, optval, len)) {
err = -EFAULT;
break;
struct bt_security sec;
struct bt_power pwr;
struct l2cap_conn *conn;
- int len, err = 0;
+ int err = 0;
+ size_t len = optlen;
u32 opt;
BT_DBG("sk %p", sk);
sec.level = BT_SECURITY_LOW;
- len = min_t(unsigned int, sizeof(sec), optlen);
+ len = min(sizeof(sec), len);
if (copy_from_user((char *) &sec, optval, len)) {
err = -EFAULT;
break;
pwr.force_active = BT_POWER_FORCE_ACTIVE_ON;
- len = min_t(unsigned int, sizeof(pwr), optlen);
+ len = min(sizeof(pwr), len);
if (copy_from_user((char *) &pwr, optval, len)) {
err = -EFAULT;
break;
struct sock *sk = sock->sk;
struct bt_security sec;
int err = 0;
- size_t len;
+ size_t len = optlen;
u32 opt;
BT_DBG("sk %p", sk);
sec.level = BT_SECURITY_LOW;
- len = min_t(unsigned int, sizeof(sec), optlen);
+ len = min(sizeof(sec), len);
if (copy_from_user((char *) &sec, optval, len)) {
err = -EFAULT;
break;
BT_DBG("tty %p id %d", tty, tty->index);
BT_DBG("dev %p dst %pMR channel %d opened %d", dev, &dev->dst,
- dev->channel, dev->port.count);
+ dev->channel, atomic_read(&dev->port.count));
err = tty_port_open(&dev->port, tty, filp);
if (err)
struct rfcomm_dev *dev = (struct rfcomm_dev *) tty->driver_data;
BT_DBG("tty %p dev %p dlc %p opened %d", tty, dev, dev->dlc,
- dev->port.count);
+ atomic_read(&dev->port.count));
tty_port_close(&dev->port, tty, filp);
}
{
int err;
+ BUILD_BUG_ON(sizeof(struct br_input_skb_cb) > FIELD_SIZEOF(struct sk_buff, cb));
+
err = stp_proto_register(&br_stp_proto);
if (err < 0) {
pr_err("bridge: can't register sap for STP\n");
.get_link_af_size = br_get_link_af_size,
};
-struct rtnl_link_ops br_link_ops __read_mostly = {
+struct rtnl_link_ops br_link_ops = {
.kind = "bridge",
.priv_size = sizeof(struct net_bridge),
.setup = br_dev_setup,
tmp.valid_hooks = t->table->valid_hooks;
}
mutex_unlock(&ebt_mutex);
- if (copy_to_user(user, &tmp, *len) != 0) {
+ if (*len > sizeof(tmp) || copy_to_user(user, &tmp, *len) != 0) {
BUGPRINT("c2u Didn't work\n");
ret = -EFAULT;
break;
goto out;
tmp.valid_hooks = t->valid_hooks;
- if (copy_to_user(user, &tmp, *len) != 0) {
+ if (*len > sizeof(tmp) || copy_to_user(user, &tmp, *len) != 0) {
ret = -EFAULT;
break;
}
tmp.entries_size = t->table->entries_size;
tmp.valid_hooks = t->table->valid_hooks;
- if (copy_to_user(user, &tmp, *len) != 0) {
+ if (*len > sizeof(tmp) || copy_to_user(user, &tmp, *len) != 0) {
ret = -EFAULT;
break;
}
#include <linux/spinlock.h>
#include <linux/slab.h>
#include <linux/pkt_sched.h>
+#include <linux/sched.h>
#include <net/caif/caif_layer.h>
#include <net/caif/cfpkt.h>
#include <net/caif/cfctrl.h>
memset(&dev_info, 0, sizeof(dev_info));
dev_info.id = 0xff;
cfsrvl_init(&this->serv, 0, &dev_info, false);
- atomic_set(&this->req_seq_no, 1);
- atomic_set(&this->rsp_seq_no, 1);
+ atomic_set_unchecked(&this->req_seq_no, 1);
+ atomic_set_unchecked(&this->rsp_seq_no, 1);
this->serv.layer.receive = cfctrl_recv;
sprintf(this->serv.layer.name, "ctrl");
this->serv.layer.ctrlcmd = cfctrl_ctrlcmd;
struct cfctrl_request_info *req)
{
spin_lock_bh(&ctrl->info_list_lock);
- atomic_inc(&ctrl->req_seq_no);
- req->sequence_no = atomic_read(&ctrl->req_seq_no);
+ atomic_inc_unchecked(&ctrl->req_seq_no);
+ req->sequence_no = atomic_read_unchecked(&ctrl->req_seq_no);
list_add_tail(&req->list, &ctrl->list);
spin_unlock_bh(&ctrl->info_list_lock);
}
if (p != first)
pr_warn("Requests are not received in order\n");
- atomic_set(&ctrl->rsp_seq_no,
+ atomic_set_unchecked(&ctrl->rsp_seq_no,
p->sequence_no);
list_del(&p->list);
goto out;
};
-static struct rtnl_link_ops ipcaif_link_ops __read_mostly = {
+static struct rtnl_link_ops ipcaif_link_ops = {
.kind = "caif",
.priv_size = sizeof(struct chnl_net),
.setup = ipcaif_net_setup,
goto inval_skb;
}
+ skb->ip_summed = CHECKSUM_UNNECESSARY;
+
+ skb_reset_mac_header(skb);
skb_reset_network_header(skb);
skb_reset_transport_header(skb);
};
/* notifier block for netdevice event */
-static struct notifier_block can_netdev_notifier __read_mostly = {
+static struct notifier_block can_netdev_notifier = {
.notifier_call = can_notifier,
};
}
/* create /proc/net/can-bcm directory */
- proc_dir = proc_mkdir("can-bcm", init_net.proc_net);
+ proc_dir = proc_mkdir_restrict("can-bcm", init_net.proc_net);
return 0;
}
"default: " __stringify(CGW_DEFAULT_HOPS) ")");
static HLIST_HEAD(cgw_list);
-static struct notifier_block notifier;
static struct kmem_cache *cgw_cache __read_mostly;
return err;
}
+static struct notifier_block notifier = {
+ .notifier_call = cgw_notifier
+};
+
static __init int cgw_module_init(void)
{
/* sanitize given module parameter */
return -ENOMEM;
/* set notifier */
- notifier.notifier_call = cgw_notifier;
register_netdevice_notifier(¬ifier);
if (__rtnl_register(PF_CAN, RTM_GETROUTE, NULL, cgw_dump_jobs, NULL)) {
void can_init_proc(void)
{
/* create /proc/net/can directory */
- can_dir = proc_mkdir("can", init_net.proc_net);
+ can_dir = proc_mkdir_restrict("can", init_net.proc_net);
if (!can_dir) {
printk(KERN_INFO "can: failed to create /proc/net/can . "
#define MAX_ADDR_STR_LEN 64 /* 54 is enough */
static char addr_str[ADDR_STR_COUNT][MAX_ADDR_STR_LEN];
-static atomic_t addr_str_seq = ATOMIC_INIT(0);
+static atomic_unchecked_t addr_str_seq = ATOMIC_INIT(0);
static struct page *zero_page; /* used in certain error cases */
struct sockaddr_in *in4 = (struct sockaddr_in *) ss;
struct sockaddr_in6 *in6 = (struct sockaddr_in6 *) ss;
- i = atomic_inc_return(&addr_str_seq) & ADDR_STR_COUNT_MASK;
+ i = atomic_inc_return_unchecked(&addr_str_seq) & ADDR_STR_COUNT_MASK;
s = addr_str[i];
switch (ss->ss_family) {
#define CMSG_COMPAT_FIRSTHDR(msg) \
(((msg)->msg_controllen) >= sizeof(struct compat_cmsghdr) ? \
- (struct compat_cmsghdr __user *)((msg)->msg_control) : \
+ (struct compat_cmsghdr __force_user *)((msg)->msg_control) : \
(struct compat_cmsghdr __user *)NULL)
#define CMSG_COMPAT_OK(ucmlen, ucmsg, mhdr) \
((ucmlen) >= sizeof(struct compat_cmsghdr) && \
(ucmlen) <= (unsigned long) \
((mhdr)->msg_controllen - \
- ((char *)(ucmsg) - (char *)(mhdr)->msg_control)))
+ ((char __force_kernel *)(ucmsg) - (char *)(mhdr)->msg_control)))
static inline struct compat_cmsghdr __user *cmsg_compat_nxthdr(struct msghdr *msg,
struct compat_cmsghdr __user *cmsg, int cmsg_len)
{
char __user *ptr = (char __user *)cmsg + CMSG_COMPAT_ALIGN(cmsg_len);
- if ((unsigned long)(ptr + 1 - (char __user *)msg->msg_control) >
+ if ((unsigned long)(ptr + 1 - (char __force_user *)msg->msg_control) >
msg->msg_controllen)
return NULL;
return (struct compat_cmsghdr __user *)ptr;
int put_cmsg_compat(struct msghdr *kmsg, int level, int type, int len, void *data)
{
- struct compat_cmsghdr __user *cm = (struct compat_cmsghdr __user *) kmsg->msg_control;
+ struct compat_cmsghdr __user *cm = (struct compat_cmsghdr __force_user *) kmsg->msg_control;
struct compat_cmsghdr cmhdr;
struct compat_timeval ctv;
struct compat_timespec cts[3];
void scm_detach_fds_compat(struct msghdr *kmsg, struct scm_cookie *scm)
{
- struct compat_cmsghdr __user *cm = (struct compat_cmsghdr __user *) kmsg->msg_control;
+ struct compat_cmsghdr __user *cm = (struct compat_cmsghdr __force_user *) kmsg->msg_control;
int fdmax = (kmsg->msg_controllen - sizeof(struct compat_cmsghdr)) / sizeof(int);
int fdnum = scm->fp->count;
struct file **fp = scm->fp->fp;
return -EFAULT;
old_fs = get_fs();
set_fs(KERNEL_DS);
- err = sock_setsockopt(sock, level, optname, (char *)&ktime, sizeof(ktime));
+ err = sock_setsockopt(sock, level, optname, (char __force_user *)&ktime, sizeof(ktime));
set_fs(old_fs);
return err;
len = sizeof(ktime);
old_fs = get_fs();
set_fs(KERNEL_DS);
- err = sock_getsockopt(sock, level, optname, (char *) &ktime, &len);
+ err = sock_getsockopt(sock, level, optname, (char __force_user *) &ktime, (int __force_user *)&len);
set_fs(old_fs);
if (!err) {
case MCAST_JOIN_GROUP:
case MCAST_LEAVE_GROUP:
{
- struct compat_group_req __user *gr32 = (void *)optval;
+ struct compat_group_req __user *gr32 = (void __user *)optval;
struct group_req __user *kgr =
compat_alloc_user_space(sizeof(struct group_req));
u32 interface;
case MCAST_BLOCK_SOURCE:
case MCAST_UNBLOCK_SOURCE:
{
- struct compat_group_source_req __user *gsr32 = (void *)optval;
+ struct compat_group_source_req __user *gsr32 = (void __user *)optval;
struct group_source_req __user *kgsr = compat_alloc_user_space(
sizeof(struct group_source_req));
u32 interface;
}
case MCAST_MSFILTER:
{
- struct compat_group_filter __user *gf32 = (void *)optval;
+ struct compat_group_filter __user *gf32 = (void __user *)optval;
struct group_filter __user *kgf;
u32 interface, fmode, numsrc;
char __user *optval, int __user *optlen,
int (*getsockopt)(struct sock *, int, int, char __user *, int __user *))
{
- struct compat_group_filter __user *gf32 = (void *)optval;
+ struct compat_group_filter __user *gf32 = (void __user *)optval;
struct group_filter __user *kgf;
int __user *koptlen;
u32 interface, fmode, numsrc;
if (call < SYS_SOCKET || call > SYS_SENDMMSG)
return -EINVAL;
- if (copy_from_user(a, args, nas[call]))
+ if (nas[call] > sizeof a || copy_from_user(a, args, nas[call]))
return -EFAULT;
a0 = a[0];
a1 = a[1];
}
kfree_skb(skb);
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
sk_mem_reclaim_partial(sk);
return err;
{
if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
if (skb_copy_ubufs(skb, GFP_ATOMIC)) {
- atomic_long_inc(&dev->rx_dropped);
+ atomic_long_inc_unchecked(&dev->rx_dropped);
kfree_skb(skb);
return NET_RX_DROP;
}
}
if (unlikely(!is_skb_forwardable(dev, skb))) {
- atomic_long_inc(&dev->rx_dropped);
+ atomic_long_inc_unchecked(&dev->rx_dropped);
kfree_skb(skb);
return NET_RX_DROP;
}
drop:
rcu_read_unlock_bh();
- atomic_long_inc(&dev->tx_dropped);
+ atomic_long_inc_unchecked(&dev->tx_dropped);
kfree_skb_list(skb);
return rc;
out:
local_irq_restore(flags);
- atomic_long_inc(&skb->dev->rx_dropped);
+ atomic_long_inc_unchecked(&skb->dev->rx_dropped);
kfree_skb(skb);
return NET_RX_DROP;
}
}
EXPORT_SYMBOL(netif_rx_ni);
-static void net_tx_action(struct softirq_action *h)
+static __latent_entropy void net_tx_action(void)
{
struct softnet_data *sd = this_cpu_ptr(&softnet_data);
ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
} else {
drop:
- atomic_long_inc(&skb->dev->rx_dropped);
+ atomic_long_inc_unchecked(&skb->dev->rx_dropped);
kfree_skb(skb);
/* Jamal, now you will not able to escape explaining
* me how you were going to use this. :-)
return work;
}
-static void net_rx_action(struct softirq_action *h)
+static __latent_entropy void net_rx_action(void)
{
struct softnet_data *sd = this_cpu_ptr(&softnet_data);
unsigned long time_limit = jiffies + 2;
} else {
netdev_stats_to_stats64(storage, &dev->stats);
}
- storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
- storage->tx_dropped += atomic_long_read(&dev->tx_dropped);
+ storage->rx_dropped += atomic_long_read_unchecked(&dev->rx_dropped);
+ storage->tx_dropped += atomic_long_read_unchecked(&dev->tx_dropped);
return storage;
}
EXPORT_SYMBOL(dev_get_stats);
no_module = !dev;
if (no_module && capable(CAP_NET_ADMIN))
no_module = request_module("netdev-%s", name);
- if (no_module && capable(CAP_SYS_MODULE))
+ if (no_module && capable(CAP_SYS_MODULE)) {
+#ifdef CONFIG_GRKERNSEC_MODHARDEN
+ ___request_module(true, "grsec_modharden_netdev", "%s", name);
+#else
request_module("%s", name);
+#endif
+ }
}
EXPORT_SYMBOL(dev_load);
/* Unkown instruction. */
default:
- goto err;
+ WARN(1, KERN_ALERT "Unknown sock filter code:%u jt:%u tf:%u k:%u\n",
+ fp->code, fp->jt, fp->jf, fp->k);
+ kfree(addrs);
+ BUG();
+ return -EINVAL;
}
insn++;
u16 *masks, memvalid = 0; /* One bit per cell, 16 cells */
int pc, ret = 0;
- BUILD_BUG_ON(BPF_MEMWORDS > 16);
+ BUILD_BUG_ON(BPF_MEMWORDS != 16);
masks = kmalloc_array(flen, sizeof(*masks), GFP_KERNEL);
if (!masks)
if (!fp)
return -ENOMEM;
- memcpy(fp->insns, fprog->filter, fsize);
+ memcpy(fp->insns, (void __force_kernel *)fprog->filter, fsize);
fp->len = fprog->len;
/* Since unattached filters are not copied back to user
static int flow_entry_valid(struct flow_cache_entry *fle,
struct netns_xfrm *xfrm)
{
- if (atomic_read(&xfrm->flow_cache_genid) != fle->genid)
+ if (atomic_read_unchecked(&xfrm->flow_cache_genid) != fle->genid)
return 0;
if (fle->object && !fle->object->ops->check(fle->object))
return 0;
hlist_add_head(&fle->u.hlist, &fcp->hash_table[hash]);
fcp->hash_count++;
}
- } else if (likely(fle->genid == atomic_read(&net->xfrm.flow_cache_genid))) {
+ } else if (likely(fle->genid == atomic_read_unchecked(&net->xfrm.flow_cache_genid))) {
flo = fle->object;
if (!flo)
goto ret_object;
}
flo = resolver(net, key, family, dir, flo, ctx);
if (fle) {
- fle->genid = atomic_read(&net->xfrm.flow_cache_genid);
+ fle->genid = atomic_read_unchecked(&net->xfrm.flow_cache_genid);
if (!IS_ERR(flo))
fle->object = flo;
else
void __user *buffer, size_t *lenp, loff_t *ppos)
{
int size, ret;
- struct ctl_table tmp = *ctl;
+ ctl_table_no_const tmp = *ctl;
tmp.extra1 = &zero;
tmp.extra2 = &unres_qlen_max;
void __user *buffer,
size_t *lenp, loff_t *ppos)
{
- struct ctl_table tmp = *ctl;
+ ctl_table_no_const tmp = *ctl;
int ret;
tmp.extra1 = &zero;
struct rtnl_link_stats64 temp;
const struct rtnl_link_stats64 *stats = dev_get_stats(dev, &temp);
- seq_printf(seq, "%6s: %7llu %7llu %4llu %4llu %4llu %5llu %10llu %9llu "
+ if (gr_proc_is_restricted())
+ seq_printf(seq, "%6s: %7llu %7llu %4llu %4llu %4llu %5llu %10llu %9llu "
+ "%8llu %7llu %4llu %4llu %4llu %5llu %7llu %10llu\n",
+ dev->name, 0ULL, 0ULL, 0ULL, 0ULL, 0ULL, 0ULL, 0ULL, 0ULL, 0ULL, 0ULL,
+ 0ULL, 0ULL, 0ULL, 0ULL, 0ULL, 0ULL);
+ else
+ seq_printf(seq, "%6s: %7llu %7llu %4llu %4llu %4llu %5llu %10llu %9llu "
"%8llu %7llu %4llu %4llu %4llu %5llu %7llu %10llu\n",
dev->name, stats->rx_bytes, stats->rx_packets,
stats->rx_errors,
return 0;
}
-static const struct seq_operations dev_seq_ops = {
+const struct seq_operations dev_seq_ops = {
.start = dev_seq_start,
.next = dev_seq_next,
.stop = dev_seq_stop,
static int softnet_seq_open(struct inode *inode, struct file *file)
{
- return seq_open(file, &softnet_seq_ops);
+ return seq_open_restrict(file, &softnet_seq_ops);
}
static const struct file_operations softnet_seq_fops = {
else
seq_printf(seq, "%04x", ntohs(pt->type));
+#ifdef CONFIG_GRKERNSEC_HIDESYM
+ seq_printf(seq, " %-8s %pf\n",
+ pt->dev ? pt->dev->name : "", NULL);
+#else
seq_printf(seq, " %-8s %pf\n",
pt->dev ? pt->dev->name : "", pt->func);
+#endif
}
return 0;
{
struct net_device *netdev = to_net_dev(dev);
return sprintf(buf, fmt_dec,
- atomic_read(&netdev->carrier_changes));
+ atomic_read_unchecked(&netdev->carrier_changes));
}
static DEVICE_ATTR_RO(carrier_changes);
int error;
LIST_HEAD(net_exit_list);
- list_add_tail(&ops->list, list);
+ pax_list_add_tail((struct list_head *)&ops->list, list);
if (ops->init || (ops->id && ops->size)) {
for_each_net(net) {
error = ops_init(ops, net);
out_undo:
/* If I have an error cleanup all namespaces I initialized */
- list_del(&ops->list);
+ pax_list_del((struct list_head *)&ops->list);
ops_exit_list(ops, &net_exit_list);
ops_free_list(ops, &net_exit_list);
return error;
struct net *net;
LIST_HEAD(net_exit_list);
- list_del(&ops->list);
+ pax_list_del((struct list_head *)&ops->list);
for_each_net(net)
list_add_tail(&net->exit_list, &net_exit_list);
ops_exit_list(ops, &net_exit_list);
mutex_lock(&net_mutex);
error = register_pernet_operations(&pernet_list, ops);
if (!error && (first_device == &pernet_list))
- first_device = &ops->list;
+ first_device = (struct list_head *)&ops->list;
mutex_unlock(&net_mutex);
return error;
}
struct udphdr *udph;
struct iphdr *iph;
struct ethhdr *eth;
- static atomic_t ip_ident;
+ static atomic_unchecked_t ip_ident;
struct ipv6hdr *ip6h;
udp_len = len + sizeof(*udph);
put_unaligned(0x45, (unsigned char *)iph);
iph->tos = 0;
put_unaligned(htons(ip_len), &(iph->tot_len));
- iph->id = htons(atomic_inc_return(&ip_ident));
+ iph->id = htons(atomic_inc_return_unchecked(&ip_ident));
iph->frag_off = 0;
iph->ttl = 64;
iph->protocol = IPPROTO_UDP;
pn->net = net;
INIT_LIST_HEAD(&pn->pktgen_threads);
pn->pktgen_exiting = false;
- pn->proc_dir = proc_mkdir(PG_PROC_DIR, pn->net->proc_net);
+ pn->proc_dir = proc_mkdir_restrict(PG_PROC_DIR, pn->net->proc_net);
if (!pn->proc_dir) {
pr_warn("cannot create /proc/net/%s\n", PG_PROC_DIR);
return -ENODEV;
rtnl_doit_func doit;
rtnl_dumpit_func dumpit;
rtnl_calcit_func calcit;
-};
+} __no_const;
static DEFINE_MUTEX(rtnl_mutex);
* to use the ops for creating device. So do not
* fill up dellink as well. That disables rtnl_dellink.
*/
- if (ops->setup && !ops->dellink)
- ops->dellink = unregister_netdevice_queue;
+ if (ops->setup && !ops->dellink) {
+ pax_open_kernel();
+ *(void **)&ops->dellink = unregister_netdevice_queue;
+ pax_close_kernel();
+ }
- list_add_tail(&ops->list, &link_ops);
+ pax_list_add_tail((struct list_head *)&ops->list, &link_ops);
return 0;
}
EXPORT_SYMBOL_GPL(__rtnl_link_register);
for_each_net(net) {
__rtnl_kill_links(net, ops);
}
- list_del(&ops->list);
+ pax_list_del((struct list_head *)&ops->list);
}
EXPORT_SYMBOL_GPL(__rtnl_link_unregister);
(dev->ifalias &&
nla_put_string(skb, IFLA_IFALIAS, dev->ifalias)) ||
nla_put_u32(skb, IFLA_CARRIER_CHANGES,
- atomic_read(&dev->carrier_changes)))
+ atomic_read_unchecked(&dev->carrier_changes)))
goto nla_put_failure;
if (1) {
if (IS_ERR(dest_net))
return PTR_ERR(dest_net);
+ err = -EPERM;
+ if (!netlink_ns_capable(skb, dest_net->user_ns, CAP_NET_ADMIN))
+ goto out;
+
dev = rtnl_create_link(dest_net, ifname, name_assign_type, ops, tb);
if (IS_ERR(dev)) {
err = PTR_ERR(dev);
int put_cmsg(struct msghdr * msg, int level, int type, int len, void *data)
{
struct cmsghdr __user *cm
- = (__force struct cmsghdr __user *)msg->msg_control;
+ = (struct cmsghdr __force_user *)msg->msg_control;
struct cmsghdr cmhdr;
int cmlen = CMSG_LEN(len);
int err;
err = -EFAULT;
if (copy_to_user(cm, &cmhdr, sizeof cmhdr))
goto out;
- if (copy_to_user(CMSG_DATA(cm), data, cmlen - sizeof(struct cmsghdr)))
+ if (copy_to_user((void __force_user *)CMSG_DATA((void __force_kernel *)cm), data, cmlen - sizeof(struct cmsghdr)))
goto out;
cmlen = CMSG_SPACE(len);
if (msg->msg_controllen < cmlen)
void scm_detach_fds(struct msghdr *msg, struct scm_cookie *scm)
{
struct cmsghdr __user *cm
- = (__force struct cmsghdr __user*)msg->msg_control;
+ = (struct cmsghdr __force_user *)msg->msg_control;
int fdmax = 0;
int fdnum = scm->fp->count;
if (fdnum < fdmax)
fdmax = fdnum;
- for (i=0, cmfptr=(__force int __user *)CMSG_DATA(cm); i<fdmax;
+ for (i=0, cmfptr=(int __force_user *)CMSG_DATA((void __force_kernel *)cm); i<fdmax;
i++, cmfptr++)
{
struct socket *sock;
__wsum skb_checksum(const struct sk_buff *skb, int offset,
int len, __wsum csum)
{
- const struct skb_checksum_ops ops = {
+ static const struct skb_checksum_ops ops = {
.update = csum_partial_ext,
.combine = csum_block_add_ext,
};
skbuff_head_cache = kmem_cache_create("skbuff_head_cache",
sizeof(struct sk_buff),
0,
- SLAB_HWCACHE_ALIGN|SLAB_PANIC,
+ SLAB_HWCACHE_ALIGN|SLAB_PANIC|
+ SLAB_NO_SANITIZE,
NULL);
skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
sizeof(struct sk_buff_fclones),
0,
- SLAB_HWCACHE_ALIGN|SLAB_PANIC,
+ SLAB_HWCACHE_ALIGN|SLAB_PANIC|
+ SLAB_NO_SANITIZE,
NULL);
}
struct sk_buff_head *list = &sk->sk_receive_queue;
if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf) {
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
trace_sock_rcvqueue_full(sk, skb);
return -ENOMEM;
}
return err;
if (!sk_rmem_schedule(sk, skb, skb->truesize)) {
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
return -ENOBUFS;
}
skb_dst_force(skb);
spin_lock_irqsave(&list->lock, flags);
- skb->dropcount = atomic_read(&sk->sk_drops);
+ skb->dropcount = atomic_read_unchecked(&sk->sk_drops);
__skb_queue_tail(list, skb);
spin_unlock_irqrestore(&list->lock, flags);
skb->dev = NULL;
if (sk_rcvqueues_full(sk, sk->sk_rcvbuf)) {
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
goto discard_and_relse;
}
if (nested)
mutex_release(&sk->sk_lock.dep_map, 1, _RET_IP_);
} else if (sk_add_backlog(sk, skb, sk->sk_rcvbuf)) {
bh_unlock_sock(sk);
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
goto discard_and_relse;
}
}
break;
+#ifndef GRKERNSEC_BPF_HARDEN
case SO_ATTACH_BPF:
ret = -EINVAL;
if (optlen == sizeof(u32)) {
ret = sk_attach_bpf(ufd, sk);
}
break;
-
+#endif
case SO_DETACH_FILTER:
ret = sk_detach_filter(sk);
break;
struct timeval tm;
} v;
- int lv = sizeof(int);
- int len;
+ unsigned int lv = sizeof(int);
+ unsigned int len;
if (get_user(len, optlen))
return -EFAULT;
- if (len < 0)
+ if (len > INT_MAX)
return -EINVAL;
memset(&v, 0, sizeof(v));
case SO_PEERNAME:
{
- char address[128];
+ char address[_K_SS_MAXSIZE];
if (sock->ops->getname(sock, (struct sockaddr *)address, &lv, 2))
return -ENOTCONN;
- if (lv < len)
+ if (lv < len || sizeof address < len)
return -EINVAL;
if (copy_to_user(optval, address, len))
return -EFAULT;
if (len > lv)
len = lv;
- if (copy_to_user(optval, &v, len))
+ if (len > sizeof(v) || copy_to_user(optval, &v, len))
return -EFAULT;
lenout:
if (put_user(len, optlen))
*/
smp_wmb();
atomic_set(&sk->sk_refcnt, 1);
- atomic_set(&sk->sk_drops, 0);
+ atomic_set_unchecked(&sk->sk_drops, 0);
}
EXPORT_SYMBOL(sock_init_data);
int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len,
int level, int type)
{
+ struct sock_extended_err ee;
struct sock_exterr_skb *serr;
struct sk_buff *skb;
int copied, err;
sock_recv_timestamp(msg, sk, skb);
serr = SKB_EXT_ERR(skb);
- put_cmsg(msg, level, type, sizeof(serr->ee), &serr->ee);
+ ee = serr->ee;
+ put_cmsg(msg, level, type, sizeof ee, &ee);
msg->msg_flags |= MSG_ERRQUEUE;
err = copied;
#include <linux/inet_diag.h>
#include <linux/sock_diag.h>
-static const struct sock_diag_handler *sock_diag_handlers[AF_MAX];
+static const struct sock_diag_handler *sock_diag_handlers[AF_MAX] __read_only;
static int (*inet_rcv_compat)(struct sk_buff *skb, struct nlmsghdr *nlh);
static DEFINE_MUTEX(sock_diag_table_mutex);
int sock_diag_check_cookie(void *sk, __u32 *cookie)
{
+#ifndef CONFIG_GRKERNSEC_HIDESYM
if ((cookie[0] != INET_DIAG_NOCOOKIE ||
cookie[1] != INET_DIAG_NOCOOKIE) &&
((u32)(unsigned long)sk != cookie[0] ||
(u32)((((unsigned long)sk) >> 31) >> 1) != cookie[1]))
return -ESTALE;
else
+#endif
return 0;
}
EXPORT_SYMBOL_GPL(sock_diag_check_cookie);
void sock_diag_save_cookie(void *sk, __u32 *cookie)
{
+#ifdef CONFIG_GRKERNSEC_HIDESYM
+ cookie[0] = 0;
+ cookie[1] = 0;
+#else
cookie[0] = (u32)(unsigned long)sk;
cookie[1] = (u32)(((unsigned long)sk >> 31) >> 1);
+#endif
}
EXPORT_SYMBOL_GPL(sock_diag_save_cookie);
mutex_lock(&sock_diag_table_mutex);
if (sock_diag_handlers[hndl->family])
err = -EBUSY;
- else
+ else {
+ pax_open_kernel();
sock_diag_handlers[hndl->family] = hndl;
+ pax_close_kernel();
+ }
mutex_unlock(&sock_diag_table_mutex);
return err;
mutex_lock(&sock_diag_table_mutex);
BUG_ON(sock_diag_handlers[family] != hnld);
+ pax_open_kernel();
sock_diag_handlers[family] = NULL;
+ pax_close_kernel();
mutex_unlock(&sock_diag_table_mutex);
}
EXPORT_SYMBOL_GPL(sock_diag_unregister);
static int zero = 0;
static int one = 1;
static int ushort_max = USHRT_MAX;
+static int min_sndbuf = SOCK_MIN_SNDBUF;
+static int min_rcvbuf = SOCK_MIN_RCVBUF;
static int net_msg_warn; /* Unused, but still a sysctl */
{
unsigned int orig_size, size;
int ret, i;
- struct ctl_table tmp = {
+ ctl_table_no_const tmp = {
.data = &size,
.maxlen = sizeof(size),
.mode = table->mode
void __user *buffer, size_t *lenp, loff_t *ppos)
{
char id[IFNAMSIZ];
- struct ctl_table tbl = {
+ ctl_table_no_const tbl = {
.data = id,
.maxlen = IFNAMSIZ,
};
static int proc_do_rss_key(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp, loff_t *ppos)
{
- struct ctl_table fake_table;
+ ctl_table_no_const fake_table;
char buf[NETDEV_RSS_KEY_LEN * 3];
snprintf(buf, sizeof(buf), "%*phC", NETDEV_RSS_KEY_LEN, netdev_rss_key);
.maxlen = sizeof(int),
.mode = 0644,
.proc_handler = proc_dointvec_minmax,
- .extra1 = &one,
+ .extra1 = &min_sndbuf,
},
{
.procname = "rmem_max",
.maxlen = sizeof(int),
.mode = 0644,
.proc_handler = proc_dointvec_minmax,
- .extra1 = &one,
+ .extra1 = &min_rcvbuf,
},
{
.procname = "wmem_default",
.maxlen = sizeof(int),
.mode = 0644,
.proc_handler = proc_dointvec_minmax,
- .extra1 = &one,
+ .extra1 = &min_sndbuf,
},
{
.procname = "rmem_default",
.maxlen = sizeof(int),
.mode = 0644,
.proc_handler = proc_dointvec_minmax,
- .extra1 = &one,
+ .extra1 = &min_rcvbuf,
},
{
.procname = "dev_weight",
.mode = 0444,
.proc_handler = proc_do_rss_key,
},
-#ifdef CONFIG_BPF_JIT
+#if defined(CONFIG_BPF_JIT) && !defined(CONFIG_GRKERNSEC_BPF_HARDEN)
{
.procname = "bpf_jit_enable",
.data = &bpf_jit_enable,
static __net_init int sysctl_core_net_init(struct net *net)
{
- struct ctl_table *tbl;
+ ctl_table_no_const *tbl = NULL;
net->core.sysctl_somaxconn = SOMAXCONN;
- tbl = netns_core_table;
if (!net_eq(net, &init_net)) {
- tbl = kmemdup(tbl, sizeof(netns_core_table), GFP_KERNEL);
+ tbl = kmemdup(netns_core_table, sizeof(netns_core_table), GFP_KERNEL);
if (tbl == NULL)
goto err_dup;
if (net->user_ns != &init_user_ns) {
tbl[0].procname = NULL;
}
- }
-
- net->core.sysctl_hdr = register_net_sysctl(net, "net/core", tbl);
+ net->core.sysctl_hdr = register_net_sysctl(net, "net/core", tbl);
+ } else
+ net->core.sysctl_hdr = register_net_sysctl(net, "net/core", netns_core_table);
if (net->core.sysctl_hdr == NULL)
goto err_reg;
return 0;
err_reg:
- if (tbl != netns_core_table)
- kfree(tbl);
+ kfree(tbl);
err_dup:
return -ENOMEM;
}
kfree(tbl);
}
-static __net_initdata struct pernet_operations sysctl_core_ops = {
+static __net_initconst struct pernet_operations sysctl_core_ops = {
.init = sysctl_core_net_init,
.exit = sysctl_core_net_exit,
};
.sysctl_rmem = sysctl_decnet_rmem,
.max_header = DN_MAX_NSP_DATA_HEADER + 64,
.obj_size = sizeof(struct dn_sock),
+ .slab_flags = SLAB_USERCOPY,
};
static struct sock *dn_alloc_sock(struct net *net, struct socket *sock, gfp_t gfp)
.extra1 = &min_t3,
.extra2 = &max_t3
},
- {0}
+ { }
},
};
if (len > *lenp) len = *lenp;
- if (copy_to_user(buffer, addr, len))
+ if (len > sizeof addr || copy_to_user(buffer, addr, len))
return -EFAULT;
*lenp = len;
if (len > *lenp) len = *lenp;
- if (copy_to_user(buffer, devname, len))
+ if (len > sizeof devname || copy_to_user(buffer, devname, len))
return -EFAULT;
*lenp = len;
return -EMSGSIZE;
}
-static struct rtnl_link_ops hsr_link_ops __read_mostly = {
+static struct rtnl_link_ops hsr_link_ops = {
.kind = "hsr",
.maxtype = IFLA_HSR_MAX,
.policy = hsr_policy,
dev_put(real_dev);
}
-static struct rtnl_link_ops lowpan_link_ops __read_mostly = {
+static struct rtnl_link_ops lowpan_link_ops = {
.kind = "lowpan",
.priv_size = sizeof(struct lowpan_dev_info),
.setup = lowpan_setup,
static int __net_init lowpan_frags_ns_sysctl_register(struct net *net)
{
- struct ctl_table *table;
+ ctl_table_no_const *table = NULL;
struct ctl_table_header *hdr;
struct netns_ieee802154_lowpan *ieee802154_lowpan =
net_ieee802154_lowpan(net);
- table = lowpan_frags_ns_ctl_table;
if (!net_eq(net, &init_net)) {
- table = kmemdup(table, sizeof(lowpan_frags_ns_ctl_table),
+ table = kmemdup(lowpan_frags_ns_ctl_table, sizeof(lowpan_frags_ns_ctl_table),
GFP_KERNEL);
if (table == NULL)
goto err_alloc;
/* Don't export sysctls to unprivileged users */
if (net->user_ns != &init_user_ns)
table[0].procname = NULL;
- }
-
- hdr = register_net_sysctl(net, "net/ieee802154/6lowpan", table);
+ hdr = register_net_sysctl(net, "net/ieee802154/6lowpan", table);
+ } else
+ hdr = register_net_sysctl(net, "net/ieee802154/6lowpan", lowpan_frags_ns_ctl_table);
if (hdr == NULL)
goto err_reg;
return 0;
err_reg:
- if (!net_eq(net, &init_net))
- kfree(table);
+ kfree(table);
err_alloc:
return -ENOMEM;
}
return ip_recv_error(sk, msg, len, addr_len);
#if IS_ENABLED(CONFIG_IPV6)
if (sk->sk_family == AF_INET6)
- return pingv6_ops.ipv6_recv_error(sk, msg, len, addr_len);
+ return pingv6_ops->ipv6_recv_error(sk, msg, len, addr_len);
#endif
return -EINVAL;
}
static struct ipv4_devconf ipv4_devconf = {
.data = {
- [IPV4_DEVCONF_ACCEPT_REDIRECTS - 1] = 1,
+ [IPV4_DEVCONF_ACCEPT_REDIRECTS - 1] = 0,
+ [IPV4_DEVCONF_RP_FILTER - 1] = 1,
[IPV4_DEVCONF_SEND_REDIRECTS - 1] = 1,
[IPV4_DEVCONF_SECURE_REDIRECTS - 1] = 1,
[IPV4_DEVCONF_SHARED_MEDIA - 1] = 1,
static struct ipv4_devconf ipv4_devconf_dflt = {
.data = {
- [IPV4_DEVCONF_ACCEPT_REDIRECTS - 1] = 1,
+ [IPV4_DEVCONF_ACCEPT_REDIRECTS - 1] = 0,
+ [IPV4_DEVCONF_RP_FILTER - 1] = 1,
[IPV4_DEVCONF_SEND_REDIRECTS - 1] = 1,
[IPV4_DEVCONF_SECURE_REDIRECTS - 1] = 1,
[IPV4_DEVCONF_SHARED_MEDIA - 1] = 1,
idx = 0;
head = &net->dev_index_head[h];
rcu_read_lock();
- cb->seq = atomic_read(&net->ipv4.dev_addr_genid) ^
+ cb->seq = atomic_read_unchecked(&net->ipv4.dev_addr_genid) ^
net->dev_base_seq;
hlist_for_each_entry_rcu(dev, head, index_hlist) {
if (idx < s_idx)
idx = 0;
head = &net->dev_index_head[h];
rcu_read_lock();
- cb->seq = atomic_read(&net->ipv4.dev_addr_genid) ^
+ cb->seq = atomic_read_unchecked(&net->ipv4.dev_addr_genid) ^
net->dev_base_seq;
hlist_for_each_entry_rcu(dev, head, index_hlist) {
if (idx < s_idx)
#define DEVINET_SYSCTL_FLUSHING_ENTRY(attr, name) \
DEVINET_SYSCTL_COMPLEX_ENTRY(attr, name, ipv4_doint_and_flush)
-static struct devinet_sysctl_table {
+static const struct devinet_sysctl_table {
struct ctl_table_header *sysctl_header;
struct ctl_table devinet_vars[__IPV4_DEVCONF_MAX];
} devinet_sysctl = {
int err;
struct ipv4_devconf *all, *dflt;
#ifdef CONFIG_SYSCTL
- struct ctl_table *tbl = ctl_forward_entry;
+ ctl_table_no_const *tbl = NULL;
struct ctl_table_header *forw_hdr;
#endif
goto err_alloc_dflt;
#ifdef CONFIG_SYSCTL
- tbl = kmemdup(tbl, sizeof(ctl_forward_entry), GFP_KERNEL);
+ tbl = kmemdup(ctl_forward_entry, sizeof(ctl_forward_entry), GFP_KERNEL);
if (tbl == NULL)
goto err_alloc_ctl;
goto err_reg_dflt;
err = -ENOMEM;
- forw_hdr = register_net_sysctl(net, "net/ipv4", tbl);
+ if (!net_eq(net, &init_net))
+ forw_hdr = register_net_sysctl(net, "net/ipv4", tbl);
+ else
+ forw_hdr = register_net_sysctl(net, "net/ipv4", ctl_forward_entry);
if (forw_hdr == NULL)
goto err_reg_ctl;
net->ipv4.forw_hdr = forw_hdr;
err_reg_dflt:
__devinet_sysctl_unregister(all);
err_reg_all:
- if (tbl != ctl_forward_entry)
- kfree(tbl);
+ kfree(tbl);
err_alloc_ctl:
#endif
if (dflt != &ipv4_devconf_dflt)
#ifdef CONFIG_IP_ROUTE_MULTIPATH
fib_sync_up(dev);
#endif
- atomic_inc(&net->ipv4.dev_addr_genid);
+ atomic_inc_unchecked(&net->ipv4.dev_addr_genid);
rt_cache_flush(dev_net(dev));
break;
case NETDEV_DOWN:
fib_del_ifaddr(ifa, NULL);
- atomic_inc(&net->ipv4.dev_addr_genid);
+ atomic_inc_unchecked(&net->ipv4.dev_addr_genid);
if (ifa->ifa_dev->ifa_list == NULL) {
/* Last address was deleted from this interface.
* Disable IP.
#ifdef CONFIG_IP_ROUTE_MULTIPATH
fib_sync_up(dev);
#endif
- atomic_inc(&net->ipv4.dev_addr_genid);
+ atomic_inc_unchecked(&net->ipv4.dev_addr_genid);
rt_cache_flush(net);
break;
case NETDEV_DOWN:
nh->nh_saddr = inet_select_addr(nh->nh_dev,
nh->nh_gw,
nh->nh_parent->fib_scope);
- nh->nh_saddr_genid = atomic_read(&net->ipv4.dev_addr_genid);
+ nh->nh_saddr_genid = atomic_read_unchecked(&net->ipv4.dev_addr_genid);
return nh->nh_saddr;
}
mutex_unlock(&inet_diag_table_mutex);
}
+static size_t inet_sk_attr_size(void)
+{
+ return nla_total_size(sizeof(struct tcp_info))
+ + nla_total_size(1) /* INET_DIAG_SHUTDOWN */
+ + nla_total_size(1) /* INET_DIAG_TOS */
+ + nla_total_size(1) /* INET_DIAG_TCLASS */
+ + nla_total_size(sizeof(struct inet_diag_meminfo))
+ + nla_total_size(sizeof(struct inet_diag_msg))
+ + nla_total_size(SK_MEMINFO_VARS * sizeof(u32))
+ + nla_total_size(TCP_CA_NAME_MAX)
+ + nla_total_size(sizeof(struct tcpvegas_info))
+ + 64;
+}
+
int inet_sk_diag_fill(struct sock *sk, struct inet_connection_sock *icsk,
struct sk_buff *skb, struct inet_diag_req_v2 *req,
struct user_namespace *user_ns,
if (err)
goto out;
- rep = nlmsg_new(sizeof(struct inet_diag_msg) +
- sizeof(struct inet_diag_meminfo) +
- sizeof(struct tcp_info) + 64, GFP_KERNEL);
+ rep = nlmsg_new(inet_sk_attr_size(), GFP_KERNEL);
if (!rep) {
err = -ENOMEM;
goto out;
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/wait.h>
+#include <linux/security.h>
#include <net/inet_connection_sock.h>
#include <net/inet_hashtables.h>
return inet_ehashfn(net, laddr, lport, faddr, fport);
}
+extern void gr_update_task_in_ip_table(const struct inet_sock *inet);
+
/*
* Allocate and initialize a new local port bind bucket.
* The bindhash mutex for snum's hash chain must be held here.
twrefcnt += inet_twsk_bind_unhash(tw, hinfo);
spin_unlock(&head->lock);
+ gr_update_task_in_ip_table(inet_sk(sk));
+
if (tw) {
inet_twsk_deschedule(tw, death_row);
while (twrefcnt) {
if (p) {
p->daddr = *daddr;
atomic_set(&p->refcnt, 1);
- atomic_set(&p->rid, 0);
+ atomic_set_unchecked(&p->rid, 0);
p->metrics[RTAX_LOCK-1] = INETPEER_METRICS_NEW;
p->rate_tokens = 0;
/* 60*HZ is arbitrary, but chosen enough high so that the first
return 0;
start = qp->rid;
- end = atomic_inc_return(&peer->rid);
+ end = atomic_inc_return_unchecked(&peer->rid);
qp->rid = end;
rc = qp->q.fragments && (end - start) > max;
static int __net_init ip4_frags_ns_ctl_register(struct net *net)
{
- struct ctl_table *table;
+ ctl_table_no_const *table = NULL;
struct ctl_table_header *hdr;
- table = ip4_frags_ns_ctl_table;
if (!net_eq(net, &init_net)) {
- table = kmemdup(table, sizeof(ip4_frags_ns_ctl_table), GFP_KERNEL);
+ table = kmemdup(ip4_frags_ns_ctl_table, sizeof(ip4_frags_ns_ctl_table), GFP_KERNEL);
if (table == NULL)
goto err_alloc;
/* Don't export sysctls to unprivileged users */
if (net->user_ns != &init_user_ns)
table[0].procname = NULL;
- }
+ hdr = register_net_sysctl(net, "net/ipv4", table);
+ } else
+ hdr = register_net_sysctl(net, "net/ipv4", ip4_frags_ns_ctl_table);
- hdr = register_net_sysctl(net, "net/ipv4", table);
if (hdr == NULL)
goto err_reg;
return 0;
err_reg:
- if (!net_eq(net, &init_net))
- kfree(table);
+ kfree(table);
err_alloc:
return -ENOMEM;
}
module_param(log_ecn_error, bool, 0644);
MODULE_PARM_DESC(log_ecn_error, "Log packets received with corrupted ECN");
-static struct rtnl_link_ops ipgre_link_ops __read_mostly;
+static struct rtnl_link_ops ipgre_link_ops;
static int ipgre_tunnel_init(struct net_device *dev);
static int ipgre_net_id __read_mostly;
[IFLA_GRE_ENCAP_DPORT] = { .type = NLA_U16 },
};
-static struct rtnl_link_ops ipgre_link_ops __read_mostly = {
+static struct rtnl_link_ops ipgre_link_ops = {
.kind = "gre",
.maxtype = IFLA_GRE_MAX,
.policy = ipgre_policy,
.fill_info = ipgre_fill_info,
};
-static struct rtnl_link_ops ipgre_tap_ops __read_mostly = {
+static struct rtnl_link_ops ipgre_tap_ops = {
.kind = "gretap",
.maxtype = IFLA_GRE_MAX,
.policy = ipgre_policy,
#include <linux/mroute.h>
#include <linux/netlink.h>
+#ifdef CONFIG_GRKERNSEC_BLACKHOLE
+extern int grsec_enable_blackhole;
+#endif
+
/*
* Process Router Attention IP option (RFC 2113)
*/
if (!raw) {
if (xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb)) {
IP_INC_STATS_BH(net, IPSTATS_MIB_INUNKNOWNPROTOS);
+#ifdef CONFIG_GRKERNSEC_BLACKHOLE
+ if (!grsec_enable_blackhole || (skb->dev->flags & IFF_LOOPBACK))
+#endif
icmp_send(skb, ICMP_DEST_UNREACH,
ICMP_PROT_UNREACH, 0);
}
len = min_t(unsigned int, len, opt->optlen);
if (put_user(len, optlen))
return -EFAULT;
- if (copy_to_user(optval, opt->__data, len))
+ if ((len > (sizeof(optbuf) - sizeof(struct ip_options))) ||
+ copy_to_user(optval, opt->__data, len))
return -EFAULT;
return 0;
}
if (sk->sk_type != SOCK_STREAM)
return -ENOPROTOOPT;
- msg.msg_control = (__force void *) optval;
+ msg.msg_control = (__force_kernel void *) optval;
msg.msg_controllen = len;
msg.msg_flags = flags;
#include <net/net_namespace.h>
#include <net/netns/generic.h>
-static struct rtnl_link_ops vti_link_ops __read_mostly;
+static struct rtnl_link_ops vti_link_ops;
static int vti_net_id __read_mostly;
static int vti_tunnel_init(struct net_device *dev);
[IFLA_VTI_REMOTE] = { .len = FIELD_SIZEOF(struct iphdr, daddr) },
};
-static struct rtnl_link_ops vti_link_ops __read_mostly = {
+static struct rtnl_link_ops vti_link_ops = {
.kind = "vti",
.maxtype = IFLA_VTI_MAX,
.policy = vti_policy,
mm_segment_t oldfs = get_fs();
set_fs(get_ds());
- res = devinet_ioctl(&init_net, cmd, (struct ifreq __user *) arg);
+ res = devinet_ioctl(&init_net, cmd, (struct ifreq __force_user *) arg);
set_fs(oldfs);
return res;
}
mm_segment_t oldfs = get_fs();
set_fs(get_ds());
- res = dev_ioctl(&init_net, cmd, (struct ifreq __user *) arg);
+ res = dev_ioctl(&init_net, cmd, (struct ifreq __force_user *) arg);
set_fs(oldfs);
return res;
}
mm_segment_t oldfs = get_fs();
set_fs(get_ds());
- res = ip_rt_ioctl(&init_net, cmd, (void __user *) arg);
+ res = ip_rt_ioctl(&init_net, cmd, (void __force_user *) arg);
set_fs(oldfs);
return res;
}
static int ipip_net_id __read_mostly;
static int ipip_tunnel_init(struct net_device *dev);
-static struct rtnl_link_ops ipip_link_ops __read_mostly;
+static struct rtnl_link_ops ipip_link_ops;
static int ipip_err(struct sk_buff *skb, u32 info)
{
[IFLA_IPTUN_ENCAP_DPORT] = { .type = NLA_U16 },
};
-static struct rtnl_link_ops ipip_link_ops __read_mostly = {
+static struct rtnl_link_ops ipip_link_ops = {
.kind = "ipip",
.maxtype = IFLA_IPTUN_MAX,
.policy = ipip_policy,
#endif
static int get_info(struct net *net, void __user *user,
- const int *len, int compat)
+ int len, int compat)
{
char name[XT_TABLE_MAXNAMELEN];
struct xt_table *t;
int ret;
- if (*len != sizeof(struct arpt_getinfo)) {
- duprintf("length %u != %Zu\n", *len,
+ if (len != sizeof(struct arpt_getinfo)) {
+ duprintf("length %u != %Zu\n", len,
sizeof(struct arpt_getinfo));
return -EINVAL;
}
info.size = private->size;
strcpy(info.name, name);
- if (copy_to_user(user, &info, *len) != 0)
+ if (copy_to_user(user, &info, len) != 0)
ret = -EFAULT;
else
ret = 0;
switch (cmd) {
case ARPT_SO_GET_INFO:
- ret = get_info(sock_net(sk), user, len, 1);
+ ret = get_info(sock_net(sk), user, *len, 1);
break;
case ARPT_SO_GET_ENTRIES:
ret = compat_get_entries(sock_net(sk), user, len);
switch (cmd) {
case ARPT_SO_GET_INFO:
- ret = get_info(sock_net(sk), user, len, 0);
+ ret = get_info(sock_net(sk), user, *len, 0);
break;
case ARPT_SO_GET_ENTRIES:
#endif
static int get_info(struct net *net, void __user *user,
- const int *len, int compat)
+ int len, int compat)
{
char name[XT_TABLE_MAXNAMELEN];
struct xt_table *t;
int ret;
- if (*len != sizeof(struct ipt_getinfo)) {
- duprintf("length %u != %zu\n", *len,
+ if (len != sizeof(struct ipt_getinfo)) {
+ duprintf("length %u != %zu\n", len,
sizeof(struct ipt_getinfo));
return -EINVAL;
}
info.size = private->size;
strcpy(info.name, name);
- if (copy_to_user(user, &info, *len) != 0)
+ if (copy_to_user(user, &info, len) != 0)
ret = -EFAULT;
else
ret = 0;
switch (cmd) {
case IPT_SO_GET_INFO:
- ret = get_info(sock_net(sk), user, len, 1);
+ ret = get_info(sock_net(sk), user, *len, 1);
break;
case IPT_SO_GET_ENTRIES:
ret = compat_get_entries(sock_net(sk), user, len);
switch (cmd) {
case IPT_SO_GET_INFO:
- ret = get_info(sock_net(sk), user, len, 0);
+ ret = get_info(sock_net(sk), user, *len, 0);
break;
case IPT_SO_GET_ENTRIES:
spin_lock_init(&cn->lock);
#ifdef CONFIG_PROC_FS
- cn->procdir = proc_mkdir("ipt_CLUSTERIP", net->proc_net);
+ cn->procdir = proc_mkdir_restrict("ipt_CLUSTERIP", net->proc_net);
if (!cn->procdir) {
pr_err("Unable to proc dir entry\n");
return -ENOMEM;
};
static struct ping_table ping_table;
-struct pingv6_ops pingv6_ops;
+struct pingv6_ops *pingv6_ops;
EXPORT_SYMBOL_GPL(pingv6_ops);
static u16 ping_port_rover;
return -ENODEV;
}
}
- has_addr = pingv6_ops.ipv6_chk_addr(net, &addr->sin6_addr, dev,
+ has_addr = pingv6_ops->ipv6_chk_addr(net, &addr->sin6_addr, dev,
scoped);
rcu_read_unlock();
}
#if IS_ENABLED(CONFIG_IPV6)
} else if (skb->protocol == htons(ETH_P_IPV6)) {
- harderr = pingv6_ops.icmpv6_err_convert(type, code, &err);
+ harderr = pingv6_ops->icmpv6_err_convert(type, code, &err);
#endif
}
info, (u8 *)icmph);
#if IS_ENABLED(CONFIG_IPV6)
} else if (family == AF_INET6) {
- pingv6_ops.ipv6_icmp_error(sk, skb, err, 0,
+ pingv6_ops->ipv6_icmp_error(sk, skb, err, 0,
info, (u8 *)icmph);
#endif
}
}
if (inet6_sk(sk)->rxopt.all)
- pingv6_ops.ip6_datagram_recv_common_ctl(sk, msg, skb);
+ pingv6_ops->ip6_datagram_recv_common_ctl(sk, msg, skb);
if (skb->protocol == htons(ETH_P_IPV6) &&
inet6_sk(sk)->rxopt.all)
- pingv6_ops.ip6_datagram_recv_specific_ctl(sk, msg, skb);
+ pingv6_ops->ip6_datagram_recv_specific_ctl(sk, msg, skb);
else if (skb->protocol == htons(ETH_P_IP) && isk->cmsg_flags)
ip_cmsg_recv(msg, skb);
#endif
from_kuid_munged(seq_user_ns(f), sock_i_uid(sp)),
0, sock_i_ino(sp),
atomic_read(&sp->sk_refcnt), sp,
- atomic_read(&sp->sk_drops));
+ atomic_read_unchecked(&sp->sk_drops));
}
static int ping_v4_seq_show(struct seq_file *seq, void *v)
int raw_rcv(struct sock *sk, struct sk_buff *skb)
{
if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb)) {
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
kfree_skb(skb);
return NET_RX_DROP;
}
static int raw_seticmpfilter(struct sock *sk, char __user *optval, int optlen)
{
+ struct icmp_filter filter;
+
if (optlen > sizeof(struct icmp_filter))
optlen = sizeof(struct icmp_filter);
- if (copy_from_user(&raw_sk(sk)->filter, optval, optlen))
+ if (copy_from_user(&filter, optval, optlen))
return -EFAULT;
+ raw_sk(sk)->filter = filter;
return 0;
}
static int raw_geticmpfilter(struct sock *sk, char __user *optval, int __user *optlen)
{
int len, ret = -EFAULT;
+ struct icmp_filter filter;
if (get_user(len, optlen))
goto out;
if (len > sizeof(struct icmp_filter))
len = sizeof(struct icmp_filter);
ret = -EFAULT;
- if (put_user(len, optlen) ||
- copy_to_user(optval, &raw_sk(sk)->filter, len))
+ filter = raw_sk(sk)->filter;
+ if (put_user(len, optlen) || len > sizeof filter || copy_to_user(optval, &filter, len))
goto out;
ret = 0;
out: return ret;
0, 0L, 0,
from_kuid_munged(seq_user_ns(seq), sock_i_uid(sp)),
0, sock_i_ino(sp),
- atomic_read(&sp->sk_refcnt), sp, atomic_read(&sp->sk_drops));
+ atomic_read(&sp->sk_refcnt), sp, atomic_read_unchecked(&sp->sk_drops));
}
static int raw_seq_show(struct seq_file *seq, void *v)
static int rt_cache_seq_open(struct inode *inode, struct file *file)
{
- return seq_open(file, &rt_cache_seq_ops);
+ return seq_open_restrict(file, &rt_cache_seq_ops);
}
static const struct file_operations rt_cache_seq_fops = {
static int rt_cpu_seq_open(struct inode *inode, struct file *file)
{
- return seq_open(file, &rt_cpu_seq_ops);
+ return seq_open_restrict(file, &rt_cpu_seq_ops);
}
static const struct file_operations rt_cpu_seq_fops = {
static int rt_acct_proc_open(struct inode *inode, struct file *file)
{
- return single_open(file, rt_acct_proc_show, NULL);
+ return single_open_restrict(file, rt_acct_proc_show, NULL);
}
static const struct file_operations rt_acct_proc_fops = {
#define IP_IDENTS_SZ 2048u
struct ip_ident_bucket {
- atomic_t id;
+ atomic_unchecked_t id;
u32 stamp32;
};
-static struct ip_ident_bucket *ip_idents __read_mostly;
+static struct ip_ident_bucket ip_idents[IP_IDENTS_SZ] __read_mostly;
/* In order to protect privacy, we add a perturbation to identifiers
* if one generator is seldom used. This makes hard for an attacker
if (old != now && cmpxchg(&bucket->stamp32, old, now) == old)
delta = prandom_u32_max(now - old);
- return atomic_add_return(segs + delta, &bucket->id) - segs;
+ return atomic_add_return_unchecked(segs + delta, &bucket->id) - segs;
}
EXPORT_SYMBOL(ip_idents_reserve);
.maxlen = sizeof(int),
.mode = 0200,
.proc_handler = ipv4_sysctl_rtcache_flush,
+ .extra1 = &init_net,
},
{ },
};
static __net_init int sysctl_route_net_init(struct net *net)
{
- struct ctl_table *tbl;
+ ctl_table_no_const *tbl = NULL;
- tbl = ipv4_route_flush_table;
if (!net_eq(net, &init_net)) {
- tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL);
+ tbl = kmemdup(ipv4_route_flush_table, sizeof(ipv4_route_flush_table), GFP_KERNEL);
if (tbl == NULL)
goto err_dup;
/* Don't export sysctls to unprivileged users */
if (net->user_ns != &init_user_ns)
tbl[0].procname = NULL;
- }
- tbl[0].extra1 = net;
+ tbl[0].extra1 = net;
+ net->ipv4.route_hdr = register_net_sysctl(net, "net/ipv4/route", tbl);
+ } else
+ net->ipv4.route_hdr = register_net_sysctl(net, "net/ipv4/route", ipv4_route_flush_table);
- net->ipv4.route_hdr = register_net_sysctl(net, "net/ipv4/route", tbl);
if (net->ipv4.route_hdr == NULL)
goto err_reg;
return 0;
err_reg:
- if (tbl != ipv4_route_flush_table)
- kfree(tbl);
+ kfree(tbl);
err_dup:
return -ENOMEM;
}
static __net_init int rt_genid_init(struct net *net)
{
- atomic_set(&net->ipv4.rt_genid, 0);
- atomic_set(&net->fnhe_genid, 0);
+ atomic_set_unchecked(&net->ipv4.rt_genid, 0);
+ atomic_set_unchecked(&net->fnhe_genid, 0);
get_random_bytes(&net->ipv4.dev_addr_genid,
sizeof(net->ipv4.dev_addr_genid));
return 0;
{
int rc = 0;
- ip_idents = kmalloc(IP_IDENTS_SZ * sizeof(*ip_idents), GFP_KERNEL);
- if (!ip_idents)
- panic("IP: failed to allocate ip_idents\n");
-
- prandom_bytes(ip_idents, IP_IDENTS_SZ * sizeof(*ip_idents));
+ prandom_bytes(ip_idents, sizeof(ip_idents));
#ifdef CONFIG_IP_ROUTE_CLASSID
ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
container_of(table->data, struct net, ipv4.ip_local_ports.range);
int ret;
int range[2];
- struct ctl_table tmp = {
+ ctl_table_no_const tmp = {
.data = &range,
.maxlen = sizeof(range),
.mode = table->mode,
int ret;
gid_t urange[2];
kgid_t low, high;
- struct ctl_table tmp = {
+ ctl_table_no_const tmp = {
.data = &urange,
.maxlen = sizeof(urange),
.mode = table->mode,
void __user *buffer, size_t *lenp, loff_t *ppos)
{
char val[TCP_CA_NAME_MAX];
- struct ctl_table tbl = {
+ ctl_table_no_const tbl = {
.data = val,
.maxlen = TCP_CA_NAME_MAX,
};
void __user *buffer, size_t *lenp,
loff_t *ppos)
{
- struct ctl_table tbl = { .maxlen = TCP_CA_BUF_MAX, };
+ ctl_table_no_const tbl = { .maxlen = TCP_CA_BUF_MAX, };
int ret;
tbl.data = kmalloc(tbl.maxlen, GFP_USER);
void __user *buffer, size_t *lenp,
loff_t *ppos)
{
- struct ctl_table tbl = { .maxlen = TCP_CA_BUF_MAX };
+ ctl_table_no_const tbl = { .maxlen = TCP_CA_BUF_MAX };
int ret;
tbl.data = kmalloc(tbl.maxlen, GFP_USER);
void __user *buffer, size_t *lenp,
loff_t *ppos)
{
- struct ctl_table tbl = { .maxlen = (TCP_FASTOPEN_KEY_LENGTH * 2 + 10) };
+ ctl_table_no_const tbl = { .maxlen = (TCP_FASTOPEN_KEY_LENGTH * 2 + 10) };
struct tcp_fastopen_context *ctxt;
int ret;
u32 user_key[4]; /* 16 bytes, matching TCP_FASTOPEN_KEY_LENGTH */
static __net_init int ipv4_sysctl_init_net(struct net *net)
{
- struct ctl_table *table;
+ ctl_table_no_const *table = NULL;
- table = ipv4_net_table;
if (!net_eq(net, &init_net)) {
int i;
- table = kmemdup(table, sizeof(ipv4_net_table), GFP_KERNEL);
+ table = kmemdup(ipv4_net_table, sizeof(ipv4_net_table), GFP_KERNEL);
if (table == NULL)
goto err_alloc;
table[i].data += (void *)net - (void *)&init_net;
}
- net->ipv4.ipv4_hdr = register_net_sysctl(net, "net/ipv4", table);
+ if (!net_eq(net, &init_net))
+ net->ipv4.ipv4_hdr = register_net_sysctl(net, "net/ipv4", table);
+ else
+ net->ipv4.ipv4_hdr = register_net_sysctl(net, "net/ipv4", ipv4_net_table);
if (net->ipv4.ipv4_hdr == NULL)
goto err_reg;
* without any lock. We want to make sure compiler wont store
* intermediate values in this location.
*/
- ACCESS_ONCE(sk->sk_pacing_rate) = min_t(u64, rate,
+ ACCESS_ONCE_RW(sk->sk_pacing_rate) = min_t(u64, rate,
sk->sk_max_pacing_rate);
}
* simplifies code)
*/
static void
-tcp_collapse(struct sock *sk, struct sk_buff_head *list,
+__intentional_overflow(5,6) tcp_collapse(struct sock *sk, struct sk_buff_head *list,
struct sk_buff *head, struct sk_buff *tail,
u32 start, u32 end)
{
tcp_paws_reject(&tp->rx_opt, 0))
goto discard_and_undo;
+#ifndef CONFIG_GRKERNSEC_NO_SIMULT_CONNECT
if (th->syn) {
/* We see SYN without ACK. It is attempt of
* simultaneous connect with crossed SYNs.
goto discard;
#endif
}
+#endif
/* "fifth, if neither of the SYN or RST bits is set then
* drop the segment and return."
*/
goto discard;
if (th->syn) {
- if (th->fin)
+ if (th->fin || th->urg || th->psh)
goto discard;
if (icsk->icsk_af_ops->conn_request(sk, skb) < 0)
return 1;
int sysctl_tcp_low_latency __read_mostly;
EXPORT_SYMBOL(sysctl_tcp_low_latency);
+#ifdef CONFIG_GRKERNSEC_BLACKHOLE
+extern int grsec_enable_blackhole;
+#endif
+
#ifdef CONFIG_TCP_MD5SIG
static int tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
__be32 daddr, __be32 saddr, const struct tcphdr *th);
return 0;
reset:
+#ifdef CONFIG_GRKERNSEC_BLACKHOLE
+ if (!grsec_enable_blackhole)
+#endif
tcp_v4_send_reset(rsk, skb);
discard:
kfree_skb(skb);
TCP_SKB_CB(skb)->sacked = 0;
sk = __inet_lookup_skb(&tcp_hashinfo, skb, th->source, th->dest);
- if (!sk)
+ if (!sk) {
+#ifdef CONFIG_GRKERNSEC_BLACKHOLE
+ ret = 1;
+#endif
goto no_tcp_socket;
-
+ }
process:
- if (sk->sk_state == TCP_TIME_WAIT)
+ if (sk->sk_state == TCP_TIME_WAIT) {
+#ifdef CONFIG_GRKERNSEC_BLACKHOLE
+ ret = 2;
+#endif
goto do_time_wait;
+ }
if (unlikely(iph->ttl < inet_sk(sk)->min_ttl)) {
NET_INC_STATS_BH(net, LINUX_MIB_TCPMINTTLDROP);
bad_packet:
TCP_INC_STATS_BH(net, TCP_MIB_INERRS);
} else {
+#ifdef CONFIG_GRKERNSEC_BLACKHOLE
+ if (!grsec_enable_blackhole || (ret == 1 &&
+ (skb->dev->flags & IFF_LOOPBACK)))
+#endif
tcp_v4_send_reset(NULL, skb);
}
#include <net/inet_common.h>
#include <net/xfrm.h>
+#ifdef CONFIG_GRKERNSEC_BLACKHOLE
+extern int grsec_enable_blackhole;
+#endif
+
int sysctl_tcp_syncookies __read_mostly = 1;
EXPORT_SYMBOL(sysctl_tcp_syncookies);
* avoid becoming vulnerable to outside attack aiming at
* resetting legit local connections.
*/
- req->rsk_ops->send_reset(sk, skb);
+#ifdef CONFIG_GRKERNSEC_BLACKHOLE
+ if (!grsec_enable_blackhole)
+#endif
+ req->rsk_ops->send_reset(sk, skb);
} else if (fastopen) { /* received a valid RST pkt */
reqsk_fastopen_remove(sk, req, true);
tcp_reset(sk);
if (cnt + width >= len)
break;
- if (copy_to_user(buf + cnt, tbuf, width))
+ if (width > sizeof tbuf || copy_to_user(buf + cnt, tbuf, width))
return -EFAULT;
cnt += width;
}
#include <linux/gfp.h>
#include <net/tcp.h>
+#ifdef CONFIG_GRKERNSEC_BLACKHOLE
+extern int grsec_lastack_retries;
+#endif
+
int sysctl_tcp_syn_retries __read_mostly = TCP_SYN_RETRIES;
int sysctl_tcp_synack_retries __read_mostly = TCP_SYNACK_RETRIES;
int sysctl_tcp_keepalive_time __read_mostly = TCP_KEEPALIVE_TIME;
}
}
+#ifdef CONFIG_GRKERNSEC_BLACKHOLE
+ if ((sk->sk_state == TCP_LAST_ACK) &&
+ (grsec_lastack_retries > 0) &&
+ (grsec_lastack_retries < retry_until))
+ retry_until = grsec_lastack_retries;
+#endif
+
if (retransmits_timed_out(sk, retry_until,
syn_set ? 0 : icsk->icsk_user_timeout, syn_set)) {
/* Has it gone just too far? */
#include <linux/types.h>
#include <linux/fcntl.h>
#include <linux/module.h>
+#include <linux/security.h>
#include <linux/socket.h>
#include <linux/sockios.h>
#include <linux/igmp.h>
#include <net/busy_poll.h>
#include "udp_impl.h"
+#ifdef CONFIG_GRKERNSEC_BLACKHOLE
+extern int grsec_enable_blackhole;
+#endif
+
struct udp_table udp_table __read_mostly;
EXPORT_SYMBOL(udp_table);
return true;
}
+extern int gr_search_udp_recvmsg(struct sock *sk, const struct sk_buff *skb);
+extern int gr_search_udp_sendmsg(struct sock *sk, struct sockaddr_in *addr);
+
/*
* This routine is called by the ICMP module when it gets some
* sort of error condition. If err < 0 then the socket should
dport = usin->sin_port;
if (dport == 0)
return -EINVAL;
+
+ err = gr_search_udp_sendmsg(sk, usin);
+ if (err)
+ return err;
} else {
if (sk->sk_state != TCP_ESTABLISHED)
return -EDESTADDRREQ;
+
+ err = gr_search_udp_sendmsg(sk, NULL);
+ if (err)
+ return err;
+
daddr = inet->inet_daddr;
dport = inet->inet_dport;
/* Open fast path for connected socket.
IS_UDPLITE(sk));
UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS,
IS_UDPLITE(sk));
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
__skb_unlink(skb, rcvq);
__skb_queue_tail(&list_kill, skb);
}
if (!skb)
goto out;
+ err = gr_search_udp_recvmsg(sk, skb);
+ if (err)
+ goto out_free;
+
ulen = skb->len - sizeof(struct udphdr);
copied = len;
if (copied > ulen)
if (unlikely(err)) {
trace_kfree_skb(skb, udp_recvmsg);
if (!peeked) {
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
UDP_INC_STATS_USER(sock_net(sk),
UDP_MIB_INERRORS, is_udplite);
}
UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite);
drop:
UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
kfree_skb(skb);
return -1;
}
skb1 = (i == final) ? skb : skb_clone(skb, GFP_ATOMIC);
if (!skb1) {
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_RCVBUFERRORS,
IS_UDPLITE(sk));
UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS,
goto csum_error;
UDP_INC_STATS_BH(net, UDP_MIB_NOPORTS, proto == IPPROTO_UDPLITE);
+#ifdef CONFIG_GRKERNSEC_BLACKHOLE
+ if (!grsec_enable_blackhole || (skb->dev->flags & IFF_LOOPBACK))
+#endif
icmp_send(skb, ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, 0);
/*
from_kuid_munged(seq_user_ns(f), sock_i_uid(sp)),
0, sock_i_ino(sp),
atomic_read(&sp->sk_refcnt), sp,
- atomic_read(&sp->sk_drops));
+ atomic_read_unchecked(&sp->sk_drops));
}
int udp4_seq_show(struct seq_file *seq, void *v)
fl4->flowi4_tos = iph->tos;
}
-static inline int xfrm4_garbage_collect(struct dst_ops *ops)
+static int xfrm4_garbage_collect(struct dst_ops *ops)
{
struct net *net = container_of(ops, struct net, xfrm.xfrm4_dst_ops);
- xfrm4_policy_afinfo.garbage_collect(net);
+ xfrm_garbage_collect_deferred(net);
return (dst_entries_get_slow(ops) > ops->gc_thresh * 2);
}
static int __net_init xfrm4_net_init(struct net *net)
{
- struct ctl_table *table;
+ ctl_table_no_const *table = NULL;
struct ctl_table_header *hdr;
- table = xfrm4_policy_table;
if (!net_eq(net, &init_net)) {
- table = kmemdup(table, sizeof(xfrm4_policy_table), GFP_KERNEL);
+ table = kmemdup(xfrm4_policy_table, sizeof(xfrm4_policy_table), GFP_KERNEL);
if (!table)
goto err_alloc;
table[0].data = &net->xfrm.xfrm4_dst_ops.gc_thresh;
- }
-
- hdr = register_net_sysctl(net, "net/ipv4", table);
+ hdr = register_net_sysctl(net, "net/ipv4", table);
+ } else
+ hdr = register_net_sysctl(net, "net/ipv4", xfrm4_policy_table);
if (!hdr)
goto err_reg;
return 0;
err_reg:
- if (!net_eq(net, &init_net))
- kfree(table);
+ kfree(table);
err_alloc:
return -ENOMEM;
}
.hop_limit = IPV6_DEFAULT_HOPLIMIT,
.mtu6 = IPV6_MIN_MTU,
.accept_ra = 1,
- .accept_redirects = 1,
+ .accept_redirects = 0,
.autoconf = 1,
.force_mld_version = 0,
.mldv1_unsolicited_report_interval = 10 * HZ,
.hop_limit = IPV6_DEFAULT_HOPLIMIT,
.mtu6 = IPV6_MIN_MTU,
.accept_ra = 1,
- .accept_redirects = 1,
+ .accept_redirects = 0,
.autoconf = 1,
.force_mld_version = 0,
.mldv1_unsolicited_report_interval = 10 * HZ,
idx = 0;
head = &net->dev_index_head[h];
rcu_read_lock();
- cb->seq = atomic_read(&net->ipv6.dev_addr_genid) ^
+ cb->seq = atomic_read_unchecked(&net->ipv6.dev_addr_genid) ^
net->dev_base_seq;
hlist_for_each_entry_rcu(dev, head, index_hlist) {
if (idx < s_idx)
p.iph.ihl = 5;
p.iph.protocol = IPPROTO_IPV6;
p.iph.ttl = 64;
- ifr.ifr_ifru.ifru_data = (__force void __user *)&p;
+ ifr.ifr_ifru.ifru_data = (void __force_user *)&p;
if (ops->ndo_do_ioctl) {
mm_segment_t oldfs = get_fs();
.release = seq_release_net,
};
+extern void register_ipv6_seq_ops_addr(struct seq_operations *addr);
+extern void unregister_ipv6_seq_ops_addr(void);
+
static int __net_init if6_proc_net_init(struct net *net)
{
- if (!proc_create("if_inet6", S_IRUGO, net->proc_net, &if6_fops))
+ register_ipv6_seq_ops_addr(&if6_seq_ops);
+ if (!proc_create("if_inet6", S_IRUGO, net->proc_net, &if6_fops)) {
+ unregister_ipv6_seq_ops_addr();
return -ENOMEM;
+ }
return 0;
}
static void __net_exit if6_proc_net_exit(struct net *net)
{
remove_proc_entry("if_inet6", net->proc_net);
+ unregister_ipv6_seq_ops_addr();
}
static struct pernet_operations if6_proc_net_ops = {
s_ip_idx = ip_idx = cb->args[2];
rcu_read_lock();
- cb->seq = atomic_read(&net->ipv6.dev_addr_genid) ^ net->dev_base_seq;
+ cb->seq = atomic_read_unchecked(&net->ipv6.dev_addr_genid) ^ net->dev_base_seq;
for (h = s_h; h < NETDEV_HASHENTRIES; h++, s_idx = 0) {
idx = 0;
head = &net->dev_index_head[h];
rt_genid_bump_ipv6(net);
break;
}
- atomic_inc(&net->ipv6.dev_addr_genid);
+ atomic_inc_unchecked(&net->ipv6.dev_addr_genid);
}
static void ipv6_ifa_notify(int event, struct inet6_ifaddr *ifp)
int *valp = ctl->data;
int val = *valp;
loff_t pos = *ppos;
- struct ctl_table lctl;
+ ctl_table_no_const lctl;
int ret;
/*
int *valp = ctl->data;
int val = *valp;
loff_t pos = *ppos;
- struct ctl_table lctl;
+ ctl_table_no_const lctl;
int ret;
/*
net->ipv6.sysctl.icmpv6_time = 1*HZ;
net->ipv6.sysctl.flowlabel_consistency = 1;
net->ipv6.sysctl.auto_flowlabels = 0;
- atomic_set(&net->ipv6.fib6_sernum, 1);
+ atomic_set_unchecked(&net->ipv6.fib6_sernum, 1);
err = ipv6_init_mibs(net);
if (err)
0,
sock_i_ino(sp),
atomic_read(&sp->sk_refcnt), sp,
- atomic_read(&sp->sk_drops));
+ atomic_read_unchecked(&sp->sk_drops));
}
struct ctl_table * __net_init ipv6_icmp_sysctl_init(struct net *net)
{
- struct ctl_table *table;
+ ctl_table_no_const *table;
table = kmemdup(ipv6_icmp_table_template,
sizeof(ipv6_icmp_table_template),
int new, old;
do {
- old = atomic_read(&net->ipv6.fib6_sernum);
+ old = atomic_read_unchecked(&net->ipv6.fib6_sernum);
new = old < INT_MAX ? old + 1 : 1;
- } while (atomic_cmpxchg(&net->ipv6.fib6_sernum,
+ } while (atomic_cmpxchg_unchecked(&net->ipv6.fib6_sernum,
old, new) != old);
return new;
}
struct net_device *fb_tunnel_dev;
};
-static struct rtnl_link_ops ip6gre_link_ops __read_mostly;
-static struct rtnl_link_ops ip6gre_tap_ops __read_mostly;
+static struct rtnl_link_ops ip6gre_link_ops;
+static struct rtnl_link_ops ip6gre_tap_ops;
static int ip6gre_tunnel_init(struct net_device *dev);
static void ip6gre_tunnel_setup(struct net_device *dev);
static void ip6gre_tunnel_link(struct ip6gre_net *ign, struct ip6_tnl *t);
}
-static struct inet6_protocol ip6gre_protocol __read_mostly = {
+static struct inet6_protocol ip6gre_protocol = {
.handler = ip6gre_rcv,
.err_handler = ip6gre_err,
.flags = INET6_PROTO_NOPOLICY|INET6_PROTO_FINAL,
[IFLA_GRE_FLAGS] = { .type = NLA_U32 },
};
-static struct rtnl_link_ops ip6gre_link_ops __read_mostly = {
+static struct rtnl_link_ops ip6gre_link_ops = {
.kind = "ip6gre",
.maxtype = IFLA_GRE_MAX,
.policy = ip6gre_policy,
.fill_info = ip6gre_fill_info,
};
-static struct rtnl_link_ops ip6gre_tap_ops __read_mostly = {
+static struct rtnl_link_ops ip6gre_tap_ops = {
.kind = "ip6gretap",
.maxtype = IFLA_GRE_MAX,
.policy = ip6gre_policy,
static int ip6_tnl_dev_init(struct net_device *dev);
static void ip6_tnl_dev_setup(struct net_device *dev);
-static struct rtnl_link_ops ip6_link_ops __read_mostly;
+static struct rtnl_link_ops ip6_link_ops;
static int ip6_tnl_net_id __read_mostly;
struct ip6_tnl_net {
[IFLA_IPTUN_PROTO] = { .type = NLA_U8 },
};
-static struct rtnl_link_ops ip6_link_ops __read_mostly = {
+static struct rtnl_link_ops ip6_link_ops = {
.kind = "ip6tnl",
.maxtype = IFLA_IPTUN_MAX,
.policy = ip6_tnl_policy,
static int vti6_dev_init(struct net_device *dev);
static void vti6_dev_setup(struct net_device *dev);
-static struct rtnl_link_ops vti6_link_ops __read_mostly;
+static struct rtnl_link_ops vti6_link_ops;
static int vti6_net_id __read_mostly;
struct vti6_net {
[IFLA_VTI_OKEY] = { .type = NLA_U32 },
};
-static struct rtnl_link_ops vti6_link_ops __read_mostly = {
+static struct rtnl_link_ops vti6_link_ops = {
.kind = "vti6",
.maxtype = IFLA_VTI_MAX,
.policy = vti6_policy,
if (sk->sk_type != SOCK_STREAM)
return -ENOPROTOOPT;
- msg.msg_control = optval;
+ msg.msg_control = (void __force_kernel *)optval;
msg.msg_controllen = len;
msg.msg_flags = flags;
#endif
static int get_info(struct net *net, void __user *user,
- const int *len, int compat)
+ int len, int compat)
{
char name[XT_TABLE_MAXNAMELEN];
struct xt_table *t;
int ret;
- if (*len != sizeof(struct ip6t_getinfo)) {
- duprintf("length %u != %zu\n", *len,
+ if (len != sizeof(struct ip6t_getinfo)) {
+ duprintf("length %u != %zu\n", len,
sizeof(struct ip6t_getinfo));
return -EINVAL;
}
info.size = private->size;
strcpy(info.name, name);
- if (copy_to_user(user, &info, *len) != 0)
+ if (copy_to_user(user, &info, len) != 0)
ret = -EFAULT;
else
ret = 0;
switch (cmd) {
case IP6T_SO_GET_INFO:
- ret = get_info(sock_net(sk), user, len, 1);
+ ret = get_info(sock_net(sk), user, *len, 1);
break;
case IP6T_SO_GET_ENTRIES:
ret = compat_get_entries(sock_net(sk), user, len);
switch (cmd) {
case IP6T_SO_GET_INFO:
- ret = get_info(sock_net(sk), user, len, 0);
+ ret = get_info(sock_net(sk), user, *len, 0);
break;
case IP6T_SO_GET_ENTRIES:
static int nf_ct_frag6_sysctl_register(struct net *net)
{
- struct ctl_table *table;
+ ctl_table_no_const *table = NULL;
struct ctl_table_header *hdr;
- table = nf_ct_frag6_sysctl_table;
if (!net_eq(net, &init_net)) {
- table = kmemdup(table, sizeof(nf_ct_frag6_sysctl_table),
+ table = kmemdup(nf_ct_frag6_sysctl_table, sizeof(nf_ct_frag6_sysctl_table),
GFP_KERNEL);
if (table == NULL)
goto err_alloc;
table[2].data = &net->nf_frag.frags.high_thresh;
table[2].extra1 = &net->nf_frag.frags.low_thresh;
table[2].extra2 = &init_net.nf_frag.frags.high_thresh;
- }
-
- hdr = register_net_sysctl(net, "net/netfilter", table);
+ hdr = register_net_sysctl(net, "net/netfilter", table);
+ } else
+ hdr = register_net_sysctl(net, "net/netfilter", nf_ct_frag6_sysctl_table);
if (hdr == NULL)
goto err_reg;
return 0;
err_reg:
- if (!net_eq(net, &init_net))
- kfree(table);
+ kfree(table);
err_alloc:
return -ENOMEM;
}
};
#endif
+static struct pingv6_ops real_pingv6_ops = {
+ .ipv6_recv_error = ipv6_recv_error,
+ .ip6_datagram_recv_common_ctl = ip6_datagram_recv_common_ctl,
+ .ip6_datagram_recv_specific_ctl = ip6_datagram_recv_specific_ctl,
+ .icmpv6_err_convert = icmpv6_err_convert,
+ .ipv6_icmp_error = ipv6_icmp_error,
+ .ipv6_chk_addr = ipv6_chk_addr,
+};
+
+static struct pingv6_ops dummy_pingv6_ops = {
+ .ipv6_recv_error = dummy_ipv6_recv_error,
+ .ip6_datagram_recv_common_ctl = dummy_ip6_datagram_recv_ctl,
+ .ip6_datagram_recv_specific_ctl = dummy_ip6_datagram_recv_ctl,
+ .icmpv6_err_convert = dummy_icmpv6_err_convert,
+ .ipv6_icmp_error = dummy_ipv6_icmp_error,
+ .ipv6_chk_addr = dummy_ipv6_chk_addr,
+};
+
int __init pingv6_init(void)
{
#ifdef CONFIG_PROC_FS
if (ret)
return ret;
#endif
- pingv6_ops.ipv6_recv_error = ipv6_recv_error;
- pingv6_ops.ip6_datagram_recv_common_ctl = ip6_datagram_recv_common_ctl;
- pingv6_ops.ip6_datagram_recv_specific_ctl =
- ip6_datagram_recv_specific_ctl;
- pingv6_ops.icmpv6_err_convert = icmpv6_err_convert;
- pingv6_ops.ipv6_icmp_error = ipv6_icmp_error;
- pingv6_ops.ipv6_chk_addr = ipv6_chk_addr;
+ pingv6_ops = &real_pingv6_ops;
return inet6_register_protosw(&pingv6_protosw);
}
*/
void pingv6_exit(void)
{
- pingv6_ops.ipv6_recv_error = dummy_ipv6_recv_error;
- pingv6_ops.ip6_datagram_recv_common_ctl = dummy_ip6_datagram_recv_ctl;
- pingv6_ops.ip6_datagram_recv_specific_ctl = dummy_ip6_datagram_recv_ctl;
- pingv6_ops.icmpv6_err_convert = dummy_icmpv6_err_convert;
- pingv6_ops.ipv6_icmp_error = dummy_ipv6_icmp_error;
- pingv6_ops.ipv6_chk_addr = dummy_ipv6_chk_addr;
#ifdef CONFIG_PROC_FS
unregister_pernet_subsys(&ping_v6_net_ops);
#endif
+ pingv6_ops = &dummy_pingv6_ops;
inet6_unregister_protosw(&pingv6_protosw);
}
if (!proc_create("snmp6", S_IRUGO, net->proc_net, &snmp6_seq_fops))
goto proc_snmp6_fail;
- net->mib.proc_net_devsnmp6 = proc_mkdir("dev_snmp6", net->proc_net);
+ net->mib.proc_net_devsnmp6 = proc_mkdir_restrict("dev_snmp6", net->proc_net);
if (!net->mib.proc_net_devsnmp6)
goto proc_dev_snmp6_fail;
return 0;
{
if ((raw6_sk(sk)->checksum || rcu_access_pointer(sk->sk_filter)) &&
skb_checksum_complete(skb)) {
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
kfree_skb(skb);
return NET_RX_DROP;
}
struct raw6_sock *rp = raw6_sk(sk);
if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb)) {
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
kfree_skb(skb);
return NET_RX_DROP;
}
if (inet->hdrincl) {
if (skb_checksum_complete(skb)) {
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
kfree_skb(skb);
return NET_RX_DROP;
}
return err;
}
-static int rawv6_send_hdrinc(struct sock *sk, void *from, int length,
+static int rawv6_send_hdrinc(struct sock *sk, void *from, unsigned int length,
struct flowi6 *fl6, struct dst_entry **dstp,
unsigned int flags)
{
static int rawv6_seticmpfilter(struct sock *sk, int level, int optname,
char __user *optval, int optlen)
{
+ struct icmp6_filter filter;
+
switch (optname) {
case ICMPV6_FILTER:
if (optlen > sizeof(struct icmp6_filter))
optlen = sizeof(struct icmp6_filter);
- if (copy_from_user(&raw6_sk(sk)->filter, optval, optlen))
+ if (copy_from_user(&filter, optval, optlen))
return -EFAULT;
+ raw6_sk(sk)->filter = filter;
return 0;
default:
return -ENOPROTOOPT;
char __user *optval, int __user *optlen)
{
int len;
+ struct icmp6_filter filter;
switch (optname) {
case ICMPV6_FILTER:
len = sizeof(struct icmp6_filter);
if (put_user(len, optlen))
return -EFAULT;
- if (copy_to_user(optval, &raw6_sk(sk)->filter, len))
+ filter = raw6_sk(sk)->filter;
+ if (len > sizeof filter || copy_to_user(optval, &filter, len))
return -EFAULT;
return 0;
default:
static int __net_init ip6_frags_ns_sysctl_register(struct net *net)
{
- struct ctl_table *table;
+ ctl_table_no_const *table = NULL;
struct ctl_table_header *hdr;
- table = ip6_frags_ns_ctl_table;
if (!net_eq(net, &init_net)) {
- table = kmemdup(table, sizeof(ip6_frags_ns_ctl_table), GFP_KERNEL);
+ table = kmemdup(ip6_frags_ns_ctl_table, sizeof(ip6_frags_ns_ctl_table), GFP_KERNEL);
if (table == NULL)
goto err_alloc;
/* Don't export sysctls to unprivileged users */
if (net->user_ns != &init_user_ns)
table[0].procname = NULL;
- }
+ hdr = register_net_sysctl(net, "net/ipv6", table);
+ } else
+ hdr = register_net_sysctl(net, "net/ipv6", ip6_frags_ns_ctl_table);
- hdr = register_net_sysctl(net, "net/ipv6", table);
if (hdr == NULL)
goto err_reg;
return 0;
err_reg:
- if (!net_eq(net, &init_net))
- kfree(table);
+ kfree(table);
err_alloc:
return -ENOMEM;
}
struct ctl_table * __net_init ipv6_route_sysctl_init(struct net *net)
{
- struct ctl_table *table;
+ ctl_table_no_const *table;
table = kmemdup(ipv6_route_table_template,
sizeof(ipv6_route_table_template),
static void ipip6_dev_free(struct net_device *dev);
static bool check_6rd(struct ip_tunnel *tunnel, const struct in6_addr *v6dst,
__be32 *v4dst);
-static struct rtnl_link_ops sit_link_ops __read_mostly;
+static struct rtnl_link_ops sit_link_ops;
static int sit_net_id __read_mostly;
struct sit_net {
unregister_netdevice_queue(dev, head);
}
-static struct rtnl_link_ops sit_link_ops __read_mostly = {
+static struct rtnl_link_ops sit_link_ops = {
.kind = "sit",
.maxtype = IFLA_IPTUN_MAX,
.policy = ipip6_policy,
static int __net_init ipv6_sysctl_net_init(struct net *net)
{
- struct ctl_table *ipv6_table;
+ ctl_table_no_const *ipv6_table;
struct ctl_table *ipv6_route_table;
struct ctl_table *ipv6_icmp_table;
int err;
}
}
+#ifdef CONFIG_GRKERNSEC_BLACKHOLE
+extern int grsec_enable_blackhole;
+#endif
+
static void tcp_v6_hash(struct sock *sk)
{
if (sk->sk_state != TCP_CLOSE) {
return 0;
reset:
+#ifdef CONFIG_GRKERNSEC_BLACKHOLE
+ if (!grsec_enable_blackhole)
+#endif
tcp_v6_send_reset(sk, skb);
discard:
if (opt_skb)
sk = __inet6_lookup_skb(&tcp_hashinfo, skb, th->source, th->dest,
inet6_iif(skb));
- if (!sk)
+ if (!sk) {
+#ifdef CONFIG_GRKERNSEC_BLACKHOLE
+ ret = 1;
+#endif
goto no_tcp_socket;
+ }
process:
- if (sk->sk_state == TCP_TIME_WAIT)
+ if (sk->sk_state == TCP_TIME_WAIT) {
+#ifdef CONFIG_GRKERNSEC_BLACKHOLE
+ ret = 2;
+#endif
goto do_time_wait;
+ }
if (hdr->hop_limit < inet6_sk(sk)->min_hopcount) {
NET_INC_STATS_BH(net, LINUX_MIB_TCPMINTTLDROP);
bad_packet:
TCP_INC_STATS_BH(net, TCP_MIB_INERRS);
} else {
+#ifdef CONFIG_GRKERNSEC_BLACKHOLE
+ if (!grsec_enable_blackhole || (ret == 1 &&
+ (skb->dev->flags & IFF_LOOPBACK)))
+#endif
tcp_v6_send_reset(NULL, skb);
}
udp_ipv6_hash_secret + net_hash_mix(net));
}
+#ifdef CONFIG_GRKERNSEC_BLACKHOLE
+extern int grsec_enable_blackhole;
+#endif
+
int ipv6_rcv_saddr_equal(const struct sock *sk, const struct sock *sk2)
{
const struct in6_addr *sk2_rcv_saddr6 = inet6_rcv_saddr(sk2);
if (unlikely(err)) {
trace_kfree_skb(skb, udpv6_recvmsg);
if (!peeked) {
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
if (is_udp4)
UDP_INC_STATS_USER(sock_net(sk),
UDP_MIB_INERRORS,
UDP6_INC_STATS_BH(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite);
drop:
UDP6_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
kfree_skb(skb);
return -1;
}
if (likely(skb1 == NULL))
skb1 = (i == final) ? skb : skb_clone(skb, GFP_ATOMIC);
if (!skb1) {
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
UDP6_INC_STATS_BH(sock_net(sk), UDP_MIB_RCVBUFERRORS,
IS_UDPLITE(sk));
UDP6_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS,
goto csum_error;
UDP6_INC_STATS_BH(net, UDP_MIB_NOPORTS, proto == IPPROTO_UDPLITE);
+#ifdef CONFIG_GRKERNSEC_BLACKHOLE
+ if (!grsec_enable_blackhole || (skb->dev->flags & IFF_LOOPBACK))
+#endif
icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_PORT_UNREACH, 0);
kfree_skb(skb);
}
}
-static inline int xfrm6_garbage_collect(struct dst_ops *ops)
+static int xfrm6_garbage_collect(struct dst_ops *ops)
{
struct net *net = container_of(ops, struct net, xfrm.xfrm6_dst_ops);
- xfrm6_policy_afinfo.garbage_collect(net);
+ xfrm_garbage_collect_deferred(net);
return dst_entries_get_fast(ops) > ops->gc_thresh * 2;
}
static int __net_init xfrm6_net_init(struct net *net)
{
- struct ctl_table *table;
+ ctl_table_no_const *table = NULL;
struct ctl_table_header *hdr;
- table = xfrm6_policy_table;
if (!net_eq(net, &init_net)) {
- table = kmemdup(table, sizeof(xfrm6_policy_table), GFP_KERNEL);
+ table = kmemdup(xfrm6_policy_table, sizeof(xfrm6_policy_table), GFP_KERNEL);
if (!table)
goto err_alloc;
table[0].data = &net->xfrm.xfrm6_dst_ops.gc_thresh;
- }
+ hdr = register_net_sysctl(net, "net/ipv6", table);
+ } else
+ hdr = register_net_sysctl(net, "net/ipv6", xfrm6_policy_table);
- hdr = register_net_sysctl(net, "net/ipv6", table);
if (!hdr)
goto err_reg;
return 0;
err_reg:
- if (!net_eq(net, &init_net))
- kfree(table);
+ kfree(table);
err_alloc:
return -ENOMEM;
}
struct proc_dir_entry *p;
int rc = -ENOMEM;
- ipx_proc_dir = proc_mkdir("ipx", init_net.proc_net);
+ ipx_proc_dir = proc_mkdir_restrict("ipx", init_net.proc_net);
if (!ipx_proc_dir)
goto out;
add_wait_queue(&port->open_wait, &wait);
pr_debug("%s(%d):block_til_ready before block on %s open_count=%d\n",
- __FILE__, __LINE__, tty->driver->name, port->count);
+ __FILE__, __LINE__, tty->driver->name, atomic_read(&port->count));
spin_lock_irqsave(&port->lock, flags);
- port->count--;
+ atomic_dec(&port->count);
port->blocked_open++;
spin_unlock_irqrestore(&port->lock, flags);
}
pr_debug("%s(%d):block_til_ready blocking on %s open_count=%d\n",
- __FILE__, __LINE__, tty->driver->name, port->count);
+ __FILE__, __LINE__, tty->driver->name, atomic_read(&port->count));
schedule();
}
spin_lock_irqsave(&port->lock, flags);
if (!tty_hung_up_p(filp))
- port->count++;
+ atomic_inc(&port->count);
port->blocked_open--;
spin_unlock_irqrestore(&port->lock, flags);
pr_debug("%s(%d):block_til_ready after blocking on %s open_count=%d\n",
- __FILE__, __LINE__, tty->driver->name, port->count);
+ __FILE__, __LINE__, tty->driver->name, atomic_read(&port->count));
if (!retval)
port->flags |= ASYNC_NORMAL_ACTIVE;
/* ++ is not atomic, so this should be protected - Jean II */
spin_lock_irqsave(&self->port.lock, flags);
- self->port.count++;
+ atomic_inc(&self->port.count);
spin_unlock_irqrestore(&self->port.lock, flags);
tty_port_tty_set(&self->port, tty);
pr_debug("%s(), %s%d, count = %d\n", __func__ , tty->driver->name,
- self->line, self->port.count);
+ self->line, atomic_read(&self->port.count));
/* Not really used by us, but lets do it anyway */
self->port.low_latency = (self->port.flags & ASYNC_LOW_LATENCY) ? 1 : 0;
tty_kref_put(port->tty);
}
port->tty = NULL;
- port->count = 0;
+ atomic_set(&port->count, 0);
spin_unlock_irqrestore(&port->lock, flags);
wake_up_interruptible(&port->open_wait);
seq_putc(m, '\n');
seq_printf(m, "Role: %s\n", self->client ? "client" : "server");
- seq_printf(m, "Open count: %d\n", self->port.count);
+ seq_printf(m, "Open count: %d\n", atomic_read(&self->port.count));
seq_printf(m, "Max data size: %d\n", self->max_data_size);
seq_printf(m, "Max header size: %d\n", self->max_header_size);
{
int i;
- proc_irda = proc_mkdir("irda", init_net.proc_net);
+ proc_irda = proc_mkdir_restrict("irda", init_net.proc_net);
if (proc_irda == NULL)
return;
{
char name[12];
- sprintf(name, "%08x", atomic_inc_return(&iucv_sk_list.autobind_name));
+ sprintf(name, "%08x", atomic_inc_return_unchecked(&iucv_sk_list.autobind_name));
while (__iucv_get_sock_by_name(name)) {
sprintf(name, "%08x",
- atomic_inc_return(&iucv_sk_list.autobind_name));
+ atomic_inc_return_unchecked(&iucv_sk_list.autobind_name));
}
memcpy(iucv->src_name, name, 8);
}
return NOTIFY_OK;
}
-static struct notifier_block __refdata iucv_cpu_notifier = {
+static struct notifier_block iucv_cpu_notifier = {
.notifier_call = iucv_cpu_notify,
};
static u32 get_acqseq(void)
{
u32 res;
- static atomic_t acqseq;
+ static atomic_unchecked_t acqseq;
do {
- res = atomic_inc_return(&acqseq);
+ res = atomic_inc_return_unchecked(&acqseq);
} while (!res);
return res;
}
struct sock *tunnel_sock;
struct l2tp_session *session;
struct list_head list;
- atomic_long_t tx_bytes;
- atomic_long_t tx_packets;
- atomic_long_t tx_dropped;
- atomic_long_t rx_bytes;
- atomic_long_t rx_packets;
- atomic_long_t rx_errors;
+ atomic_long_unchecked_t tx_bytes;
+ atomic_long_unchecked_t tx_packets;
+ atomic_long_unchecked_t tx_dropped;
+ atomic_long_unchecked_t rx_bytes;
+ atomic_long_unchecked_t rx_packets;
+ atomic_long_unchecked_t rx_errors;
};
/* via l2tp_session_priv() */
int ret = l2tp_xmit_skb(session, skb, session->hdr_len);
if (likely(ret == NET_XMIT_SUCCESS)) {
- atomic_long_add(len, &priv->tx_bytes);
- atomic_long_inc(&priv->tx_packets);
+ atomic_long_add_unchecked(len, &priv->tx_bytes);
+ atomic_long_inc_unchecked(&priv->tx_packets);
} else {
- atomic_long_inc(&priv->tx_dropped);
+ atomic_long_inc_unchecked(&priv->tx_dropped);
}
return NETDEV_TX_OK;
}
{
struct l2tp_eth *priv = netdev_priv(dev);
- stats->tx_bytes = atomic_long_read(&priv->tx_bytes);
- stats->tx_packets = atomic_long_read(&priv->tx_packets);
- stats->tx_dropped = atomic_long_read(&priv->tx_dropped);
- stats->rx_bytes = atomic_long_read(&priv->rx_bytes);
- stats->rx_packets = atomic_long_read(&priv->rx_packets);
- stats->rx_errors = atomic_long_read(&priv->rx_errors);
+ stats->tx_bytes = atomic_long_read_unchecked(&priv->tx_bytes);
+ stats->tx_packets = atomic_long_read_unchecked(&priv->tx_packets);
+ stats->tx_dropped = atomic_long_read_unchecked(&priv->tx_dropped);
+ stats->rx_bytes = atomic_long_read_unchecked(&priv->rx_bytes);
+ stats->rx_packets = atomic_long_read_unchecked(&priv->rx_packets);
+ stats->rx_errors = atomic_long_read_unchecked(&priv->rx_errors);
return stats;
}
nf_reset(skb);
if (dev_forward_skb(dev, skb) == NET_RX_SUCCESS) {
- atomic_long_inc(&priv->rx_packets);
- atomic_long_add(data_len, &priv->rx_bytes);
+ atomic_long_inc_unchecked(&priv->rx_packets);
+ atomic_long_add_unchecked(data_len, &priv->rx_bytes);
} else {
- atomic_long_inc(&priv->rx_errors);
+ atomic_long_inc_unchecked(&priv->rx_errors);
}
return;
error:
- atomic_long_inc(&priv->rx_errors);
+ atomic_long_inc_unchecked(&priv->rx_errors);
kfree_skb(skb);
}
int rc = -ENOMEM;
struct proc_dir_entry *p;
- llc_proc_dir = proc_mkdir("llc", init_net.proc_net);
+ llc_proc_dir = proc_mkdir_restrict("llc", init_net.proc_net);
if (!llc_proc_dir)
goto out;
ret = ieee80211_vif_use_channel(sdata, chandef,
IEEE80211_CHANCTX_EXCLUSIVE);
}
- } else if (local->open_count == local->monitors) {
+ } else if (local_read(&local->open_count) == local->monitors) {
local->_oper_chandef = *chandef;
ieee80211_hw_config(local, 0);
}
else
local->probe_req_reg--;
- if (!local->open_count)
+ if (!local_read(&local->open_count))
break;
ieee80211_queue_work(&local->hw, &local->reconfig_filter);
if (chanctx_conf) {
*chandef = sdata->vif.bss_conf.chandef;
ret = 0;
- } else if (local->open_count > 0 &&
- local->open_count == local->monitors &&
+ } else if (local_read(&local->open_count) > 0 &&
+ local_read(&local->open_count) == local->monitors &&
sdata->vif.type == NL80211_IFTYPE_MONITOR) {
if (local->use_chanctx)
*chandef = local->monitor_chandef;
#include <net/ieee80211_radiotap.h>
#include <net/cfg80211.h>
#include <net/mac80211.h>
+#include <asm/local.h>
#include "key.h"
#include "sta_info.h"
#include "debug.h"
/* also used to protect ampdu_ac_queue and amdpu_ac_stop_refcnt */
spinlock_t queue_stop_reason_lock;
- int open_count;
+ local_t open_count;
int monitors, cooked_mntrs;
/* number of interfaces with corresponding FIF_ flags */
int fif_fcsfail, fif_plcpfail, fif_control, fif_other_bss, fif_pspoll,
break;
}
- if (local->open_count == 0) {
+ if (local_read(&local->open_count) == 0) {
res = drv_start(local);
if (res)
goto err_del_bss;
res = drv_add_interface(local, sdata);
if (res)
goto err_stop;
- } else if (local->monitors == 0 && local->open_count == 0) {
+ } else if (local->monitors == 0 && local_read(&local->open_count) == 0) {
res = ieee80211_add_virtual_monitor(local);
if (res)
goto err_stop;
atomic_inc(&local->iff_promiscs);
if (coming_up)
- local->open_count++;
+ local_inc(&local->open_count);
if (hw_reconf_flags)
ieee80211_hw_config(local, hw_reconf_flags);
err_del_interface:
drv_remove_interface(local, sdata);
err_stop:
- if (!local->open_count)
+ if (!local_read(&local->open_count))
drv_stop(local);
err_del_bss:
sdata->bss = NULL;
}
if (going_down)
- local->open_count--;
+ local_dec(&local->open_count);
switch (sdata->vif.type) {
case NL80211_IFTYPE_AP_VLAN:
}
spin_unlock_irqrestore(&local->queue_stop_reason_lock, flags);
- if (local->open_count == 0)
+ if (local_read(&local->open_count) == 0)
ieee80211_clear_tx_pending(local);
/*
if (cancel_scan)
flush_delayed_work(&local->scan_work);
- if (local->open_count == 0) {
+ if (local_read(&local->open_count) == 0) {
ieee80211_stop_device(local);
/* no reconfiguring after stop! */
ieee80211_configure_filter(local);
ieee80211_hw_config(local, hw_reconf_flags);
- if (local->monitors == local->open_count)
+ if (local->monitors == local_read(&local->open_count))
ieee80211_add_virtual_monitor(local);
}
changed &= ~(IEEE80211_CONF_CHANGE_CHANNEL |
IEEE80211_CONF_CHANGE_POWER);
- if (changed && local->open_count) {
+ if (changed && local_read(&local->open_count)) {
ret = drv_config(local, changed);
/*
* Goal:
struct ieee80211_sub_if_data *sdata;
struct sta_info *sta;
- if (!local->open_count)
+ if (!local_read(&local->open_count))
goto suspend;
ieee80211_scan_cancel(local);
cancel_work_sync(&local->dynamic_ps_enable_work);
del_timer_sync(&local->dynamic_ps_timer);
- local->wowlan = wowlan && local->open_count;
+ local->wowlan = wowlan && local_read(&local->open_count);
if (local->wowlan) {
int err = drv_suspend(local, wowlan);
if (err < 0) {
WARN_ON(!list_empty(&local->chanctx_list));
/* stop hardware - this must stop RX */
- if (local->open_count)
+ if (local_read(&local->open_count))
ieee80211_stop_device(local);
suspend:
ASSERT_RTNL();
- if (local->open_count)
+ if (local_read(&local->open_count))
return -EBUSY;
if (local->hw.flags & IEEE80211_HW_HAS_RATE_CONTROL) {
}
#endif
/* everything else happens only if HW was up & running */
- if (!local->open_count)
+ if (!local_read(&local->open_count))
goto wake_up;
/*
local->in_reconfig = false;
barrier();
- if (local->monitors == local->open_count && local->monitors > 0)
+ if (local->monitors == local_read(&local->open_count) && local->monitors > 0)
ieee80211_add_virtual_monitor(local);
/*
To compile it as a module, choose M here. If unsure, say N.
+config NETFILTER_XT_MATCH_GRADM
+ tristate '"gradm" match support'
+ depends on NETFILTER_XTABLES && NETFILTER_ADVANCED
+ depends on GRKERNSEC && !GRKERNSEC_NO_RBAC
+ ---help---
+ The gradm match allows to match on grsecurity RBAC being enabled.
+ It is useful when iptables rules are applied early on bootup to
+ prevent connections to the machine (except from a trusted host)
+ while the RBAC system is disabled.
+
config NETFILTER_XT_MATCH_HASHLIMIT
tristate '"hashlimit" match support'
depends on (IP6_NF_IPTABLES || IP6_NF_IPTABLES=n)
obj-$(CONFIG_NETFILTER_XT_MATCH_DSCP) += xt_dscp.o
obj-$(CONFIG_NETFILTER_XT_MATCH_ECN) += xt_ecn.o
obj-$(CONFIG_NETFILTER_XT_MATCH_ESP) += xt_esp.o
+obj-$(CONFIG_NETFILTER_XT_MATCH_GRADM) += xt_gradm.o
obj-$(CONFIG_NETFILTER_XT_MATCH_HASHLIMIT) += xt_hashlimit.o
obj-$(CONFIG_NETFILTER_XT_MATCH_HELPER) += xt_helper.o
obj-$(CONFIG_NETFILTER_XT_MATCH_HL) += xt_hl.o
return ret;
}
-static struct nf_sockopt_ops so_set __read_mostly = {
+static struct nf_sockopt_ops so_set = {
.pf = PF_INET,
.get_optmin = SO_IP_SET,
.get_optmax = SO_IP_SET + 1,
/* Increase the refcnt counter of the dest */
ip_vs_dest_hold(dest);
- conn_flags = atomic_read(&dest->conn_flags);
+ conn_flags = atomic_read_unchecked(&dest->conn_flags);
if (cp->protocol != IPPROTO_UDP)
conn_flags &= ~IP_VS_CONN_F_ONE_PACKET;
flags = cp->flags;
cp->control = NULL;
atomic_set(&cp->n_control, 0);
- atomic_set(&cp->in_pkts, 0);
+ atomic_set_unchecked(&cp->in_pkts, 0);
cp->packet_xmit = NULL;
cp->app = NULL;
/* Don't drop the entry if its number of incoming packets is not
located in [0, 8] */
- i = atomic_read(&cp->in_pkts);
+ i = atomic_read_unchecked(&cp->in_pkts);
if (i > 8 || i < 0) return 0;
if (!todrop_rate[i]) return 0;
ret = cp->packet_xmit(skb, cp, pd->pp, iph);
/* do not touch skb anymore */
- atomic_inc(&cp->in_pkts);
+ atomic_inc_unchecked(&cp->in_pkts);
ip_vs_conn_put(cp);
return ret;
}
if (cp->flags & IP_VS_CONN_F_ONE_PACKET)
pkts = sysctl_sync_threshold(ipvs);
else
- pkts = atomic_add_return(1, &cp->in_pkts);
+ pkts = atomic_add_return_unchecked(1, &cp->in_pkts);
if (ipvs->sync_state & IP_VS_STATE_MASTER)
ip_vs_sync_conn(net, cp, pkts);
*/
ip_vs_rs_hash(ipvs, dest);
}
- atomic_set(&dest->conn_flags, conn_flags);
+ atomic_set_unchecked(&dest->conn_flags, conn_flags);
/* bind the service */
old_svc = rcu_dereference_protected(dest->svc, 1);
* align with netns init in ip_vs_control_net_init()
*/
-static struct ctl_table vs_vars[] = {
+static ctl_table_no_const vs_vars[] __read_only = {
{
.procname = "amemthresh",
.maxlen = sizeof(int),
" %-7s %-6d %-10d %-10d\n",
&dest->addr.in6,
ntohs(dest->port),
- ip_vs_fwd_name(atomic_read(&dest->conn_flags)),
+ ip_vs_fwd_name(atomic_read_unchecked(&dest->conn_flags)),
atomic_read(&dest->weight),
atomic_read(&dest->activeconns),
atomic_read(&dest->inactconns));
"%-7s %-6d %-10d %-10d\n",
ntohl(dest->addr.ip),
ntohs(dest->port),
- ip_vs_fwd_name(atomic_read(&dest->conn_flags)),
+ ip_vs_fwd_name(atomic_read_unchecked(&dest->conn_flags)),
atomic_read(&dest->weight),
atomic_read(&dest->activeconns),
atomic_read(&dest->inactconns));
entry.addr = dest->addr.ip;
entry.port = dest->port;
- entry.conn_flags = atomic_read(&dest->conn_flags);
+ entry.conn_flags = atomic_read_unchecked(&dest->conn_flags);
entry.weight = atomic_read(&dest->weight);
entry.u_threshold = dest->u_threshold;
entry.l_threshold = dest->l_threshold;
if (nla_put(skb, IPVS_DEST_ATTR_ADDR, sizeof(dest->addr), &dest->addr) ||
nla_put_be16(skb, IPVS_DEST_ATTR_PORT, dest->port) ||
nla_put_u32(skb, IPVS_DEST_ATTR_FWD_METHOD,
- (atomic_read(&dest->conn_flags) &
+ (atomic_read_unchecked(&dest->conn_flags) &
IP_VS_CONN_F_FWD_MASK)) ||
nla_put_u32(skb, IPVS_DEST_ATTR_WEIGHT,
atomic_read(&dest->weight)) ||
{
int idx;
struct netns_ipvs *ipvs = net_ipvs(net);
- struct ctl_table *tbl;
+ ctl_table_no_const *tbl;
atomic_set(&ipvs->dropentry, 0);
spin_lock_init(&ipvs->dropentry_lock);
* IPVS LBLC sysctl table
*/
#ifdef CONFIG_SYSCTL
-static struct ctl_table vs_vars_table[] = {
+static ctl_table_no_const vs_vars_table[] __read_only = {
{
.procname = "lblc_expiration",
.data = NULL,
* IPVS LBLCR sysctl table
*/
-static struct ctl_table vs_vars_table[] = {
+static ctl_table_no_const vs_vars_table[] __read_only = {
{
.procname = "lblcr_expiration",
.data = NULL,
cp = cp->control;
if (cp) {
if (cp->flags & IP_VS_CONN_F_TEMPLATE)
- pkts = atomic_add_return(1, &cp->in_pkts);
+ pkts = atomic_add_return_unchecked(1, &cp->in_pkts);
else
pkts = sysctl_sync_threshold(ipvs);
ip_vs_sync_conn(net, cp->control, pkts);
if (!cp)
return;
if (cp->flags & IP_VS_CONN_F_TEMPLATE)
- pkts = atomic_add_return(1, &cp->in_pkts);
+ pkts = atomic_add_return_unchecked(1, &cp->in_pkts);
else
pkts = sysctl_sync_threshold(ipvs);
goto sloop;
if (opt)
memcpy(&cp->in_seq, opt, sizeof(*opt));
- atomic_set(&cp->in_pkts, sysctl_sync_threshold(ipvs));
+ atomic_set_unchecked(&cp->in_pkts, sysctl_sync_threshold(ipvs));
cp->state = state;
cp->old_state = cp->state;
/*
else
rc = NF_ACCEPT;
/* do not touch skb anymore */
- atomic_inc(&cp->in_pkts);
+ atomic_inc_unchecked(&cp->in_pkts);
goto out;
}
else
rc = NF_ACCEPT;
/* do not touch skb anymore */
- atomic_inc(&cp->in_pkts);
+ atomic_inc_unchecked(&cp->in_pkts);
goto out;
}
#ifdef CONFIG_SYSCTL
static int nf_conntrack_acct_init_sysctl(struct net *net)
{
- struct ctl_table *table;
+ ctl_table_no_const *table;
table = kmemdup(acct_sysctl_table, sizeof(acct_sysctl_table),
GFP_KERNEL);
#define DYING_NULLS_VAL ((1<<30)+1)
#define TEMPLATE_NULLS_VAL ((1<<30)+2)
+#ifdef CONFIG_GRKERNSEC_HIDESYM
+static atomic_unchecked_t conntrack_cache_id = ATOMIC_INIT(0);
+#endif
+
int nf_conntrack_init_net(struct net *net)
{
int ret = -ENOMEM;
if (!net->ct.stat)
goto err_pcpu_lists;
+#ifdef CONFIG_GRKERNSEC_HIDESYM
+ net->ct.slabname = kasprintf(GFP_KERNEL, "nf_conntrack_%08x", atomic_inc_return_unchecked(&conntrack_cache_id));
+#else
net->ct.slabname = kasprintf(GFP_KERNEL, "nf_conntrack_%p", net);
+#endif
if (!net->ct.slabname)
goto err_slabname;
#ifdef CONFIG_SYSCTL
static int nf_conntrack_event_init_sysctl(struct net *net)
{
- struct ctl_table *table;
+ ctl_table_no_const *table;
table = kmemdup(event_sysctl_table, sizeof(event_sysctl_table),
GFP_KERNEL);
static int nf_conntrack_helper_init_sysctl(struct net *net)
{
- struct ctl_table *table;
+ ctl_table_no_const *table;
table = kmemdup(helper_sysctl_table, sizeof(helper_sysctl_table),
GFP_KERNEL);
static void
nf_ct_unregister_sysctl(struct ctl_table_header **header,
- struct ctl_table **table,
+ ctl_table_no_const **table,
unsigned int users)
{
if (users > 0)
static int nf_conntrack_standalone_init_sysctl(struct net *net)
{
- struct ctl_table *table;
+ ctl_table_no_const *table;
table = kmemdup(nf_ct_sysctl_table, sizeof(nf_ct_sysctl_table),
GFP_KERNEL);
#ifdef CONFIG_SYSCTL
static int nf_conntrack_tstamp_init_sysctl(struct net *net)
{
- struct ctl_table *table;
+ ctl_table_no_const *table;
table = kmemdup(tstamp_sysctl_table, sizeof(tstamp_sysctl_table),
GFP_KERNEL);
#ifdef CONFIG_SYSCTL
static char nf_log_sysctl_fnames[NFPROTO_NUMPROTO-NFPROTO_UNSPEC][3];
-static struct ctl_table nf_log_sysctl_table[NFPROTO_NUMPROTO+1];
+static ctl_table_no_const nf_log_sysctl_table[NFPROTO_NUMPROTO+1] __read_only;
static int nf_log_proc_dostring(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp, loff_t *ppos)
rcu_assign_pointer(net->nf.nf_loggers[tindex], logger);
mutex_unlock(&nf_log_mutex);
} else {
+ ctl_table_no_const nf_log_table = *table;
+
mutex_lock(&nf_log_mutex);
logger = nft_log_dereference(net->nf.nf_loggers[tindex]);
if (!logger)
- table->data = "NONE";
+ nf_log_table.data = "NONE";
else
- table->data = logger->name;
- r = proc_dostring(table, write, buffer, lenp, ppos);
+ nf_log_table.data = logger->name;
+ r = proc_dostring(&nf_log_table, write, buffer, lenp, ppos);
mutex_unlock(&nf_log_mutex);
}
}
}
- list_add(®->list, &nf_sockopts);
+ pax_list_add((struct list_head *)®->list, &nf_sockopts);
out:
mutex_unlock(&nf_sockopt_mutex);
return ret;
void nf_unregister_sockopt(struct nf_sockopt_ops *reg)
{
mutex_lock(&nf_sockopt_mutex);
- list_del(®->list);
+ pax_list_del((struct list_head *)®->list);
mutex_unlock(&nf_sockopt_mutex);
}
EXPORT_SYMBOL(nf_unregister_sockopt);
struct nfnl_log_net {
spinlock_t instances_lock;
struct hlist_head instance_table[INSTANCE_BUCKETS];
- atomic_t global_seq;
+ atomic_unchecked_t global_seq;
};
static struct nfnl_log_net *nfnl_log_pernet(struct net *net)
/* global sequence number */
if ((inst->flags & NFULNL_CFG_F_SEQ_GLOBAL) &&
nla_put_be32(inst->skb, NFULA_SEQ_GLOBAL,
- htonl(atomic_inc_return(&log->global_seq))))
+ htonl(atomic_inc_return_unchecked(&log->global_seq))))
goto nla_put_failure;
if (data_len) {
{
struct hashlimit_net *hashlimit_net = hashlimit_pernet(net);
- hashlimit_net->ipt_hashlimit = proc_mkdir("ipt_hashlimit", net->proc_net);
+ hashlimit_net->ipt_hashlimit = proc_mkdir_restrict("ipt_hashlimit", net->proc_net);
if (!hashlimit_net->ipt_hashlimit)
return -ENOMEM;
#if IS_ENABLED(CONFIG_IP6_NF_IPTABLES)
- hashlimit_net->ip6t_hashlimit = proc_mkdir("ip6t_hashlimit", net->proc_net);
+ hashlimit_net->ip6t_hashlimit = proc_mkdir_restrict("ip6t_hashlimit", net->proc_net);
if (!hashlimit_net->ip6t_hashlimit) {
remove_proc_entry("ipt_hashlimit", net->proc_net);
return -ENOMEM;
{
struct recent_net *recent_net = recent_pernet(net);
- recent_net->xt_recent = proc_mkdir("xt_recent", net->proc_net);
+ recent_net->xt_recent = proc_mkdir_restrict("xt_recent", net->proc_net);
if (!recent_net->xt_recent)
return -ENOMEM;
return 0;
#include <linux/module.h>
struct xt_statistic_priv {
- atomic_t count;
+ atomic_unchecked_t count;
} ____cacheline_aligned_in_smp;
MODULE_LICENSE("GPL");
break;
case XT_STATISTIC_MODE_NTH:
do {
- oval = atomic_read(&info->master->count);
+ oval = atomic_read_unchecked(&info->master->count);
nval = (oval == info->u.nth.every) ? 0 : oval + 1;
- } while (atomic_cmpxchg(&info->master->count, oval, nval) != oval);
+ } while (atomic_cmpxchg_unchecked(&info->master->count, oval, nval) != oval);
if (nval == 0)
ret = !ret;
break;
info->master = kzalloc(sizeof(*info->master), GFP_KERNEL);
if (info->master == NULL)
return -ENOMEM;
- atomic_set(&info->master->count, info->u.nth.count);
+ atomic_set_unchecked(&info->master->count, info->u.nth.count);
return 0;
}
sk->sk_error_report(sk);
}
}
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
}
static void netlink_rcv_wake(struct sock *sk)
sk_wmem_alloc_get(s),
nlk->cb_running,
atomic_read(&s->sk_refcnt),
- atomic_read(&s->sk_drops),
+ atomic_read_unchecked(&s->sk_drops),
sock_i_ino(s)
);
.ndo_get_stats64 = internal_dev_get_stats,
};
-static struct rtnl_link_ops internal_dev_link_ops __read_mostly = {
+static struct rtnl_link_ops internal_dev_link_ops = {
.kind = "openvswitch",
};
* netdev-stats can be directly read over netlink-ioctl.
*/
- stats->rx_errors = atomic_long_read(&vport->err_stats.rx_errors);
- stats->tx_errors = atomic_long_read(&vport->err_stats.tx_errors);
- stats->tx_dropped = atomic_long_read(&vport->err_stats.tx_dropped);
- stats->rx_dropped = atomic_long_read(&vport->err_stats.rx_dropped);
+ stats->rx_errors = atomic_long_read_unchecked(&vport->err_stats.rx_errors);
+ stats->tx_errors = atomic_long_read_unchecked(&vport->err_stats.tx_errors);
+ stats->tx_dropped = atomic_long_read_unchecked(&vport->err_stats.tx_dropped);
+ stats->rx_dropped = atomic_long_read_unchecked(&vport->err_stats.rx_dropped);
for_each_possible_cpu(i) {
const struct pcpu_sw_netstats *percpu_stats;
{
switch (err_type) {
case VPORT_E_RX_DROPPED:
- atomic_long_inc(&vport->err_stats.rx_dropped);
+ atomic_long_inc_unchecked(&vport->err_stats.rx_dropped);
break;
case VPORT_E_RX_ERROR:
- atomic_long_inc(&vport->err_stats.rx_errors);
+ atomic_long_inc_unchecked(&vport->err_stats.rx_errors);
break;
case VPORT_E_TX_DROPPED:
- atomic_long_inc(&vport->err_stats.tx_dropped);
+ atomic_long_inc_unchecked(&vport->err_stats.tx_dropped);
break;
case VPORT_E_TX_ERROR:
- atomic_long_inc(&vport->err_stats.tx_errors);
+ atomic_long_inc_unchecked(&vport->err_stats.tx_errors);
break;
}
/* The following definitions are for implementers of vport devices: */
struct vport_err_stats {
- atomic_long_t rx_dropped;
- atomic_long_t rx_errors;
- atomic_long_t tx_dropped;
- atomic_long_t tx_errors;
+ atomic_long_unchecked_t rx_dropped;
+ atomic_long_unchecked_t rx_errors;
+ atomic_long_unchecked_t tx_dropped;
+ atomic_long_unchecked_t tx_errors;
};
/**
* struct vport_portids - array of netlink portids of a vport.
return ret;
drop:
- atomic_long_inc(&dev->tx_dropped);
+ atomic_long_inc_unchecked(&dev->tx_dropped);
kfree_skb(skb);
return NET_XMIT_DROP;
}
spin_lock(&sk->sk_receive_queue.lock);
po->stats.stats1.tp_packets++;
- skb->dropcount = atomic_read(&sk->sk_drops);
+ skb->dropcount = atomic_read_unchecked(&sk->sk_drops);
__skb_queue_tail(&sk->sk_receive_queue, skb);
spin_unlock(&sk->sk_receive_queue.lock);
sk->sk_data_ready(sk);
drop_n_acct:
spin_lock(&sk->sk_receive_queue.lock);
po->stats.stats1.tp_drops++;
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
spin_unlock(&sk->sk_receive_queue.lock);
drop_n_restore:
case PACKET_HDRLEN:
if (len > sizeof(int))
len = sizeof(int);
- if (copy_from_user(&val, optval, len))
+ if (len > sizeof(val) || copy_from_user(&val, optval, len))
return -EFAULT;
switch (val) {
case TPACKET_V1:
len = lv;
if (put_user(len, optlen))
return -EFAULT;
- if (copy_to_user(optval, data, len))
+ if (len > sizeof(st) || copy_to_user(optval, data, len))
return -EFAULT;
return 0;
}
case PNS_PEP_CTRL_REQ:
if (skb_queue_len(&pn->ctrlreq_queue) >= PNPIPE_CTRLREQ_MAX) {
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
break;
}
__skb_pull(skb, 4);
}
if (pn->rx_credits == 0) {
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
err = -ENOBUFS;
break;
}
}
if (pn->rx_credits == 0) {
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
err = NET_RX_DROP;
break;
}
from_kuid_munged(seq_user_ns(seq), sock_i_uid(sk)),
sock_i_ino(sk),
atomic_read(&sk->sk_refcnt), sk,
- atomic_read(&sk->sk_drops));
+ atomic_read_unchecked(&sk->sk_drops));
}
seq_pad(seq, '\n');
return 0;
{
int ret;
int range[2] = {local_port_range[0], local_port_range[1]};
- struct ctl_table tmp = {
+ ctl_table_no_const tmp = {
.data = &range,
.maxlen = sizeof(range),
.mode = table->mode,
* finds that the saved generation number is smaller than the global generation
* number, it wakes up the process.
*/
-static atomic_t rds_cong_generation = ATOMIC_INIT(0);
+static atomic_unchecked_t rds_cong_generation = ATOMIC_INIT(0);
/*
* Congestion monitoring
rdsdebug("waking map %p for %pI4\n",
map, &map->m_addr);
rds_stats_inc(s_cong_update_received);
- atomic_inc(&rds_cong_generation);
+ atomic_inc_unchecked(&rds_cong_generation);
if (waitqueue_active(&map->m_waitq))
wake_up(&map->m_waitq);
if (waitqueue_active(&rds_poll_waitq))
int rds_cong_updated_since(unsigned long *recent)
{
- unsigned long gen = atomic_read(&rds_cong_generation);
+ unsigned long gen = atomic_read_unchecked(&rds_cong_generation);
if (likely(*recent == gen))
return 0;
/* sending acks */
unsigned long i_ack_flags;
#ifdef KERNEL_HAS_ATOMIC64
- atomic64_t i_ack_next; /* next ACK to send */
+ atomic64_unchecked_t i_ack_next; /* next ACK to send */
#else
spinlock_t i_ack_lock; /* protect i_ack_next */
u64 i_ack_next; /* next ACK to send */
/* Clear the ACK state */
clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);
#ifdef KERNEL_HAS_ATOMIC64
- atomic64_set(&ic->i_ack_next, 0);
+ atomic64_set_unchecked(&ic->i_ack_next, 0);
#else
ic->i_ack_next = 0;
#endif
static void rds_ib_set_ack(struct rds_ib_connection *ic, u64 seq,
int ack_required)
{
- atomic64_set(&ic->i_ack_next, seq);
+ atomic64_set_unchecked(&ic->i_ack_next, seq);
if (ack_required) {
smp_mb__before_atomic();
set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
smp_mb__after_atomic();
- return atomic64_read(&ic->i_ack_next);
+ return atomic64_read_unchecked(&ic->i_ack_next);
}
#endif
/* sending acks */
unsigned long i_ack_flags;
#ifdef KERNEL_HAS_ATOMIC64
- atomic64_t i_ack_next; /* next ACK to send */
+ atomic64_unchecked_t i_ack_next; /* next ACK to send */
#else
spinlock_t i_ack_lock; /* protect i_ack_next */
u64 i_ack_next; /* next ACK to send */
/* Clear the ACK state */
clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);
#ifdef KERNEL_HAS_ATOMIC64
- atomic64_set(&ic->i_ack_next, 0);
+ atomic64_set_unchecked(&ic->i_ack_next, 0);
#else
ic->i_ack_next = 0;
#endif
int *unpinned);
static void rds_iw_destroy_fastreg(struct rds_iw_mr_pool *pool, struct rds_iw_mr *ibmr);
-static int rds_iw_get_device(struct rds_sock *rs, struct rds_iw_device **rds_iwdev, struct rdma_cm_id **cm_id)
+static int rds_iw_get_device(struct sockaddr_in *src, struct sockaddr_in *dst,
+ struct rds_iw_device **rds_iwdev,
+ struct rdma_cm_id **cm_id)
{
struct rds_iw_device *iwdev;
struct rds_iw_cm_id *i_cm_id;
src_addr->sin_port,
dst_addr->sin_addr.s_addr,
dst_addr->sin_port,
- rs->rs_bound_addr,
- rs->rs_bound_port,
- rs->rs_conn_addr,
- rs->rs_conn_port);
+ src->sin_addr.s_addr,
+ src->sin_port,
+ dst->sin_addr.s_addr,
+ dst->sin_port);
#ifdef WORKING_TUPLE_DETECTION
- if (src_addr->sin_addr.s_addr == rs->rs_bound_addr &&
- src_addr->sin_port == rs->rs_bound_port &&
- dst_addr->sin_addr.s_addr == rs->rs_conn_addr &&
- dst_addr->sin_port == rs->rs_conn_port) {
+ if (src_addr->sin_addr.s_addr == src->sin_addr.s_addr &&
+ src_addr->sin_port == src->sin_port &&
+ dst_addr->sin_addr.s_addr == dst->sin_addr.s_addr &&
+ dst_addr->sin_port == dst->sin_port) {
#else
/* FIXME - needs to compare the local and remote
* ipaddr/port tuple, but the ipaddr is the only
* zero'ed. It doesn't appear to be properly populated
* during connection setup...
*/
- if (src_addr->sin_addr.s_addr == rs->rs_bound_addr) {
+ if (src_addr->sin_addr.s_addr == src->sin_addr.s_addr) {
#endif
spin_unlock_irq(&iwdev->spinlock);
*rds_iwdev = iwdev;
{
struct sockaddr_in *src_addr, *dst_addr;
struct rds_iw_device *rds_iwdev_old;
- struct rds_sock rs;
struct rdma_cm_id *pcm_id;
int rc;
src_addr = (struct sockaddr_in *)&cm_id->route.addr.src_addr;
dst_addr = (struct sockaddr_in *)&cm_id->route.addr.dst_addr;
- rs.rs_bound_addr = src_addr->sin_addr.s_addr;
- rs.rs_bound_port = src_addr->sin_port;
- rs.rs_conn_addr = dst_addr->sin_addr.s_addr;
- rs.rs_conn_port = dst_addr->sin_port;
-
- rc = rds_iw_get_device(&rs, &rds_iwdev_old, &pcm_id);
+ rc = rds_iw_get_device(src_addr, dst_addr, &rds_iwdev_old, &pcm_id);
if (rc)
rds_iw_remove_cm_id(rds_iwdev, cm_id);
struct rds_iw_device *rds_iwdev;
struct rds_iw_mr *ibmr = NULL;
struct rdma_cm_id *cm_id;
+ struct sockaddr_in src = {
+ .sin_addr.s_addr = rs->rs_bound_addr,
+ .sin_port = rs->rs_bound_port,
+ };
+ struct sockaddr_in dst = {
+ .sin_addr.s_addr = rs->rs_conn_addr,
+ .sin_port = rs->rs_conn_port,
+ };
int ret;
- ret = rds_iw_get_device(rs, &rds_iwdev, &cm_id);
+ ret = rds_iw_get_device(&src, &dst, &rds_iwdev, &cm_id);
if (ret || !cm_id) {
ret = -ENODEV;
goto out;
static void rds_iw_set_ack(struct rds_iw_connection *ic, u64 seq,
int ack_required)
{
- atomic64_set(&ic->i_ack_next, seq);
+ atomic64_set_unchecked(&ic->i_ack_next, seq);
if (ack_required) {
smp_mb__before_atomic();
set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
smp_mb__after_atomic();
- return atomic64_read(&ic->i_ack_next);
+ return atomic64_read_unchecked(&ic->i_ack_next);
}
#endif
void (*sync_mr)(void *trans_private, int direction);
void (*free_mr)(void *trans_private, int invalidate);
void (*flush_mrs)(void);
-};
+} __do_const;
struct rds_sock {
struct sock rs_sk;
int val = 1;
set_fs(KERNEL_DS);
- sock->ops->setsockopt(sock, SOL_TCP, TCP_NODELAY, (char __user *)&val,
+ sock->ops->setsockopt(sock, SOL_TCP, TCP_NODELAY, (char __force_user *)&val,
sizeof(val));
set_fs(oldfs);
}
oldfs = get_fs();
set_fs(KERNEL_DS);
- sock->ops->setsockopt(sock, SOL_TCP, TCP_CORK, (char __user *)&val,
+ sock->ops->setsockopt(sock, SOL_TCP, TCP_CORK, (char __force_user *)&val,
sizeof(val));
set_fs(oldfs);
}
__be32 rxrpc_epoch;
/* current debugging ID */
-atomic_t rxrpc_debug_id;
+atomic_unchecked_t rxrpc_debug_id;
/* count of skbs currently in use */
atomic_t rxrpc_n_skbs;
_enter("{%d,%d,%d,%d},",
call->acks_hard, call->acks_unacked,
- atomic_read(&call->sequence),
+ atomic_read_unchecked(&call->sequence),
CIRC_CNT(call->acks_head, call->acks_tail, call->acks_winsz));
stop = 0;
/* each Tx packet has a new serial number */
sp->hdr.serial =
- htonl(atomic_inc_return(&call->conn->serial));
+ htonl(atomic_inc_return_unchecked(&call->conn->serial));
hdr = (struct rxrpc_header *) txb->head;
hdr->serial = sp->hdr.serial;
*/
static void rxrpc_clear_tx_window(struct rxrpc_call *call)
{
- rxrpc_rotate_tx_window(call, atomic_read(&call->sequence));
+ rxrpc_rotate_tx_window(call, atomic_read_unchecked(&call->sequence));
}
/*
latest = ntohl(sp->hdr.serial);
hard = ntohl(ack.firstPacket);
- tx = atomic_read(&call->sequence);
+ tx = atomic_read_unchecked(&call->sequence);
_proto("Rx ACK %%%u { m=%hu f=#%u p=#%u s=%%%u r=%s n=%u }",
latest,
goto maybe_reschedule;
send_ACK_with_skew:
- ack.maxSkew = htons(atomic_read(&call->conn->hi_serial) -
+ ack.maxSkew = htons(atomic_read_unchecked(&call->conn->hi_serial) -
ntohl(ack.serial));
send_ACK:
mtu = call->conn->trans->peer->if_mtu;
ackinfo.rxMTU = htonl(rxrpc_rx_mtu);
ackinfo.jumbo_max = htonl(rxrpc_rx_jumbo_max);
- hdr.serial = htonl(atomic_inc_return(&call->conn->serial));
+ hdr.serial = htonl(atomic_inc_return_unchecked(&call->conn->serial));
_proto("Tx ACK %%%u { m=%hu f=#%u p=#%u s=%%%u r=%s n=%u }",
ntohl(hdr.serial),
ntohs(ack.maxSkew),
send_message:
_debug("send message");
- hdr.serial = htonl(atomic_inc_return(&call->conn->serial));
+ hdr.serial = htonl(atomic_inc_return_unchecked(&call->conn->serial));
_proto("Tx %s %%%u", rxrpc_pkts[hdr.type], ntohl(hdr.serial));
send_message_2:
spin_lock_init(&call->lock);
rwlock_init(&call->state_lock);
atomic_set(&call->usage, 1);
- call->debug_id = atomic_inc_return(&rxrpc_debug_id);
+ call->debug_id = atomic_inc_return_unchecked(&rxrpc_debug_id);
call->state = RXRPC_CALL_CLIENT_SEND_REQUEST;
memset(&call->sock_node, 0xed, sizeof(call->sock_node));
rwlock_init(&conn->lock);
spin_lock_init(&conn->state_lock);
atomic_set(&conn->usage, 1);
- conn->debug_id = atomic_inc_return(&rxrpc_debug_id);
+ conn->debug_id = atomic_inc_return_unchecked(&rxrpc_debug_id);
conn->avail_calls = RXRPC_MAXCALLS;
conn->size_align = 4;
conn->header_size = sizeof(struct rxrpc_header);
len = iov[0].iov_len + iov[1].iov_len;
- hdr.serial = htonl(atomic_inc_return(&conn->serial));
+ hdr.serial = htonl(atomic_inc_return_unchecked(&conn->serial));
_proto("Tx CONN ABORT %%%u { %d }", ntohl(hdr.serial), abort_code);
ret = kernel_sendmsg(conn->trans->local->socket, &msg, iov, 2, len);
/* track the latest serial number on this connection for ACK packet
* information */
serial = ntohl(sp->hdr.serial);
- hi_serial = atomic_read(&call->conn->hi_serial);
+ hi_serial = atomic_read_unchecked(&call->conn->hi_serial);
while (serial > hi_serial)
- hi_serial = atomic_cmpxchg(&call->conn->hi_serial, hi_serial,
+ hi_serial = atomic_cmpxchg_unchecked(&call->conn->hi_serial, hi_serial,
serial);
/* request ACK generation for any ACK or DATA packet that requests
int error; /* error code for local abort */
int debug_id; /* debug ID for printks */
unsigned int call_counter; /* call ID counter */
- atomic_t serial; /* packet serial number counter */
- atomic_t hi_serial; /* highest serial number received */
+ atomic_unchecked_t serial; /* packet serial number counter */
+ atomic_unchecked_t hi_serial; /* highest serial number received */
u8 avail_calls; /* number of calls available */
u8 size_align; /* data size alignment (for security) */
u8 header_size; /* rxrpc + security header size */
spinlock_t lock;
rwlock_t state_lock; /* lock for state transition */
atomic_t usage;
- atomic_t sequence; /* Tx data packet sequence counter */
+ atomic_unchecked_t sequence; /* Tx data packet sequence counter */
u32 abort_code; /* local/remote abort code */
enum { /* current state of call */
RXRPC_CALL_CLIENT_SEND_REQUEST, /* - client sending request phase */
*/
extern atomic_t rxrpc_n_skbs;
extern __be32 rxrpc_epoch;
-extern atomic_t rxrpc_debug_id;
+extern atomic_unchecked_t rxrpc_debug_id;
extern struct workqueue_struct *rxrpc_workqueue;
/*
spin_lock_init(&local->lock);
rwlock_init(&local->services_lock);
atomic_set(&local->usage, 1);
- local->debug_id = atomic_inc_return(&rxrpc_debug_id);
+ local->debug_id = atomic_inc_return_unchecked(&rxrpc_debug_id);
memcpy(&local->srx, srx, sizeof(*srx));
}
memset(skb_put(skb, pad), 0, pad);
}
- seq = atomic_inc_return(&call->sequence);
+ seq = atomic_inc_return_unchecked(&call->sequence);
sp->hdr.epoch = conn->epoch;
sp->hdr.cid = call->cid;
sp->hdr.callNumber = call->call_id;
sp->hdr.seq = htonl(seq);
sp->hdr.serial =
- htonl(atomic_inc_return(&conn->serial));
+ htonl(atomic_inc_return_unchecked(&conn->serial));
sp->hdr.type = RXRPC_PACKET_TYPE_DATA;
sp->hdr.userStatus = 0;
sp->hdr.securityIndex = conn->security_ix;
INIT_LIST_HEAD(&peer->error_targets);
spin_lock_init(&peer->lock);
atomic_set(&peer->usage, 1);
- peer->debug_id = atomic_inc_return(&rxrpc_debug_id);
+ peer->debug_id = atomic_inc_return_unchecked(&rxrpc_debug_id);
memcpy(&peer->srx, srx, sizeof(*srx));
rxrpc_assess_MTU_size(peer);
atomic_read(&conn->usage),
rxrpc_conn_states[conn->state],
key_serial(conn->key),
- atomic_read(&conn->serial),
- atomic_read(&conn->hi_serial));
+ atomic_read_unchecked(&conn->serial),
+ atomic_read_unchecked(&conn->hi_serial));
return 0;
}
spin_lock_init(&trans->client_lock);
rwlock_init(&trans->conn_lock);
atomic_set(&trans->usage, 1);
- trans->debug_id = atomic_inc_return(&rxrpc_debug_id);
+ trans->debug_id = atomic_inc_return_unchecked(&rxrpc_debug_id);
if (peer->srx.transport.family == AF_INET) {
switch (peer->srx.transport_type) {
len = iov[0].iov_len + iov[1].iov_len;
- hdr.serial = htonl(atomic_inc_return(&conn->serial));
+ hdr.serial = htonl(atomic_inc_return_unchecked(&conn->serial));
_proto("Tx CHALLENGE %%%u", ntohl(hdr.serial));
ret = kernel_sendmsg(conn->trans->local->socket, &msg, iov, 2, len);
len = iov[0].iov_len + iov[1].iov_len + iov[2].iov_len;
- hdr->serial = htonl(atomic_inc_return(&conn->serial));
+ hdr->serial = htonl(atomic_inc_return_unchecked(&conn->serial));
_proto("Tx RESPONSE %%%u", ntohl(hdr->serial));
ret = kernel_sendmsg(conn->trans->local->socket, &msg, iov, 3, len);
if (test_and_clear_bit(__LINK_STATE_NOCARRIER, &dev->state)) {
if (dev->reg_state == NETREG_UNINITIALIZED)
return;
- atomic_inc(&dev->carrier_changes);
+ atomic_inc_unchecked(&dev->carrier_changes);
linkwatch_fire_event(dev);
if (netif_running(dev))
__netdev_watchdog_up(dev);
if (!test_and_set_bit(__LINK_STATE_NOCARRIER, &dev->state)) {
if (dev->reg_state == NETREG_UNINITIALIZED)
return;
- atomic_inc(&dev->carrier_changes);
+ atomic_inc_unchecked(&dev->carrier_changes);
linkwatch_fire_event(dev);
}
}
.flags = INET6_PROTO_NOPOLICY | INET6_PROTO_FINAL,
};
-static struct sctp_af sctp_af_inet6 = {
+static struct sctp_af sctp_af_inet6 __read_only = {
.sa_family = AF_INET6,
.sctp_xmit = sctp_v6_xmit,
.setsockopt = ipv6_setsockopt,
#endif
};
-static struct sctp_pf sctp_pf_inet6 = {
+static struct sctp_pf sctp_pf_inet6 __read_only = {
.event_msgname = sctp_inet6_event_msgname,
.skb_msgname = sctp_inet6_skb_msgname,
.af_supported = sctp_inet6_af_supported,
void sctp_v6_pf_exit(void)
{
- list_del(&sctp_af_inet6.list);
+ pax_list_del(&sctp_af_inet6.list);
}
/* Initialize IPv6 support and register with socket layer. */
return 0;
}
+ pax_open_kernel();
INIT_LIST_HEAD(&af->list);
- list_add_tail(&af->list, &sctp_address_families);
+ pax_close_kernel();
+ pax_list_add_tail(&af->list, &sctp_address_families);
return 1;
}
static struct sctp_af sctp_af_inet;
-static struct sctp_pf sctp_pf_inet = {
+static struct sctp_pf sctp_pf_inet __read_only = {
.event_msgname = sctp_inet_event_msgname,
.skb_msgname = sctp_inet_skb_msgname,
.af_supported = sctp_inet_af_supported,
};
/* IPv4 address related functions. */
-static struct sctp_af sctp_af_inet = {
+static struct sctp_af sctp_af_inet __read_only = {
.sa_family = AF_INET,
.sctp_xmit = sctp_v4_xmit,
.setsockopt = ip_setsockopt,
static void sctp_v4_pf_exit(void)
{
- list_del(&sctp_af_inet.list);
+ pax_list_del(&sctp_af_inet.list);
}
static int sctp_v4_protosw_init(void)
sctp_generate_timeout_event(asoc, SCTP_EVENT_TIMEOUT_SACK);
}
-sctp_timer_event_t *sctp_timer_events[SCTP_NUM_TIMEOUT_TYPES] = {
+sctp_timer_event_t * const sctp_timer_events[SCTP_NUM_TIMEOUT_TYPES] = {
NULL,
sctp_generate_t1_cookie_event,
sctp_generate_t1_init_event,
{
struct sctp_association *asoc;
struct sctp_ulpevent *event;
+ struct sctp_event_subscribe subscribe;
if (optlen > sizeof(struct sctp_event_subscribe))
return -EINVAL;
- if (copy_from_user(&sctp_sk(sk)->subscribe, optval, optlen))
+ if (copy_from_user(&subscribe, optval, optlen))
return -EFAULT;
+ sctp_sk(sk)->subscribe = subscribe;
if (sctp_sk(sk)->subscribe.sctp_data_io_event)
pr_warn_ratelimited(DEPRECATED "%s (pid %d) "
static int sctp_getsockopt_events(struct sock *sk, int len, char __user *optval,
int __user *optlen)
{
+ struct sctp_event_subscribe subscribe;
+
if (len <= 0)
return -EINVAL;
if (len > sizeof(struct sctp_event_subscribe))
len = sizeof(struct sctp_event_subscribe);
if (put_user(len, optlen))
return -EFAULT;
- if (copy_to_user(optval, &sctp_sk(sk)->subscribe, len))
+ subscribe = sctp_sk(sk)->subscribe;
+ if (copy_to_user(optval, &subscribe, len))
return -EFAULT;
return 0;
}
*/
static int sctp_getsockopt_autoclose(struct sock *sk, int len, char __user *optval, int __user *optlen)
{
+ __u32 autoclose;
+
/* Applicable to UDP-style socket only */
if (sctp_style(sk, TCP))
return -EOPNOTSUPP;
len = sizeof(int);
if (put_user(len, optlen))
return -EFAULT;
- if (copy_to_user(optval, &sctp_sk(sk)->autoclose, sizeof(int)))
+ autoclose = sctp_sk(sk)->autoclose;
+ if (copy_to_user(optval, &autoclose, sizeof(int)))
return -EFAULT;
return 0;
}
*/
static int sctp_getsockopt_initmsg(struct sock *sk, int len, char __user *optval, int __user *optlen)
{
+ struct sctp_initmsg initmsg;
+
if (len < sizeof(struct sctp_initmsg))
return -EINVAL;
len = sizeof(struct sctp_initmsg);
if (put_user(len, optlen))
return -EFAULT;
- if (copy_to_user(optval, &sctp_sk(sk)->initmsg, len))
+ initmsg = sctp_sk(sk)->initmsg;
+ if (copy_to_user(optval, &initmsg, len))
return -EFAULT;
return 0;
}
->addr_to_user(sp, &temp);
if (space_left < addrlen)
return -ENOMEM;
+ if (addrlen > sizeof(temp) || addrlen < 0)
+ return -EFAULT;
if (copy_to_user(to, &temp, addrlen))
return -EFAULT;
to += addrlen;
loff_t *ppos)
{
struct net *net = current->nsproxy->net_ns;
- struct ctl_table tbl;
+ ctl_table_no_const tbl;
bool changed = false;
char *none = "none";
char tmp[8];
struct net *net = current->nsproxy->net_ns;
unsigned int min = *(unsigned int *) ctl->extra1;
unsigned int max = *(unsigned int *) ctl->extra2;
- struct ctl_table tbl;
+ ctl_table_no_const tbl;
int ret, new_value;
memset(&tbl, 0, sizeof(struct ctl_table));
struct net *net = current->nsproxy->net_ns;
unsigned int min = *(unsigned int *) ctl->extra1;
unsigned int max = *(unsigned int *) ctl->extra2;
- struct ctl_table tbl;
+ ctl_table_no_const tbl;
int ret, new_value;
memset(&tbl, 0, sizeof(struct ctl_table));
loff_t *ppos)
{
struct net *net = current->nsproxy->net_ns;
- struct ctl_table tbl;
+ ctl_table_no_const tbl;
int new_value, ret;
memset(&tbl, 0, sizeof(struct ctl_table));
int sctp_sysctl_net_register(struct net *net)
{
- struct ctl_table *table;
+ ctl_table_no_const *table;
int i;
table = kmemdup(sctp_net_table, sizeof(sctp_net_table), GFP_KERNEL);
#include <linux/magic.h>
#include <linux/slab.h>
#include <linux/xattr.h>
+#include <linux/in.h>
#include <asm/uaccess.h>
#include <asm/unistd.h>
#include <linux/atalk.h>
#include <net/busy_poll.h>
#include <linux/errqueue.h>
+#include <linux/grsock.h>
#ifdef CONFIG_NET_RX_BUSY_POLL
unsigned int sysctl_net_busy_read __read_mostly;
*/
static DEFINE_SPINLOCK(net_family_lock);
-static const struct net_proto_family __rcu *net_families[NPROTO] __read_mostly;
+const struct net_proto_family __rcu *net_families[NPROTO] __read_mostly;
/*
* Statistics counters of the socket lists
&sockfs_dentry_operations, SOCKFS_MAGIC);
}
-static struct vfsmount *sock_mnt __read_mostly;
+struct vfsmount *sock_mnt __read_mostly;
static struct file_system_type sock_fs_type = {
.name = "sockfs",
return -EAFNOSUPPORT;
if (type < 0 || type >= SOCK_MAX)
return -EINVAL;
+ if (protocol < 0)
+ return -EINVAL;
/* Compatibility.
if (err)
return err;
+ if(!kern && !gr_search_socket(family, type, protocol)) {
+ if (rcu_access_pointer(net_families[family]) == NULL)
+ return -EAFNOSUPPORT;
+ else
+ return -EACCES;
+ }
+
+ if (!kern && gr_handle_sock_all(family, type, protocol)) {
+ if (rcu_access_pointer(net_families[family]) == NULL)
+ return -EAFNOSUPPORT;
+ else
+ return -EACCES;
+ }
+
/*
* Allocate the socket and allow the family to set things up. if
* the protocol is 0, the family is instructed to select an appropriate
if (sock) {
err = move_addr_to_kernel(umyaddr, addrlen, &address);
if (err >= 0) {
+ if (gr_handle_sock_server((struct sockaddr *)&address)) {
+ err = -EACCES;
+ goto error;
+ }
+ err = gr_search_bind(sock, (struct sockaddr_in *)&address);
+ if (err)
+ goto error;
+
err = security_socket_bind(sock,
(struct sockaddr *)&address,
addrlen);
(struct sockaddr *)
&address, addrlen);
}
+error:
fput_light(sock->file, fput_needed);
}
return err;
if ((unsigned int)backlog > somaxconn)
backlog = somaxconn;
+ if (gr_handle_sock_server_other(sock->sk)) {
+ err = -EPERM;
+ goto error;
+ }
+
+ err = gr_search_listen(sock);
+ if (err)
+ goto error;
+
err = security_socket_listen(sock, backlog);
if (!err)
err = sock->ops->listen(sock, backlog);
+error:
fput_light(sock->file, fput_needed);
}
return err;
newsock->type = sock->type;
newsock->ops = sock->ops;
+ if (gr_handle_sock_server_other(sock->sk)) {
+ err = -EPERM;
+ sock_release(newsock);
+ goto out_put;
+ }
+
+ err = gr_search_accept(sock);
+ if (err) {
+ sock_release(newsock);
+ goto out_put;
+ }
+
/*
* We don't need try_module_get here, as the listening socket (sock)
* has the protocol module (sock->ops->owner) held.
fd_install(newfd, newfile);
err = newfd;
+ gr_attach_curr_ip(newsock->sk);
+
out_put:
fput_light(sock->file, fput_needed);
out:
int, addrlen)
{
struct socket *sock;
+ struct sockaddr *sck;
struct sockaddr_storage address;
int err, fput_needed;
if (err < 0)
goto out_put;
+ sck = (struct sockaddr *)&address;
+
+ if (gr_handle_sock_client(sck)) {
+ err = -EACCES;
+ goto out_put;
+ }
+
+ err = gr_search_connect(sock, (struct sockaddr_in *)sck);
+ if (err)
+ goto out_put;
+
err =
security_socket_connect(sock, (struct sockaddr *)&address, addrlen);
if (err)
* the protocol.
*/
+asmlinkage long sys_sendto(int, void __user *, size_t, unsigned, struct sockaddr __user *, int);
+
SYSCALL_DEFINE6(sendto, int, fd, void __user *, buff, size_t, len,
unsigned int, flags, struct sockaddr __user *, addr,
int, addr_len)
if (len > INT_MAX)
len = INT_MAX;
+ if (unlikely(!access_ok(VERIFY_READ, buff, len)))
+ return -EFAULT;
sock = sockfd_lookup_light(fd, &err, &fput_needed);
if (!sock)
goto out;
struct socket *sock;
struct iovec iov;
struct msghdr msg;
- struct sockaddr_storage address;
+ struct sockaddr_storage address = { };
int err, err2;
int fput_needed;
if (size > INT_MAX)
size = INT_MAX;
+ if (unlikely(!access_ok(VERIFY_WRITE, ubuf, size)))
+ return -EFAULT;
sock = sockfd_lookup_light(fd, &err, &fput_needed);
if (!sock)
goto out;
* checking falls down on this.
*/
if (copy_from_user(ctl_buf,
- (void __user __force *)msg_sys->msg_control,
+ (void __force_user *)msg_sys->msg_control,
ctl_len))
goto out_freectl;
msg_sys->msg_control = ctl_buf;
ssize_t err;
/* kernel mode address */
- struct sockaddr_storage addr;
+ struct sockaddr_storage addr = { };
/* user mode address pointers */
struct sockaddr __user *uaddr;
ifr = compat_alloc_user_space(buf_size);
rxnfc = (void __user *)ifr + ALIGN(sizeof(struct ifreq), 8);
- if (copy_in_user(&ifr->ifr_name, &ifr32->ifr_name, IFNAMSIZ))
+ if (copy_in_user(ifr->ifr_name, ifr32->ifr_name, IFNAMSIZ))
return -EFAULT;
if (put_user(convert_in ? rxnfc : compat_ptr(data),
old_fs = get_fs();
set_fs(KERNEL_DS);
err = dev_ioctl(net, cmd,
- (struct ifreq __user __force *) &kifr);
+ (struct ifreq __force_user *) &kifr);
set_fs(old_fs);
return err;
old_fs = get_fs();
set_fs(KERNEL_DS);
- err = dev_ioctl(net, cmd, (void __user __force *)&ifr);
+ err = dev_ioctl(net, cmd, (void __force_user *)&ifr);
set_fs(old_fs);
if (cmd == SIOCGIFMAP && !err) {
ret |= get_user(rtdev, &(ur4->rt_dev));
if (rtdev) {
ret |= copy_from_user(devname, compat_ptr(rtdev), 15);
- r4.rt_dev = (char __user __force *)devname;
+ r4.rt_dev = (char __force_user *)devname;
devname[15] = 0;
} else
r4.rt_dev = NULL;
int __user *uoptlen;
int err;
- uoptval = (char __user __force *) optval;
- uoptlen = (int __user __force *) optlen;
+ uoptval = (char __force_user *) optval;
+ uoptlen = (int __force_user *) optlen;
set_fs(KERNEL_DS);
if (level == SOL_SOCKET)
char __user *uoptval;
int err;
- uoptval = (char __user __force *) optval;
+ uoptval = (char __force_user *) optval;
set_fs(KERNEL_DS);
if (level == SOL_SOCKET)
uint64_t *handle)
{
struct rsc rsci, *rscp = NULL;
- static atomic64_t ctxhctr;
+ static atomic64_unchecked_t ctxhctr = ATOMIC64_INIT(0);
long long ctxh;
struct gss_api_mech *gm = NULL;
time_t expiry;
status = -ENOMEM;
/* the handle needs to be just a unique id,
* use a static counter */
- ctxh = atomic64_inc_return(&ctxhctr);
+ ctxh = atomic64_inc_return_unchecked(&ctxhctr);
/* make a copy for the caller */
*handle = ctxh;
struct sunrpc_net *sn;
sn = net_generic(net, sunrpc_net_id);
- cd->u.procfs.proc_ent = proc_mkdir(cd->name, sn->proc_net_rpc);
+ cd->u.procfs.proc_ent = proc_mkdir_restrict(cd->name, sn->proc_net_rpc);
if (cd->u.procfs.proc_ent == NULL)
goto out_nomem;
cd->u.procfs.channel_ent = NULL;
(RPC_IS_ASYNC(task) ? "async" : "sync"));
/* Increment call count */
- task->tk_msg.rpc_proc->p_count++;
+ pax_open_kernel();
+ (*(unsigned int *)&task->tk_msg.rpc_proc->p_count)++;
+ pax_close_kernel();
clnt->cl_stats->rpccnt++;
task->tk_action = call_reserve;
}
#if IS_ENABLED(CONFIG_SUNRPC_DEBUG) || IS_ENABLED(CONFIG_TRACEPOINTS)
static void rpc_task_set_debuginfo(struct rpc_task *task)
{
- static atomic_t rpc_pid;
+ static atomic_unchecked_t rpc_pid;
- task->tk_pid = atomic_inc_return(&rpc_pid);
+ task->tk_pid = atomic_inc_return_unchecked(&rpc_pid);
}
#else
static inline void rpc_task_set_debuginfo(struct rpc_task *task)
dprintk("RPC: registering /proc/net/rpc\n");
sn = net_generic(net, sunrpc_net_id);
- sn->proc_net_rpc = proc_mkdir("rpc", net->proc_net);
+ sn->proc_net_rpc = proc_mkdir_restrict("rpc", net->proc_net);
if (sn->proc_net_rpc == NULL)
return -ENOMEM;
svc_putnl(resv, RPC_SUCCESS);
/* Bump per-procedure stats counter */
- procp->pc_count++;
+ pax_open_kernel();
+ (*(unsigned int *)&procp->pc_count)++;
+ pax_close_kernel();
/* Initialize storage for argp and resp */
memset(rqstp->rq_argp, 0, procp->pc_argsize);
struct group_info *gi;
};
-static int unix_gid_hash(kuid_t uid)
+static int __intentional_overflow(-1) unix_gid_hash(kuid_t uid)
{
return hash_long(from_kuid(&init_user_ns, uid), GID_HASHBITS);
}
(*bpp)[-1] = '\n';
}
-static struct unix_gid *unix_gid_lookup(struct cache_detail *cd, kuid_t uid);
+static struct unix_gid * __intentional_overflow(-1) unix_gid_lookup(struct cache_detail *cd, kuid_t uid);
static int unix_gid_parse(struct cache_detail *cd,
char *mesg, int mlen)
static unsigned int min_max_inline = 4096;
static unsigned int max_max_inline = 65536;
-atomic_t rdma_stat_recv;
-atomic_t rdma_stat_read;
-atomic_t rdma_stat_write;
-atomic_t rdma_stat_sq_starve;
-atomic_t rdma_stat_rq_starve;
-atomic_t rdma_stat_rq_poll;
-atomic_t rdma_stat_rq_prod;
-atomic_t rdma_stat_sq_poll;
-atomic_t rdma_stat_sq_prod;
+atomic_unchecked_t rdma_stat_recv;
+atomic_unchecked_t rdma_stat_read;
+atomic_unchecked_t rdma_stat_write;
+atomic_unchecked_t rdma_stat_sq_starve;
+atomic_unchecked_t rdma_stat_rq_starve;
+atomic_unchecked_t rdma_stat_rq_poll;
+atomic_unchecked_t rdma_stat_rq_prod;
+atomic_unchecked_t rdma_stat_sq_poll;
+atomic_unchecked_t rdma_stat_sq_prod;
/* Temporary NFS request map and context caches */
struct kmem_cache *svc_rdma_map_cachep;
len -= *ppos;
if (len > *lenp)
len = *lenp;
- if (len && copy_to_user(buffer, str_buf, len))
+ if (len > sizeof str_buf || (len && copy_to_user(buffer, str_buf, len)))
return -EFAULT;
*lenp = len;
*ppos += len;
{
.procname = "rdma_stat_read",
.data = &rdma_stat_read,
- .maxlen = sizeof(atomic_t),
+ .maxlen = sizeof(atomic_unchecked_t),
.mode = 0644,
.proc_handler = read_reset_stat,
},
{
.procname = "rdma_stat_recv",
.data = &rdma_stat_recv,
- .maxlen = sizeof(atomic_t),
+ .maxlen = sizeof(atomic_unchecked_t),
.mode = 0644,
.proc_handler = read_reset_stat,
},
{
.procname = "rdma_stat_write",
.data = &rdma_stat_write,
- .maxlen = sizeof(atomic_t),
+ .maxlen = sizeof(atomic_unchecked_t),
.mode = 0644,
.proc_handler = read_reset_stat,
},
{
.procname = "rdma_stat_sq_starve",
.data = &rdma_stat_sq_starve,
- .maxlen = sizeof(atomic_t),
+ .maxlen = sizeof(atomic_unchecked_t),
.mode = 0644,
.proc_handler = read_reset_stat,
},
{
.procname = "rdma_stat_rq_starve",
.data = &rdma_stat_rq_starve,
- .maxlen = sizeof(atomic_t),
+ .maxlen = sizeof(atomic_unchecked_t),
.mode = 0644,
.proc_handler = read_reset_stat,
},
{
.procname = "rdma_stat_rq_poll",
.data = &rdma_stat_rq_poll,
- .maxlen = sizeof(atomic_t),
+ .maxlen = sizeof(atomic_unchecked_t),
.mode = 0644,
.proc_handler = read_reset_stat,
},
{
.procname = "rdma_stat_rq_prod",
.data = &rdma_stat_rq_prod,
- .maxlen = sizeof(atomic_t),
+ .maxlen = sizeof(atomic_unchecked_t),
.mode = 0644,
.proc_handler = read_reset_stat,
},
{
.procname = "rdma_stat_sq_poll",
.data = &rdma_stat_sq_poll,
- .maxlen = sizeof(atomic_t),
+ .maxlen = sizeof(atomic_unchecked_t),
.mode = 0644,
.proc_handler = read_reset_stat,
},
{
.procname = "rdma_stat_sq_prod",
.data = &rdma_stat_sq_prod,
- .maxlen = sizeof(atomic_t),
+ .maxlen = sizeof(atomic_unchecked_t),
.mode = 0644,
.proc_handler = read_reset_stat,
},
*page_no = pg_no;
*page_offset = pg_off;
ret = read;
- atomic_inc(&rdma_stat_read);
+ atomic_inc_unchecked(&rdma_stat_read);
return ret;
err:
svc_rdma_unmap_dma(ctxt);
*page_no = pg_no;
*page_offset = pg_off;
ret = read;
- atomic_inc(&rdma_stat_read);
+ atomic_inc_unchecked(&rdma_stat_read);
return ret;
err:
svc_rdma_unmap_dma(ctxt);
dto_q);
list_del_init(&ctxt->dto_q);
} else {
- atomic_inc(&rdma_stat_rq_starve);
+ atomic_inc_unchecked(&rdma_stat_rq_starve);
clear_bit(XPT_DATA, &xprt->xpt_flags);
ctxt = NULL;
}
dprintk("svcrdma: processing ctxt=%p on xprt=%p, rqstp=%p, status=%d\n",
ctxt, rdma_xprt, rqstp, ctxt->wc_status);
BUG_ON(ctxt->wc_status != IB_WC_SUCCESS);
- atomic_inc(&rdma_stat_recv);
+ atomic_inc_unchecked(&rdma_stat_recv);
/* Build up the XDR from the receive buffers. */
rdma_build_arg_xdr(rqstp, ctxt, ctxt->byte_len);
write_wr.wr.rdma.remote_addr = to;
/* Post It */
- atomic_inc(&rdma_stat_write);
+ atomic_inc_unchecked(&rdma_stat_write);
if (svc_rdma_send(xprt, &write_wr))
goto err;
return write_len - bc;
return;
ib_req_notify_cq(xprt->sc_rq_cq, IB_CQ_NEXT_COMP);
- atomic_inc(&rdma_stat_rq_poll);
+ atomic_inc_unchecked(&rdma_stat_rq_poll);
while ((ret = ib_poll_cq(xprt->sc_rq_cq, 1, &wc)) > 0) {
ctxt = (struct svc_rdma_op_ctxt *)(unsigned long)wc.wr_id;
}
if (ctxt)
- atomic_inc(&rdma_stat_rq_prod);
+ atomic_inc_unchecked(&rdma_stat_rq_prod);
set_bit(XPT_DATA, &xprt->sc_xprt.xpt_flags);
/*
return;
ib_req_notify_cq(xprt->sc_sq_cq, IB_CQ_NEXT_COMP);
- atomic_inc(&rdma_stat_sq_poll);
+ atomic_inc_unchecked(&rdma_stat_sq_poll);
while ((ret = ib_poll_cq(cq, ARRAY_SIZE(wc_a), wc_a)) > 0) {
int i;
}
if (ctxt)
- atomic_inc(&rdma_stat_sq_prod);
+ atomic_inc_unchecked(&rdma_stat_sq_prod);
}
static void sq_comp_handler(struct ib_cq *cq, void *cq_context)
spin_lock_bh(&xprt->sc_lock);
if (xprt->sc_sq_depth < atomic_read(&xprt->sc_sq_count) + wr_count) {
spin_unlock_bh(&xprt->sc_lock);
- atomic_inc(&rdma_stat_sq_starve);
+ atomic_inc_unchecked(&rdma_stat_sq_starve);
/* See if we can opportunistically reap SQ WR to make room */
sq_cq_reap(xprt);
kgid_t root_gid = make_kgid(net->user_ns, 0);
/* Allow network administrator to have same access as root. */
- if (ns_capable(net->user_ns, CAP_NET_ADMIN) ||
+ if (ns_capable_nolog(net->user_ns, CAP_NET_ADMIN) ||
uid_eq(root_uid, current_euid())) {
int mode = (table->mode >> 6) & 7;
return (mode << 6) | (mode << 3) | mode;
struct tipc_subscriber *subscriber = sub->subscriber;
struct kvec msg_sect;
- msg_sect.iov_base = (void *)&sub->evt;
+ msg_sect.iov_base = &sub->evt;
msg_sect.iov_len = sizeof(struct tipc_event);
sub->evt.event = htohl(event, sub->swap);
sub->evt.found_lower = htohl(found_lower, sub->swap);
err = -ECONNREFUSED;
if (!S_ISSOCK(inode->i_mode))
goto put_fail;
+
+ if (!gr_acl_handle_unix(path.dentry, path.mnt)) {
+ err = -EACCES;
+ goto put_fail;
+ }
+
u = unix_find_socket_byinode(inode);
if (!u)
goto put_fail;
if (u) {
struct dentry *dentry;
dentry = unix_sk(u)->path.dentry;
+
+ if (!gr_handle_chroot_unix(pid_vnr(u->sk_peer_pid))) {
+ err = -EPERM;
+ sock_put(u);
+ goto fail;
+ }
+
if (dentry)
touch_atime(&unix_sk(u)->path);
} else
*/
err = security_path_mknod(&path, dentry, mode, 0);
if (!err) {
+ if (!gr_acl_handle_mknod(dentry, path.dentry, path.mnt, mode)) {
+ err = -EACCES;
+ goto out;
+ }
err = vfs_mknod(path.dentry->d_inode, dentry, mode, 0);
if (!err) {
res->mnt = mntget(path.mnt);
res->dentry = dget(dentry);
+ gr_handle_create(dentry, path.mnt);
}
}
+out:
done_path_create(&path, dentry);
return err;
}
writable = unix_writable(sk);
other = unix_peer_get(sk);
if (other) {
- if (unix_peer(other) != sk) {
+ unix_state_lock(other);
+ if (!sock_flag(other, SOCK_DEAD) && unix_peer(other) != sk) {
+ unix_state_unlock(other);
sock_poll_wait(file, &unix_sk(other)->peer_wait, wait);
if (unix_recvq_full(other))
writable = 0;
- }
+ } else
+ unix_state_unlock(other);
sock_put(other);
}
seq_puts(seq, "Num RefCount Protocol Flags Type St "
"Inode Path\n");
else {
- struct sock *s = v;
+ struct sock *s = v, *peer;
struct unix_sock *u = unix_sk(s);
unix_state_lock(s);
+ peer = unix_peer(s);
+ unix_state_unlock(s);
+
+ unix_state_double_lock(s, peer);
seq_printf(seq, "%pK: %08X %08X %08X %04X %02X %5lu",
s,
}
for ( ; i < len; i++)
seq_putc(seq, u->addr->name->sun_path[i]);
- }
- unix_state_unlock(s);
+ } else if (peer)
+ seq_printf(seq, " P%lu", sock_i_ino(peer));
+
+ unix_state_double_unlock(s, peer);
seq_putc(seq, '\n');
}
int __net_init unix_sysctl_register(struct net *net)
{
- struct ctl_table *table;
+ ctl_table_no_const *table;
table = kmemdup(unix_table, sizeof(unix_table), GFP_KERNEL);
if (table == NULL)
/* Socket control packet based operations. */
struct vmci_transport_notify_ops vmci_transport_notify_pkt_ops = {
- vmci_transport_notify_pkt_socket_init,
- vmci_transport_notify_pkt_socket_destruct,
- vmci_transport_notify_pkt_poll_in,
- vmci_transport_notify_pkt_poll_out,
- vmci_transport_notify_pkt_handle_pkt,
- vmci_transport_notify_pkt_recv_init,
- vmci_transport_notify_pkt_recv_pre_block,
- vmci_transport_notify_pkt_recv_pre_dequeue,
- vmci_transport_notify_pkt_recv_post_dequeue,
- vmci_transport_notify_pkt_send_init,
- vmci_transport_notify_pkt_send_pre_block,
- vmci_transport_notify_pkt_send_pre_enqueue,
- vmci_transport_notify_pkt_send_post_enqueue,
- vmci_transport_notify_pkt_process_request,
- vmci_transport_notify_pkt_process_negotiate,
+ .socket_init = vmci_transport_notify_pkt_socket_init,
+ .socket_destruct = vmci_transport_notify_pkt_socket_destruct,
+ .poll_in = vmci_transport_notify_pkt_poll_in,
+ .poll_out = vmci_transport_notify_pkt_poll_out,
+ .handle_notify_pkt = vmci_transport_notify_pkt_handle_pkt,
+ .recv_init = vmci_transport_notify_pkt_recv_init,
+ .recv_pre_block = vmci_transport_notify_pkt_recv_pre_block,
+ .recv_pre_dequeue = vmci_transport_notify_pkt_recv_pre_dequeue,
+ .recv_post_dequeue = vmci_transport_notify_pkt_recv_post_dequeue,
+ .send_init = vmci_transport_notify_pkt_send_init,
+ .send_pre_block = vmci_transport_notify_pkt_send_pre_block,
+ .send_pre_enqueue = vmci_transport_notify_pkt_send_pre_enqueue,
+ .send_post_enqueue = vmci_transport_notify_pkt_send_post_enqueue,
+ .process_request = vmci_transport_notify_pkt_process_request,
+ .process_negotiate = vmci_transport_notify_pkt_process_negotiate,
};
/* Socket always on control packet based operations. */
struct vmci_transport_notify_ops vmci_transport_notify_pkt_q_state_ops = {
- vmci_transport_notify_pkt_socket_init,
- vmci_transport_notify_pkt_socket_destruct,
- vmci_transport_notify_pkt_poll_in,
- vmci_transport_notify_pkt_poll_out,
- vmci_transport_notify_pkt_handle_pkt,
- vmci_transport_notify_pkt_recv_init,
- vmci_transport_notify_pkt_recv_pre_block,
- vmci_transport_notify_pkt_recv_pre_dequeue,
- vmci_transport_notify_pkt_recv_post_dequeue,
- vmci_transport_notify_pkt_send_init,
- vmci_transport_notify_pkt_send_pre_block,
- vmci_transport_notify_pkt_send_pre_enqueue,
- vmci_transport_notify_pkt_send_post_enqueue,
- vmci_transport_notify_pkt_process_request,
- vmci_transport_notify_pkt_process_negotiate,
+ .socket_init = vmci_transport_notify_pkt_socket_init,
+ .socket_destruct = vmci_transport_notify_pkt_socket_destruct,
+ .poll_in = vmci_transport_notify_pkt_poll_in,
+ .poll_out = vmci_transport_notify_pkt_poll_out,
+ .handle_notify_pkt = vmci_transport_notify_pkt_handle_pkt,
+ .recv_init = vmci_transport_notify_pkt_recv_init,
+ .recv_pre_block = vmci_transport_notify_pkt_recv_pre_block,
+ .recv_pre_dequeue = vmci_transport_notify_pkt_recv_pre_dequeue,
+ .recv_post_dequeue = vmci_transport_notify_pkt_recv_post_dequeue,
+ .send_init = vmci_transport_notify_pkt_send_init,
+ .send_pre_block = vmci_transport_notify_pkt_send_pre_block,
+ .send_pre_enqueue = vmci_transport_notify_pkt_send_pre_enqueue,
+ .send_post_enqueue = vmci_transport_notify_pkt_send_post_enqueue,
+ .process_request = vmci_transport_notify_pkt_process_request,
+ .process_negotiate = vmci_transport_notify_pkt_process_negotiate,
};
*/
/* Support for very large requests */
- if ((descr->flags & IW_DESCR_FLAG_NOMAX) &&
- (user_length > descr->max_tokens)) {
+ if (user_length > descr->max_tokens) {
/* Allow userspace to GET more than max so
* we can support any size GET requests.
* There is still a limit : -ENOMEM.
}
}
- if (IW_IS_GET(cmd) && !(descr->flags & IW_DESCR_FLAG_NOMAX)) {
- /*
- * If this is a GET, but not NOMAX, it means that the extra
- * data is not bounded by userspace, but by max_tokens. Thus
- * set the length to max_tokens. This matches the extra data
- * allocation.
- * The driver should fill it with the number of tokens it
- * provided, and it may check iwp->length rather than having
- * knowledge of max_tokens. If the driver doesn't change the
- * iwp->length, this ioctl just copies back max_token tokens
- * filled with zeroes. Hopefully the driver isn't claiming
- * them to be valid data.
- */
- iwp->length = descr->max_tokens;
- }
-
err = handler(dev, info, (union iwreq_data *) iwp, extra);
iwp->length += essid_compat;
.mode = 0644,
.proc_handler = proc_dointvec,
},
- { 0, },
+ { },
};
void __init x25_register_sysctl(void)
int __init x25_proc_init(void)
{
- if (!proc_mkdir("x25", init_net.proc_net))
+ if (!proc_mkdir_restrict("x25", init_net.proc_net))
return -ENOMEM;
if (!proc_create("x25/route", S_IRUGO, init_net.proc_net,
{
policy->walk.dead = 1;
- atomic_inc(&policy->genid);
+ atomic_inc_unchecked(&policy->genid);
if (del_timer(&policy->polq.hold_timer))
xfrm_pol_put(policy);
else
hlist_add_head(&policy->bydst, chain);
__xfrm_policy_link(policy, dir);
- atomic_inc(&net->xfrm.flow_cache_genid);
+ atomic_inc_unchecked(&net->xfrm.flow_cache_genid);
/* After previous checking, family can either be AF_INET or AF_INET6 */
if (policy->family == AF_INET)
xdst->num_pols = num_pols;
memcpy(xdst->pols, pols, sizeof(struct xfrm_policy *) * num_pols);
- xdst->policy_genid = atomic_read(&pols[0]->genid);
+ xdst->policy_genid = atomic_read_unchecked(&pols[0]->genid);
return xdst;
}
}
EXPORT_SYMBOL(xfrm_garbage_collect);
-static void xfrm_garbage_collect_deferred(struct net *net)
+void xfrm_garbage_collect_deferred(struct net *net)
{
flow_cache_flush_deferred(net);
}
+EXPORT_SYMBOL(xfrm_garbage_collect_deferred);
static void xfrm_init_pmtu(struct dst_entry *dst)
{
if (xdst->xfrm_genid != dst->xfrm->genid)
return 0;
if (xdst->num_pols > 0 &&
- xdst->policy_genid != atomic_read(&xdst->pols[0]->genid))
+ xdst->policy_genid != atomic_read_unchecked(&xdst->pols[0]->genid))
return 0;
mtu = dst_mtu(dst->child);
dst_ops->link_failure = xfrm_link_failure;
if (likely(dst_ops->neigh_lookup == NULL))
dst_ops->neigh_lookup = xfrm_neigh_lookup;
- if (likely(afinfo->garbage_collect == NULL))
- afinfo->garbage_collect = xfrm_garbage_collect_deferred;
rcu_assign_pointer(xfrm_policy_afinfo[afinfo->family], afinfo);
}
spin_unlock(&xfrm_policy_afinfo_lock);
dst_ops->check = NULL;
dst_ops->negative_advice = NULL;
dst_ops->link_failure = NULL;
- afinfo->garbage_collect = NULL;
}
return err;
}
sizeof(pol->xfrm_vec[i].saddr));
pol->xfrm_vec[i].encap_family = mp->new_family;
/* flush bundles */
- atomic_inc(&pol->genid);
+ atomic_inc_unchecked(&pol->genid);
}
}
if (unlikely(afinfo == NULL))
return -EAFNOSUPPORT;
- typemap = afinfo->type_map;
+ typemap = (const struct xfrm_type **)afinfo->type_map;
spin_lock_bh(&xfrm_type_lock);
- if (likely(typemap[type->proto] == NULL))
+ if (likely(typemap[type->proto] == NULL)) {
+ pax_open_kernel();
typemap[type->proto] = type;
- else
+ pax_close_kernel();
+ } else
err = -EEXIST;
spin_unlock_bh(&xfrm_type_lock);
xfrm_state_put_afinfo(afinfo);
if (unlikely(afinfo == NULL))
return -EAFNOSUPPORT;
- typemap = afinfo->type_map;
+ typemap = (const struct xfrm_type **)afinfo->type_map;
spin_lock_bh(&xfrm_type_lock);
if (unlikely(typemap[type->proto] != type))
err = -ENOENT;
- else
+ else {
+ pax_open_kernel();
typemap[type->proto] = NULL;
+ pax_close_kernel();
+ }
spin_unlock_bh(&xfrm_type_lock);
xfrm_state_put_afinfo(afinfo);
return err;
static const struct xfrm_type *xfrm_get_type(u8 proto, unsigned short family)
{
struct xfrm_state_afinfo *afinfo;
- const struct xfrm_type **typemap;
const struct xfrm_type *type;
int modload_attempted = 0;
afinfo = xfrm_state_get_afinfo(family);
if (unlikely(afinfo == NULL))
return NULL;
- typemap = afinfo->type_map;
- type = typemap[proto];
+ type = afinfo->type_map[proto];
if (unlikely(type && !try_module_get(type->owner)))
type = NULL;
if (!type && !modload_attempted) {
return -EAFNOSUPPORT;
err = -EEXIST;
- modemap = afinfo->mode_map;
+ modemap = (struct xfrm_mode **)afinfo->mode_map;
spin_lock_bh(&xfrm_mode_lock);
if (modemap[mode->encap])
goto out;
if (!try_module_get(afinfo->owner))
goto out;
- mode->afinfo = afinfo;
+ pax_open_kernel();
+ *(const void **)&mode->afinfo = afinfo;
modemap[mode->encap] = mode;
+ pax_close_kernel();
err = 0;
out:
return -EAFNOSUPPORT;
err = -ENOENT;
- modemap = afinfo->mode_map;
+ modemap = (struct xfrm_mode **)afinfo->mode_map;
spin_lock_bh(&xfrm_mode_lock);
if (likely(modemap[mode->encap] == mode)) {
+ pax_open_kernel();
modemap[mode->encap] = NULL;
+ pax_close_kernel();
module_put(mode->afinfo->owner);
err = 0;
}
u32 xfrm_get_acqseq(void)
{
u32 res;
- static atomic_t acqseq;
+ static atomic_unchecked_t acqseq;
do {
- res = atomic_inc_return(&acqseq);
+ res = atomic_inc_return_unchecked(&acqseq);
} while (!res);
return res;
int __net_init xfrm_sysctl_init(struct net *net)
{
- struct ctl_table *table;
+ ctl_table_no_const *table;
__xfrm_sysctl_init(net);
# cc-ldoption
# Usage: ldflags += $(call cc-ldoption, -Wl$(comma)--hash-style=both)
cc-ldoption = $(call try-run,\
- $(CC) $(1) -nostdlib -x c /dev/null -o "$$TMP",$(1),$(2))
+ $(CC) $(1) -Wl,-r -nostdlib -x c /dev/null -o "$$TMP",$(1),$(2))
# ld-option
# Usage: LDFLAGS += $(call ld-option, -X)
endif
# Do not include host rules unless needed
-ifneq ($(hostprogs-y)$(hostprogs-m),)
+ifneq ($(hostprogs-y)$(hostprogs-m)$(hostlibs-y)$(hostlibs-m)$(hostcxxlibs-y)$(hostcxxlibs-m),)
include scripts/Makefile.host
endif
__clean-files := $(extra-y) $(extra-m) $(extra-) \
$(always) $(targets) $(clean-files) \
$(host-progs) \
- $(hostprogs-y) $(hostprogs-m) $(hostprogs-)
+ $(hostprogs-y) $(hostprogs-m) $(hostprogs-) \
+ $(hostlibs-y) $(hostlibs-m) $(hostlibs-)
__clean-files := $(filter-out $(no-clean-files), $(__clean-files))
# Will compile qconf as a C++ program, and menu as a C program.
# They are linked as C++ code to the executable qconf
+# hostprogs-y := conf
+# conf-objs := conf.o libkconfig.so
+# libkconfig-objs := expr.o type.o
+# Will create a shared library named libkconfig.so that consists of
+# expr.o and type.o (they are both compiled as C code and the object files
+# are made as position independent code).
+# conf.c is compiled as a C program, and conf.o is linked together with
+# libkconfig.so as the executable conf.
+# Note: Shared libraries consisting of C++ files are not supported
+
__hostprogs := $(sort $(hostprogs-y) $(hostprogs-m))
+__hostlibs := $(sort $(hostlibs-y) $(hostlibs-m))
+__hostcxxlibs := $(sort $(hostcxxlibs-y) $(hostcxxlibs-m))
# C code
# Executables compiled from a single .c file
# C++ Object (.o) files compiled from .cc files
host-cxxobjs := $(sort $(foreach m,$(host-cxxmulti),$($(m)-cxxobjs)))
+# Shared libaries (only .c supported)
+# Shared libraries (.so) - all .so files referenced in "xxx-objs"
+host-cshlib := $(sort $(filter %.so, $(host-cobjs)))
+host-cshlib += $(sort $(filter %.so, $(__hostlibs)))
+host-cxxshlib := $(sort $(filter %.so, $(__hostcxxlibs)))
+# Remove .so files from "xxx-objs"
+host-cobjs := $(filter-out %.so,$(host-cobjs))
+host-cxxobjs := $(filter-out %.so,$(host-cxxobjs))
+
+# Object (.o) files used by the shared libaries
+host-cshobjs := $(sort $(foreach m,$(host-cshlib),$($(m:.so=-objs))))
+host-cxxshobjs := $(sort $(foreach m,$(host-cxxshlib),$($(m:.so=-objs))))
+
# output directory for programs/.o files
# hostprogs-y := tools/build may have been specified.
# Retrieve also directory of .o files from prog-objs or prog-cxxobjs notation
host-cobjs := $(addprefix $(obj)/,$(host-cobjs))
host-cxxmulti := $(addprefix $(obj)/,$(host-cxxmulti))
host-cxxobjs := $(addprefix $(obj)/,$(host-cxxobjs))
+host-cshlib := $(addprefix $(obj)/,$(host-cshlib))
+host-cxxshlib := $(addprefix $(obj)/,$(host-cxxshlib))
+host-cshobjs := $(addprefix $(obj)/,$(host-cshobjs))
+host-cxxshobjs := $(addprefix $(obj)/,$(host-cxxshobjs))
host-objdirs := $(addprefix $(obj)/,$(host-objdirs))
obj-dirs += $(host-objdirs)
$(host-cxxobjs): $(obj)/%.o: $(src)/%.cc FORCE
$(call if_changed_dep,host-cxxobjs)
+# Compile .c file, create position independent .o file
+# host-cshobjs -> .o
+quiet_cmd_host-cshobjs = HOSTCC -fPIC $@
+ cmd_host-cshobjs = $(HOSTCC) $(hostc_flags) -fPIC -c -o $@ $<
+$(host-cshobjs): $(obj)/%.o: $(src)/%.c FORCE
+ $(call if_changed_dep,host-cshobjs)
+
+# Compile .c file, create position independent .o file
+# host-cxxshobjs -> .o
+quiet_cmd_host-cxxshobjs = HOSTCXX -fPIC $@
+ cmd_host-cxxshobjs = $(HOSTCXX) $(hostcxx_flags) -fPIC -c -o $@ $<
+$(host-cxxshobjs): $(obj)/%.o: $(src)/%.c FORCE
+ $(call if_changed_dep,host-cxxshobjs)
+
+# Link a shared library, based on position independent .o files
+# *.o -> .so shared library (host-cshlib)
+quiet_cmd_host-cshlib = HOSTLLD -shared $@
+ cmd_host-cshlib = $(HOSTCC) $(HOSTLDFLAGS) -shared -o $@ \
+ $(addprefix $(obj)/,$($(@F:.so=-objs))) \
+ $(HOST_LOADLIBES) $(HOSTLOADLIBES_$(@F))
+$(host-cshlib): $(obj)/%: $(host-cshobjs) FORCE
+ $(call if_changed,host-cshlib)
+
+# Link a shared library, based on position independent .o files
+# *.o -> .so shared library (host-cxxshlib)
+quiet_cmd_host-cxxshlib = HOSTLLD -shared $@
+ cmd_host-cxxshlib = $(HOSTCXX) $(HOSTLDFLAGS) -shared -o $@ \
+ $(addprefix $(obj)/,$($(@F:.so=-objs))) \
+ $(HOST_LOADLIBES) $(HOSTLOADLIBES_$(@F))
+$(host-cxxshlib): $(obj)/%: $(host-cxxshobjs) FORCE
+ $(call if_changed,host-cxxshlib)
+
targets += $(host-csingle) $(host-cmulti) $(host-cobjs)\
- $(host-cxxmulti) $(host-cxxobjs)
+ $(host-cxxmulti) $(host-cxxobjs) $(host-cshlib) $(host-cshobjs) $(host-cxxshlib) $(host-cxxshobjs)
/*
* Lookup a value in the configuration string.
*/
-static int is_defined_config(const char *name, int len, unsigned int hash)
+static int is_defined_config(const char *name, unsigned int len, unsigned int hash)
{
struct item *aux;
/*
* Record the use of a CONFIG_* word.
*/
-static void use_config(const char *m, int slen)
+static void use_config(const char *m, unsigned int slen)
{
unsigned int hash = strhash(m, slen);
- int c, i;
+ unsigned int c, i;
if (is_defined_config(m, slen, hash))
return;
static void parse_config_file(const char *map, size_t len)
{
- const int *end = (const int *) (map + len);
+ const unsigned int *end = (const unsigned int *) (map + len);
/* start at +1, so that p can never be < map */
- const int *m = (const int *) map + 1;
+ const unsigned int *m = (const unsigned int *) map + 1;
const char *p, *q;
for (; m < end; m++) {
static void traps(void)
{
static char test[] __attribute__((aligned(sizeof(int)))) = "CONF";
- int *p = (int *)test;
+ unsigned int *p = (unsigned int *)test;
if (*p != INT_CONF) {
fprintf(stderr, "fixdep: sizeof(int) != 4 or wrong endianness? %#x\n",
FILE="$(basename "$i")"
sed -r \
-e 's/([ \t(])(__user|__force|__iomem)[ \t]/\1/g' \
+ -e 's/__intentional_overflow\([- \t,0-9]*\)//g' \
-e 's/__attribute_const__([ \t]|$)/\1/g' \
-e 's@^#include <linux/compiler.h>@@' \
-e 's/(^|[^a-zA-Z0-9])__packed([^a-zA-Z0-9_]|$)/\1__attribute__((packed))\2/g' \
fi;
# final build of init/
-${MAKE} -f "${srctree}/scripts/Makefile.build" obj=init
+${MAKE} -f "${srctree}/scripts/Makefile.build" obj=init GCC_PLUGINS_CFLAGS="${GCC_PLUGINS_CFLAGS}" GCC_PLUGINS_AFLAGS="${GCC_PLUGINS_AFLAGS}"
kallsymso=""
kallsyms_vmlinux=""
unsigned long size, unsigned long id_size,
void *symval)
{
- int i;
+ unsigned int i;
if (size % id_size || size < id_size) {
fatal("%s: sizeof(struct %s_device_id)=%lu is not a modulo "
/* USB is special because the bcdDevice can be matched against a numeric range */
/* Looks like "usb:vNpNdNdcNdscNdpNicNiscNipNinN" */
static void do_usb_entry(void *symval,
- unsigned int bcdDevice_initial, int bcdDevice_initial_digits,
+ unsigned int bcdDevice_initial, unsigned int bcdDevice_initial_digits,
unsigned char range_lo, unsigned char range_hi,
unsigned char max, struct module *mod)
{
{
unsigned int devlo, devhi;
unsigned char chi, clo, max;
- int ndigits;
+ unsigned int ndigits;
DEF_FIELD(symval, usb_device_id, match_flags);
DEF_FIELD(symval, usb_device_id, idVendor);
for (i = 0; i < count; i++) {
DEF_FIELD_ADDR(symval + i*id_size, pnp_device_id, id);
char acpi_id[sizeof(*id)];
- int j;
+ unsigned int j;
buf_printf(&mod->dev_table_buf,
"MODULE_ALIAS(\"pnp:d%s*\");\n", *id);
for (j = 0; j < PNP_MAX_DEVICES; j++) {
const char *id = (char *)(*devs)[j].id;
- int i2, j2;
+ unsigned int i2, j2;
int dup = 0;
if (!id[0])
/* add an individual alias for every device entry */
if (!dup) {
char acpi_id[PNP_ID_LEN];
- int k;
+ unsigned int k;
buf_printf(&mod->dev_table_buf,
"MODULE_ALIAS(\"pnp:d%s*\");\n", id);
static int do_dmi_entry(const char *filename, void *symval,
char *alias)
{
- int i, j;
+ unsigned int i, j;
DEF_FIELD_ADDR(symval, dmi_system_id, matches);
sprintf(alias, "dmi*");
ANY_INIT_TO_ANY_EXIT,
ANY_EXIT_TO_ANY_INIT,
EXPORT_TO_INIT_EXIT,
+ DATA_TO_TEXT
};
struct sectioncheck {
.tosec = { INIT_SECTIONS, EXIT_SECTIONS, NULL },
.mismatch = EXPORT_TO_INIT_EXIT,
.symbol_white_list = { DEFAULT_SYMBOL_WHITE_LIST, NULL },
+},
+/* Do not reference code from writable data */
+{
+ .fromsec = { DATA_SECTIONS, NULL },
+ .tosec = { TEXT_SECTIONS, NULL },
+ .mismatch = DATA_TO_TEXT
}
};
continue;
if (ELF_ST_TYPE(sym->st_info) == STT_SECTION)
continue;
- if (sym->st_value == addr)
- return sym;
/* Find a symbol nearby - addr are maybe negative */
d = sym->st_value - addr;
+ if (d == 0)
+ return sym;
if (d < 0)
d = addr - sym->st_value;
if (d < distance) {
tosym, prl_to, prl_to, tosym);
free(prl_to);
break;
+ case DATA_TO_TEXT:
+#if 0
+ fprintf(stderr,
+ "The %s %s:%s references\n"
+ "the %s %s:%s%s\n",
+ from, fromsec, fromsym, to, tosec, tosym, to_p);
+#endif
+ break;
}
fprintf(stderr, "\n");
}
static void check_sec_ref(struct module *mod, const char *modname,
struct elf_info *elf)
{
- int i;
+ unsigned int i;
Elf_Shdr *sechdrs = elf->sechdrs;
/* Walk through all sections */
va_end(ap);
}
-void buf_write(struct buffer *buf, const char *s, int len)
+void buf_write(struct buffer *buf, const char *s, unsigned int len)
{
if (buf->size - buf->pos < len) {
buf->size += len + SZ;
if (fstat(fileno(file), &st) < 0)
goto close_write;
- if (st.st_size != b->pos)
+ if (st.st_size != (off_t)b->pos)
goto close_write;
tmp = NOFAIL(malloc(b->pos));
struct buffer {
char *p;
- int pos;
- int size;
+ unsigned int pos;
+ unsigned int size;
};
void __attribute__((format(printf, 2, 3)))
buf_printf(struct buffer *buf, const char *fmt, ...);
void
-buf_write(struct buffer *buf, const char *s, int len);
+buf_write(struct buffer *buf, const char *s, unsigned int len);
struct module {
struct module *next;
goto out;
}
- if (write(fd, sum, strlen(sum)+1) != strlen(sum)+1) {
+ if (write(fd, sum, strlen(sum)+1) != (ssize_t)strlen(sum)+1) {
warn("writing sum in %s failed: %s\n",
filename, strerror(errno));
goto out;
SECTIONS {
/DISCARD/ : { *(.discard) }
+ .rodata : {
+ *(.rodata) *(.rodata.*)
+ *(.data..read_only)
+ }
__ksymtab : { *(SORT(___ksymtab+*)) }
__ksymtab_gpl : { *(SORT(___ksymtab_gpl+*)) }
__ksymtab_unused : { *(SORT(___ksymtab_unused+*)) }
ln -sf $(srctree) $(KERNELPATH)
$(CONFIG_SHELL) $(MKSPEC) >$(objtree)/kernel.spec
$(CONFIG_SHELL) $(srctree)/scripts/setlocalversion --save-scmversion
- tar -cz $(RCS_TAR_IGNORE) -f $(KERNELPATH).tar.gz $(TAR_CONTENT)
+ tar --owner=root --group=root -cz $(RCS_TAR_IGNORE) -f $(KERNELPATH).tar.gz $(TAR_CONTENT)
rm $(KERNELPATH)
rm -f $(objtree)/.scmversion
$(CONFIG_SHELL) $(srctree)/scripts/mkversion > $(objtree)/.tmp_version
(cd $srctree; find arch/$SRCARCH -name module.lds -o -name Kbuild.platforms -o -name Platform) >> "$objtree/debian/hdrsrcfiles"
(cd $srctree; find $(find arch/$SRCARCH -name include -o -name scripts -type d) -type f) >> "$objtree/debian/hdrsrcfiles"
(cd $objtree; find arch/$SRCARCH/include Module.symvers include scripts -type f) >> "$objtree/debian/hdrobjfiles"
+(cd $objtree; find tools/gcc -name \*.so >> "$objtree/debian/hdrobjfiles")
destdir=$kernel_headers_dir/usr/src/linux-headers-$version
mkdir -p "$destdir"
(cd $srctree; tar -c -f - -T -) < "$objtree/debian/hdrsrcfiles" | (cd $destdir; tar -xf -)
echo "mkdir -p "'$RPM_BUILD_ROOT'"/usr/src/kernels/$KERNELRELEASE"
echo "EXCLUDES=\"$RCS_TAR_IGNORE --exclude .tmp_versions --exclude=*vmlinux* --exclude=*.o --exclude=*.ko --exclude=*.cmd --exclude=Documentation --exclude=firmware --exclude .config.old --exclude .missing-syscalls.d\""
echo "tar "'$EXCLUDES'" -cf- . | (cd "'$RPM_BUILD_ROOT'"/usr/src/kernels/$KERNELRELEASE;tar xvf -)"
-echo 'cd $RPM_BUILD_ROOT'"/lib/modules/$KERNELRELEASE"
-echo "ln -sf /usr/src/kernels/$KERNELRELEASE build"
-echo "ln -sf /usr/src/kernels/$KERNELRELEASE source"
fi
echo ""
echo "%clean"
echo 'rm -rf $RPM_BUILD_ROOT'
echo ""
+echo "%pre"
+echo 'chmod -f 0500 /boot'
+echo 'if [ -d /lib/modules ]; then'
+echo 'chmod -f 0500 /lib/modules'
+echo 'fi'
+echo 'if [ -d /lib32/modules ]; then'
+echo 'chmod -f 0500 /lib32/modules'
+echo 'fi'
+echo 'if [ -d /lib64/modules ]; then'
+echo 'chmod -f 0500 /lib64/modules'
+echo 'fi'
+echo ""
+echo "%post devel"
+echo "ln -sf /usr/src/kernels/$KERNELRELEASE /lib/modules/$KERNELRELEASE/build"
+echo "ln -sf /usr/src/kernels/$KERNELRELEASE /lib/modules/$KERNELRELEASE/source"
+echo ""
echo "%post"
-echo "if [ -x /sbin/installkernel -a -r /boot/vmlinuz-$KERNELRELEASE -a -r /boot/System.map-$KERNELRELEASE ]; then"
-echo "cp /boot/vmlinuz-$KERNELRELEASE /boot/vmlinuz-$KERNELRELEASE-rpm"
-echo "cp /boot/System.map-$KERNELRELEASE /boot/System.map-$KERNELRELEASE-rpm"
-echo "rm -f /boot/vmlinuz-$KERNELRELEASE /boot/System.map-$KERNELRELEASE"
-echo "/sbin/installkernel $KERNELRELEASE /boot/vmlinuz-$KERNELRELEASE-rpm /boot/System.map-$KERNELRELEASE-rpm"
-echo "rm -f /boot/vmlinuz-$KERNELRELEASE-rpm /boot/System.map-$KERNELRELEASE-rpm"
+echo "if [ -x /sbin/dracut ]; then"
+echo '/sbin/new-kernel-pkg --dracut --mkinitrd --depmod --install --make-default '"$KERNELRELEASE"' || exit $?'
+echo "else"
+echo '/sbin/new-kernel-pkg --mkinitrd --depmod --install --make-default '"$KERNELRELEASE"' || exit $?'
echo "fi"
echo ""
echo "%files"
-echo '%defattr (-, root, root)'
+echo '%defattr (400, root, root, 500)'
echo "%dir /lib/modules"
-echo "/lib/modules/$KERNELRELEASE"
echo "%exclude /lib/modules/$KERNELRELEASE/build"
echo "%exclude /lib/modules/$KERNELRELEASE/source"
+echo "/lib/modules/$KERNELRELEASE"
echo "/lib/firmware/$KERNELRELEASE"
echo "/boot/*"
echo ""
echo ""
if ! $PREBUILT; then
echo "%files devel"
-echo '%defattr (-, root, root)'
+echo '%defattr (400, root, root, 500)'
+echo "%dir /lib/modules/$KERNELRELEASE"
echo "/usr/src/kernels/$KERNELRELEASE"
-echo "/lib/modules/$KERNELRELEASE/build"
-echo "/lib/modules/$KERNELRELEASE/source"
echo ""
fi
fprintf(out, " * Linux logo %s\n", logoname);
fputs(" */\n\n", out);
fputs("#include <linux/linux_logo.h>\n\n", out);
- fprintf(out, "static unsigned char %s_data[] __initdata = {\n",
+ fprintf(out, "static unsigned char %s_data[] = {\n",
logoname);
}
static void write_footer(void)
{
fputs("\n};\n\n", out);
- fprintf(out, "const struct linux_logo %s __initconst = {\n", logoname);
+ fprintf(out, "const struct linux_logo %s = {\n", logoname);
fprintf(out, "\t.type\t\t= %s,\n", logo_types[logo_type]);
fprintf(out, "\t.width\t\t= %d,\n", logo_width);
fprintf(out, "\t.height\t\t= %d,\n", logo_height);
fputs("\n};\n\n", out);
/* write logo clut */
- fprintf(out, "static unsigned char %s_clut[] __initdata = {\n",
+ fprintf(out, "static unsigned char %s_clut[] = {\n",
logoname);
write_hex_cnt = 0;
for (i = 0; i < logo_clutsize; i++) {
const char *secstrtab;
const char *strtab;
char *extab_image;
- int extab_index = 0;
- int i;
- int idx;
+ unsigned int extab_index = 0;
+ unsigned int i;
+ unsigned int idx;
unsigned int num_sections;
unsigned int secindex_strings;
fi
# ignore userspace tools
-ignore="$ignore ( -path ${tree}tools ) -prune -o"
+ignore="$ignore ( -path \"${tree}tools/[^g]*\" ) -prune -o"
# Find all available archs
find_all_archs()
menu "Security options"
+menu "Grsecurity"
+
+ config ARCH_TRACK_EXEC_LIMIT
+ bool
+
+ config PAX_KERNEXEC_PLUGIN
+ bool
+
+ config PAX_PER_CPU_PGD
+ bool
+
+ config TASK_SIZE_MAX_SHIFT
+ int
+ depends on X86_64
+ default 47 if !PAX_PER_CPU_PGD
+ default 42 if PAX_PER_CPU_PGD
+
+ config PAX_ENABLE_PAE
+ bool
+ default y if (X86_32 && (MPENTIUM4 || MK8 || MPSC || MCORE2 || MATOM))
+
+ config PAX_USERCOPY_SLABS
+ bool
+
+config GRKERNSEC
+ bool "Grsecurity"
+ select CRYPTO
+ select CRYPTO_SHA256
+ select PROC_FS
+ select STOP_MACHINE
+ select TTY
+ select DEBUG_KERNEL
+ select DEBUG_LIST
+ help
+ If you say Y here, you will be able to configure many features
+ that will enhance the security of your system. It is highly
+ recommended that you say Y here and read through the help
+ for each option so that you fully understand the features and
+ can evaluate their usefulness for your machine.
+
+choice
+ prompt "Configuration Method"
+ depends on GRKERNSEC
+ default GRKERNSEC_CONFIG_CUSTOM
+ help
+
+config GRKERNSEC_CONFIG_AUTO
+ bool "Automatic"
+ help
+ If you choose this configuration method, you'll be able to answer a small
+ number of simple questions about how you plan to use this kernel.
+ The settings of grsecurity and PaX will be automatically configured for
+ the highest commonly-used settings within the provided constraints.
+
+ If you require additional configuration, custom changes can still be made
+ from the "custom configuration" menu.
+
+config GRKERNSEC_CONFIG_CUSTOM
+ bool "Custom"
+ help
+ If you choose this configuration method, you'll be able to configure all
+ grsecurity and PaX settings manually. Via this method, no options are
+ automatically enabled.
+
+endchoice
+
+choice
+ prompt "Usage Type"
+ depends on (GRKERNSEC && GRKERNSEC_CONFIG_AUTO)
+ default GRKERNSEC_CONFIG_SERVER
+ help
+
+config GRKERNSEC_CONFIG_SERVER
+ bool "Server"
+ help
+ Choose this option if you plan to use this kernel on a server.
+
+config GRKERNSEC_CONFIG_DESKTOP
+ bool "Desktop"
+ help
+ Choose this option if you plan to use this kernel on a desktop.
+
+endchoice
+
+choice
+ prompt "Virtualization Type"
+ depends on (GRKERNSEC && X86 && GRKERNSEC_CONFIG_AUTO)
+ default GRKERNSEC_CONFIG_VIRT_NONE
+ help
+
+config GRKERNSEC_CONFIG_VIRT_NONE
+ bool "None"
+ help
+ Choose this option if this kernel will be run on bare metal.
+
+config GRKERNSEC_CONFIG_VIRT_GUEST
+ bool "Guest"
+ help
+ Choose this option if this kernel will be run as a VM guest.
+
+config GRKERNSEC_CONFIG_VIRT_HOST
+ bool "Host"
+ help
+ Choose this option if this kernel will be run as a VM host.
+
+endchoice
+
+choice
+ prompt "Virtualization Hardware"
+ depends on (GRKERNSEC && X86 && GRKERNSEC_CONFIG_AUTO && (GRKERNSEC_CONFIG_VIRT_GUEST || GRKERNSEC_CONFIG_VIRT_HOST))
+ help
+
+config GRKERNSEC_CONFIG_VIRT_EPT
+ bool "EPT/RVI Processor Support"
+ depends on X86
+ help
+ Choose this option if your CPU supports the EPT or RVI features of 2nd-gen
+ hardware virtualization. This allows for additional kernel hardening protections
+ to operate without additional performance impact.
+
+ To see if your Intel processor supports EPT, see:
+ http://ark.intel.com/Products/VirtualizationTechnology
+ (Most Core i3/5/7 support EPT)
+
+ To see if your AMD processor supports RVI, see:
+ http://support.amd.com/us/kbarticles/Pages/GPU120AMDRVICPUsHyperVWin8.aspx
+
+config GRKERNSEC_CONFIG_VIRT_SOFT
+ bool "First-gen/No Hardware Virtualization"
+ help
+ Choose this option if you use an Atom/Pentium/Core 2 processor that either doesn't
+ support hardware virtualization or doesn't support the EPT/RVI extensions.
+
+endchoice
+
+choice
+ prompt "Virtualization Software"
+ depends on (GRKERNSEC && GRKERNSEC_CONFIG_AUTO && (GRKERNSEC_CONFIG_VIRT_GUEST || GRKERNSEC_CONFIG_VIRT_HOST))
+ help
+
+config GRKERNSEC_CONFIG_VIRT_XEN
+ bool "Xen"
+ help
+ Choose this option if this kernel is running as a Xen guest or host.
+
+config GRKERNSEC_CONFIG_VIRT_VMWARE
+ bool "VMWare"
+ help
+ Choose this option if this kernel is running as a VMWare guest or host.
+
+config GRKERNSEC_CONFIG_VIRT_KVM
+ bool "KVM"
+ help
+ Choose this option if this kernel is running as a KVM guest or host.
+
+config GRKERNSEC_CONFIG_VIRT_VIRTUALBOX
+ bool "VirtualBox"
+ help
+ Choose this option if this kernel is running as a VirtualBox guest or host.
+
+config GRKERNSEC_CONFIG_VIRT_HYPERV
+ bool "Hyper-V"
+ help
+ Choose this option if this kernel is running as a Hyper-V guest.
+
+endchoice
+
+choice
+ prompt "Required Priorities"
+ depends on (GRKERNSEC && GRKERNSEC_CONFIG_AUTO)
+ default GRKERNSEC_CONFIG_PRIORITY_PERF
+ help
+
+config GRKERNSEC_CONFIG_PRIORITY_PERF
+ bool "Performance"
+ help
+ Choose this option if performance is of highest priority for this deployment
+ of grsecurity. Features like UDEREF on a 64bit kernel, kernel stack clearing,
+ clearing of structures intended for userland, and freed memory sanitizing will
+ be disabled.
+
+config GRKERNSEC_CONFIG_PRIORITY_SECURITY
+ bool "Security"
+ help
+ Choose this option if security is of highest priority for this deployment of
+ grsecurity. UDEREF, kernel stack clearing, clearing of structures intended
+ for userland, and freed memory sanitizing will be enabled for this kernel.
+ In a worst-case scenario, these features can introduce a 20% performance hit
+ (UDEREF on x64 contributing half of this hit).
+
+endchoice
+
+menu "Default Special Groups"
+depends on (GRKERNSEC && GRKERNSEC_CONFIG_AUTO)
+
+config GRKERNSEC_PROC_GID
+ int "GID exempted from /proc restrictions"
+ default 1001
+ help
+ Setting this GID determines which group will be exempted from
+ grsecurity's /proc restrictions, allowing users of the specified
+ group to view network statistics and the existence of other users'
+ processes on the system. This GID may also be chosen at boot time
+ via "grsec_proc_gid=" on the kernel commandline.
+
+config GRKERNSEC_TPE_UNTRUSTED_GID
+ int "GID for TPE-untrusted users"
+ depends on GRKERNSEC_CONFIG_SERVER && GRKERNSEC_TPE && !GRKERNSEC_TPE_INVERT
+ default 1005
+ help
+ Setting this GID determines which group untrusted users should
+ be added to. These users will be placed under grsecurity's Trusted Path
+ Execution mechanism, preventing them from executing their own binaries.
+ The users will only be able to execute binaries in directories owned and
+ writable only by the root user. If the sysctl option is enabled, a sysctl
+ option with name "tpe_gid" is created.
+
+config GRKERNSEC_TPE_TRUSTED_GID
+ int "GID for TPE-trusted users"
+ depends on GRKERNSEC_CONFIG_SERVER && GRKERNSEC_TPE && GRKERNSEC_TPE_INVERT
+ default 1005
+ help
+ Setting this GID determines what group TPE restrictions will be
+ *disabled* for. If the sysctl option is enabled, a sysctl option
+ with name "tpe_gid" is created.
+
+config GRKERNSEC_SYMLINKOWN_GID
+ int "GID for users with kernel-enforced SymlinksIfOwnerMatch"
+ depends on GRKERNSEC_CONFIG_SERVER
+ default 1006
+ help
+ Setting this GID determines what group kernel-enforced
+ SymlinksIfOwnerMatch will be enabled for. If the sysctl option
+ is enabled, a sysctl option with name "symlinkown_gid" is created.
+
+
+endmenu
+
+menu "Customize Configuration"
+depends on GRKERNSEC
+
+menu "PaX"
+
+config PAX
+ bool "Enable various PaX features"
+ default y if GRKERNSEC_CONFIG_AUTO
+ depends on GRKERNSEC && (ALPHA || ARM || AVR32 || IA64 || MIPS || PARISC || PPC || SPARC || X86)
+ help
+ This allows you to enable various PaX features. PaX adds
+ intrusion prevention mechanisms to the kernel that reduce
+ the risks posed by exploitable memory corruption bugs.
+
+menu "PaX Control"
+ depends on PAX
+
+config PAX_SOFTMODE
+ bool 'Support soft mode'
+ help
+ Enabling this option will allow you to run PaX in soft mode, that
+ is, PaX features will not be enforced by default, only on executables
+ marked explicitly. You must also enable PT_PAX_FLAGS or XATTR_PAX_FLAGS
+ support as they are the only way to mark executables for soft mode use.
+
+ Soft mode can be activated by using the "pax_softmode=1" kernel command
+ line option on boot. Furthermore you can control various PaX features
+ at runtime via the entries in /proc/sys/kernel/pax.
+
+config PAX_EI_PAX
+ bool 'Use legacy ELF header marking'
+ default y if GRKERNSEC_CONFIG_AUTO
+ help
+ Enabling this option will allow you to control PaX features on
+ a per executable basis via the 'chpax' utility available at
+ http://pax.grsecurity.net/. The control flags will be read from
+ an otherwise reserved part of the ELF header. This marking has
+ numerous drawbacks (no support for soft-mode, toolchain does not
+ know about the non-standard use of the ELF header) therefore it
+ has been deprecated in favour of PT_PAX_FLAGS and XATTR_PAX_FLAGS
+ support.
+
+ Note that if you enable PT_PAX_FLAGS or XATTR_PAX_FLAGS marking
+ support as well, they will override the legacy EI_PAX marks.
+
+ If you enable none of the marking options then all applications
+ will run with PaX enabled on them by default.
+
+config PAX_PT_PAX_FLAGS
+ bool 'Use ELF program header marking'
+ default y if GRKERNSEC_CONFIG_AUTO
+ help
+ Enabling this option will allow you to control PaX features on
+ a per executable basis via the 'paxctl' utility available at
+ http://pax.grsecurity.net/. The control flags will be read from
+ a PaX specific ELF program header (PT_PAX_FLAGS). This marking
+ has the benefits of supporting both soft mode and being fully
+ integrated into the toolchain (the binutils patch is available
+ from http://pax.grsecurity.net).
+
+ Note that if you enable the legacy EI_PAX marking support as well,
+ the EI_PAX marks will be overridden by the PT_PAX_FLAGS marks.
+
+ If you enable both PT_PAX_FLAGS and XATTR_PAX_FLAGS support then you
+ must make sure that the marks are the same if a binary has both marks.
+
+ If you enable none of the marking options then all applications
+ will run with PaX enabled on them by default.
+
+config PAX_XATTR_PAX_FLAGS
+ bool 'Use filesystem extended attributes marking'
+ default y if GRKERNSEC_CONFIG_AUTO
+ select CIFS_XATTR if CIFS
+ select F2FS_FS_XATTR if F2FS_FS
+ select EXT2_FS_XATTR if EXT2_FS
+ select EXT3_FS_XATTR if EXT3_FS
+ select JFFS2_FS_XATTR if JFFS2_FS
+ select REISERFS_FS_XATTR if REISERFS_FS
+ select SQUASHFS_XATTR if SQUASHFS
+ select TMPFS_XATTR if TMPFS
+ help
+ Enabling this option will allow you to control PaX features on
+ a per executable basis via the 'setfattr' utility. The control
+ flags will be read from the user.pax.flags extended attribute of
+ the file. This marking has the benefit of supporting binary-only
+ applications that self-check themselves (e.g., skype) and would
+ not tolerate chpax/paxctl changes. The main drawback is that
+ extended attributes are not supported by some filesystems (e.g.,
+ isofs, udf, vfat) so copying files through such filesystems will
+ lose the extended attributes and these PaX markings.
+
+ Note that if you enable the legacy EI_PAX marking support as well,
+ the EI_PAX marks will be overridden by the XATTR_PAX_FLAGS marks.
+
+ If you enable both PT_PAX_FLAGS and XATTR_PAX_FLAGS support then you
+ must make sure that the marks are the same if a binary has both marks.
+
+ If you enable none of the marking options then all applications
+ will run with PaX enabled on them by default.
+
+choice
+ prompt 'MAC system integration'
+ default PAX_HAVE_ACL_FLAGS
+ help
+ Mandatory Access Control systems have the option of controlling
+ PaX flags on a per executable basis, choose the method supported
+ by your particular system.
+
+ - "none": if your MAC system does not interact with PaX,
+ - "direct": if your MAC system defines pax_set_initial_flags() itself,
+ - "hook": if your MAC system uses the pax_set_initial_flags_func callback.
+
+ NOTE: this option is for developers/integrators only.
+
+ config PAX_NO_ACL_FLAGS
+ bool 'none'
+
+ config PAX_HAVE_ACL_FLAGS
+ bool 'direct'
+
+ config PAX_HOOK_ACL_FLAGS
+ bool 'hook'
+endchoice
+
+endmenu
+
+menu "Non-executable pages"
+ depends on PAX
+
+config PAX_NOEXEC
+ bool "Enforce non-executable pages"
+ default y if GRKERNSEC_CONFIG_AUTO
+ depends on ALPHA || (ARM && (CPU_V6 || CPU_V6K || CPU_V7)) || IA64 || MIPS || PARISC || PPC || S390 || SPARC || X86
+ help
+ By design some architectures do not allow for protecting memory
+ pages against execution or even if they do, Linux does not make
+ use of this feature. In practice this means that if a page is
+ readable (such as the stack or heap) it is also executable.
+
+ There is a well known exploit technique that makes use of this
+ fact and a common programming mistake where an attacker can
+ introduce code of his choice somewhere in the attacked program's
+ memory (typically the stack or the heap) and then execute it.
+
+ If the attacked program was running with different (typically
+ higher) privileges than that of the attacker, then he can elevate
+ his own privilege level (e.g. get a root shell, write to files for
+ which he does not have write access to, etc).
+
+ Enabling this option will let you choose from various features
+ that prevent the injection and execution of 'foreign' code in
+ a program.
+
+ This will also break programs that rely on the old behaviour and
+ expect that dynamically allocated memory via the malloc() family
+ of functions is executable (which it is not). Notable examples
+ are the XFree86 4.x server, the java runtime and wine.
+
+config PAX_PAGEEXEC
+ bool "Paging based non-executable pages"
+ default y if GRKERNSEC_CONFIG_AUTO
+ depends on PAX_NOEXEC && (!X86_32 || M586 || M586TSC || M586MMX || M686 || MPENTIUMII || MPENTIUMIII || MPENTIUMM || MCORE2 || MATOM || MPENTIUM4 || MPSC || MK7 || MK8 || MWINCHIPC6 || MWINCHIP2 || MWINCHIP3D || MVIAC3_2 || MVIAC7)
+ select ARCH_TRACK_EXEC_LIMIT if X86_32
+ help
+ This implementation is based on the paging feature of the CPU.
+ On i386 without hardware non-executable bit support there is a
+ variable but usually low performance impact, however on Intel's
+ P4 core based CPUs it is very high so you should not enable this
+ for kernels meant to be used on such CPUs.
+
+ On alpha, avr32, ia64, parisc, sparc, sparc64, x86_64 and i386
+ with hardware non-executable bit support there is no performance
+ impact, on ppc the impact is negligible.
+
+ Note that several architectures require various emulations due to
+ badly designed userland ABIs, this will cause a performance impact
+ but will disappear as soon as userland is fixed. For example, ppc
+ userland MUST have been built with secure-plt by a recent toolchain.
+
+config PAX_SEGMEXEC
+ bool "Segmentation based non-executable pages"
+ default y if GRKERNSEC_CONFIG_AUTO
+ depends on PAX_NOEXEC && X86_32
+ help
+ This implementation is based on the segmentation feature of the
+ CPU and has a very small performance impact, however applications
+ will be limited to a 1.5 GB address space instead of the normal
+ 3 GB.
+
+config PAX_EMUTRAMP
+ bool "Emulate trampolines"
+ default y if PARISC || GRKERNSEC_CONFIG_AUTO
+ depends on (PAX_PAGEEXEC || PAX_SEGMEXEC) && (PARISC || X86)
+ help
+ There are some programs and libraries that for one reason or
+ another attempt to execute special small code snippets from
+ non-executable memory pages. Most notable examples are the
+ signal handler return code generated by the kernel itself and
+ the GCC trampolines.
+
+ If you enabled CONFIG_PAX_PAGEEXEC or CONFIG_PAX_SEGMEXEC then
+ such programs will no longer work under your kernel.
+
+ As a remedy you can say Y here and use the 'chpax' or 'paxctl'
+ utilities to enable trampoline emulation for the affected programs
+ yet still have the protection provided by the non-executable pages.
+
+ On parisc you MUST enable this option and EMUSIGRT as well, otherwise
+ your system will not even boot.
+
+ Alternatively you can say N here and use the 'chpax' or 'paxctl'
+ utilities to disable CONFIG_PAX_PAGEEXEC and CONFIG_PAX_SEGMEXEC
+ for the affected files.
+
+ NOTE: enabling this feature *may* open up a loophole in the
+ protection provided by non-executable pages that an attacker
+ could abuse. Therefore the best solution is to not have any
+ files on your system that would require this option. This can
+ be achieved by not using libc5 (which relies on the kernel
+ signal handler return code) and not using or rewriting programs
+ that make use of the nested function implementation of GCC.
+ Skilled users can just fix GCC itself so that it implements
+ nested function calls in a way that does not interfere with PaX.
+
+config PAX_EMUSIGRT
+ bool "Automatically emulate sigreturn trampolines"
+ depends on PAX_EMUTRAMP && PARISC
+ default y
+ help
+ Enabling this option will have the kernel automatically detect
+ and emulate signal return trampolines executing on the stack
+ that would otherwise lead to task termination.
+
+ This solution is intended as a temporary one for users with
+ legacy versions of libc (libc5, glibc 2.0, uClibc before 0.9.17,
+ Modula-3 runtime, etc) or executables linked to such, basically
+ everything that does not specify its own SA_RESTORER function in
+ normal executable memory like glibc 2.1+ does.
+
+ On parisc you MUST enable this option, otherwise your system will
+ not even boot.
+
+ NOTE: this feature cannot be disabled on a per executable basis
+ and since it *does* open up a loophole in the protection provided
+ by non-executable pages, the best solution is to not have any
+ files on your system that would require this option.
+
+config PAX_MPROTECT
+ bool "Restrict mprotect()"
+ default y if GRKERNSEC_CONFIG_AUTO
+ depends on (PAX_PAGEEXEC || PAX_SEGMEXEC)
+ help
+ Enabling this option will prevent programs from
+ - changing the executable status of memory pages that were
+ not originally created as executable,
+ - making read-only executable pages writable again,
+ - creating executable pages from anonymous memory,
+ - making read-only-after-relocations (RELRO) data pages writable again.
+
+ You should say Y here to complete the protection provided by
+ the enforcement of non-executable pages.
+
+ NOTE: you can use the 'chpax' or 'paxctl' utilities to control
+ this feature on a per file basis.
+
+config PAX_MPROTECT_COMPAT
+ bool "Use legacy/compat protection demoting (read help)"
+ default y if (GRKERNSEC_CONFIG_AUTO && GRKERNSEC_CONFIG_DESKTOP)
+ depends on PAX_MPROTECT
+ help
+ The current implementation of PAX_MPROTECT denies RWX allocations/mprotects
+ by sending the proper error code to the application. For some broken
+ userland, this can cause problems with Python or other applications. The
+ current implementation however allows for applications like clamav to
+ detect if JIT compilation/execution is allowed and to fall back gracefully
+ to an interpreter-based mode if it does not. While we encourage everyone
+ to use the current implementation as-is and push upstream to fix broken
+ userland (note that the RWX logging option can assist with this), in some
+ environments this may not be possible. Having to disable MPROTECT
+ completely on certain binaries reduces the security benefit of PaX,
+ so this option is provided for those environments to revert to the old
+ behavior.
+
+config PAX_ELFRELOCS
+ bool "Allow ELF text relocations (read help)"
+ depends on PAX_MPROTECT
+ default n
+ help
+ Non-executable pages and mprotect() restrictions are effective
+ in preventing the introduction of new executable code into an
+ attacked task's address space. There remain only two venues
+ for this kind of attack: if the attacker can execute already
+ existing code in the attacked task then he can either have it
+ create and mmap() a file containing his code or have it mmap()
+ an already existing ELF library that does not have position
+ independent code in it and use mprotect() on it to make it
+ writable and copy his code there. While protecting against
+ the former approach is beyond PaX, the latter can be prevented
+ by having only PIC ELF libraries on one's system (which do not
+ need to relocate their code). If you are sure this is your case,
+ as is the case with all modern Linux distributions, then leave
+ this option disabled. You should say 'n' here.
+
+config PAX_ETEXECRELOCS
+ bool "Allow ELF ET_EXEC text relocations"
+ depends on PAX_MPROTECT && (ALPHA || IA64 || PARISC)
+ select PAX_ELFRELOCS
+ default y
+ help
+ On some architectures there are incorrectly created applications
+ that require text relocations and would not work without enabling
+ this option. If you are an alpha, ia64 or parisc user, you should
+ enable this option and disable it once you have made sure that
+ none of your applications need it.
+
+config PAX_EMUPLT
+ bool "Automatically emulate ELF PLT"
+ depends on PAX_MPROTECT && (ALPHA || PARISC || SPARC)
+ default y
+ help
+ Enabling this option will have the kernel automatically detect
+ and emulate the Procedure Linkage Table entries in ELF files.
+ On some architectures such entries are in writable memory, and
+ become non-executable leading to task termination. Therefore
+ it is mandatory that you enable this option on alpha, parisc,
+ sparc and sparc64, otherwise your system would not even boot.
+
+ NOTE: this feature *does* open up a loophole in the protection
+ provided by the non-executable pages, therefore the proper
+ solution is to modify the toolchain to produce a PLT that does
+ not need to be writable.
+
+config PAX_DLRESOLVE
+ bool 'Emulate old glibc resolver stub'
+ depends on PAX_EMUPLT && SPARC
+ default n
+ help
+ This option is needed if userland has an old glibc (before 2.4)
+ that puts a 'save' instruction into the runtime generated resolver
+ stub that needs special emulation.
+
+config PAX_KERNEXEC
+ bool "Enforce non-executable kernel pages"
+ default y if GRKERNSEC_CONFIG_AUTO && (!X86 || GRKERNSEC_CONFIG_VIRT_NONE || (GRKERNSEC_CONFIG_VIRT_EPT && GRKERNSEC_CONFIG_VIRT_GUEST) || (GRKERNSEC_CONFIG_VIRT_EPT && GRKERNSEC_CONFIG_VIRT_KVM))
+ depends on (X86 || (ARM && (CPU_V6 || CPU_V6K || CPU_V7) && !(ARM_LPAE && MODULES))) && !XEN
+ select PAX_PER_CPU_PGD if X86_64 || (X86_32 && X86_PAE)
+ select PAX_KERNEXEC_PLUGIN if X86_64
+ select ARM_KERNMEM_PERMS if ARM
+ help
+ This is the kernel land equivalent of PAGEEXEC and MPROTECT,
+ that is, enabling this option will make it harder to inject
+ and execute 'foreign' code in kernel memory itself.
+
+ Note that on amd64, CONFIG_EFI enabled with "efi=old_map" on
+ the kernel command-line will result in an RWX physical map.
+
+ Likewise, the EFI runtime services are necessarily mapped as
+ RWX. If CONFIG_EFI is enabled on an EFI-capable system, it
+ is recommended that you boot with "noefi" on the kernel
+ command-line if possible to eliminate the mapping.
+
+choice
+ prompt "Return Address Instrumentation Method"
+ default PAX_KERNEXEC_PLUGIN_METHOD_BTS
+ depends on PAX_KERNEXEC_PLUGIN
+ help
+ Select the method used to instrument function pointer dereferences.
+ Note that binary modules cannot be instrumented by this approach.
+
+ Note that the implementation requires a gcc with plugin support,
+ i.e., gcc 4.5 or newer. You may need to install the supporting
+ headers explicitly in addition to the normal gcc package.
+
+ config PAX_KERNEXEC_PLUGIN_METHOD_BTS
+ bool "bts"
+ help
+ This method is compatible with binary only modules but has
+ a higher runtime overhead.
+
+ config PAX_KERNEXEC_PLUGIN_METHOD_OR
+ bool "or"
+ depends on !PARAVIRT
+ help
+ This method is incompatible with binary only modules but has
+ a lower runtime overhead.
+endchoice
+
+config PAX_KERNEXEC_PLUGIN_METHOD
+ string
+ default "bts" if PAX_KERNEXEC_PLUGIN_METHOD_BTS
+ default "or" if PAX_KERNEXEC_PLUGIN_METHOD_OR
+ default ""
+
+config PAX_KERNEXEC_MODULE_TEXT
+ int "Minimum amount of memory reserved for module code"
+ default "8" if (!GRKERNSEC_CONFIG_AUTO || GRKERNSEC_CONFIG_SERVER)
+ default "12" if (GRKERNSEC_CONFIG_AUTO && GRKERNSEC_CONFIG_DESKTOP)
+ depends on PAX_KERNEXEC && X86_32
+ help
+ Due to implementation details the kernel must reserve a fixed
+ amount of memory for runtime allocated code (such as modules)
+ at compile time that cannot be changed at runtime. Here you
+ can specify the minimum amount in MB that will be reserved.
+ Due to the same implementation details this size will always
+ be rounded up to the next 2/4 MB boundary (depends on PAE) so
+ the actually available memory for runtime allocated code will
+ usually be more than this minimum.
+
+ The default 4 MB should be enough for most users but if you have
+ an excessive number of modules (e.g., most distribution configs
+ compile many drivers as modules) or use huge modules such as
+ nvidia's kernel driver, you will need to adjust this amount.
+ A good rule of thumb is to look at your currently loaded kernel
+ modules and add up their sizes.
+
+endmenu
+
+menu "Address Space Layout Randomization"
+ depends on PAX
+
+config PAX_ASLR
+ bool "Address Space Layout Randomization"
+ default y if GRKERNSEC_CONFIG_AUTO
+ help
+ Many if not most exploit techniques rely on the knowledge of
+ certain addresses in the attacked program. The following options
+ will allow the kernel to apply a certain amount of randomization
+ to specific parts of the program thereby forcing an attacker to
+ guess them in most cases. Any failed guess will most likely crash
+ the attacked program which allows the kernel to detect such attempts
+ and react on them. PaX itself provides no reaction mechanisms,
+ instead it is strongly encouraged that you make use of grsecurity's
+ (http://www.grsecurity.net/) built-in crash detection features or
+ develop one yourself.
+
+ By saying Y here you can choose to randomize the following areas:
+ - top of the task's kernel stack
+ - top of the task's userland stack
+ - base address for mmap() requests that do not specify one
+ (this includes all libraries)
+ - base address of the main executable
+
+ It is strongly recommended to say Y here as address space layout
+ randomization has negligible impact on performance yet it provides
+ a very effective protection.
+
+ NOTE: you can use the 'chpax' or 'paxctl' utilities to control
+ this feature on a per file basis.
+
+config PAX_RANDKSTACK
+ bool "Randomize kernel stack base"
+ default y if GRKERNSEC_CONFIG_AUTO && !(GRKERNSEC_CONFIG_VIRT_HOST && GRKERNSEC_CONFIG_VIRT_VIRTUALBOX)
+ depends on X86_TSC && X86
+ help
+ By saying Y here the kernel will randomize every task's kernel
+ stack on every system call. This will not only force an attacker
+ to guess it but also prevent him from making use of possible
+ leaked information about it.
+
+ Since the kernel stack is a rather scarce resource, randomization
+ may cause unexpected stack overflows, therefore you should very
+ carefully test your system. Note that once enabled in the kernel
+ configuration, this feature cannot be disabled on a per file basis.
+
+config PAX_RANDUSTACK
+ bool
+
+config PAX_RANDMMAP
+ bool "Randomize user stack and mmap() bases"
+ default y if GRKERNSEC_CONFIG_AUTO
+ depends on PAX_ASLR
+ select PAX_RANDUSTACK
+ help
+ By saying Y here the kernel will randomize every task's userland
+ stack and use a randomized base address for mmap() requests that
+ do not specify one themselves.
+
+ The stack randomization is done in two steps where the second
+ one may apply a big amount of shift to the top of the stack and
+ cause problems for programs that want to use lots of memory (more
+ than 2.5 GB if SEGMEXEC is not active, or 1.25 GB when it is).
+
+ As a result of mmap randomization all dynamically loaded libraries
+ will appear at random addresses and therefore be harder to exploit
+ by a technique where an attacker attempts to execute library code
+ for his purposes (e.g. spawn a shell from an exploited program that
+ is running at an elevated privilege level).
+
+ Furthermore, if a program is relinked as a dynamic ELF file, its
+ base address will be randomized as well, completing the full
+ randomization of the address space layout. Attacking such programs
+ becomes a guess game. You can find an example of doing this at
+ http://pax.grsecurity.net/et_dyn.tar.gz and practical samples at
+ http://www.grsecurity.net/grsec-gcc-specs.tar.gz .
+
+ NOTE: you can use the 'chpax' or 'paxctl' utilities to control this
+ feature on a per file basis.
+
+endmenu
+
+menu "Miscellaneous hardening features"
+
+config PAX_MEMORY_SANITIZE
+ bool "Sanitize all freed memory"
+ default y if (GRKERNSEC_CONFIG_AUTO && GRKERNSEC_CONFIG_PRIORITY_SECURITY)
+ help
+ By saying Y here the kernel will erase memory pages and slab objects
+ as soon as they are freed. This in turn reduces the lifetime of data
+ stored in them, making it less likely that sensitive information such
+ as passwords, cryptographic secrets, etc stay in memory for too long.
+
+ This is especially useful for programs whose runtime is short, long
+ lived processes and the kernel itself benefit from this as long as
+ they ensure timely freeing of memory that may hold sensitive
+ information.
+
+ A nice side effect of the sanitization of slab objects is the
+ reduction of possible info leaks caused by padding bytes within the
+ leaky structures. Use-after-free bugs for structures containing
+ pointers can also be detected as dereferencing the sanitized pointer
+ will generate an access violation.
+
+ The tradeoff is performance impact, on a single CPU system kernel
+ compilation sees a 3% slowdown, other systems and workloads may vary
+ and you are advised to test this feature on your expected workload
+ before deploying it.
+
+ The slab sanitization feature excludes a few slab caches per default
+ for performance reasons. To extend the feature to cover those as
+ well, pass "pax_sanitize_slab=full" as kernel command line parameter.
+
+ To reduce the performance penalty by sanitizing pages only, albeit
+ limiting the effectiveness of this feature at the same time, slab
+ sanitization can be disabled with the kernel command line parameter
+ "pax_sanitize_slab=off".
+
+ Note that this feature does not protect data stored in live pages,
+ e.g., process memory swapped to disk may stay there for a long time.
+
+config PAX_MEMORY_STACKLEAK
+ bool "Sanitize kernel stack"
+ default y if (GRKERNSEC_CONFIG_AUTO && GRKERNSEC_CONFIG_PRIORITY_SECURITY)
+ depends on X86
+ help
+ By saying Y here the kernel will erase the kernel stack before it
+ returns from a system call. This in turn reduces the information
+ that a kernel stack leak bug can reveal.
+
+ Note that such a bug can still leak information that was put on
+ the stack by the current system call (the one eventually triggering
+ the bug) but traces of earlier system calls on the kernel stack
+ cannot leak anymore.
+
+ The tradeoff is performance impact: on a single CPU system kernel
+ compilation sees a 1% slowdown, other systems and workloads may vary
+ and you are advised to test this feature on your expected workload
+ before deploying it.
+
+ Note that the full feature requires a gcc with plugin support,
+ i.e., gcc 4.5 or newer. You may need to install the supporting
+ headers explicitly in addition to the normal gcc package. Using
+ older gcc versions means that functions with large enough stack
+ frames may leave uninitialized memory behind that may be exposed
+ to a later syscall leaking the stack.
+
+config PAX_MEMORY_STRUCTLEAK
+ bool "Forcibly initialize local variables copied to userland"
+ default y if (GRKERNSEC_CONFIG_AUTO && GRKERNSEC_CONFIG_PRIORITY_SECURITY)
+ help
+ By saying Y here the kernel will zero initialize some local
+ variables that are going to be copied to userland. This in
+ turn prevents unintended information leakage from the kernel
+ stack should later code forget to explicitly set all parts of
+ the copied variable.
+
+ The tradeoff is less performance impact than PAX_MEMORY_STACKLEAK
+ at a much smaller coverage.
+
+ Note that the implementation requires a gcc with plugin support,
+ i.e., gcc 4.5 or newer. You may need to install the supporting
+ headers explicitly in addition to the normal gcc package.
+
+config PAX_MEMORY_UDEREF
+ bool "Prevent invalid userland pointer dereference"
+ default y if GRKERNSEC_CONFIG_AUTO && !(X86_64 && GRKERNSEC_CONFIG_PRIORITY_PERF) && (!X86 || GRKERNSEC_CONFIG_VIRT_NONE || GRKERNSEC_CONFIG_VIRT_EPT)
+ depends on (X86 || (ARM && (CPU_V6 || CPU_V6K || CPU_V7) && !ARM_LPAE)) && !UML_X86 && !XEN
+ select PAX_PER_CPU_PGD if X86_64
+ help
+ By saying Y here the kernel will be prevented from dereferencing
+ userland pointers in contexts where the kernel expects only kernel
+ pointers. This is both a useful runtime debugging feature and a
+ security measure that prevents exploiting a class of kernel bugs.
+
+ The tradeoff is that some virtualization solutions may experience
+ a huge slowdown and therefore you should not enable this feature
+ for kernels meant to run in such environments. Whether a given VM
+ solution is affected or not is best determined by simply trying it
+ out, the performance impact will be obvious right on boot as this
+ mechanism engages from very early on. A good rule of thumb is that
+ VMs running on CPUs without hardware virtualization support (i.e.,
+ the majority of IA-32 CPUs) will likely experience the slowdown.
+
+ On X86_64 the kernel will make use of PCID support when available
+ (Intel's Westmere, Sandy Bridge, etc) for better security (default)
+ or performance impact. Pass pax_weakuderef on the kernel command
+ line to choose the latter.
+
+config PAX_REFCOUNT
+ bool "Prevent various kernel object reference counter overflows"
+ default y if GRKERNSEC_CONFIG_AUTO
+ depends on GRKERNSEC && ((ARM && (CPU_V6 || CPU_V6K || CPU_V7)) || MIPS || PPC || SPARC64 || X86)
+ help
+ By saying Y here the kernel will detect and prevent overflowing
+ various (but not all) kinds of object reference counters. Such
+ overflows can normally occur due to bugs only and are often, if
+ not always, exploitable.
+
+ The tradeoff is that data structures protected by an overflowed
+ refcount will never be freed and therefore will leak memory. Note
+ that this leak also happens even without this protection but in
+ that case the overflow can eventually trigger the freeing of the
+ data structure while it is still being used elsewhere, resulting
+ in the exploitable situation that this feature prevents.
+
+ Since this has a negligible performance impact, you should enable
+ this feature.
+
+config PAX_CONSTIFY_PLUGIN
+ bool "Automatically constify eligible structures"
+ default y
+ depends on !UML && PAX_KERNEXEC
+ help
+ By saying Y here the compiler will automatically constify a class
+ of types that contain only function pointers. This reduces the
+ kernel's attack surface and also produces a better memory layout.
+
+ Note that the implementation requires a gcc with plugin support,
+ i.e., gcc 4.5 or newer. You may need to install the supporting
+ headers explicitly in addition to the normal gcc package.
+
+ Note that if some code really has to modify constified variables
+ then the source code will have to be patched to allow it. Examples
+ can be found in PaX itself (the no_const attribute) and for some
+ out-of-tree modules at http://www.grsecurity.net/~paxguy1/ .
+
+config PAX_USERCOPY
+ bool "Harden heap object copies between kernel and userland"
+ default y if GRKERNSEC_CONFIG_AUTO
+ depends on ARM || IA64 || PPC || SPARC || X86
+ depends on GRKERNSEC && (SLAB || SLUB || SLOB)
+ select PAX_USERCOPY_SLABS
+ help
+ By saying Y here the kernel will enforce the size of heap objects
+ when they are copied in either direction between the kernel and
+ userland, even if only a part of the heap object is copied.
+
+ Specifically, this checking prevents information leaking from the
+ kernel heap during kernel to userland copies (if the kernel heap
+ object is otherwise fully initialized) and prevents kernel heap
+ overflows during userland to kernel copies.
+
+ Note that the current implementation provides the strictest bounds
+ checks for the SLUB allocator.
+
+ Enabling this option also enables per-slab cache protection against
+ data in a given cache being copied into/out of via userland
+ accessors. Though the whitelist of regions will be reduced over
+ time, it notably protects important data structures like task structs.
+
+ If frame pointers are enabled on x86, this option will also restrict
+ copies into and out of the kernel stack to local variables within a
+ single frame.
+
+ Since this has a negligible performance impact, you should enable
+ this feature.
+
+config PAX_USERCOPY_DEBUG
+ bool
+ depends on X86 && PAX_USERCOPY
+ default n
+
+config PAX_SIZE_OVERFLOW
+ bool "Prevent various integer overflows in function size parameters"
+ default y if GRKERNSEC_CONFIG_AUTO
+ depends on X86
+ help
+ By saying Y here the kernel recomputes expressions of function
+ arguments marked by a size_overflow attribute with double integer
+ precision (DImode/TImode for 32/64 bit integer types).
+
+ The recomputed argument is checked against TYPE_MAX and an event
+ is logged on overflow and the triggering process is killed.
+
+ Homepage: http://www.grsecurity.net/~ephox/overflow_plugin/
+
+ Note that the implementation requires a gcc with plugin support,
+ i.e., gcc 4.5 or newer. You may need to install the supporting
+ headers explicitly in addition to the normal gcc package.
+
+config PAX_LATENT_ENTROPY
+ bool "Generate some entropy during boot and runtime"
+ default y if GRKERNSEC_CONFIG_AUTO
+ help
+ By saying Y here the kernel will instrument some kernel code to
+ extract some entropy from both original and artificially created
+ program state. This will help especially embedded systems where
+ there is little 'natural' source of entropy normally. The cost
+ is some slowdown of the boot process and fork and irq processing.
+
+ When pax_extra_latent_entropy is passed on the kernel command line,
+ entropy will be extracted from up to the first 4GB of RAM while the
+ runtime memory allocator is being initialized. This costs even more
+ slowdown of the boot process.
+
+ Note that the implementation requires a gcc with plugin support,
+ i.e., gcc 4.5 or newer. You may need to install the supporting
+ headers explicitly in addition to the normal gcc package.
+
+ Note that entropy extracted this way is not cryptographically
+ secure!
+
+endmenu
+
+endmenu
+
+source grsecurity/Kconfig
+
+endmenu
+
+endmenu
+
source security/keys/Kconfig
config SECURITY_DMESG_RESTRICT
config LSM_MMAP_MIN_ADDR
int "Low address space for LSM to protect from user allocation"
depends on SECURITY && SECURITY_SELINUX
- default 32768 if ARM || (ARM64 && COMPAT)
+ default 32768 if ALPHA || ARM || (ARM64 && COMPAT) || PARISC || SPARC32
default 65536
help
This is the portion of low virtual memory which should be protected
int aa_path_link(struct aa_profile *profile, struct dentry *old_dentry,
struct path *new_dir, struct dentry *new_dentry)
{
- struct path link = { new_dir->mnt, new_dentry };
- struct path target = { new_dir->mnt, old_dentry };
+ struct path link = { .mnt = new_dir->mnt, .dentry = new_dentry };
+ struct path target = { .mnt = new_dir->mnt, .dentry = old_dentry };
struct path_cond cond = {
old_dentry->d_inode->i_uid,
old_dentry->d_inode->i_mode
struct dentry *dentry, u32 mask,
struct path_cond *cond)
{
- struct path path = { dir->mnt, dentry };
+ struct path path = { .mnt = dir->mnt, .dentry = dentry };
return common_perm(op, &path, mask, cond);
}
static int common_perm_mnt_dentry(int op, struct vfsmount *mnt,
struct dentry *dentry, u32 mask)
{
- struct path path = { mnt, dentry };
+ struct path path = { .mnt = mnt, .dentry = dentry };
struct path_cond cond = { dentry->d_inode->i_uid,
dentry->d_inode->i_mode
};
profile = aa_current_profile();
if (!unconfined(profile)) {
- struct path old_path = { old_dir->mnt, old_dentry };
- struct path new_path = { new_dir->mnt, new_dentry };
+ struct path old_path = { .mnt = old_dir->mnt, .dentry = old_dentry };
+ struct path new_path = { .mnt = new_dir->mnt, .dentry = new_dentry };
struct path_cond cond = { old_dentry->d_inode->i_uid,
old_dentry->d_inode->i_mode
};
return error;
}
-static struct security_operations apparmor_ops = {
+static struct security_operations apparmor_ops __read_only = {
.name = "apparmor",
.ptrace_access_check = apparmor_ptrace_access_check,
return 0;
}
+/* returns:
+ 1 for suid privilege
+ 2 for sgid privilege
+ 3 for fscap privilege
+*/
+int is_privileged_binary(const struct dentry *dentry)
+{
+ struct cpu_vfs_cap_data capdata;
+ struct inode *inode = dentry->d_inode;
+
+ if (!inode || S_ISDIR(inode->i_mode))
+ return 0;
+
+ if (inode->i_mode & S_ISUID)
+ return 1;
+ if ((inode->i_mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP))
+ return 2;
+
+ if (!get_vfs_caps_from_disk(dentry, &capdata)) {
+ if (!cap_isclear(capdata.inheritable) || !cap_isclear(capdata.permitted))
+ return 3;
+ }
+
+ return 0;
+}
+
/*
* Attempt to get the on-exec apply capability sets for an executable file from
* its xattrs and, if present, apply them to the proposed credentials being
const struct cred *cred = current_cred();
kuid_t root_uid = make_kuid(cred->user_ns, 0);
+ if (gr_acl_enable_at_secure())
+ return 1;
+
if (!uid_eq(cred->uid, root_uid)) {
if (bprm->cap_effective)
return 1;
extern spinlock_t ima_queue_lock;
struct ima_h_table {
- atomic_long_t len; /* number of stored measurements in the list */
- atomic_long_t violations;
+ atomic_long_unchecked_t len; /* number of stored measurements in the list */
+ atomic_long_unchecked_t violations;
struct hlist_head queue[IMA_MEASURE_HTABLE_SIZE];
};
extern struct ima_h_table ima_htable;
int result;
/* can overflow, only indicator */
- atomic_long_inc(&ima_htable.violations);
+ atomic_long_inc_unchecked(&ima_htable.violations);
result = ima_alloc_init_template(NULL, file, filename,
NULL, 0, &entry);
static int valid_policy = 1;
#define TMPBUFLEN 12
static ssize_t ima_show_htable_value(char __user *buf, size_t count,
- loff_t *ppos, atomic_long_t *val)
+ loff_t *ppos, atomic_long_unchecked_t *val)
{
char tmpbuf[TMPBUFLEN];
ssize_t len;
- len = scnprintf(tmpbuf, TMPBUFLEN, "%li\n", atomic_long_read(val));
+ len = scnprintf(tmpbuf, TMPBUFLEN, "%li\n", atomic_long_read_unchecked(val));
return simple_read_from_buffer(buf, count, ppos, tmpbuf, len);
}
INIT_LIST_HEAD(&qe->later);
list_add_tail_rcu(&qe->later, &ima_measurements);
- atomic_long_inc(&ima_htable.len);
+ atomic_long_inc_unchecked(&ima_htable.len);
key = ima_hash_key(entry->digest);
hlist_add_head_rcu(&qe->hnext, &ima_htable.queue[key]);
return 0;
if (ret == 0)
goto no_payload_free;
- ret = keyctl_instantiate_key_common(id, iov, ioc, ret, ringid);
+ ret = keyctl_instantiate_key_common(id, (const struct iovec __force_user *)iov, ioc, ret, ringid);
err:
if (iov != iovstack)
kfree(iov);
extern long keyctl_invalidate_key(key_serial_t);
extern long keyctl_instantiate_key_common(key_serial_t,
- const struct iovec *,
+ const struct iovec __user *,
unsigned, size_t, key_serial_t);
#ifdef CONFIG_PERSISTENT_KEYRINGS
extern long keyctl_get_persistent(uid_t, key_serial_t);
atomic_set(&key->usage, 1);
init_rwsem(&key->sem);
- lockdep_set_class(&key->sem, &type->lock_class);
+ lockdep_set_class(&key->sem, (struct lock_class_key *)&type->lock_class);
key->index_key.type = type;
key->user = user;
key->quotalen = quotalen;
struct key_type *p;
int ret;
- memset(&ktype->lock_class, 0, sizeof(ktype->lock_class));
+ pax_open_kernel();
+ memset((void *)&ktype->lock_class, 0, sizeof(ktype->lock_class));
+ pax_close_kernel();
ret = -EEXIST;
down_write(&key_types_sem);
}
/* store the type */
- list_add(&ktype->link, &key_types_list);
+ pax_list_add((struct list_head *)&ktype->link, &key_types_list);
pr_notice("Key type %s registered\n", ktype->name);
ret = 0;
void unregister_key_type(struct key_type *ktype)
{
down_write(&key_types_sem);
- list_del_init(&ktype->link);
+ pax_list_del_init((struct list_head *)&ktype->link);
downgrade_write(&key_types_sem);
key_gc_keytype(ktype);
pr_notice("Key type %s unregistered\n", ktype->name);
0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
/* add the special key types */
- list_add_tail(&key_type_keyring.link, &key_types_list);
- list_add_tail(&key_type_dead.link, &key_types_list);
- list_add_tail(&key_type_user.link, &key_types_list);
- list_add_tail(&key_type_logon.link, &key_types_list);
+ pax_list_add_tail((struct list_head *)&key_type_keyring.link, &key_types_list);
+ pax_list_add_tail((struct list_head *)&key_type_dead.link, &key_types_list);
+ pax_list_add_tail((struct list_head *)&key_type_user.link, &key_types_list);
+ pax_list_add_tail((struct list_head *)&key_type_logon.link, &key_types_list);
/* record the root user tracking */
rb_link_node(&root_key_user.node,
/*
* Copy the iovec data from userspace
*/
-static long copy_from_user_iovec(void *buffer, const struct iovec *iov,
+static long copy_from_user_iovec(void *buffer, const struct iovec __user *iov,
unsigned ioc)
{
for (; ioc > 0; ioc--) {
* If successful, 0 will be returned.
*/
long keyctl_instantiate_key_common(key_serial_t id,
- const struct iovec *payload_iov,
+ const struct iovec __user *payload_iov,
unsigned ioc,
size_t plen,
key_serial_t ringid)
[0].iov_len = plen
};
- return keyctl_instantiate_key_common(id, iov, 1, plen, ringid);
+ return keyctl_instantiate_key_common(id, (const struct iovec __force_user *)iov, 1, plen, ringid);
}
return keyctl_instantiate_key_common(id, NULL, 0, 0, ringid);
if (ret == 0)
goto no_payload_free;
- ret = keyctl_instantiate_key_common(id, iov, ioc, ret, ringid);
+ ret = keyctl_instantiate_key_common(id, (const struct iovec __force_user *)iov, ioc, ret, ringid);
err:
if (iov != iovstack)
kfree(iov);
link_prealloc_failed:
mutex_unlock(&user->cons_lock);
+ key_put(key);
kleave(" = %d [prelink]", ret);
return ret;
*/
static void update_mmap_min_addr(void)
{
+#ifndef SPARC
#ifdef CONFIG_LSM_MMAP_MIN_ADDR
if (dac_mmap_min_addr > CONFIG_LSM_MMAP_MIN_ADDR)
mmap_min_addr = dac_mmap_min_addr;
#else
mmap_min_addr = dac_mmap_min_addr;
#endif
+#endif
}
/*
static __initdata char chosen_lsm[SECURITY_NAME_MAX + 1] =
CONFIG_DEFAULT_SECURITY;
-static struct security_operations *security_ops;
-static struct security_operations default_security_ops = {
+struct security_operations *security_ops __read_only;
+struct security_operations default_security_ops __read_only = {
.name = "default",
};
return 0;
}
-void reset_security_ops(void)
-{
- security_ops = &default_security_ops;
-}
-
/* Save user chosen LSM */
static int __init choose_lsm(char *str)
{
struct avc_cache {
struct hlist_head slots[AVC_CACHE_SLOTS]; /* head for avc_node->list */
spinlock_t slots_lock[AVC_CACHE_SLOTS]; /* lock for writes */
- atomic_t lru_hint; /* LRU hint for reclaim scan */
+ atomic_unchecked_t lru_hint; /* LRU hint for reclaim scan */
atomic_t active_nodes;
u32 latest_notif; /* latest revocation notification */
};
spin_lock_init(&avc_cache.slots_lock[i]);
}
atomic_set(&avc_cache.active_nodes, 0);
- atomic_set(&avc_cache.lru_hint, 0);
+ atomic_set_unchecked(&avc_cache.lru_hint, 0);
avc_node_cachep = kmem_cache_create("avc_node", sizeof(struct avc_node),
0, SLAB_PANIC, NULL);
spinlock_t *lock;
for (try = 0, ecx = 0; try < AVC_CACHE_SLOTS; try++) {
- hvalue = atomic_inc_return(&avc_cache.lru_hint) & (AVC_CACHE_SLOTS - 1);
+ hvalue = atomic_inc_return_unchecked(&avc_cache.lru_hint) & (AVC_CACHE_SLOTS - 1);
head = &avc_cache.slots[hvalue];
lock = &avc_cache.slots_lock[hvalue];
#endif
-static struct security_operations selinux_ops = {
+static struct security_operations selinux_ops __read_only = {
+ .name = "selinux",
.name = "selinux",
.ptrace_access_check = selinux_ptrace_access_check,
#ifdef CONFIG_SECURITY_SELINUX_DISABLE
static int selinux_disabled;
+extern struct security_operations *security_ops;
+extern struct security_operations default_security_ops;
+
int selinux_disable(void)
{
if (ss_initialized) {
selinux_disabled = 1;
selinux_enabled = 0;
- reset_security_ops();
+ pax_open_kernel();
+ security_ops = &default_security_ops;
+ pax_close_kernel();
/* Try to destroy the avc node cache */
avc_disable();
rtnl_lock();
for_each_net(net) {
- atomic_inc(&net->xfrm.flow_cache_genid);
+ atomic_inc_unchecked(&net->xfrm.flow_cache_genid);
rt_genid_bump_all(net);
}
rtnl_unlock();
return 0;
}
-struct security_operations smack_ops = {
+struct security_operations smack_ops __read_only = {
.name = "smack",
.ptrace_access_check = smack_ptrace_access_check,
{
struct tomoyo_request_info r;
struct tomoyo_obj_info obj = {
- .path1 = *path,
+ .path1 = { .mnt = path->mnt, .dentry = path->dentry },
};
int error = -ENOMEM;
struct tomoyo_path_info buf;
struct tomoyo_path_info buf;
struct tomoyo_request_info r;
struct tomoyo_obj_info obj = {
- .path1 = *path,
+ .path1 = { .mnt = path->mnt, .dentry = path->dentry },
};
int idx;
{
struct tomoyo_request_info r;
struct tomoyo_obj_info obj = {
- .path1 = *path,
+ .path1 = { .mnt = path->mnt, .dentry = path->dentry },
};
int error;
struct tomoyo_path_info buf;
{
struct tomoyo_request_info r;
struct tomoyo_obj_info obj = {
- .path1 = *path,
+ .path1 = { .mnt = path->mnt, .dentry = path->dentry },
};
int error = -ENOMEM;
struct tomoyo_path_info buf;
struct tomoyo_path_info buf2;
struct tomoyo_request_info r;
struct tomoyo_obj_info obj = {
- .path1 = *path1,
- .path2 = *path2,
+ .path1 = { .mnt = path1->mnt, .dentry = path1->dentry },
+ .path2 = { .mnt = path2->mnt, .dentry = path2->dentry }
};
int idx;
type == tomoyo_mounts[TOMOYO_MOUNT_MOVE]) {
need_dev = -1; /* dev_name is a directory */
} else {
+ if (!capable(CAP_SYS_ADMIN)) {
+ error = -EPERM;
+ goto out;
+ }
fstype = get_fs_type(type);
if (!fstype) {
error = -ENODEV;
*/
static int tomoyo_inode_getattr(struct vfsmount *mnt, struct dentry *dentry)
{
- struct path path = { mnt, dentry };
+ struct path path = { .mnt = mnt, .dentry = dentry };
return tomoyo_path_perm(TOMOYO_TYPE_GETATTR, &path, NULL);
}
*/
static int tomoyo_path_unlink(struct path *parent, struct dentry *dentry)
{
- struct path path = { parent->mnt, dentry };
+ struct path path = { .mnt = parent->mnt, .dentry = dentry };
return tomoyo_path_perm(TOMOYO_TYPE_UNLINK, &path, NULL);
}
static int tomoyo_path_mkdir(struct path *parent, struct dentry *dentry,
umode_t mode)
{
- struct path path = { parent->mnt, dentry };
+ struct path path = { .mnt = parent->mnt, .dentry = dentry };
return tomoyo_path_number_perm(TOMOYO_TYPE_MKDIR, &path,
mode & S_IALLUGO);
}
*/
static int tomoyo_path_rmdir(struct path *parent, struct dentry *dentry)
{
- struct path path = { parent->mnt, dentry };
+ struct path path = { .mnt = parent->mnt, .dentry = dentry };
return tomoyo_path_perm(TOMOYO_TYPE_RMDIR, &path, NULL);
}
static int tomoyo_path_symlink(struct path *parent, struct dentry *dentry,
const char *old_name)
{
- struct path path = { parent->mnt, dentry };
+ struct path path = { .mnt = parent->mnt, .dentry = dentry };
return tomoyo_path_perm(TOMOYO_TYPE_SYMLINK, &path, old_name);
}
static int tomoyo_path_mknod(struct path *parent, struct dentry *dentry,
umode_t mode, unsigned int dev)
{
- struct path path = { parent->mnt, dentry };
+ struct path path = { .mnt = parent->mnt, .dentry = dentry };
int type = TOMOYO_TYPE_CREATE;
const unsigned int perm = mode & S_IALLUGO;
static int tomoyo_path_link(struct dentry *old_dentry, struct path *new_dir,
struct dentry *new_dentry)
{
- struct path path1 = { new_dir->mnt, old_dentry };
- struct path path2 = { new_dir->mnt, new_dentry };
+ struct path path1 = { .mnt = new_dir->mnt, .dentry = old_dentry };
+ struct path path2 = { .mnt = new_dir->mnt, .dentry = new_dentry };
return tomoyo_path2_perm(TOMOYO_TYPE_LINK, &path1, &path2);
}
struct path *new_parent,
struct dentry *new_dentry)
{
- struct path path1 = { old_parent->mnt, old_dentry };
- struct path path2 = { new_parent->mnt, new_dentry };
+ struct path path1 = { .mnt = old_parent->mnt, .dentry = old_dentry };
+ struct path path2 = { .mnt = new_parent->mnt, .dentry = new_dentry };
return tomoyo_path2_perm(TOMOYO_TYPE_RENAME, &path1, &path2);
}
*/
static int tomoyo_sb_umount(struct vfsmount *mnt, int flags)
{
- struct path path = { mnt, mnt->mnt_root };
+ struct path path = { .mnt = mnt, .dentry = mnt->mnt_root };
return tomoyo_path_perm(TOMOYO_TYPE_UMOUNT, &path, NULL);
}
* tomoyo_security_ops is a "struct security_operations" which is used for
* registering TOMOYO.
*/
-static struct security_operations tomoyo_security_ops = {
+static struct security_operations tomoyo_security_ops __read_only = {
.name = "tomoyo",
.cred_alloc_blank = tomoyo_cred_alloc_blank,
.cred_prepare = tomoyo_cred_prepare,
config SECURITY_YAMA
bool "Yama support"
- depends on SECURITY
+ depends on SECURITY && !GRKERNSEC
select SECURITYFS
select SECURITY_PATH
default n
}
#ifndef CONFIG_SECURITY_YAMA_STACKED
-static struct security_operations yama_ops = {
+static struct security_operations yama_ops __read_only = {
.name = "yama",
.ptrace_access_check = yama_ptrace_access_check,
#endif
#ifdef CONFIG_SYSCTL
+static int zero __read_only;
+static int max_scope __read_only = YAMA_SCOPE_NO_ATTACH;
+
static int yama_dointvec_minmax(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp, loff_t *ppos)
{
- int rc;
+ ctl_table_no_const yama_table;
if (write && !capable(CAP_SYS_PTRACE))
return -EPERM;
- rc = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
- if (rc)
- return rc;
-
+ yama_table = *table;
/* Lock the max value if it ever gets set. */
- if (write && *(int *)table->data == *(int *)table->extra2)
- table->extra1 = table->extra2;
-
- return rc;
+ if (ptrace_scope == max_scope)
+ yama_table.extra1 = &max_scope;
+ return proc_dointvec_minmax(&yama_table, write, buffer, lenp, ppos);
}
-static int zero;
-static int max_scope = YAMA_SCOPE_NO_ATTACH;
-
struct ctl_path yama_sysctl_path[] = {
{ .procname = "kernel", },
{ .procname = "yama", },
spdif_locked:1,
analog_locked:1,
original_mute:2;
- int open_count;
+ local_t open_count;
struct codec_info *codec_info;
/* mutex serializes concurrent access to the device
struct onyx *onyx = cii->codec_data;
mutex_lock(&onyx->mutex);
- onyx->open_count++;
+ local_inc(&onyx->open_count);
mutex_unlock(&onyx->mutex);
return 0;
struct onyx *onyx = cii->codec_data;
mutex_lock(&onyx->mutex);
- onyx->open_count--;
- if (!onyx->open_count)
+ if (local_dec_and_test(&onyx->open_count))
onyx->spdif_locked = onyx->analog_locked = 0;
mutex_unlock(&onyx->mutex);
#include <linux/i2c.h>
#include <asm/pmac_low_i2c.h>
#include <asm/prom.h>
+#include <asm/local.h>
/* PCM3052 register definitions */
if (in_kernel) {
mm_segment_t fs;
fs = snd_enter_user();
- ret = snd_pcm_lib_write(substream, (void __force __user *)ptr, frames);
+ ret = snd_pcm_lib_write(substream, (void __force_user *)ptr, frames);
snd_leave_user(fs);
} else {
- ret = snd_pcm_lib_write(substream, (void __force __user *)ptr, frames);
+ ret = snd_pcm_lib_write(substream, (void __force_user *)ptr, frames);
}
if (ret != -EPIPE && ret != -ESTRPIPE)
break;
if (in_kernel) {
mm_segment_t fs;
fs = snd_enter_user();
- ret = snd_pcm_lib_read(substream, (void __force __user *)ptr, frames);
+ ret = snd_pcm_lib_read(substream, (void __force_user *)ptr, frames);
snd_leave_user(fs);
} else {
- ret = snd_pcm_lib_read(substream, (void __force __user *)ptr, frames);
+ ret = snd_pcm_lib_read(substream, (void __force_user *)ptr, frames);
}
if (ret == -EPIPE) {
if (runtime->status->state == SNDRV_PCM_STATE_DRAINING) {
struct snd_pcm_plugin_channel *channels;
size_t oss_frame_bytes = (runtime->oss.plugin_first->src_width * runtime->oss.plugin_first->src_format.channels) / 8;
if (!in_kernel) {
- if (copy_from_user(runtime->oss.buffer, (const char __force __user *)buf, bytes))
+ if (copy_from_user(runtime->oss.buffer, (const char __force_user *)buf, bytes))
return -EFAULT;
buf = runtime->oss.buffer;
}
}
} else {
tmp = snd_pcm_oss_write2(substream,
- (const char __force *)buf,
+ (const char __force_kernel *)buf,
runtime->oss.period_bytes, 0);
if (tmp <= 0)
goto err;
struct snd_pcm_runtime *runtime = substream->runtime;
snd_pcm_sframes_t frames, frames1;
#ifdef CONFIG_SND_PCM_OSS_PLUGINS
- char __user *final_dst = (char __force __user *)buf;
+ char __user *final_dst = (char __force_user *)buf;
if (runtime->oss.plugin_first) {
struct snd_pcm_plugin_channel *channels;
size_t oss_frame_bytes = (runtime->oss.plugin_last->dst_width * runtime->oss.plugin_last->dst_format.channels) / 8;
xfer += tmp;
runtime->oss.buffer_used -= tmp;
} else {
- tmp = snd_pcm_oss_read2(substream, (char __force *)buf,
+ tmp = snd_pcm_oss_read2(substream, (char __force_kernel *)buf,
runtime->oss.period_bytes, 0);
if (tmp <= 0)
goto err;
size1);
size1 /= runtime->channels; /* frames */
fs = snd_enter_user();
- snd_pcm_lib_write(substream, (void __force __user *)runtime->oss.buffer, size1);
+ snd_pcm_lib_write(substream, (void __force_user *)runtime->oss.buffer, size1);
snd_leave_user(fs);
}
} else if (runtime->access == SNDRV_PCM_ACCESS_RW_NONINTERLEAVED) {
int err;
fs = snd_enter_user();
- err = snd_pcm_delay(substream, &delay);
+ err = snd_pcm_delay(substream, (snd_pcm_sframes_t __force_user *)&delay);
snd_leave_user(fs);
if (err < 0)
return err;
switch (substream->stream) {
case SNDRV_PCM_STREAM_PLAYBACK:
result = snd_pcm_playback_ioctl1(NULL, substream, cmd,
- (void __user *)arg);
+ (void __force_user *)arg);
break;
case SNDRV_PCM_STREAM_CAPTURE:
result = snd_pcm_capture_ioctl1(NULL, substream, cmd,
- (void __user *)arg);
+ (void __force_user *)arg);
break;
default:
result = -EINVAL;
{
int rc;
static struct snd_seq_dev_ops ops = {
- snd_seq_oss_synth_register,
- snd_seq_oss_synth_unregister,
+ .init_device = snd_seq_oss_synth_register,
+ .free_device = snd_seq_oss_synth_unregister,
};
snd_seq_autoload_lock();
int argsize; /* argument size */
/* operators */
- struct snd_seq_dev_ops ops;
+ struct snd_seq_dev_ops *ops;
/* registered devices */
struct list_head dev_list; /* list of devices */
mutex_lock(&ops->reg_mutex);
/* copy driver operators */
- ops->ops = *entry;
+ ops->ops = entry;
ops->driver |= DRIVER_LOADED;
ops->argsize = argsize;
dev->name, ops->id, ops->argsize, dev->argsize);
return -EINVAL;
}
- if (ops->ops.init_device(dev) >= 0) {
+ if (ops->ops->init_device(dev) >= 0) {
dev->status = SNDRV_SEQ_DEVICE_REGISTERED;
ops->num_init_devices++;
} else {
dev->name, ops->id, ops->argsize, dev->argsize);
return -EINVAL;
}
- if ((result = ops->ops.free_device(dev)) >= 0 || result == -ENXIO) {
+ if ((result = ops->ops->free_device(dev)) >= 0 || result == -ENXIO) {
dev->status = SNDRV_SEQ_DEVICE_FREE;
dev->driver_data = NULL;
ops->num_init_devices--;
static int __init alsa_seq_midi_init(void)
{
static struct snd_seq_dev_ops ops = {
- snd_seq_midisynth_register_port,
- snd_seq_midisynth_unregister_port,
+ .init_device = snd_seq_midisynth_register_port,
+ .free_device = snd_seq_midisynth_unregister_port,
};
memset(&synths, 0, sizeof(synths));
snd_seq_autoload_lock();
case SNDRV_MINOR_TIMER: str = "snd-timer"; break;
default: return;
}
- request_module(str);
+ request_module("%s", str);
}
#endif /* modular kernel */
#include <sound/initval.h>
#include <sound/rawmidi.h>
#include <sound/control.h>
+#include <asm/local.h>
#define CARD_NAME "Miditerminal 4140"
#define DRIVER_NAME "MTS64"
struct pardevice *pardev;
int pardev_claimed;
- int open_count;
+ local_t open_count;
int current_midi_output_port;
int current_midi_input_port;
u8 mode[MTS64_NUM_INPUT_PORTS];
{
struct mts64 *mts = substream->rmidi->private_data;
- if (mts->open_count == 0) {
+ if (local_read(&mts->open_count) == 0) {
/* We don't need a spinlock here, because this is just called
if the device has not been opened before.
So there aren't any IRQs from the device */
msleep(50);
}
- ++(mts->open_count);
+ local_inc(&mts->open_count);
return 0;
}
struct mts64 *mts = substream->rmidi->private_data;
unsigned long flags;
- --(mts->open_count);
- if (mts->open_count == 0) {
+ if (local_dec_return(&mts->open_count) == 0) {
/* We need the spinlock_irqsave here because we can still
have IRQs at this point */
spin_lock_irqsave(&mts->lock, flags);
msleep(500);
- } else if (mts->open_count < 0)
- mts->open_count = 0;
+ } else if (local_read(&mts->open_count) < 0)
+ local_set(&mts->open_count, 0);
return 0;
}
{
static struct snd_seq_dev_ops ops =
{
- snd_opl3_seq_new_device,
- snd_opl3_seq_delete_device
+ .init_device = snd_opl3_seq_new_device,
+ .free_device = snd_opl3_seq_delete_device
};
return snd_seq_device_register_driver(SNDRV_SEQ_DEV_ID_OPL3, &ops,
MODULE_DESCRIPTION("OPL4 driver");
MODULE_LICENSE("GPL");
-static void inline snd_opl4_wait(struct snd_opl4 *opl4)
+static inline void snd_opl4_wait(struct snd_opl4 *opl4)
{
int timeout = 10;
while ((inb(opl4->fm_port) & OPL4_STATUS_BUSY) && --timeout > 0)
static int __init alsa_opl4_synth_init(void)
{
static struct snd_seq_dev_ops ops = {
- snd_opl4_seq_new_device,
- snd_opl4_seq_delete_device
+ .init_device = snd_opl4_seq_new_device,
+ .free_device = snd_opl4_seq_delete_device
};
return snd_seq_device_register_driver(SNDRV_SEQ_DEV_ID_OPL4, &ops,
#include <sound/initval.h>
#include <sound/rawmidi.h>
#include <sound/control.h>
+#include <asm/local.h>
#define CARD_NAME "Portman 2x4"
#define DRIVER_NAME "portman"
struct pardevice *pardev;
int pardev_claimed;
- int open_count;
+ local_t open_count;
int mode[PORTMAN_NUM_INPUT_PORTS];
struct snd_rawmidi_substream *midi_input[PORTMAN_NUM_INPUT_PORTS];
};
ptr = s->pcm_buffer_pointer + frames;
if (ptr >= pcm->runtime->buffer_size)
ptr -= pcm->runtime->buffer_size;
- ACCESS_ONCE(s->pcm_buffer_pointer) = ptr;
+ ACCESS_ONCE_RW(s->pcm_buffer_pointer) = ptr;
s->pcm_period_pointer += frames;
if (s->pcm_period_pointer >= pcm->runtime->period_size) {
*/
void amdtp_stream_update(struct amdtp_stream *s)
{
- ACCESS_ONCE(s->source_node_id_field) =
+ ACCESS_ONCE_RW(s->source_node_id_field) =
(fw_parent_device(s->unit)->card->node_id & 0x3f) << 24;
}
EXPORT_SYMBOL(amdtp_stream_update);
static inline void amdtp_stream_pcm_trigger(struct amdtp_stream *s,
struct snd_pcm_substream *pcm)
{
- ACCESS_ONCE(s->pcm) = pcm;
+ ACCESS_ONCE_RW(s->pcm) = pcm;
}
/**
struct snd_rawmidi_substream *midi)
{
if (port < s->midi_ports)
- ACCESS_ONCE(s->midi[port]) = midi;
+ ACCESS_ONCE_RW(s->midi[port]) = midi;
}
static inline bool cip_sfc_is_base_44100(enum cip_sfc sfc)
ptr += count;
if (ptr >= runtime->buffer_size)
ptr -= runtime->buffer_size;
- ACCESS_ONCE(isight->buffer_pointer) = ptr;
+ ACCESS_ONCE_RW(isight->buffer_pointer) = ptr;
isight->period_counter += count;
if (isight->period_counter >= runtime->period_size) {
if (err < 0)
return err;
- ACCESS_ONCE(isight->pcm_active) = true;
+ ACCESS_ONCE_RW(isight->pcm_active) = true;
return 0;
}
{
struct isight *isight = substream->private_data;
- ACCESS_ONCE(isight->pcm_active) = false;
+ ACCESS_ONCE_RW(isight->pcm_active) = false;
mutex_lock(&isight->mutex);
isight_stop_streaming(isight);
switch (cmd) {
case SNDRV_PCM_TRIGGER_START:
- ACCESS_ONCE(isight->pcm_running) = true;
+ ACCESS_ONCE_RW(isight->pcm_running) = true;
break;
case SNDRV_PCM_TRIGGER_STOP:
- ACCESS_ONCE(isight->pcm_running) = false;
+ ACCESS_ONCE_RW(isight->pcm_running) = false;
break;
default:
return -EINVAL;
{
struct scs *scs = stream->rmidi->private_data;
- ACCESS_ONCE(scs->output) = up ? stream : NULL;
+ ACCESS_ONCE_RW(scs->output) = up ? stream : NULL;
if (up) {
scs->output_idle = false;
tasklet_schedule(&scs->tasklet);
{
struct scs *scs = stream->rmidi->private_data;
- ACCESS_ONCE(scs->input) = up ? stream : NULL;
+ ACCESS_ONCE_RW(scs->input) = up ? stream : NULL;
}
static void scs_input_escaped_byte(struct snd_rawmidi_substream *stream,
snd_card_disconnect(scs->card);
- ACCESS_ONCE(scs->output) = NULL;
- ACCESS_ONCE(scs->input) = NULL;
+ ACCESS_ONCE_RW(scs->output) = NULL;
+ ACCESS_ONCE_RW(scs->input) = NULL;
wait_event(scs->idle_wait, scs->output_idle);
{
static struct snd_seq_dev_ops ops = {
- snd_emu8000_new_device,
- snd_emu8000_delete_device,
+ .init_device = snd_emu8000_new_device,
+ .free_device = snd_emu8000_delete_device,
};
return snd_seq_device_register_driver(SNDRV_SEQ_DEV_ID_EMU8000, &ops,
sizeof(struct snd_emu8000*));
buf16 = (signed short *)(localbuf + localoffs);
while (c)
{
- locallen = (c >= LBUFCOPYSIZE ? LBUFCOPYSIZE : c);
+ locallen = ((unsigned)c >= LBUFCOPYSIZE ? LBUFCOPYSIZE : c);
if (copy_from_user(lbuf8,
userbuf+useroffs + p,
locallen))
{
struct cs4297a_state *s;
u32 pwr, id;
- mm_segment_t fs;
int rval;
u64 cfg;
int mdio_val;
if (!rval) {
char *sb1250_duart_present;
+#if 0
+ mm_segment_t fs;
fs = get_fs();
set_fs(KERNEL_DS);
-#if 0
val = SOUND_MASK_LINE;
mixer_ioctl(s, SOUND_MIXER_WRITE_RECSRC, (unsigned long) &val);
for (i = 0; i < ARRAY_SIZE(initvol); i++) {
val = initvol[i].vol;
mixer_ioctl(s, initvol[i].mixch, (unsigned long) &val);
}
+ set_fs(fs);
// cs4297a_write_ac97(s, 0x18, 0x0808);
#else
// cs4297a_write_ac97(s, 0x5e, 0x180);
cs4297a_write_ac97(s, 0x02, 0x0808);
cs4297a_write_ac97(s, 0x18, 0x0808);
#endif
- set_fs(fs);
list_add(&s->list, &cs4297a_devs);
{
static struct snd_seq_dev_ops ops = {
- snd_emu10k1_synth_new_device,
- snd_emu10k1_synth_delete_device,
+ .init_device = snd_emu10k1_synth_new_device,
+ .free_device = snd_emu10k1_synth_delete_device,
};
return snd_seq_device_register_driver(SNDRV_SEQ_DEV_ID_EMU10K1_SYNTH, &ops,
sizeof(struct snd_emu10k1_synth_arg));
/* FIXME: set_fs() hack for obtaining user-space TLV data */
mm_segment_t fs = get_fs();
set_fs(get_ds());
- if (!kctl->tlv.c(kctl, 0, sizeof(_tlv), _tlv))
+ if (!kctl->tlv.c(kctl, 0, sizeof(_tlv), (unsigned int __force_user *)_tlv))
tlv = _tlv;
set_fs(fs);
} else if (kctl->vd[0].access & SNDRV_CTL_ELEM_ACCESS_TLV_READ)
spinlock_t reg_lock;
spinlock_t voice_lock;
wait_queue_head_t interrupt_sleep;
- atomic_t interrupt_sleep_count;
+ atomic_unchecked_t interrupt_sleep_count;
struct snd_info_entry *proc_entry;
const struct firmware *dsp_microcode;
const struct firmware *controller_microcode;
if ((snd_ymfpci_readl(chip, YDSXGR_STATUS) & 2) == 0)
break;
}
- if (atomic_read(&chip->interrupt_sleep_count)) {
- atomic_set(&chip->interrupt_sleep_count, 0);
+ if (atomic_read_unchecked(&chip->interrupt_sleep_count)) {
+ atomic_set_unchecked(&chip->interrupt_sleep_count, 0);
wake_up(&chip->interrupt_sleep);
}
__end:
continue;
init_waitqueue_entry(&wait, current);
add_wait_queue(&chip->interrupt_sleep, &wait);
- atomic_inc(&chip->interrupt_sleep_count);
+ atomic_inc_unchecked(&chip->interrupt_sleep_count);
schedule_timeout_uninterruptible(msecs_to_jiffies(50));
remove_wait_queue(&chip->interrupt_sleep, &wait);
}
snd_ymfpci_writel(chip, YDSXGR_MODE, mode);
spin_unlock(&chip->reg_lock);
- if (atomic_read(&chip->interrupt_sleep_count)) {
- atomic_set(&chip->interrupt_sleep_count, 0);
+ if (atomic_read_unchecked(&chip->interrupt_sleep_count)) {
+ atomic_set_unchecked(&chip->interrupt_sleep_count, 0);
wake_up(&chip->interrupt_sleep);
}
}
spin_lock_init(&chip->reg_lock);
spin_lock_init(&chip->voice_lock);
init_waitqueue_head(&chip->interrupt_sleep);
- atomic_set(&chip->interrupt_sleep_count, 0);
+ atomic_set_unchecked(&chip->interrupt_sleep_count, 0);
chip->card = card;
chip->pci = pci;
chip->irq = -1;
if (ret)
return ret;
- ops->warm_reset = snd_soc_ac97_warm_reset;
- ops->reset = snd_soc_ac97_reset;
+ pax_open_kernel();
+ *(void **)&ops->warm_reset = snd_soc_ac97_warm_reset;
+ *(void **)&ops->reset = snd_soc_ac97_reset;
+ pax_close_kernel();
snd_ac97_rst_cfg = cfg;
return 0;
* MIDI emulation operators
*/
static struct snd_midi_op emux_ops = {
- snd_emux_note_on,
- snd_emux_note_off,
- snd_emux_key_press,
- snd_emux_terminate_note,
- snd_emux_control,
- snd_emux_nrpn,
- snd_emux_sysex,
+ .note_on = snd_emux_note_on,
+ .note_off = snd_emux_note_off,
+ .key_press = snd_emux_key_press,
+ .note_terminate = snd_emux_terminate_note,
+ .control = snd_emux_control,
+ .nrpn = snd_emux_nrpn,
+ .sysex = snd_emux_sysex,
};
# define unlikely(x) __builtin_expect(!!(x), 0)
#endif
+#ifndef __size_overflow
+# define __size_overflow(...)
+#endif
+
+#ifndef __intentional_overflow
+# define __intentional_overflow(...)
+#endif
+
#define ACCESS_ONCE(x) (*(volatile typeof(x) *)&(x))
#endif /* _TOOLS_LINUX_COMPILER_H */
LIBFILE = libapikfs.a
-CFLAGS = -ggdb3 -Wall -Wextra -std=gnu99 -Werror -O6 -D_FORTIFY_SOURCE=2 $(EXTRA_WARNINGS) $(EXTRA_CFLAGS) -fPIC
+CFLAGS = -ggdb3 -Wall -Wextra -std=gnu99 -Werror -O6 -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 $(EXTRA_WARNINGS) $(EXTRA_CFLAGS) -fPIC
EXTLIBS = -lelf -lpthread -lrt -lm
ALL_CFLAGS = $(CFLAGS) $(BASIC_CFLAGS) -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64
ALL_LDFLAGS = $(LDFLAGS)
#define altinstruction_entry #
+ .macro pax_force_retaddr rip=0, reload=0
+ .endm
+
#endif
({ \
typeof(ptr) __pu_ptr = (ptr); \
__chk_user_ptr(__pu_ptr, sizeof(*__pu_ptr)); \
- ACCESS_ONCE(*(__pu_ptr)) = x; \
+ ACCESS_ONCE_RW(*(__pu_ptr)) = x; \
0; \
})
static cpumask_var_t cpus_hardware_enabled;
static int kvm_usage_count = 0;
-static atomic_t hardware_enable_failed;
+static atomic_unchecked_t hardware_enable_failed;
struct kmem_cache *kvm_vcpu_cache;
EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
-static __read_mostly struct preempt_ops kvm_preempt_ops;
+static void kvm_sched_in(struct preempt_notifier *pn, int cpu);
+static void kvm_sched_out(struct preempt_notifier *pn, struct task_struct *next);
+static struct preempt_ops kvm_preempt_ops = {
+ .sched_in = kvm_sched_in,
+ .sched_out = kvm_sched_out,
+};
struct dentry *kvm_debugfs_dir;
/* We can read the guest memory with __xxx_user() later on. */
if ((mem->slot < KVM_USER_MEM_SLOTS) &&
((mem->userspace_addr & (PAGE_SIZE - 1)) ||
- !access_ok(VERIFY_WRITE,
+ !access_ok_noprefault(VERIFY_WRITE,
(void __user *)(unsigned long)mem->userspace_addr,
mem->memory_size)))
goto out;
int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
{
- const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0)));
+ int r;
+ unsigned long addr;
- return kvm_write_guest_page(kvm, gfn, zero_page, offset, len);
+ addr = gfn_to_hva(kvm, gfn);
+ if (kvm_is_error_hva(addr))
+ return -EFAULT;
+ r = __clear_user((void __user *)addr + offset, len);
+ if (r)
+ return -EFAULT;
+ mark_page_dirty(kvm, gfn);
+ return 0;
}
EXPORT_SYMBOL_GPL(kvm_clear_guest_page);
return 0;
}
-static struct file_operations kvm_vcpu_fops = {
+static file_operations_no_const kvm_vcpu_fops __read_only = {
.release = kvm_vcpu_release,
.unlocked_ioctl = kvm_vcpu_ioctl,
#ifdef CONFIG_COMPAT
}
#endif
-static struct file_operations kvm_vm_fops = {
+static file_operations_no_const kvm_vm_fops __read_only = {
.release = kvm_vm_release,
.unlocked_ioctl = kvm_vm_ioctl,
#ifdef CONFIG_COMPAT
return r;
}
-static struct file_operations kvm_chardev_ops = {
+static file_operations_no_const kvm_chardev_ops __read_only = {
.unlocked_ioctl = kvm_dev_ioctl,
.compat_ioctl = kvm_dev_ioctl,
.llseek = noop_llseek,
if (r) {
cpumask_clear_cpu(cpu, cpus_hardware_enabled);
- atomic_inc(&hardware_enable_failed);
+ atomic_inc_unchecked(&hardware_enable_failed);
printk(KERN_INFO "kvm: enabling virtualization on "
"CPU%d failed\n", cpu);
}
kvm_usage_count++;
if (kvm_usage_count == 1) {
- atomic_set(&hardware_enable_failed, 0);
+ atomic_set_unchecked(&hardware_enable_failed, 0);
on_each_cpu(hardware_enable_nolock, NULL, 1);
- if (atomic_read(&hardware_enable_failed)) {
+ if (atomic_read_unchecked(&hardware_enable_failed)) {
hardware_disable_all_nolock();
r = -EBUSY;
}
kvm_arch_vcpu_put(vcpu);
}
-int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
+int kvm_init(const void *opaque, unsigned vcpu_size, unsigned vcpu_align,
struct module *module)
{
int r;
if (!vcpu_align)
vcpu_align = __alignof__(struct kvm_vcpu);
kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align,
- 0, NULL);
+ SLAB_USERCOPY, NULL);
if (!kvm_vcpu_cache) {
r = -ENOMEM;
goto out_free_3;
if (r)
goto out_free;
+ pax_open_kernel();
kvm_chardev_ops.owner = module;
kvm_vm_fops.owner = module;
kvm_vcpu_fops.owner = module;
+ pax_close_kernel();
r = misc_register(&kvm_dev);
if (r) {
register_syscore_ops(&kvm_syscore_ops);
- kvm_preempt_ops.sched_in = kvm_sched_in;
- kvm_preempt_ops.sched_out = kvm_sched_out;
-
r = kvm_init_debug();
if (r) {
printk(KERN_ERR "kvm: create debugfs files failed\n");
projects / people / ms / linux.git / commitdiff
