[thirdparty/kernel/linux.git] / arch / x86 / lib / hweight.S

/* SPDX-License-Identifier: GPL-2.0 */
#include <linux/linkage.h>
#include <asm/export.h>

#include <asm/asm.h>

/*
 * unsigned int __sw_hweight32(unsigned int w)
 * %rdi: w
 */
ENTRY(__sw_hweight32)

#ifdef CONFIG_X86_64
	movl %edi, %eax				# w
#endif
	__ASM_SIZE(push,) %__ASM_REG(dx)
	movl %eax, %edx				# w -> t
	shrl %edx				# t >>= 1
	andl $0x55555555, %edx			# t &= 0x55555555
	subl %edx, %eax				# w -= t

	movl %eax, %edx				# w -> t
	shrl $2, %eax				# w_tmp >>= 2
	andl $0x33333333, %edx			# t	&= 0x33333333
	andl $0x33333333, %eax			# w_tmp &= 0x33333333
	addl %edx, %eax				# w = w_tmp + t

	movl %eax, %edx				# w -> t
	shrl $4, %edx				# t >>= 4
	addl %edx, %eax				# w_tmp += t
	andl  $0x0f0f0f0f, %eax			# w_tmp &= 0x0f0f0f0f
	imull $0x01010101, %eax, %eax		# w_tmp *= 0x01010101
	shrl $24, %eax				# w = w_tmp >> 24
	__ASM_SIZE(pop,) %__ASM_REG(dx)
	ret
ENDPROC(__sw_hweight32)
EXPORT_SYMBOL(__sw_hweight32)

ENTRY(__sw_hweight64)
#ifdef CONFIG_X86_64
	pushq   %rdi
	pushq   %rdx

	movq    %rdi, %rdx                      # w -> t
	movabsq $0x5555555555555555, %rax
	shrq    %rdx                            # t >>= 1
	andq    %rdx, %rax                      # t &= 0x5555555555555555
	movabsq $0x3333333333333333, %rdx
	subq    %rax, %rdi                      # w -= t

	movq    %rdi, %rax                      # w -> t
	shrq    $2, %rdi                        # w_tmp >>= 2
	andq    %rdx, %rax                      # t     &= 0x3333333333333333
	andq    %rdi, %rdx                      # w_tmp &= 0x3333333333333333
	addq    %rdx, %rax                      # w = w_tmp + t

	movq    %rax, %rdx                      # w -> t
	shrq    $4, %rdx                        # t >>= 4
	addq    %rdx, %rax                      # w_tmp += t
	movabsq $0x0f0f0f0f0f0f0f0f, %rdx
	andq    %rdx, %rax                      # w_tmp &= 0x0f0f0f0f0f0f0f0f
	movabsq $0x0101010101010101, %rdx
	imulq   %rdx, %rax                      # w_tmp *= 0x0101010101010101
	shrq    $56, %rax                       # w = w_tmp >> 56

	popq    %rdx
	popq    %rdi
	ret
#else /* CONFIG_X86_32 */
	/* We're getting an u64 arg in (%eax,%edx): unsigned long hweight64(__u64 w) */
	pushl   %ecx

	call    __sw_hweight32
	movl    %eax, %ecx                      # stash away result
	movl    %edx, %eax                      # second part of input
	call    __sw_hweight32
	addl    %ecx, %eax                      # result

	popl    %ecx
	ret
#endif
ENDPROC(__sw_hweight64)
EXPORT_SYMBOL(__sw_hweight64)
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
f5967101	2	#include <linux/linkage.h>
784d5699	3	#include <asm/export.h>
f5967101 BP	4
	5	#include <asm/asm.h>
	6
	7	/*
	8	* unsigned int __sw_hweight32(unsigned int w)
	9	* %rdi: w
	10	*/
	11	ENTRY(__sw_hweight32)
	12
	13	#ifdef CONFIG_X86_64
	14	movl %edi, %eax # w
	15	#endif
	16	__ASM_SIZE(push,) %__ASM_REG(dx)
	17	movl %eax, %edx # w -> t
	18	shrl %edx # t >>= 1
	19	andl $0x55555555, %edx # t &= 0x55555555
	20	subl %edx, %eax # w -= t
	21
	22	movl %eax, %edx # w -> t
	23	shrl $2, %eax # w_tmp >>= 2
	24	andl $0x33333333, %edx # t &= 0x33333333
	25	andl $0x33333333, %eax # w_tmp &= 0x33333333
	26	addl %edx, %eax # w = w_tmp + t
	27
	28	movl %eax, %edx # w -> t
	29	shrl $4, %edx # t >>= 4
	30	addl %edx, %eax # w_tmp += t
	31	andl $0x0f0f0f0f, %eax # w_tmp &= 0x0f0f0f0f
	32	imull $0x01010101, %eax, %eax # w_tmp *= 0x01010101
	33	shrl $24, %eax # w = w_tmp >> 24
	34	__ASM_SIZE(pop,) %__ASM_REG(dx)
	35	ret
	36	ENDPROC(__sw_hweight32)
784d5699	37	EXPORT_SYMBOL(__sw_hweight32)
f5967101 BP	38
	39	ENTRY(__sw_hweight64)
	40	#ifdef CONFIG_X86_64
65ea11ec	41	pushq %rdi
f5967101 BP	42	pushq %rdx
	43
	44	movq %rdi, %rdx # w -> t
	45	movabsq $0x5555555555555555, %rax
	46	shrq %rdx # t >>= 1
	47	andq %rdx, %rax # t &= 0x5555555555555555
	48	movabsq $0x3333333333333333, %rdx
	49	subq %rax, %rdi # w -= t
	50
	51	movq %rdi, %rax # w -> t
	52	shrq $2, %rdi # w_tmp >>= 2
	53	andq %rdx, %rax # t &= 0x3333333333333333
	54	andq %rdi, %rdx # w_tmp &= 0x3333333333333333
	55	addq %rdx, %rax # w = w_tmp + t
	56
	57	movq %rax, %rdx # w -> t
	58	shrq $4, %rdx # t >>= 4
	59	addq %rdx, %rax # w_tmp += t
	60	movabsq $0x0f0f0f0f0f0f0f0f, %rdx
	61	andq %rdx, %rax # w_tmp &= 0x0f0f0f0f0f0f0f0f
	62	movabsq $0x0101010101010101, %rdx
	63	imulq %rdx, %rax # w_tmp *= 0x0101010101010101
	64	shrq $56, %rax # w = w_tmp >> 56
	65
	66	popq %rdx
65ea11ec	67	popq %rdi
f5967101 BP	68	ret
	69	#else /* CONFIG_X86_32 */
	70	/* We're getting an u64 arg in (%eax,%edx): unsigned long hweight64(__u64 w) */
	71	pushl %ecx
	72
	73	call __sw_hweight32
	74	movl %eax, %ecx # stash away result
	75	movl %edx, %eax # second part of input
	76	call __sw_hweight32
	77	addl %ecx, %eax # result
	78
	79	popl %ecx
	80	ret
	81	#endif
	82	ENDPROC(__sw_hweight64)
784d5699	83	EXPORT_SYMBOL(__sw_hweight64)