[thirdparty/openssl.git] / crypto / ppccap.c

/*
 * Copyright 2009-2021 The OpenSSL Project Authors. All Rights Reserved.
 *
 * Licensed under the Apache License 2.0 (the "License").  You may not use
 * this file except in compliance with the License.  You can obtain a copy
 * in the file LICENSE in the source distribution or at
 * https://www.openssl.org/source/license.html
 */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <setjmp.h>
#include <signal.h>
#include <unistd.h>
#if defined(__linux) || defined(_AIX)
# include <sys/utsname.h>
#endif
#if defined(_AIX53)     /* defined even on post-5.3 */
# include <sys/systemcfg.h>
# if !defined(__power_set)
#  define __power_set(a) (_system_configuration.implementation & (a))
# endif
#endif
#if defined(__APPLE__) && defined(__MACH__)
# include <sys/types.h>
# include <sys/sysctl.h>
#endif
#include <openssl/crypto.h>
#include <openssl/bn.h>
#include <internal/cryptlib.h>
#include <crypto/chacha.h>
#include "bn/bn_local.h"

#include "ppc_arch.h"

unsigned int OPENSSL_ppccap_P = 0;

static sigset_t all_masked;


#ifdef OPENSSL_BN_ASM_MONT
int bn_mul_mont(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp,
                const BN_ULONG *np, const BN_ULONG *n0, int num)
{
    int bn_mul_mont_int(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp,
                        const BN_ULONG *np, const BN_ULONG *n0, int num);
    int bn_mul4x_mont_int(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp,
                          const BN_ULONG *np, const BN_ULONG *n0, int num);
    int bn_mul_mont_fixed_n6(BN_ULONG *rp, const BN_ULONG *ap,
                             const BN_ULONG *bp, const BN_ULONG *np,
                             const BN_ULONG *n0, int num);
    int bn_mul_mont_300_fixed_n6(BN_ULONG *rp, const BN_ULONG *ap,
                                 const BN_ULONG *bp, const BN_ULONG *np,
                                 const BN_ULONG *n0, int num);

    if (num < 4)
        return 0;

    if ((num & 3) == 0)
        return bn_mul4x_mont_int(rp, ap, bp, np, n0, num);

    /*
     * There used to be [optional] call to bn_mul_mont_fpu64 here,
     * but above subroutine is faster on contemporary processors.
     * Formulation means that there might be old processors where
     * FPU code path would be faster, POWER6 perhaps, but there was
     * no opportunity to figure it out...
     */

    if (num == 6)
        if (OPENSSL_ppccap_P & PPC_MADD300)
            return bn_mul_mont_300_fixed_n6(rp, ap, bp, np, n0, num);
        else
            return bn_mul_mont_fixed_n6(rp, ap, bp, np, n0, num);

    return bn_mul_mont_int(rp, ap, bp, np, n0, num);
}
#endif
void sha256_block_p8(void *ctx, const void *inp, size_t len);
void sha256_block_ppc(void *ctx, const void *inp, size_t len);
void sha256_block_data_order(void *ctx, const void *inp, size_t len);
void sha256_block_data_order(void *ctx, const void *inp, size_t len)
{
    OPENSSL_ppccap_P & PPC_CRYPTO207 ? sha256_block_p8(ctx, inp, len) :
        sha256_block_ppc(ctx, inp, len);
}

void sha512_block_p8(void *ctx, const void *inp, size_t len);
void sha512_block_ppc(void *ctx, const void *inp, size_t len);
void sha512_block_data_order(void *ctx, const void *inp, size_t len);
void sha512_block_data_order(void *ctx, const void *inp, size_t len)
{
    OPENSSL_ppccap_P & PPC_CRYPTO207 ? sha512_block_p8(ctx, inp, len) :
        sha512_block_ppc(ctx, inp, len);
}

#ifndef FIPS_MODULE
# ifndef OPENSSL_NO_CHACHA
void ChaCha20_ctr32_int(unsigned char *out, const unsigned char *inp,
                        size_t len, const unsigned int key[8],
                        const unsigned int counter[4]);
void ChaCha20_ctr32_vmx(unsigned char *out, const unsigned char *inp,
                        size_t len, const unsigned int key[8],
                        const unsigned int counter[4]);
void ChaCha20_ctr32_vsx(unsigned char *out, const unsigned char *inp,
                        size_t len, const unsigned int key[8],
                        const unsigned int counter[4]);
void ChaCha20_ctr32(unsigned char *out, const unsigned char *inp,
                    size_t len, const unsigned int key[8],
                    const unsigned int counter[4])
{
    OPENSSL_ppccap_P & PPC_CRYPTO207
        ? ChaCha20_ctr32_vsx(out, inp, len, key, counter)
        : OPENSSL_ppccap_P & PPC_ALTIVEC
            ? ChaCha20_ctr32_vmx(out, inp, len, key, counter)
            : ChaCha20_ctr32_int(out, inp, len, key, counter);
}
# endif

# ifndef OPENSSL_NO_POLY1305
void poly1305_init_int(void *ctx, const unsigned char key[16]);
void poly1305_blocks(void *ctx, const unsigned char *inp, size_t len,
                         unsigned int padbit);
void poly1305_emit(void *ctx, unsigned char mac[16],
                       const unsigned int nonce[4]);
void poly1305_init_fpu(void *ctx, const unsigned char key[16]);
void poly1305_blocks_fpu(void *ctx, const unsigned char *inp, size_t len,
                         unsigned int padbit);
void poly1305_emit_fpu(void *ctx, unsigned char mac[16],
                       const unsigned int nonce[4]);
void poly1305_init_vsx(void *ctx, const unsigned char key[16]);
void poly1305_blocks_vsx(void *ctx, const unsigned char *inp, size_t len,
                         unsigned int padbit);
void poly1305_emit_vsx(void *ctx, unsigned char mac[16],
                       const unsigned int nonce[4]);
int poly1305_init(void *ctx, const unsigned char key[16], void *func[2]);
int poly1305_init(void *ctx, const unsigned char key[16], void *func[2])
{
    if (OPENSSL_ppccap_P & PPC_CRYPTO207) {
        poly1305_init_int(ctx, key);
        func[0] = (void*)(uintptr_t)poly1305_blocks_vsx;
        func[1] = (void*)(uintptr_t)poly1305_emit;
    } else if (sizeof(size_t) == 4 && (OPENSSL_ppccap_P & PPC_FPU)) {
        poly1305_init_fpu(ctx, key);
        func[0] = (void*)(uintptr_t)poly1305_blocks_fpu;
        func[1] = (void*)(uintptr_t)poly1305_emit_fpu;
    } else {
        poly1305_init_int(ctx, key);
        func[0] = (void*)(uintptr_t)poly1305_blocks;
        func[1] = (void*)(uintptr_t)poly1305_emit;
    }
    return 1;
}
# endif
#endif /* FIPS_MODULE */

#ifdef ECP_NISTZ256_ASM
void ecp_nistz256_mul_mont(unsigned long res[4], const unsigned long a[4],
                           const unsigned long b[4]);

void ecp_nistz256_to_mont(unsigned long res[4], const unsigned long in[4]);
void ecp_nistz256_to_mont(unsigned long res[4], const unsigned long in[4])
{
    static const unsigned long RR[] = { 0x0000000000000003U,
                                        0xfffffffbffffffffU,
                                        0xfffffffffffffffeU,
                                        0x00000004fffffffdU };

    ecp_nistz256_mul_mont(res, in, RR);
}

void ecp_nistz256_from_mont(unsigned long res[4], const unsigned long in[4]);
void ecp_nistz256_from_mont(unsigned long res[4], const unsigned long in[4])
{
    static const unsigned long one[] = { 1, 0, 0, 0 };

    ecp_nistz256_mul_mont(res, in, one);
}
#endif

static sigjmp_buf ill_jmp;
static void ill_handler(int sig)
{
    siglongjmp(ill_jmp, sig);
}

void OPENSSL_fpu_probe(void);
void OPENSSL_ppc64_probe(void);
void OPENSSL_altivec_probe(void);
void OPENSSL_crypto207_probe(void);
void OPENSSL_madd300_probe(void);

long OPENSSL_rdtsc_mftb(void);
long OPENSSL_rdtsc_mfspr268(void);

uint32_t OPENSSL_rdtsc(void)
{
    if (OPENSSL_ppccap_P & PPC_MFTB)
        return OPENSSL_rdtsc_mftb();
    else if (OPENSSL_ppccap_P & PPC_MFSPR268)
        return OPENSSL_rdtsc_mfspr268();
    else
        return 0;
}

size_t OPENSSL_instrument_bus_mftb(unsigned int *, size_t);
size_t OPENSSL_instrument_bus_mfspr268(unsigned int *, size_t);

size_t OPENSSL_instrument_bus(unsigned int *out, size_t cnt)
{
    if (OPENSSL_ppccap_P & PPC_MFTB)
        return OPENSSL_instrument_bus_mftb(out, cnt);
    else if (OPENSSL_ppccap_P & PPC_MFSPR268)
        return OPENSSL_instrument_bus_mfspr268(out, cnt);
    else
        return 0;
}

size_t OPENSSL_instrument_bus2_mftb(unsigned int *, size_t, size_t);
size_t OPENSSL_instrument_bus2_mfspr268(unsigned int *, size_t, size_t);

size_t OPENSSL_instrument_bus2(unsigned int *out, size_t cnt, size_t max)
{
    if (OPENSSL_ppccap_P & PPC_MFTB)
        return OPENSSL_instrument_bus2_mftb(out, cnt, max);
    else if (OPENSSL_ppccap_P & PPC_MFSPR268)
        return OPENSSL_instrument_bus2_mfspr268(out, cnt, max);
    else
        return 0;
}

#if defined(__GLIBC__) && defined(__GLIBC_PREREQ)
# if __GLIBC_PREREQ(2, 16)
#  include <sys/auxv.h>
#  define OSSL_IMPLEMENT_GETAUXVAL
# endif
#endif

#if defined(__FreeBSD__)
# include <sys/param.h>
# if __FreeBSD_version >= 1200000
#  include <sys/auxv.h>
#  define OSSL_IMPLEMENT_GETAUXVAL

static unsigned long getauxval(unsigned long key)
{
  unsigned long val = 0ul;

  if (elf_aux_info((int)key, &val, sizeof(val)) != 0)
    return 0ul;

  return val;
}
# endif
#endif

/* I wish <sys/auxv.h> was universally available */
#define HWCAP                   16      /* AT_HWCAP */
#define HWCAP_PPC64             (1U << 30)
#define HWCAP_ALTIVEC           (1U << 28)
#define HWCAP_FPU               (1U << 27)
#define HWCAP_POWER6_EXT        (1U << 9)
#define HWCAP_VSX               (1U << 7)

#define HWCAP2                  26      /* AT_HWCAP2 */
#define HWCAP_VEC_CRYPTO        (1U << 25)
#define HWCAP_ARCH_3_00         (1U << 23)

# if defined(__GNUC__) && __GNUC__>=2
__attribute__ ((constructor))
# endif
void OPENSSL_cpuid_setup(void)
{
    char *e;
    struct sigaction ill_oact, ill_act;
    sigset_t oset;
    static int trigger = 0;

    if (trigger)
        return;
    trigger = 1;

    if ((e = getenv("OPENSSL_ppccap"))) {
        OPENSSL_ppccap_P = strtoul(e, NULL, 0);
        return;
    }

    OPENSSL_ppccap_P = 0;

#if defined(_AIX)
    OPENSSL_ppccap_P |= PPC_FPU;

    if (sizeof(size_t) == 4) {
        struct utsname uts;
# if defined(_SC_AIX_KERNEL_BITMODE)
        if (sysconf(_SC_AIX_KERNEL_BITMODE) != 64)
            return;
# endif
        if (uname(&uts) != 0 || atoi(uts.version) < 6)
            return;
    }

# if defined(__power_set)
    /*
     * Value used in __power_set is a single-bit 1<<n one denoting
     * specific processor class. Incidentally 0xffffffff<<n can be
     * used to denote specific processor and its successors.
     */
    if (sizeof(size_t) == 4) {
        /* In 32-bit case PPC_FPU64 is always fastest [if option] */
        if (__power_set(0xffffffffU<<13))       /* POWER5 and later */
            OPENSSL_ppccap_P |= PPC_FPU64;
    } else {
        /* In 64-bit case PPC_FPU64 is fastest only on POWER6 */
        if (__power_set(0x1U<<14))              /* POWER6 */
            OPENSSL_ppccap_P |= PPC_FPU64;
    }

    if (__power_set(0xffffffffU<<14))           /* POWER6 and later */
        OPENSSL_ppccap_P |= PPC_ALTIVEC;

    if (__power_set(0xffffffffU<<16))           /* POWER8 and later */
        OPENSSL_ppccap_P |= PPC_CRYPTO207;

    if (__power_set(0xffffffffU<<17))           /* POWER9 and later */
        OPENSSL_ppccap_P |= PPC_MADD300;

    return;
# endif
#endif

#if defined(__APPLE__) && defined(__MACH__)
    OPENSSL_ppccap_P |= PPC_FPU;

    {
        int val;
        size_t len = sizeof(val);

        if (sysctlbyname("hw.optional.64bitops", &val, &len, NULL, 0) == 0) {
            if (val)
                OPENSSL_ppccap_P |= PPC_FPU64;
        }

        len = sizeof(val);
        if (sysctlbyname("hw.optional.altivec", &val, &len, NULL, 0) == 0) {
            if (val)
                OPENSSL_ppccap_P |= PPC_ALTIVEC;
        }

        return;
    }
#endif

#ifdef OSSL_IMPLEMENT_GETAUXVAL
    {
        unsigned long hwcap = getauxval(HWCAP);
        unsigned long hwcap2 = getauxval(HWCAP2);

        if (hwcap & HWCAP_FPU) {
            OPENSSL_ppccap_P |= PPC_FPU;

            if (sizeof(size_t) == 4) {
                /* In 32-bit case PPC_FPU64 is always fastest [if option] */
                if (hwcap & HWCAP_PPC64)
                    OPENSSL_ppccap_P |= PPC_FPU64;
            } else {
                /* In 64-bit case PPC_FPU64 is fastest only on POWER6 */
                if (hwcap & HWCAP_POWER6_EXT)
                    OPENSSL_ppccap_P |= PPC_FPU64;
            }
        }

        if (hwcap & HWCAP_ALTIVEC) {
            OPENSSL_ppccap_P |= PPC_ALTIVEC;

            if ((hwcap & HWCAP_VSX) && (hwcap2 & HWCAP_VEC_CRYPTO))
                OPENSSL_ppccap_P |= PPC_CRYPTO207;
        }

        if (hwcap2 & HWCAP_ARCH_3_00) {
            OPENSSL_ppccap_P |= PPC_MADD300;
        }
    }
#endif

    sigfillset(&all_masked);
    sigdelset(&all_masked, SIGILL);
    sigdelset(&all_masked, SIGTRAP);
#ifdef SIGEMT
    sigdelset(&all_masked, SIGEMT);
#endif
    sigdelset(&all_masked, SIGFPE);
    sigdelset(&all_masked, SIGBUS);
    sigdelset(&all_masked, SIGSEGV);

    memset(&ill_act, 0, sizeof(ill_act));
    ill_act.sa_handler = ill_handler;
    ill_act.sa_mask = all_masked;

    sigprocmask(SIG_SETMASK, &ill_act.sa_mask, &oset);
    sigaction(SIGILL, &ill_act, &ill_oact);

#ifndef OSSL_IMPLEMENT_GETAUXVAL
    if (sigsetjmp(ill_jmp,1) == 0) {
        OPENSSL_fpu_probe();
        OPENSSL_ppccap_P |= PPC_FPU;

        if (sizeof(size_t) == 4) {
# ifdef __linux
            struct utsname uts;
            if (uname(&uts) == 0 && strcmp(uts.machine, "ppc64") == 0)
# endif
                if (sigsetjmp(ill_jmp, 1) == 0) {
                    OPENSSL_ppc64_probe();
                    OPENSSL_ppccap_P |= PPC_FPU64;
                }
        } else {
            /*
             * Wanted code detecting POWER6 CPU and setting PPC_FPU64
             */
        }
    }

    if (sigsetjmp(ill_jmp, 1) == 0) {
        OPENSSL_altivec_probe();
        OPENSSL_ppccap_P |= PPC_ALTIVEC;
        if (sigsetjmp(ill_jmp, 1) == 0) {
            OPENSSL_crypto207_probe();
            OPENSSL_ppccap_P |= PPC_CRYPTO207;
        }
    }

    if (sigsetjmp(ill_jmp, 1) == 0) {
        OPENSSL_madd300_probe();
        OPENSSL_ppccap_P |= PPC_MADD300;
    }
#endif

    if (sigsetjmp(ill_jmp, 1) == 0) {
        OPENSSL_rdtsc_mftb();
        OPENSSL_ppccap_P |= PPC_MFTB;
    } else if (sigsetjmp(ill_jmp, 1) == 0) {
        OPENSSL_rdtsc_mfspr268();
        OPENSSL_ppccap_P |= PPC_MFSPR268;
    }

    sigaction(SIGILL, &ill_oact, NULL);
    sigprocmask(SIG_SETMASK, &oset, NULL);
}
Commit	Line	Data
b1322259	1	/*
4333b89f	2	* Copyright 2009-2021 The OpenSSL Project Authors. All Rights Reserved.
b1322259	3	*
0e9725bc	4	* Licensed under the Apache License 2.0 (the "License"). You may not use
b1322259 RS	5	* this file except in compliance with the License. You can obtain a copy
	6	* in the file LICENSE in the source distribution or at
	7	* https://www.openssl.org/source/license.html
	8	*/
	9
b4b48a10 AP	10	#include <stdio.h>
	11	#include <stdlib.h>
	12	#include <string.h>
	13	#include <setjmp.h>
	14	#include <signal.h>
fd054957	15	#include <unistd.h>
d5630dd6	16	#if defined(__linux) \|\| defined(_AIX)
0f113f3e	17	# include <sys/utsname.h>
78c3e205	18	#endif
2688d999 AP	19	#if defined(_AIX53) /* defined even on post-5.3 */
	20	# include <sys/systemcfg.h>
	21	# if !defined(__power_set)
	22	# define __power_set(a) (_system_configuration.implementation & (a))
	23	# endif
	24	#endif
0bd93bbe AP	25	#if defined(__APPLE__) && defined(__MACH__)
	26	# include <sys/types.h>
	27	# include <sys/sysctl.h>
	28	#endif
11252459	29	#include <openssl/crypto.h>
10b0b5ec	30	#include <openssl/bn.h>
8c8fbca9	31	#include <internal/cryptlib.h>
25f2138b	32	#include <crypto/chacha.h>
706457b7	33	#include "bn/bn_local.h"
b4b48a10	34
0e716d92	35	#include "ppc_arch.h"
b4b48a10	36
07f3e4f3	37	unsigned int OPENSSL_ppccap_P = 0;
b4b48a10 AP	38
	39	static sigset_t all_masked;
	40
a8140a42 MC	41
a8140a42 MC	42	#ifdef OPENSSL_BN_ASM_MONT
10b0b5ec RL	43	int bn_mul_mont(BN_ULONG rp, const BN_ULONG ap, const BN_ULONG *bp,
	44	const BN_ULONG np, const BN_ULONG n0, int num)
	45	{
	46	int bn_mul_mont_int(BN_ULONG rp, const BN_ULONG ap, const BN_ULONG *bp,
	47	const BN_ULONG np, const BN_ULONG n0, int num);
	48	int bn_mul4x_mont_int(BN_ULONG rp, const BN_ULONG ap, const BN_ULONG *bp,
	49	const BN_ULONG np, const BN_ULONG n0, int num);
0d40ca47 MS	50	int bn_mul_mont_fixed_n6(BN_ULONG rp, const BN_ULONG ap,
	51	const BN_ULONG bp, const BN_ULONG np,
	52	const BN_ULONG *n0, int num);
	53	int bn_mul_mont_300_fixed_n6(BN_ULONG rp, const BN_ULONG ap,
	54	const BN_ULONG bp, const BN_ULONG np,
	55	const BN_ULONG *n0, int num);
10b0b5ec RL	56
	57	if (num < 4)
	58	return 0;
	59
	60	if ((num & 3) == 0)
	61	return bn_mul4x_mont_int(rp, ap, bp, np, n0, num);
	62
	63	/*
	64	* There used to be [optional] call to bn_mul_mont_fpu64 here,
	65	* but above subroutine is faster on contemporary processors.
	66	* Formulation means that there might be old processors where
	67	* FPU code path would be faster, POWER6 perhaps, but there was
	68	* no opportunity to figure it out...
	69	*/
	70
0d40ca47 MS	71	if (num == 6)
	72	if (OPENSSL_ppccap_P & PPC_MADD300)
	73	return bn_mul_mont_300_fixed_n6(rp, ap, bp, np, n0, num);
	74	else
	75	return bn_mul_mont_fixed_n6(rp, ap, bp, np, n0, num);
	76
10b0b5ec RL	77	return bn_mul_mont_int(rp, ap, bp, np, n0, num);
10b0b5ec RL	78	}
a8140a42	79	#endif
10b0b5ec RL	80	void sha256_block_p8(void ctx, const void inp, size_t len);
10b0b5ec RL	81	void sha256_block_ppc(void ctx, const void inp, size_t len);
8c8fbca9	82	void sha256_block_data_order(void ctx, const void inp, size_t len);
10b0b5ec RL	83	void sha256_block_data_order(void ctx, const void inp, size_t len)
	84	{
	85	OPENSSL_ppccap_P & PPC_CRYPTO207 ? sha256_block_p8(ctx, inp, len) :
	86	sha256_block_ppc(ctx, inp, len);
	87	}
	88
	89	void sha512_block_p8(void ctx, const void inp, size_t len);
	90	void sha512_block_ppc(void ctx, const void inp, size_t len);
8c8fbca9	91	void sha512_block_data_order(void ctx, const void inp, size_t len);
10b0b5ec RL	92	void sha512_block_data_order(void ctx, const void inp, size_t len)
	93	{
	94	OPENSSL_ppccap_P & PPC_CRYPTO207 ? sha512_block_p8(ctx, inp, len) :
	95	sha512_block_ppc(ctx, inp, len);
	96	}
	97
f844f9eb	98	#ifndef FIPS_MODULE
41525ed6	99	# ifndef OPENSSL_NO_CHACHA
10b0b5ec RL	100	void ChaCha20_ctr32_int(unsigned char out, const unsigned char inp,
	101	size_t len, const unsigned int key[8],
	102	const unsigned int counter[4]);
	103	void ChaCha20_ctr32_vmx(unsigned char out, const unsigned char inp,
	104	size_t len, const unsigned int key[8],
	105	const unsigned int counter[4]);
316d527f AP	106	void ChaCha20_ctr32_vsx(unsigned char out, const unsigned char inp,
	107	size_t len, const unsigned int key[8],
	108	const unsigned int counter[4]);
10b0b5ec RL	109	void ChaCha20_ctr32(unsigned char out, const unsigned char inp,
	110	size_t len, const unsigned int key[8],
	111	const unsigned int counter[4])
	112	{
316d527f AP	113	OPENSSL_ppccap_P & PPC_CRYPTO207
	114	? ChaCha20_ctr32_vsx(out, inp, len, key, counter)
	115	: OPENSSL_ppccap_P & PPC_ALTIVEC
	116	? ChaCha20_ctr32_vmx(out, inp, len, key, counter)
	117	: ChaCha20_ctr32_int(out, inp, len, key, counter);
10b0b5ec	118	}
41525ed6	119	# endif
10b0b5ec	120
41525ed6	121	# ifndef OPENSSL_NO_POLY1305
10b0b5ec RL	122	void poly1305_init_int(void *ctx, const unsigned char key[16]);
	123	void poly1305_blocks(void ctx, const unsigned char inp, size_t len,
	124	unsigned int padbit);
	125	void poly1305_emit(void *ctx, unsigned char mac[16],
	126	const unsigned int nonce[4]);
	127	void poly1305_init_fpu(void *ctx, const unsigned char key[16]);
	128	void poly1305_blocks_fpu(void ctx, const unsigned char inp, size_t len,
	129	unsigned int padbit);
	130	void poly1305_emit_fpu(void *ctx, unsigned char mac[16],
	131	const unsigned int nonce[4]);
a28e4890 AP	132	void poly1305_init_vsx(void *ctx, const unsigned char key[16]);
	133	void poly1305_blocks_vsx(void ctx, const unsigned char inp, size_t len,
	134	unsigned int padbit);
	135	void poly1305_emit_vsx(void *ctx, unsigned char mac[16],
	136	const unsigned int nonce[4]);
8c8fbca9	137	int poly1305_init(void ctx, const unsigned char key[16], void func[2]);
10b0b5ec RL	138	int poly1305_init(void ctx, const unsigned char key[16], void func[2])
10b0b5ec RL	139	{
a28e4890 AP	140	if (OPENSSL_ppccap_P & PPC_CRYPTO207) {
	141	poly1305_init_int(ctx, key);
	142	func[0] = (void*)(uintptr_t)poly1305_blocks_vsx;
	143	func[1] = (void*)(uintptr_t)poly1305_emit;
	144	} else if (sizeof(size_t) == 4 && (OPENSSL_ppccap_P & PPC_FPU)) {
10b0b5ec	145	poly1305_init_fpu(ctx, key);
8c8fbca9 BE	146	func[0] = (void*)(uintptr_t)poly1305_blocks_fpu;
8c8fbca9 BE	147	func[1] = (void*)(uintptr_t)poly1305_emit_fpu;
10b0b5ec RL	148	} else {
10b0b5ec RL	149	poly1305_init_int(ctx, key);
8c8fbca9 BE	150	func[0] = (void*)(uintptr_t)poly1305_blocks;
8c8fbca9 BE	151	func[1] = (void*)(uintptr_t)poly1305_emit;
10b0b5ec RL	152	}
	153	return 1;
	154	}
41525ed6	155	# endif
f844f9eb	156	#endif /* FIPS_MODULE */
10b0b5ec	157
a9612d6c	158	#ifdef ECP_NISTZ256_ASM
10b0b5ec RL	159	void ecp_nistz256_mul_mont(unsigned long res[4], const unsigned long a[4],
	160	const unsigned long b[4]);
	161
	162	void ecp_nistz256_to_mont(unsigned long res[4], const unsigned long in[4]);
	163	void ecp_nistz256_to_mont(unsigned long res[4], const unsigned long in[4])
	164	{
	165	static const unsigned long RR[] = { 0x0000000000000003U,
	166	0xfffffffbffffffffU,
	167	0xfffffffffffffffeU,
	168	0x00000004fffffffdU };
	169
	170	ecp_nistz256_mul_mont(res, in, RR);
	171	}
	172
	173	void ecp_nistz256_from_mont(unsigned long res[4], const unsigned long in[4]);
	174	void ecp_nistz256_from_mont(unsigned long res[4], const unsigned long in[4])
	175	{
	176	static const unsigned long one[] = { 1, 0, 0, 0 };
	177
	178	ecp_nistz256_mul_mont(res, in, one);
	179	}
a9612d6c	180	#endif
10b0b5ec	181
b4b48a10	182	static sigjmp_buf ill_jmp;
0f113f3e MC	183	static void ill_handler(int sig)
	184	{
	185	siglongjmp(ill_jmp, sig);
	186	}
b4b48a10	187
2688d999	188	void OPENSSL_fpu_probe(void);
70b76d39	189	void OPENSSL_ppc64_probe(void);
fd054957	190	void OPENSSL_altivec_probe(void);
de51e830	191	void OPENSSL_crypto207_probe(void);
53385e1f	192	void OPENSSL_madd300_probe(void);
70b76d39	193
c8f37048 BE	194	long OPENSSL_rdtsc_mftb(void);
	195	long OPENSSL_rdtsc_mfspr268(void);
	196
	197	uint32_t OPENSSL_rdtsc(void)
	198	{
	199	if (OPENSSL_ppccap_P & PPC_MFTB)
	200	return OPENSSL_rdtsc_mftb();
	201	else if (OPENSSL_ppccap_P & PPC_MFSPR268)
	202	return OPENSSL_rdtsc_mfspr268();
	203	else
	204	return 0;
	205	}
	206
	207	size_t OPENSSL_instrument_bus_mftb(unsigned int *, size_t);
	208	size_t OPENSSL_instrument_bus_mfspr268(unsigned int *, size_t);
	209
	210	size_t OPENSSL_instrument_bus(unsigned int *out, size_t cnt)
	211	{
	212	if (OPENSSL_ppccap_P & PPC_MFTB)
	213	return OPENSSL_instrument_bus_mftb(out, cnt);
	214	else if (OPENSSL_ppccap_P & PPC_MFSPR268)
	215	return OPENSSL_instrument_bus_mfspr268(out, cnt);
	216	else
	217	return 0;
	218	}
	219
	220	size_t OPENSSL_instrument_bus2_mftb(unsigned int *, size_t, size_t);
	221	size_t OPENSSL_instrument_bus2_mfspr268(unsigned int *, size_t, size_t);
	222
	223	size_t OPENSSL_instrument_bus2(unsigned int *out, size_t cnt, size_t max)
	224	{
	225	if (OPENSSL_ppccap_P & PPC_MFTB)
	226	return OPENSSL_instrument_bus2_mftb(out, cnt, max);
	227	else if (OPENSSL_ppccap_P & PPC_MFSPR268)
	228	return OPENSSL_instrument_bus2_mfspr268(out, cnt, max);
	229	else
	230	return 0;
	231	}
	232
5f40dd15 RL	233	#if defined(__GLIBC__) && defined(__GLIBC_PREREQ)
	234	# if __GLIBC_PREREQ(2, 16)
	235	# include <sys/auxv.h>
	236	# define OSSL_IMPLEMENT_GETAUXVAL
	237	# endif
2688d999 AP	238	#endif
2688d999 AP	239
b57ec739 DC	240	#if defined(__FreeBSD__)
	241	# include <sys/param.h>
	242	# if __FreeBSD_version >= 1200000
	243	# include <sys/auxv.h>
	244	# define OSSL_IMPLEMENT_GETAUXVAL
	245
	246	static unsigned long getauxval(unsigned long key)
	247	{
	248	unsigned long val = 0ul;
	249
	250	if (elf_aux_info((int)key, &val, sizeof(val)) != 0)
	251	return 0ul;
	252
	253	return val;
	254	}
	255	# endif
	256	#endif
	257
2688d999 AP	258	/* I wish <sys/auxv.h> was universally available */
	259	#define HWCAP 16 /* AT_HWCAP */
	260	#define HWCAP_PPC64 (1U << 30)
	261	#define HWCAP_ALTIVEC (1U << 28)
	262	#define HWCAP_FPU (1U << 27)
	263	#define HWCAP_POWER6_EXT (1U << 9)
	264	#define HWCAP_VSX (1U << 7)
	265
	266	#define HWCAP2 26 /* AT_HWCAP2 */
	267	#define HWCAP_VEC_CRYPTO (1U << 25)
e0e53282	268	#define HWCAP_ARCH_3_00 (1U << 23)
2688d999 AP	269
	270	# if defined(__GNUC__) && __GNUC__>=2
	271	__attribute__ ((constructor))
	272	# endif
b4b48a10	273	void OPENSSL_cpuid_setup(void)
0f113f3e MC	274	{
	275	char *e;
	276	struct sigaction ill_oact, ill_act;
	277	sigset_t oset;
	278	static int trigger = 0;
	279
	280	if (trigger)
	281	return;
	282	trigger = 1;
	283
0f113f3e MC	284	if ((e = getenv("OPENSSL_ppccap"))) {
	285	OPENSSL_ppccap_P = strtoul(e, NULL, 0);
	286	return;
	287	}
b4b48a10	288
0f113f3e	289	OPENSSL_ppccap_P = 0;
6415dd7b	290
fd054957	291	#if defined(_AIX)
2688d999 AP	292	OPENSSL_ppccap_P \|= PPC_FPU;
2688d999 AP	293
0f113f3e MC	294	if (sizeof(size_t) == 4) {
0f113f3e MC	295	struct utsname uts;
fd054957	296	# if defined(_SC_AIX_KERNEL_BITMODE)
0f113f3e MC	297	if (sysconf(_SC_AIX_KERNEL_BITMODE) != 64)
0f113f3e MC	298	return;
fd054957	299	# endif
0f113f3e MC	300	if (uname(&uts) != 0 \|\| atoi(uts.version) < 6)
	301	return;
	302	}
2688d999 AP	303
	304	# if defined(__power_set)
	305	/*
	306	* Value used in __power_set is a single-bit 1<<n one denoting
	307	* specific processor class. Incidentally 0xffffffff<<n can be
	308	* used to denote specific processor and its successors.
	309	*/
	310	if (sizeof(size_t) == 4) {
	311	/* In 32-bit case PPC_FPU64 is always fastest [if option] */
	312	if (__power_set(0xffffffffU<<13)) /* POWER5 and later */
	313	OPENSSL_ppccap_P \|= PPC_FPU64;
	314	} else {
	315	/* In 64-bit case PPC_FPU64 is fastest only on POWER6 */
	316	if (__power_set(0x1U<<14)) /* POWER6 */
	317	OPENSSL_ppccap_P \|= PPC_FPU64;
	318	}
	319
	320	if (__power_set(0xffffffffU<<14)) /* POWER6 and later */
	321	OPENSSL_ppccap_P \|= PPC_ALTIVEC;
	322
	323	if (__power_set(0xffffffffU<<16)) /* POWER8 and later */
	324	OPENSSL_ppccap_P \|= PPC_CRYPTO207;
	325
e0e53282 AP	326	if (__power_set(0xffffffffU<<17)) /* POWER9 and later */
	327	OPENSSL_ppccap_P \|= PPC_MADD300;
	328
2688d999 AP	329	return;
	330	# endif
	331	#endif
	332
0bd93bbe AP	333	#if defined(__APPLE__) && defined(__MACH__)
	334	OPENSSL_ppccap_P \|= PPC_FPU;
	335
	336	{
	337	int val;
	338	size_t len = sizeof(val);
	339
	340	if (sysctlbyname("hw.optional.64bitops", &val, &len, NULL, 0) == 0) {
	341	if (val)
	342	OPENSSL_ppccap_P \|= PPC_FPU64;
	343	}
	344
	345	len = sizeof(val);
	346	if (sysctlbyname("hw.optional.altivec", &val, &len, NULL, 0) == 0) {
	347	if (val)
	348	OPENSSL_ppccap_P \|= PPC_ALTIVEC;
	349	}
	350
	351	return;
	352	}
	353	#endif
	354
5f40dd15 RL	355	#ifdef OSSL_IMPLEMENT_GETAUXVAL
5f40dd15 RL	356	{
2688d999	357	unsigned long hwcap = getauxval(HWCAP);
99592c73	358	unsigned long hwcap2 = getauxval(HWCAP2);
2688d999 AP	359
2688d999 AP	360	if (hwcap & HWCAP_FPU) {
dccd20d1	361	OPENSSL_ppccap_P \|= PPC_FPU;
2688d999 AP	362
	363	if (sizeof(size_t) == 4) {
	364	/* In 32-bit case PPC_FPU64 is always fastest [if option] */
	365	if (hwcap & HWCAP_PPC64)
	366	OPENSSL_ppccap_P \|= PPC_FPU64;
	367	} else {
	368	/* In 64-bit case PPC_FPU64 is fastest only on POWER6 */
	369	if (hwcap & HWCAP_POWER6_EXT)
	370	OPENSSL_ppccap_P \|= PPC_FPU64;
	371	}
	372	}
	373
	374	if (hwcap & HWCAP_ALTIVEC) {
	375	OPENSSL_ppccap_P \|= PPC_ALTIVEC;
	376
99592c73	377	if ((hwcap & HWCAP_VSX) && (hwcap2 & HWCAP_VEC_CRYPTO))
2688d999 AP	378	OPENSSL_ppccap_P \|= PPC_CRYPTO207;
	379	}
	380
99592c73	381	if (hwcap2 & HWCAP_ARCH_3_00) {
e0e53282 AP	382	OPENSSL_ppccap_P \|= PPC_MADD300;
e0e53282 AP	383	}
2688d999	384	}
5f40dd15	385	#endif
2688d999 AP	386
	387	sigfillset(&all_masked);
	388	sigdelset(&all_masked, SIGILL);
	389	sigdelset(&all_masked, SIGTRAP);
	390	#ifdef SIGEMT
	391	sigdelset(&all_masked, SIGEMT);
fd054957	392	#endif
2688d999 AP	393	sigdelset(&all_masked, SIGFPE);
	394	sigdelset(&all_masked, SIGBUS);
	395	sigdelset(&all_masked, SIGSEGV);
fd054957	396
0f113f3e MC	397	memset(&ill_act, 0, sizeof(ill_act));
	398	ill_act.sa_handler = ill_handler;
	399	ill_act.sa_mask = all_masked;
6415dd7b	400
0f113f3e MC	401	sigprocmask(SIG_SETMASK, &ill_act.sa_mask, &oset);
0f113f3e MC	402	sigaction(SIGILL, &ill_act, &ill_oact);
6415dd7b	403
c8f37048	404	#ifndef OSSL_IMPLEMENT_GETAUXVAL
2688d999 AP	405	if (sigsetjmp(ill_jmp,1) == 0) {
	406	OPENSSL_fpu_probe();
	407	OPENSSL_ppccap_P \|= PPC_FPU;
	408
	409	if (sizeof(size_t) == 4) {
c8f37048	410	# ifdef __linux
2688d999 AP	411	struct utsname uts;
2688d999 AP	412	if (uname(&uts) == 0 && strcmp(uts.machine, "ppc64") == 0)
c8f37048	413	# endif
2688d999 AP	414	if (sigsetjmp(ill_jmp, 1) == 0) {
	415	OPENSSL_ppc64_probe();
	416	OPENSSL_ppccap_P \|= PPC_FPU64;
	417	}
	418	} else {
	419	/*
	420	* Wanted code detecting POWER6 CPU and setting PPC_FPU64
	421	*/
	422	}
0f113f3e MC	423	}
	424
	425	if (sigsetjmp(ill_jmp, 1) == 0) {
	426	OPENSSL_altivec_probe();
	427	OPENSSL_ppccap_P \|= PPC_ALTIVEC;
	428	if (sigsetjmp(ill_jmp, 1) == 0) {
	429	OPENSSL_crypto207_probe();
	430	OPENSSL_ppccap_P \|= PPC_CRYPTO207;
	431	}
	432	}
	433
e0e53282 AP	434	if (sigsetjmp(ill_jmp, 1) == 0) {
	435	OPENSSL_madd300_probe();
	436	OPENSSL_ppccap_P \|= PPC_MADD300;
	437	}
c8f37048 BE	438	#endif
	439
	440	if (sigsetjmp(ill_jmp, 1) == 0) {
	441	OPENSSL_rdtsc_mftb();
	442	OPENSSL_ppccap_P \|= PPC_MFTB;
	443	} else if (sigsetjmp(ill_jmp, 1) == 0) {
	444	OPENSSL_rdtsc_mfspr268();
	445	OPENSSL_ppccap_P \|= PPC_MFSPR268;
	446	}
e0e53282	447
0f113f3e MC	448	sigaction(SIGILL, &ill_oact, NULL);
	449	sigprocmask(SIG_SETMASK, &oset, NULL);
	450	}