[thirdparty/openssl.git] / crypto / ppccap.c

/*
 * Copyright 2009-2018 The OpenSSL Project Authors. All Rights Reserved.
 *
 * Licensed under the Apache License 2.0 (the "License").  You may not use
 * this file except in compliance with the License.  You can obtain a copy
 * in the file LICENSE in the source distribution or at
 * https://www.openssl.org/source/license.html
 */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <setjmp.h>
#include <signal.h>
#include <unistd.h>
#if defined(__linux) || defined(_AIX)
# include <sys/utsname.h>
#endif
#if defined(_AIX53)     /* defined even on post-5.3 */
# include <sys/systemcfg.h>
# if !defined(__power_set)
#  define __power_set(a) (_system_configuration.implementation & (a))
# endif
#endif
#if defined(__APPLE__) && defined(__MACH__)
# include <sys/types.h>
# include <sys/sysctl.h>
#endif
#include <openssl/crypto.h>
#include <openssl/bn.h>
#include <internal/cryptlib.h>
#include <internal/chacha.h>
#include "bn/bn_lcl.h"

#include "ppc_arch.h"

unsigned int OPENSSL_ppccap_P = 0;

static sigset_t all_masked;


#ifdef OPENSSL_BN_ASM_MONT
int bn_mul_mont(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp,
                const BN_ULONG *np, const BN_ULONG *n0, int num)
{
    int bn_mul_mont_int(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp,
                        const BN_ULONG *np, const BN_ULONG *n0, int num);
    int bn_mul4x_mont_int(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp,
                          const BN_ULONG *np, const BN_ULONG *n0, int num);

    if (num < 4)
        return 0;

    if ((num & 3) == 0)
        return bn_mul4x_mont_int(rp, ap, bp, np, n0, num);

    /*
     * There used to be [optional] call to bn_mul_mont_fpu64 here,
     * but above subroutine is faster on contemporary processors.
     * Formulation means that there might be old processors where
     * FPU code path would be faster, POWER6 perhaps, but there was
     * no opportunity to figure it out...
     */

    return bn_mul_mont_int(rp, ap, bp, np, n0, num);
}
#endif
void sha256_block_p8(void *ctx, const void *inp, size_t len);
void sha256_block_ppc(void *ctx, const void *inp, size_t len);
void sha256_block_data_order(void *ctx, const void *inp, size_t len);
void sha256_block_data_order(void *ctx, const void *inp, size_t len)
{
    OPENSSL_ppccap_P & PPC_CRYPTO207 ? sha256_block_p8(ctx, inp, len) :
        sha256_block_ppc(ctx, inp, len);
}

void sha512_block_p8(void *ctx, const void *inp, size_t len);
void sha512_block_ppc(void *ctx, const void *inp, size_t len);
void sha512_block_data_order(void *ctx, const void *inp, size_t len);
void sha512_block_data_order(void *ctx, const void *inp, size_t len)
{
    OPENSSL_ppccap_P & PPC_CRYPTO207 ? sha512_block_p8(ctx, inp, len) :
        sha512_block_ppc(ctx, inp, len);
}

/*
 * TODO(3.0): Temporarily disabled some assembler that hasn't been brought into
 * the FIPS module yet.
 */
#ifndef FIPS_MODE
# ifndef OPENSSL_NO_CHACHA
void ChaCha20_ctr32_int(unsigned char *out, const unsigned char *inp,
                        size_t len, const unsigned int key[8],
                        const unsigned int counter[4]);
void ChaCha20_ctr32_vmx(unsigned char *out, const unsigned char *inp,
                        size_t len, const unsigned int key[8],
                        const unsigned int counter[4]);
void ChaCha20_ctr32_vsx(unsigned char *out, const unsigned char *inp,
                        size_t len, const unsigned int key[8],
                        const unsigned int counter[4]);
void ChaCha20_ctr32(unsigned char *out, const unsigned char *inp,
                    size_t len, const unsigned int key[8],
                    const unsigned int counter[4])
{
    OPENSSL_ppccap_P & PPC_CRYPTO207
        ? ChaCha20_ctr32_vsx(out, inp, len, key, counter)
        : OPENSSL_ppccap_P & PPC_ALTIVEC
            ? ChaCha20_ctr32_vmx(out, inp, len, key, counter)
            : ChaCha20_ctr32_int(out, inp, len, key, counter);
}
# endif

# ifndef OPENSSL_NO_POLY1305
void poly1305_init_int(void *ctx, const unsigned char key[16]);
void poly1305_blocks(void *ctx, const unsigned char *inp, size_t len,
                         unsigned int padbit);
void poly1305_emit(void *ctx, unsigned char mac[16],
                       const unsigned int nonce[4]);
void poly1305_init_fpu(void *ctx, const unsigned char key[16]);
void poly1305_blocks_fpu(void *ctx, const unsigned char *inp, size_t len,
                         unsigned int padbit);
void poly1305_emit_fpu(void *ctx, unsigned char mac[16],
                       const unsigned int nonce[4]);
void poly1305_init_vsx(void *ctx, const unsigned char key[16]);
void poly1305_blocks_vsx(void *ctx, const unsigned char *inp, size_t len,
                         unsigned int padbit);
void poly1305_emit_vsx(void *ctx, unsigned char mac[16],
                       const unsigned int nonce[4]);
int poly1305_init(void *ctx, const unsigned char key[16], void *func[2]);
int poly1305_init(void *ctx, const unsigned char key[16], void *func[2])
{
    if (OPENSSL_ppccap_P & PPC_CRYPTO207) {
        poly1305_init_int(ctx, key);
        func[0] = (void*)(uintptr_t)poly1305_blocks_vsx;
        func[1] = (void*)(uintptr_t)poly1305_emit;
    } else if (sizeof(size_t) == 4 && (OPENSSL_ppccap_P & PPC_FPU)) {
        poly1305_init_fpu(ctx, key);
        func[0] = (void*)(uintptr_t)poly1305_blocks_fpu;
        func[1] = (void*)(uintptr_t)poly1305_emit_fpu;
    } else {
        poly1305_init_int(ctx, key);
        func[0] = (void*)(uintptr_t)poly1305_blocks;
        func[1] = (void*)(uintptr_t)poly1305_emit;
    }
    return 1;
}
# endif
#endif /* FIPS_MODE */

#ifdef ECP_NISTZ256_ASM
void ecp_nistz256_mul_mont(unsigned long res[4], const unsigned long a[4],
                           const unsigned long b[4]);

void ecp_nistz256_to_mont(unsigned long res[4], const unsigned long in[4]);
void ecp_nistz256_to_mont(unsigned long res[4], const unsigned long in[4])
{
    static const unsigned long RR[] = { 0x0000000000000003U,
                                        0xfffffffbffffffffU,
                                        0xfffffffffffffffeU,
                                        0x00000004fffffffdU };

    ecp_nistz256_mul_mont(res, in, RR);
}

void ecp_nistz256_from_mont(unsigned long res[4], const unsigned long in[4]);
void ecp_nistz256_from_mont(unsigned long res[4], const unsigned long in[4])
{
    static const unsigned long one[] = { 1, 0, 0, 0 };

    ecp_nistz256_mul_mont(res, in, one);
}
#endif

static sigjmp_buf ill_jmp;
static void ill_handler(int sig)
{
    siglongjmp(ill_jmp, sig);
}

void OPENSSL_fpu_probe(void);
void OPENSSL_ppc64_probe(void);
void OPENSSL_altivec_probe(void);
void OPENSSL_crypto207_probe(void);
void OPENSSL_madd300_probe(void);

long OPENSSL_rdtsc_mftb(void);
long OPENSSL_rdtsc_mfspr268(void);

uint32_t OPENSSL_rdtsc(void)
{
    if (OPENSSL_ppccap_P & PPC_MFTB)
        return OPENSSL_rdtsc_mftb();
    else if (OPENSSL_ppccap_P & PPC_MFSPR268)
        return OPENSSL_rdtsc_mfspr268();
    else
        return 0;
}

size_t OPENSSL_instrument_bus_mftb(unsigned int *, size_t);
size_t OPENSSL_instrument_bus_mfspr268(unsigned int *, size_t);

size_t OPENSSL_instrument_bus(unsigned int *out, size_t cnt)
{
    if (OPENSSL_ppccap_P & PPC_MFTB)
        return OPENSSL_instrument_bus_mftb(out, cnt);
    else if (OPENSSL_ppccap_P & PPC_MFSPR268)
        return OPENSSL_instrument_bus_mfspr268(out, cnt);
    else
        return 0;
}

size_t OPENSSL_instrument_bus2_mftb(unsigned int *, size_t, size_t);
size_t OPENSSL_instrument_bus2_mfspr268(unsigned int *, size_t, size_t);

size_t OPENSSL_instrument_bus2(unsigned int *out, size_t cnt, size_t max)
{
    if (OPENSSL_ppccap_P & PPC_MFTB)
        return OPENSSL_instrument_bus2_mftb(out, cnt, max);
    else if (OPENSSL_ppccap_P & PPC_MFSPR268)
        return OPENSSL_instrument_bus2_mfspr268(out, cnt, max);
    else
        return 0;
}

#if defined(__GLIBC__) && defined(__GLIBC_PREREQ)
# if __GLIBC_PREREQ(2, 16)
#  include <sys/auxv.h>
#  define OSSL_IMPLEMENT_GETAUXVAL
# endif
#endif

/* I wish <sys/auxv.h> was universally available */
#define HWCAP                   16      /* AT_HWCAP */
#define HWCAP_PPC64             (1U << 30)
#define HWCAP_ALTIVEC           (1U << 28)
#define HWCAP_FPU               (1U << 27)
#define HWCAP_POWER6_EXT        (1U << 9)
#define HWCAP_VSX               (1U << 7)

#define HWCAP2                  26      /* AT_HWCAP2 */
#define HWCAP_VEC_CRYPTO        (1U << 25)
#define HWCAP_ARCH_3_00         (1U << 23)

# if defined(__GNUC__) && __GNUC__>=2
__attribute__ ((constructor))
# endif
void OPENSSL_cpuid_setup(void)
{
    char *e;
    struct sigaction ill_oact, ill_act;
    sigset_t oset;
    static int trigger = 0;

    if (trigger)
        return;
    trigger = 1;

    if ((e = getenv("OPENSSL_ppccap"))) {
        OPENSSL_ppccap_P = strtoul(e, NULL, 0);
        return;
    }

    OPENSSL_ppccap_P = 0;

#if defined(_AIX)
    OPENSSL_ppccap_P |= PPC_FPU;

    if (sizeof(size_t) == 4) {
        struct utsname uts;
# if defined(_SC_AIX_KERNEL_BITMODE)
        if (sysconf(_SC_AIX_KERNEL_BITMODE) != 64)
            return;
# endif
        if (uname(&uts) != 0 || atoi(uts.version) < 6)
            return;
    }

# if defined(__power_set)
    /*
     * Value used in __power_set is a single-bit 1<<n one denoting
     * specific processor class. Incidentally 0xffffffff<<n can be
     * used to denote specific processor and its successors.
     */
    if (sizeof(size_t) == 4) {
        /* In 32-bit case PPC_FPU64 is always fastest [if option] */
        if (__power_set(0xffffffffU<<13))       /* POWER5 and later */
            OPENSSL_ppccap_P |= PPC_FPU64;
    } else {
        /* In 64-bit case PPC_FPU64 is fastest only on POWER6 */
        if (__power_set(0x1U<<14))              /* POWER6 */
            OPENSSL_ppccap_P |= PPC_FPU64;
    }

    if (__power_set(0xffffffffU<<14))           /* POWER6 and later */
        OPENSSL_ppccap_P |= PPC_ALTIVEC;

    if (__power_set(0xffffffffU<<16))           /* POWER8 and later */
        OPENSSL_ppccap_P |= PPC_CRYPTO207;

    if (__power_set(0xffffffffU<<17))           /* POWER9 and later */
        OPENSSL_ppccap_P |= PPC_MADD300;

    return;
# endif
#endif

#if defined(__APPLE__) && defined(__MACH__)
    OPENSSL_ppccap_P |= PPC_FPU;

    {
        int val;
        size_t len = sizeof(val);

        if (sysctlbyname("hw.optional.64bitops", &val, &len, NULL, 0) == 0) {
            if (val)
                OPENSSL_ppccap_P |= PPC_FPU64;
        }

        len = sizeof(val);
        if (sysctlbyname("hw.optional.altivec", &val, &len, NULL, 0) == 0) {
            if (val)
                OPENSSL_ppccap_P |= PPC_ALTIVEC;
        }

        return;
    }
#endif

#ifdef OSSL_IMPLEMENT_GETAUXVAL
    {
        unsigned long hwcap = getauxval(HWCAP);
        unsigned long hwcap2 = getauxval(HWCAP2);

        if (hwcap & HWCAP_FPU) {
            OPENSSL_ppccap_P |= PPC_FPU;

            if (sizeof(size_t) == 4) {
                /* In 32-bit case PPC_FPU64 is always fastest [if option] */
                if (hwcap & HWCAP_PPC64)
                    OPENSSL_ppccap_P |= PPC_FPU64;
            } else {
                /* In 64-bit case PPC_FPU64 is fastest only on POWER6 */
                if (hwcap & HWCAP_POWER6_EXT)
                    OPENSSL_ppccap_P |= PPC_FPU64;
            }
        }

        if (hwcap & HWCAP_ALTIVEC) {
            OPENSSL_ppccap_P |= PPC_ALTIVEC;

            if ((hwcap & HWCAP_VSX) && (hwcap2 & HWCAP_VEC_CRYPTO))
                OPENSSL_ppccap_P |= PPC_CRYPTO207;
        }

        if (hwcap2 & HWCAP_ARCH_3_00) {
            OPENSSL_ppccap_P |= PPC_MADD300;
        }
    }
#endif

    sigfillset(&all_masked);
    sigdelset(&all_masked, SIGILL);
    sigdelset(&all_masked, SIGTRAP);
#ifdef SIGEMT
    sigdelset(&all_masked, SIGEMT);
#endif
    sigdelset(&all_masked, SIGFPE);
    sigdelset(&all_masked, SIGBUS);
    sigdelset(&all_masked, SIGSEGV);

    memset(&ill_act, 0, sizeof(ill_act));
    ill_act.sa_handler = ill_handler;
    ill_act.sa_mask = all_masked;

    sigprocmask(SIG_SETMASK, &ill_act.sa_mask, &oset);
    sigaction(SIGILL, &ill_act, &ill_oact);

#ifndef OSSL_IMPLEMENT_GETAUXVAL
    if (sigsetjmp(ill_jmp,1) == 0) {
        OPENSSL_fpu_probe();
        OPENSSL_ppccap_P |= PPC_FPU;

        if (sizeof(size_t) == 4) {
# ifdef __linux
            struct utsname uts;
            if (uname(&uts) == 0 && strcmp(uts.machine, "ppc64") == 0)
# endif
                if (sigsetjmp(ill_jmp, 1) == 0) {
                    OPENSSL_ppc64_probe();
                    OPENSSL_ppccap_P |= PPC_FPU64;
                }
        } else {
            /*
             * Wanted code detecting POWER6 CPU and setting PPC_FPU64
             */
        }
    }

    if (sigsetjmp(ill_jmp, 1) == 0) {
        OPENSSL_altivec_probe();
        OPENSSL_ppccap_P |= PPC_ALTIVEC;
        if (sigsetjmp(ill_jmp, 1) == 0) {
            OPENSSL_crypto207_probe();
            OPENSSL_ppccap_P |= PPC_CRYPTO207;
        }
    }

    if (sigsetjmp(ill_jmp, 1) == 0) {
        OPENSSL_madd300_probe();
        OPENSSL_ppccap_P |= PPC_MADD300;
    }
#endif

    if (sigsetjmp(ill_jmp, 1) == 0) {
        OPENSSL_rdtsc_mftb();
        OPENSSL_ppccap_P |= PPC_MFTB;
    } else if (sigsetjmp(ill_jmp, 1) == 0) {
        OPENSSL_rdtsc_mfspr268();
        OPENSSL_ppccap_P |= PPC_MFSPR268;
    }

    sigaction(SIGILL, &ill_oact, NULL);
    sigprocmask(SIG_SETMASK, &oset, NULL);
}
Commit	Line	Data
b1322259	1	/*
83cf7abf	2	* Copyright 2009-2018 The OpenSSL Project Authors. All Rights Reserved.
b1322259	3	*
0e9725bc	4	* Licensed under the Apache License 2.0 (the "License"). You may not use
b1322259 RS	5	* this file except in compliance with the License. You can obtain a copy
	6	* in the file LICENSE in the source distribution or at
	7	* https://www.openssl.org/source/license.html
	8	*/
	9
b4b48a10 AP	10	#include <stdio.h>
	11	#include <stdlib.h>
	12	#include <string.h>
	13	#include <setjmp.h>
	14	#include <signal.h>
fd054957	15	#include <unistd.h>
d5630dd6	16	#if defined(__linux) \|\| defined(_AIX)
0f113f3e	17	# include <sys/utsname.h>
78c3e205	18	#endif
2688d999 AP	19	#if defined(_AIX53) /* defined even on post-5.3 */
	20	# include <sys/systemcfg.h>
	21	# if !defined(__power_set)
	22	# define __power_set(a) (_system_configuration.implementation & (a))
	23	# endif
	24	#endif
0bd93bbe AP	25	#if defined(__APPLE__) && defined(__MACH__)
	26	# include <sys/types.h>
	27	# include <sys/sysctl.h>
	28	#endif
11252459	29	#include <openssl/crypto.h>
10b0b5ec	30	#include <openssl/bn.h>
8c8fbca9 BE	31	#include <internal/cryptlib.h>
	32	#include <internal/chacha.h>
	33	#include "bn/bn_lcl.h"
b4b48a10	34
0e716d92	35	#include "ppc_arch.h"
b4b48a10	36
07f3e4f3	37	unsigned int OPENSSL_ppccap_P = 0;
b4b48a10 AP	38
	39	static sigset_t all_masked;
	40
a8140a42 MC	41
a8140a42 MC	42	#ifdef OPENSSL_BN_ASM_MONT
10b0b5ec RL	43	int bn_mul_mont(BN_ULONG rp, const BN_ULONG ap, const BN_ULONG *bp,
	44	const BN_ULONG np, const BN_ULONG n0, int num)
	45	{
	46	int bn_mul_mont_int(BN_ULONG rp, const BN_ULONG ap, const BN_ULONG *bp,
	47	const BN_ULONG np, const BN_ULONG n0, int num);
	48	int bn_mul4x_mont_int(BN_ULONG rp, const BN_ULONG ap, const BN_ULONG *bp,
	49	const BN_ULONG np, const BN_ULONG n0, int num);
	50
	51	if (num < 4)
	52	return 0;
	53
	54	if ((num & 3) == 0)
	55	return bn_mul4x_mont_int(rp, ap, bp, np, n0, num);
	56
	57	/*
	58	* There used to be [optional] call to bn_mul_mont_fpu64 here,
	59	* but above subroutine is faster on contemporary processors.
	60	* Formulation means that there might be old processors where
	61	* FPU code path would be faster, POWER6 perhaps, but there was
	62	* no opportunity to figure it out...
	63	*/
	64
	65	return bn_mul_mont_int(rp, ap, bp, np, n0, num);
	66	}
a8140a42	67	#endif
10b0b5ec RL	68	void sha256_block_p8(void ctx, const void inp, size_t len);
10b0b5ec RL	69	void sha256_block_ppc(void ctx, const void inp, size_t len);
8c8fbca9	70	void sha256_block_data_order(void ctx, const void inp, size_t len);
10b0b5ec RL	71	void sha256_block_data_order(void ctx, const void inp, size_t len)
	72	{
	73	OPENSSL_ppccap_P & PPC_CRYPTO207 ? sha256_block_p8(ctx, inp, len) :
	74	sha256_block_ppc(ctx, inp, len);
	75	}
	76
	77	void sha512_block_p8(void ctx, const void inp, size_t len);
	78	void sha512_block_ppc(void ctx, const void inp, size_t len);
8c8fbca9	79	void sha512_block_data_order(void ctx, const void inp, size_t len);
10b0b5ec RL	80	void sha512_block_data_order(void ctx, const void inp, size_t len)
	81	{
	82	OPENSSL_ppccap_P & PPC_CRYPTO207 ? sha512_block_p8(ctx, inp, len) :
	83	sha512_block_ppc(ctx, inp, len);
	84	}
	85
55a9ca5c RL	86	/*
	87	* TODO(3.0): Temporarily disabled some assembler that hasn't been brought into
	88	* the FIPS module yet.
	89	*/
	90	#ifndef FIPS_MODE
41525ed6	91	# ifndef OPENSSL_NO_CHACHA
10b0b5ec RL	92	void ChaCha20_ctr32_int(unsigned char out, const unsigned char inp,
	93	size_t len, const unsigned int key[8],
	94	const unsigned int counter[4]);
	95	void ChaCha20_ctr32_vmx(unsigned char out, const unsigned char inp,
	96	size_t len, const unsigned int key[8],
	97	const unsigned int counter[4]);
316d527f AP	98	void ChaCha20_ctr32_vsx(unsigned char out, const unsigned char inp,
	99	size_t len, const unsigned int key[8],
	100	const unsigned int counter[4]);
10b0b5ec RL	101	void ChaCha20_ctr32(unsigned char out, const unsigned char inp,
	102	size_t len, const unsigned int key[8],
	103	const unsigned int counter[4])
	104	{
316d527f AP	105	OPENSSL_ppccap_P & PPC_CRYPTO207
	106	? ChaCha20_ctr32_vsx(out, inp, len, key, counter)
	107	: OPENSSL_ppccap_P & PPC_ALTIVEC
	108	? ChaCha20_ctr32_vmx(out, inp, len, key, counter)
	109	: ChaCha20_ctr32_int(out, inp, len, key, counter);
10b0b5ec	110	}
41525ed6	111	# endif
10b0b5ec	112
41525ed6	113	# ifndef OPENSSL_NO_POLY1305
10b0b5ec RL	114	void poly1305_init_int(void *ctx, const unsigned char key[16]);
	115	void poly1305_blocks(void ctx, const unsigned char inp, size_t len,
	116	unsigned int padbit);
	117	void poly1305_emit(void *ctx, unsigned char mac[16],
	118	const unsigned int nonce[4]);
	119	void poly1305_init_fpu(void *ctx, const unsigned char key[16]);
	120	void poly1305_blocks_fpu(void ctx, const unsigned char inp, size_t len,
	121	unsigned int padbit);
	122	void poly1305_emit_fpu(void *ctx, unsigned char mac[16],
	123	const unsigned int nonce[4]);
a28e4890 AP	124	void poly1305_init_vsx(void *ctx, const unsigned char key[16]);
	125	void poly1305_blocks_vsx(void ctx, const unsigned char inp, size_t len,
	126	unsigned int padbit);
	127	void poly1305_emit_vsx(void *ctx, unsigned char mac[16],
	128	const unsigned int nonce[4]);
8c8fbca9	129	int poly1305_init(void ctx, const unsigned char key[16], void func[2]);
10b0b5ec RL	130	int poly1305_init(void ctx, const unsigned char key[16], void func[2])
10b0b5ec RL	131	{
a28e4890 AP	132	if (OPENSSL_ppccap_P & PPC_CRYPTO207) {
	133	poly1305_init_int(ctx, key);
	134	func[0] = (void*)(uintptr_t)poly1305_blocks_vsx;
	135	func[1] = (void*)(uintptr_t)poly1305_emit;
	136	} else if (sizeof(size_t) == 4 && (OPENSSL_ppccap_P & PPC_FPU)) {
10b0b5ec	137	poly1305_init_fpu(ctx, key);
8c8fbca9 BE	138	func[0] = (void*)(uintptr_t)poly1305_blocks_fpu;
8c8fbca9 BE	139	func[1] = (void*)(uintptr_t)poly1305_emit_fpu;
10b0b5ec RL	140	} else {
10b0b5ec RL	141	poly1305_init_int(ctx, key);
8c8fbca9 BE	142	func[0] = (void*)(uintptr_t)poly1305_blocks;
8c8fbca9 BE	143	func[1] = (void*)(uintptr_t)poly1305_emit;
10b0b5ec RL	144	}
	145	return 1;
	146	}
41525ed6	147	# endif
a9612d6c	148	#endif /* FIPS_MODE */
10b0b5ec	149
a9612d6c	150	#ifdef ECP_NISTZ256_ASM
10b0b5ec RL	151	void ecp_nistz256_mul_mont(unsigned long res[4], const unsigned long a[4],
	152	const unsigned long b[4]);
	153
	154	void ecp_nistz256_to_mont(unsigned long res[4], const unsigned long in[4]);
	155	void ecp_nistz256_to_mont(unsigned long res[4], const unsigned long in[4])
	156	{
	157	static const unsigned long RR[] = { 0x0000000000000003U,
	158	0xfffffffbffffffffU,
	159	0xfffffffffffffffeU,
	160	0x00000004fffffffdU };
	161
	162	ecp_nistz256_mul_mont(res, in, RR);
	163	}
	164
	165	void ecp_nistz256_from_mont(unsigned long res[4], const unsigned long in[4]);
	166	void ecp_nistz256_from_mont(unsigned long res[4], const unsigned long in[4])
	167	{
	168	static const unsigned long one[] = { 1, 0, 0, 0 };
	169
	170	ecp_nistz256_mul_mont(res, in, one);
	171	}
a9612d6c	172	#endif
10b0b5ec	173
b4b48a10	174	static sigjmp_buf ill_jmp;
0f113f3e MC	175	static void ill_handler(int sig)
	176	{
	177	siglongjmp(ill_jmp, sig);
	178	}
b4b48a10	179
2688d999	180	void OPENSSL_fpu_probe(void);
70b76d39	181	void OPENSSL_ppc64_probe(void);
fd054957	182	void OPENSSL_altivec_probe(void);
de51e830	183	void OPENSSL_crypto207_probe(void);
53385e1f	184	void OPENSSL_madd300_probe(void);
70b76d39	185
c8f37048 BE	186	long OPENSSL_rdtsc_mftb(void);
	187	long OPENSSL_rdtsc_mfspr268(void);
	188
	189	uint32_t OPENSSL_rdtsc(void)
	190	{
	191	if (OPENSSL_ppccap_P & PPC_MFTB)
	192	return OPENSSL_rdtsc_mftb();
	193	else if (OPENSSL_ppccap_P & PPC_MFSPR268)
	194	return OPENSSL_rdtsc_mfspr268();
	195	else
	196	return 0;
	197	}
	198
	199	size_t OPENSSL_instrument_bus_mftb(unsigned int *, size_t);
	200	size_t OPENSSL_instrument_bus_mfspr268(unsigned int *, size_t);
	201
	202	size_t OPENSSL_instrument_bus(unsigned int *out, size_t cnt)
	203	{
	204	if (OPENSSL_ppccap_P & PPC_MFTB)
	205	return OPENSSL_instrument_bus_mftb(out, cnt);
	206	else if (OPENSSL_ppccap_P & PPC_MFSPR268)
	207	return OPENSSL_instrument_bus_mfspr268(out, cnt);
	208	else
	209	return 0;
	210	}
	211
	212	size_t OPENSSL_instrument_bus2_mftb(unsigned int *, size_t, size_t);
	213	size_t OPENSSL_instrument_bus2_mfspr268(unsigned int *, size_t, size_t);
	214
	215	size_t OPENSSL_instrument_bus2(unsigned int *out, size_t cnt, size_t max)
	216	{
	217	if (OPENSSL_ppccap_P & PPC_MFTB)
	218	return OPENSSL_instrument_bus2_mftb(out, cnt, max);
	219	else if (OPENSSL_ppccap_P & PPC_MFSPR268)
	220	return OPENSSL_instrument_bus2_mfspr268(out, cnt, max);
	221	else
	222	return 0;
	223	}
	224
5f40dd15 RL	225	#if defined(__GLIBC__) && defined(__GLIBC_PREREQ)
	226	# if __GLIBC_PREREQ(2, 16)
	227	# include <sys/auxv.h>
	228	# define OSSL_IMPLEMENT_GETAUXVAL
	229	# endif
2688d999 AP	230	#endif
	231
	232	/* I wish <sys/auxv.h> was universally available */
	233	#define HWCAP 16 /* AT_HWCAP */
	234	#define HWCAP_PPC64 (1U << 30)
	235	#define HWCAP_ALTIVEC (1U << 28)
	236	#define HWCAP_FPU (1U << 27)
	237	#define HWCAP_POWER6_EXT (1U << 9)
	238	#define HWCAP_VSX (1U << 7)
	239
	240	#define HWCAP2 26 /* AT_HWCAP2 */
	241	#define HWCAP_VEC_CRYPTO (1U << 25)
e0e53282	242	#define HWCAP_ARCH_3_00 (1U << 23)
2688d999 AP	243
	244	# if defined(__GNUC__) && __GNUC__>=2
	245	__attribute__ ((constructor))
	246	# endif
b4b48a10	247	void OPENSSL_cpuid_setup(void)
0f113f3e MC	248	{
	249	char *e;
	250	struct sigaction ill_oact, ill_act;
	251	sigset_t oset;
	252	static int trigger = 0;
	253
	254	if (trigger)
	255	return;
	256	trigger = 1;
	257
0f113f3e MC	258	if ((e = getenv("OPENSSL_ppccap"))) {
	259	OPENSSL_ppccap_P = strtoul(e, NULL, 0);
	260	return;
	261	}
b4b48a10	262
0f113f3e	263	OPENSSL_ppccap_P = 0;
6415dd7b	264
fd054957	265	#if defined(_AIX)
2688d999 AP	266	OPENSSL_ppccap_P \|= PPC_FPU;
2688d999 AP	267
0f113f3e MC	268	if (sizeof(size_t) == 4) {
0f113f3e MC	269	struct utsname uts;
fd054957	270	# if defined(_SC_AIX_KERNEL_BITMODE)
0f113f3e MC	271	if (sysconf(_SC_AIX_KERNEL_BITMODE) != 64)
0f113f3e MC	272	return;
fd054957	273	# endif
0f113f3e MC	274	if (uname(&uts) != 0 \|\| atoi(uts.version) < 6)
	275	return;
	276	}
2688d999 AP	277
	278	# if defined(__power_set)
	279	/*
	280	* Value used in __power_set is a single-bit 1<<n one denoting
	281	* specific processor class. Incidentally 0xffffffff<<n can be
	282	* used to denote specific processor and its successors.
	283	*/
	284	if (sizeof(size_t) == 4) {
	285	/* In 32-bit case PPC_FPU64 is always fastest [if option] */
	286	if (__power_set(0xffffffffU<<13)) /* POWER5 and later */
	287	OPENSSL_ppccap_P \|= PPC_FPU64;
	288	} else {
	289	/* In 64-bit case PPC_FPU64 is fastest only on POWER6 */
	290	if (__power_set(0x1U<<14)) /* POWER6 */
	291	OPENSSL_ppccap_P \|= PPC_FPU64;
	292	}
	293
	294	if (__power_set(0xffffffffU<<14)) /* POWER6 and later */
	295	OPENSSL_ppccap_P \|= PPC_ALTIVEC;
	296
	297	if (__power_set(0xffffffffU<<16)) /* POWER8 and later */
	298	OPENSSL_ppccap_P \|= PPC_CRYPTO207;
	299
e0e53282 AP	300	if (__power_set(0xffffffffU<<17)) /* POWER9 and later */
	301	OPENSSL_ppccap_P \|= PPC_MADD300;
	302
2688d999 AP	303	return;
	304	# endif
	305	#endif
	306
0bd93bbe AP	307	#if defined(__APPLE__) && defined(__MACH__)
	308	OPENSSL_ppccap_P \|= PPC_FPU;
	309
	310	{
	311	int val;
	312	size_t len = sizeof(val);
	313
	314	if (sysctlbyname("hw.optional.64bitops", &val, &len, NULL, 0) == 0) {
	315	if (val)
	316	OPENSSL_ppccap_P \|= PPC_FPU64;
	317	}
	318
	319	len = sizeof(val);
	320	if (sysctlbyname("hw.optional.altivec", &val, &len, NULL, 0) == 0) {
	321	if (val)
	322	OPENSSL_ppccap_P \|= PPC_ALTIVEC;
	323	}
	324
	325	return;
	326	}
	327	#endif
	328
5f40dd15 RL	329	#ifdef OSSL_IMPLEMENT_GETAUXVAL
5f40dd15 RL	330	{
2688d999	331	unsigned long hwcap = getauxval(HWCAP);
99592c73	332	unsigned long hwcap2 = getauxval(HWCAP2);
2688d999 AP	333
2688d999 AP	334	if (hwcap & HWCAP_FPU) {
dccd20d1	335	OPENSSL_ppccap_P \|= PPC_FPU;
2688d999 AP	336
	337	if (sizeof(size_t) == 4) {
	338	/* In 32-bit case PPC_FPU64 is always fastest [if option] */
	339	if (hwcap & HWCAP_PPC64)
	340	OPENSSL_ppccap_P \|= PPC_FPU64;
	341	} else {
	342	/* In 64-bit case PPC_FPU64 is fastest only on POWER6 */
	343	if (hwcap & HWCAP_POWER6_EXT)
	344	OPENSSL_ppccap_P \|= PPC_FPU64;
	345	}
	346	}
	347
	348	if (hwcap & HWCAP_ALTIVEC) {
	349	OPENSSL_ppccap_P \|= PPC_ALTIVEC;
	350
99592c73	351	if ((hwcap & HWCAP_VSX) && (hwcap2 & HWCAP_VEC_CRYPTO))
2688d999 AP	352	OPENSSL_ppccap_P \|= PPC_CRYPTO207;
	353	}
	354
99592c73	355	if (hwcap2 & HWCAP_ARCH_3_00) {
e0e53282 AP	356	OPENSSL_ppccap_P \|= PPC_MADD300;
e0e53282 AP	357	}
2688d999	358	}
5f40dd15	359	#endif
2688d999 AP	360
	361	sigfillset(&all_masked);
	362	sigdelset(&all_masked, SIGILL);
	363	sigdelset(&all_masked, SIGTRAP);
	364	#ifdef SIGEMT
	365	sigdelset(&all_masked, SIGEMT);
fd054957	366	#endif
2688d999 AP	367	sigdelset(&all_masked, SIGFPE);
	368	sigdelset(&all_masked, SIGBUS);
	369	sigdelset(&all_masked, SIGSEGV);
fd054957	370
0f113f3e MC	371	memset(&ill_act, 0, sizeof(ill_act));
	372	ill_act.sa_handler = ill_handler;
	373	ill_act.sa_mask = all_masked;
6415dd7b	374
0f113f3e MC	375	sigprocmask(SIG_SETMASK, &ill_act.sa_mask, &oset);
0f113f3e MC	376	sigaction(SIGILL, &ill_act, &ill_oact);
6415dd7b	377
c8f37048	378	#ifndef OSSL_IMPLEMENT_GETAUXVAL
2688d999 AP	379	if (sigsetjmp(ill_jmp,1) == 0) {
	380	OPENSSL_fpu_probe();
	381	OPENSSL_ppccap_P \|= PPC_FPU;
	382
	383	if (sizeof(size_t) == 4) {
c8f37048	384	# ifdef __linux
2688d999 AP	385	struct utsname uts;
2688d999 AP	386	if (uname(&uts) == 0 && strcmp(uts.machine, "ppc64") == 0)
c8f37048	387	# endif
2688d999 AP	388	if (sigsetjmp(ill_jmp, 1) == 0) {
	389	OPENSSL_ppc64_probe();
	390	OPENSSL_ppccap_P \|= PPC_FPU64;
	391	}
	392	} else {
	393	/*
	394	* Wanted code detecting POWER6 CPU and setting PPC_FPU64
	395	*/
	396	}
0f113f3e MC	397	}
	398
	399	if (sigsetjmp(ill_jmp, 1) == 0) {
	400	OPENSSL_altivec_probe();
	401	OPENSSL_ppccap_P \|= PPC_ALTIVEC;
	402	if (sigsetjmp(ill_jmp, 1) == 0) {
	403	OPENSSL_crypto207_probe();
	404	OPENSSL_ppccap_P \|= PPC_CRYPTO207;
	405	}
	406	}
	407
e0e53282 AP	408	if (sigsetjmp(ill_jmp, 1) == 0) {
	409	OPENSSL_madd300_probe();
	410	OPENSSL_ppccap_P \|= PPC_MADD300;
	411	}
c8f37048 BE	412	#endif
	413
	414	if (sigsetjmp(ill_jmp, 1) == 0) {
	415	OPENSSL_rdtsc_mftb();
	416	OPENSSL_ppccap_P \|= PPC_MFTB;
	417	} else if (sigsetjmp(ill_jmp, 1) == 0) {
	418	OPENSSL_rdtsc_mfspr268();
	419	OPENSSL_ppccap_P \|= PPC_MFSPR268;
	420	}
e0e53282	421
0f113f3e MC	422	sigaction(SIGILL, &ill_oact, NULL);
	423	sigprocmask(SIG_SETMASK, &oset, NULL);
	424	}