/* fat.h - setups for fat binaries.
   Generated by configure - DO NOT EDIT.  */

#ifndef OPERATION_add_n
#undef  mpn_add_n
#define mpn_add_n  (*__gmpn_cpuvec.add_n)
#endif
DECL_add_n (__MPN(add_n_init));

#ifndef OPERATION_addmul_1
#undef  mpn_addmul_1
#define mpn_addmul_1  (*__gmpn_cpuvec.addmul_1)
#endif
DECL_addmul_1 (__MPN(addmul_1_init));

#ifndef OPERATION_bdiv_dbm1c
#undef  mpn_bdiv_dbm1c
#define mpn_bdiv_dbm1c  (*__gmpn_cpuvec.bdiv_dbm1c)
#endif
DECL_bdiv_dbm1c (__MPN(bdiv_dbm1c_init));

#ifndef OPERATION_com
#undef  mpn_com
#define mpn_com  (*__gmpn_cpuvec.com)
#endif
DECL_com (__MPN(com_init));

#ifndef OPERATION_copyd
#undef  mpn_copyd
#define mpn_copyd  (*__gmpn_cpuvec.copyd)
#endif
DECL_copyd (__MPN(copyd_init));

#ifndef OPERATION_copyi
#undef  mpn_copyi
#define mpn_copyi  (*__gmpn_cpuvec.copyi)
#endif
DECL_copyi (__MPN(copyi_init));

#ifndef OPERATION_dive_1
#undef  mpn_divexact_1
#define mpn_divexact_1  (*__gmpn_cpuvec.divexact_1)
#endif
DECL_divexact_1 (__MPN(divexact_1_init));

#ifndef OPERATION_divrem_1
#undef  mpn_divrem_1
#define mpn_divrem_1  (*__gmpn_cpuvec.divrem_1)
#endif
DECL_divrem_1 (__MPN(divrem_1_init));

#ifndef OPERATION_gcd_1
#undef  mpn_gcd_1
#define mpn_gcd_1  (*__gmpn_cpuvec.gcd_1)
#endif
DECL_gcd_1 (__MPN(gcd_1_init));

#ifndef OPERATION_lshift
#undef  mpn_lshift
#define mpn_lshift  (*__gmpn_cpuvec.lshift)
#endif
DECL_lshift (__MPN(lshift_init));

#ifndef OPERATION_lshiftc
#undef  mpn_lshiftc
#define mpn_lshiftc  (*__gmpn_cpuvec.lshiftc)
#endif
DECL_lshiftc (__MPN(lshiftc_init));

#ifndef OPERATION_mod_1
#undef  mpn_mod_1
#define mpn_mod_1  (*__gmpn_cpuvec.mod_1)
#endif
DECL_mod_1 (__MPN(mod_1_init));

#ifndef OPERATION_mod_1_1
#undef  mpn_mod_1_1p
#define mpn_mod_1_1p  (*__gmpn_cpuvec.mod_1_1p)
#endif
DECL_mod_1_1p (__MPN(mod_1_1p_init));

#ifndef OPERATION_mod_1_1_cps
#undef  mpn_mod_1_1p_cps
#define mpn_mod_1_1p_cps  (*__gmpn_cpuvec.mod_1_1p_cps)
#endif
DECL_mod_1_1p_cps (__MPN(mod_1_1p_cps_init));

#ifndef OPERATION_mod_1_2
#undef  mpn_mod_1s_2p
#define mpn_mod_1s_2p  (*__gmpn_cpuvec.mod_1s_2p)
#endif
DECL_mod_1s_2p (__MPN(mod_1s_2p_init));

#ifndef OPERATION_mod_1_2_cps
#undef  mpn_mod_1s_2p_cps
#define mpn_mod_1s_2p_cps  (*__gmpn_cpuvec.mod_1s_2p_cps)
#endif
DECL_mod_1s_2p_cps (__MPN(mod_1s_2p_cps_init));

#ifndef OPERATION_mod_1_4
#undef  mpn_mod_1s_4p
#define mpn_mod_1s_4p  (*__gmpn_cpuvec.mod_1s_4p)
#endif
DECL_mod_1s_4p (__MPN(mod_1s_4p_init));

#ifndef OPERATION_mod_1_4_cps
#undef  mpn_mod_1s_4p_cps
#define mpn_mod_1s_4p_cps  (*__gmpn_cpuvec.mod_1s_4p_cps)
#endif
DECL_mod_1s_4p_cps (__MPN(mod_1s_4p_cps_init));

#ifndef OPERATION_mod_34lsub1
#undef  mpn_mod_34lsub1
#define mpn_mod_34lsub1  (*__gmpn_cpuvec.mod_34lsub1)
#endif
DECL_mod_34lsub1 (__MPN(mod_34lsub1_init));

#ifndef OPERATION_mode1o
#undef  mpn_modexact_1c_odd
#define mpn_modexact_1c_odd  (*__gmpn_cpuvec.modexact_1c_odd)
#endif
DECL_modexact_1c_odd (__MPN(modexact_1c_odd_init));

#ifndef OPERATION_mul_1
#undef  mpn_mul_1
#define mpn_mul_1  (*__gmpn_cpuvec.mul_1)
#endif
DECL_mul_1 (__MPN(mul_1_init));

#ifndef OPERATION_mul_basecase
#undef  mpn_mul_basecase
#define mpn_mul_basecase  (*__gmpn_cpuvec.mul_basecase)
#endif
DECL_mul_basecase (__MPN(mul_basecase_init));

#ifndef OPERATION_mullo_basecase
#undef  mpn_mullo_basecase
#define mpn_mullo_basecase  (*__gmpn_cpuvec.mullo_basecase)
#endif
DECL_mullo_basecase (__MPN(mullo_basecase_init));

#ifndef OPERATION_pre_divrem_1
#undef  mpn_preinv_divrem_1
#define mpn_preinv_divrem_1  (*__gmpn_cpuvec.preinv_divrem_1)
#endif
DECL_preinv_divrem_1 (__MPN(preinv_divrem_1_init));

#ifndef OPERATION_pre_mod_1
#undef  mpn_preinv_mod_1
#define mpn_preinv_mod_1  (*__gmpn_cpuvec.preinv_mod_1)
#endif
DECL_preinv_mod_1 (__MPN(preinv_mod_1_init));

#ifndef OPERATION_redc_1
#undef  mpn_redc_1
#define mpn_redc_1  (*__gmpn_cpuvec.redc_1)
#endif
DECL_redc_1 (__MPN(redc_1_init));

#ifndef OPERATION_redc_2
#undef  mpn_redc_2
#define mpn_redc_2  (*__gmpn_cpuvec.redc_2)
#endif
DECL_redc_2 (__MPN(redc_2_init));

#ifndef OPERATION_rshift
#undef  mpn_rshift
#define mpn_rshift  (*__gmpn_cpuvec.rshift)
#endif
DECL_rshift (__MPN(rshift_init));

#ifndef OPERATION_sqr_basecase
#undef  mpn_sqr_basecase
#define mpn_sqr_basecase  (*__gmpn_cpuvec.sqr_basecase)
#endif
DECL_sqr_basecase (__MPN(sqr_basecase_init));

#ifndef OPERATION_sub_n
#undef  mpn_sub_n
#define mpn_sub_n  (*__gmpn_cpuvec.sub_n)
#endif
DECL_sub_n (__MPN(sub_n_init));

#ifndef OPERATION_submul_1
#undef  mpn_submul_1
#define mpn_submul_1  (*__gmpn_cpuvec.submul_1)
#endif
DECL_submul_1 (__MPN(submul_1_init));

/* variable thresholds */
#undef  MUL_TOOM22_THRESHOLD
#define MUL_TOOM22_THRESHOLD  CPUVEC_THRESHOLD (mul_toom22_threshold)
#undef  MUL_TOOM33_THRESHOLD
#define MUL_TOOM33_THRESHOLD  CPUVEC_THRESHOLD (mul_toom33_threshold)
#undef  SQR_TOOM2_THRESHOLD
#define SQR_TOOM2_THRESHOLD  CPUVEC_THRESHOLD (sqr_toom2_threshold)
#undef  SQR_TOOM3_THRESHOLD
#define SQR_TOOM3_THRESHOLD  CPUVEC_THRESHOLD (sqr_toom3_threshold)
#undef  BMOD_1_TO_MOD_1_THRESHOLD
#define BMOD_1_TO_MOD_1_THRESHOLD  CPUVEC_THRESHOLD (bmod_1_to_mod_1_threshold)

/* Copy all fields into __gmpn_cpuvec.
   memcpy is not used because it might operate byte-wise (depending on its
   implementation), and we need the function pointer writes to be atomic.
   volatile discourages the compiler from trying to optimize this.  */
#define CPUVEC_INSTALL(vec) \
  do { \
    volatile struct cpuvec_t *p = &__gmpn_cpuvec; \
    p->add_n = vec.add_n; \
    p->addmul_1 = vec.addmul_1; \
    p->bdiv_dbm1c = vec.bdiv_dbm1c; \
    p->com = vec.com; \
    p->copyd = vec.copyd; \
    p->copyi = vec.copyi; \
    p->divexact_1 = vec.divexact_1; \
    p->divrem_1 = vec.divrem_1; \
    p->gcd_1 = vec.gcd_1; \
    p->lshift = vec.lshift; \
    p->lshiftc = vec.lshiftc; \
    p->mod_1 = vec.mod_1; \
    p->mod_1_1p = vec.mod_1_1p; \
    p->mod_1_1p_cps = vec.mod_1_1p_cps; \
    p->mod_1s_2p = vec.mod_1s_2p; \
    p->mod_1s_2p_cps = vec.mod_1s_2p_cps; \
    p->mod_1s_4p = vec.mod_1s_4p; \
    p->mod_1s_4p_cps = vec.mod_1s_4p_cps; \
    p->mod_34lsub1 = vec.mod_34lsub1; \
    p->modexact_1c_odd = vec.modexact_1c_odd; \
    p->mul_1 = vec.mul_1; \
    p->mul_basecase = vec.mul_basecase; \
    p->mullo_basecase = vec.mullo_basecase; \
    p->preinv_divrem_1 = vec.preinv_divrem_1; \
    p->preinv_mod_1 = vec.preinv_mod_1; \
    p->redc_1 = vec.redc_1; \
    p->redc_2 = vec.redc_2; \
    p->rshift = vec.rshift; \
    p->sqr_basecase = vec.sqr_basecase; \
    p->sub_n = vec.sub_n; \
    p->submul_1 = vec.submul_1; \
    p->mul_toom22_threshold = vec.mul_toom22_threshold; \
    p->mul_toom33_threshold = vec.mul_toom33_threshold; \
    p->sqr_toom2_threshold = vec.sqr_toom2_threshold; \
    p->sqr_toom3_threshold = vec.sqr_toom3_threshold; \
    p->bmod_1_to_mod_1_threshold = vec.bmod_1_to_mod_1_threshold; \
  } while (0)

/* A helper to check all fields are filled. */
#define ASSERT_CPUVEC(vec) \
  do { \
    ASSERT (vec.add_n != NULL); \
    ASSERT (vec.addmul_1 != NULL); \
    ASSERT (vec.bdiv_dbm1c != NULL); \
    ASSERT (vec.com != NULL); \
    ASSERT (vec.copyd != NULL); \
    ASSERT (vec.copyi != NULL); \
    ASSERT (vec.divexact_1 != NULL); \
    ASSERT (vec.divrem_1 != NULL); \
    ASSERT (vec.gcd_1 != NULL); \
    ASSERT (vec.lshift != NULL); \
    ASSERT (vec.lshiftc != NULL); \
    ASSERT (vec.mod_1 != NULL); \
    ASSERT (vec.mod_1_1p != NULL); \
    ASSERT (vec.mod_1_1p_cps != NULL); \
    ASSERT (vec.mod_1s_2p != NULL); \
    ASSERT (vec.mod_1s_2p_cps != NULL); \
    ASSERT (vec.mod_1s_4p != NULL); \
    ASSERT (vec.mod_1s_4p_cps != NULL); \
    ASSERT (vec.mod_34lsub1 != NULL); \
    ASSERT (vec.modexact_1c_odd != NULL); \
    ASSERT (vec.mul_1 != NULL); \
    ASSERT (vec.mul_basecase != NULL); \
    ASSERT (vec.mullo_basecase != NULL); \
    ASSERT (vec.preinv_divrem_1 != NULL); \
    ASSERT (vec.preinv_mod_1 != NULL); \
    ASSERT (vec.redc_1 != NULL); \
    ASSERT (vec.redc_2 != NULL); \
    ASSERT (vec.rshift != NULL); \
    ASSERT (vec.sqr_basecase != NULL); \
    ASSERT (vec.sub_n != NULL); \
    ASSERT (vec.submul_1 != NULL); \
    ASSERT (vec.mul_toom22_threshold != 0); \
    ASSERT (vec.mul_toom33_threshold != 0); \
    ASSERT (vec.sqr_toom2_threshold != 0); \
    ASSERT (vec.sqr_toom3_threshold != 0); \
    ASSERT (vec.bmod_1_to_mod_1_threshold != 0); \
  } while (0)

/* Call ITERATE(field) for each fat threshold field. */
#define ITERATE_FAT_THRESHOLDS() \
  do { \
    ITERATE (MUL_TOOM22_THRESHOLD, mul_toom22_threshold); \
    ITERATE (MUL_TOOM33_THRESHOLD, mul_toom33_threshold); \
    ITERATE (SQR_TOOM2_THRESHOLD, sqr_toom2_threshold); \
    ITERATE (SQR_TOOM3_THRESHOLD, sqr_toom3_threshold); \
    ITERATE (BMOD_1_TO_MOD_1_THRESHOLD, bmod_1_to_mod_1_threshold); \
  } while (0)

DECL_add_n (__gmpn_add_n_x86);
DECL_addmul_1 (__gmpn_addmul_1_x86);
DECL_bdiv_dbm1c (__gmpn_bdiv_dbm1c_x86);
DECL_copyd (__gmpn_copyd_x86);
DECL_copyi (__gmpn_copyi_x86);
DECL_divexact_1 (__gmpn_divexact_1_x86);
DECL_divrem_1 (__gmpn_divrem_1_x86);
DECL_lshift (__gmpn_lshift_x86);
DECL_mod_34lsub1 (__gmpn_mod_34lsub1_x86);
DECL_mul_1 (__gmpn_mul_1_x86);
DECL_mul_basecase (__gmpn_mul_basecase_x86);
DECL_rshift (__gmpn_rshift_x86);
DECL_sqr_basecase (__gmpn_sqr_basecase_x86);
DECL_sub_n (__gmpn_sub_n_x86);
DECL_submul_1 (__gmpn_submul_1_x86);

#define CPUVEC_SETUP_x86 \
  do { \
    decided_cpuvec.add_n = __gmpn_add_n_x86; \
    decided_cpuvec.addmul_1 = __gmpn_addmul_1_x86; \
    decided_cpuvec.bdiv_dbm1c = __gmpn_bdiv_dbm1c_x86; \
    decided_cpuvec.copyd = __gmpn_copyd_x86; \
    decided_cpuvec.copyi = __gmpn_copyi_x86; \
    decided_cpuvec.divexact_1 = __gmpn_divexact_1_x86; \
    decided_cpuvec.divrem_1 = __gmpn_divrem_1_x86; \
    decided_cpuvec.lshift = __gmpn_lshift_x86; \
    decided_cpuvec.mod_34lsub1 = __gmpn_mod_34lsub1_x86; \
    decided_cpuvec.mul_1 = __gmpn_mul_1_x86; \
    decided_cpuvec.mul_basecase = __gmpn_mul_basecase_x86; \
    decided_cpuvec.rshift = __gmpn_rshift_x86; \
    decided_cpuvec.sqr_basecase = __gmpn_sqr_basecase_x86; \
    decided_cpuvec.sub_n = __gmpn_sub_n_x86; \
    decided_cpuvec.submul_1 = __gmpn_submul_1_x86; \
  } while (0)

DECL_com (__gmpn_com_fat);
DECL_gcd_1 (__gmpn_gcd_1_fat);
DECL_lshiftc (__gmpn_lshiftc_fat);
DECL_mod_1 (__gmpn_mod_1_fat);
DECL_mod_1_1p (__gmpn_mod_1_1p_fat);
DECL_mod_1_1p_cps (__gmpn_mod_1_1p_cps_fat);
DECL_mod_1s_2p (__gmpn_mod_1s_2p_fat);
DECL_mod_1s_2p_cps (__gmpn_mod_1s_2p_cps_fat);
DECL_mod_1s_4p (__gmpn_mod_1s_4p_fat);
DECL_mod_1s_4p_cps (__gmpn_mod_1s_4p_cps_fat);
DECL_modexact_1c_odd (__gmpn_modexact_1c_odd_fat);
DECL_mullo_basecase (__gmpn_mullo_basecase_fat);
DECL_redc_1 (__gmpn_redc_1_fat);
DECL_redc_2 (__gmpn_redc_2_fat);

#define CPUVEC_SETUP_fat \
  do { \
    decided_cpuvec.mul_toom22_threshold = 20; \
    decided_cpuvec.mul_toom33_threshold = 130; \
    decided_cpuvec.sqr_toom2_threshold = 30; \
    decided_cpuvec.sqr_toom3_threshold = 200; \
    decided_cpuvec.bmod_1_to_mod_1_threshold = 20; \
    decided_cpuvec.com = __gmpn_com_fat; \
    decided_cpuvec.gcd_1 = __gmpn_gcd_1_fat; \
    decided_cpuvec.lshiftc = __gmpn_lshiftc_fat; \
    decided_cpuvec.mod_1 = __gmpn_mod_1_fat; \
    decided_cpuvec.mod_1_1p = __gmpn_mod_1_1p_fat; \
    decided_cpuvec.mod_1_1p_cps = __gmpn_mod_1_1p_cps_fat; \
    decided_cpuvec.mod_1s_2p = __gmpn_mod_1s_2p_fat; \
    decided_cpuvec.mod_1s_2p_cps = __gmpn_mod_1s_2p_cps_fat; \
    decided_cpuvec.mod_1s_4p = __gmpn_mod_1s_4p_fat; \
    decided_cpuvec.mod_1s_4p_cps = __gmpn_mod_1s_4p_cps_fat; \
    decided_cpuvec.modexact_1c_odd = __gmpn_modexact_1c_odd_fat; \
    decided_cpuvec.mullo_basecase = __gmpn_mullo_basecase_fat; \
    decided_cpuvec.redc_1 = __gmpn_redc_1_fat; \
    decided_cpuvec.redc_2 = __gmpn_redc_2_fat; \
  } while (0)


#define CPUVEC_SETUP_i486 \
  do { \
    decided_cpuvec.mul_toom22_threshold = 18; \
    decided_cpuvec.mul_toom33_threshold = 228; \
    decided_cpuvec.sqr_toom2_threshold = 49; \
    decided_cpuvec.sqr_toom3_threshold = 238; \
  } while (0)

DECL_add_n (__gmpn_add_n_k6);
DECL_addmul_1 (__gmpn_addmul_1_k6);
DECL_divrem_1 (__gmpn_divrem_1_k6);
DECL_gcd_1 (__gmpn_gcd_1_k6);
DECL_mod_34lsub1 (__gmpn_mod_34lsub1_k6);
DECL_modexact_1c_odd (__gmpn_modexact_1c_odd_k6);
DECL_mul_1 (__gmpn_mul_1_k6);
DECL_mul_basecase (__gmpn_mul_basecase_k6);
DECL_preinv_mod_1 (__gmpn_preinv_mod_1_k6);
DECL_sqr_basecase (__gmpn_sqr_basecase_k6);
DECL_sub_n (__gmpn_sub_n_k6);
DECL_submul_1 (__gmpn_submul_1_k6);

#define CPUVEC_SETUP_k6 \
  do { \
    decided_cpuvec.mul_toom22_threshold = 20; \
    decided_cpuvec.mul_toom33_threshold = 69; \
    decided_cpuvec.sqr_toom2_threshold = 32; \
    decided_cpuvec.sqr_toom3_threshold = 97; \
    decided_cpuvec.add_n = __gmpn_add_n_k6; \
    decided_cpuvec.addmul_1 = __gmpn_addmul_1_k6; \
    decided_cpuvec.divrem_1 = __gmpn_divrem_1_k6; \
    decided_cpuvec.gcd_1 = __gmpn_gcd_1_k6; \
    decided_cpuvec.mod_34lsub1 = __gmpn_mod_34lsub1_k6; \
    decided_cpuvec.modexact_1c_odd = __gmpn_modexact_1c_odd_k6; \
    decided_cpuvec.mul_1 = __gmpn_mul_1_k6; \
    decided_cpuvec.mul_basecase = __gmpn_mul_basecase_k6; \
    decided_cpuvec.preinv_mod_1 = __gmpn_preinv_mod_1_k6; \
    decided_cpuvec.sqr_basecase = __gmpn_sqr_basecase_k6; \
    decided_cpuvec.sub_n = __gmpn_sub_n_k6; \
    decided_cpuvec.submul_1 = __gmpn_submul_1_k6; \
  } while (0)

DECL_com (__gmpn_com_k6_mmx);
DECL_divexact_1 (__gmpn_divexact_1_k6_mmx);
DECL_lshift (__gmpn_lshift_k6_mmx);
DECL_rshift (__gmpn_rshift_k6_mmx);

#define CPUVEC_SETUP_k6_mmx \
  do { \
    decided_cpuvec.com = __gmpn_com_k6_mmx; \
    decided_cpuvec.divexact_1 = __gmpn_divexact_1_k6_mmx; \
    decided_cpuvec.lshift = __gmpn_lshift_k6_mmx; \
    decided_cpuvec.rshift = __gmpn_rshift_k6_mmx; \
  } while (0)

DECL_copyd (__gmpn_copyd_k6_k62mmx);
DECL_lshift (__gmpn_lshift_k6_k62mmx);
DECL_rshift (__gmpn_rshift_k6_k62mmx);

#define CPUVEC_SETUP_k6_k62mmx \
  do { \
    decided_cpuvec.copyd = __gmpn_copyd_k6_k62mmx; \
    decided_cpuvec.lshift = __gmpn_lshift_k6_k62mmx; \
    decided_cpuvec.rshift = __gmpn_rshift_k6_k62mmx; \
  } while (0)

DECL_add_n (__gmpn_add_n_k7);
DECL_addmul_1 (__gmpn_addmul_1_k7);
DECL_divexact_1 (__gmpn_divexact_1_k7);
DECL_gcd_1 (__gmpn_gcd_1_k7);
DECL_mod_1_1p (__gmpn_mod_1_1p_k7);
DECL_mod_1_1p_cps (__gmpn_mod_1_1p_cps_k7);
DECL_mod_1s_4p (__gmpn_mod_1s_4p_k7);
DECL_mod_1s_4p_cps (__gmpn_mod_1s_4p_cps_k7);
DECL_mod_34lsub1 (__gmpn_mod_34lsub1_k7);
DECL_modexact_1c_odd (__gmpn_modexact_1c_odd_k7);
DECL_mul_1 (__gmpn_mul_1_k7);
DECL_mul_basecase (__gmpn_mul_basecase_k7);
DECL_sqr_basecase (__gmpn_sqr_basecase_k7);
DECL_sub_n (__gmpn_sub_n_k7);
DECL_submul_1 (__gmpn_submul_1_k7);

#define CPUVEC_SETUP_k7 \
  do { \
    decided_cpuvec.mul_toom22_threshold = 28; \
    decided_cpuvec.mul_toom33_threshold = 85; \
    decided_cpuvec.sqr_toom2_threshold = 50; \
    decided_cpuvec.sqr_toom3_threshold = 83; \
    decided_cpuvec.bmod_1_to_mod_1_threshold = 24; \
    decided_cpuvec.add_n = __gmpn_add_n_k7; \
    decided_cpuvec.addmul_1 = __gmpn_addmul_1_k7; \
    decided_cpuvec.divexact_1 = __gmpn_divexact_1_k7; \
    decided_cpuvec.gcd_1 = __gmpn_gcd_1_k7; \
    decided_cpuvec.mod_1_1p = __gmpn_mod_1_1p_k7; \
    decided_cpuvec.mod_1_1p_cps = __gmpn_mod_1_1p_cps_k7; \
    decided_cpuvec.mod_1s_4p = __gmpn_mod_1s_4p_k7; \
    decided_cpuvec.mod_1s_4p_cps = __gmpn_mod_1s_4p_cps_k7; \
    decided_cpuvec.mod_34lsub1 = __gmpn_mod_34lsub1_k7; \
    decided_cpuvec.modexact_1c_odd = __gmpn_modexact_1c_odd_k7; \
    decided_cpuvec.mul_1 = __gmpn_mul_1_k7; \
    decided_cpuvec.mul_basecase = __gmpn_mul_basecase_k7; \
    decided_cpuvec.sqr_basecase = __gmpn_sqr_basecase_k7; \
    decided_cpuvec.sub_n = __gmpn_sub_n_k7; \
    decided_cpuvec.submul_1 = __gmpn_submul_1_k7; \
  } while (0)

DECL_com (__gmpn_com_k7_mmx);
DECL_copyd (__gmpn_copyd_k7_mmx);
DECL_copyi (__gmpn_copyi_k7_mmx);
DECL_divrem_1 (__gmpn_divrem_1_k7_mmx);
DECL_preinv_divrem_1 (__gmpn_preinv_divrem_1_k7_mmx);
DECL_lshift (__gmpn_lshift_k7_mmx);
DECL_rshift (__gmpn_rshift_k7_mmx);

#define CPUVEC_SETUP_k7_mmx \
  do { \
    decided_cpuvec.com = __gmpn_com_k7_mmx; \
    decided_cpuvec.copyd = __gmpn_copyd_k7_mmx; \
    decided_cpuvec.copyi = __gmpn_copyi_k7_mmx; \
    decided_cpuvec.divrem_1 = __gmpn_divrem_1_k7_mmx; \
    decided_cpuvec.preinv_divrem_1 = __gmpn_preinv_divrem_1_k7_mmx; \
    decided_cpuvec.lshift = __gmpn_lshift_k7_mmx; \
    decided_cpuvec.rshift = __gmpn_rshift_k7_mmx; \
  } while (0)


#define CPUVEC_SETUP_k8 \
  do { \
    decided_cpuvec.mul_toom22_threshold = 26; \
    decided_cpuvec.mul_toom33_threshold = 81; \
    decided_cpuvec.sqr_toom2_threshold = 46; \
    decided_cpuvec.sqr_toom3_threshold = 81; \
    decided_cpuvec.bmod_1_to_mod_1_threshold = 40; \
  } while (0)


#define CPUVEC_SETUP_k10 \
  do { \
    decided_cpuvec.mul_toom22_threshold = 26; \
    decided_cpuvec.mul_toom33_threshold = 85; \
    decided_cpuvec.sqr_toom2_threshold = 40; \
    decided_cpuvec.sqr_toom3_threshold = 81; \
    decided_cpuvec.bmod_1_to_mod_1_threshold = 31; \
  } while (0)


#define CPUVEC_SETUP_bobcat \
  do { \
    decided_cpuvec.mul_toom22_threshold = 28; \
    decided_cpuvec.mul_toom33_threshold = 90; \
    decided_cpuvec.sqr_toom2_threshold = 38; \
    decided_cpuvec.sqr_toom3_threshold = 89; \
    decided_cpuvec.bmod_1_to_mod_1_threshold = 42; \
  } while (0)

DECL_add_n (__gmpn_add_n_pentium);
DECL_addmul_1 (__gmpn_addmul_1_pentium);
DECL_com (__gmpn_com_pentium);
DECL_copyd (__gmpn_copyd_pentium);
DECL_copyi (__gmpn_copyi_pentium);
DECL_divexact_1 (__gmpn_divexact_1_pentium);
DECL_lshift (__gmpn_lshift_pentium);
DECL_mod_34lsub1 (__gmpn_mod_34lsub1_pentium);
DECL_modexact_1c_odd (__gmpn_modexact_1c_odd_pentium);
DECL_mul_1 (__gmpn_mul_1_pentium);
DECL_mul_basecase (__gmpn_mul_basecase_pentium);
DECL_rshift (__gmpn_rshift_pentium);
DECL_sqr_basecase (__gmpn_sqr_basecase_pentium);
DECL_sub_n (__gmpn_sub_n_pentium);
DECL_submul_1 (__gmpn_submul_1_pentium);

#define CPUVEC_SETUP_pentium \
  do { \
    decided_cpuvec.mul_toom22_threshold = 16; \
    decided_cpuvec.mul_toom33_threshold = 90; \
    decided_cpuvec.sqr_toom2_threshold = 22; \
    decided_cpuvec.sqr_toom3_threshold = 122; \
    decided_cpuvec.add_n = __gmpn_add_n_pentium; \
    decided_cpuvec.addmul_1 = __gmpn_addmul_1_pentium; \
    decided_cpuvec.com = __gmpn_com_pentium; \
    decided_cpuvec.copyd = __gmpn_copyd_pentium; \
    decided_cpuvec.copyi = __gmpn_copyi_pentium; \
    decided_cpuvec.divexact_1 = __gmpn_divexact_1_pentium; \
    decided_cpuvec.lshift = __gmpn_lshift_pentium; \
    decided_cpuvec.mod_34lsub1 = __gmpn_mod_34lsub1_pentium; \
    decided_cpuvec.modexact_1c_odd = __gmpn_modexact_1c_odd_pentium; \
    decided_cpuvec.mul_1 = __gmpn_mul_1_pentium; \
    decided_cpuvec.mul_basecase = __gmpn_mul_basecase_pentium; \
    decided_cpuvec.rshift = __gmpn_rshift_pentium; \
    decided_cpuvec.sqr_basecase = __gmpn_sqr_basecase_pentium; \
    decided_cpuvec.sub_n = __gmpn_sub_n_pentium; \
    decided_cpuvec.submul_1 = __gmpn_submul_1_pentium; \
  } while (0)

DECL_lshift (__gmpn_lshift_pentium_mmx);
DECL_mul_1 (__gmpn_mul_1_pentium_mmx);
DECL_rshift (__gmpn_rshift_pentium_mmx);

#define CPUVEC_SETUP_pentium_mmx \
  do { \
    decided_cpuvec.mul_toom22_threshold = 16; \
    decided_cpuvec.mul_toom33_threshold = 53; \
    decided_cpuvec.sqr_toom2_threshold = 20; \
    decided_cpuvec.sqr_toom3_threshold = 73; \
    decided_cpuvec.bmod_1_to_mod_1_threshold = 51; \
    decided_cpuvec.lshift = __gmpn_lshift_pentium_mmx; \
    decided_cpuvec.mul_1 = __gmpn_mul_1_pentium_mmx; \
    decided_cpuvec.rshift = __gmpn_rshift_pentium_mmx; \
  } while (0)

DECL_add_n (__gmpn_add_n_p6);
DECL_addmul_1 (__gmpn_addmul_1_p6);
DECL_copyd (__gmpn_copyd_p6);
DECL_divexact_1 (__gmpn_divexact_1_p6);
DECL_gcd_1 (__gmpn_gcd_1_p6);
DECL_mod_34lsub1 (__gmpn_mod_34lsub1_p6);
DECL_modexact_1c_odd (__gmpn_modexact_1c_odd_p6);
DECL_mul_basecase (__gmpn_mul_basecase_p6);
DECL_sqr_basecase (__gmpn_sqr_basecase_p6);
DECL_sub_n (__gmpn_sub_n_p6);
DECL_submul_1 (__gmpn_submul_1_p6);

#define CPUVEC_SETUP_p6 \
  do { \
    decided_cpuvec.mul_toom22_threshold = 20; \
    decided_cpuvec.mul_toom33_threshold = 74; \
    decided_cpuvec.sqr_toom2_threshold = 30; \
    decided_cpuvec.sqr_toom3_threshold = 101; \
    decided_cpuvec.bmod_1_to_mod_1_threshold = 21; \
    decided_cpuvec.add_n = __gmpn_add_n_p6; \
    decided_cpuvec.addmul_1 = __gmpn_addmul_1_p6; \
    decided_cpuvec.copyd = __gmpn_copyd_p6; \
    decided_cpuvec.divexact_1 = __gmpn_divexact_1_p6; \
    decided_cpuvec.gcd_1 = __gmpn_gcd_1_p6; \
    decided_cpuvec.mod_34lsub1 = __gmpn_mod_34lsub1_p6; \
    decided_cpuvec.modexact_1c_odd = __gmpn_modexact_1c_odd_p6; \
    decided_cpuvec.mul_basecase = __gmpn_mul_basecase_p6; \
    decided_cpuvec.sqr_basecase = __gmpn_sqr_basecase_p6; \
    decided_cpuvec.sub_n = __gmpn_sub_n_p6; \
    decided_cpuvec.submul_1 = __gmpn_submul_1_p6; \
  } while (0)

DECL_divrem_1 (__gmpn_divrem_1_p6_mmx);
DECL_preinv_divrem_1 (__gmpn_preinv_divrem_1_p6_mmx);
DECL_lshift (__gmpn_lshift_p6_mmx);
DECL_rshift (__gmpn_rshift_p6_mmx);

#define CPUVEC_SETUP_p6_mmx \
  do { \
    decided_cpuvec.mul_toom22_threshold = 22; \
    decided_cpuvec.mul_toom33_threshold = 73; \
    decided_cpuvec.sqr_toom2_threshold = 30; \
    decided_cpuvec.sqr_toom3_threshold = 81; \
    decided_cpuvec.bmod_1_to_mod_1_threshold = 49; \
    decided_cpuvec.divrem_1 = __gmpn_divrem_1_p6_mmx; \
    decided_cpuvec.preinv_divrem_1 = __gmpn_preinv_divrem_1_p6_mmx; \
    decided_cpuvec.lshift = __gmpn_lshift_p6_mmx; \
    decided_cpuvec.rshift = __gmpn_rshift_p6_mmx; \
  } while (0)


#define CPUVEC_SETUP_p6_p3mmx \
  do { \
  } while (0)

DECL_addmul_1 (__gmpn_addmul_1_p6_sse2);
DECL_mod_1_1p (__gmpn_mod_1_1p_p6_sse2);
DECL_mod_1s_4p (__gmpn_mod_1s_4p_p6_sse2);
DECL_mul_1 (__gmpn_mul_1_p6_sse2);
DECL_mul_basecase (__gmpn_mul_basecase_p6_sse2);
DECL_sqr_basecase (__gmpn_sqr_basecase_p6_sse2);
DECL_submul_1 (__gmpn_submul_1_p6_sse2);

#define CPUVEC_SETUP_p6_sse2 \
  do { \
    decided_cpuvec.mul_toom22_threshold = 20; \
    decided_cpuvec.mul_toom33_threshold = 77; \
    decided_cpuvec.sqr_toom2_threshold = 30; \
    decided_cpuvec.sqr_toom3_threshold = 101; \
    decided_cpuvec.bmod_1_to_mod_1_threshold = 21; \
    decided_cpuvec.addmul_1 = __gmpn_addmul_1_p6_sse2; \
    decided_cpuvec.mod_1_1p = __gmpn_mod_1_1p_p6_sse2; \
    decided_cpuvec.mod_1s_4p = __gmpn_mod_1s_4p_p6_sse2; \
    decided_cpuvec.mul_1 = __gmpn_mul_1_p6_sse2; \
    decided_cpuvec.mul_basecase = __gmpn_mul_basecase_p6_sse2; \
    decided_cpuvec.sqr_basecase = __gmpn_sqr_basecase_p6_sse2; \
    decided_cpuvec.submul_1 = __gmpn_submul_1_p6_sse2; \
  } while (0)

DECL_copyd (__gmpn_copyd_pentium4);
DECL_copyi (__gmpn_copyi_pentium4);

#define CPUVEC_SETUP_pentium4 \
  do { \
    decided_cpuvec.copyd = __gmpn_copyd_pentium4; \
    decided_cpuvec.copyi = __gmpn_copyi_pentium4; \
  } while (0)

DECL_lshift (__gmpn_lshift_pentium4_mmx);
DECL_rshift (__gmpn_rshift_pentium4_mmx);

#define CPUVEC_SETUP_pentium4_mmx \
  do { \
    decided_cpuvec.lshift = __gmpn_lshift_pentium4_mmx; \
    decided_cpuvec.rshift = __gmpn_rshift_pentium4_mmx; \
  } while (0)

DECL_add_n (__gmpn_add_n_pentium4_sse2);
DECL_addmul_1 (__gmpn_addmul_1_pentium4_sse2);
DECL_bdiv_dbm1c (__gmpn_bdiv_dbm1c_pentium4_sse2);
DECL_divexact_1 (__gmpn_divexact_1_pentium4_sse2);
DECL_divrem_1 (__gmpn_divrem_1_pentium4_sse2);
DECL_preinv_divrem_1 (__gmpn_preinv_divrem_1_pentium4_sse2);
DECL_mod_1_1p (__gmpn_mod_1_1p_pentium4_sse2);
DECL_mod_1_1p_cps (__gmpn_mod_1_1p_cps_pentium4_sse2);
DECL_mod_1s_4p (__gmpn_mod_1s_4p_pentium4_sse2);
DECL_mod_1s_4p_cps (__gmpn_mod_1s_4p_cps_pentium4_sse2);
DECL_mod_34lsub1 (__gmpn_mod_34lsub1_pentium4_sse2);
DECL_modexact_1c_odd (__gmpn_modexact_1c_odd_pentium4_sse2);
DECL_mul_1 (__gmpn_mul_1_pentium4_sse2);
DECL_mul_basecase (__gmpn_mul_basecase_pentium4_sse2);
DECL_sqr_basecase (__gmpn_sqr_basecase_pentium4_sse2);
DECL_sub_n (__gmpn_sub_n_pentium4_sse2);
DECL_submul_1 (__gmpn_submul_1_pentium4_sse2);

#define CPUVEC_SETUP_pentium4_sse2 \
  do { \
    decided_cpuvec.mul_toom22_threshold = 29; \
    decided_cpuvec.mul_toom33_threshold = 107; \
    decided_cpuvec.sqr_toom2_threshold = 48; \
    decided_cpuvec.sqr_toom3_threshold = 173; \
    decided_cpuvec.bmod_1_to_mod_1_threshold = 20; \
    decided_cpuvec.add_n = __gmpn_add_n_pentium4_sse2; \
    decided_cpuvec.addmul_1 = __gmpn_addmul_1_pentium4_sse2; \
    decided_cpuvec.bdiv_dbm1c = __gmpn_bdiv_dbm1c_pentium4_sse2; \
    decided_cpuvec.divexact_1 = __gmpn_divexact_1_pentium4_sse2; \
    decided_cpuvec.divrem_1 = __gmpn_divrem_1_pentium4_sse2; \
    decided_cpuvec.preinv_divrem_1 = __gmpn_preinv_divrem_1_pentium4_sse2; \
    decided_cpuvec.mod_1_1p = __gmpn_mod_1_1p_pentium4_sse2; \
    decided_cpuvec.mod_1_1p_cps = __gmpn_mod_1_1p_cps_pentium4_sse2; \
    decided_cpuvec.mod_1s_4p = __gmpn_mod_1s_4p_pentium4_sse2; \
    decided_cpuvec.mod_1s_4p_cps = __gmpn_mod_1s_4p_cps_pentium4_sse2; \
    decided_cpuvec.mod_34lsub1 = __gmpn_mod_34lsub1_pentium4_sse2; \
    decided_cpuvec.modexact_1c_odd = __gmpn_modexact_1c_odd_pentium4_sse2; \
    decided_cpuvec.mul_1 = __gmpn_mul_1_pentium4_sse2; \
    decided_cpuvec.mul_basecase = __gmpn_mul_basecase_pentium4_sse2; \
    decided_cpuvec.sqr_basecase = __gmpn_sqr_basecase_pentium4_sse2; \
    decided_cpuvec.sub_n = __gmpn_sub_n_pentium4_sse2; \
    decided_cpuvec.submul_1 = __gmpn_submul_1_pentium4_sse2; \
  } while (0)


#define CPUVEC_SETUP_core2 \
  do { \
    decided_cpuvec.mul_toom22_threshold = 24; \
    decided_cpuvec.mul_toom33_threshold = 93; \
    decided_cpuvec.sqr_toom2_threshold = 34; \
    decided_cpuvec.sqr_toom3_threshold = 116; \
    decided_cpuvec.bmod_1_to_mod_1_threshold = 19; \
  } while (0)


#define CPUVEC_SETUP_coreinhm \
  do { \
    decided_cpuvec.mul_toom22_threshold = 28; \
    decided_cpuvec.mul_toom33_threshold = 81; \
    decided_cpuvec.sqr_toom2_threshold = 42; \
    decided_cpuvec.sqr_toom3_threshold = 149; \
    decided_cpuvec.bmod_1_to_mod_1_threshold = 16; \
  } while (0)


#define CPUVEC_SETUP_coreisbr \
  do { \
    decided_cpuvec.mul_toom22_threshold = 28; \
    decided_cpuvec.mul_toom33_threshold = 101; \
    decided_cpuvec.sqr_toom2_threshold = 48; \
    decided_cpuvec.sqr_toom3_threshold = 165; \
    decided_cpuvec.bmod_1_to_mod_1_threshold = 18; \
  } while (0)

DECL_add_n (__gmpn_add_n_atom);
DECL_divexact_1 (__gmpn_divexact_1_atom);
DECL_lshift (__gmpn_lshift_atom);
DECL_lshiftc (__gmpn_lshiftc_atom);
DECL_mod_34lsub1 (__gmpn_mod_34lsub1_atom);
DECL_modexact_1c_odd (__gmpn_modexact_1c_odd_atom);
DECL_rshift (__gmpn_rshift_atom);
DECL_sub_n (__gmpn_sub_n_atom);

#define CPUVEC_SETUP_atom \
  do { \
    decided_cpuvec.mul_toom22_threshold = 22; \
    decided_cpuvec.mul_toom33_threshold = 81; \
    decided_cpuvec.sqr_toom2_threshold = 32; \
    decided_cpuvec.sqr_toom3_threshold = 109; \
    decided_cpuvec.bmod_1_to_mod_1_threshold = 33; \
    decided_cpuvec.add_n = __gmpn_add_n_atom; \
    decided_cpuvec.divexact_1 = __gmpn_divexact_1_atom; \
    decided_cpuvec.lshift = __gmpn_lshift_atom; \
    decided_cpuvec.lshiftc = __gmpn_lshiftc_atom; \
    decided_cpuvec.mod_34lsub1 = __gmpn_mod_34lsub1_atom; \
    decided_cpuvec.modexact_1c_odd = __gmpn_modexact_1c_odd_atom; \
    decided_cpuvec.rshift = __gmpn_rshift_atom; \
    decided_cpuvec.sub_n = __gmpn_sub_n_atom; \
  } while (0)

DECL_copyd (__gmpn_copyd_atom_mmx);
DECL_copyi (__gmpn_copyi_atom_mmx);

#define CPUVEC_SETUP_atom_mmx \
  do { \
    decided_cpuvec.copyd = __gmpn_copyd_atom_mmx; \
    decided_cpuvec.copyi = __gmpn_copyi_atom_mmx; \
  } while (0)

DECL_addmul_1 (__gmpn_addmul_1_atom_sse2);
DECL_bdiv_dbm1c (__gmpn_bdiv_dbm1c_atom_sse2);
DECL_divrem_1 (__gmpn_divrem_1_atom_sse2);
DECL_mod_1_1p (__gmpn_mod_1_1p_atom_sse2);
DECL_mod_1s_4p (__gmpn_mod_1s_4p_atom_sse2);
DECL_mul_1 (__gmpn_mul_1_atom_sse2);
DECL_mul_basecase (__gmpn_mul_basecase_atom_sse2);
DECL_sqr_basecase (__gmpn_sqr_basecase_atom_sse2);
DECL_submul_1 (__gmpn_submul_1_atom_sse2);

#define CPUVEC_SETUP_atom_sse2 \
  do { \
    decided_cpuvec.addmul_1 = __gmpn_addmul_1_atom_sse2; \
    decided_cpuvec.bdiv_dbm1c = __gmpn_bdiv_dbm1c_atom_sse2; \
    decided_cpuvec.divrem_1 = __gmpn_divrem_1_atom_sse2; \
    decided_cpuvec.mod_1_1p = __gmpn_mod_1_1p_atom_sse2; \
    decided_cpuvec.mod_1s_4p = __gmpn_mod_1s_4p_atom_sse2; \
    decided_cpuvec.mul_1 = __gmpn_mul_1_atom_sse2; \
    decided_cpuvec.mul_basecase = __gmpn_mul_basecase_atom_sse2; \
    decided_cpuvec.sqr_basecase = __gmpn_sqr_basecase_atom_sse2; \
    decided_cpuvec.submul_1 = __gmpn_submul_1_atom_sse2; \
  } while (0)


#define CPUVEC_SETUP_nano \
  do { \
    decided_cpuvec.mul_toom22_threshold = 16; \
    decided_cpuvec.mul_toom33_threshold = 132; \
    decided_cpuvec.sqr_toom2_threshold = 28; \
    decided_cpuvec.sqr_toom3_threshold = 194; \
    decided_cpuvec.bmod_1_to_mod_1_threshold = 32; \
  } while (0)

#define MUL_TOOM22_THRESHOLD_LIMIT  29
#define MUL_TOOM33_THRESHOLD_LIMIT  228
#define SQR_TOOM2_THRESHOLD_LIMIT  50
#define SQR_TOOM3_THRESHOLD_LIMIT  238
#define BMOD_1_TO_MOD_1_THRESHOLD_LIMIT  51
