/*********************************************************************/

/* Copyright 2009, 2010 The University of Texas at Austin.           */

/* All rights reserved.                                              */

/*                                                                   */

/* Redistribution and use in source and binary forms, with or        */

/* without modification, are permitted provided that the following   */

/* conditions are met:                                               */

/*                                                                   */

/*   1. Redistributions of source code must retain the above         */

/*      copyright notice, this list of conditions and the following  */

/*      disclaimer.                                                  */

/*                                                                   */

/*   2. Redistributions in binary form must reproduce the above      */

/*      copyright notice, this list of conditions and the following  */

/*      disclaimer in the documentation and/or other materials       */

/*      provided with the distribution.                              */

/*                                                                   */

/*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

/*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

/*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

/*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

/*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

/*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

/*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

/*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

/*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

/*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

/*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

/*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

/*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

/*    POSSIBILITY OF SUCH DAMAGE.                                    */

/*                                                                   */

/* The views and conclusions contained in the software and           */

/* documentation are those of the authors and should not be          */

/* interpreted as representing official policies, either expressed   */

/* or implied, of The University of Texas at Austin.                 */

/*********************************************************************/

#include <stdio.h></stdio.h>

#include <string.h></string.h>

#include "common.h"

#ifdef BUILD_KERNEL

#include "kernelTS.h"

#endif

#undef DEBUG

static void init_parameter(void);

gotoblas_t TABLE_NAME = {

  DTB_DEFAULT_ENTRIES ,

  GEMM_DEFAULT_OFFSET_A, GEMM_DEFAULT_OFFSET_B, GEMM_DEFAULT_ALIGN,

  0, 0, 0,

  SGEMM_DEFAULT_UNROLL_M, SGEMM_DEFAULT_UNROLL_N, MAX(SGEMM_DEFAULT_UNROLL_M, SGEMM_DEFAULT_UNROLL_N),

#ifdef HAVE_EXCLUSIVE_CACHE

  1,

#else

  0,

#endif

  samax_kTS,  samin_kTS,  smax_kTS,  smin_kTS,

  isamax_kTS, isamin_kTS, ismax_kTS, ismin_kTS,

  snrm2_kTS,  sasum_kTS,  scopy_kTS, sdot_kTS,

  dsdot_kTS,

  srot_kTS,   saxpy_kTS,  sscal_kTS, sswap_kTS,

  sgemv_nTS,  sgemv_tTS, sger_kTS,

  ssymv_LTS, ssymv_UTS,

  sgemm_kernelTS, sgemm_betaTS, 

#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N

  sgemm_incopyTS, sgemm_itcopyTS, 

#else

  sgemm_oncopyTS, sgemm_otcopyTS,

#endif

  sgemm_oncopyTS, sgemm_otcopyTS,

  strsm_kernel_LNTS, strsm_kernel_LTTS, strsm_kernel_RNTS, strsm_kernel_RTTS,

#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N

  strsm_iunucopyTS, strsm_iunncopyTS, strsm_iutucopyTS, strsm_iutncopyTS,

  strsm_ilnucopyTS, strsm_ilnncopyTS, strsm_iltucopyTS, strsm_iltncopyTS,

#else

  strsm_ounucopyTS, strsm_ounncopyTS, strsm_outucopyTS, strsm_outncopyTS,

  strsm_olnucopyTS, strsm_olnncopyTS, strsm_oltucopyTS, strsm_oltncopyTS,

#endif

  strsm_ounucopyTS, strsm_ounncopyTS, strsm_outucopyTS, strsm_outncopyTS,

  strsm_olnucopyTS, strsm_olnncopyTS, strsm_oltucopyTS, strsm_oltncopyTS,

  strmm_kernel_RNTS, strmm_kernel_RTTS, strmm_kernel_LNTS, strmm_kernel_LTTS,

#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N

  strmm_iunucopyTS, strmm_iunncopyTS, strmm_iutucopyTS, strmm_iutncopyTS,

  strmm_ilnucopyTS, strmm_ilnncopyTS, strmm_iltucopyTS, strmm_iltncopyTS,

#else

  strmm_ounucopyTS, strmm_ounncopyTS, strmm_outucopyTS, strmm_outncopyTS,

  strmm_olnucopyTS, strmm_olnncopyTS, strmm_oltucopyTS, strmm_oltncopyTS,

#endif

  strmm_ounucopyTS, strmm_ounncopyTS, strmm_outucopyTS, strmm_outncopyTS,

  strmm_olnucopyTS, strmm_olnncopyTS, strmm_oltucopyTS, strmm_oltncopyTS,

#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N

  ssymm_iutcopyTS, ssymm_iltcopyTS, 

#else

  ssymm_outcopyTS, ssymm_oltcopyTS,

#endif

  ssymm_outcopyTS, ssymm_oltcopyTS,

#ifndef NO_LAPACK

  sneg_tcopyTS, slaswp_ncopyTS,

#else

  NULL,NULL,

#endif

  0, 0, 0,

  DGEMM_DEFAULT_UNROLL_M, DGEMM_DEFAULT_UNROLL_N, MAX(DGEMM_DEFAULT_UNROLL_M, DGEMM_DEFAULT_UNROLL_N),

  damax_kTS,  damin_kTS,  dmax_kTS,  dmin_kTS,

  idamax_kTS, idamin_kTS, idmax_kTS, idmin_kTS,

  dnrm2_kTS,  dasum_kTS,  dcopy_kTS, ddot_kTS,

  drot_kTS,   daxpy_kTS,  dscal_kTS, dswap_kTS,

  dgemv_nTS,  dgemv_tTS,  dger_kTS,

  dsymv_LTS,  dsymv_UTS,

  dgemm_kernelTS, dgemm_betaTS, 

#if DGEMM_DEFAULT_UNROLL_M != DGEMM_DEFAULT_UNROLL_N

  dgemm_incopyTS, dgemm_itcopyTS, 

#else

  dgemm_oncopyTS, dgemm_otcopyTS,

#endif

  dgemm_oncopyTS, dgemm_otcopyTS,

  dtrsm_kernel_LNTS, dtrsm_kernel_LTTS, dtrsm_kernel_RNTS, dtrsm_kernel_RTTS,

#if DGEMM_DEFAULT_UNROLL_M != DGEMM_DEFAULT_UNROLL_N

  dtrsm_iunucopyTS, dtrsm_iunncopyTS, dtrsm_iutucopyTS, dtrsm_iutncopyTS,

  dtrsm_ilnucopyTS, dtrsm_ilnncopyTS, dtrsm_iltucopyTS, dtrsm_iltncopyTS,

#else

  dtrsm_ounucopyTS, dtrsm_ounncopyTS, dtrsm_outucopyTS, dtrsm_outncopyTS,

  dtrsm_olnucopyTS, dtrsm_olnncopyTS, dtrsm_oltucopyTS, dtrsm_oltncopyTS,

#endif

  dtrsm_ounucopyTS, dtrsm_ounncopyTS, dtrsm_outucopyTS, dtrsm_outncopyTS,

  dtrsm_olnucopyTS, dtrsm_olnncopyTS, dtrsm_oltucopyTS, dtrsm_oltncopyTS,

  dtrmm_kernel_RNTS, dtrmm_kernel_RTTS, dtrmm_kernel_LNTS, dtrmm_kernel_LTTS,

#if DGEMM_DEFAULT_UNROLL_M != DGEMM_DEFAULT_UNROLL_N

  dtrmm_iunucopyTS, dtrmm_iunncopyTS, dtrmm_iutucopyTS, dtrmm_iutncopyTS,

  dtrmm_ilnucopyTS, dtrmm_ilnncopyTS, dtrmm_iltucopyTS, dtrmm_iltncopyTS,

#else

  dtrmm_ounucopyTS, dtrmm_ounncopyTS, dtrmm_outucopyTS, dtrmm_outncopyTS,

  dtrmm_olnucopyTS, dtrmm_olnncopyTS, dtrmm_oltucopyTS, dtrmm_oltncopyTS,

#endif

  dtrmm_ounucopyTS, dtrmm_ounncopyTS, dtrmm_outucopyTS, dtrmm_outncopyTS,

  dtrmm_olnucopyTS, dtrmm_olnncopyTS, dtrmm_oltucopyTS, dtrmm_oltncopyTS,

#if DGEMM_DEFAULT_UNROLL_M != DGEMM_DEFAULT_UNROLL_N

  dsymm_iutcopyTS, dsymm_iltcopyTS, 

#else

  dsymm_outcopyTS, dsymm_oltcopyTS,

#endif

  dsymm_outcopyTS, dsymm_oltcopyTS,

#ifndef NO_LAPACK

  dneg_tcopyTS, dlaswp_ncopyTS,

#else

  NULL, NULL,

#endif

#ifdef EXPRECISION

  0, 0, 0,

  QGEMM_DEFAULT_UNROLL_M, QGEMM_DEFAULT_UNROLL_N, MAX(QGEMM_DEFAULT_UNROLL_M, QGEMM_DEFAULT_UNROLL_N),

  qamax_kTS,  qamin_kTS,  qmax_kTS,  qmin_kTS,

  iqamax_kTS, iqamin_kTS, iqmax_kTS, iqmin_kTS,

  qnrm2_kTS,  qasum_kTS,  qcopy_kTS, qdot_kTS,

  qrot_kTS,   qaxpy_kTS,  qscal_kTS, qswap_kTS,

  qgemv_nTS,  qgemv_tTS,  qger_kTS,

  qsymv_LTS,  qsymv_UTS,

  qgemm_kernelTS, qgemm_betaTS, 

#if QGEMM_DEFAULT_UNROLL_M != QGEMM_DEFAULT_UNROLL_N

  qgemm_incopyTS, qgemm_itcopyTS, 

#else

  qgemm_oncopyTS, qgemm_otcopyTS,

#endif

  qgemm_oncopyTS, qgemm_otcopyTS,

  qtrsm_kernel_LNTS, qtrsm_kernel_LTTS, qtrsm_kernel_RNTS, qtrsm_kernel_RTTS,

#if QGEMM_DEFAULT_UNROLL_M != QGEMM_DEFAULT_UNROLL_N

  qtrsm_iunucopyTS, qtrsm_iunncopyTS, qtrsm_iutucopyTS, qtrsm_iutncopyTS,

  qtrsm_ilnucopyTS, qtrsm_ilnncopyTS, qtrsm_iltucopyTS, qtrsm_iltncopyTS,

#else

  qtrsm_ounucopyTS, qtrsm_ounncopyTS, qtrsm_outucopyTS, qtrsm_outncopyTS,

  qtrsm_olnucopyTS, qtrsm_olnncopyTS, qtrsm_oltucopyTS, qtrsm_oltncopyTS,

#endif

  qtrsm_ounucopyTS, qtrsm_ounncopyTS, qtrsm_outucopyTS, qtrsm_outncopyTS,

  qtrsm_olnucopyTS, qtrsm_olnncopyTS, qtrsm_oltucopyTS, qtrsm_oltncopyTS,

  qtrmm_kernel_RNTS, qtrmm_kernel_RTTS, qtrmm_kernel_LNTS, qtrmm_kernel_LTTS,

#if QGEMM_DEFAULT_UNROLL_M != QGEMM_DEFAULT_UNROLL_N

  qtrmm_iunucopyTS, qtrmm_iunncopyTS, qtrmm_iutucopyTS, qtrmm_iutncopyTS,

  qtrmm_ilnucopyTS, qtrmm_ilnncopyTS, qtrmm_iltucopyTS, qtrmm_iltncopyTS,

#else

  qtrmm_ounucopyTS, qtrmm_ounncopyTS, qtrmm_outucopyTS, qtrmm_outncopyTS,

  qtrmm_olnucopyTS, qtrmm_olnncopyTS, qtrmm_oltucopyTS, qtrmm_oltncopyTS,

#endif

  qtrmm_ounucopyTS, qtrmm_ounncopyTS, qtrmm_outucopyTS, qtrmm_outncopyTS,

  qtrmm_olnucopyTS, qtrmm_olnncopyTS, qtrmm_oltucopyTS, qtrmm_oltncopyTS,

#if QGEMM_DEFAULT_UNROLL_M != QGEMM_DEFAULT_UNROLL_N

  qsymm_iutcopyTS, qsymm_iltcopyTS, 

#else

  qsymm_outcopyTS, qsymm_oltcopyTS,

#endif

  qsymm_outcopyTS, qsymm_oltcopyTS,

#ifndef NO_LAPACK

  qneg_tcopyTS, qlaswp_ncopyTS,

#else

  NULL, NULL,

#endif

#endif

  0, 0, 0,

  CGEMM_DEFAULT_UNROLL_M, CGEMM_DEFAULT_UNROLL_N, MAX(CGEMM_DEFAULT_UNROLL_M, CGEMM_DEFAULT_UNROLL_N),

  camax_kTS, camin_kTS, icamax_kTS, icamin_kTS,

  cnrm2_kTS, casum_kTS, ccopy_kTS,

  cdotu_kTS, cdotc_kTS, csrot_kTS,

  caxpy_kTS, caxpyc_kTS, cscal_kTS, cswap_kTS, 

  cgemv_nTS, cgemv_tTS, cgemv_rTS, cgemv_cTS, 

  cgemv_oTS, cgemv_uTS, cgemv_sTS, cgemv_dTS, 

  cgeru_kTS, cgerc_kTS, cgerv_kTS, cgerd_kTS, 

  csymv_LTS, csymv_UTS,

  chemv_LTS, chemv_UTS, chemv_MTS, chemv_VTS,

  cgemm_kernel_nTS, cgemm_kernel_lTS, cgemm_kernel_rTS, cgemm_kernel_bTS,

  cgemm_betaTS,

#if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N

  cgemm_incopyTS, cgemm_itcopyTS,

#else

  cgemm_oncopyTS, cgemm_otcopyTS,

#endif

  cgemm_oncopyTS, cgemm_otcopyTS,

  ctrsm_kernel_LNTS, ctrsm_kernel_LTTS, ctrsm_kernel_LRTS, ctrsm_kernel_LCTS,

  ctrsm_kernel_RNTS, ctrsm_kernel_RTTS, ctrsm_kernel_RRTS, ctrsm_kernel_RCTS,

#if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N

  ctrsm_iunucopyTS,  ctrsm_iunncopyTS,  ctrsm_iutucopyTS,  ctrsm_iutncopyTS,

  ctrsm_ilnucopyTS,  ctrsm_ilnncopyTS,  ctrsm_iltucopyTS,  ctrsm_iltncopyTS,

#else

  ctrsm_ounucopyTS,  ctrsm_ounncopyTS,  ctrsm_outucopyTS,  ctrsm_outncopyTS,

  ctrsm_olnucopyTS,  ctrsm_olnncopyTS,  ctrsm_oltucopyTS,  ctrsm_oltncopyTS,

#endif

  ctrsm_ounucopyTS,  ctrsm_ounncopyTS,  ctrsm_outucopyTS,  ctrsm_outncopyTS,

  ctrsm_olnucopyTS,  ctrsm_olnncopyTS,  ctrsm_oltucopyTS,  ctrsm_oltncopyTS,

  ctrmm_kernel_RNTS,  ctrmm_kernel_RTTS,  ctrmm_kernel_RRTS,  ctrmm_kernel_RCTS,

  ctrmm_kernel_LNTS,  ctrmm_kernel_LTTS,  ctrmm_kernel_LRTS,  ctrmm_kernel_LCTS,

#if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N

  ctrmm_iunucopyTS,  ctrmm_iunncopyTS,  ctrmm_iutucopyTS,  ctrmm_iutncopyTS,

  ctrmm_ilnucopyTS,  ctrmm_ilnncopyTS,  ctrmm_iltucopyTS,  ctrmm_iltncopyTS,

#else

  ctrmm_ounucopyTS,  ctrmm_ounncopyTS,  ctrmm_outucopyTS,  ctrmm_outncopyTS,

  ctrmm_olnucopyTS,  ctrmm_olnncopyTS,  ctrmm_oltucopyTS,  ctrmm_oltncopyTS,

#endif

  ctrmm_ounucopyTS,  ctrmm_ounncopyTS,  ctrmm_outucopyTS,  ctrmm_outncopyTS,

  ctrmm_olnucopyTS,  ctrmm_olnncopyTS,  ctrmm_oltucopyTS,  ctrmm_oltncopyTS,

#if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N

  csymm_iutcopyTS,  csymm_iltcopyTS,

#else

  csymm_outcopyTS,  csymm_oltcopyTS,

#endif

  csymm_outcopyTS,  csymm_oltcopyTS,

#if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N

  chemm_iutcopyTS,  chemm_iltcopyTS,

#else

  chemm_outcopyTS,  chemm_oltcopyTS,

#endif

  chemm_outcopyTS,  chemm_oltcopyTS,

  cgemm3m_kernelTS,

  cgemm3m_incopybTS,  cgemm3m_incopyrTS,

  cgemm3m_incopyiTS,  cgemm3m_itcopybTS,

  cgemm3m_itcopyrTS,  cgemm3m_itcopyiTS,

  cgemm3m_oncopybTS,  cgemm3m_oncopyrTS,

  cgemm3m_oncopyiTS,  cgemm3m_otcopybTS,

  cgemm3m_otcopyrTS,  cgemm3m_otcopyiTS,

  csymm3m_iucopybTS,  csymm3m_ilcopybTS,

  csymm3m_iucopyrTS,  csymm3m_ilcopyrTS,

  csymm3m_iucopyiTS,  csymm3m_ilcopyiTS,

  csymm3m_oucopybTS,  csymm3m_olcopybTS,

  csymm3m_oucopyrTS,  csymm3m_olcopyrTS,

  csymm3m_oucopyiTS,  csymm3m_olcopyiTS,

  chemm3m_iucopybTS,  chemm3m_ilcopybTS,

  chemm3m_iucopyrTS,  chemm3m_ilcopyrTS,

  chemm3m_iucopyiTS,  chemm3m_ilcopyiTS, 

  chemm3m_oucopybTS,  chemm3m_olcopybTS,

  chemm3m_oucopyrTS,  chemm3m_olcopyrTS,

  chemm3m_oucopyiTS,  chemm3m_olcopyiTS,

#ifndef NO_LAPACK

  cneg_tcopyTS, claswp_ncopyTS,

#else

  NULL, NULL,

#endif

  0, 0, 0,

  ZGEMM_DEFAULT_UNROLL_M, ZGEMM_DEFAULT_UNROLL_N, MAX(ZGEMM_DEFAULT_UNROLL_M, ZGEMM_DEFAULT_UNROLL_N),

  zamax_kTS, zamin_kTS, izamax_kTS, izamin_kTS,

  znrm2_kTS, zasum_kTS, zcopy_kTS,

  zdotu_kTS, zdotc_kTS, zdrot_kTS,

  zaxpy_kTS, zaxpyc_kTS, zscal_kTS, zswap_kTS, 

  zgemv_nTS, zgemv_tTS, zgemv_rTS, zgemv_cTS, 

  zgemv_oTS, zgemv_uTS, zgemv_sTS, zgemv_dTS, 

  zgeru_kTS, zgerc_kTS, zgerv_kTS, zgerd_kTS, 

  zsymv_LTS, zsymv_UTS,

  zhemv_LTS, zhemv_UTS, zhemv_MTS, zhemv_VTS,

  zgemm_kernel_nTS, zgemm_kernel_lTS, zgemm_kernel_rTS, zgemm_kernel_bTS,

  zgemm_betaTS,

#if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N

  zgemm_incopyTS, zgemm_itcopyTS,

#else

  zgemm_oncopyTS, zgemm_otcopyTS,

#endif

  zgemm_oncopyTS, zgemm_otcopyTS,

  ztrsm_kernel_LNTS, ztrsm_kernel_LTTS, ztrsm_kernel_LRTS, ztrsm_kernel_LCTS,

  ztrsm_kernel_RNTS, ztrsm_kernel_RTTS, ztrsm_kernel_RRTS, ztrsm_kernel_RCTS,

#if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N

  ztrsm_iunucopyTS,  ztrsm_iunncopyTS,  ztrsm_iutucopyTS,  ztrsm_iutncopyTS,

  ztrsm_ilnucopyTS,  ztrsm_ilnncopyTS,  ztrsm_iltucopyTS,  ztrsm_iltncopyTS,

#else

  ztrsm_ounucopyTS,  ztrsm_ounncopyTS,  ztrsm_outucopyTS,  ztrsm_outncopyTS,

  ztrsm_olnucopyTS,  ztrsm_olnncopyTS,  ztrsm_oltucopyTS,  ztrsm_oltncopyTS,

#endif

  ztrsm_ounucopyTS,  ztrsm_ounncopyTS,  ztrsm_outucopyTS,  ztrsm_outncopyTS,

  ztrsm_olnucopyTS,  ztrsm_olnncopyTS,  ztrsm_oltucopyTS,  ztrsm_oltncopyTS,

  ztrmm_kernel_RNTS,  ztrmm_kernel_RTTS,  ztrmm_kernel_RRTS,  ztrmm_kernel_RCTS,

  ztrmm_kernel_LNTS,  ztrmm_kernel_LTTS,  ztrmm_kernel_LRTS,  ztrmm_kernel_LCTS,

#if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N

  ztrmm_iunucopyTS,  ztrmm_iunncopyTS,  ztrmm_iutucopyTS,  ztrmm_iutncopyTS,

  ztrmm_ilnucopyTS,  ztrmm_ilnncopyTS,  ztrmm_iltucopyTS,  ztrmm_iltncopyTS,

#else

  ztrmm_ounucopyTS,  ztrmm_ounncopyTS,  ztrmm_outucopyTS,  ztrmm_outncopyTS,

  ztrmm_olnucopyTS,  ztrmm_olnncopyTS,  ztrmm_oltucopyTS,  ztrmm_oltncopyTS,

#endif

  ztrmm_ounucopyTS,  ztrmm_ounncopyTS,  ztrmm_outucopyTS,  ztrmm_outncopyTS,

  ztrmm_olnucopyTS,  ztrmm_olnncopyTS,  ztrmm_oltucopyTS,  ztrmm_oltncopyTS,

#if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N

  zsymm_iutcopyTS,  zsymm_iltcopyTS,

#else

  zsymm_outcopyTS,  zsymm_oltcopyTS,

#endif

  zsymm_outcopyTS,  zsymm_oltcopyTS,

#if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N

  zhemm_iutcopyTS,  zhemm_iltcopyTS,

#else

  zhemm_outcopyTS,  zhemm_oltcopyTS,

#endif

  zhemm_outcopyTS,  zhemm_oltcopyTS,

  zgemm3m_kernelTS,

  zgemm3m_incopybTS,  zgemm3m_incopyrTS,

  zgemm3m_incopyiTS,  zgemm3m_itcopybTS,

  zgemm3m_itcopyrTS,  zgemm3m_itcopyiTS,

  zgemm3m_oncopybTS,  zgemm3m_oncopyrTS,

  zgemm3m_oncopyiTS,  zgemm3m_otcopybTS,

  zgemm3m_otcopyrTS,  zgemm3m_otcopyiTS,

  zsymm3m_iucopybTS,  zsymm3m_ilcopybTS,

  zsymm3m_iucopyrTS,  zsymm3m_ilcopyrTS,

  zsymm3m_iucopyiTS,  zsymm3m_ilcopyiTS,

  zsymm3m_oucopybTS,  zsymm3m_olcopybTS,

  zsymm3m_oucopyrTS,  zsymm3m_olcopyrTS,

  zsymm3m_oucopyiTS,  zsymm3m_olcopyiTS,

  zhemm3m_iucopybTS,  zhemm3m_ilcopybTS,

  zhemm3m_iucopyrTS,  zhemm3m_ilcopyrTS,

  zhemm3m_iucopyiTS,  zhemm3m_ilcopyiTS, 

  zhemm3m_oucopybTS,  zhemm3m_olcopybTS,

  zhemm3m_oucopyrTS,  zhemm3m_olcopyrTS,

  zhemm3m_oucopyiTS,  zhemm3m_olcopyiTS,

#ifndef NO_LAPACK

  zneg_tcopyTS, zlaswp_ncopyTS,

#else

  NULL, NULL,

#endif

#ifdef EXPRECISION

  0, 0, 0,

  XGEMM_DEFAULT_UNROLL_M, XGEMM_DEFAULT_UNROLL_N, MAX(XGEMM_DEFAULT_UNROLL_M, XGEMM_DEFAULT_UNROLL_N),

  xamax_kTS, xamin_kTS, ixamax_kTS, ixamin_kTS,

  xnrm2_kTS, xasum_kTS, xcopy_kTS,

  xdotu_kTS, xdotc_kTS, xqrot_kTS,

  xaxpy_kTS, xaxpyc_kTS, xscal_kTS, xswap_kTS, 

  xgemv_nTS, xgemv_tTS, xgemv_rTS, xgemv_cTS, 

  xgemv_oTS, xgemv_uTS, xgemv_sTS, xgemv_dTS, 

  xgeru_kTS, xgerc_kTS, xgerv_kTS, xgerd_kTS, 

  xsymv_LTS, xsymv_UTS,

  xhemv_LTS, xhemv_UTS, xhemv_MTS, xhemv_VTS,

  xgemm_kernel_nTS, xgemm_kernel_lTS, xgemm_kernel_rTS, xgemm_kernel_bTS,

  xgemm_betaTS,

#if XGEMM_DEFAULT_UNROLL_M != XGEMM_DEFAULT_UNROLL_N

  xgemm_incopyTS, xgemm_itcopyTS,

#else

  xgemm_oncopyTS, xgemm_otcopyTS,

#endif

  xgemm_oncopyTS, xgemm_otcopyTS,

  xtrsm_kernel_LNTS, xtrsm_kernel_LTTS, xtrsm_kernel_LRTS, xtrsm_kernel_LCTS,

  xtrsm_kernel_RNTS, xtrsm_kernel_RTTS, xtrsm_kernel_RRTS, xtrsm_kernel_RCTS,

#if XGEMM_DEFAULT_UNROLL_M != XGEMM_DEFAULT_UNROLL_N

  xtrsm_iunucopyTS,  xtrsm_iunncopyTS,  xtrsm_iutucopyTS,  xtrsm_iutncopyTS,

  xtrsm_ilnucopyTS,  xtrsm_ilnncopyTS,  xtrsm_iltucopyTS,  xtrsm_iltncopyTS,

#else

  xtrsm_ounucopyTS,  xtrsm_ounncopyTS,  xtrsm_outucopyTS,  xtrsm_outncopyTS,

  xtrsm_olnucopyTS,  xtrsm_olnncopyTS,  xtrsm_oltucopyTS,  xtrsm_oltncopyTS,

#endif

  xtrsm_ounucopyTS,  xtrsm_ounncopyTS,  xtrsm_outucopyTS,  xtrsm_outncopyTS,

  xtrsm_olnucopyTS,  xtrsm_olnncopyTS,  xtrsm_oltucopyTS,  xtrsm_oltncopyTS,

  xtrmm_kernel_RNTS,  xtrmm_kernel_RTTS,  xtrmm_kernel_RRTS,  xtrmm_kernel_RCTS,

  xtrmm_kernel_LNTS,  xtrmm_kernel_LTTS,  xtrmm_kernel_LRTS,  xtrmm_kernel_LCTS,

#if XGEMM_DEFAULT_UNROLL_M != XGEMM_DEFAULT_UNROLL_N

  xtrmm_iunucopyTS,  xtrmm_iunncopyTS,  xtrmm_iutucopyTS,  xtrmm_iutncopyTS,

  xtrmm_ilnucopyTS,  xtrmm_ilnncopyTS,  xtrmm_iltucopyTS,  xtrmm_iltncopyTS,

#else

  xtrmm_ounucopyTS,  xtrmm_ounncopyTS,  xtrmm_outucopyTS,  xtrmm_outncopyTS,

  xtrmm_olnucopyTS,  xtrmm_olnncopyTS,  xtrmm_oltucopyTS,  xtrmm_oltncopyTS,

#endif

  xtrmm_ounucopyTS,  xtrmm_ounncopyTS,  xtrmm_outucopyTS,  xtrmm_outncopyTS,

  xtrmm_olnucopyTS,  xtrmm_olnncopyTS,  xtrmm_oltucopyTS,  xtrmm_oltncopyTS,

#if XGEMM_DEFAULT_UNROLL_M != XGEMM_DEFAULT_UNROLL_N

  xsymm_iutcopyTS,  xsymm_iltcopyTS,

#else

  xsymm_outcopyTS,  xsymm_oltcopyTS,

#endif

  xsymm_outcopyTS,  xsymm_oltcopyTS,

#if XGEMM_DEFAULT_UNROLL_M != XGEMM_DEFAULT_UNROLL_N

  xhemm_iutcopyTS,  xhemm_iltcopyTS,

#else

  xhemm_outcopyTS,  xhemm_oltcopyTS,

#endif

  xhemm_outcopyTS,  xhemm_oltcopyTS,

  xgemm3m_kernelTS,

  xgemm3m_incopybTS,  xgemm3m_incopyrTS,

  xgemm3m_incopyiTS,  xgemm3m_itcopybTS,

  xgemm3m_itcopyrTS,  xgemm3m_itcopyiTS,

  xgemm3m_oncopybTS,  xgemm3m_oncopyrTS,

  xgemm3m_oncopyiTS,  xgemm3m_otcopybTS,

  xgemm3m_otcopyrTS,  xgemm3m_otcopyiTS,

  xsymm3m_iucopybTS,  xsymm3m_ilcopybTS,

  xsymm3m_iucopyrTS,  xsymm3m_ilcopyrTS,

  xsymm3m_iucopyiTS,  xsymm3m_ilcopyiTS,

  xsymm3m_oucopybTS,  xsymm3m_olcopybTS,

  xsymm3m_oucopyrTS,  xsymm3m_olcopyrTS,

  xsymm3m_oucopyiTS,  xsymm3m_olcopyiTS,

  xhemm3m_iucopybTS,  xhemm3m_ilcopybTS,

  xhemm3m_iucopyrTS,  xhemm3m_ilcopyrTS,

  xhemm3m_iucopyiTS,  xhemm3m_ilcopyiTS, 

  xhemm3m_oucopybTS,  xhemm3m_olcopybTS,

  xhemm3m_oucopyrTS,  xhemm3m_olcopyrTS,

  xhemm3m_oucopyiTS,  xhemm3m_olcopyiTS,

#ifndef NO_LAPACK

  xneg_tcopyTS, xlaswp_ncopyTS,

#else

  NULL, NULL,

#endif

#endif

  init_parameter,

  SNUMOPT, DNUMOPT, QNUMOPT,

};

#ifdef ARCH_X86

static int get_l2_size_old(void){

  int i, eax, ebx, ecx, edx, cpuid_level;

  int info[15];

  cpuid(2, &eax, &ebx, &ecx, &edx);

  info[ 0] = BITMASK(eax,  8, 0xff);

  info[ 1] = BITMASK(eax, 16, 0xff);

  info[ 2] = BITMASK(eax, 24, 0xff);

  info[ 3] = BITMASK(ebx,  0, 0xff);

  info[ 4] = BITMASK(ebx,  8, 0xff);

  info[ 5] = BITMASK(ebx, 16, 0xff);

  info[ 6] = BITMASK(ebx, 24, 0xff);

  info[ 7] = BITMASK(ecx,  0, 0xff);

  info[ 8] = BITMASK(ecx,  8, 0xff);

  info[ 9] = BITMASK(ecx, 16, 0xff);

  info[10] = BITMASK(ecx, 24, 0xff);

  info[11] = BITMASK(edx,  0, 0xff);

  info[12] = BITMASK(edx,  8, 0xff);

  info[13] = BITMASK(edx, 16, 0xff);

  info[14] = BITMASK(edx, 24, 0xff);

  for (i = 0; i < 15; i++){

    switch (info[i]){

      /* This table is from http://www.sandpile.org/ia32/cpuid.htm */

    case 0x1a :

      return 96;

    case 0x39 :

    case 0x3b :

    case 0x41 :

    case 0x79 :

    case 0x81 :

      return 128;

    case 0x3a :

      return 192;

    case 0x21 :

    case 0x3c :

    case 0x42 :

    case 0x7a :

    case 0x7e :

    case 0x82 :

      return 256;

    case 0x3d :

      return 384;

    case 0x3e :

    case 0x43 :

    case 0x7b :

    case 0x7f :

    case 0x83 :

    case 0x86 :

      return 512;

    case 0x44 :

    case 0x78 :

    case 0x7c :

    case 0x84 :

    case 0x87 :

      return 1024;

    case 0x45 :

    case 0x7d :

    case 0x85 :

      return 2048;

    case 0x48 :

      return 3184;

    case 0x49 :

      return 4096;

    case 0x4e :

      return 6144;

    }

  }

  return 0;

}

#endif

static __inline__ int get_l2_size(void){

  int eax, ebx, ecx, edx, l2;

  cpuid(0x80000006, &eax, &ebx, &ecx, &edx);

  l2 = BITMASK(ecx, 16, 0xffff);

#ifndef ARCH_X86

  return l2;

#else

  if (l2 > 0) return l2;

  return get_l2_size_old();

#endif

}

static __inline__ int get_l3_size(void){

  int eax, ebx, ecx, edx;

  cpuid(0x80000006, &eax, &ebx, &ecx, &edx);

  return BITMASK(edx, 18, 0x3fff) * 512;

}

static void init_parameter(void) {

  int l2 = get_l2_size();

  TABLE_NAME.sgemm_q = SGEMM_DEFAULT_Q;

  TABLE_NAME.dgemm_q = DGEMM_DEFAULT_Q;

  TABLE_NAME.cgemm_q = CGEMM_DEFAULT_Q;

  TABLE_NAME.zgemm_q = ZGEMM_DEFAULT_Q;

#ifdef EXPRECISION

  TABLE_NAME.qgemm_q = QGEMM_DEFAULT_Q;

  TABLE_NAME.xgemm_q = XGEMM_DEFAULT_Q;

#endif

#if defined(CORE_KATMAI)  || defined(CORE_COPPERMINE) || defined(CORE_BANIAS) || defined(CORE_YONAH)

#ifdef DEBUG

  fprintf(stderr, "Katmai, Coppermine, Banias\n");

#endif

  TABLE_NAME.sgemm_p =  64 * (l2 >> 7);

  TABLE_NAME.dgemm_p =  32 * (l2 >> 7);

  TABLE_NAME.cgemm_p =  32 * (l2 >> 7);

  TABLE_NAME.zgemm_p =  16 * (l2 >> 7);

#ifdef EXPRECISION

  TABLE_NAME.qgemm_p =  16 * (l2 >> 7);

  TABLE_NAME.xgemm_p =   8 * (l2 >> 7);

#endif

#endif

#ifdef CORE_NORTHWOOD

#ifdef DEBUG

  fprintf(stderr, "Northwood\n");

#endif

  TABLE_NAME.sgemm_p =  96 * (l2 >> 7);

  TABLE_NAME.dgemm_p =  48 * (l2 >> 7);

  TABLE_NAME.cgemm_p =  48 * (l2 >> 7);

  TABLE_NAME.zgemm_p =  24 * (l2 >> 7);

#ifdef EXPRECISION

  TABLE_NAME.qgemm_p =  24 * (l2 >> 7);

  TABLE_NAME.xgemm_p =  12 * (l2 >> 7);

#endif

#endif

#ifdef ATOM

#ifdef DEBUG

  fprintf(stderr, "Atom\n");

#endif

  TABLE_NAME.sgemm_p = 256;

  TABLE_NAME.dgemm_p = 128;

  TABLE_NAME.cgemm_p = 128;

  TABLE_NAME.zgemm_p =  64;

#ifdef EXPRECISION

  TABLE_NAME.qgemm_p =  64;

  TABLE_NAME.xgemm_p =  32;

#endif

#endif

#ifdef CORE_PRESCOTT

#ifdef DEBUG

  fprintf(stderr, "Prescott\n");

#endif

  TABLE_NAME.sgemm_p =  56 * (l2 >> 7);

  TABLE_NAME.dgemm_p =  28 * (l2 >> 7);

  TABLE_NAME.cgemm_p =  28 * (l2 >> 7);

  TABLE_NAME.zgemm_p =  14 * (l2 >> 7);

#ifdef EXPRECISION

  TABLE_NAME.qgemm_p =  14 * (l2 >> 7);

  TABLE_NAME.xgemm_p =   7 * (l2 >> 7);

#endif

#endif

#ifdef CORE2

#ifdef DEBUG

  fprintf(stderr, "Core2\n");

#endif

  TABLE_NAME.sgemm_p =  92 * (l2 >> 9);

  TABLE_NAME.dgemm_p =  46 * (l2 >> 9);

  TABLE_NAME.cgemm_p =  46 * (l2 >> 9);

  TABLE_NAME.zgemm_p =  23 * (l2 >> 9);

#ifdef EXPRECISION

  TABLE_NAME.qgemm_p =  92 * (l2 >> 9);

  TABLE_NAME.xgemm_p =  46 * (l2 >> 9);

#endif

#endif

#ifdef PENRYN

#ifdef DEBUG

  fprintf(stderr, "Penryn\n");

#endif

  TABLE_NAME.sgemm_p =  42 * (l2 >> 9) + 8;

  TABLE_NAME.dgemm_p =  42 * (l2 >> 9) + 8;

  TABLE_NAME.cgemm_p =  21 * (l2 >> 9) + 4;

  TABLE_NAME.zgemm_p =  21 * (l2 >> 9) + 4;

#ifdef EXPRECISION

  TABLE_NAME.qgemm_p =  42 * (l2 >> 9) + 8;

  TABLE_NAME.xgemm_p =  21 * (l2 >> 9) + 4;

#endif

#endif

#ifdef NEHALEM

#ifdef DEBUG

  fprintf(stderr, "Nehalem\n");

#endif

  TABLE_NAME.sgemm_p = SGEMM_DEFAULT_P;

  TABLE_NAME.dgemm_p = DGEMM_DEFAULT_P;

  TABLE_NAME.cgemm_p = CGEMM_DEFAULT_P;

  TABLE_NAME.zgemm_p = ZGEMM_DEFAULT_P;

#ifdef EXPRECISION

  TABLE_NAME.qgemm_p = QGEMM_DEFAULT_P;

  TABLE_NAME.xgemm_p = XGEMM_DEFAULT_P;

#endif

#endif

#ifdef OPTERON

#ifdef DEBUG

  fprintf(stderr, "Opteron\n");

#endif

  TABLE_NAME.sgemm_p = 224 +  56 * (l2 >> 7);

  TABLE_NAME.dgemm_p = 112 +  28 * (l2 >> 7);

  TABLE_NAME.cgemm_p = 112 +  28 * (l2 >> 7);

  TABLE_NAME.zgemm_p =  56 +  14 * (l2 >> 7);

#ifdef EXPRECISION

  TABLE_NAME.qgemm_p =  56 +  14 * (l2 >> 7);

  TABLE_NAME.xgemm_p =  28 +   7 * (l2 >> 7);

#endif

#endif

#ifdef BARCELONA

#ifdef DEBUG

  fprintf(stderr, "Barcelona\n");

#endif

  TABLE_NAME.sgemm_p = SGEMM_DEFAULT_P;

  TABLE_NAME.dgemm_p = DGEMM_DEFAULT_P;

  TABLE_NAME.cgemm_p = CGEMM_DEFAULT_P;

  TABLE_NAME.zgemm_p = ZGEMM_DEFAULT_P;

#ifdef EXPRECISION

  TABLE_NAME.qgemm_p = QGEMM_DEFAULT_P;

  TABLE_NAME.xgemm_p = XGEMM_DEFAULT_P;

#endif

#endif

#ifdef NANO

#ifdef DEBUG

  fprintf(stderr, "NANO\n");

#endif

  TABLE_NAME.sgemm_p = SGEMM_DEFAULT_P;

  TABLE_NAME.dgemm_p = DGEMM_DEFAULT_P;

  TABLE_NAME.cgemm_p = CGEMM_DEFAULT_P;

  TABLE_NAME.zgemm_p = ZGEMM_DEFAULT_P;

#ifdef EXPRECISION

  TABLE_NAME.qgemm_p = QGEMM_DEFAULT_P;

  TABLE_NAME.xgemm_p = XGEMM_DEFAULT_P;

#endif

#endif

  TABLE_NAME.sgemm_p = (TABLE_NAME.sgemm_p + SGEMM_DEFAULT_UNROLL_M - 1) & ~(SGEMM_DEFAULT_UNROLL_M - 1);

  TABLE_NAME.dgemm_p = (TABLE_NAME.dgemm_p + DGEMM_DEFAULT_UNROLL_M - 1) & ~(DGEMM_DEFAULT_UNROLL_M - 1);

  TABLE_NAME.cgemm_p = (TABLE_NAME.cgemm_p + CGEMM_DEFAULT_UNROLL_M - 1) & ~(CGEMM_DEFAULT_UNROLL_M - 1);

  TABLE_NAME.zgemm_p = (TABLE_NAME.zgemm_p + ZGEMM_DEFAULT_UNROLL_M - 1) & ~(ZGEMM_DEFAULT_UNROLL_M - 1);

#ifdef QUAD_PRECISION

  TABLE_NAME.qgemm_p = (TABLE_NAME.qgemm_p + QGEMM_DEFAULT_UNROLL_M - 1) & ~(QGEMM_DEFAULT_UNROLL_M - 1);

  TABLE_NAME.xgemm_p = (TABLE_NAME.xgemm_p + XGEMM_DEFAULT_UNROLL_M - 1) & ~(XGEMM_DEFAULT_UNROLL_M - 1);

#endif

#ifdef DEBUG

  fprintf(stderr, "L2 = %8d DGEMM_P  .. %d\n", l2, TABLE_NAME.dgemm_p);

#endif

  TABLE_NAME.sgemm_r = (((BUFFER_SIZE - 

			       ((TABLE_NAME.sgemm_p * TABLE_NAME.sgemm_q *  4 + TABLE_NAME.offsetA 

				 + TABLE_NAME.align) & ~TABLE_NAME.align)

			       ) / (TABLE_NAME.sgemm_q *  4) - 15) & ~15);

  TABLE_NAME.dgemm_r = (((BUFFER_SIZE - 

			       ((TABLE_NAME.dgemm_p * TABLE_NAME.dgemm_q *  8 + TABLE_NAME.offsetA 

				 + TABLE_NAME.align) & ~TABLE_NAME.align)

			       ) / (TABLE_NAME.dgemm_q *  8) - 15) & ~15);

#ifdef EXPRECISION

  TABLE_NAME.qgemm_r = (((BUFFER_SIZE - 

			       ((TABLE_NAME.qgemm_p * TABLE_NAME.qgemm_q * 16 + TABLE_NAME.offsetA 

				 + TABLE_NAME.align) & ~TABLE_NAME.align)

			       ) / (TABLE_NAME.qgemm_q * 16) - 15) & ~15);

#endif

  TABLE_NAME.cgemm_r = (((BUFFER_SIZE - 

			       ((TABLE_NAME.cgemm_p * TABLE_NAME.cgemm_q *  8 + TABLE_NAME.offsetA 

				 + TABLE_NAME.align) & ~TABLE_NAME.align)

			       ) / (TABLE_NAME.cgemm_q *  8) - 15) & ~15);

  TABLE_NAME.zgemm_r = (((BUFFER_SIZE - 

			       ((TABLE_NAME.zgemm_p * TABLE_NAME.zgemm_q * 16 + TABLE_NAME.offsetA 

				 + TABLE_NAME.align) & ~TABLE_NAME.align)

			       ) / (TABLE_NAME.zgemm_q * 16) - 15) & ~15);

#ifdef EXPRECISION

  TABLE_NAME.xgemm_r = (((BUFFER_SIZE - 

			       ((TABLE_NAME.xgemm_p * TABLE_NAME.xgemm_q * 32 + TABLE_NAME.offsetA 

				 + TABLE_NAME.align) & ~TABLE_NAME.align)	

		       ) / (TABLE_NAME.xgemm_q * 32) - 15) & ~15);

#endif

}