Blame thirdparty/openblas/xianyi-OpenBLAS-e6e87a2/driver/level3/level3_thread.c

kusano 2b45e8
/*********************************************************************/
kusano 2b45e8
/* Copyright 2009, 2010 The University of Texas at Austin.           */
kusano 2b45e8
/* All rights reserved.                                              */
kusano 2b45e8
/*                                                                   */
kusano 2b45e8
/* Redistribution and use in source and binary forms, with or        */
kusano 2b45e8
/* without modification, are permitted provided that the following   */
kusano 2b45e8
/* conditions are met:                                               */
kusano 2b45e8
/*                                                                   */
kusano 2b45e8
/*   1. Redistributions of source code must retain the above         */
kusano 2b45e8
/*      copyright notice, this list of conditions and the following  */
kusano 2b45e8
/*      disclaimer.                                                  */
kusano 2b45e8
/*                                                                   */
kusano 2b45e8
/*   2. Redistributions in binary form must reproduce the above      */
kusano 2b45e8
/*      copyright notice, this list of conditions and the following  */
kusano 2b45e8
/*      disclaimer in the documentation and/or other materials       */
kusano 2b45e8
/*      provided with the distribution.                              */
kusano 2b45e8
/*                                                                   */
kusano 2b45e8
/*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */
kusano 2b45e8
/*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */
kusano 2b45e8
/*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */
kusano 2b45e8
/*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */
kusano 2b45e8
/*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */
kusano 2b45e8
/*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */
kusano 2b45e8
/*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */
kusano 2b45e8
/*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */
kusano 2b45e8
/*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */
kusano 2b45e8
/*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */
kusano 2b45e8
/*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */
kusano 2b45e8
/*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */
kusano 2b45e8
/*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */
kusano 2b45e8
/*    POSSIBILITY OF SUCH DAMAGE.                                    */
kusano 2b45e8
/*                                                                   */
kusano 2b45e8
/* The views and conclusions contained in the software and           */
kusano 2b45e8
/* documentation are those of the authors and should not be          */
kusano 2b45e8
/* interpreted as representing official policies, either expressed   */
kusano 2b45e8
/* or implied, of The University of Texas at Austin.                 */
kusano 2b45e8
/*********************************************************************/
kusano 2b45e8
kusano 2b45e8
#ifndef CACHE_LINE_SIZE
kusano 2b45e8
#define CACHE_LINE_SIZE 8
kusano 2b45e8
#endif
kusano 2b45e8
kusano 2b45e8
#ifndef DIVIDE_RATE
kusano 2b45e8
#define DIVIDE_RATE 2
kusano 2b45e8
#endif
kusano 2b45e8
kusano 2b45e8
#ifndef SWITCH_RATIO
kusano 2b45e8
#define SWITCH_RATIO 2
kusano 2b45e8
#endif
kusano 2b45e8
kusano 2b45e8
#ifndef GEMM_LOCAL
kusano 2b45e8
#if   defined(NN)
kusano 2b45e8
#define GEMM_LOCAL    GEMM_NN
kusano 2b45e8
#elif defined(NT)
kusano 2b45e8
#define GEMM_LOCAL    GEMM_NT
kusano 2b45e8
#elif defined(NR)
kusano 2b45e8
#define GEMM_LOCAL    GEMM_NR
kusano 2b45e8
#elif defined(NC)
kusano 2b45e8
#define GEMM_LOCAL    GEMM_NC
kusano 2b45e8
#elif defined(TN)
kusano 2b45e8
#define GEMM_LOCAL    GEMM_TN
kusano 2b45e8
#elif defined(TT)
kusano 2b45e8
#define GEMM_LOCAL    GEMM_TT
kusano 2b45e8
#elif defined(TR)
kusano 2b45e8
#define GEMM_LOCAL    GEMM_TR
kusano 2b45e8
#elif defined(TC)
kusano 2b45e8
#define GEMM_LOCAL    GEMM_TC
kusano 2b45e8
#elif defined(RN)
kusano 2b45e8
#define GEMM_LOCAL    GEMM_RN
kusano 2b45e8
#elif defined(RT)
kusano 2b45e8
#define GEMM_LOCAL    GEMM_RT
kusano 2b45e8
#elif defined(RR)
kusano 2b45e8
#define GEMM_LOCAL    GEMM_RR
kusano 2b45e8
#elif defined(RC)
kusano 2b45e8
#define GEMM_LOCAL    GEMM_RC
kusano 2b45e8
#elif defined(CN)
kusano 2b45e8
#define GEMM_LOCAL    GEMM_CN
kusano 2b45e8
#elif defined(CT)
kusano 2b45e8
#define GEMM_LOCAL    GEMM_CT
kusano 2b45e8
#elif defined(CR)
kusano 2b45e8
#define GEMM_LOCAL    GEMM_CR
kusano 2b45e8
#elif defined(CC)
kusano 2b45e8
#define GEMM_LOCAL    GEMM_CC
kusano 2b45e8
#endif
kusano 2b45e8
#endif
kusano 2b45e8
kusano 2b45e8
typedef struct {
kusano 2b45e8
  volatile BLASLONG working[MAX_CPU_NUMBER][CACHE_LINE_SIZE * DIVIDE_RATE];
kusano 2b45e8
} job_t;
kusano 2b45e8
kusano 2b45e8
kusano 2b45e8
#ifndef BETA_OPERATION
kusano 2b45e8
#ifndef COMPLEX
kusano 2b45e8
#define BETA_OPERATION(M_FROM, M_TO, N_FROM, N_TO, BETA, C, LDC) \
kusano 2b45e8
	GEMM_BETA((M_TO) - (M_FROM), (N_TO - N_FROM), 0, \
kusano 2b45e8
		  BETA[0], NULL, 0, NULL, 0, \
kusano 2b45e8
		  (FLOAT *)(C) + ((M_FROM) + (N_FROM) * (LDC)) * COMPSIZE, LDC)
kusano 2b45e8
#else
kusano 2b45e8
#define BETA_OPERATION(M_FROM, M_TO, N_FROM, N_TO, BETA, C, LDC) \
kusano 2b45e8
	GEMM_BETA((M_TO) - (M_FROM), (N_TO - N_FROM), 0, \
kusano 2b45e8
		  BETA[0], BETA[1], NULL, 0, NULL, 0, \
kusano 2b45e8
		  (FLOAT *)(C) + ((M_FROM) + (N_FROM) * (LDC)) * COMPSIZE, LDC)
kusano 2b45e8
#endif
kusano 2b45e8
#endif
kusano 2b45e8
kusano 2b45e8
#ifndef ICOPY_OPERATION
kusano 2b45e8
#if defined(NN) || defined(NT) || defined(NC) || defined(NR) || \
kusano 2b45e8
    defined(RN) || defined(RT) || defined(RC) || defined(RR)
kusano 2b45e8
#define ICOPY_OPERATION(M, N, A, LDA, X, Y, BUFFER) GEMM_ITCOPY(M, N, (FLOAT *)(A) + ((Y) + (X) * (LDA)) * COMPSIZE, LDA, BUFFER);
kusano 2b45e8
#else
kusano 2b45e8
#define ICOPY_OPERATION(M, N, A, LDA, X, Y, BUFFER) GEMM_INCOPY(M, N, (FLOAT *)(A) + ((X) + (Y) * (LDA)) * COMPSIZE, LDA, BUFFER);
kusano 2b45e8
#endif
kusano 2b45e8
#endif
kusano 2b45e8
kusano 2b45e8
#ifndef OCOPY_OPERATION
kusano 2b45e8
#if defined(NN) || defined(TN) || defined(CN) || defined(RN) || \
kusano 2b45e8
    defined(NR) || defined(TR) || defined(CR) || defined(RR)
kusano 2b45e8
#define OCOPY_OPERATION(M, N, A, LDA, X, Y, BUFFER) GEMM_ONCOPY(M, N, (FLOAT *)(A) + ((X) + (Y) * (LDA)) * COMPSIZE, LDA, BUFFER);
kusano 2b45e8
#else
kusano 2b45e8
#define OCOPY_OPERATION(M, N, A, LDA, X, Y, BUFFER) GEMM_OTCOPY(M, N, (FLOAT *)(A) + ((Y) + (X) * (LDA)) * COMPSIZE, LDA, BUFFER);
kusano 2b45e8
#endif
kusano 2b45e8
#endif
kusano 2b45e8
kusano 2b45e8
#ifndef KERNEL_FUNC
kusano 2b45e8
#if defined(NN) || defined(NT) || defined(TN) || defined(TT)
kusano 2b45e8
#define KERNEL_FUNC	GEMM_KERNEL_N
kusano 2b45e8
#endif
kusano 2b45e8
#if defined(CN) || defined(CT) || defined(RN) || defined(RT)
kusano 2b45e8
#define KERNEL_FUNC	GEMM_KERNEL_L
kusano 2b45e8
#endif
kusano 2b45e8
#if defined(NC) || defined(TC) || defined(NR) || defined(TR)
kusano 2b45e8
#define KERNEL_FUNC	GEMM_KERNEL_R
kusano 2b45e8
#endif
kusano 2b45e8
#if defined(CC) || defined(CR) || defined(RC) || defined(RR)
kusano 2b45e8
#define KERNEL_FUNC	GEMM_KERNEL_B
kusano 2b45e8
#endif
kusano 2b45e8
#endif
kusano 2b45e8
kusano 2b45e8
#ifndef KERNEL_OPERATION
kusano 2b45e8
#ifndef COMPLEX
kusano 2b45e8
#define KERNEL_OPERATION(M, N, K, ALPHA, SA, SB, C, LDC, X, Y) \
kusano 2b45e8
	KERNEL_FUNC(M, N, K, ALPHA[0], SA, SB, (FLOAT *)(C) + ((X) + (Y) * LDC) * COMPSIZE, LDC)
kusano 2b45e8
#else
kusano 2b45e8
#define KERNEL_OPERATION(M, N, K, ALPHA, SA, SB, C, LDC, X, Y) \
kusano 2b45e8
	KERNEL_FUNC(M, N, K, ALPHA[0], ALPHA[1], SA, SB, (FLOAT *)(C) + ((X) + (Y) * LDC) * COMPSIZE, LDC)
kusano 2b45e8
#endif
kusano 2b45e8
#endif
kusano 2b45e8
kusano 2b45e8
#ifndef FUSED_KERNEL_OPERATION
kusano 2b45e8
#if defined(NN) || defined(TN) || defined(CN) || defined(RN) || \
kusano 2b45e8
    defined(NR) || defined(TR) || defined(CR) || defined(RR)
kusano 2b45e8
#ifndef COMPLEX
kusano 2b45e8
#define FUSED_KERNEL_OPERATION(M, N, K, ALPHA, SA, SB, B, LDB, C, LDC, I, J, L) \
kusano 2b45e8
	FUSED_GEMM_KERNEL_N(M, N, K, ALPHA[0], SA, SB, \
kusano 2b45e8
	(FLOAT *)(B) + ((L) + (J) * LDB) * COMPSIZE, LDB, (FLOAT *)(C) + ((I) + (J) * LDC) * COMPSIZE, LDC)
kusano 2b45e8
#else
kusano 2b45e8
#define FUSED_KERNEL_OPERATION(M, N, K, ALPHA, SA, SB, B, LDB, C, LDC, I, J, L) \
kusano 2b45e8
	FUSED_GEMM_KERNEL_N(M, N, K, ALPHA[0], ALPHA[1], SA, SB, \
kusano 2b45e8
	(FLOAT *)(B) + ((L) + (J) * LDB) * COMPSIZE, LDB, (FLOAT *)(C) + ((I) + (J) * LDC) * COMPSIZE, LDC)
kusano 2b45e8
kusano 2b45e8
#endif
kusano 2b45e8
#else
kusano 2b45e8
#ifndef COMPLEX
kusano 2b45e8
#define FUSED_KERNEL_OPERATION(M, N, K, ALPHA, SA, SB, B, LDB, C, LDC, I, J, L) \
kusano 2b45e8
	FUSED_GEMM_KERNEL_T(M, N, K, ALPHA[0], SA, SB, \
kusano 2b45e8
	(FLOAT *)(B) + ((J) + (L) * LDB) * COMPSIZE, LDB, (FLOAT *)(C) + ((I) + (J) * LDC) * COMPSIZE, LDC)
kusano 2b45e8
#else
kusano 2b45e8
#define FUSED_KERNEL_OPERATION(M, N, K, ALPHA, SA, SB, B, LDB, C, LDC, I, J, L) \
kusano 2b45e8
	FUSED_GEMM_KERNEL_T(M, N, K, ALPHA[0], ALPHA[1], SA, SB, \
kusano 2b45e8
	(FLOAT *)(B) + ((J) + (L) * LDB) * COMPSIZE, LDB, (FLOAT *)(C) + ((I) + (J) * LDC) * COMPSIZE, LDC)
kusano 2b45e8
#endif
kusano 2b45e8
#endif
kusano 2b45e8
#endif
kusano 2b45e8
kusano 2b45e8
#ifndef A
kusano 2b45e8
#define A	args -> a
kusano 2b45e8
#endif
kusano 2b45e8
#ifndef LDA
kusano 2b45e8
#define LDA	args -> lda
kusano 2b45e8
#endif
kusano 2b45e8
#ifndef B
kusano 2b45e8
#define B	args -> b
kusano 2b45e8
#endif
kusano 2b45e8
#ifndef LDB
kusano 2b45e8
#define LDB	args -> ldb
kusano 2b45e8
#endif
kusano 2b45e8
#ifndef C
kusano 2b45e8
#define C	args -> c
kusano 2b45e8
#endif
kusano 2b45e8
#ifndef LDC
kusano 2b45e8
#define LDC	args -> ldc
kusano 2b45e8
#endif
kusano 2b45e8
#ifndef M
kusano 2b45e8
#define M	args -> m
kusano 2b45e8
#endif
kusano 2b45e8
#ifndef N
kusano 2b45e8
#define N	args -> n
kusano 2b45e8
#endif
kusano 2b45e8
#ifndef K
kusano 2b45e8
#define K	args -> k
kusano 2b45e8
#endif
kusano 2b45e8
kusano 2b45e8
#ifdef TIMING
kusano 2b45e8
#define START_RPCC()		rpcc_counter = rpcc()
kusano 2b45e8
#define STOP_RPCC(COUNTER)	COUNTER  += rpcc() - rpcc_counter
kusano 2b45e8
#else
kusano 2b45e8
#define START_RPCC()
kusano 2b45e8
#define STOP_RPCC(COUNTER)
kusano 2b45e8
#endif
kusano 2b45e8
kusano 2b45e8
static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLOAT *sb, BLASLONG mypos){
kusano 2b45e8
kusano 2b45e8
  FLOAT *buffer[DIVIDE_RATE];
kusano 2b45e8
kusano 2b45e8
  BLASLONG k, lda, ldb, ldc;
kusano 2b45e8
  BLASLONG m_from, m_to, n_from, n_to, N_from, N_to;
kusano 2b45e8
kusano 2b45e8
  FLOAT *alpha, *beta;
kusano 2b45e8
  FLOAT *a, *b, *c;
kusano 2b45e8
  job_t *job = (job_t *)args -> common;
kusano 2b45e8
  BLASLONG xxx, bufferside;
kusano 2b45e8
kusano 2b45e8
  BLASLONG ls, min_l, jjs, min_jj;
kusano 2b45e8
  BLASLONG is, min_i, div_n;
kusano 2b45e8
kusano 2b45e8
  BLASLONG i, current;
kusano 2b45e8
  BLASLONG l1stride, l2size;
kusano 2b45e8
kusano 2b45e8
#ifdef TIMING
kusano 2b45e8
  BLASULONG rpcc_counter;
kusano 2b45e8
  BLASULONG copy_A = 0;
kusano 2b45e8
  BLASULONG copy_B = 0;
kusano 2b45e8
  BLASULONG kernel = 0;
kusano 2b45e8
  BLASULONG waiting1 = 0;
kusano 2b45e8
  BLASULONG waiting2 = 0;
kusano 2b45e8
  BLASULONG waiting3 = 0;
kusano 2b45e8
  BLASULONG waiting6[MAX_CPU_NUMBER];
kusano 2b45e8
  BLASULONG ops    = 0;
kusano 2b45e8
kusano 2b45e8
  for (i = 0; i < args -> nthreads; i++) waiting6[i] = 0;
kusano 2b45e8
#endif
kusano 2b45e8
kusano 2b45e8
  k = K;
kusano 2b45e8
kusano 2b45e8
  a = (FLOAT *)A;
kusano 2b45e8
  b = (FLOAT *)B;
kusano 2b45e8
  c = (FLOAT *)C;
kusano 2b45e8
kusano 2b45e8
  lda = LDA;
kusano 2b45e8
  ldb = LDB;
kusano 2b45e8
  ldc = LDC;
kusano 2b45e8
kusano 2b45e8
  alpha = (FLOAT *)args -> alpha;
kusano 2b45e8
  beta  = (FLOAT *)args -> beta;
kusano 2b45e8
kusano 2b45e8
  m_from = 0;
kusano 2b45e8
  m_to   = M;
kusano 2b45e8
kusano 2b45e8
  if (range_m) {
kusano 2b45e8
    m_from = range_m[0];
kusano 2b45e8
    m_to   = range_m[1];
kusano 2b45e8
  }
kusano 2b45e8
kusano 2b45e8
  n_from = 0;
kusano 2b45e8
  n_to   = N;
kusano 2b45e8
kusano 2b45e8
  N_from = 0;
kusano 2b45e8
  N_to   = N;
kusano 2b45e8
kusano 2b45e8
  if (range_n) {
kusano 2b45e8
    n_from = range_n[mypos + 0];
kusano 2b45e8
    n_to   = range_n[mypos + 1];
kusano 2b45e8
kusano 2b45e8
    N_from = range_n[0];
kusano 2b45e8
    N_to   = range_n[args -> nthreads];
kusano 2b45e8
  }
kusano 2b45e8
kusano 2b45e8
  if (beta) {
kusano 2b45e8
#ifndef COMPLEX
kusano 2b45e8
    if (beta[0] != ONE)
kusano 2b45e8
#else
kusano 2b45e8
    if ((beta[0] != ONE) || (beta[1] != ZERO))
kusano 2b45e8
#endif
kusano 2b45e8
      BETA_OPERATION(m_from, m_to, N_from, N_to, beta, c, ldc);
kusano 2b45e8
  }
kusano 2b45e8
kusano 2b45e8
  if ((k == 0) || (alpha == NULL)) return 0;
kusano 2b45e8
kusano 2b45e8
  if ((alpha[0] == ZERO)
kusano 2b45e8
#ifdef COMPLEX
kusano 2b45e8
      && (alpha[1] == ZERO)
kusano 2b45e8
#endif
kusano 2b45e8
      ) return 0;
kusano 2b45e8
kusano 2b45e8
  l2size = GEMM_P * GEMM_Q;
kusano 2b45e8
kusano 2b45e8
#if 0
kusano 2b45e8
  fprintf(stderr, "Thread[%ld]  m_from : %ld m_to : %ld n_from : %ld n_to : %ld N_from : %ld N_to : %ld\n",
kusano 2b45e8
	  mypos, m_from, m_to, n_from, n_to, N_from, N_to);
kusano 2b45e8
kusano 2b45e8
  fprintf(stderr, "GEMM: P = %4ld  Q = %4ld  R = %4ld\n", (BLASLONG)GEMM_P, (BLASLONG)GEMM_Q, (BLASLONG)GEMM_R);
kusano 2b45e8
kusano 2b45e8
#endif
kusano 2b45e8
kusano 2b45e8
  div_n = (n_to - n_from + DIVIDE_RATE - 1) / DIVIDE_RATE;
kusano 2b45e8
  
kusano 2b45e8
  buffer[0] = sb;
kusano 2b45e8
  for (i = 1; i < DIVIDE_RATE; i++) {
kusano 2b45e8
    buffer[i] = buffer[i - 1] + GEMM_Q * ((div_n + GEMM_UNROLL_N - 1) & ~(GEMM_UNROLL_N - 1)) * COMPSIZE;
kusano 2b45e8
  }
kusano 2b45e8
  
kusano 2b45e8
kusano 2b45e8
  for(ls = 0; ls < k; ls += min_l){
kusano 2b45e8
kusano 2b45e8
    min_l = k - ls;
kusano 2b45e8
kusano 2b45e8
    if (min_l >= GEMM_Q * 2) {
kusano 2b45e8
      min_l  = GEMM_Q;
kusano 2b45e8
    } else {
kusano 2b45e8
      if (min_l > GEMM_Q) min_l = (min_l + 1) / 2;
kusano 2b45e8
    }
kusano 2b45e8
kusano 2b45e8
    l1stride = 1;
kusano 2b45e8
    min_i = m_to - m_from;
kusano 2b45e8
    
kusano 2b45e8
    if (min_i >= GEMM_P * 2) {
kusano 2b45e8
      min_i = GEMM_P;
kusano 2b45e8
    } else {
kusano 2b45e8
      if (min_i > GEMM_P) {
kusano 2b45e8
	min_i = (min_i / 2 + GEMM_UNROLL_M - 1) & ~(GEMM_UNROLL_M - 1);
kusano 2b45e8
      } else {
kusano 2b45e8
	if (args -> nthreads == 1) l1stride = 0;
kusano 2b45e8
      }
kusano 2b45e8
    }
kusano 2b45e8
kusano 2b45e8
    START_RPCC();
kusano 2b45e8
    
kusano 2b45e8
    ICOPY_OPERATION(min_l, min_i, a, lda, ls, m_from, sa);
kusano 2b45e8
    
kusano 2b45e8
    STOP_RPCC(copy_A);
kusano 2b45e8
    
kusano 2b45e8
    div_n = (n_to - n_from + DIVIDE_RATE - 1) / DIVIDE_RATE;
kusano 2b45e8
    
kusano 2b45e8
    for (xxx = n_from, bufferside = 0; xxx < n_to; xxx += div_n, bufferside ++) {
kusano 2b45e8
      
kusano 2b45e8
      START_RPCC();
kusano 2b45e8
      
kusano 2b45e8
      /* Make sure if no one is using buffer */
kusano 2b45e8
      for (i = 0; i < args -> nthreads; i++)
kusano 2b45e8
	while (job[mypos].working[i][CACHE_LINE_SIZE * bufferside]) {YIELDING;};
kusano 2b45e8
      
kusano 2b45e8
      STOP_RPCC(waiting1);
kusano 2b45e8
      
kusano 2b45e8
#if defined(FUSED_GEMM) && !defined(TIMING)
kusano 2b45e8
kusano 2b45e8
      FUSED_KERNEL_OPERATION(min_i, MIN(n_to, xxx + div_n) - xxx, min_l, alpha,
kusano 2b45e8
			     sa, buffer[bufferside], b, ldb, c, ldc, m_from, xxx, ls);
kusano 2b45e8
kusano 2b45e8
#else
kusano 2b45e8
kusano 2b45e8
      for(jjs = xxx; jjs < MIN(n_to, xxx + div_n); jjs += min_jj){
kusano 2b45e8
	min_jj = MIN(n_to, xxx + div_n) - jjs;
kusano 2b45e8
	if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
kusano 2b45e8
	
kusano 2b45e8
	START_RPCC();
kusano 2b45e8
	
kusano 2b45e8
	OCOPY_OPERATION(min_l, min_jj, b, ldb, ls, jjs, 
kusano 2b45e8
			buffer[bufferside] + min_l * (jjs - xxx) * COMPSIZE * l1stride);
kusano 2b45e8
	
kusano 2b45e8
	STOP_RPCC(copy_B);
kusano 2b45e8
	
kusano 2b45e8
	START_RPCC();
kusano 2b45e8
	
kusano 2b45e8
	KERNEL_OPERATION(min_i, min_jj, min_l, alpha,
kusano 2b45e8
			 sa, buffer[bufferside] + min_l * (jjs - xxx) * COMPSIZE * l1stride,
kusano 2b45e8
			 c, ldc, m_from, jjs);
kusano 2b45e8
	  
kusano 2b45e8
	STOP_RPCC(kernel);
kusano 2b45e8
kusano 2b45e8
#ifdef TIMING
kusano 2b45e8
	  ops += 2 * min_i * min_jj * min_l;
kusano 2b45e8
#endif
kusano 2b45e8
kusano 2b45e8
      }
kusano 2b45e8
#endif
kusano 2b45e8
	
kusano 2b45e8
      for (i = 0; i < args -> nthreads; i++) job[mypos].working[i][CACHE_LINE_SIZE * bufferside] = (BLASLONG)buffer[bufferside];
kusano 2b45e8
      WMB;
kusano 2b45e8
    }
kusano 2b45e8
kusano 2b45e8
    current = mypos;
kusano 2b45e8
    
kusano 2b45e8
    do {
kusano 2b45e8
      current ++;
kusano 2b45e8
      if (current >= args -> nthreads) current = 0;
kusano 2b45e8
      
kusano 2b45e8
      div_n = (range_n[current + 1]  - range_n[current] + DIVIDE_RATE - 1) / DIVIDE_RATE;
kusano 2b45e8
      
kusano 2b45e8
      for (xxx = range_n[current], bufferside = 0; xxx < range_n[current + 1]; xxx += div_n, bufferside ++) {
kusano 2b45e8
	
kusano 2b45e8
	if (current != mypos) {
kusano 2b45e8
	  
kusano 2b45e8
	  START_RPCC();
kusano 2b45e8
	  
kusano 2b45e8
	  /* thread has to wait */
kusano 2b45e8
	  while(job[current].working[mypos][CACHE_LINE_SIZE * bufferside] == 0) {YIELDING;};
kusano 2b45e8
	  
kusano 2b45e8
	  STOP_RPCC(waiting2);
kusano 2b45e8
	    
kusano 2b45e8
	  START_RPCC();
kusano 2b45e8
kusano 2b45e8
	  KERNEL_OPERATION(min_i, MIN(range_n[current + 1]  - xxx,  div_n), min_l, alpha,
kusano 2b45e8
			   sa, (FLOAT *)job[current].working[mypos][CACHE_LINE_SIZE * bufferside],
kusano 2b45e8
			   c, ldc, m_from, xxx);
kusano 2b45e8
kusano 2b45e8
	STOP_RPCC(kernel);
kusano 2b45e8
#ifdef TIMING
kusano 2b45e8
	  ops += 2 * min_i * MIN(range_n[current + 1]  - xxx,  div_n) * min_l;
kusano 2b45e8
#endif
kusano 2b45e8
	}
kusano 2b45e8
	
kusano 2b45e8
	if (m_to - m_from == min_i) {
kusano 2b45e8
	  job[current].working[mypos][CACHE_LINE_SIZE * bufferside] &= 0;
kusano 2b45e8
	}
kusano 2b45e8
      }
kusano 2b45e8
    } while (current != mypos);
kusano 2b45e8
    
kusano 2b45e8
kusano 2b45e8
    for(is = m_from + min_i; is < m_to; is += min_i){
kusano 2b45e8
      min_i = m_to - is;
kusano 2b45e8
kusano 2b45e8
      if (min_i >= GEMM_P * 2) {
kusano 2b45e8
	min_i = GEMM_P;
kusano 2b45e8
      } else 
kusano 2b45e8
	if (min_i > GEMM_P) {
kusano 2b45e8
	  min_i = ((min_i + 1) / 2 + GEMM_UNROLL_M - 1) & ~(GEMM_UNROLL_M - 1);
kusano 2b45e8
	}
kusano 2b45e8
      
kusano 2b45e8
      START_RPCC();
kusano 2b45e8
      
kusano 2b45e8
      ICOPY_OPERATION(min_l, min_i, a, lda, ls, is, sa);
kusano 2b45e8
      
kusano 2b45e8
      STOP_RPCC(copy_A);
kusano 2b45e8
      
kusano 2b45e8
      current = mypos;
kusano 2b45e8
      do {
kusano 2b45e8
	
kusano 2b45e8
	div_n = (range_n[current + 1]  - range_n[current] + DIVIDE_RATE - 1) / DIVIDE_RATE;
kusano 2b45e8
	
kusano 2b45e8
	for (xxx = range_n[current], bufferside = 0; xxx < range_n[current + 1]; xxx += div_n, bufferside ++) {
kusano 2b45e8
	  
kusano 2b45e8
	  START_RPCC();
kusano 2b45e8
	  
kusano 2b45e8
	  KERNEL_OPERATION(min_i, MIN(range_n[current + 1] - xxx, div_n), min_l, alpha,
kusano 2b45e8
			   sa, (FLOAT *)job[current].working[mypos][CACHE_LINE_SIZE * bufferside],
kusano 2b45e8
			   c, ldc, is, xxx);
kusano 2b45e8
	  
kusano 2b45e8
	STOP_RPCC(kernel);
kusano 2b45e8
kusano 2b45e8
#ifdef TIMING
kusano 2b45e8
	ops += 2 * min_i * MIN(range_n[current + 1]  - xxx, div_n) * min_l;
kusano 2b45e8
#endif
kusano 2b45e8
kusano 2b45e8
	if (is + min_i >= m_to) {
kusano 2b45e8
	  /* Thread doesn't need this buffer any more */
kusano 2b45e8
	  job[current].working[mypos][CACHE_LINE_SIZE * bufferside] &= 0;
kusano 2b45e8
	  WMB;
kusano 2b45e8
	}
kusano 2b45e8
	}
kusano 2b45e8
	
kusano 2b45e8
	current ++;
kusano 2b45e8
	if (current >= args -> nthreads) current = 0;
kusano 2b45e8
	
kusano 2b45e8
      } while (current != mypos);
kusano 2b45e8
      
kusano 2b45e8
    }
kusano 2b45e8
    
kusano 2b45e8
  }
kusano 2b45e8
  
kusano 2b45e8
  START_RPCC();
kusano 2b45e8
kusano 2b45e8
  for (i = 0; i < args -> nthreads; i++) {
kusano 2b45e8
    for (xxx = 0; xxx < DIVIDE_RATE; xxx++) {
kusano 2b45e8
      while (job[mypos].working[i][CACHE_LINE_SIZE * xxx] ) {YIELDING;};
kusano 2b45e8
    }
kusano 2b45e8
  }
kusano 2b45e8
kusano 2b45e8
  STOP_RPCC(waiting3);
kusano 2b45e8
kusano 2b45e8
#ifdef TIMING
kusano 2b45e8
  BLASLONG waiting = waiting1 + waiting2 + waiting3;
kusano 2b45e8
  BLASLONG total = copy_A + copy_B + kernel + waiting;
kusano 2b45e8
kusano 2b45e8
  fprintf(stderr, "GEMM   [%2ld] Copy_A : %6.2f  Copy_B : %6.2f  Wait1 : %6.2f Wait2 : %6.2f Wait3 : %6.2f Kernel : %6.2f",
kusano 2b45e8
	  mypos, (double)copy_A /(double)total * 100., (double)copy_B /(double)total * 100.,
kusano 2b45e8
	  (double)waiting1 /(double)total * 100.,
kusano 2b45e8
	  (double)waiting2 /(double)total * 100.,
kusano 2b45e8
	  (double)waiting3 /(double)total * 100.,
kusano 2b45e8
	  (double)ops/(double)kernel / 4. * 100.);
kusano 2b45e8
kusano 2b45e8
#if 0
kusano 2b45e8
  fprintf(stderr, "GEMM   [%2ld] Copy_A : %6.2ld  Copy_B : %6.2ld  Wait : %6.2ld\n",
kusano 2b45e8
	  mypos, copy_A, copy_B, waiting);
kusano 2b45e8
kusano 2b45e8
  fprintf(stderr, "Waiting[%2ld] %6.2f %6.2f %6.2f\n",
kusano 2b45e8
	  mypos,
kusano 2b45e8
	  (double)waiting1/(double)waiting * 100.,
kusano 2b45e8
	  (double)waiting2/(double)waiting * 100.,
kusano 2b45e8
	  (double)waiting3/(double)waiting * 100.);
kusano 2b45e8
#endif
kusano 2b45e8
  fprintf(stderr, "\n");
kusano 2b45e8
#endif
kusano 2b45e8
kusano 2b45e8
  return 0;
kusano 2b45e8
}
kusano 2b45e8
kusano 2b45e8
static int gemm_driver(blas_arg_t *args, BLASLONG *range_m, BLASLONG
kusano 2b45e8
		       *range_n, FLOAT *sa, FLOAT *sb, BLASLONG mypos){
kusano 2b45e8
kusano 2b45e8
  blas_arg_t newarg;
kusano 2b45e8
kusano 2b45e8
  job_t          job[MAX_CPU_NUMBER];
kusano 2b45e8
  blas_queue_t queue[MAX_CPU_NUMBER];
kusano 2b45e8
kusano 2b45e8
  BLASLONG range_M[MAX_CPU_NUMBER + 1];
kusano 2b45e8
  BLASLONG range_N[MAX_CPU_NUMBER + 1];
kusano 2b45e8
kusano 2b45e8
  BLASLONG num_cpu_m, num_cpu_n;
kusano 2b45e8
kusano 2b45e8
  BLASLONG nthreads = args -> nthreads;
kusano 2b45e8
kusano 2b45e8
  BLASLONG width, i, j, k, js;
kusano 2b45e8
  BLASLONG m, n, n_from, n_to;
kusano 2b45e8
  int  mode;
kusano 2b45e8
kusano 2b45e8
#ifndef COMPLEX
kusano 2b45e8
#ifdef XDOUBLE
kusano 2b45e8
  mode  =  BLAS_XDOUBLE | BLAS_REAL | BLAS_NODE;
kusano 2b45e8
#elif defined(DOUBLE)
kusano 2b45e8
  mode  =  BLAS_DOUBLE  | BLAS_REAL | BLAS_NODE;
kusano 2b45e8
#else
kusano 2b45e8
  mode  =  BLAS_SINGLE  | BLAS_REAL | BLAS_NODE;
kusano 2b45e8
#endif  
kusano 2b45e8
#else
kusano 2b45e8
#ifdef XDOUBLE
kusano 2b45e8
  mode  =  BLAS_XDOUBLE | BLAS_COMPLEX | BLAS_NODE;
kusano 2b45e8
#elif defined(DOUBLE)
kusano 2b45e8
  mode  =  BLAS_DOUBLE  | BLAS_COMPLEX | BLAS_NODE;
kusano 2b45e8
#else
kusano 2b45e8
  mode  =  BLAS_SINGLE  | BLAS_COMPLEX | BLAS_NODE;
kusano 2b45e8
#endif  
kusano 2b45e8
#endif
kusano 2b45e8
kusano 2b45e8
  newarg.m        = args -> m;
kusano 2b45e8
  newarg.n        = args -> n;
kusano 2b45e8
  newarg.k        = args -> k;
kusano 2b45e8
  newarg.a        = args -> a;
kusano 2b45e8
  newarg.b        = args -> b;
kusano 2b45e8
  newarg.c        = args -> c;
kusano 2b45e8
  newarg.lda      = args -> lda;
kusano 2b45e8
  newarg.ldb      = args -> ldb;
kusano 2b45e8
  newarg.ldc      = args -> ldc;
kusano 2b45e8
  newarg.alpha    = args -> alpha;
kusano 2b45e8
  newarg.beta     = args -> beta;
kusano 2b45e8
  newarg.nthreads = args -> nthreads;
kusano 2b45e8
  newarg.common   = (void *)job;
kusano 2b45e8
   
kusano 2b45e8
#ifdef PARAMTEST
kusano 2b45e8
  newarg.gemm_p  = args -> gemm_p;
kusano 2b45e8
  newarg.gemm_q  = args -> gemm_q;
kusano 2b45e8
  newarg.gemm_r  = args -> gemm_r;
kusano 2b45e8
#endif
kusano 2b45e8
kusano 2b45e8
  if (!range_m) {
kusano 2b45e8
    range_M[0] = 0;
kusano 2b45e8
    m          = args -> m;
kusano 2b45e8
  } else {
kusano 2b45e8
    range_M[0] = range_m[0];
kusano 2b45e8
    m          = range_m[1] - range_m[0];
kusano 2b45e8
  }
kusano 2b45e8
kusano 2b45e8
  num_cpu_m  = 0;
kusano 2b45e8
kusano 2b45e8
  while (m > 0){
kusano 2b45e8
    
kusano 2b45e8
    width  = blas_quickdivide(m + nthreads - num_cpu_m - 1, nthreads - num_cpu_m);
kusano 2b45e8
kusano 2b45e8
    m -= width;
kusano 2b45e8
    if (m < 0) width = width + m;
kusano 2b45e8
kusano 2b45e8
    range_M[num_cpu_m + 1] = range_M[num_cpu_m] + width;
kusano 2b45e8
kusano 2b45e8
    num_cpu_m ++;
kusano 2b45e8
  }
kusano 2b45e8
kusano 2b45e8
  for (i = 0; i < num_cpu_m; i++) {
kusano 2b45e8
    queue[i].mode    = mode;
kusano 2b45e8
    queue[i].routine = inner_thread;
kusano 2b45e8
    queue[i].args    = &newarg;
kusano 2b45e8
    queue[i].range_m = &range_M[i];
kusano 2b45e8
    queue[i].range_n = &range_N[0];
kusano 2b45e8
    queue[i].sa      = NULL;
kusano 2b45e8
    queue[i].sb      = NULL;
kusano 2b45e8
    queue[i].next    = &queue[i + 1];
kusano 2b45e8
  }
kusano 2b45e8
  
kusano 2b45e8
  queue[0].sa = sa;
kusano 2b45e8
  queue[0].sb = sb;
kusano 2b45e8
  
kusano 2b45e8
  if (!range_n) {
kusano 2b45e8
    n_from = 0;
kusano 2b45e8
    n_to   = args -> n;
kusano 2b45e8
  } else {
kusano 2b45e8
    n_from = range_n[0];
kusano 2b45e8
    n_to   = range_n[1];
kusano 2b45e8
  }
kusano 2b45e8
kusano 2b45e8
  for(js = n_from; js < n_to; js += GEMM_R * nthreads){
kusano 2b45e8
    n = n_to - js;
kusano 2b45e8
    if (n > GEMM_R * nthreads) n = GEMM_R * nthreads;
kusano 2b45e8
    
kusano 2b45e8
    range_N[0] = js;
kusano 2b45e8
kusano 2b45e8
    num_cpu_n  = 0;
kusano 2b45e8
kusano 2b45e8
    while (n > 0){
kusano 2b45e8
      
kusano 2b45e8
      width  = blas_quickdivide(n + nthreads - num_cpu_n - 1, nthreads - num_cpu_n);
kusano 2b45e8
      
kusano 2b45e8
      n -= width;
kusano 2b45e8
      if (n < 0) width = width + n;
kusano 2b45e8
      
kusano 2b45e8
      range_N[num_cpu_n + 1] = range_N[num_cpu_n] + width;
kusano 2b45e8
      
kusano 2b45e8
      num_cpu_n ++;
kusano 2b45e8
    }
kusano 2b45e8
    
kusano 2b45e8
    for (j = 0; j < num_cpu_m; j++) {
kusano 2b45e8
      for (i = 0; i < num_cpu_m; i++) {
kusano 2b45e8
	for (k = 0; k < DIVIDE_RATE; k++) {
kusano 2b45e8
	  job[j].working[i][CACHE_LINE_SIZE * k] = 0;
kusano 2b45e8
	}
kusano 2b45e8
      }
kusano 2b45e8
    }
kusano 2b45e8
    
kusano 2b45e8
    queue[num_cpu_m - 1].next = NULL;
kusano 2b45e8
kusano 2b45e8
    exec_blas(num_cpu_m, queue);
kusano 2b45e8
  }
kusano 2b45e8
kusano 2b45e8
  return 0;
kusano 2b45e8
}
kusano 2b45e8
kusano 2b45e8
int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLOAT *sb, BLASLONG mypos){
kusano 2b45e8
kusano 2b45e8
  BLASLONG m = args -> m;
kusano 2b45e8
  BLASLONG n = args -> n;
kusano 2b45e8
  BLASLONG nthreads = args -> nthreads;
kusano 2b45e8
  BLASLONG divN, divT;
kusano 2b45e8
  int mode;
kusano 2b45e8
  
kusano 2b45e8
  if (nthreads  == 1) {
kusano 2b45e8
    GEMM_LOCAL(args, range_m, range_n, sa, sb, 0); 
kusano 2b45e8
    return 0;
kusano 2b45e8
  }
kusano 2b45e8
kusano 2b45e8
  if (range_m) {
kusano 2b45e8
    BLASLONG m_from = *(((BLASLONG *)range_m) + 0);
kusano 2b45e8
    BLASLONG m_to   = *(((BLASLONG *)range_m) + 1);
kusano 2b45e8
kusano 2b45e8
    m = m_to - m_from;
kusano 2b45e8
  }
kusano 2b45e8
kusano 2b45e8
  if (range_n) {
kusano 2b45e8
    BLASLONG n_from = *(((BLASLONG *)range_n) + 0);
kusano 2b45e8
    BLASLONG n_to   = *(((BLASLONG *)range_n) + 1);
kusano 2b45e8
kusano 2b45e8
    n = n_to - n_from;
kusano 2b45e8
  }
kusano 2b45e8
kusano 2b45e8
  if ((args -> m < nthreads * SWITCH_RATIO) || (args -> n < nthreads * SWITCH_RATIO)) {
kusano 2b45e8
    GEMM_LOCAL(args, range_m, range_n, sa, sb, 0);
kusano 2b45e8
    return 0;
kusano 2b45e8
  }
kusano 2b45e8
kusano 2b45e8
  divT = nthreads;
kusano 2b45e8
  divN = 1;
kusano 2b45e8
kusano 2b45e8
#if 0
kusano 2b45e8
  while ((GEMM_P * divT > m * SWITCH_RATIO) && (divT > 1)) {
kusano 2b45e8
    do {
kusano 2b45e8
      divT --;
kusano 2b45e8
      divN = 1;
kusano 2b45e8
      while (divT * divN < nthreads) divN ++;
kusano 2b45e8
    } while ((divT * divN != nthreads) && (divT > 1));
kusano 2b45e8
  }
kusano 2b45e8
#endif
kusano 2b45e8
kusano 2b45e8
  // fprintf(stderr, "divN = %4ld  divT = %4ld\n", divN, divT);
kusano 2b45e8
kusano 2b45e8
  args -> nthreads = divT;
kusano 2b45e8
kusano 2b45e8
  if (divN == 1){
kusano 2b45e8
kusano 2b45e8
    gemm_driver(args, range_m, range_n, sa, sb, 0);
kusano 2b45e8
  } else {
kusano 2b45e8
#ifndef COMPLEX
kusano 2b45e8
#ifdef XDOUBLE
kusano 2b45e8
    mode  =  BLAS_XDOUBLE | BLAS_REAL;
kusano 2b45e8
#elif defined(DOUBLE)
kusano 2b45e8
    mode  =  BLAS_DOUBLE  | BLAS_REAL;
kusano 2b45e8
#else
kusano 2b45e8
    mode  =  BLAS_SINGLE  | BLAS_REAL;
kusano 2b45e8
#endif  
kusano 2b45e8
#else
kusano 2b45e8
#ifdef XDOUBLE
kusano 2b45e8
    mode  =  BLAS_XDOUBLE | BLAS_COMPLEX;
kusano 2b45e8
#elif defined(DOUBLE)
kusano 2b45e8
    mode  =  BLAS_DOUBLE  | BLAS_COMPLEX;
kusano 2b45e8
#else
kusano 2b45e8
    mode  =  BLAS_SINGLE  | BLAS_COMPLEX;
kusano 2b45e8
#endif  
kusano 2b45e8
#endif
kusano 2b45e8
    
kusano 2b45e8
#if defined(TN) || defined(TT) || defined(TR) || defined(TC) || \
kusano 2b45e8
    defined(CN) || defined(CT) || defined(CR) || defined(CC)
kusano 2b45e8
    mode |= (BLAS_TRANSA_T);
kusano 2b45e8
#endif
kusano 2b45e8
#if defined(NT) || defined(TT) || defined(RT) || defined(CT) || \
kusano 2b45e8
    defined(NC) || defined(TC) || defined(RC) || defined(CC)
kusano 2b45e8
    mode |= (BLAS_TRANSB_T);
kusano 2b45e8
#endif
kusano 2b45e8
    
kusano 2b45e8
#ifdef OS_WINDOWS
kusano 2b45e8
    gemm_thread_n(mode, args, range_m, range_n, GEMM_LOCAL,  sa, sb, divN); 
kusano 2b45e8
#else
kusano 2b45e8
    gemm_thread_n(mode, args, range_m, range_n, gemm_driver, sa, sb, divN); 
kusano 2b45e8
#endif
kusano 2b45e8
kusano 2b45e8
  }
kusano 2b45e8
kusano 2b45e8
  return 0;
kusano 2b45e8
}