|
kusano |
2b45e8 |
/*********************************************************************/
|
|
kusano |
2b45e8 |
/* Copyright 2009, 2010 The University of Texas at Austin. */
|
|
kusano |
2b45e8 |
/* All rights reserved. */
|
|
kusano |
2b45e8 |
/* */
|
|
kusano |
2b45e8 |
/* Redistribution and use in source and binary forms, with or */
|
|
kusano |
2b45e8 |
/* without modification, are permitted provided that the following */
|
|
kusano |
2b45e8 |
/* conditions are met: */
|
|
kusano |
2b45e8 |
/* */
|
|
kusano |
2b45e8 |
/* 1. Redistributions of source code must retain the above */
|
|
kusano |
2b45e8 |
/* copyright notice, this list of conditions and the following */
|
|
kusano |
2b45e8 |
/* disclaimer. */
|
|
kusano |
2b45e8 |
/* */
|
|
kusano |
2b45e8 |
/* 2. Redistributions in binary form must reproduce the above */
|
|
kusano |
2b45e8 |
/* copyright notice, this list of conditions and the following */
|
|
kusano |
2b45e8 |
/* disclaimer in the documentation and/or other materials */
|
|
kusano |
2b45e8 |
/* provided with the distribution. */
|
|
kusano |
2b45e8 |
/* */
|
|
kusano |
2b45e8 |
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
|
|
kusano |
2b45e8 |
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
|
|
kusano |
2b45e8 |
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
|
|
kusano |
2b45e8 |
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
|
|
kusano |
2b45e8 |
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
|
|
kusano |
2b45e8 |
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
|
|
kusano |
2b45e8 |
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
|
|
kusano |
2b45e8 |
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
|
|
kusano |
2b45e8 |
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
|
|
kusano |
2b45e8 |
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
|
|
kusano |
2b45e8 |
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
|
|
kusano |
2b45e8 |
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
|
|
kusano |
2b45e8 |
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
|
|
kusano |
2b45e8 |
/* POSSIBILITY OF SUCH DAMAGE. */
|
|
kusano |
2b45e8 |
/* */
|
|
kusano |
2b45e8 |
/* The views and conclusions contained in the software and */
|
|
kusano |
2b45e8 |
/* documentation are those of the authors and should not be */
|
|
kusano |
2b45e8 |
/* interpreted as representing official policies, either expressed */
|
|
kusano |
2b45e8 |
/* or implied, of The University of Texas at Austin. */
|
|
kusano |
2b45e8 |
/*********************************************************************/
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
#include <stdio.h></stdio.h>
|
|
kusano |
2b45e8 |
#include <stdlib.h></stdlib.h>
|
|
kusano |
2b45e8 |
//#include <sys mman.h=""></sys>
|
|
kusano |
2b45e8 |
#include "common.h"
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
#ifndef USE_OPENMP
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
#include "blas_server.c"
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
#else
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
int blas_server_avail = 0;
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
void goto_set_num_threads(int num_threads) {
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
if (num_threads < 1) num_threads = blas_num_threads;
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
if (num_threads > MAX_CPU_NUMBER) num_threads = MAX_CPU_NUMBER;
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
if (num_threads > blas_num_threads) {
|
|
kusano |
2b45e8 |
blas_num_threads = num_threads;
|
|
kusano |
2b45e8 |
}
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
blas_cpu_number = num_threads;
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
omp_set_num_threads(blas_cpu_number);
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
#if defined(ARCH_MIPS64)
|
|
kusano |
2b45e8 |
//set parameters for different number of threads.
|
|
kusano |
2b45e8 |
blas_set_parameter();
|
|
kusano |
2b45e8 |
#endif
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
}
|
|
kusano |
2b45e8 |
void openblas_set_num_threads(int num_threads) {
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
goto_set_num_threads(num_threads);
|
|
kusano |
2b45e8 |
}
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
int blas_thread_init(void){
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
blas_get_cpu_number();
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
blas_server_avail = 1;
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
return 0;
|
|
kusano |
2b45e8 |
}
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
int BLASFUNC(blas_thread_shutdown)(void){
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
blas_server_avail = 0;
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
return 0;
|
|
kusano |
2b45e8 |
}
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
static void legacy_exec(void *func, int mode, blas_arg_t *args, void *sb){
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
if (!(mode & BLAS_COMPLEX)){
|
|
kusano |
2b45e8 |
#ifdef EXPRECISION
|
|
kusano |
2b45e8 |
if (mode & BLAS_XDOUBLE){
|
|
kusano |
2b45e8 |
/* REAL / Extended Double */
|
|
kusano |
2b45e8 |
void (*afunc)(BLASLONG, BLASLONG, BLASLONG, xdouble,
|
|
kusano |
2b45e8 |
xdouble *, BLASLONG, xdouble *, BLASLONG,
|
|
kusano |
2b45e8 |
xdouble *, BLASLONG, void *) = func;
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
afunc(args -> m, args -> n, args -> k,
|
|
kusano |
2b45e8 |
((xdouble *)args -> alpha)[0],
|
|
kusano |
2b45e8 |
args -> a, args -> lda,
|
|
kusano |
2b45e8 |
args -> b, args -> ldb,
|
|
kusano |
2b45e8 |
args -> c, args -> ldc, sb);
|
|
kusano |
2b45e8 |
} else
|
|
kusano |
2b45e8 |
#endif
|
|
kusano |
2b45e8 |
if (mode & BLAS_DOUBLE){
|
|
kusano |
2b45e8 |
/* REAL / Double */
|
|
kusano |
2b45e8 |
void (*afunc)(BLASLONG, BLASLONG, BLASLONG, double,
|
|
kusano |
2b45e8 |
double *, BLASLONG, double *, BLASLONG,
|
|
kusano |
2b45e8 |
double *, BLASLONG, void *) = func;
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
afunc(args -> m, args -> n, args -> k,
|
|
kusano |
2b45e8 |
((double *)args -> alpha)[0],
|
|
kusano |
2b45e8 |
args -> a, args -> lda,
|
|
kusano |
2b45e8 |
args -> b, args -> ldb,
|
|
kusano |
2b45e8 |
args -> c, args -> ldc, sb);
|
|
kusano |
2b45e8 |
} else {
|
|
kusano |
2b45e8 |
/* REAL / Single */
|
|
kusano |
2b45e8 |
void (*afunc)(BLASLONG, BLASLONG, BLASLONG, float,
|
|
kusano |
2b45e8 |
float *, BLASLONG, float *, BLASLONG,
|
|
kusano |
2b45e8 |
float *, BLASLONG, void *) = func;
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
afunc(args -> m, args -> n, args -> k,
|
|
kusano |
2b45e8 |
((float *)args -> alpha)[0],
|
|
kusano |
2b45e8 |
args -> a, args -> lda,
|
|
kusano |
2b45e8 |
args -> b, args -> ldb,
|
|
kusano |
2b45e8 |
args -> c, args -> ldc, sb);
|
|
kusano |
2b45e8 |
}
|
|
kusano |
2b45e8 |
} else {
|
|
kusano |
2b45e8 |
#ifdef EXPRECISION
|
|
kusano |
2b45e8 |
if (mode & BLAS_XDOUBLE){
|
|
kusano |
2b45e8 |
/* COMPLEX / Extended Double */
|
|
kusano |
2b45e8 |
void (*afunc)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble,
|
|
kusano |
2b45e8 |
xdouble *, BLASLONG, xdouble *, BLASLONG,
|
|
kusano |
2b45e8 |
xdouble *, BLASLONG, void *) = func;
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
afunc(args -> m, args -> n, args -> k,
|
|
kusano |
2b45e8 |
((xdouble *)args -> alpha)[0],
|
|
kusano |
2b45e8 |
((xdouble *)args -> alpha)[1],
|
|
kusano |
2b45e8 |
args -> a, args -> lda,
|
|
kusano |
2b45e8 |
args -> b, args -> ldb,
|
|
kusano |
2b45e8 |
args -> c, args -> ldc, sb);
|
|
kusano |
2b45e8 |
} else
|
|
kusano |
2b45e8 |
#endif
|
|
kusano |
2b45e8 |
if (mode & BLAS_DOUBLE){
|
|
kusano |
2b45e8 |
/* COMPLEX / Double */
|
|
kusano |
2b45e8 |
void (*afunc)(BLASLONG, BLASLONG, BLASLONG, double, double,
|
|
kusano |
2b45e8 |
double *, BLASLONG, double *, BLASLONG,
|
|
kusano |
2b45e8 |
double *, BLASLONG, void *) = func;
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
afunc(args -> m, args -> n, args -> k,
|
|
kusano |
2b45e8 |
((double *)args -> alpha)[0],
|
|
kusano |
2b45e8 |
((double *)args -> alpha)[1],
|
|
kusano |
2b45e8 |
args -> a, args -> lda,
|
|
kusano |
2b45e8 |
args -> b, args -> ldb,
|
|
kusano |
2b45e8 |
args -> c, args -> ldc, sb);
|
|
kusano |
2b45e8 |
} else {
|
|
kusano |
2b45e8 |
/* COMPLEX / Single */
|
|
kusano |
2b45e8 |
void (*afunc)(BLASLONG, BLASLONG, BLASLONG, float, float,
|
|
kusano |
2b45e8 |
float *, BLASLONG, float *, BLASLONG,
|
|
kusano |
2b45e8 |
float *, BLASLONG, void *) = func;
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
afunc(args -> m, args -> n, args -> k,
|
|
kusano |
2b45e8 |
((float *)args -> alpha)[0],
|
|
kusano |
2b45e8 |
((float *)args -> alpha)[1],
|
|
kusano |
2b45e8 |
args -> a, args -> lda,
|
|
kusano |
2b45e8 |
args -> b, args -> ldb,
|
|
kusano |
2b45e8 |
args -> c, args -> ldc, sb);
|
|
kusano |
2b45e8 |
}
|
|
kusano |
2b45e8 |
}
|
|
kusano |
2b45e8 |
}
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
static void exec_threads(blas_queue_t *queue){
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
void *buffer, *sa, *sb;
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
buffer = NULL;
|
|
kusano |
2b45e8 |
sa = queue -> sa;
|
|
kusano |
2b45e8 |
sb = queue -> sb;
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
#ifdef CONSISTENT_FPCSR
|
|
kusano |
2b45e8 |
__asm__ __volatile__ ("ldmxcsr %0" : : "m" (queue -> sse_mode));
|
|
kusano |
2b45e8 |
__asm__ __volatile__ ("fldcw %0" : : "m" (queue -> x87_mode));
|
|
kusano |
2b45e8 |
#endif
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
if ((sa == NULL) && (sb == NULL) && ((queue -> mode & BLAS_PTHREAD) == 0)) {
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
buffer = blas_memory_alloc(2);
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
if (sa == NULL) sa = (void *)((BLASLONG)buffer + GEMM_OFFSET_A);
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
if (sb == NULL) {
|
|
kusano |
2b45e8 |
if (!(queue -> mode & BLAS_COMPLEX)){
|
|
kusano |
2b45e8 |
#ifdef EXPRECISION
|
|
kusano |
2b45e8 |
if (queue -> mode & BLAS_XDOUBLE){
|
|
kusano |
2b45e8 |
sb = (void *)(((BLASLONG)sa + ((QGEMM_P * QGEMM_Q * sizeof(xdouble)
|
|
kusano |
2b45e8 |
+ GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B);
|
|
kusano |
2b45e8 |
} else
|
|
kusano |
2b45e8 |
#endif
|
|
kusano |
2b45e8 |
if (queue -> mode & BLAS_DOUBLE){
|
|
kusano |
2b45e8 |
sb = (void *)(((BLASLONG)sa + ((DGEMM_P * DGEMM_Q * sizeof(double)
|
|
kusano |
2b45e8 |
+ GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B);
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
} else {
|
|
kusano |
2b45e8 |
sb = (void *)(((BLASLONG)sa + ((SGEMM_P * SGEMM_Q * sizeof(float)
|
|
kusano |
2b45e8 |
+ GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B);
|
|
kusano |
2b45e8 |
}
|
|
kusano |
2b45e8 |
} else {
|
|
kusano |
2b45e8 |
#ifdef EXPRECISION
|
|
kusano |
2b45e8 |
if (queue -> mode & BLAS_XDOUBLE){
|
|
kusano |
2b45e8 |
sb = (void *)(((BLASLONG)sa + ((XGEMM_P * XGEMM_Q * 2 * sizeof(xdouble)
|
|
kusano |
2b45e8 |
+ GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B);
|
|
kusano |
2b45e8 |
} else
|
|
kusano |
2b45e8 |
#endif
|
|
kusano |
2b45e8 |
if (queue -> mode & BLAS_DOUBLE){
|
|
kusano |
2b45e8 |
sb = (void *)(((BLASLONG)sa + ((ZGEMM_P * ZGEMM_Q * 2 * sizeof(double)
|
|
kusano |
2b45e8 |
+ GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B);
|
|
kusano |
2b45e8 |
} else {
|
|
kusano |
2b45e8 |
sb = (void *)(((BLASLONG)sa + ((CGEMM_P * CGEMM_Q * 2 * sizeof(float)
|
|
kusano |
2b45e8 |
+ GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B);
|
|
kusano |
2b45e8 |
}
|
|
kusano |
2b45e8 |
}
|
|
kusano |
2b45e8 |
}
|
|
kusano |
2b45e8 |
}
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
if (queue -> mode & BLAS_LEGACY) {
|
|
kusano |
2b45e8 |
legacy_exec(queue -> routine, queue -> mode, queue -> args, sb);
|
|
kusano |
2b45e8 |
} else
|
|
kusano |
2b45e8 |
if (queue -> mode & BLAS_PTHREAD) {
|
|
kusano |
2b45e8 |
void (*pthreadcompat)(void *) = queue -> routine;
|
|
kusano |
2b45e8 |
(pthreadcompat)(queue -> args);
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
} else {
|
|
kusano |
2b45e8 |
int (*routine)(blas_arg_t *, void *, void *, void *, void *, BLASLONG) = queue -> routine;
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
(routine)(queue -> args, queue -> range_m, queue -> range_n, sa, sb, queue -> position);
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
}
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
if (buffer != NULL) blas_memory_free(buffer);
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
}
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
int exec_blas(BLASLONG num, blas_queue_t *queue){
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
BLASLONG i;
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
if ((num <= 0) || (queue == NULL)) return 0;
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
#ifdef CONSISTENT_FPCSR
|
|
kusano |
2b45e8 |
for (i = 0; i < num; i ++) {
|
|
kusano |
2b45e8 |
__asm__ __volatile__ ("fnstcw %0" : "=m" (queue[i].x87_mode));
|
|
kusano |
2b45e8 |
__asm__ __volatile__ ("stmxcsr %0" : "=m" (queue[i].sse_mode));
|
|
kusano |
2b45e8 |
}
|
|
kusano |
2b45e8 |
#endif
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
#pragma omp parallel for schedule(static)
|
|
kusano |
2b45e8 |
for (i = 0; i < num; i ++) {
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
#ifndef USE_SIMPLE_THREADED_LEVEL3
|
|
kusano |
2b45e8 |
queue[i].position = i;
|
|
kusano |
2b45e8 |
#endif
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
exec_threads(&queue[i]);
|
|
kusano |
2b45e8 |
}
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
return 0;
|
|
kusano |
2b45e8 |
}
|
|
kusano |
2b45e8 |
|
|
kusano |
2b45e8 |
#endif
|