/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/
#define ASSEMBLER
#include "common.h"
#undef ZERO
#define ALPHA 0
#define FZERO 16
#define M r3
#define N r4
#define K r5
#ifdef linux
#define A r6
#define B r7
#define C r8
#define LDC r9
#define OFFSET r10
#endif
#define TEMP r11
#define KK r14
#define INCM1 r15
#define INCM3 r16
#define INCM5 r17
#define INCM7 r18
#define INC2 r19
#define INC r20
#define INC4 r21
#define I r22
#define J r23
#define AO r24
#define BO r25
#define AO2 r26
#define BO2 r27
#define CO1 r28
#define CO2 r29
#define ZERO r31
#ifndef NEEDPARAM
#define A1 f16
#define A2 f17
#define A3 f18
#define A4 f19
#define A5 f20
#define A6 f21
#define A7 f22
#define A8 f23
#define A9 f24
#define A10 f25
#define B1 f26
#define B2 f27
#define B3 f28
#define B4 f29
#define B5 f30
#define B6 f31
#define AP B6
#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
defined(NR) || defined(NC) || defined(TR) || defined(TC)
#define FXCPMADD fxcpmadd
#define FXCSMADD fxcxnpma
#else
#define FXCPMADD fxcpnsma
#define FXCSMADD fxcxma
#endif
PROLOGUE
PROFCODE
li r0, -16
stfpdux f14, SP, r0
stfpdux f15, SP, r0
stfpdux f16, SP, r0
stfpdux f17, SP, r0
stfpdux f18, SP, r0
stfpdux f19, SP, r0
stfpdux f20, SP, r0
stfpdux f21, SP, r0
stfpdux f22, SP, r0
stfpdux f23, SP, r0
stfpdux f24, SP, r0
stfpdux f25, SP, r0
stfpdux f26, SP, r0
stfpdux f27, SP, r0
stfpdux f28, SP, r0
stfpdux f29, SP, r0
stfpdux f30, SP, r0
stfpdux f31, SP, r0
stwu r31, -4(SP)
stwu r30, -4(SP)
stwu r29, -4(SP)
stwu r28, -4(SP)
stwu r27, -4(SP)
stwu r26, -4(SP)
stwu r25, -4(SP)
stwu r24, -4(SP)
stwu r23, -4(SP)
stwu r22, -4(SP)
stwu r21, -4(SP)
stwu r20, -4(SP)
stwu r19, -4(SP)
stwu r18, -4(SP)
stwu r17, -4(SP)
stwu r16, -4(SP)
stwu r15, -4(SP)
stwu r14, -4(SP)
li r0, 0
stwu r0, -4(SP)
stwu r0, -4(SP)
stfdu f2, -8(SP)
stfdu f1, -8(SP)
slwi LDC, LDC, ZBASE_SHIFT
cmpwi cr0, M, 0
ble .L999
cmpwi cr0, N, 0
ble .L999
cmpwi cr0, K, 0
ble .L999
#if defined(TRMMKERNEL) && !defined(LEFT)
neg KK, OFFSET
#endif
andi. r0, C, 2 * SIZE - 1
bne .L1000
li INC, 1 * SIZE
li INC2, 2 * SIZE
li INC4, 4 * SIZE
li INCM1, -1 * SIZE
li INCM3, -2 * SIZE
li INCM5, -4 * SIZE
li INCM7, -6 * SIZE
addi C, C, - 2 * SIZE
srawi. J, N, 1
ble .L50
.align 4
.L10:
mr CO1, C
add CO2, C, LDC
add C, CO2, LDC
#if defined(TRMMKERNEL) && defined(LEFT)
mr KK, OFFSET
#endif
addi AO, A, -4 * SIZE
li r0, FZERO
lfpsx f0, SP, r0
srawi. I, M, 2
ble .L20
.align 4
.L11:
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
addi AO2, AO, 2 * SIZE
fpmr f4, f0
addi BO, B, - 4 * SIZE
fpmr f8, f0
addi BO2, B, - 2 * SIZE
fpmr f12, f0
#else
slwi TEMP, KK, 2 + ZBASE_SHIFT
slwi r0, KK, 1 + ZBASE_SHIFT
add AO, AO, TEMP
add BO, B, r0
addi AO2, AO, 2 * SIZE
fpmr f4, f0
addi BO, BO, - 4 * SIZE
fpmr f8, f0
addi BO2, BO, 2 * SIZE
fpmr f12, f0
#endif
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub TEMP, K, KK
#elif defined(LEFT)
addi TEMP, KK, 4
#else
addi TEMP, KK, 2
#endif
srawi. r0, TEMP, 2
fpmr f1, f0
mtspr CTR, r0
ble .L14
#else
addi AO2, AO, 2 * SIZE
fpmr f4, f0
addi BO, B, - 4 * SIZE
fpmr f8, f0
addi BO2, B, - 2 * SIZE
fpmr f12, f0
srawi. r0, K, 2
fpmr f1, f0
mtspr CTR, r0
ble .L14
#endif
LFPDUX A1, AO, INC4
fpmr f5, f0
LFPDUX A3, AO, INC4
fpmr f9, f0
LFPDUX B1, BO, INC4
fpmr f13, f0
LFPDUX A5, AO, INC4
fpmr f2, f0
LFPDUX A6, AO, INC4
fpmr f6, f0
LFPDUX B3, BO, INC4
fpmr f10, f0
LFPDUX A7, AO, INC4
fpmr f14, f0
LFPDUX A8, AO, INC4
fpmr f3, f0
LFPDUX B5, BO, INC4
fpmr f7, f0
LFPDUX A9, AO, INC4
fpmr f11, f0
LFPDUX A2, AO2, INC4
fpmr f15, f0
LFPDUX B2, BO2, INC4
bdz- .L13
.align 4
.L12:
## 1 ##
FXCPMADD f0, B1, A1, f0
nop
FXCSMADD f4, B1, A1, f4
nop
FXCPMADD f8, B2, A1, f8
LFPDUX B4, BO2, INC4
FXCSMADD f12, B2, A1, f12
LFPDUX B6, BO, INC4
FXCPMADD f1, B1, A2, f1
nop
FXCSMADD f5, B1, A2, f5
LFPDUX A4, AO2, INC4
FXCPMADD f9, B2, A2, f9
LFPDUX A10, AO, INC4
FXCSMADD f13, B2, A2, f13
nop
FXCPMADD f2, B1, A3, f2
nop
FXCSMADD f6, B1, A3, f6
nop
FXCPMADD f10, B2, A3, f10
nop
FXCSMADD f14, B2, A3, f14
nop
FXCPMADD f3, B1, A4, f3
nop
FXCSMADD f7, B1, A4, f7
LFPDUX A2, AO2, INC4
FXCPMADD f11, B2, A4, f11
LFPDUX A1, AO, INC4
FXCSMADD f15, B2, A4, f15
nop
## 2 ##
FXCPMADD f0, B3, A5, f0
nop
FXCSMADD f4, B3, A5, f4
nop
FXCPMADD f8, B4, A5, f8
LFPDUX B2, BO2, INC4
FXCSMADD f12, B4, A5, f12
LFPDUX B1, BO, INC4
FXCPMADD f1, B3, A2, f1
nop
FXCSMADD f5, B3, A2, f5
LFPDUX A4, AO2, INC4
FXCPMADD f9, B4, A2, f9
LFPDUX A3, AO, INC4
FXCSMADD f13, B4, A2, f13
nop
FXCPMADD f2, B3, A6, f2
nop
FXCSMADD f6, B3, A6, f6
nop
FXCPMADD f10, B4, A6, f10
nop
FXCSMADD f14, B4, A6, f14
nop
FXCPMADD f3, B3, A4, f3
nop
FXCSMADD f7, B3, A4, f7
LFPDUX A2, AO2, INC4
FXCPMADD f11, B4, A4, f11
LFPDUX A5, AO, INC4
FXCSMADD f15, B4, A4, f15
nop
## 3 ##
FXCPMADD f0, B5, A7, f0
nop
FXCSMADD f4, B5, A7, f4
nop
FXCPMADD f8, B2, A7, f8
LFPDUX B4, BO2, INC4
FXCSMADD f12, B2, A7, f12
LFPDUX B3, BO, INC4
FXCPMADD f1, B5, A2, f1
nop
FXCSMADD f5, B5, A2, f5
LFPDUX A4, AO2, INC4
FXCPMADD f9, B2, A2, f9
LFPDUX A6, AO, INC4
FXCSMADD f13, B2, A2, f13
nop
FXCPMADD f2, B5, A8, f2
nop
FXCSMADD f6, B5, A8, f6
nop
FXCPMADD f10, B2, A8, f10
nop
FXCSMADD f14, B2, A8, f14
nop
FXCPMADD f3, B5, A4, f3
nop
FXCSMADD f7, B5, A4, f7
LFPDUX A2, AO2, INC4
FXCPMADD f11, B2, A4, f11
LFPDUX A7, AO, INC4
FXCSMADD f15, B2, A4, f15
nop
## 4 ##
FXCPMADD f0, B6, A9, f0
nop
FXCSMADD f4, B6, A9, f4
nop
FXCPMADD f8, B4, A9, f8
LFPDUX B2, BO2, INC4
FXCSMADD f12, B4, A9, f12
LFPDUX B5, BO, INC4
FXCPMADD f1, B6, A2, f1
nop
FXCSMADD f5, B6, A2, f5
LFPDUX A4, AO2, INC4
FXCPMADD f9, B4, A2, f9
LFPDUX A8, AO, INC4
FXCSMADD f13, B4, A2, f13
nop
FXCPMADD f2, B6, A10, f2
nop
FXCSMADD f6, B6, A10, f6
nop
FXCPMADD f10, B4, A10, f10
nop
FXCSMADD f14, B4, A10, f14
nop
FXCPMADD f3, B6, A4, f3
LFPDUX A2, AO2, INC4
FXCSMADD f7, B6, A4, f7
LFPDUX A9, AO, INC4
FXCPMADD f11, B4, A4, f11
nop
FXCSMADD f15, B4, A4, f15
bdnz+ .L12
.align 4
.L13:
## 1 ##
FXCPMADD f0, B1, A1, f0
nop
FXCSMADD f4, B1, A1, f4
nop
FXCPMADD f8, B2, A1, f8
LFPDUX B4, BO2, INC4
FXCSMADD f12, B2, A1, f12
LFPDUX B6, BO, INC4
FXCPMADD f1, B1, A2, f1
nop
FXCSMADD f5, B1, A2, f5
LFPDUX A4, AO2, INC4
FXCPMADD f9, B2, A2, f9
LFPDUX A10, AO, INC4
FXCSMADD f13, B2, A2, f13
nop
FXCPMADD f2, B1, A3, f2
nop
FXCSMADD f6, B1, A3, f6
nop
FXCPMADD f10, B2, A3, f10
nop
FXCSMADD f14, B2, A3, f14
nop
FXCPMADD f3, B1, A4, f3
nop
FXCSMADD f7, B1, A4, f7
LFPDUX A2, AO2, INC4
FXCPMADD f11, B2, A4, f11
#ifndef TRMMKERNEL
LFPDUX A1, CO1, INC2
#else
nop
#endif
FXCSMADD f15, B2, A4, f15
nop
## 2 ##
FXCPMADD f0, B3, A5, f0
nop
FXCSMADD f4, B3, A5, f4
nop
FXCPMADD f8, B4, A5, f8
LFPDUX B2, BO2, INC4
FXCSMADD f12, B4, A5, f12
#ifndef TRMMKERNEL
LFPDUX B1, CO1, INC2
#else
nop
#endif
FXCPMADD f1, B3, A2, f1
nop
FXCSMADD f5, B3, A2, f5
LFPDUX A4, AO2, INC4
FXCPMADD f9, B4, A2, f9
#ifndef TRMMKERNEL
LFPDUX A3, CO1, INC2
#else
nop
#endif
FXCSMADD f13, B4, A2, f13
nop
FXCPMADD f2, B3, A6, f2
nop
FXCSMADD f6, B3, A6, f6
nop
FXCPMADD f10, B4, A6, f10
nop
FXCSMADD f14, B4, A6, f14
nop
FXCPMADD f3, B3, A4, f3
nop
FXCSMADD f7, B3, A4, f7
LFPDUX A2, AO2, INC4
FXCPMADD f11, B4, A4, f11
#ifndef TRMMKERNEL
LFPDUX A5, CO1, INC2
#else
nop
#endif
FXCSMADD f15, B4, A4, f15
nop
## 3 ##
FXCPMADD f0, B5, A7, f0
nop
FXCSMADD f4, B5, A7, f4
nop
FXCPMADD f8, B2, A7, f8
LFPDUX B4, BO2, INC4
FXCSMADD f12, B2, A7, f12
#ifndef TRMMKERNEL
LFPDUX B3, CO2, INC2
#else
nop
#endif
FXCPMADD f1, B5, A2, f1
nop
FXCSMADD f5, B5, A2, f5
LFPDUX A4, AO2, INC4
FXCPMADD f9, B2, A2, f9
#ifndef TRMMKERNEL
LFPDUX A6, CO2, INC2
#else
nop
#endif
FXCSMADD f13, B2, A2, f13
FXCPMADD f2, B5, A8, f2
nop
FXCSMADD f6, B5, A8, f6
nop
FXCPMADD f10, B2, A8, f10
nop
FXCSMADD f14, B2, A8, f14
nop
FXCPMADD f3, B5, A4, f3
nop
FXCSMADD f7, B5, A4, f7
LFPDUX A2, AO2, INC4
FXCPMADD f11, B2, A4, f11
#ifndef TRMMKERNEL
LFPDUX A7, CO2, INC2
#else
nop
#endif
FXCSMADD f15, B2, A4, f15
nop
## 4 ##
FXCPMADD f0, B6, A9, f0
nop
FXCSMADD f4, B6, A9, f4
nop
FXCPMADD f8, B4, A9, f8
#ifndef TRMMKERNEL
LFPDUX B2, CO2, INC2
#else
nop
#endif
FXCSMADD f12, B4, A9, f12
FXCPMADD f1, B6, A2, f1
nop
FXCSMADD f5, B6, A2, f5
LFPDUX A4, AO2, INC4
FXCPMADD f9, B4, A2, f9
nop
FXCSMADD f13, B4, A2, f13
nop
FXCPMADD f2, B6, A10, f2
FXCSMADD f6, B6, A10, f6
FXCPMADD f10, B4, A10, f10
FXCSMADD f14, B4, A10, f14
FXCPMADD f3, B6, A4, f3
FXCSMADD f7, B6, A4, f7
FXCPMADD f11, B4, A4, f11
FXCSMADD f15, B4, A4, f15
.align 4
.L14:
li r0, ALPHA
lfpdx AP, SP, r0
#ifdef TRMMKERNEL
li r0, FZERO
lfpsx f30, SP, r0
#endif
#if defined(TRMMKERNEL)
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub TEMP, K, KK
#elif defined(LEFT)
addi TEMP, KK, 4
#else
addi TEMP, KK, 2
#endif
andi. r0, TEMP, 3
mtspr CTR, r0
ble+ .L18
cmpwi cr0, TEMP, 3
bgt+ .L15
#else
andi. r0, K, 3
mtspr CTR, r0
ble+ .L18
cmpwi cr0, K, 3
bgt+ .L15
#endif
#ifndef TRMMKERNEL
LFPDUX A1, CO1, INC2
fpmr f5, f0
LFPDUX B1, CO1, INC2
fpmr f9, f0
LFPDUX A3, CO1, INC2
fpmr f13, f0
LFPDUX A5, CO1, INC2
fpmr f2, f0
LFPDUX B3, CO2, INC2
fpmr f6, f0
LFPDUX A6, CO2, INC2
fpmr f10, f0
LFPDUX A7, CO2, INC2
fpmr f14, f0
LFPDUX B2, CO2, INC2
fpmr f3, f0
#else
fpmr f5, f0
fpmr f9, f0
fpmr f13, f0
fpmr f2, f0
fpmr f6, f0
fpmr f10, f0
fpmr f14, f0
fpmr f3, f0
#endif
fpmr f7, f0
fpmr f11, f0
fpmr f15, f0
.align 4
.L15:
LFPDUX A2, AO, INC4
LFPDUX A4, AO2, INC4
LFPDUX A10, BO, INC4
LFPDUX B4, BO2, INC4
bdz- .L17
.align 4
.L16:
FXCPMADD f0, A10, A2, f0
FXCSMADD f4, A10, A2, f4
FXCPMADD f8, B4, A2, f8
FXCSMADD f12, B4, A2, f12
LFPDUX A2, AO, INC4
FXCPMADD f1, A10, A4, f1
FXCSMADD f5, A10, A4, f5
FXCPMADD f9, B4, A4, f9
FXCSMADD f13, B4, A4, f13
LFPDUX A4, AO2, INC4
FXCPMADD f2, A10, A2, f2
FXCSMADD f6, A10, A2, f6
FXCPMADD f10, B4, A2, f10
FXCSMADD f14, B4, A2, f14
LFPDUX A2, AO, INC4
FXCPMADD f3, A10, A4, f3
FXCSMADD f7, A10, A4, f7
LFPDUX A10, BO, INC4
FXCPMADD f11, B4, A4, f11
FXCSMADD f15, B4, A4, f15
LFPDUX A4, AO2, INC4
LFPDUX B4, BO2, INC4
bdnz+ .L16
.align 4
.L17:
FXCPMADD f0, A10, A2, f0
FXCSMADD f4, A10, A2, f4
FXCPMADD f8, B4, A2, f8
FXCSMADD f12, B4, A2, f12
LFPDUX A2, AO, INC4
FXCPMADD f1, A10, A4, f1
FXCSMADD f5, A10, A4, f5
FXCPMADD f9, B4, A4, f9
FXCSMADD f13, B4, A4, f13
LFPDUX A4, AO2, INC4
FXCPMADD f2, A10, A2, f2
FXCSMADD f6, A10, A2, f6
FXCPMADD f10, B4, A2, f10
FXCSMADD f14, B4, A2, f14
FXCPMADD f3, A10, A4, f3
FXCSMADD f7, A10, A4, f7
FXCPMADD f11, B4, A4, f11
FXCSMADD f15, B4, A4, f15
.align 4
.L18:
#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
defined(RN) || defined(RT) || defined(CN) || defined(CT)
fpadd f0, f0, f4
fpadd f8, f8, f12
fpadd f1, f1, f5
fpadd f9, f9, f13
fpadd f2, f2, f6
fpadd f10, f10, f14
fpadd f3, f3, f7
fpadd f11, f11, f15
#else
fpsub f0, f0, f4
fpsub f8, f8, f12
fpsub f1, f1, f5
fpsub f9, f9, f13
fpsub f2, f2, f6
fpsub f10, f10, f14
fpsub f3, f3, f7
fpsub f11, f11, f15
#endif
#ifndef TRMMKERNEL
fxcpmadd A1, f0, AP, A1
fxcpmadd B1, f1, AP, B1
fxcpmadd A3, f2, AP, A3
fxcpmadd A5, f3, AP, A5
fxcxnpma f0, f0, AP, A1
fxcpmadd B3, f8, AP, B3
fxcxnpma f1, f1, AP, B1
fxcpmadd A6, f9, AP, A6
fxcxnpma f2, f2, AP, A3
fxcpmadd A7, f10, AP, A7
fxcxnpma f3, f3, AP, A5
fxcpmadd B2, f11, AP, B2
fxcxnpma f8, f8, AP, B3
STFPDUX f0, CO1, INCM7
fxcxnpma f9, f9, AP, A6
STFPDUX f1, CO1, INC2
fxcxnpma f10, f10, AP, A7
STFPDUX f2, CO1, INC2
fxcxnpma f11, f11, AP, B2
STFPDUX f3, CO1, INC2
STFPDUX f8, CO2, INCM7
STFPDUX f9, CO2, INC2
STFPDUX f10, CO2, INC2
STFPDUX f11, CO2, INC2
#else
fxcpmadd f12, f0, AP, f30
fxcpmadd f13, f1, AP, f30
fxcpmadd f14, f2, AP, f30
fxcpmadd f15, f3, AP, f30
fxcxnpma f0, f0, AP, f12
fxcxnpma f1, f1, AP, f13
fxcxnpma f2, f2, AP, f14
fxcxnpma f3, f3, AP, f15
fxcpmadd f16, f8, AP, f30
fxcpmadd f17, f9, AP, f30
fxcpmadd f18, f10, AP, f30
fxcpmadd f19, f11, AP, f30
fxcxnpma f8, f8, AP, f16
fxcxnpma f9, f9, AP, f17
fxcxnpma f10, f10, AP, f18
fxcxnpma f11, f11, AP, f19
STFPDUX f0, CO1, INC2
STFPDUX f1, CO1, INC2
STFPDUX f2, CO1, INC2
STFPDUX f3, CO1, INC2
STFPDUX f8, CO2, INC2
STFPDUX f9, CO2, INC2
STFPDUX f10, CO2, INC2
STFPDUX f11, CO2, INC2
#endif
#ifdef TRMMKERNEL
#if ( defined(LEFT) && defined(TRANSA)) || \
(!defined(LEFT) && !defined(TRANSA))
sub TEMP, K, KK
#ifdef LEFT
addi TEMP, TEMP, -4
#else
addi TEMP, TEMP, -2
#endif
slwi r0, TEMP, 2 + ZBASE_SHIFT
slwi TEMP, TEMP, 1 + ZBASE_SHIFT
add AO, AO, r0
add BO, BO, TEMP
#endif
#ifdef LEFT
addi KK, KK, 4
#endif
#endif
addic. I, I, -1
li r0, FZERO
lfpsx f0, SP, r0
bgt+ .L11
.align 4
.L20:
andi. I, M, 2
beq .L30
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
addi AO2, AO, 2 * SIZE
fpmr f4, f0
addi BO, B, - 4 * SIZE
fpmr f8, f0
addi BO2, B, - 2 * SIZE
fpmr f12, f0
#else
slwi TEMP, KK, 1 + ZBASE_SHIFT
slwi r0, KK, 1 + ZBASE_SHIFT
add AO, AO, TEMP
add BO, B, r0
addi AO2, AO, 2 * SIZE
fpmr f4, f0
addi BO, BO, - 4 * SIZE
fpmr f8, f0
addi BO2, BO, 2 * SIZE
fpmr f12, f0
#endif
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub TEMP, K, KK
#elif defined(LEFT)
addi TEMP, KK, 2
#else
addi TEMP, KK, 2
#endif
srawi. r0, TEMP, 2
fpmr f1, f0
fpmr f5, f0
fpmr f9, f0
mtspr CTR, r0
fpmr f13, f0
ble .L24
#else
addi AO2, AO, 2 * SIZE
fpmr f4, f0
addi BO, B, - 4 * SIZE
fpmr f8, f0
addi BO2, B, - 2 * SIZE
fpmr f12, f0
srawi. r0, K, 2
fpmr f1, f0
fpmr f5, f0
fpmr f9, f0
mtspr CTR, r0
fpmr f13, f0
ble .L24
#endif
LFPDUX A1, AO, INC4
LFPDUX B1, BO, INC4
LFPDUX A2, AO2, INC4
LFPDUX B2, BO2, INC4
LFPDUX A3, AO, INC4
LFPDUX B3, BO, INC4
LFPDUX A4, AO2, INC4
LFPDUX B4, BO2, INC4
LFPDUX A5, AO, INC4
LFPDUX B5, BO, INC4
LFPDUX A6, AO2, INC4
LFPDUX B6, BO2, INC4
LFPDUX A7, AO, INC4
LFPDUX A9, BO, INC4
LFPDUX A10, BO2, INC4
bdz- .L23
.align 4
.L22:
FXCPMADD f0, B1, A1, f0
nop
FXCSMADD f4, B1, A1, f4
LFPDUX A8, AO2, INC4
FXCPMADD f8, B2, A1, f8
nop
FXCSMADD f12, B2, A1, f12
LFPDUX A1, AO, INC4
FXCPMADD f1, B1, A2, f1
nop
FXCSMADD f5, B1, A2, f5
LFPDUX B1, BO, INC4
FXCPMADD f9, B2, A2, f9
nop
FXCSMADD f13, B2, A2, f13
LFPDUX B2, BO2, INC4
FXCPMADD f0, B3, A3, f0
nop
FXCSMADD f4, B3, A3, f4
LFPDUX A2, AO2, INC4
FXCPMADD f8, B4, A3, f8
nop
FXCSMADD f12, B4, A3, f12
LFPDUX A3, AO, INC4
FXCPMADD f1, B3, A4, f1
nop
FXCSMADD f5, B3, A4, f5
LFPDUX B3, BO, INC4
FXCPMADD f9, B4, A4, f9
nop
FXCSMADD f13, B4, A4, f13
LFPDUX B4, BO2, INC4
FXCPMADD f0, B5, A5, f0
nop
FXCSMADD f4, B5, A5, f4
LFPDUX A4, AO2, INC4
FXCPMADD f8, B6, A5, f8
nop
FXCSMADD f12, B6, A5, f12
LFPDUX A5, AO, INC4
FXCPMADD f1, B5, A6, f1
nop
FXCSMADD f5, B5, A6, f5
LFPDUX B5, BO, INC4
FXCPMADD f9, B6, A6, f9
nop
FXCSMADD f13, B6, A6, f13
LFPDUX B6, BO2, INC4
FXCPMADD f0, A9, A7, f0
nop
FXCSMADD f4, A9, A7, f4
LFPDUX A6, AO2, INC4
FXCPMADD f8, A10, A7, f8
nop
FXCSMADD f12, A10, A7, f12
LFPDUX A7, AO, INC4
FXCPMADD f1, A9, A8, f1
nop
FXCSMADD f5, A9, A8, f5
LFPDUX A9, BO, INC4
FXCPMADD f9, A10, A8, f9
nop
FXCSMADD f13, A10, A8, f13
LFPDUX A10, BO2, INC4
bdnz+ .L22
.align 4
.L23:
FXCPMADD f0, B1, A1, f0
FXCSMADD f4, B1, A1, f4
LFPDUX A8, AO2, INC4
FXCPMADD f8, B2, A1, f8
FXCSMADD f12, B2, A1, f12
FXCPMADD f1, B1, A2, f1
FXCSMADD f5, B1, A2, f5
FXCPMADD f9, B2, A2, f9
FXCSMADD f13, B2, A2, f13
FXCPMADD f0, B3, A3, f0
FXCSMADD f4, B3, A3, f4
FXCPMADD f8, B4, A3, f8
FXCSMADD f12, B4, A3, f12
FXCPMADD f1, B3, A4, f1
FXCSMADD f5, B3, A4, f5
FXCPMADD f9, B4, A4, f9
FXCSMADD f13, B4, A4, f13
FXCPMADD f0, B5, A5, f0
FXCSMADD f4, B5, A5, f4
FXCPMADD f8, B6, A5, f8
FXCSMADD f12, B6, A5, f12
FXCPMADD f1, B5, A6, f1
FXCSMADD f5, B5, A6, f5
FXCPMADD f9, B6, A6, f9
FXCSMADD f13, B6, A6, f13
FXCPMADD f0, A9, A7, f0
FXCSMADD f4, A9, A7, f4
FXCPMADD f8, A10, A7, f8
FXCSMADD f12, A10, A7, f12
FXCPMADD f1, A9, A8, f1
FXCSMADD f5, A9, A8, f5
FXCPMADD f9, A10, A8, f9
FXCSMADD f13, A10, A8, f13
.align 4
.L24:
li r0, ALPHA
lfpdx AP, SP, r0
#ifdef TRMMKERNEL
li r0, FZERO
lfpsx f30, SP, r0
#endif
#if defined(TRMMKERNEL)
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub TEMP, K, KK
#elif defined(LEFT)
addi TEMP, KK, 2
#else
addi TEMP, KK, 2
#endif
andi. r0, TEMP, 3
mtspr CTR, r0
#else
andi. r0, K, 3
mtspr CTR, r0
#endif
ble+ .L28
LFPDUX A1, AO, INC4
LFPDUX A2, AO2, INC4
LFPDUX B1, BO, INC4
LFPDUX B2, BO2, INC4
bdz- .L27
.align 4
.L26:
FXCPMADD f0, B1, A1, f0
FXCSMADD f4, B1, A1, f4
FXCPMADD f8, B2, A1, f8
FXCSMADD f12, B2, A1, f12
LFPDUX A1, AO, INC4
FXCPMADD f1, B1, A2, f1
FXCSMADD f5, B1, A2, f5
LFPDUX B1, BO, INC4
FXCPMADD f9, B2, A2, f9
FXCSMADD f13, B2, A2, f13
LFPDUX A2, AO2, INC4
LFPDUX B2, BO2, INC4
bdnz+ .L26
.align 4
.L27:
FXCPMADD f0, B1, A1, f0
FXCSMADD f4, B1, A1, f4
FXCPMADD f8, B2, A1, f8
FXCSMADD f12, B2, A1, f12
FXCPMADD f1, B1, A2, f1
FXCSMADD f5, B1, A2, f5
FXCPMADD f9, B2, A2, f9
FXCSMADD f13, B2, A2, f13
.align 4
.L28:
#ifndef TRMMKERNEL
LFPDUX A1, CO1, INC2
LFPDUX A2, CO1, INC2
LFPDUX A3, CO2, INC2
LFPDUX A4, CO2, INC2
#endif
#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
defined(RN) || defined(RT) || defined(CN) || defined(CT)
fpadd f0, f0, f4
fpadd f8, f8, f12
fpadd f1, f1, f5
fpadd f9, f9, f13
#else
fpsub f0, f0, f4
fpsub f8, f8, f12
fpsub f1, f1, f5
fpsub f9, f9, f13
#endif
#ifndef TRMMKERNEL
fxcpmadd A1, f0, AP, A1
fxcpmadd A2, f1, AP, A2
fxcpmadd A3, f8, AP, A3
fxcpmadd A4, f9, AP, A4
fxcxnpma f0, f0, AP, A1
fxcxnpma f1, f1, AP, A2
fxcxnpma f8, f8, AP, A3
fxcxnpma f9, f9, AP, A4
STFPDUX f0, CO1, INCM3
STFPDUX f1, CO1, INC2
STFPDUX f8, CO2, INCM3
STFPDUX f9, CO2, INC2
#else
fxcpmadd f12, f0, AP, f30
fxcpmadd f13, f1, AP, f30
fxcpmadd f14, f8, AP, f30
fxcpmadd f15, f9, AP, f30
fxcxnpma f0, f0, AP, f12
fxcxnpma f1, f1, AP, f13
fxcxnpma f8, f8, AP, f14
fxcxnpma f9, f9, AP, f15
STFPDUX f0, CO1, INC2
STFPDUX f1, CO1, INC2
STFPDUX f8, CO2, INC2
STFPDUX f9, CO2, INC2
#endif
#ifdef TRMMKERNEL
#if ( defined(LEFT) && defined(TRANSA)) || \
(!defined(LEFT) && !defined(TRANSA))
sub TEMP, K, KK
#ifdef LEFT
addi TEMP, TEMP, -2
#else
addi TEMP, TEMP, -2
#endif
slwi r0, TEMP, 1 + ZBASE_SHIFT
slwi TEMP, TEMP, 1 + ZBASE_SHIFT
add AO, AO, r0
add BO, BO, TEMP
#endif
#ifdef LEFT
addi KK, KK, 2
#endif
#endif
li r0, FZERO
lfpsx f0, SP, r0
.align 4
.L30:
andi. I, M, 1
beq .L49
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
addi AO2, AO, 2 * SIZE
fpmr f1, f0
addi BO, B, - 4 * SIZE
fpmr f2, f0
addi BO2, B, - 2 * SIZE
fpmr f3, f0
#else
slwi TEMP, KK, 0 + ZBASE_SHIFT
slwi r0, KK, 1 + ZBASE_SHIFT
add AO, AO, TEMP
add BO, B, r0
addi AO2, AO, 2 * SIZE
fpmr f1, f0
addi BO, BO, - 4 * SIZE
fpmr f2, f0
addi BO2, BO, 2 * SIZE
fpmr f3, f0
#endif
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub TEMP, K, KK
#elif defined(LEFT)
addi TEMP, KK, 1
#else
addi TEMP, KK, 2
#endif
srawi. r0, TEMP, 2
mtspr CTR, r0
ble .L34
#else
addi AO2, AO, 2 * SIZE
fpmr f1, f0
addi BO, B, - 4 * SIZE
fpmr f2, f0
addi BO2, B, - 2 * SIZE
fpmr f3, f0
srawi. r0, K, 2
mtspr CTR, r0
ble .L34
#endif
LFPDUX A1, AO, INC4
LFPDUX B1, BO, INC4
LFPDUX B2, BO2, INC4
LFPDUX A2, AO2, INC4
LFPDUX B3, BO, INC4
LFPDUX B4, BO2, INC4
LFPDUX A3, AO, INC4
LFPDUX A5, BO, INC4
LFPDUX A6, BO2, INC4
LFPDUX A4, AO2, INC4
LFPDUX A7, BO, INC4
LFPDUX A8, BO2, INC4
bdz- .L33
.align 4
.L32:
FXCPMADD f0, B1, A1, f0
FXCSMADD f1, B1, A1, f1
LFPDUX B1, BO, INC4
FXCPMADD f2, B2, A1, f2
FXCSMADD f3, B2, A1, f3
LFPDUX B2, BO2, INC4
LFPDUX A1, AO, INC4
FXCPMADD f0, B3, A2, f0
FXCSMADD f1, B3, A2, f1
LFPDUX B3, BO, INC4
FXCPMADD f2, B4, A2, f2
FXCSMADD f3, B4, A2, f3
LFPDUX B4, BO2, INC4
LFPDUX A2, AO2, INC4
FXCPMADD f0, A5, A3, f0
FXCSMADD f1, A5, A3, f1
LFPDUX A5, BO, INC4
FXCPMADD f2, A6, A3, f2
FXCSMADD f3, A6, A3, f3
LFPDUX A6, BO2, INC4
LFPDUX A3, AO, INC4
FXCPMADD f0, A7, A4, f0
FXCSMADD f1, A7, A4, f1
LFPDUX A7, BO, INC4
FXCPMADD f2, A8, A4, f2
FXCSMADD f3, A8, A4, f3
LFPDUX A8, BO2, INC4
LFPDUX A4, AO2, INC4
bdnz+ .L32
.align 4
.L33:
FXCPMADD f0, B1, A1, f0
FXCSMADD f1, B1, A1, f1
FXCPMADD f2, B2, A1, f2
FXCSMADD f3, B2, A1, f3
FXCPMADD f0, B3, A2, f0
FXCSMADD f1, B3, A2, f1
FXCPMADD f2, B4, A2, f2
FXCSMADD f3, B4, A2, f3
FXCPMADD f0, A5, A3, f0
FXCSMADD f1, A5, A3, f1
FXCPMADD f2, A6, A3, f2
FXCSMADD f3, A6, A3, f3
FXCPMADD f0, A7, A4, f0
FXCSMADD f1, A7, A4, f1
FXCPMADD f2, A8, A4, f2
FXCSMADD f3, A8, A4, f3
.align 4
.L34:
li r0, ALPHA
lfpdx AP, SP, r0
#ifdef TRMMKERNEL
li r0, FZERO
lfpsx f30, SP, r0
#endif
#if defined(TRMMKERNEL)
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub TEMP, K, KK
#elif defined(LEFT)
addi TEMP, KK, 1
#else
addi TEMP, KK, 2
#endif
andi. r0, TEMP, 3
mtspr CTR, r0
#else
andi. r0, K, 3
mtspr CTR, r0
#endif
ble+ .L38
LFPDX A1, AO, INC4
LFPDUX B1, BO, INC4
LFPDUX B2, BO2, INC4
add AO, AO, INC2
bdz- .L37
.align 4
.L36:
FXCPMADD f0, B1, A1, f0
FXCSMADD f1, B1, A1, f1
LFPDUX B1, BO, INC4
FXCPMADD f2, B2, A1, f2
FXCSMADD f3, B2, A1, f3
LFPDX A1, AO, INC4
LFPDUX B2, BO2, INC4
add AO, AO, INC2
bdnz+ .L36
.align 4
.L37:
FXCPMADD f0, B1, A1, f0
FXCSMADD f1, B1, A1, f1
FXCPMADD f2, B2, A1, f2
FXCSMADD f3, B2, A1, f3
.align 4
.L38:
#ifndef TRMMKERNEL
LFPDX A1, CO1, INC2
LFPDX A2, CO2, INC2
#endif
#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
defined(RN) || defined(RT) || defined(CN) || defined(CT)
fpadd f0, f0, f1
fpadd f2, f2, f3
#else
fpsub f0, f0, f1
fpsub f2, f2, f3
#endif
#ifndef TRMMKERNEL
fxcpmadd A1, f0, AP, A1
fxcpmadd A2, f2, AP, A2
fxcxnpma f0, f0, AP, A1
fxcxnpma f2, f2, AP, A2
#else
fxcpmadd f12, f0, AP, f30
fxcpmadd f13, f2, AP, f30
fxcxnpma f0, f0, AP, f12
fxcxnpma f2, f2, AP, f13
#endif
STFPDUX f0, CO1, INC2
STFPDUX f2, CO2, INC2
#ifdef TRMMKERNEL
#if ( defined(LEFT) && defined(TRANSA)) || \
(!defined(LEFT) && !defined(TRANSA))
sub TEMP, K, KK
#ifdef LEFT
addi TEMP, TEMP, -1
#else
addi TEMP, TEMP, -2
#endif
slwi r0, TEMP, 0 + ZBASE_SHIFT
slwi TEMP, TEMP, 1 + ZBASE_SHIFT
add AO, AO, r0
add BO, BO, TEMP
#endif
#ifdef LEFT
addi KK, KK, 1
#endif
#endif
li r0, FZERO
lfpsx f0, SP, r0
.align 4
.L49:
#if defined(TRMMKERNEL) && !defined(LEFT)
addi KK, KK, 2
#endif
addi B, BO, 4 * SIZE
addic. J, J, -1
bgt+ .L10
.align 4
.L50:
andi. J, N, 1
beq .L999
mr CO1, C
#if defined(TRMMKERNEL) && defined(LEFT)
mr KK, OFFSET
#endif
addi AO, A, -2 * SIZE
li r0, FZERO
lfpsx f0, SP, r0
srawi. I, M, 2
ble .L60
.align 4
.L51:
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
fpmr f4, f0
addi BO, B, - 2 * SIZE
fpmr f1, f0
fpmr f5, f0
fpmr f2, f0
fpmr f6, f0
#else
slwi TEMP, KK, 2 + ZBASE_SHIFT
slwi r0, KK, 0 + ZBASE_SHIFT
add AO, AO, TEMP
add BO, B, r0
fpmr f4, f0
addi BO, BO, - 2 * SIZE
fpmr f1, f0
fpmr f5, f0
fpmr f2, f0
fpmr f6, f0
#endif
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub TEMP, K, KK
#elif defined(LEFT)
addi TEMP, KK, 4
#else
addi TEMP, KK, 1
#endif
srawi. r0, TEMP, 2
fpmr f3, f0
mtspr CTR, r0
fpmr f7, f0
ble .L54
#else
srawi. r0, K, 2
fpmr f4, f0
addi BO, B, - 2 * SIZE
fpmr f1, f0
fpmr f5, f0
fpmr f2, f0
fpmr f6, f0
fpmr f3, f0
mtspr CTR, r0
fpmr f7, f0
ble .L54
#endif
LFPDUX B1, BO, INC2
LFPDUX A1, AO, INC2
LFPDUX A2, AO, INC2
LFPDUX B2, BO, INC2
LFPDUX A3, AO, INC2
LFPDUX A4, AO, INC2
LFPDUX B3, BO, INC2
LFPDUX A5, AO, INC2
LFPDUX A6, AO, INC2
LFPDUX A7, AO, INC2
LFPDUX A8, AO, INC2
bdz- .L53
.align 4
.L52:
FXCPMADD f0, B1, A1, f0
LFPDUX B4, BO, INC2
FXCSMADD f4, B1, A1, f4
LFPDUX A1, AO, INC2
FXCPMADD f1, B1, A2, f1
nop
FXCSMADD f5, B1, A2, f5
LFPDUX A2, AO, INC2
FXCPMADD f2, B1, A3, f2
nop
FXCSMADD f6, B1, A3, f6
LFPDUX A3, AO, INC2
FXCPMADD f3, B1, A4, f3
nop
FXCSMADD f7, B1, A4, f7
LFPDUX A4, AO, INC2
FXCPMADD f0, B2, A5, f0
LFPDUX B1, BO, INC2
FXCSMADD f4, B2, A5, f4
LFPDUX A5, AO, INC2
FXCPMADD f1, B2, A6, f1
nop
FXCSMADD f5, B2, A6, f5
LFPDUX A6, AO, INC2
FXCPMADD f2, B2, A7, f2
nop
FXCSMADD f6, B2, A7, f6
LFPDUX A7, AO, INC2
FXCPMADD f3, B2, A8, f3
nop
FXCSMADD f7, B2, A8, f7
LFPDUX A8, AO, INC2
FXCPMADD f0, B3, A1, f0
LFPDUX B2, BO, INC2
FXCSMADD f4, B3, A1, f4
LFPDUX A1, AO, INC2
FXCPMADD f1, B3, A2, f1
nop
FXCSMADD f5, B3, A2, f5
LFPDUX A2, AO, INC2
FXCPMADD f2, B3, A3, f2
nop
FXCSMADD f6, B3, A3, f6
LFPDUX A3, AO, INC2
FXCPMADD f3, B3, A4, f3
nop
FXCSMADD f7, B3, A4, f7
LFPDUX A4, AO, INC2
FXCPMADD f0, B4, A5, f0
LFPDUX B3, BO, INC2
FXCSMADD f4, B4, A5, f4
LFPDUX A5, AO, INC2
FXCPMADD f1, B4, A6, f1
nop
FXCSMADD f5, B4, A6, f5
LFPDUX A6, AO, INC2
FXCPMADD f2, B4, A7, f2
nop
FXCSMADD f6, B4, A7, f6
LFPDUX A7, AO, INC2
FXCPMADD f3, B4, A8, f3
nop
FXCSMADD f7, B4, A8, f7
LFPDUX A8, AO, INC2
bdnz+ .L52
.align 4
.L53:
FXCPMADD f0, B1, A1, f0
LFPDUX B4, BO, INC2
FXCSMADD f4, B1, A1, f4
LFPDUX A1, AO, INC2
FXCPMADD f1, B1, A2, f1
nop
FXCSMADD f5, B1, A2, f5
LFPDUX A2, AO, INC2
FXCPMADD f2, B1, A3, f2
nop
FXCSMADD f6, B1, A3, f6
LFPDUX A3, AO, INC2
FXCPMADD f3, B1, A4, f3
nop
FXCSMADD f7, B1, A4, f7
LFPDUX A4, AO, INC2
FXCPMADD f0, B2, A5, f0
nop
FXCSMADD f4, B2, A5, f4
LFPDUX A5, AO, INC2
FXCPMADD f1, B2, A6, f1
nop
FXCSMADD f5, B2, A6, f5
LFPDUX A6, AO, INC2
FXCPMADD f2, B2, A7, f2
nop
FXCSMADD f6, B2, A7, f6
LFPDUX A7, AO, INC2
FXCPMADD f3, B2, A8, f3
nop
FXCSMADD f7, B2, A8, f7
LFPDUX A8, AO, INC2
FXCPMADD f0, B3, A1, f0
FXCSMADD f4, B3, A1, f4
FXCPMADD f1, B3, A2, f1
FXCSMADD f5, B3, A2, f5
FXCPMADD f2, B3, A3, f2
FXCSMADD f6, B3, A3, f6
FXCPMADD f3, B3, A4, f3
FXCSMADD f7, B3, A4, f7
FXCPMADD f0, B4, A5, f0
FXCSMADD f4, B4, A5, f4
FXCPMADD f1, B4, A6, f1
FXCSMADD f5, B4, A6, f5
FXCPMADD f2, B4, A7, f2
FXCSMADD f6, B4, A7, f6
FXCPMADD f3, B4, A8, f3
FXCSMADD f7, B4, A8, f7
.align 4
.L54:
li r0, ALPHA
lfpdx AP, SP, r0
#ifdef TRMMKERNEL
li r0, FZERO
lfpsx f30, SP, r0
#endif
#if defined(TRMMKERNEL)
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub TEMP, K, KK
#elif defined(LEFT)
addi TEMP, KK, 4
#else
addi TEMP, KK, 1
#endif
andi. r0, TEMP, 3
mtspr CTR, r0
#else
andi. r0, K, 3
mtspr CTR, r0
#endif
ble+ .L58
LFPDUX A1, AO, INC2
LFPDUX B1, BO, INC2
LFPDUX A2, AO, INC2
LFPDUX A3, AO, INC2
LFPDUX A4, AO, INC2
bdz- .L57
.align 4
.L56:
FXCPMADD f0, B1, A1, f0
FXCSMADD f4, B1, A1, f4
LFPDUX A1, AO, INC2
FXCPMADD f1, B1, A2, f1
FXCSMADD f5, B1, A2, f5
LFPDUX A2, AO, INC2
FXCPMADD f2, B1, A3, f2
FXCSMADD f6, B1, A3, f6
LFPDUX A3, AO, INC2
FXCPMADD f3, B1, A4, f3
FXCSMADD f7, B1, A4, f7
LFPDUX A4, AO, INC2
LFPDUX B1, BO, INC2
bdnz+ .L56
.align 4
.L57:
FXCPMADD f0, B1, A1, f0
FXCSMADD f4, B1, A1, f4
FXCPMADD f1, B1, A2, f1
FXCSMADD f5, B1, A2, f5
FXCPMADD f2, B1, A3, f2
FXCSMADD f6, B1, A3, f6
FXCPMADD f3, B1, A4, f3
FXCSMADD f7, B1, A4, f7
.align 4
.L58:
#ifndef TRMMKERNEL
LFPDUX A1, CO1, INC2
LFPDUX A2, CO1, INC2
LFPDUX A3, CO1, INC2
LFPDUX A4, CO1, INC2
#endif
#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
defined(RN) || defined(RT) || defined(CN) || defined(CT)
fpadd f0, f0, f4
fpadd f1, f1, f5
fpadd f2, f2, f6
fpadd f3, f3, f7
#else
fpsub f0, f0, f4
fpsub f1, f1, f5
fpsub f2, f2, f6
fpsub f3, f3, f7
#endif
#ifndef TRMMKERNEL
fxcpmadd A1, f0, AP, A1
fxcpmadd A2, f1, AP, A2
fxcpmadd A3, f2, AP, A3
fxcpmadd A4, f3, AP, A4
fxcxnpma f0, f0, AP, A1
fxcxnpma f1, f1, AP, A2
fxcxnpma f2, f2, AP, A3
fxcxnpma f3, f3, AP, A4
STFPDUX f0, CO1, INCM7
STFPDUX f1, CO1, INC2
STFPDUX f2, CO1, INC2
STFPDUX f3, CO1, INC2
#else
fxcpmadd f12, f0, AP, f30
fxcpmadd f13, f1, AP, f30
fxcpmadd f14, f2, AP, f30
fxcpmadd f15, f3, AP, f30
fxcxnpma f0, f0, AP, f12
fxcxnpma f1, f1, AP, f13
fxcxnpma f2, f2, AP, f14
fxcxnpma f3, f3, AP, f15
STFPDUX f0, CO1, INC2
STFPDUX f1, CO1, INC2
STFPDUX f2, CO1, INC2
STFPDUX f3, CO1, INC2
#endif
#ifdef TRMMKERNEL
#if ( defined(LEFT) && defined(TRANSA)) || \
(!defined(LEFT) && !defined(TRANSA))
sub TEMP, K, KK
#ifdef LEFT
addi TEMP, TEMP, -4
#else
addi TEMP, TEMP, -1
#endif
slwi r0, TEMP, 2 + ZBASE_SHIFT
slwi TEMP, TEMP, 0 + ZBASE_SHIFT
add AO, AO, r0
add BO, BO, TEMP
#endif
#ifdef LEFT
addi KK, KK, 4
#endif
#endif
addic. I, I, -1
li r0, FZERO
lfpsx f0, SP, r0
bgt+ .L51
.align 4
.L60:
andi. I, M, 2
beq .L70
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
addi BO, B, - 2 * SIZE
fpmr f1, f0
#else
slwi TEMP, KK, 1 + ZBASE_SHIFT
slwi r0, KK, 0 + ZBASE_SHIFT
add AO, AO, TEMP
add BO, B, r0
addi BO, BO, - 2 * SIZE
fpmr f1, f0
#endif
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub TEMP, K, KK
#elif defined(LEFT)
addi TEMP, KK, 2
#else
addi TEMP, KK, 1
#endif
srawi. r0, TEMP, 2
fpmr f2, f0
mtspr CTR, r0
fpmr f3, f0
ble .L64
#else
srawi. r0, K, 2
fpmr f1, f0
addi BO, B, - 2 * SIZE
fpmr f2, f0
mtspr CTR, r0
fpmr f3, f0
ble .L64
#endif
LFPDUX B1, BO, INC2
LFPDUX A1, AO, INC2
LFPDUX A2, AO, INC2
LFPDUX B2, BO, INC2
LFPDUX A3, AO, INC2
LFPDUX A4, AO, INC2
LFPDUX B3, BO, INC2
LFPDUX A5, AO, INC2
LFPDUX A6, AO, INC2
LFPDUX B4, BO, INC2
LFPDUX A7, AO, INC2
LFPDUX A8, AO, INC2
bdz- .L63
.align 4
.L62:
FXCPMADD f0, B1, A1, f0
FXCSMADD f2, B1, A1, f2
LFPDUX A1, AO, INC2
FXCPMADD f1, B1, A2, f1
FXCSMADD f3, B1, A2, f3
LFPDUX A2, AO, INC2
LFPDUX B1, BO, INC2
FXCPMADD f0, B2, A3, f0
FXCSMADD f2, B2, A3, f2
LFPDUX A3, AO, INC2
FXCPMADD f1, B2, A4, f1
FXCSMADD f3, B2, A4, f3
LFPDUX A4, AO, INC2
LFPDUX B2, BO, INC2
FXCPMADD f0, B3, A5, f0
FXCSMADD f2, B3, A5, f2
LFPDUX A5, AO, INC2
FXCPMADD f1, B3, A6, f1
FXCSMADD f3, B3, A6, f3
LFPDUX A6, AO, INC2
LFPDUX B3, BO, INC2
FXCPMADD f0, B4, A7, f0
FXCSMADD f2, B4, A7, f2
LFPDUX A7, AO, INC2
FXCPMADD f1, B4, A8, f1
FXCSMADD f3, B4, A8, f3
LFPDUX A8, AO, INC2
LFPDUX B4, BO, INC2
bdnz+ .L62
.align 4
.L63:
FXCPMADD f0, B1, A1, f0
FXCSMADD f2, B1, A1, f2
FXCPMADD f1, B1, A2, f1
FXCSMADD f3, B1, A2, f3
FXCPMADD f0, B2, A3, f0
FXCSMADD f2, B2, A3, f2
FXCPMADD f1, B2, A4, f1
FXCSMADD f3, B2, A4, f3
FXCPMADD f0, B3, A5, f0
FXCSMADD f2, B3, A5, f2
FXCPMADD f1, B3, A6, f1
FXCSMADD f3, B3, A6, f3
FXCPMADD f0, B4, A7, f0
FXCSMADD f2, B4, A7, f2
FXCPMADD f1, B4, A8, f1
FXCSMADD f3, B4, A8, f3
.align 4
.L64:
li r0, ALPHA
lfpdx AP, SP, r0
#ifdef TRMMKERNEL
li r0, FZERO
lfpsx f30, SP, r0
#endif
#if defined(TRMMKERNEL)
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub TEMP, K, KK
#elif defined(LEFT)
addi TEMP, KK, 2
#else
addi TEMP, KK, 1
#endif
andi. r0, TEMP, 3
mtspr CTR, r0
#else
andi. r0, K, 3
mtspr CTR, r0
#endif
ble+ .L68
LFPDUX A1, AO, INC2
LFPDUX B1, BO, INC2
LFPDUX A2, AO, INC2
bdz- .L67
.align 4
.L66:
FXCPMADD f0, B1, A1, f0
FXCSMADD f2, B1, A1, f2
LFPDUX A1, AO, INC2
FXCPMADD f1, B1, A2, f1
FXCSMADD f3, B1, A2, f3
LFPDUX B1, BO, INC2
LFPDUX A2, AO, INC2
bdnz+ .L66
.align 4
.L67:
FXCPMADD f0, B1, A1, f0
FXCSMADD f2, B1, A1, f2
FXCPMADD f1, B1, A2, f1
FXCSMADD f3, B1, A2, f3
.align 4
.L68:
#ifndef TRMMKERNEL
LFPDUX A1, CO1, INC2
LFPDUX A2, CO1, INC2
#endif
#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
defined(RN) || defined(RT) || defined(CN) || defined(CT)
fpadd f0, f0, f2
fpadd f1, f1, f3
#else
fpsub f0, f0, f2
fpsub f1, f1, f3
#endif
#ifndef TRMMKERNEL
fxcpmadd A1, f0, AP, A1
fxcpmadd A2, f1, AP, A2
fxcxnpma f0, f0, AP, A1
fxcxnpma f1, f1, AP, A2
STFPDUX f0, CO1, INCM3
STFPDUX f1, CO1, INC2
#else
fxcpmadd f12, f0, AP, f30
fxcpmadd f13, f1, AP, f30
fxcxnpma f0, f0, AP, f12
fxcxnpma f1, f1, AP, f13
STFPDUX f0, CO1, INC2
STFPDUX f1, CO1, INC2
#endif
#ifdef TRMMKERNEL
#if ( defined(LEFT) && defined(TRANSA)) || \
(!defined(LEFT) && !defined(TRANSA))
sub TEMP, K, KK
#ifdef LEFT
addi TEMP, TEMP, -2
#else
addi TEMP, TEMP, -1
#endif
slwi r0, TEMP, 1 + ZBASE_SHIFT
slwi TEMP, TEMP, 0 + ZBASE_SHIFT
add AO, AO, r0
add BO, BO, TEMP
#endif
#ifdef LEFT
addi KK, KK, 2
#endif
#endif
li r0, FZERO
lfpsx f0, SP, r0
.align 4
.L70:
andi. I, M, 1
beq .L89
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
addi BO, B, - 2 * SIZE
fpmr f1, f0
#else
slwi TEMP, KK, 0 + ZBASE_SHIFT
slwi r0, KK, 0 + ZBASE_SHIFT
add AO, AO, TEMP
add BO, B, r0
addi BO, BO, - 2 * SIZE
fpmr f1, f0
#endif
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub TEMP, K, KK
#elif defined(LEFT)
addi TEMP, KK, 1
#else
addi TEMP, KK, 1
#endif
srawi. r0, TEMP, 3
fpmr f2, f0
mtspr CTR, r0
fpmr f3, f0
ble .L74
#else
addi BO, B, - 2 * SIZE
fpmr f1, f0
srawi. r0, K, 3
fpmr f2, f0
mtspr CTR, r0
fpmr f3, f0
ble .L74
#endif
LFPDUX A1, AO, INC2
LFPDUX B1, BO, INC2
LFPDUX A2, AO, INC2
LFPDUX B2, BO, INC2
LFPDUX A3, AO, INC2
LFPDUX B3, BO, INC2
LFPDUX A4, AO, INC2
LFPDUX B4, BO, INC2
LFPDUX A5, AO, INC2
LFPDUX B5, BO, INC2
LFPDUX A6, AO, INC2
LFPDUX B6, BO, INC2
LFPDUX A7, AO, INC2
LFPDUX A9, BO, INC2
LFPDUX A8, AO, INC2
LFPDUX A10, BO, INC2
bdz- .L73
.align 4
.L72:
FXCPMADD f0, B1, A1, f0
FXCSMADD f1, B1, A1, f1
LFPDUX A1, AO, INC2
LFPDUX B1, BO, INC2
FXCPMADD f2, B2, A2, f2
FXCSMADD f3, B2, A2, f3
LFPDUX A2, AO, INC2
LFPDUX B2, BO, INC2
FXCPMADD f0, B3, A3, f0
FXCSMADD f1, B3, A3, f1
LFPDUX A3, AO, INC2
LFPDUX B3, BO, INC2
FXCPMADD f2, B4, A4, f2
FXCSMADD f3, B4, A4, f3
LFPDUX A4, AO, INC2
LFPDUX B4, BO, INC2
FXCPMADD f0, B5, A5, f0
FXCSMADD f1, B5, A5, f1
LFPDUX A5, AO, INC2
LFPDUX B5, BO, INC2
FXCPMADD f2, B6, A6, f2
FXCSMADD f3, B6, A6, f3
LFPDUX A6, AO, INC2
LFPDUX B6, BO, INC2
FXCPMADD f0, A9, A7, f0
FXCSMADD f1, A9, A7, f1
LFPDUX A7, AO, INC2
LFPDUX A9, BO, INC2
FXCPMADD f2, A10, A8, f2
FXCSMADD f3, A10, A8, f3
LFPDUX A8, AO, INC2
LFPDUX A10, BO, INC2
bdnz+ .L72
.align 4
.L73:
FXCPMADD f0, B1, A1, f0
FXCSMADD f1, B1, A1, f1
FXCPMADD f2, B2, A2, f2
FXCSMADD f3, B2, A2, f3
FXCPMADD f0, B3, A3, f0
FXCSMADD f1, B3, A3, f1
FXCPMADD f2, B4, A4, f2
FXCSMADD f3, B4, A4, f3
FXCPMADD f0, B5, A5, f0
FXCSMADD f1, B5, A5, f1
FXCPMADD f2, B6, A6, f2
FXCSMADD f3, B6, A6, f3
FXCPMADD f0, A9, A7, f0
FXCSMADD f1, A9, A7, f1
FXCPMADD f2, A10, A8, f2
FXCSMADD f3, A10, A8, f3
.align 4
.L74:
li r0, ALPHA
lfpdx AP, SP, r0
#ifdef TRMMKERNEL
li r0, FZERO
lfpsx f30, SP, r0
#endif
#if defined(TRMMKERNEL)
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub TEMP, K, KK
#elif defined(LEFT)
addi TEMP, KK, 1
#else
addi TEMP, KK, 1
#endif
andi. r0, TEMP, 7
mtspr CTR, r0
#else
andi. r0, K, 7
mtspr CTR, r0
#endif
ble+ .L78
LFPDUX A1, AO, INC2
LFPDUX B1, BO, INC2
bdz- .L77
.align 4
.L76:
FXCPMADD f0, B1, A1, f0
FXCSMADD f1, B1, A1, f1
LFPDUX A1, AO, INC2
LFPDUX B1, BO, INC2
bdnz+ .L76
.align 4
.L77:
FXCPMADD f0, B1, A1, f0
FXCSMADD f1, B1, A1, f1
.align 4
.L78:
#ifndef TRMMKERNEL
LFPDX A1, CO1, INC2
#endif
fpadd f0, f0, f2
fpadd f1, f1, f3
#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
defined(RN) || defined(RT) || defined(CN) || defined(CT)
fpadd f0, f0, f1
#else
fpsub f0, f0, f1
#endif
#ifndef TRMMKERNEL
fxcpmadd A1, f0, AP, A1
fxcxnpma f0, f0, AP, A1
#else
fxcpmadd f12, f0, AP, f30
fxcxnpma f0, f0, AP, f12
#endif
STFPDUX f0, CO1, INC2
li r0, FZERO
lfpsx f0, SP, r0
.align 4
.L89:
addi B, BO, 2 * SIZE
.align 4
.L999:
addi SP, SP, 20
lwzu r14, 4(SP)
lwzu r15, 4(SP)
lwzu r16, 4(SP)
lwzu r17, 4(SP)
lwzu r18, 4(SP)
lwzu r19, 4(SP)
lwzu r20, 4(SP)
lwzu r21, 4(SP)
lwzu r22, 4(SP)
lwzu r23, 4(SP)
lwzu r24, 4(SP)
lwzu r25, 4(SP)
lwzu r26, 4(SP)
lwzu r27, 4(SP)
lwzu r28, 4(SP)
lwzu r29, 4(SP)
lwzu r30, 4(SP)
lwzu r31, 4(SP)
subi SP, SP, 12
li r0, 16
lfpdux f31, SP, r0
lfpdux f30, SP, r0
lfpdux f29, SP, r0
lfpdux f28, SP, r0
lfpdux f27, SP, r0
lfpdux f26, SP, r0
lfpdux f25, SP, r0
lfpdux f24, SP, r0
lfpdux f23, SP, r0
lfpdux f22, SP, r0
lfpdux f21, SP, r0
lfpdux f20, SP, r0
lfpdux f19, SP, r0
lfpdux f18, SP, r0
lfpdux f17, SP, r0
lfpdux f16, SP, r0
lfpdux f15, SP, r0
lfpdux f14, SP, r0
addi SP, SP, 16
blr
.align 4
.L1000:
li INC, 1 * SIZE
li INC2, 2 * SIZE
li INC4, 4 * SIZE
li INCM1, -1 * SIZE
li INCM3, -3 * SIZE
li INCM5, -5 * SIZE
li INCM7, -7 * SIZE
addi C, C, - 1 * SIZE
srawi. J, N, 1
ble .L1050
.align 4
.L1010:
mr CO1, C
add CO2, C, LDC
add C, CO2, LDC
#if defined(TRMMKERNEL) && defined(LEFT)
mr KK, OFFSET
#endif
addi AO, A, -4 * SIZE
li r0, FZERO
lfpsx f0, SP, r0
srawi. I, M, 2
ble .L1020
.align 4
.L1011:
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
addi AO2, AO, 2 * SIZE
fpmr f4, f0
addi BO, B, - 4 * SIZE
fpmr f8, f0
addi BO2, B, - 2 * SIZE
fpmr f12, f0
#else
slwi TEMP, KK, 2 + ZBASE_SHIFT
slwi r0, KK, 1 + ZBASE_SHIFT
add AO, AO, TEMP
add BO, B, r0
addi AO2, AO, 2 * SIZE
fpmr f4, f0
addi BO, BO, - 4 * SIZE
fpmr f8, f0
addi BO2, BO, 2 * SIZE
fpmr f12, f0
#endif
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub TEMP, K, KK
#elif defined(LEFT)
addi TEMP, KK, 4
#else
addi TEMP, KK, 2
#endif
srawi. r0, TEMP, 2
fpmr f1, f0
mtspr CTR, r0
ble .L1014
#else
addi AO2, AO, 2 * SIZE
fpmr f4, f0
addi BO, B, - 4 * SIZE
fpmr f8, f0
addi BO2, B, - 2 * SIZE
fpmr f12, f0
srawi. r0, K, 2
fpmr f1, f0
mtspr CTR, r0
ble .L1014
#endif
LFPDUX A1, AO, INC4
fpmr f5, f0
LFPDUX A3, AO, INC4
fpmr f9, f0
LFPDUX B1, BO, INC4
fpmr f13, f0
LFPDUX A5, AO, INC4
fpmr f2, f0
LFPDUX A6, AO, INC4
fpmr f6, f0
LFPDUX B3, BO, INC4
fpmr f10, f0
LFPDUX A7, AO, INC4
fpmr f14, f0
LFPDUX A8, AO, INC4
fpmr f3, f0
LFPDUX B5, BO, INC4
fpmr f7, f0
LFPDUX A9, AO, INC4
fpmr f11, f0
LFPDUX A2, AO2, INC4
fpmr f15, f0
LFPDUX B2, BO2, INC4
bdz- .L1013
.align 4
.L1012:
## 1 ##
FXCPMADD f0, B1, A1, f0
nop
FXCSMADD f4, B1, A1, f4
nop
FXCPMADD f8, B2, A1, f8
LFPDUX B4, BO2, INC4
FXCSMADD f12, B2, A1, f12
LFPDUX B6, BO, INC4
FXCPMADD f1, B1, A2, f1
nop
FXCSMADD f5, B1, A2, f5
LFPDUX A4, AO2, INC4
FXCPMADD f9, B2, A2, f9
LFPDUX A10, AO, INC4
FXCSMADD f13, B2, A2, f13
nop
FXCPMADD f2, B1, A3, f2
nop
FXCSMADD f6, B1, A3, f6
nop
FXCPMADD f10, B2, A3, f10
nop
FXCSMADD f14, B2, A3, f14
nop
FXCPMADD f3, B1, A4, f3
nop
FXCSMADD f7, B1, A4, f7
LFPDUX A2, AO2, INC4
FXCPMADD f11, B2, A4, f11
LFPDUX A1, AO, INC4
FXCSMADD f15, B2, A4, f15
nop
## 2 ##
FXCPMADD f0, B3, A5, f0
nop
FXCSMADD f4, B3, A5, f4
nop
FXCPMADD f8, B4, A5, f8
LFPDUX B2, BO2, INC4
FXCSMADD f12, B4, A5, f12
LFPDUX B1, BO, INC4
FXCPMADD f1, B3, A2, f1
nop
FXCSMADD f5, B3, A2, f5
LFPDUX A4, AO2, INC4
FXCPMADD f9, B4, A2, f9
LFPDUX A3, AO, INC4
FXCSMADD f13, B4, A2, f13
nop
FXCPMADD f2, B3, A6, f2
nop
FXCSMADD f6, B3, A6, f6
nop
FXCPMADD f10, B4, A6, f10
nop
FXCSMADD f14, B4, A6, f14
nop
FXCPMADD f3, B3, A4, f3
nop
FXCSMADD f7, B3, A4, f7
LFPDUX A2, AO2, INC4
FXCPMADD f11, B4, A4, f11
LFPDUX A5, AO, INC4
FXCSMADD f15, B4, A4, f15
nop
## 3 ##
FXCPMADD f0, B5, A7, f0
nop
FXCSMADD f4, B5, A7, f4
nop
FXCPMADD f8, B2, A7, f8
LFPDUX B4, BO2, INC4
FXCSMADD f12, B2, A7, f12
LFPDUX B3, BO, INC4
FXCPMADD f1, B5, A2, f1
nop
FXCSMADD f5, B5, A2, f5
LFPDUX A4, AO2, INC4
FXCPMADD f9, B2, A2, f9
LFPDUX A6, AO, INC4
FXCSMADD f13, B2, A2, f13
nop
FXCPMADD f2, B5, A8, f2
nop
FXCSMADD f6, B5, A8, f6
nop
FXCPMADD f10, B2, A8, f10
nop
FXCSMADD f14, B2, A8, f14
nop
FXCPMADD f3, B5, A4, f3
nop
FXCSMADD f7, B5, A4, f7
LFPDUX A2, AO2, INC4
FXCPMADD f11, B2, A4, f11
LFPDUX A7, AO, INC4
FXCSMADD f15, B2, A4, f15
nop
## 4 ##
FXCPMADD f0, B6, A9, f0
nop
FXCSMADD f4, B6, A9, f4
nop
FXCPMADD f8, B4, A9, f8
LFPDUX B2, BO2, INC4
FXCSMADD f12, B4, A9, f12
LFPDUX B5, BO, INC4
FXCPMADD f1, B6, A2, f1
nop
FXCSMADD f5, B6, A2, f5
LFPDUX A4, AO2, INC4
FXCPMADD f9, B4, A2, f9
LFPDUX A8, AO, INC4
FXCSMADD f13, B4, A2, f13
nop
FXCPMADD f2, B6, A10, f2
nop
FXCSMADD f6, B6, A10, f6
nop
FXCPMADD f10, B4, A10, f10
nop
FXCSMADD f14, B4, A10, f14
nop
FXCPMADD f3, B6, A4, f3
LFPDUX A2, AO2, INC4
FXCSMADD f7, B6, A4, f7
LFPDUX A9, AO, INC4
FXCPMADD f11, B4, A4, f11
nop
FXCSMADD f15, B4, A4, f15
bdnz+ .L1012
.align 4
.L1013:
## 1 ##
FXCPMADD f0, B1, A1, f0
nop
FXCSMADD f4, B1, A1, f4
nop
FXCPMADD f8, B2, A1, f8
LFPDUX B4, BO2, INC4
FXCSMADD f12, B2, A1, f12
LFPDUX B6, BO, INC4
FXCPMADD f1, B1, A2, f1
nop
FXCSMADD f5, B1, A2, f5
LFPDUX A4, AO2, INC4
FXCPMADD f9, B2, A2, f9
LFPDUX A10, AO, INC4
FXCSMADD f13, B2, A2, f13
nop
FXCPMADD f2, B1, A3, f2
nop
FXCSMADD f6, B1, A3, f6
nop
FXCPMADD f10, B2, A3, f10
nop
FXCSMADD f14, B2, A3, f14
nop
FXCPMADD f3, B1, A4, f3
nop
FXCSMADD f7, B1, A4, f7
LFPDUX A2, AO2, INC4
FXCPMADD f11, B2, A4, f11
#ifndef TRMMKERNEL
LFDUX A1, CO1, INC
#else
nop
#endif
FXCSMADD f15, B2, A4, f15
nop
## 2 ##
FXCPMADD f0, B3, A5, f0
nop
FXCSMADD f4, B3, A5, f4
nop
FXCPMADD f8, B4, A5, f8
LFPDUX B2, BO2, INC4
FXCSMADD f12, B4, A5, f12
#ifndef TRMMKERNEL
LFDUX B1, CO1, INC2
#else
nop
#endif
FXCPMADD f1, B3, A2, f1
nop
FXCSMADD f5, B3, A2, f5
LFPDUX A4, AO2, INC4
FXCPMADD f9, B4, A2, f9
#ifndef TRMMKERNEL
LFDUX A3, CO1, INC2
#else
nop
#endif
FXCSMADD f13, B4, A2, f13
nop
FXCPMADD f2, B3, A6, f2
nop
FXCSMADD f6, B3, A6, f6
nop
FXCPMADD f10, B4, A6, f10
nop
FXCSMADD f14, B4, A6, f14
nop
FXCPMADD f3, B3, A4, f3
nop
FXCSMADD f7, B3, A4, f7
LFPDUX A2, AO2, INC4
FXCPMADD f11, B4, A4, f11
#ifndef TRMMKERNEL
LFDUX A5, CO1, INC2
#else
nop
#endif
FXCSMADD f15, B4, A4, f15
nop
## 3 ##
FXCPMADD f0, B5, A7, f0
nop
FXCSMADD f4, B5, A7, f4
nop
FXCPMADD f8, B2, A7, f8
LFPDUX B4, BO2, INC4
FXCSMADD f12, B2, A7, f12
#ifndef TRMMKERNEL
LFSDUX A1, CO1, INCM5
#else
nop
#endif
FXCPMADD f1, B5, A2, f1
nop
FXCSMADD f5, B5, A2, f5
LFPDUX A4, AO2, INC4
FXCPMADD f9, B2, A2, f9
#ifndef TRMMKERNEL
LFSDUX B1, CO1, INC2
#else
nop
#endif
FXCSMADD f13, B2, A2, f13
nop
FXCPMADD f2, B5, A8, f2
nop
FXCSMADD f6, B5, A8, f6
nop
FXCPMADD f10, B2, A8, f10
nop
FXCSMADD f14, B2, A8, f14
nop
FXCPMADD f3, B5, A4, f3
nop
FXCSMADD f7, B5, A4, f7
LFPDUX A2, AO2, INC4
FXCPMADD f11, B2, A4, f11
#ifndef TRMMKERNEL
LFSDUX A3, CO1, INC2
#else
nop
#endif
FXCSMADD f15, B2, A4, f15
nop
## 4 ##
FXCPMADD f0, B6, A9, f0
nop
FXCSMADD f4, B6, A9, f4
nop
FXCPMADD f8, B4, A9, f8
#ifndef TRMMKERNEL
LFSDUX A5, CO1, INC2
#else
nop
#endif
FXCSMADD f12, B4, A9, f12
#ifndef TRMMKERNEL
LFDUX B3, CO2, INC
#else
nop
#endif
FXCPMADD f1, B6, A2, f1
nop
FXCSMADD f5, B6, A2, f5
LFPDUX A4, AO2, INC4
FXCPMADD f9, B4, A2, f9
#ifndef TRMMKERNEL
LFDUX A6, CO2, INC2
#else
nop
#endif
FXCSMADD f13, B4, A2, f13
nop
FXCPMADD f2, B6, A10, f2
nop
FXCSMADD f6, B6, A10, f6
nop
FXCPMADD f10, B4, A10, f10
nop
FXCSMADD f14, B4, A10, f14
#ifndef TRMMKERNEL
LFDUX A7, CO2, INC2
#else
nop
#endif
FXCPMADD f3, B6, A4, f3
nop
FXCSMADD f7, B6, A4, f7
nop
FXCPMADD f11, B4, A4, f11
nop
FXCSMADD f15, B4, A4, f15
#ifndef TRMMKERNEL
LFDUX B2, CO2, INC2
#else
nop
#endif
.align 4
.L1014:
li r0, ALPHA
lfpdx AP, SP, r0
#ifdef TRMMKERNEL
li r0, FZERO
lfpsx f30, SP, r0
#endif
#if defined(TRMMKERNEL)
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub TEMP, K, KK
#elif defined(LEFT)
addi TEMP, KK, 4
#else
addi TEMP, KK, 2
#endif
andi. r0, TEMP, 3
mtspr CTR, r0
ble+ .L1018
cmpwi cr0, TEMP, 3
bgt+ .L1015
#else
andi. r0, K, 3
mtspr CTR, r0
ble+ .L1018
cmpwi cr0, K, 3
bgt+ .L1015
#endif
#ifndef TRMMKERNEL
LFDUX A1, CO1, INC
fpmr f5, f0
LFDUX B1, CO1, INC2
fpmr f9, f0
LFDUX A3, CO1, INC2
fpmr f13, f0
LFDUX A5, CO1, INC2
fpmr f2, f0
LFSDUX A1, CO1, INCM5
fpmr f6, f0
LFSDUX B1, CO1, INC2
fpmr f10, f0
LFSDUX A3, CO1, INC2
fpmr f14, f0
LFSDUX A5, CO1, INC2
fpmr f3, f0
LFDUX B3, CO2, INC
fpmr f7, f0
LFDUX A6, CO2, INC2
fpmr f11, f0
LFDUX A7, CO2, INC2
fpmr f15, f0
LFDUX B2, CO2, INC2
#else
fpmr f5, f0
fpmr f9, f0
fpmr f13, f0
fpmr f2, f0
fpmr f6, f0
fpmr f10, f0
fpmr f14, f0
fpmr f3, f0
fpmr f7, f0
fpmr f11, f0
fpmr f15, f0
#endif
.align 4
.L1015:
LFPDUX A2, AO, INC4
LFPDUX A4, AO2, INC4
LFPDUX A10, BO, INC4
LFPDUX B4, BO2, INC4
bdz- .L1017
.align 4
.L1016:
FXCPMADD f0, A10, A2, f0
FXCSMADD f4, A10, A2, f4
FXCPMADD f8, B4, A2, f8
FXCSMADD f12, B4, A2, f12
LFPDUX A2, AO, INC4
FXCPMADD f1, A10, A4, f1
FXCSMADD f5, A10, A4, f5
FXCPMADD f9, B4, A4, f9
FXCSMADD f13, B4, A4, f13
LFPDUX A4, AO2, INC4
FXCPMADD f2, A10, A2, f2
FXCSMADD f6, A10, A2, f6
FXCPMADD f10, B4, A2, f10
FXCSMADD f14, B4, A2, f14
LFPDUX A2, AO, INC4
FXCPMADD f3, A10, A4, f3
FXCSMADD f7, A10, A4, f7
LFPDUX A10, BO, INC4
FXCPMADD f11, B4, A4, f11
FXCSMADD f15, B4, A4, f15
LFPDUX A4, AO2, INC4
LFPDUX B4, BO2, INC4
bdnz+ .L1016
.align 4
.L1017:
FXCPMADD f0, A10, A2, f0
FXCSMADD f4, A10, A2, f4
FXCPMADD f8, B4, A2, f8
FXCSMADD f12, B4, A2, f12
LFPDUX A2, AO, INC4
FXCPMADD f1, A10, A4, f1
FXCSMADD f5, A10, A4, f5
FXCPMADD f9, B4, A4, f9
FXCSMADD f13, B4, A4, f13
LFPDUX A4, AO2, INC4
FXCPMADD f2, A10, A2, f2
FXCSMADD f6, A10, A2, f6
FXCPMADD f10, B4, A2, f10
FXCSMADD f14, B4, A2, f14
FXCPMADD f3, A10, A4, f3
FXCSMADD f7, A10, A4, f7
FXCPMADD f11, B4, A4, f11
FXCSMADD f15, B4, A4, f15
.align 4
.L1018:
#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
defined(RN) || defined(RT) || defined(CN) || defined(CT)
fpadd f0, f0, f4
fpadd f8, f8, f12
fpadd f1, f1, f5
fpadd f9, f9, f13
fpadd f2, f2, f6
fpadd f10, f10, f14
fpadd f3, f3, f7
fpadd f11, f11, f15
#else
fpsub f0, f0, f4
fpsub f8, f8, f12
fpsub f1, f1, f5
fpsub f9, f9, f13
fpsub f2, f2, f6
fpsub f10, f10, f14
fpsub f3, f3, f7
fpsub f11, f11, f15
#endif
#ifndef TRMMKERNEL
fxcpmadd A1, f0, AP, A1
LFSDUX B3, CO2, INCM5
fxcpmadd B1, f1, AP, B1
LFSDUX A6, CO2, INC2
fxcpmadd A3, f2, AP, A3
LFSDUX A7, CO2, INC2
fxcpmadd A5, f3, AP, A5
LFSDUX B2, CO2, INC2
fxcxnpma f0, f0, AP, A1
fxcpmadd B3, f8, AP, B3
fxcxnpma f1, f1, AP, B1
fxcpmadd A6, f9, AP, A6
fxcxnpma f2, f2, AP, A3
fxcpmadd A7, f10, AP, A7
fxcxnpma f3, f3, AP, A5
STFDUX f0, CO1, INCM7
fxcpmadd B2, f11, AP, B2
STFSDUX f0, CO1, INC
fxcxnpma f8, f8, AP, B3
STFDUX f1, CO1, INC
STFSDUX f1, CO1, INC
fxcxnpma f9, f9, AP, A6
STFDUX f2, CO1, INC
STFSDUX f2, CO1, INC
fxcxnpma f10, f10, AP, A7
STFDUX f3, CO1, INC
STFSDUX f3, CO1, INC
fxcxnpma f11, f11, AP, B2
STFDUX f8, CO2, INCM7
#else
fxcpmadd f12, f0, AP, f30
fxcpmadd f13, f1, AP, f30
fxcpmadd f14, f2, AP, f30
fxcpmadd f15, f3, AP, f30
fxcxnpma f0, f0, AP, f12
fxcxnpma f1, f1, AP, f13
fxcxnpma f2, f2, AP, f14
fxcxnpma f3, f3, AP, f15
fxcpmadd f16, f8, AP, f30
fxcpmadd f17, f9, AP, f30
fxcpmadd f18, f10, AP, f30
fxcpmadd f19, f11, AP, f30
fxcxnpma f8, f8, AP, f16
fxcxnpma f9, f9, AP, f17
fxcxnpma f10, f10, AP, f18
fxcxnpma f11, f11, AP, f19
STFDUX f0, CO1, INC
STFSDUX f0, CO1, INC
STFDUX f1, CO1, INC
STFSDUX f1, CO1, INC
STFDUX f2, CO1, INC
STFSDUX f2, CO1, INC
STFDUX f3, CO1, INC
STFSDUX f3, CO1, INC
STFDUX f8, CO2, INC
#endif
STFSDUX f8, CO2, INC
STFDUX f9, CO2, INC
STFSDUX f9, CO2, INC
STFDUX f10, CO2, INC
STFSDUX f10, CO2, INC
STFDUX f11, CO2, INC
STFSDUX f11, CO2, INC
#ifdef TRMMKERNEL
#if ( defined(LEFT) && defined(TRANSA)) || \
(!defined(LEFT) && !defined(TRANSA))
sub TEMP, K, KK
#ifdef LEFT
addi TEMP, TEMP, -4
#else
addi TEMP, TEMP, -2
#endif
slwi r0, TEMP, 2 + ZBASE_SHIFT
slwi TEMP, TEMP, 1 + ZBASE_SHIFT
add AO, AO, r0
add BO, BO, TEMP
#endif
#ifdef LEFT
addi KK, KK, 4
#endif
#endif
addic. I, I, -1
li r0, FZERO
lfpsx f0, SP, r0
bgt+ .L1011
.align 4
.L1020:
andi. I, M, 2
beq .L1030
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
addi AO2, AO, 2 * SIZE
fpmr f4, f0
addi BO, B, - 4 * SIZE
fpmr f8, f0
addi BO2, B, - 2 * SIZE
fpmr f12, f0
#else
slwi TEMP, KK, 1 + ZBASE_SHIFT
slwi r0, KK, 1 + ZBASE_SHIFT
add AO, AO, TEMP
add BO, B, r0
addi AO2, AO, 2 * SIZE
fpmr f4, f0
addi BO, BO, - 4 * SIZE
fpmr f8, f0
addi BO2, BO, 2 * SIZE
fpmr f12, f0
#endif
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub TEMP, K, KK
#elif defined(LEFT)
addi TEMP, KK, 2
#else
addi TEMP, KK, 2
#endif
srawi. r0, TEMP, 2
fpmr f1, f0
fpmr f5, f0
fpmr f9, f0
mtspr CTR, r0
fpmr f13, f0
ble .L1024
#else
addi AO2, AO, 2 * SIZE
fpmr f4, f0
addi BO, B, - 4 * SIZE
fpmr f8, f0
addi BO2, B, - 2 * SIZE
fpmr f12, f0
srawi. r0, K, 2
fpmr f1, f0
fpmr f5, f0
fpmr f9, f0
mtspr CTR, r0
fpmr f13, f0
ble .L1024
#endif
LFPDUX A1, AO, INC4
LFPDUX B1, BO, INC4
LFPDUX A2, AO2, INC4
LFPDUX B2, BO2, INC4
LFPDUX A3, AO, INC4
LFPDUX B3, BO, INC4
LFPDUX A4, AO2, INC4
LFPDUX B4, BO2, INC4
LFPDUX A5, AO, INC4
LFPDUX B5, BO, INC4
LFPDUX A6, AO2, INC4
LFPDUX B6, BO2, INC4
LFPDUX A7, AO, INC4
LFPDUX A9, BO, INC4
LFPDUX A10, BO2, INC4
bdz- .L1023
.align 4
.L1022:
FXCPMADD f0, B1, A1, f0
nop
FXCSMADD f4, B1, A1, f4
LFPDUX A8, AO2, INC4
FXCPMADD f8, B2, A1, f8
nop
FXCSMADD f12, B2, A1, f12
LFPDUX A1, AO, INC4
FXCPMADD f1, B1, A2, f1
nop
FXCSMADD f5, B1, A2, f5
LFPDUX B1, BO, INC4
FXCPMADD f9, B2, A2, f9
nop
FXCSMADD f13, B2, A2, f13
LFPDUX B2, BO2, INC4
FXCPMADD f0, B3, A3, f0
nop
FXCSMADD f4, B3, A3, f4
LFPDUX A2, AO2, INC4
FXCPMADD f8, B4, A3, f8
nop
FXCSMADD f12, B4, A3, f12
LFPDUX A3, AO, INC4
FXCPMADD f1, B3, A4, f1
nop
FXCSMADD f5, B3, A4, f5
LFPDUX B3, BO, INC4
FXCPMADD f9, B4, A4, f9
nop
FXCSMADD f13, B4, A4, f13
LFPDUX B4, BO2, INC4
FXCPMADD f0, B5, A5, f0
nop
FXCSMADD f4, B5, A5, f4
LFPDUX A4, AO2, INC4
FXCPMADD f8, B6, A5, f8
nop
FXCSMADD f12, B6, A5, f12
LFPDUX A5, AO, INC4
FXCPMADD f1, B5, A6, f1
nop
FXCSMADD f5, B5, A6, f5
LFPDUX B5, BO, INC4
FXCPMADD f9, B6, A6, f9
nop
FXCSMADD f13, B6, A6, f13
LFPDUX B6, BO2, INC4
FXCPMADD f0, A9, A7, f0
nop
FXCSMADD f4, A9, A7, f4
LFPDUX A6, AO2, INC4
FXCPMADD f8, A10, A7, f8
nop
FXCSMADD f12, A10, A7, f12
LFPDUX A7, AO, INC4
FXCPMADD f1, A9, A8, f1
nop
FXCSMADD f5, A9, A8, f5
LFPDUX A9, BO, INC4
FXCPMADD f9, A10, A8, f9
nop
FXCSMADD f13, A10, A8, f13
LFPDUX A10, BO2, INC4
bdnz+ .L1022
.align 4
.L1023:
FXCPMADD f0, B1, A1, f0
FXCSMADD f4, B1, A1, f4
LFPDUX A8, AO2, INC4
FXCPMADD f8, B2, A1, f8
FXCSMADD f12, B2, A1, f12
FXCPMADD f1, B1, A2, f1
FXCSMADD f5, B1, A2, f5
FXCPMADD f9, B2, A2, f9
FXCSMADD f13, B2, A2, f13
FXCPMADD f0, B3, A3, f0
FXCSMADD f4, B3, A3, f4
FXCPMADD f8, B4, A3, f8
FXCSMADD f12, B4, A3, f12
FXCPMADD f1, B3, A4, f1
FXCSMADD f5, B3, A4, f5
FXCPMADD f9, B4, A4, f9
FXCSMADD f13, B4, A4, f13
FXCPMADD f0, B5, A5, f0
FXCSMADD f4, B5, A5, f4
FXCPMADD f8, B6, A5, f8
FXCSMADD f12, B6, A5, f12
FXCPMADD f1, B5, A6, f1
FXCSMADD f5, B5, A6, f5
FXCPMADD f9, B6, A6, f9
FXCSMADD f13, B6, A6, f13
FXCPMADD f0, A9, A7, f0
FXCSMADD f4, A9, A7, f4
FXCPMADD f8, A10, A7, f8
FXCSMADD f12, A10, A7, f12
FXCPMADD f1, A9, A8, f1
FXCSMADD f5, A9, A8, f5
FXCPMADD f9, A10, A8, f9
FXCSMADD f13, A10, A8, f13
.align 4
.L1024:
li r0, ALPHA
lfpdx AP, SP, r0
#ifdef TRMMKERNEL
li r0, FZERO
lfpsx f30, SP, r0
#endif
#if defined(TRMMKERNEL)
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub TEMP, K, KK
#elif defined(LEFT)
addi TEMP, KK, 2
#else
addi TEMP, KK, 2
#endif
andi. r0, TEMP, 3
mtspr CTR, r0
#else
andi. r0, K, 3
mtspr CTR, r0
#endif
ble+ .L1028
LFPDUX A1, AO, INC4
LFPDUX A2, AO2, INC4
LFPDUX B1, BO, INC4
LFPDUX B2, BO2, INC4
bdz- .L1027
.align 4
.L1026:
FXCPMADD f0, B1, A1, f0
FXCSMADD f4, B1, A1, f4
FXCPMADD f8, B2, A1, f8
FXCSMADD f12, B2, A1, f12
LFPDUX A1, AO, INC4
FXCPMADD f1, B1, A2, f1
FXCSMADD f5, B1, A2, f5
LFPDUX B1, BO, INC4
FXCPMADD f9, B2, A2, f9
FXCSMADD f13, B2, A2, f13
LFPDUX A2, AO2, INC4
LFPDUX B2, BO2, INC4
bdnz+ .L1026
.align 4
.L1027:
FXCPMADD f0, B1, A1, f0
FXCSMADD f4, B1, A1, f4
FXCPMADD f8, B2, A1, f8
FXCSMADD f12, B2, A1, f12
FXCPMADD f1, B1, A2, f1
FXCSMADD f5, B1, A2, f5
FXCPMADD f9, B2, A2, f9
FXCSMADD f13, B2, A2, f13
.align 4
.L1028:
#ifndef TRMMKERNEL
LFDUX A1, CO1, INC
LFDUX A2, CO1, INC2
LFDUX A3, CO2, INC
LFDUX A4, CO2, INC2
LFSDUX A1, CO1, INCM1
LFSDUX A2, CO1, INC2
LFSDUX A3, CO2, INCM1
LFSDUX A4, CO2, INC2
#endif
#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
defined(RN) || defined(RT) || defined(CN) || defined(CT)
fpadd f0, f0, f4
fpadd f8, f8, f12
fpadd f1, f1, f5
fpadd f9, f9, f13
#else
fpsub f0, f0, f4
fpsub f8, f8, f12
fpsub f1, f1, f5
fpsub f9, f9, f13
#endif
#ifndef TRMMKERNEL
fxcpmadd A1, f0, AP, A1
fxcpmadd A2, f1, AP, A2
fxcpmadd A3, f8, AP, A3
fxcpmadd A4, f9, AP, A4
fxcxnpma f0, f0, AP, A1
fxcxnpma f1, f1, AP, A2
fxcxnpma f8, f8, AP, A3
fxcxnpma f9, f9, AP, A4
STFDUX f0, CO1, INCM3
STFSDUX f0, CO1, INC
STFDUX f1, CO1, INC
STFSDUX f1, CO1, INC
STFDUX f8, CO2, INCM3
STFSDUX f8, CO2, INC
STFDUX f9, CO2, INC
STFSDUX f9, CO2, INC
#else
fxcpmadd f12, f0, AP, f30
fxcpmadd f13, f1, AP, f30
fxcpmadd f14, f8, AP, f30
fxcpmadd f15, f9, AP, f30
fxcxnpma f0, f0, AP, f12
fxcxnpma f1, f1, AP, f13
fxcxnpma f8, f8, AP, f14
fxcxnpma f9, f9, AP, f15
STFDUX f0, CO1, INC
STFSDUX f0, CO1, INC
STFDUX f1, CO1, INC
STFSDUX f1, CO1, INC
STFDUX f8, CO2, INC
STFSDUX f8, CO2, INC
STFDUX f9, CO2, INC
STFSDUX f9, CO2, INC
#endif
#ifdef TRMMKERNEL
#if ( defined(LEFT) && defined(TRANSA)) || \
(!defined(LEFT) && !defined(TRANSA))
sub TEMP, K, KK
#ifdef LEFT
addi TEMP, TEMP, -2
#else
addi TEMP, TEMP, -2
#endif
slwi r0, TEMP, 1 + ZBASE_SHIFT
slwi TEMP, TEMP, 1 + ZBASE_SHIFT
add AO, AO, r0
add BO, BO, TEMP
#endif
#ifdef LEFT
addi KK, KK, 2
#endif
#endif
li r0, FZERO
lfpsx f0, SP, r0
.align 4
.L1030:
andi. I, M, 1
beq .L1049
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
addi AO2, AO, 2 * SIZE
fpmr f1, f0
addi BO, B, - 4 * SIZE
fpmr f2, f0
addi BO2, B, - 2 * SIZE
fpmr f3, f0
#else
slwi TEMP, KK, 0 + ZBASE_SHIFT
slwi r0, KK, 1 + ZBASE_SHIFT
add AO, AO, TEMP
add BO, B, r0
addi AO2, AO, 2 * SIZE
fpmr f1, f0
addi BO, BO, - 4 * SIZE
fpmr f2, f0
addi BO2, BO, 2 * SIZE
fpmr f3, f0
#endif
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub TEMP, K, KK
#elif defined(LEFT)
addi TEMP, KK, 1
#else
addi TEMP, KK, 2
#endif
srawi. r0, TEMP, 2
mtspr CTR, r0
ble .L1034
#else
addi AO2, AO, 2 * SIZE
fpmr f1, f0
addi BO, B, - 4 * SIZE
fpmr f2, f0
addi BO2, B, - 2 * SIZE
fpmr f3, f0
srawi. r0, K, 2
mtspr CTR, r0
ble .L1034
#endif
LFPDUX A1, AO, INC4
LFPDUX B1, BO, INC4
LFPDUX B2, BO2, INC4
LFPDUX A2, AO2, INC4
LFPDUX B3, BO, INC4
LFPDUX B4, BO2, INC4
LFPDUX A3, AO, INC4
LFPDUX A5, BO, INC4
LFPDUX A6, BO2, INC4
LFPDUX A4, AO2, INC4
LFPDUX A7, BO, INC4
LFPDUX A8, BO2, INC4
bdz- .L1033
.align 4
.L1032:
FXCPMADD f0, B1, A1, f0
FXCSMADD f1, B1, A1, f1
LFPDUX B1, BO, INC4
FXCPMADD f2, B2, A1, f2
FXCSMADD f3, B2, A1, f3
LFPDUX B2, BO2, INC4
LFPDUX A1, AO, INC4
FXCPMADD f0, B3, A2, f0
FXCSMADD f1, B3, A2, f1
LFPDUX B3, BO, INC4
FXCPMADD f2, B4, A2, f2
FXCSMADD f3, B4, A2, f3
LFPDUX B4, BO2, INC4
LFPDUX A2, AO2, INC4
FXCPMADD f0, A5, A3, f0
FXCSMADD f1, A5, A3, f1
LFPDUX A5, BO, INC4
FXCPMADD f2, A6, A3, f2
FXCSMADD f3, A6, A3, f3
LFPDUX A6, BO2, INC4
LFPDUX A3, AO, INC4
FXCPMADD f0, A7, A4, f0
FXCSMADD f1, A7, A4, f1
LFPDUX A7, BO, INC4
FXCPMADD f2, A8, A4, f2
FXCSMADD f3, A8, A4, f3
LFPDUX A8, BO2, INC4
LFPDUX A4, AO2, INC4
bdnz+ .L1032
.align 4
.L1033:
FXCPMADD f0, B1, A1, f0
FXCSMADD f1, B1, A1, f1
FXCPMADD f2, B2, A1, f2
FXCSMADD f3, B2, A1, f3
FXCPMADD f0, B3, A2, f0
FXCSMADD f1, B3, A2, f1
FXCPMADD f2, B4, A2, f2
FXCSMADD f3, B4, A2, f3
FXCPMADD f0, A5, A3, f0
FXCSMADD f1, A5, A3, f1
FXCPMADD f2, A6, A3, f2
FXCSMADD f3, A6, A3, f3
FXCPMADD f0, A7, A4, f0
FXCSMADD f1, A7, A4, f1
FXCPMADD f2, A8, A4, f2
FXCSMADD f3, A8, A4, f3
.align 4
.L1034:
li r0, ALPHA
lfpdx AP, SP, r0
#ifdef TRMMKERNEL
li r0, FZERO
lfpsx f30, SP, r0
#endif
#if defined(TRMMKERNEL)
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub TEMP, K, KK
#elif defined(LEFT)
addi TEMP, KK, 1
#else
addi TEMP, KK, 2
#endif
andi. r0, TEMP, 3
mtspr CTR, r0
#else
andi. r0, K, 3
mtspr CTR, r0
#endif
ble+ .L1038
LFPDX A1, AO, INC4
LFPDUX B1, BO, INC4
LFPDUX B2, BO2, INC4
add AO, AO, INC2
bdz- .L1037
.align 4
.L1036:
FXCPMADD f0, B1, A1, f0
FXCSMADD f1, B1, A1, f1
LFPDUX B1, BO, INC4
FXCPMADD f2, B2, A1, f2
FXCSMADD f3, B2, A1, f3
LFPDX A1, AO, INC4
LFPDUX B2, BO2, INC4
add AO, AO, INC2
bdnz+ .L1036
.align 4
.L1037:
FXCPMADD f0, B1, A1, f0
FXCSMADD f1, B1, A1, f1
FXCPMADD f2, B2, A1, f2
FXCSMADD f3, B2, A1, f3
.align 4
.L1038:
#ifndef TRMMKERNEL
LFDUX A1, CO1, INC
LFDUX A2, CO2, INC
LFSDUX A1, CO1, INC
LFSDUX A2, CO2, INC
#endif
#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
defined(RN) || defined(RT) || defined(CN) || defined(CT)
fpadd f0, f0, f1
fpadd f2, f2, f3
#else
fpsub f0, f0, f1
fpsub f2, f2, f3
#endif
#ifndef TRMMKERNEL
fxcpmadd A1, f0, AP, A1
fxcpmadd A2, f2, AP, A2
fxcxnpma f0, f0, AP, A1
fxcxnpma f2, f2, AP, A2
STFDUX f0, CO1, INCM1
STFSDUX f0, CO1, INC
STFDUX f2, CO2, INCM1
STFSDUX f2, CO2, INC
#else
fxcpmadd f12, f0, AP, f30
fxcpmadd f13, f2, AP, f30
fxcxnpma f0, f0, AP, f12
fxcxnpma f2, f2, AP, f13
STFDUX f0, CO1, INC
STFSDUX f0, CO1, INC
STFDUX f2, CO2, INC
STFSDUX f2, CO2, INC
#endif
#ifdef TRMMKERNEL
#if ( defined(LEFT) && defined(TRANSA)) || \
(!defined(LEFT) && !defined(TRANSA))
sub TEMP, K, KK
#ifdef LEFT
addi TEMP, TEMP, -1
#else
addi TEMP, TEMP, -2
#endif
slwi r0, TEMP, 0 + ZBASE_SHIFT
slwi TEMP, TEMP, 1 + ZBASE_SHIFT
add AO, AO, r0
add BO, BO, TEMP
#endif
#ifdef LEFT
addi KK, KK, 1
#endif
#endif
li r0, FZERO
lfpsx f0, SP, r0
.align 4
.L1049:
#if defined(TRMMKERNEL) && !defined(LEFT)
addi KK, KK, 2
#endif
addi B, BO, 4 * SIZE
addic. J, J, -1
bgt+ .L1010
.align 4
.L1050:
andi. J, N, 1
beq .L10999
mr CO1, C
#if defined(TRMMKERNEL) && defined(LEFT)
mr KK, OFFSET
#endif
addi AO, A, -2 * SIZE
li r0, FZERO
lfpsx f0, SP, r0
srawi. I, M, 2
ble .L1060
.align 4
.L1051:
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
fpmr f4, f0
addi BO, B, - 2 * SIZE
fpmr f1, f0
fpmr f5, f0
fpmr f2, f0
fpmr f6, f0
#else
slwi TEMP, KK, 2 + ZBASE_SHIFT
slwi r0, KK, 0 + ZBASE_SHIFT
add AO, AO, TEMP
add BO, B, r0
fpmr f4, f0
addi BO, BO, - 2 * SIZE
fpmr f1, f0
fpmr f5, f0
fpmr f2, f0
fpmr f6, f0
#endif
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub TEMP, K, KK
#elif defined(LEFT)
addi TEMP, KK, 4
#else
addi TEMP, KK, 1
#endif
srawi. r0, TEMP, 2
fpmr f3, f0
mtspr CTR, r0
fpmr f7, f0
ble .L1054
#else
srawi. r0, K, 2
fpmr f4, f0
addi BO, B, - 2 * SIZE
fpmr f1, f0
fpmr f5, f0
fpmr f2, f0
fpmr f6, f0
fpmr f3, f0
mtspr CTR, r0
fpmr f7, f0
ble .L1054
#endif
LFPDUX B1, BO, INC2
LFPDUX A1, AO, INC2
LFPDUX A2, AO, INC2
LFPDUX B2, BO, INC2
LFPDUX A3, AO, INC2
LFPDUX A4, AO, INC2
LFPDUX B3, BO, INC2
LFPDUX A5, AO, INC2
LFPDUX A6, AO, INC2
LFPDUX A7, AO, INC2
LFPDUX A8, AO, INC2
bdz- .L1053
.align 4
.L1052:
FXCPMADD f0, B1, A1, f0
LFPDUX B4, BO, INC2
FXCSMADD f4, B1, A1, f4
LFPDUX A1, AO, INC2
FXCPMADD f1, B1, A2, f1
nop
FXCSMADD f5, B1, A2, f5
LFPDUX A2, AO, INC2
FXCPMADD f2, B1, A3, f2
nop
FXCSMADD f6, B1, A3, f6
LFPDUX A3, AO, INC2
FXCPMADD f3, B1, A4, f3
nop
FXCSMADD f7, B1, A4, f7
LFPDUX A4, AO, INC2
FXCPMADD f0, B2, A5, f0
LFPDUX B1, BO, INC2
FXCSMADD f4, B2, A5, f4
LFPDUX A5, AO, INC2
FXCPMADD f1, B2, A6, f1
nop
FXCSMADD f5, B2, A6, f5
LFPDUX A6, AO, INC2
FXCPMADD f2, B2, A7, f2
nop
FXCSMADD f6, B2, A7, f6
LFPDUX A7, AO, INC2
FXCPMADD f3, B2, A8, f3
nop
FXCSMADD f7, B2, A8, f7
LFPDUX A8, AO, INC2
FXCPMADD f0, B3, A1, f0
LFPDUX B2, BO, INC2
FXCSMADD f4, B3, A1, f4
LFPDUX A1, AO, INC2
FXCPMADD f1, B3, A2, f1
nop
FXCSMADD f5, B3, A2, f5
LFPDUX A2, AO, INC2
FXCPMADD f2, B3, A3, f2
nop
FXCSMADD f6, B3, A3, f6
LFPDUX A3, AO, INC2
FXCPMADD f3, B3, A4, f3
nop
FXCSMADD f7, B3, A4, f7
LFPDUX A4, AO, INC2
FXCPMADD f0, B4, A5, f0
LFPDUX B3, BO, INC2
FXCSMADD f4, B4, A5, f4
LFPDUX A5, AO, INC2
FXCPMADD f1, B4, A6, f1
nop
FXCSMADD f5, B4, A6, f5
LFPDUX A6, AO, INC2
FXCPMADD f2, B4, A7, f2
nop
FXCSMADD f6, B4, A7, f6
LFPDUX A7, AO, INC2
FXCPMADD f3, B4, A8, f3
nop
FXCSMADD f7, B4, A8, f7
LFPDUX A8, AO, INC2
bdnz+ .L1052
.align 4
.L1053:
FXCPMADD f0, B1, A1, f0
LFPDUX B4, BO, INC2
FXCSMADD f4, B1, A1, f4
LFPDUX A1, AO, INC2
FXCPMADD f1, B1, A2, f1
nop
FXCSMADD f5, B1, A2, f5
LFPDUX A2, AO, INC2
FXCPMADD f2, B1, A3, f2
nop
FXCSMADD f6, B1, A3, f6
LFPDUX A3, AO, INC2
FXCPMADD f3, B1, A4, f3
nop
FXCSMADD f7, B1, A4, f7
LFPDUX A4, AO, INC2
FXCPMADD f0, B2, A5, f0
nop
FXCSMADD f4, B2, A5, f4
LFPDUX A5, AO, INC2
FXCPMADD f1, B2, A6, f1
nop
FXCSMADD f5, B2, A6, f5
LFPDUX A6, AO, INC2
FXCPMADD f2, B2, A7, f2
nop
FXCSMADD f6, B2, A7, f6
LFPDUX A7, AO, INC2
FXCPMADD f3, B2, A8, f3
nop
FXCSMADD f7, B2, A8, f7
LFPDUX A8, AO, INC2
FXCPMADD f0, B3, A1, f0
FXCSMADD f4, B3, A1, f4
FXCPMADD f1, B3, A2, f1
FXCSMADD f5, B3, A2, f5
FXCPMADD f2, B3, A3, f2
FXCSMADD f6, B3, A3, f6
FXCPMADD f3, B3, A4, f3
FXCSMADD f7, B3, A4, f7
FXCPMADD f0, B4, A5, f0
FXCSMADD f4, B4, A5, f4
FXCPMADD f1, B4, A6, f1
FXCSMADD f5, B4, A6, f5
FXCPMADD f2, B4, A7, f2
FXCSMADD f6, B4, A7, f6
FXCPMADD f3, B4, A8, f3
FXCSMADD f7, B4, A8, f7
.align 4
.L1054:
li r0, ALPHA
lfpdx AP, SP, r0
#ifdef TRMMKERNEL
li r0, FZERO
lfpsx f30, SP, r0
#endif
#if defined(TRMMKERNEL)
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub TEMP, K, KK
#elif defined(LEFT)
addi TEMP, KK, 4
#else
addi TEMP, KK, 1
#endif
andi. r0, TEMP, 3
mtspr CTR, r0
#else
andi. r0, K, 3
mtspr CTR, r0
#endif
ble+ .L1058
LFPDUX A1, AO, INC2
LFPDUX B1, BO, INC2
LFPDUX A2, AO, INC2
LFPDUX A3, AO, INC2
LFPDUX A4, AO, INC2
bdz- .L1057
.align 4
.L1056:
FXCPMADD f0, B1, A1, f0
FXCSMADD f4, B1, A1, f4
LFPDUX A1, AO, INC2
FXCPMADD f1, B1, A2, f1
FXCSMADD f5, B1, A2, f5
LFPDUX A2, AO, INC2
FXCPMADD f2, B1, A3, f2
FXCSMADD f6, B1, A3, f6
LFPDUX A3, AO, INC2
FXCPMADD f3, B1, A4, f3
FXCSMADD f7, B1, A4, f7
LFPDUX A4, AO, INC2
LFPDUX B1, BO, INC2
bdnz+ .L1056
.align 4
.L1057:
FXCPMADD f0, B1, A1, f0
FXCSMADD f4, B1, A1, f4
FXCPMADD f1, B1, A2, f1
FXCSMADD f5, B1, A2, f5
FXCPMADD f2, B1, A3, f2
FXCSMADD f6, B1, A3, f6
FXCPMADD f3, B1, A4, f3
FXCSMADD f7, B1, A4, f7
.align 4
.L1058:
#ifndef TRMMKERNEL
LFDUX A1, CO1, INC
LFDUX A2, CO1, INC2
LFDUX A3, CO1, INC2
LFDUX A4, CO1, INC2
LFSDUX A1, CO1, INCM5
LFSDUX A2, CO1, INC2
LFSDUX A3, CO1, INC2
LFSDUX A4, CO1, INC2
#endif
#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
defined(RN) || defined(RT) || defined(CN) || defined(CT)
fpadd f0, f0, f4
fpadd f1, f1, f5
fpadd f2, f2, f6
fpadd f3, f3, f7
#else
fpsub f0, f0, f4
fpsub f1, f1, f5
fpsub f2, f2, f6
fpsub f3, f3, f7
#endif
#ifndef TRMMKERNEL
fxcpmadd A1, f0, AP, A1
fxcpmadd A2, f1, AP, A2
fxcpmadd A3, f2, AP, A3
fxcpmadd A4, f3, AP, A4
fxcxnpma f0, f0, AP, A1
fxcxnpma f1, f1, AP, A2
fxcxnpma f2, f2, AP, A3
fxcxnpma f3, f3, AP, A4
STFDUX f0, CO1, INCM7
STFSDUX f0, CO1, INC
STFDUX f1, CO1, INC
STFSDUX f1, CO1, INC
STFDUX f2, CO1, INC
STFSDUX f2, CO1, INC
STFDUX f3, CO1, INC
STFSDUX f3, CO1, INC
#else
fxcpmadd f12, f0, AP, f30
fxcpmadd f13, f1, AP, f30
fxcpmadd f14, f2, AP, f30
fxcpmadd f15, f3, AP, f30
fxcxnpma f0, f0, AP, f12
fxcxnpma f1, f1, AP, f13
fxcxnpma f2, f2, AP, f14
fxcxnpma f3, f3, AP, f15
STFDUX f0, CO1, INC
STFSDUX f0, CO1, INC
STFDUX f1, CO1, INC
STFSDUX f1, CO1, INC
STFDUX f2, CO1, INC
STFSDUX f2, CO1, INC
STFDUX f3, CO1, INC
STFSDUX f3, CO1, INC
#endif
#ifdef TRMMKERNEL
#if ( defined(LEFT) && defined(TRANSA)) || \
(!defined(LEFT) && !defined(TRANSA))
sub TEMP, K, KK
#ifdef LEFT
addi TEMP, TEMP, -4
#else
addi TEMP, TEMP, -1
#endif
slwi r0, TEMP, 2 + ZBASE_SHIFT
slwi TEMP, TEMP, 0 + ZBASE_SHIFT
add AO, AO, r0
add BO, BO, TEMP
#endif
#ifdef LEFT
addi KK, KK, 4
#endif
#endif
addic. I, I, -1
li r0, FZERO
lfpsx f0, SP, r0
bgt+ .L1051
.align 4
.L1060:
andi. I, M, 2
beq .L1070
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
addi BO, B, - 2 * SIZE
fpmr f1, f0
#else
slwi TEMP, KK, 1 + ZBASE_SHIFT
slwi r0, KK, 0 + ZBASE_SHIFT
add AO, AO, TEMP
add BO, B, r0
addi BO, BO, - 2 * SIZE
fpmr f1, f0
#endif
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub TEMP, K, KK
#elif defined(LEFT)
addi TEMP, KK, 2
#else
addi TEMP, KK, 1
#endif
srawi. r0, TEMP, 2
fpmr f2, f0
mtspr CTR, r0
fpmr f3, f0
ble .L1064
#else
srawi. r0, K, 2
fpmr f1, f0
addi BO, B, - 2 * SIZE
fpmr f2, f0
mtspr CTR, r0
fpmr f3, f0
ble .L1064
#endif
LFPDUX B1, BO, INC2
LFPDUX A1, AO, INC2
LFPDUX A2, AO, INC2
LFPDUX B2, BO, INC2
LFPDUX A3, AO, INC2
LFPDUX A4, AO, INC2
LFPDUX B3, BO, INC2
LFPDUX A5, AO, INC2
LFPDUX A6, AO, INC2
LFPDUX B4, BO, INC2
LFPDUX A7, AO, INC2
LFPDUX A8, AO, INC2
bdz- .L1063
.align 4
.L1062:
FXCPMADD f0, B1, A1, f0
FXCSMADD f2, B1, A1, f2
LFPDUX A1, AO, INC2
FXCPMADD f1, B1, A2, f1
FXCSMADD f3, B1, A2, f3
LFPDUX A2, AO, INC2
LFPDUX B1, BO, INC2
FXCPMADD f0, B2, A3, f0
FXCSMADD f2, B2, A3, f2
LFPDUX A3, AO, INC2
FXCPMADD f1, B2, A4, f1
FXCSMADD f3, B2, A4, f3
LFPDUX A4, AO, INC2
LFPDUX B2, BO, INC2
FXCPMADD f0, B3, A5, f0
FXCSMADD f2, B3, A5, f2
LFPDUX A5, AO, INC2
FXCPMADD f1, B3, A6, f1
FXCSMADD f3, B3, A6, f3
LFPDUX A6, AO, INC2
LFPDUX B3, BO, INC2
FXCPMADD f0, B4, A7, f0
FXCSMADD f2, B4, A7, f2
LFPDUX A7, AO, INC2
FXCPMADD f1, B4, A8, f1
FXCSMADD f3, B4, A8, f3
LFPDUX A8, AO, INC2
LFPDUX B4, BO, INC2
bdnz+ .L1062
.align 4
.L1063:
FXCPMADD f0, B1, A1, f0
FXCSMADD f2, B1, A1, f2
FXCPMADD f1, B1, A2, f1
FXCSMADD f3, B1, A2, f3
FXCPMADD f0, B2, A3, f0
FXCSMADD f2, B2, A3, f2
FXCPMADD f1, B2, A4, f1
FXCSMADD f3, B2, A4, f3
FXCPMADD f0, B3, A5, f0
FXCSMADD f2, B3, A5, f2
FXCPMADD f1, B3, A6, f1
FXCSMADD f3, B3, A6, f3
FXCPMADD f0, B4, A7, f0
FXCSMADD f2, B4, A7, f2
FXCPMADD f1, B4, A8, f1
FXCSMADD f3, B4, A8, f3
.align 4
.L1064:
li r0, ALPHA
lfpdx AP, SP, r0
#ifdef TRMMKERNEL
li r0, FZERO
lfpsx f30, SP, r0
#endif
#if defined(TRMMKERNEL)
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub TEMP, K, KK
#elif defined(LEFT)
addi TEMP, KK, 2
#else
addi TEMP, KK, 1
#endif
andi. r0, TEMP, 3
mtspr CTR, r0
#else
andi. r0, K, 3
mtspr CTR, r0
#endif
ble+ .L1068
LFPDUX A1, AO, INC2
LFPDUX B1, BO, INC2
LFPDUX A2, AO, INC2
bdz- .L1067
.align 4
.L1066:
FXCPMADD f0, B1, A1, f0
FXCSMADD f2, B1, A1, f2
LFPDUX A1, AO, INC2
FXCPMADD f1, B1, A2, f1
FXCSMADD f3, B1, A2, f3
LFPDUX B1, BO, INC2
LFPDUX A2, AO, INC2
bdnz+ .L1066
.align 4
.L1067:
FXCPMADD f0, B1, A1, f0
FXCSMADD f2, B1, A1, f2
FXCPMADD f1, B1, A2, f1
FXCSMADD f3, B1, A2, f3
.align 4
.L1068:
#ifndef TRMMKERNEL
LFDUX A1, CO1, INC
LFDUX A2, CO1, INC2
LFSDUX A1, CO1, INCM1
LFSDUX A2, CO1, INC2
#endif
#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
defined(RN) || defined(RT) || defined(CN) || defined(CT)
fpadd f0, f0, f2
fpadd f1, f1, f3
#else
fpsub f0, f0, f2
fpsub f1, f1, f3
#endif
#ifndef TRMMKERNEL
fxcpmadd A1, f0, AP, A1
fxcpmadd A2, f1, AP, A2
fxcxnpma f0, f0, AP, A1
fxcxnpma f1, f1, AP, A2
STFDUX f0, CO1, INCM3
STFSDUX f0, CO1, INC
STFDUX f1, CO1, INC
STFSDUX f1, CO1, INC
#else
fxcpmadd f12, f0, AP, f30
fxcpmadd f13, f1, AP, f30
fxcxnpma f0, f0, AP, f12
fxcxnpma f1, f1, AP, f13
STFDUX f0, CO1, INC
STFSDUX f0, CO1, INC
STFDUX f1, CO1, INC
STFSDUX f1, CO1, INC
#endif
#ifdef TRMMKERNEL
#if ( defined(LEFT) && defined(TRANSA)) || \
(!defined(LEFT) && !defined(TRANSA))
sub TEMP, K, KK
#ifdef LEFT
addi TEMP, TEMP, -2
#else
addi TEMP, TEMP, -1
#endif
slwi r0, TEMP, 1 + ZBASE_SHIFT
slwi TEMP, TEMP, 0 + ZBASE_SHIFT
add AO, AO, r0
add BO, BO, TEMP
#endif
#ifdef LEFT
addi KK, KK, 2
#endif
#endif
li r0, FZERO
lfpsx f0, SP, r0
.align 4
.L1070:
andi. I, M, 1
beq .L1089
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
addi BO, B, - 2 * SIZE
fpmr f1, f0
#else
slwi TEMP, KK, 0 + ZBASE_SHIFT
slwi r0, KK, 0 + ZBASE_SHIFT
add AO, AO, TEMP
add BO, B, r0
addi BO, BO, - 2 * SIZE
fpmr f1, f0
#endif
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub TEMP, K, KK
#elif defined(LEFT)
addi TEMP, KK, 1
#else
addi TEMP, KK, 1
#endif
srawi. r0, TEMP, 3
fpmr f2, f0
mtspr CTR, r0
fpmr f3, f0
ble .L1074
#else
addi BO, B, - 2 * SIZE
fpmr f1, f0
srawi. r0, K, 3
fpmr f2, f0
mtspr CTR, r0
fpmr f3, f0
ble .L1074
#endif
LFPDUX A1, AO, INC2
LFPDUX B1, BO, INC2
LFPDUX A2, AO, INC2
LFPDUX B2, BO, INC2
LFPDUX A3, AO, INC2
LFPDUX B3, BO, INC2
LFPDUX A4, AO, INC2
LFPDUX B4, BO, INC2
LFPDUX A5, AO, INC2
LFPDUX B5, BO, INC2
LFPDUX A6, AO, INC2
LFPDUX B6, BO, INC2
LFPDUX A7, AO, INC2
LFPDUX A9, BO, INC2
LFPDUX A8, AO, INC2
LFPDUX A10, BO, INC2
bdz- .L1073
.align 4
.L1072:
FXCPMADD f0, B1, A1, f0
FXCSMADD f1, B1, A1, f1
LFPDUX A1, AO, INC2
LFPDUX B1, BO, INC2
FXCPMADD f2, B2, A2, f2
FXCSMADD f3, B2, A2, f3
LFPDUX A2, AO, INC2
LFPDUX B2, BO, INC2
FXCPMADD f0, B3, A3, f0
FXCSMADD f1, B3, A3, f1
LFPDUX A3, AO, INC2
LFPDUX B3, BO, INC2
FXCPMADD f2, B4, A4, f2
FXCSMADD f3, B4, A4, f3
LFPDUX A4, AO, INC2
LFPDUX B4, BO, INC2
FXCPMADD f0, B5, A5, f0
FXCSMADD f1, B5, A5, f1
LFPDUX A5, AO, INC2
LFPDUX B5, BO, INC2
FXCPMADD f2, B6, A6, f2
FXCSMADD f3, B6, A6, f3
LFPDUX A6, AO, INC2
LFPDUX B6, BO, INC2
FXCPMADD f0, A9, A7, f0
FXCSMADD f1, A9, A7, f1
LFPDUX A7, AO, INC2
LFPDUX A9, BO, INC2
FXCPMADD f2, A10, A8, f2
FXCSMADD f3, A10, A8, f3
LFPDUX A8, AO, INC2
LFPDUX A10, BO, INC2
bdnz+ .L1072
.align 4
.L1073:
FXCPMADD f0, B1, A1, f0
FXCSMADD f1, B1, A1, f1
FXCPMADD f2, B2, A2, f2
FXCSMADD f3, B2, A2, f3
FXCPMADD f0, B3, A3, f0
FXCSMADD f1, B3, A3, f1
FXCPMADD f2, B4, A4, f2
FXCSMADD f3, B4, A4, f3
FXCPMADD f0, B5, A5, f0
FXCSMADD f1, B5, A5, f1
FXCPMADD f2, B6, A6, f2
FXCSMADD f3, B6, A6, f3
FXCPMADD f0, A9, A7, f0
FXCSMADD f1, A9, A7, f1
FXCPMADD f2, A10, A8, f2
FXCSMADD f3, A10, A8, f3
.align 4
.L1074:
li r0, ALPHA
lfpdx AP, SP, r0
#ifdef TRMMKERNEL
li r0, FZERO
lfpsx f30, SP, r0
#endif
#if defined(TRMMKERNEL)
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub TEMP, K, KK
#elif defined(LEFT)
addi TEMP, KK, 1
#else
addi TEMP, KK, 1
#endif
andi. r0, TEMP, 7
mtspr CTR, r0
#else
andi. r0, K, 7
mtspr CTR, r0
#endif
ble+ .L1078
LFPDUX A1, AO, INC2
LFPDUX B1, BO, INC2
bdz- .L1077
.align 4
.L1076:
FXCPMADD f0, B1, A1, f0
FXCSMADD f1, B1, A1, f1
LFPDUX A1, AO, INC2
LFPDUX B1, BO, INC2
bdnz+ .L1076
.align 4
.L1077:
FXCPMADD f0, B1, A1, f0
FXCSMADD f1, B1, A1, f1
.align 4
.L1078:
#ifndef TRMMKERNEL
LFDUX A1, CO1, INC
LFDUX A2, CO1, INC
#endif
fpadd f0, f0, f2
fpadd f1, f1, f3
fsmfp A1, A2
#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
defined(RN) || defined(RT) || defined(CN) || defined(CT)
fpadd f0, f0, f1
#else
fpsub f0, f0, f1
#endif
#ifndef TRMMKERNEL
fxcpmadd A1, f0, AP, A1
fxcxnpma f0, f0, AP, A1
STFDUX f0, CO1, INCM1
STFSDUX f0, CO1, INC
#else
fxcpmadd f12, f0, AP, f30
fxcxnpma f0, f0, AP, f12
STFDUX f0, CO1, INC
STFSDUX f0, CO1, INC
#endif
li r0, FZERO
lfpsx f0, SP, r0
.align 4
.L1089:
addi B, BO, 2 * SIZE
.align 4
.L10999:
addi SP, SP, 20
lwzu r14, 4(SP)
lwzu r15, 4(SP)
lwzu r16, 4(SP)
lwzu r17, 4(SP)
lwzu r18, 4(SP)
lwzu r19, 4(SP)
lwzu r20, 4(SP)
lwzu r21, 4(SP)
lwzu r22, 4(SP)
lwzu r23, 4(SP)
lwzu r24, 4(SP)
lwzu r25, 4(SP)
lwzu r26, 4(SP)
lwzu r27, 4(SP)
lwzu r28, 4(SP)
lwzu r29, 4(SP)
lwzu r30, 4(SP)
lwzu r31, 4(SP)
subi SP, SP, 12
li r0, 16
lfpdux f31, SP, r0
lfpdux f30, SP, r0
lfpdux f29, SP, r0
lfpdux f28, SP, r0
lfpdux f27, SP, r0
lfpdux f26, SP, r0
lfpdux f25, SP, r0
lfpdux f24, SP, r0
lfpdux f23, SP, r0
lfpdux f22, SP, r0
lfpdux f21, SP, r0
lfpdux f20, SP, r0
lfpdux f19, SP, r0
lfpdux f18, SP, r0
lfpdux f17, SP, r0
lfpdux f16, SP, r0
lfpdux f15, SP, r0
lfpdux f14, SP, r0
addi SP, SP, 16
blr
.align 4
EPILOGUE
#endif