/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/
#define ASSEMBLER
#include "common.h"
#define N r3
#define X r6
#define INCX r7
#define Y r8
#define INCY r9
#define YY r4
#define INCX2 r5
#define INCY2 r10
#define X1 r11
#define Y1 INCX
#define YY1 INCY
#define ALPHA f1
#define A1 f0
#define A2 f8
#define A3 f2
#define A4 f3
#define A5 f4
#define A6 f5
#define A7 f6
#define A8 f7
#define A9 f25
#define B1 f9
#define B2 f10
#define B3 f11
#define B4 f12
#define B5 f13
#define B6 f14
#define B7 f15
#define B8 f16
#define C1 f17
#define C2 f18
#define C3 f19
#define C4 f20
#define C5 f21
#define C6 f22
#define C7 f23
#define C8 f24
#define ALPHA_R ALPHA
#define ALPHA_I A9
#ifndef CONJ
#define ADD1 FNMSUB
#define ADD2 FMADD
#else
#define ADD1 FMADD
#define ADD2 FNMSUB
#endif
#ifndef CONJ
#define FXMADD1 fxcpmadd
#define FXMADD2 fxcxnpma
#else
#define FXMADD1 fxcpnsma
#define FXMADD2 fxcxma
#endif
PROLOGUE
PROFCODE
li r10, -16
stfpdux f14, SP, r10
stfpdux f15, SP, r10
stfpdux f16, SP, r10
stfpdux f17, SP, r10
stfpdux f18, SP, r10
stfpdux f19, SP, r10
stfpdux f20, SP, r10
stfpdux f21, SP, r10
stfpdux f22, SP, r10
stfpdux f23, SP, r10
stfpdux f24, SP, r10
stfpdux f25, SP, r10
fsmfp ALPHA, f2
slwi INCX, INCX, BASE_SHIFT
slwi INCY, INCY, BASE_SHIFT
add INCX2, INCX, INCX
add INCY2, INCY, INCY
cmpwi cr0, N, 0
ble LL(999)
andi. r0, X, 2 * SIZE - 1
bne LL(100)
andi. r0, Y, 2 * SIZE - 1
bne LL(100)
sub X, X, INCX2
sub Y, Y, INCY2
mr YY, Y
srawi. r0, N, 3
mtspr CTR, r0
beq- LL(15)
LFPDUX A1, X, INCX2
LFPDUX B1, Y, INCY2
LFPDUX A2, X, INCX2
LFPDUX B2, Y, INCY2
LFPDUX A3, X, INCX2
LFPDUX B3, Y, INCY2
LFPDUX A4, X, INCX2
LFPDUX B4, Y, INCY2
LFPDUX A5, X, INCX2
LFPDUX B5, Y, INCY2
LFPDUX A6, X, INCX2
LFPDUX B6, Y, INCY2
LFPDUX A7, X, INCX2
LFPDUX B7, Y, INCY2
LFPDUX A8, X, INCX2
LFPDUX B8, Y, INCY2
bdz LL(13)
.align 4
LL(12):
FXMADD1 C1, ALPHA, A1, B1
LFPDUX B1, Y, INCY2
FXMADD1 C2, ALPHA, A2, B2
LFPDUX B2, Y, INCY2
FXMADD1 C3, ALPHA, A3, B3
LFPDUX B3, Y, INCY2
FXMADD1 C4, ALPHA, A4, B4
LFPDUX B4, Y, INCY2
FXMADD1 C5, ALPHA, A5, B5
LFPDUX B5, Y, INCY2
FXMADD1 C6, ALPHA, A6, B6
LFPDUX B6, Y, INCY2
FXMADD1 C7, ALPHA, A7, B7
LFPDUX B7, Y, INCY2
FXMADD1 C8, ALPHA, A8, B8
LFPDUX B8, Y, INCY2
FXMADD2 C1, ALPHA, A1, C1
LFPDUX A1, X, INCX2
FXMADD2 C2, ALPHA, A2, C2
LFPDUX A2, X, INCX2
FXMADD2 C3, ALPHA, A3, C3
LFPDUX A3, X, INCX2
FXMADD2 C4, ALPHA, A4, C4
LFPDUX A4, X, INCX2
FXMADD2 C5, ALPHA, A5, C5
LFPDUX A5, X, INCX2
FXMADD2 C6, ALPHA, A6, C6
LFPDUX A6, X, INCX2
FXMADD2 C7, ALPHA, A7, C7
LFPDUX A7, X, INCX2
FXMADD2 C8, ALPHA, A8, C8
LFPDUX A8, X, INCX2
STFPDUX C1, YY, INCY2
STFPDUX C2, YY, INCY2
STFPDUX C3, YY, INCY2
STFPDUX C4, YY, INCY2
STFPDUX C5, YY, INCY2
STFPDUX C6, YY, INCY2
STFPDUX C7, YY, INCY2
STFPDUX C8, YY, INCY2
bdnz LL(12)
.align 4
LL(13):
FXMADD1 C1, ALPHA, A1, B1
FXMADD1 C2, ALPHA, A2, B2
FXMADD1 C3, ALPHA, A3, B3
FXMADD1 C4, ALPHA, A4, B4
FXMADD1 C5, ALPHA, A5, B5
FXMADD1 C6, ALPHA, A6, B6
FXMADD1 C7, ALPHA, A7, B7
FXMADD1 C8, ALPHA, A8, B8
FXMADD2 C1, ALPHA, A1, C1
FXMADD2 C2, ALPHA, A2, C2
FXMADD2 C3, ALPHA, A3, C3
FXMADD2 C4, ALPHA, A4, C4
FXMADD2 C5, ALPHA, A5, C5
FXMADD2 C6, ALPHA, A6, C6
STFPDUX C1, YY, INCY2
FXMADD2 C7, ALPHA, A7, C7
STFPDUX C2, YY, INCY2
FXMADD2 C8, ALPHA, A8, C8
STFPDUX C3, YY, INCY2
STFPDUX C4, YY, INCY2
STFPDUX C5, YY, INCY2
STFPDUX C6, YY, INCY2
STFPDUX C7, YY, INCY2
STFPDUX C8, YY, INCY2
.align 4
LL(15):
andi. r0, N, 7
beq LL(999)
andi. r0, N, 4
beq LL(16)
LFPDUX A1, X, INCX2
LFPDUX B1, Y, INCY2
LFPDUX A2, X, INCX2
LFPDUX B2, Y, INCY2
LFPDUX A3, X, INCX2
LFPDUX B3, Y, INCY2
LFPDUX A4, X, INCX2
LFPDUX B4, Y, INCY2
FXMADD1 C1, ALPHA, A1, B1
FXMADD1 C2, ALPHA, A2, B2
FXMADD1 C3, ALPHA, A3, B3
FXMADD1 C4, ALPHA, A4, B4
FXMADD2 C1, ALPHA, A1, C1
FXMADD2 C2, ALPHA, A2, C2
FXMADD2 C3, ALPHA, A3, C3
FXMADD2 C4, ALPHA, A4, C4
STFPDUX C1, YY, INCY2
STFPDUX C2, YY, INCY2
STFPDUX C3, YY, INCY2
STFPDUX C4, YY, INCY2
.align 4
LL(16):
andi. r0, N, 2
beq LL(17)
LFPDUX A1, X, INCX2
LFPDUX B1, Y, INCY2
LFPDUX A2, X, INCX2
LFPDUX B2, Y, INCY2
FXMADD1 C1, ALPHA, A1, B1
FXMADD1 C2, ALPHA, A2, B2
FXMADD2 C1, ALPHA, A1, C1
FXMADD2 C2, ALPHA, A2, C2
STFPDUX C1, YY, INCY2
STFPDUX C2, YY, INCY2
.align 4
LL(17):
andi. r0, N, 1
beq LL(999)
LFPDUX A1, X, INCX2
LFPDUX B1, Y, INCY2
FXMADD1 C1, ALPHA, A1, B1
FXMADD2 C1, ALPHA, A1, C1
STFPDUX C1, YY, INCY2
b LL(999)
.align 4
LL(100):
fsmtp ALPHA_I, ALPHA_R
sub X, X, INCX2
sub Y, Y, INCY2
addi X1, X, SIZE
addi Y1, Y, SIZE
mr YY, Y
mr YY1, Y1
srawi. r0, N, 2
mtspr CTR, r0
beq- LL(115)
LFDUX A1, X, INCX2
LFDUX A2, X1, INCX2
LFDUX B1, Y, INCY2
LFDUX B2, Y1, INCY2
LFDUX A3, X, INCX2
LFDUX A4, X1, INCX2
LFDUX B3, Y, INCY2
LFDUX B4, Y1, INCY2
LFDUX A5, X, INCX2
LFDUX A6, X1, INCX2
LFDUX B5, Y, INCY2
LFDUX B6, Y1, INCY2
LFDUX A7, X, INCX2
LFDUX A8, X1, INCX2
LFDUX B7, Y, INCY2
LFDUX B8, Y1, INCY2
bdz LL(113)
.align 4
LL(112):
FMADD C1, ALPHA_R, A1, B1
LFDUX B1, Y, INCY2
FMADD C2, ALPHA_I, A1, B2
LFDUX A1, X, INCX2
FMADD C3, ALPHA_R, A3, B3
LFDUX B3, Y, INCY2
FMADD C4, ALPHA_I, A3, B4
LFDUX A3, X, INCX2
FMADD C5, ALPHA_R, A5, B5
LFDUX B5, Y, INCY2
FMADD C6, ALPHA_I, A5, B6
LFDUX A5, X, INCX2
FMADD C7, ALPHA_R, A7, B7
LFDUX B7, Y, INCY2
FMADD C8, ALPHA_I, A7, B8
LFDUX A7, X, INCX2
ADD1 C1, ALPHA_I, A2, C1
LFDUX B2, Y1, INCY2
ADD2 C2, ALPHA_R, A2, C2
LFDUX A2, X1, INCX2
ADD1 C3, ALPHA_I, A4, C3
LFDUX B4, Y1, INCY2
ADD2 C4, ALPHA_R, A4, C4
LFDUX A4, X1, INCX2
ADD1 C5, ALPHA_I, A6, C5
LFDUX B6, Y1, INCY2
ADD2 C6, ALPHA_R, A6, C6
LFDUX A6, X1, INCX2
ADD1 C7, ALPHA_I, A8, C7
LFDUX B8, Y1, INCY2
ADD2 C8, ALPHA_R, A8, C8
LFDUX A8, X1, INCX2
STFDUX C1, YY, INCY2
STFDUX C2, YY1, INCY2
STFDUX C3, YY, INCY2
STFDUX C4, YY1, INCY2
STFDUX C5, YY, INCY2
STFDUX C6, YY1, INCY2
STFDUX C7, YY, INCY2
STFDUX C8, YY1, INCY2
bdnz LL(112)
.align 4
LL(113):
FMADD C1, ALPHA_R, A1, B1
FMADD C2, ALPHA_I, A1, B2
FMADD C3, ALPHA_R, A3, B3
FMADD C4, ALPHA_I, A3, B4
FMADD C5, ALPHA_R, A5, B5
FMADD C6, ALPHA_I, A5, B6
FMADD C7, ALPHA_R, A7, B7
FMADD C8, ALPHA_I, A7, B8
ADD1 C1, ALPHA_I, A2, C1
ADD2 C2, ALPHA_R, A2, C2
ADD1 C3, ALPHA_I, A4, C3
ADD2 C4, ALPHA_R, A4, C4
ADD1 C5, ALPHA_I, A6, C5
ADD2 C6, ALPHA_R, A6, C6
STFDUX C1, YY, INCY2
ADD1 C7, ALPHA_I, A8, C7
STFDUX C2, YY1, INCY2
ADD2 C8, ALPHA_R, A8, C8
STFDUX C3, YY, INCY2
STFDUX C4, YY1, INCY2
STFDUX C5, YY, INCY2
STFDUX C6, YY1, INCY2
STFDUX C7, YY, INCY2
STFDUX C8, YY1, INCY2
.align 4
LL(115):
andi. r0, N, 3
beq LL(999)
andi. r0, N, 2
beq LL(117)
LFDUX A1, X, INCX2
LFDUX A2, X1, INCX2
LFDUX B1, Y, INCY2
LFDUX B2, Y1, INCY2
LFDUX A3, X, INCX2
FMADD C1, ALPHA_R, A1, B1
LFDUX A4, X1, INCX2
FMADD C2, ALPHA_I, A1, B2
LFDUX B3, Y, INCY2
FMADD C3, ALPHA_R, A3, B3
LFDUX B4, Y1, INCY2
FMADD C4, ALPHA_I, A3, B4
ADD1 C1, ALPHA_I, A2, C1
ADD2 C2, ALPHA_R, A2, C2
STFDUX C1, YY, INCY2
ADD1 C3, ALPHA_I, A4, C3
STFDUX C2, YY1, INCY2
ADD2 C4, ALPHA_R, A4, C4
STFDUX C3, YY, INCY2
STFDUX C4, YY1, INCY2
.align 4
LL(117):
andi. r0, N, 1
beq LL(999)
LFDUX A1, X, INCX2
LFDUX A2, X1, INCX2
LFDUX B1, Y, INCY2
LFDUX B2, Y1, INCY2
FMADD C1, ALPHA_R, A1, B1
FMADD C2, ALPHA_I, A1, B2
ADD1 C1, ALPHA_I, A2, C1
ADD2 C2, ALPHA_R, A2, C2
STFDUX C1, YY, INCY2
STFDUX C2, YY1, INCY2
.align 4
LL(999):
li r10, 16
subi SP, SP, 16
lfpdux f25, SP, r10
lfpdux f24, SP, r10
lfpdux f23, SP, r10
lfpdux f22, SP, r10
lfpdux f21, SP, r10
lfpdux f20, SP, r10
lfpdux f19, SP, r10
lfpdux f18, SP, r10
lfpdux f17, SP, r10
lfpdux f16, SP, r10
lfpdux f15, SP, r10
lfpdux f14, SP, r10
addi SP, SP, 16
blr
EPILOGUE