/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/
#define ASSEMBLER
#include "common.h"
#define N r3
#define X r4
#define INCX r5
#define Y r6
#define INCY r7
#define INCX2 r8
#define INCY2 r9
#define C1 f1
#define C2 f0
#define C3 f2
#define C4 f3
#define A1 f4
#define A2 f5
#define A3 f6
#define A4 f7
#define A5 f8
#define A6 f9
#define A7 f10
#define A8 f11
#define A9 f20
#define B1 f12
#define B2 f13
#define B3 f14
#define B4 f15
#define B5 f16
#define B6 f17
#define B7 f18
#define B8 f19
#define B9 f20
PROLOGUE
PROFCODE
li r10, -16
stfpdux f14, SP, r10
stfpdux f15, SP, r10
stfpdux f16, SP, r10
stfpdux f17, SP, r10
stfpdux f18, SP, r10
stfpdux f19, SP, r10
stfpdux f20, SP, r10
li r10, 0
stwu r10, -4(SP)
stwu r10, -4(SP)
stwu r10, -4(SP)
stwu r10, -4(SP)
#ifdef F_INTERFACE
LDINT N, 0(N)
LDINT INCX, 0(INCX)
LDINT INCY, 0(INCY)
#endif
lfpdx C1, SP, r10 # Zero clear
slwi INCX, INCX, BASE_SHIFT
add INCX2, INCX, INCX
fpmr C2, C1
slwi INCY, INCY, BASE_SHIFT
fpmr C3, C1
add INCY2, INCY, INCY
fpmr C4, C1
cmpwi cr0, N, 0
ble LL(999)
cmpwi cr0, INCX, SIZE
bne LL(100)
cmpwi cr0, INCY, SIZE
bne LL(100)
/* X is aligned, Y is aligned */
LL(10):
andi. r0, X, 2 * SIZE - 1
bne LL(30)
andi. r0, Y, 2 * SIZE - 1
bne LL(20)
sub X, X, INCX2
sub Y, Y, INCY2
srawi. r0, N, 4
mtspr CTR, r0
beq- LL(15)
LFPDUX A1, X, INCX2
LFPDUX B1, Y, INCY2
LFPDUX A2, X, INCX2
LFPDUX B2, Y, INCY2
LFPDUX A3, X, INCX2
LFPDUX B3, Y, INCY2
LFPDUX A4, X, INCX2
LFPDUX B4, Y, INCY2
LFPDUX A5, X, INCX2
LFPDUX B5, Y, INCY2
LFPDUX A6, X, INCX2
LFPDUX B6, Y, INCY2
LFPDUX A7, X, INCX2
LFPDUX B7, Y, INCY2
LFPDUX A8, X, INCX2
LFPDUX B8, Y, INCY2
bdz LL(14)
.align 4
LL(13):
fpmadd C1, A1, B1, C1
LFPDUX A1, X, INCX2
LFPDUX B1, Y, INCY2
fpmadd C2, A2, B2, C2
LFPDUX A2, X, INCX2
LFPDUX B2, Y, INCY2
fpmadd C3, A3, B3, C3
LFPDUX A3, X, INCX2
LFPDUX B3, Y, INCY2
fpmadd C4, A4, B4, C4
LFPDUX A4, X, INCX2
LFPDUX B4, Y, INCY2
fpmadd C1, A5, B5, C1
LFPDUX A5, X, INCX2
LFPDUX B5, Y, INCY2
fpmadd C2, A6, B6, C2
LFPDUX A6, X, INCX2
LFPDUX B6, Y, INCY2
fpmadd C3, A7, B7, C3
LFPDUX A7, X, INCX2
LFPDUX B7, Y, INCY2
fpmadd C4, A8, B8, C4
LFPDUX A8, X, INCX2
LFPDUX B8, Y, INCY2
bdnz LL(13)
.align 4
LL(14):
fpmadd C1, A1, B1, C1
fpmadd C2, A2, B2, C2
fpmadd C3, A3, B3, C3
fpmadd C4, A4, B4, C4
fpmadd C1, A5, B5, C1
fpmadd C2, A6, B6, C2
fpmadd C3, A7, B7, C3
fpmadd C4, A8, B8, C4
.align 4
LL(15):
andi. r0, N, 15
beq LL(999)
andi. r0, N, 8
beq LL(16)
LFPDUX A1, X, INCX2
LFPDUX B1, Y, INCY2
LFPDUX A2, X, INCX2
LFPDUX B2, Y, INCY2
LFPDUX A3, X, INCX2
LFPDUX B3, Y, INCY2
LFPDUX A4, X, INCX2
LFPDUX B4, Y, INCY2
fpmadd C1, A1, B1, C1
fpmadd C2, A2, B2, C2
fpmadd C3, A3, B3, C3
fpmadd C4, A4, B4, C4
.align 4
LL(16):
andi. r0, N, 4
beq LL(17)
LFPDUX A1, X, INCX2
LFPDUX B1, Y, INCY2
LFPDUX A2, X, INCX2
LFPDUX B2, Y, INCY2
fpmadd C1, A1, B1, C1
fpmadd C2, A2, B2, C2
.align 4
LL(17):
andi. r0, N, 2
beq LL(18)
LFPDUX A1, X, INCX2
LFPDUX B1, Y, INCY2
fpmadd C1, A1, B1, C1
.align 4
LL(18):
andi. r0, N, 1
beq LL(999)
LFDUX A1, X, INCX2
LFDUX B1, Y, INCY2
fmadd C1, A1, B1, C1
b LL(999)
.align 4
/* X is aligned, Y is NOT aligned */
LL(20):
LFD B1, 0 * SIZE(Y)
sub X, X, INCX2
sub Y, Y, INCY
srawi. r0, N, 4
mtspr CTR, r0
beq- LL(25)
LFPDUX A1, X, INCX2
LFXDUX B2, Y, INCY2
LFPDUX A2, X, INCX2
LFXDUX B3, Y, INCY2
LFPDUX A3, X, INCX2
LFXDUX B4, Y, INCY2
LFPDUX A4, X, INCX2
LFXDUX B5, Y, INCY2
LFPDUX A5, X, INCX2
LFXDUX B6, Y, INCY2
LFPDUX A6, X, INCX2
LFXDUX B7, Y, INCY2
LFPDUX A7, X, INCX2
fsmr B1, B2
LFXDUX B8, Y, INCY2
fsmr B2, B3
LFPDUX A8, X, INCX2
fsmr B3, B4
bdz LL(24)
.align 4
LL(23):
fpmadd C1, A1, B1, C1
LFPDUX A1, X, INCX2
fsmr B4, B5
LFXDUX B9, Y, INCY2
fpmadd C2, A2, B2, C2
LFPDUX A2, X, INCX2
fsmr B5, B6
LFXDUX B2, Y, INCY2
fpmadd C3, A3, B3, C3
LFXDUX B3, Y, INCY2
fsmr B6, B7
LFPDUX A3, X, INCX2
fpmadd C4, A4, B4, C4
LFXDUX B4, Y, INCY2
fsmr B7, B8
LFPDUX A4, X, INCX2
fpmadd C1, A5, B5, C1
LFXDUX B5, Y, INCY2
fsmr B8, B9
LFPDUX A5, X, INCX2
fpmadd C2, A6, B6, C2
LFXDUX B6, Y, INCY2
fpmr B1, B9
LFPDUX A6, X, INCX2
fpmadd C3, A7, B7, C3
LFXDUX B7, Y, INCY2
fsmr B1, B2
LFPDUX A7, X, INCX2
fpmadd C4, A8, B8, C4
LFXDUX B8, Y, INCY2
fsmr B2, B3
LFPDUX A8, X, INCX2
fsmr B3, B4
bdnz LL(23)
.align 4
LL(24):
LFXDUX B9, Y, INCY2
fpmadd C1, A1, B1, C1
fsmr B4, B5
fpmadd C2, A2, B2, C2
fsmr B5, B6
fpmadd C3, A3, B3, C3
fsmr B6, B7
fpmadd C4, A4, B4, C4
fsmr B7, B8
fpmadd C1, A5, B5, C1
fsmr B8, B9
fpmadd C2, A6, B6, C2
fpmr B1, B9
fpmadd C3, A7, B7, C3
fpmadd C4, A8, B8, C4
.align 4
LL(25):
andi. r0, N, 15
beq LL(999)
andi. r0, N, 8
beq LL(26)
LFPDUX A1, X, INCX2
LFXDUX B2, Y, INCY2
LFPDUX A2, X, INCX2
LFXDUX B3, Y, INCY2
LFPDUX A3, X, INCX2
LFXDUX B4, Y, INCY2
LFPDUX A4, X, INCX2
LFXDUX B5, Y, INCY2
fsmr B1, B2
fsmr B2, B3
fsmr B3, B4
fsmr B4, B5
fpmadd C1, A1, B1, C1
fpmadd C2, A2, B2, C2
fpmadd C3, A3, B3, C3
fpmadd C4, A4, B4, C4
fpmr B1, B5
.align 4
LL(26):
andi. r0, N, 4
beq LL(27)
LFPDUX A1, X, INCX2
LFXDUX B2, Y, INCY2
LFPDUX A2, X, INCX2
LFXDUX B3, Y, INCY2
fsmr B1, B2
fsmr B2, B3
fpmadd C1, A1, B1, C1
fpmr B1, B3
fpmadd C2, A2, B2, C2
.align 4
LL(27):
andi. r0, N, 2
beq LL(28)
LFPDUX A1, X, INCX2
LFXDUX B2, Y, INCY2
fsmr B1, B2
fpmadd C1, A1, B1, C1
fpmr B1, B2
.align 4
LL(28):
andi. r0, N, 1
beq LL(999)
LFDUX A1, X, INCX2
fmadd C1, A1, B1, C1
b LL(999)
.align 4
/* X is not aligned, Y is aligned */
LL(30):
andi. r0, Y, 2 * SIZE - 1
bne LL(40)
LFD A1, 0 * SIZE(X)
sub X, X, INCX
sub Y, Y, INCY2
srawi. r0, N, 4
mtspr CTR, r0
beq- LL(35)
LFXDUX A2, X, INCX2
LFPDUX B1, Y, INCY2
LFXDUX A3, X, INCX2
LFPDUX B2, Y, INCY2
LFXDUX A4, X, INCX2
LFPDUX B3, Y, INCY2
LFXDUX A5, X, INCX2
LFPDUX B4, Y, INCY2
LFXDUX A6, X, INCX2
LFPDUX B5, Y, INCY2
LFXDUX A7, X, INCX2
LFPDUX B6, Y, INCY2
LFXDUX A8, X, INCX2
fsmr A1, A2
LFPDUX B7, Y, INCY2
fsmr A2, A3
LFPDUX B8, Y, INCY2
fsmr A3, A4
bdz LL(34)
.align 4
LL(33):
fpmadd C1, A1, B1, C1
LFXDUX A9, X, INCX2
fsmr A4, A5
LFPDUX B1, Y, INCY2
fpmadd C2, A2, B2, C2
LFXDUX A2, X, INCX2
fsmr A5, A6
LFPDUX B2, Y, INCY2
fpmadd C3, A3, B3, C3
LFXDUX A3, X, INCX2
fsmr A6, A7
LFPDUX B3, Y, INCY2
fpmadd C4, A4, B4, C4
LFXDUX A4, X, INCX2
fsmr A7, A8
LFPDUX B4, Y, INCY2
fpmadd C1, A5, B5, C1
LFXDUX A5, X, INCX2
fsmr A8, A9
LFPDUX B5, Y, INCY2
fpmadd C2, A6, B6, C2
LFXDUX A6, X, INCX2
fpmr A1, A9
LFPDUX B6, Y, INCY2
fpmadd C3, A7, B7, C3
LFXDUX A7, X, INCX2
fsmr A1, A2
LFPDUX B7, Y, INCY2
fpmadd C4, A8, B8, C4
LFXDUX A8, X, INCX2
fsmr A2, A3
LFPDUX B8, Y, INCY2
fsmr A3, A4
bdnz LL(33)
.align 4
LL(34):
LFXDUX A9, X, INCX2
fpmadd C1, A1, B1, C1
fsmr A4, A5
fpmadd C2, A2, B2, C2
fsmr A5, A6
fpmadd C3, A3, B3, C3
fsmr A6, A7
fpmadd C4, A4, B4, C4
fsmr A7, A8
fpmadd C1, A5, B5, C1
fsmr A8, A9
fpmadd C2, A6, B6, C2
fpmr A1, A9
fpmadd C3, A7, B7, C3
fpmadd C4, A8, B8, C4
.align 4
LL(35):
andi. r0, N, 15
beq LL(999)
andi. r0, N, 8
beq LL(36)
LFXDUX A2, X, INCX2
LFPDUX B1, Y, INCY2
LFXDUX A3, X, INCX2
LFPDUX B2, Y, INCY2
LFXDUX A4, X, INCX2
LFPDUX B3, Y, INCY2
LFXDUX A5, X, INCX2
LFPDUX B4, Y, INCY2
fsmr A1, A2
fsmr A2, A3
fsmr A3, A4
fsmr A4, A5
fpmadd C1, A1, B1, C1
fpmr A1, A5
fpmadd C2, A2, B2, C2
fpmadd C3, A3, B3, C3
fpmadd C4, A4, B4, C4
.align 4
LL(36):
andi. r0, N, 4
beq LL(37)
LFXDUX A2, X, INCX2
LFPDUX B1, Y, INCY2
LFXDUX A3, X, INCX2
LFPDUX B2, Y, INCY2
fsmr A1, A2
fsmr A2, A3
fpmadd C1, A1, B1, C1
fpmr A1, A3
fpmadd C2, A2, B2, C2
.align 4
LL(37):
andi. r0, N, 2
beq LL(38)
LFXDUX A2, X, INCX2
LFPDUX B1, Y, INCY2
fsmr A1, A2
fpmadd C1, A1, B1, C1
fpmr A1, A2
.align 4
LL(38):
andi. r0, N, 1
beq LL(999)
LFDUX B1, Y, INCY2
fmadd C1, A1, B1, C1
b LL(999)
.align 4
/* X is NOT aligned, Y is NOT aligned */
LL(40):
LFD A1, 0 * SIZE(X)
LFD B1, 0 * SIZE(Y)
sub X, X, INCX
sub Y, Y, INCY
addi N, N, -1
cmpwi cr0, N, 0
fmadd C1, A1, B1, C1
ble LL(999)
srawi. r0, N, 4
mtspr CTR, r0
beq- LL(45)
LFPDUX A1, X, INCX2
LFPDUX B1, Y, INCY2
LFPDUX A2, X, INCX2
LFPDUX B2, Y, INCY2
LFPDUX A3, X, INCX2
LFPDUX B3, Y, INCY2
LFPDUX A4, X, INCX2
LFPDUX B4, Y, INCY2
LFPDUX A5, X, INCX2
LFPDUX B5, Y, INCY2
LFPDUX A6, X, INCX2
LFPDUX B6, Y, INCY2
LFPDUX A7, X, INCX2
LFPDUX B7, Y, INCY2
LFPDUX A8, X, INCX2
LFPDUX B8, Y, INCY2
bdz LL(44)
.align 4
LL(43):
fpmadd C1, A1, B1, C1
LFPDUX A1, X, INCX2
LFPDUX B1, Y, INCY2
fpmadd C2, A2, B2, C2
LFPDUX A2, X, INCX2
LFPDUX B2, Y, INCY2
fpmadd C3, A3, B3, C3
LFPDUX A3, X, INCX2
LFPDUX B3, Y, INCY2
fpmadd C4, A4, B4, C4
LFPDUX A4, X, INCX2
LFPDUX B4, Y, INCY2
fpmadd C1, A5, B5, C1
LFPDUX A5, X, INCX2
LFPDUX B5, Y, INCY2
fpmadd C2, A6, B6, C2
LFPDUX A6, X, INCX2
LFPDUX B6, Y, INCY2
fpmadd C3, A7, B7, C3
LFPDUX A7, X, INCX2
LFPDUX B7, Y, INCY2
fpmadd C4, A8, B8, C4
LFPDUX A8, X, INCX2
LFPDUX B8, Y, INCY2
bdnz LL(43)
.align 4
LL(44):
fpmadd C1, A1, B1, C1
fpmadd C2, A2, B2, C2
fpmadd C3, A3, B3, C3
fpmadd C4, A4, B4, C4
fpmadd C1, A5, B5, C1
fpmadd C2, A6, B6, C2
fpmadd C3, A7, B7, C3
fpmadd C4, A8, B8, C4
.align 4
LL(45):
andi. r0, N, 15
beq LL(999)
andi. r0, N, 8
beq LL(46)
LFPDUX A1, X, INCX2
LFPDUX B1, Y, INCY2
LFPDUX A2, X, INCX2
LFPDUX B2, Y, INCY2
LFPDUX A3, X, INCX2
LFPDUX B3, Y, INCY2
LFPDUX A4, X, INCX2
LFPDUX B4, Y, INCY2
fpmadd C1, A1, B1, C1
fpmadd C2, A2, B2, C2
fpmadd C3, A3, B3, C3
fpmadd C4, A4, B4, C4
.align 4
LL(46):
andi. r0, N, 4
beq LL(47)
LFPDUX A1, X, INCX2
LFPDUX B1, Y, INCY2
LFPDUX A2, X, INCX2
LFPDUX B2, Y, INCY2
fpmadd C1, A1, B1, C1
fpmadd C2, A2, B2, C2
.align 4
LL(47):
andi. r0, N, 2
beq LL(48)
LFPDUX A1, X, INCX2
LFPDUX B1, Y, INCY2
fpmadd C1, A1, B1, C1
.align 4
LL(48):
andi. r0, N, 1
beq LL(999)
LFDUX A1, X, INCX2
LFDUX B1, Y, INCY2
fmadd C1, A1, B1, C1
b LL(999)
.align 4
LL(100):
#ifdef F_INTERFACE
cmpwi cr0, INCX, 0
bge+ LL(101)
subi r0, N, 1
mullw r0, r0, INCX
sub X, X, r0
.align 4
LL(101):
cmpwi cr0, INCY, 0
bge+ LL(102)
subi r0, N, 1
mullw r0, r0, INCY
sub Y, Y, r0
.align 4
LL(102):
#endif
sub X, X, INCX
sub Y, Y, INCY
srawi. r0, N, 3
mtspr CTR, r0
beq- LL(105)
LFDUX A1, X, INCX
LFDUX B1, Y, INCY
LFDUX A2, X, INCX
LFDUX B2, Y, INCY
LFDUX A3, X, INCX
LFDUX B3, Y, INCY
LFDUX A4, X, INCX
LFDUX B4, Y, INCY
LFDUX A5, X, INCX
LFDUX B5, Y, INCY
LFDUX A6, X, INCX
LFDUX B6, Y, INCY
LFDUX A7, X, INCX
LFDUX B7, Y, INCY
LFDUX A8, X, INCX
LFDUX B8, Y, INCY
bdz LL(104)
.align 4
LL(103):
fmadd C1, A1, B1, C1
LFDUX A1, X, INCX
LFDUX B1, Y, INCY
fmadd C2, A2, B2, C2
LFDUX A2, X, INCX
LFDUX B2, Y, INCY
fmadd C3, A3, B3, C3
LFDUX A3, X, INCX
LFDUX B3, Y, INCY
fmadd C4, A4, B4, C4
LFDUX A4, X, INCX
LFDUX B4, Y, INCY
fmadd C1, A5, B5, C1
LFDUX A5, X, INCX
LFDUX B5, Y, INCY
fmadd C2, A6, B6, C2
LFDUX A6, X, INCX
LFDUX B6, Y, INCY
fmadd C3, A7, B7, C3
LFDUX A7, X, INCX
LFDUX B7, Y, INCY
fmadd C4, A8, B8, C4
LFDUX A8, X, INCX
LFDUX B8, Y, INCY
bdnz LL(103)
.align 4
LL(104):
fmadd C1, A1, B1, C1
fmadd C2, A2, B2, C2
fmadd C3, A3, B3, C3
fmadd C4, A4, B4, C4
fmadd C1, A5, B5, C1
fmadd C2, A6, B6, C2
fmadd C3, A7, B7, C3
fmadd C4, A8, B8, C4
.align 4
LL(105):
andi. r0, N, 7
beq LL(999)
andi. r0, N, 4
beq LL(107)
LFDUX A1, X, INCX
LFDUX B1, Y, INCY
LFDUX A2, X, INCX
LFDUX B2, Y, INCY
LFDUX A3, X, INCX
LFDUX B3, Y, INCY
LFDUX A4, X, INCX
LFDUX B4, Y, INCY
fmadd C1, A1, B1, C1
fmadd C2, A2, B2, C2
fmadd C3, A3, B3, C3
fmadd C4, A4, B4, C4
.align 4
LL(107):
andi. r0, N, 2
beq LL(108)
LFDUX A1, X, INCX
LFDUX B1, Y, INCY
LFDUX A2, X, INCX
LFDUX B2, Y, INCY
fmadd C1, A1, B1, C1
fmadd C2, A2, B2, C2
.align 4
LL(108):
andi. r0, N, 1
beq LL(999)
LFDUX A1, X, INCX
LFDUX B1, Y, INCY
fmadd C1, A1, B1, C1
.align 4
LL(999):
li r10, 16
fpadd C1, C1, C2
fpadd C3, C3, C4
fpadd C1, C1, C3
lfpdux f20, SP, r10
lfpdux f19, SP, r10
lfpdux f18, SP, r10
lfpdux f17, SP, r10
fsmtp C2, C1
lfpdux f16, SP, r10
lfpdux f15, SP, r10
lfpdux f14, SP, r10
fadd C1, C1, C2
addi SP, SP, 16
blr
EPILOGUE