/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/
#define ASSEMBLER
#include "common.h"
#define N r3
#define X r6
#define INCX r7
#define INCX2 r4
#define XX r5
#define Y r8
#define YY r9
#define ALPHA f1
#define ALPHA_I f2
#define A1 f0
#define A2 f16
#define A3 f17
#define A4 f3
#define A5 f4
#define A6 f5
#define A7 f6
#define A8 f7
#define B1 f8
#define B2 f9
#define B3 f10
#define B4 f11
#define B5 f12
#define B6 f13
#define B7 f14
#define B8 f15
PROLOGUE
PROFCODE
li r10, -16
stfpdux f14, SP, r10
stfpdux f15, SP, r10
stfpdux f16, SP, r10
stfpdux f17, SP, r10
li r10, 0
stwu r10, -4(SP)
stwu r10, -4(SP)
stwu r10, -4(SP)
stwu r10, -4(SP)
lfpdx A1, SP, r10 # Zero clear
fsmfp ALPHA, ALPHA_I
slwi INCX, INCX, BASE_SHIFT
add INCX2, INCX, INCX
cmpwi cr0, N, 0
ble LL(999)
cmpwi cr0, INCX, SIZE
bne LL(100)
fcmpu cr7, ALPHA, A1
bne cr7, LL(50)
fscmp cr7, ALPHA, A1
bne cr7, LL(50)
andi. r0, X, 2 * SIZE - 1
bne LL(20)
sub X, X, INCX2
srawi. r0, N, 2
mtspr CTR, r0
beq- LL(15)
.align 4
LL(12):
STFPDUX A1, X, INCX2
STFPDUX A1, X, INCX2
STFPDUX A1, X, INCX2
STFPDUX A1, X, INCX2
bdnz LL(12)
.align 4
LL(15):
andi. r0, N, 3
beq LL(999)
andi. r0, N, 2
beq LL(17)
STFPDUX A1, X, INCX2
STFPDUX A1, X, INCX2
.align 4
LL(17):
andi. r0, N, 1
beq LL(999)
STFPDUX A1, X, INCX2
b LL(999)
.align 4
LL(20):
sub X, X, INCX2
STFDX A1, X, INCX2
addi X, X, SIZE
addi N, N, -1
cmpwi cr0, N, 0
ble LL(29)
srawi. r0, N, 2
mtspr CTR, r0
beq- LL(25)
.align 4
LL(22):
STFPDUX A1, X, INCX2
STFPDUX A1, X, INCX2
STFPDUX A1, X, INCX2
STFPDUX A1, X, INCX2
bdnz LL(22)
.align 4
LL(25):
andi. r0, N, 3
beq LL(29)
andi. r0, N, 2
beq LL(27)
STFPDUX A1, X, INCX2
STFPDUX A1, X, INCX2
.align 4
LL(27):
andi. r0, N, 1
beq LL(29)
STFPDUX A1, X, INCX2
.align 4
LL(29):
STFDX A1, X, INCX2
b LL(999)
.align 4
LL(50):
sub Y, X, INCX2
sub X, X, INCX2
andi. r0, X, 2 * SIZE - 1
bne LL(60)
srawi. r0, N, 3
mtspr CTR, r0
beq- LL(55)
LFPDUX A1, X, INCX2
LFPDUX A2, X, INCX2
LFPDUX A3, X, INCX2
LFPDUX A4, X, INCX2
LFPDUX A5, X, INCX2
fxpmul B1, ALPHA, A1
LFPDUX A6, X, INCX2
fxpmul B2, ALPHA, A2
LFPDUX A7, X, INCX2
fxpmul B3, ALPHA, A3
LFPDUX A8, X, INCX2
fxpmul B4, ALPHA, A4
fxpmul B5, ALPHA, A5
fxcxnpma B1, ALPHA, A1, B1
fxcxnpma B2, ALPHA, A2, B2
bdz LL(53)
.align 4
LL(52):
fxcxnpma B3, ALPHA, A3, B3
LFPDUX A1, X, INCX2
fxpmul B6, ALPHA, A6
STFPDUX B1, Y, INCX2
fxcxnpma B4, ALPHA, A4, B4
LFPDUX A2, X, INCX2
fxpmul B7, ALPHA, A7
STFPDUX B2, Y, INCX2
fxcxnpma B5, ALPHA, A5, B5
LFPDUX A3, X, INCX2
fxpmul B8, ALPHA, A8
STFPDUX B3, Y, INCX2
fxcxnpma B6, ALPHA, A6, B6
LFPDUX A4, X, INCX2
fxpmul B1, ALPHA, A1
STFPDUX B4, Y, INCX2
fxcxnpma B7, ALPHA, A7, B7
LFPDUX A5, X, INCX2
fxpmul B2, ALPHA, A2
STFPDUX B5, Y, INCX2
fxcxnpma B8, ALPHA, A8, B8
LFPDUX A6, X, INCX2
fxpmul B3, ALPHA, A3
STFPDUX B6, Y, INCX2
fxcxnpma B1, ALPHA, A1, B1
LFPDUX A7, X, INCX2
fxpmul B4, ALPHA, A4
STFPDUX B7, Y, INCX2
fxcxnpma B2, ALPHA, A2, B2
LFPDUX A8, X, INCX2
fxpmul B5, ALPHA, A5
STFPDUX B8, Y, INCX2
bdnz LL(52)
.align 4
LL(53):
fxcxnpma B3, ALPHA, A3, B3
fxpmul B6, ALPHA, A6
STFPDUX B1, Y, INCX2
fxcxnpma B4, ALPHA, A4, B4
fxpmul B7, ALPHA, A7
STFPDUX B2, Y, INCX2
fxcxnpma B5, ALPHA, A5, B5
fxpmul B8, ALPHA, A8
STFPDUX B3, Y, INCX2
fxcxnpma B6, ALPHA, A6, B6
STFPDUX B4, Y, INCX2
fxcxnpma B7, ALPHA, A7, B7
STFPDUX B5, Y, INCX2
fxcxnpma B8, ALPHA, A8, B8
STFPDUX B6, Y, INCX2
STFPDUX B7, Y, INCX2
STFPDUX B8, Y, INCX2
.align 4
LL(55):
andi. r0, N, 7
beq LL(999)
andi. r0, N, 4
beq LL(56)
LFPDUX A1, X, INCX2
LFPDUX A2, X, INCX2
LFPDUX A3, X, INCX2
LFPDUX A4, X, INCX2
fxpmul B1, ALPHA, A1
fxpmul B2, ALPHA, A2
fxpmul B3, ALPHA, A3
fxpmul B4, ALPHA, A4
fxcxnpma B1, ALPHA, A1, B1
fxcxnpma B2, ALPHA, A2, B2
fxcxnpma B3, ALPHA, A3, B3
fxcxnpma B4, ALPHA, A4, B4
STFPDUX B1, Y, INCX2
STFPDUX B2, Y, INCX2
STFPDUX B3, Y, INCX2
STFPDUX B4, Y, INCX2
.align 4
LL(56):
andi. r0, N, 2
beq LL(57)
LFPDUX A1, X, INCX2
LFPDUX A2, X, INCX2
fxpmul B1, ALPHA, A1
fxpmul B2, ALPHA, A2
fxcxnpma B1, ALPHA, A1, B1
fxcxnpma B2, ALPHA, A2, B2
STFPDUX B1, Y, INCX2
STFPDUX B2, Y, INCX2
.align 4
LL(57):
andi. r0, N, 1
beq LL(999)
LFPDUX A1, X, INCX2
fxpmul B1, ALPHA, A1
fxcxnpma B1, ALPHA, A1, B1
STFPDUX B1, Y, INCX2
b LL(999)
.align 4
LL(60):
addi XX, X, SIZE
addi YY, Y, SIZE
srawi. r0, N, 2
mtspr CTR, r0
beq- LL(65)
LFDUX A1, X, INCX2
LFDUX A2, XX, INCX2
LFDUX A3, X, INCX2
LFDUX A4, XX, INCX2
LFDUX A5, X, INCX2
fmul B1, ALPHA, A1
LFDUX A6, XX, INCX2
fmul B2, ALPHA_I, A1
LFDUX A7, X, INCX2
fmul B3, ALPHA, A3
LFDUX A8, XX, INCX2
fmul B4, ALPHA_I, A3
fmul B5, ALPHA, A5
fnmsub B1, ALPHA_I, A2, B1
fmadd B2, ALPHA , A2, B2
bdz LL(63)
.align 4
LL(62):
fnmsub B3, ALPHA_I, A4, B3
LFDUX A1, X, INCX2
fmul B6, ALPHA_I, A5
STFDUX B1, Y, INCX2
fmadd B4, ALPHA , A4, B4
LFDUX A2, XX, INCX2
fmul B7, ALPHA, A7
STFDUX B2, YY, INCX2
fnmsub B5, ALPHA_I, A6, B5
LFDUX A3, X, INCX2
fmul B8, ALPHA_I, A7
STFDUX B3, Y, INCX2
fmadd B6, ALPHA , A6, B6
LFDUX A4, XX, INCX2
fmul B1, ALPHA, A1
STFDUX B4, YY, INCX2
fnmsub B7, ALPHA_I, A8, B7
LFDUX A5, X, INCX2
fmul B2, ALPHA_I, A1
STFDUX B5, Y, INCX2
fmadd B8, ALPHA , A8, B8
LFDUX A6, XX, INCX2
fmul B3, ALPHA, A3
STFDUX B6, YY, INCX2
fnmsub B1, ALPHA_I, A2, B1
LFDUX A7, X, INCX2
fmul B4, ALPHA_I, A3
STFDUX B7, Y, INCX2
fmadd B2, ALPHA , A2, B2
LFDUX A8, XX, INCX2
fmul B5, ALPHA, A5
STFDUX B8, YY, INCX2
bdnz LL(62)
.align 4
LL(63):
fnmsub B3, ALPHA_I, A4, B3
fmul B6, ALPHA_I, A5
STFDUX B1, Y, INCX2
fmadd B4, ALPHA , A4, B4
fmul B7, ALPHA, A7
STFDUX B2, YY, INCX2
fnmsub B5, ALPHA_I, A6, B5
fmul B8, ALPHA_I, A7
STFDUX B3, Y, INCX2
fmadd B6, ALPHA , A6, B6
STFDUX B4, YY, INCX2
fnmsub B7, ALPHA_I, A8, B7
STFDUX B5, Y, INCX2
fmadd B8, ALPHA , A8, B8
STFDUX B6, YY, INCX2
STFDUX B7, Y, INCX2
STFDUX B8, YY, INCX2
.align 4
LL(65):
andi. r0, N, 3
beq LL(999)
andi. r0, N, 2
beq LL(67)
LFDUX A1, X, INCX2
LFDUX A2, XX, INCX2
LFDUX A3, X, INCX2
LFDUX A4, XX, INCX2
fmul B1, ALPHA, A1
fmul B2, ALPHA, A2
fmul B3, ALPHA, A3
fmul B4, ALPHA, A4
fnmsub B1, ALPHA_I, A2, B1
fmadd B2, ALPHA_I, A1, B2
fnmsub B3, ALPHA_I, A4, B3
fmadd B4, ALPHA_I, A3, B4
STFDUX B1, Y, INCX2
STFDUX B2, YY, INCX2
STFDUX B3, Y, INCX2
STFDUX B4, YY, INCX2
.align 4
LL(67):
andi. r0, N, 1
beq LL(999)
LFDUX A1, X, INCX2
LFDUX A2, XX, INCX2
fmul B1, ALPHA, A1
fmul B2, ALPHA, A2
fnmsub B1, ALPHA_I, A2, B1
fmadd B2, ALPHA_I, A1, B2
STFDUX B1, Y, INCX2
STFDUX B2, YY, INCX2
b LL(999)
.align 4
LL(100):
fcmpu cr7, ALPHA, A1
bne cr7, LL(150)
fscmp cr7, ALPHA, A1
bne cr7, LL(150)
andi. r0, X, 2 * SIZE - 1
bne LL(120)
sub X, X, INCX2
srawi. r0, N, 2
mtspr CTR, r0
beq- LL(115)
.align 4
LL(112):
STFPDUX A1, X, INCX2
STFPDUX A1, X, INCX2
STFPDUX A1, X, INCX2
STFPDUX A1, X, INCX2
bdnz LL(112)
.align 4
LL(115):
andi. r0, N, 3
beq LL(999)
andi. r0, N, 2
beq LL(117)
STFPDUX A1, X, INCX2
STFPDUX A1, X, INCX2
.align 4
LL(117):
andi. r0, N, 1
beq LL(999)
STFPDUX A1, X, INCX2
b LL(999)
.align 4
LL(120):
subi INCX2, INCX2, SIZE
li INCX, SIZE
sub X, X, INCX2
srawi. r0, N, 2
mtspr CTR, r0
beq- LL(125)
.align 4
LL(122):
STFDUX A1, X, INCX2
STFDUX A1, X, INCX
STFDUX A1, X, INCX2
STFDUX A1, X, INCX
STFDUX A1, X, INCX2
STFDUX A1, X, INCX
STFDUX A1, X, INCX2
STFDUX A1, X, INCX
bdnz LL(122)
.align 4
LL(125):
andi. r0, N, 3
beq LL(999)
andi. r0, N, 2
beq LL(127)
STFDUX A1, X, INCX2
STFDUX A1, X, INCX
STFDUX A1, X, INCX2
STFDUX A1, X, INCX
.align 4
LL(127):
andi. r0, N, 1
beq LL(999)
STFDUX A1, X, INCX2
STFDUX A1, X, INCX
b LL(999)
.align 4
LL(150):
sub Y, X, INCX2
sub X, X, INCX2
andi. r0, X, 2 * SIZE - 1
bne LL(160)
srawi. r0, N, 3
mtspr CTR, r0
beq- LL(155)
LFPDUX A1, X, INCX2
LFPDUX A2, X, INCX2
LFPDUX A3, X, INCX2
LFPDUX A4, X, INCX2
LFPDUX A5, X, INCX2
fxpmul B1, ALPHA, A1
LFPDUX A6, X, INCX2
fxpmul B2, ALPHA, A2
LFPDUX A7, X, INCX2
fxpmul B3, ALPHA, A3
LFPDUX A8, X, INCX2
fxpmul B4, ALPHA, A4
fxpmul B5, ALPHA, A5
fxcxnpma B1, ALPHA, A1, B1
fxcxnpma B2, ALPHA, A2, B2
bdz LL(153)
.align 4
LL(152):
fxcxnpma B3, ALPHA, A3, B3
LFPDUX A1, X, INCX2
fxpmul B6, ALPHA, A6
STFPDUX B1, Y, INCX2
fxcxnpma B4, ALPHA, A4, B4
LFPDUX A2, X, INCX2
fxpmul B7, ALPHA, A7
STFPDUX B2, Y, INCX2
fxcxnpma B5, ALPHA, A5, B5
LFPDUX A3, X, INCX2
fxpmul B8, ALPHA, A8
STFPDUX B3, Y, INCX2
fxcxnpma B6, ALPHA, A6, B6
LFPDUX A4, X, INCX2
fxpmul B1, ALPHA, A1
STFPDUX B4, Y, INCX2
fxcxnpma B7, ALPHA, A7, B7
LFPDUX A5, X, INCX2
fxpmul B2, ALPHA, A2
STFPDUX B5, Y, INCX2
fxcxnpma B8, ALPHA, A8, B8
LFPDUX A6, X, INCX2
fxpmul B3, ALPHA, A3
STFPDUX B6, Y, INCX2
fxcxnpma B1, ALPHA, A1, B1
LFPDUX A7, X, INCX2
fxpmul B4, ALPHA, A4
STFPDUX B7, Y, INCX2
fxcxnpma B2, ALPHA, A2, B2
LFPDUX A8, X, INCX2
fxpmul B5, ALPHA, A5
STFPDUX B8, Y, INCX2
bdnz LL(152)
.align 4
LL(153):
fxcxnpma B3, ALPHA, A3, B3
fxpmul B6, ALPHA, A6
STFPDUX B1, Y, INCX2
fxcxnpma B4, ALPHA, A4, B4
fxpmul B7, ALPHA, A7
STFPDUX B2, Y, INCX2
fxcxnpma B5, ALPHA, A5, B5
fxpmul B8, ALPHA, A8
STFPDUX B3, Y, INCX2
fxcxnpma B6, ALPHA, A6, B6
STFPDUX B4, Y, INCX2
fxcxnpma B7, ALPHA, A7, B7
STFPDUX B5, Y, INCX2
fxcxnpma B8, ALPHA, A8, B8
STFPDUX B6, Y, INCX2
STFPDUX B7, Y, INCX2
STFPDUX B8, Y, INCX2
.align 4
LL(155):
andi. r0, N, 7
beq LL(999)
andi. r0, N, 4
beq LL(156)
LFPDUX A1, X, INCX2
LFPDUX A2, X, INCX2
LFPDUX A3, X, INCX2
LFPDUX A4, X, INCX2
fxpmul B1, ALPHA, A1
fxpmul B2, ALPHA, A2
fxpmul B3, ALPHA, A3
fxpmul B4, ALPHA, A4
fxcxnpma B1, ALPHA, A1, B1
fxcxnpma B2, ALPHA, A2, B2
fxcxnpma B3, ALPHA, A3, B3
fxcxnpma B4, ALPHA, A4, B4
STFPDUX B1, Y, INCX2
STFPDUX B2, Y, INCX2
STFPDUX B3, Y, INCX2
STFPDUX B4, Y, INCX2
.align 4
LL(156):
andi. r0, N, 2
beq LL(157)
LFPDUX A1, X, INCX2
LFPDUX A2, X, INCX2
fxpmul B1, ALPHA, A1
fxpmul B2, ALPHA, A2
fxcxnpma B1, ALPHA, A1, B1
fxcxnpma B2, ALPHA, A2, B2
STFPDUX B1, Y, INCX2
STFPDUX B2, Y, INCX2
.align 4
LL(157):
andi. r0, N, 1
beq LL(999)
LFPDUX A1, X, INCX2
fxpmul B1, ALPHA, A1
fxcxnpma B1, ALPHA, A1, B1
STFPDUX B1, Y, INCX2
b LL(999)
.align 4
LL(160):
addi XX, X, SIZE
addi YY, Y, SIZE
srawi. r0, N, 2
mtspr CTR, r0
beq- LL(165)
LFDUX A1, X, INCX2
LFDUX A2, XX, INCX2
LFDUX A3, X, INCX2
LFDUX A4, XX, INCX2
LFDUX A5, X, INCX2
fmul B1, ALPHA, A1
LFDUX A6, XX, INCX2
fmul B2, ALPHA_I, A1
LFDUX A7, X, INCX2
fmul B3, ALPHA, A3
LFDUX A8, XX, INCX2
fmul B4, ALPHA_I, A3
fmul B5, ALPHA, A5
fnmsub B1, ALPHA_I, A2, B1
fmadd B2, ALPHA , A2, B2
bdz LL(163)
.align 4
LL(162):
fnmsub B3, ALPHA_I, A4, B3
LFDUX A1, X, INCX2
fmul B6, ALPHA_I, A5
STFDUX B1, Y, INCX2
fmadd B4, ALPHA , A4, B4
LFDUX A2, XX, INCX2
fmul B7, ALPHA, A7
STFDUX B2, YY, INCX2
fnmsub B5, ALPHA_I, A6, B5
LFDUX A3, X, INCX2
fmul B8, ALPHA_I, A7
STFDUX B3, Y, INCX2
fmadd B6, ALPHA , A6, B6
LFDUX A4, XX, INCX2
fmul B1, ALPHA, A1
STFDUX B4, YY, INCX2
fnmsub B7, ALPHA_I, A8, B7
LFDUX A5, X, INCX2
fmul B2, ALPHA_I, A1
STFDUX B5, Y, INCX2
fmadd B8, ALPHA , A8, B8
LFDUX A6, XX, INCX2
fmul B3, ALPHA, A3
STFDUX B6, YY, INCX2
fnmsub B1, ALPHA_I, A2, B1
LFDUX A7, X, INCX2
fmul B4, ALPHA_I, A3
STFDUX B7, Y, INCX2
fmadd B2, ALPHA , A2, B2
LFDUX A8, XX, INCX2
fmul B5, ALPHA, A5
STFDUX B8, YY, INCX2
bdnz LL(162)
.align 4
LL(163):
fnmsub B3, ALPHA_I, A4, B3
fmul B6, ALPHA_I, A5
STFDUX B1, Y, INCX2
fmadd B4, ALPHA , A4, B4
fmul B7, ALPHA, A7
STFDUX B2, YY, INCX2
fnmsub B5, ALPHA_I, A6, B5
fmul B8, ALPHA_I, A7
STFDUX B3, Y, INCX2
fmadd B6, ALPHA , A6, B6
STFDUX B4, YY, INCX2
fnmsub B7, ALPHA_I, A8, B7
STFDUX B5, Y, INCX2
fmadd B8, ALPHA , A8, B8
STFDUX B6, YY, INCX2
STFDUX B7, Y, INCX2
STFDUX B8, YY, INCX2
.align 4
LL(165):
andi. r0, N, 3
beq LL(999)
andi. r0, N, 2
beq LL(167)
LFDUX A1, X, INCX2
LFDUX A2, XX, INCX2
LFDUX A3, X, INCX2
LFDUX A4, XX, INCX2
fmul B1, ALPHA, A1
fmul B2, ALPHA, A2
fmul B3, ALPHA, A3
fmul B4, ALPHA, A4
fnmsub B1, ALPHA_I, A2, B1
fmadd B2, ALPHA_I, A1, B2
fnmsub B3, ALPHA_I, A4, B3
fmadd B4, ALPHA_I, A3, B4
STFDUX B1, Y, INCX2
STFDUX B2, YY, INCX2
STFDUX B3, Y, INCX2
STFDUX B4, YY, INCX2
.align 4
LL(167):
andi. r0, N, 1
beq LL(999)
LFDUX A1, X, INCX2
LFDUX A2, XX, INCX2
fmul B1, ALPHA, A1
fmul B2, ALPHA, A2
fnmsub B1, ALPHA_I, A2, B1
fmadd B2, ALPHA_I, A1, B2
STFDUX B1, Y, INCX2
STFDUX B2, YY, INCX2
.align 4
LL(999):
li r10, 16
lfpdux f17, SP, r10
lfpdux f16, SP, r10
lfpdux f15, SP, r10
lfpdux f14, SP, r10
addi SP, SP, 16
blr
EPILOGUE