/*********************************************************************/

/* Copyright 2005-2010 The University of Texas at Austin.           */

/* All rights reserved.                                              */

/*                                                                   */

/* Redistribution and use in source and binary forms, with or        */

/* without modification, are permitted provided that the following   */

/* conditions are met:                                               */

/*                                                                   */

/*   1. Redistributions of source code must retain the above         */

/*      copyright notice, this list of conditions and the following  */

/*      disclaimer.                                                  */

/*                                                                   */

/*   2. Redistributions in binary form must reproduce the above      */

/*      copyright notice, this list of conditions and the following  */

/*      disclaimer in the documentation and/or other materials       */

/*      provided with the distribution.                              */

/*                                                                   */

/*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

/*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

/*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

/*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

/*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

/*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

/*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

/*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

/*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

/*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

/*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

/*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

/*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

/*    POSSIBILITY OF SUCH DAMAGE.                                    */

/*                                                                   */

/* The views and conclusions contained in the software and           */

/* documentation are those of the authors and should not be          */

/* interpreted as representing official policies, either expressed   */

/* or implied, of The University of Texas at Austin.                 */

/*********************************************************************/

#define ASSEMBLER

#include "common.h"

#define APREFETCHSIZE 24

#define APREFETCH_CATEGORY 0

#define M	%i0

#define N	%i1

#define K	%i2

#define A	%i5

#define B	%i3

#define C	%i4

#define LDC	%o0

#define AO	%o1

#define BO	%o2

#define I	%o3

#define J	%o4

#define L	%o5

#define C1	%l0

#define C2	%l1

#define C3	%l2

#define C4	%l3

#define OFFSET	%l4

#define	KK	%l5

#define TEMP1	%l6

#define TEMP2	%l7

#define AORIG	%o7

#ifdef DOUBLE

#define c01	%f0

#define c02	%f2

#define c03	%f4

#define c04	%f6

#define c05	%f8

#define c06	%f10

#define c07	%f12

#define c08	%f14

#define c09	%f16

#define c10	%f18

#define c11	%f20

#define c12	%f22

#define c13	%f24

#define c14	%f26

#define c15	%f28

#define c16	%f30

#define a1	%f32

#define a2	%f34

#define a3	%f36

#define a4	%f38

#define a5	%f40

#define b1	%f42

#define b2	%f44

#define b3	%f46

#define b4	%f48

#define b5	%f50

#define b6	%f52

#define b7	%f54

#define b8	%f56

#define b9	%f58

#define cc01	0

#define cc02	2

#define cc03	4

#define cc04	6

#define cc05	8

#define cc06	10

#define cc07	12

#define cc08	14

#define cc09	16

#define cc10	18

#define cc11	20

#define cc12	22

#define cc13	24

#define cc14	26

#define cc15	28

#define cc16	30

#define aa1	 1

#define aa2	 3

#define aa3	 5

#define aa4	 7

#define aa5	 9

#define bb1	11

#define bb2	13

#define bb3	15

#define bb4	17

#define bb5	19

#define bb6	21

#define bb7	23

#define bb8	25

#define bb9	27

#else

#define c01	%f0

#define c02	%f1

#define c03	%f2

#define c04	%f3

#define c05	%f4

#define c06	%f5

#define c07	%f6

#define c08	%f7

#define c09	%f8

#define c10	%f9

#define c11	%f10

#define c12	%f11

#define c13	%f12

#define c14	%f13

#define c15	%f14

#define c16	%f15

#define a1	%f16

#define a2	%f17

#define a3	%f18

#define a4	%f19

#define a5	%f20

#define b1	%f21

#define b2	%f22

#define b3	%f23

#define b4	%f24

#define b5	%f25

#define b6	%f26

#define b7	%f27

#define b8	%f28

#define b9	%f29

#define cc01	0

#define cc02	1

#define cc03	2

#define cc04	3

#define cc05	4

#define cc06	5

#define cc07	6

#define cc08	7

#define cc09	8

#define cc10	9

#define cc11	10

#define cc12	11

#define cc13	12

#define cc14	13

#define cc15	14

#define cc16	15

#define aa1	16

#define aa2	17

#define aa3	18

#define aa4	19

#define aa5	20

#define bb1	21

#define bb2	22

#define bb3	23

#define bb4	24

#define bb5	25

#define bb6	26

#define bb7	27

#define bb8	28

#define bb9	29

#endif

#ifndef CONJ

#define FMADD1	FMADD

#define FMADD2	FMADD

#define FMADD3	FMADD

#define FMADD4	FNMSUB

#else

#if defined(LN) || defined(LT)

#define FMADD1	FMADD

#define FMADD2	FNMSUB

#define FMADD3	FMADD

#define FMADD4	FMADD

#endif

#if defined(RN) || defined(RT)

#define FMADD1	FMADD

#define FMADD2	FMADD

#define FMADD3	FNMSUB

#define FMADD4	FMADD

#endif

#endif

        .register %g2, #scratch

        .register %g3, #scratch

	PROLOGUE

	SAVESP

#ifndef __64BIT__

#ifdef DOUBLE

	ld	[%sp + STACK_START + 32], A

	ld	[%sp + STACK_START + 36], B

	ld	[%sp + STACK_START + 40], C

	ld	[%sp + STACK_START + 44], LDC

	ld	[%sp + STACK_START + 48], OFFSET

#else

	ld	[%sp + STACK_START + 28], B

	ld	[%sp + STACK_START + 32], C

	ld	[%sp + STACK_START + 36], LDC

	ld	[%sp + STACK_START + 40], OFFSET

#endif

#else

	ldx	[%sp + STACK_START + 56], B

	ldx	[%sp + STACK_START + 64], C

	ldx	[%sp + STACK_START + 72], LDC

	ldx	[%sp + STACK_START + 80], OFFSET

#endif

	cmp	M, 0

	ble,pn	%icc, .LL999

	nop

	sll	LDC, ZBASE_SHIFT, LDC

#ifdef LN

	smul	M, K, TEMP1

	sll	TEMP1, ZBASE_SHIFT, TEMP1

	add	A, TEMP1, A

	sll	M, ZBASE_SHIFT, TEMP1

	add	C, TEMP1, C

#endif

#ifdef RN

	neg	OFFSET, KK

#endif

#ifdef RT

	smul	N, K, TEMP1

	sll	TEMP1, ZBASE_SHIFT, TEMP1

	add	B, TEMP1, B

	smul	N, LDC, TEMP1

	add	C, TEMP1, C

	sub	N, OFFSET, KK

#endif

	and	N, 1, J

	cmp	J, 0

	ble,pn	%icc, .LL20

	nop

#ifdef RT

	sll	K, ZBASE_SHIFT, TEMP1

	sub	B, TEMP1, B

#endif

#ifndef RT

	mov	C,  C1

	add	C,  LDC, C

#else

	sub	C,  LDC, C1

	sub	C,  LDC, C

#endif

#ifdef LN

	add	M, OFFSET, KK

#endif

#ifdef LT

	mov	OFFSET, KK

#endif

#if defined(LN) || defined(RT)

	mov	A, AORIG

#else

	mov	A, AO

#endif

	mov	M, I

	.align 4

.LL32:

#if defined(LT) || defined(RN)

	mov	B, BO

#else

#ifdef LN

	sll	K,  ZBASE_SHIFT, TEMP1

	sub	AORIG, TEMP1, AORIG

#endif

	sll	KK, ZBASE_SHIFT + 0, TEMP1

	add	AORIG, TEMP1, AO

	add	B,     TEMP1, BO

#endif

	LDF	[AO +  0 * SIZE], a1

	LDF	[AO +  1 * SIZE], a2

	LDF	[AO +  2 * SIZE], a3

	LDF	[AO +  3 * SIZE], a4

	LDF	[BO +  0 * SIZE], b1

	LDF	[BO +  1 * SIZE], b2

	LDF	[BO +  2 * SIZE], b3

	FCLR	(cc01)

	LDF	[BO +  3 * SIZE], b4

	FCLR	(cc02)

	LDF	[BO +  4 * SIZE], b5

	FCLR	(cc03)

	LDF	[BO +  5 * SIZE], b6

	FCLR	(cc04)

	LDF	[BO +  6 * SIZE], b7

	FCLR	(cc05)

	LDF	[BO +  7 * SIZE], b8

	FCLR	(cc06)

	prefetch [C1 + 2 * SIZE], 3

	FCLR	(cc07)

#if defined(LT) || defined(RN)

	sra	KK, 2, L

#else

	sub	K, KK, L

	sra	L,  2, L

#endif

	cmp	L,  0

	ble,pn	%icc, .LL35

	FCLR	(cc08)

	.align 4

.LL33:

	FMADD1	(aa1, bb1, cc01, cc01)

	prefetch [AO + (APREFETCHSIZE +  0) * SIZE], APREFETCH_CATEGORY

	FMADD2	(aa2, bb1, cc02, cc02)

	LDF	[BO +  8 * SIZE], b1

	FMADD3	(aa1, bb2, cc03, cc03)

	LDF	[AO +  4 * SIZE], a1

	FMADD4	(aa2, bb2, cc04, cc04)

	LDF	[AO +  5 * SIZE], a2

	FMADD1	(aa3, bb3, cc01, cc01)

	LDF	[BO +  9 * SIZE], b2

	FMADD2	(aa4, bb3, cc02, cc02)

	LDF	[BO + 10 * SIZE], b3

	FMADD3	(aa3, bb4, cc03, cc03)

	LDF	[AO +  6 * SIZE], a3

	FMADD4	(aa4, bb4, cc04, cc04)

	LDF	[AO +  7 * SIZE], a4

	FMADD1	(aa1, bb5, cc01, cc01)

	LDF	[BO + 11 * SIZE], b4

	FMADD2	(aa2, bb5, cc02, cc02)

	LDF	[BO + 12 * SIZE], b5

	FMADD3	(aa1, bb6, cc03, cc03)

	LDF	[AO +  8 * SIZE], a1

	FMADD4	(aa2, bb6, cc04, cc04)

	LDF	[AO +  9 * SIZE], a2

	FMADD1	(aa3, bb7, cc01, cc01)

	LDF	[BO + 13 * SIZE], b6

	FMADD2	(aa4, bb7, cc02, cc02)

	LDF	[BO + 14 * SIZE], b7

	FMADD3	(aa3, bb8, cc03, cc03)

	LDF	[AO + 10 * SIZE], a3

	FMADD4	(aa4, bb8, cc04, cc04)

	LDF	[AO + 11 * SIZE], a4

	add	AO,  8 * SIZE, AO

	add	L, -1, L

	add	BO,  8 * SIZE, BO

	cmp	L, 0

	bg,pt	%icc, .LL33

	LDF	[BO +  7 * SIZE], b8

	.align 4

.LL35:

#if defined(LT) || defined(RN)

	and	KK, 3, L

#else

	sub	K, KK, L

	and	L,  3, L

#endif

	cmp	L,  0

	ble,a,pn %icc, .LL38

	nop

	.align 4

.LL37:

	FMADD1	(aa1, bb1, cc01, cc01)

	add	L, -1, L

	FMADD2	(aa2, bb1, cc02, cc02)

	LDF	[BO + 2 * SIZE], b1

	FMADD3	(aa1, bb2, cc03, cc03)

	LDF	[AO + 2 * SIZE], a1

	FMADD4	(aa2, bb2, cc04, cc04)

	LDF	[AO + 3 * SIZE], a2

	add	AO, 2 * SIZE, AO

	cmp	L, 0

	add	BO, 2 * SIZE, BO

	bg,pt	%icc, .LL37

	LDF	[BO + 1 * SIZE], b2

	.align 4

.LL38:

	FADD	c01, c04, c01

	FADD	c02, c03, c02

#if defined(LN) || defined(RT)

	sub	KK, 1, TEMP1

	sll	TEMP1, ZBASE_SHIFT, TEMP1

	add	AORIG, TEMP1, AO

	add	B,     TEMP1, BO

#endif

#if defined(LN) || defined(LT)

	LDF	[BO +  0 * SIZE], a1

	LDF	[BO +  1 * SIZE], a2

#else

	LDF	[AO +  0 * SIZE], a1

	LDF	[AO +  1 * SIZE], a2

#endif

	FSUB	a1, c01, c01

	FSUB	a2, c02, c02

#if defined(LN) || defined(LT)

	LDF	[AO +  0 * SIZE], a1

	LDF	[AO +  1 * SIZE], a2

#else

	LDF	[BO +  0 * SIZE], a1

	LDF	[BO +  1 * SIZE], a2

#endif

	FMUL	a1, c01, b1

	FMUL	a2, c01, b2

#ifndef CONJ

	FNMSUB	(aa2, cc02, bb1, cc01)

	FMADD	(aa1, cc02, bb2, cc02)

#else

	FMADD	(aa2, cc02, bb1, cc01)

	FMSUB	(aa1, cc02, bb2, cc02)

#endif

#ifdef LN

	add	C1, -2 * SIZE, C1

#endif

#if defined(LN) || defined(LT)

	STF	c01, [BO +  0 * SIZE]

	STF	c02, [BO +  1 * SIZE]

#else

	STF	c01, [AO +  0 * SIZE]

	STF	c02, [AO +  1 * SIZE]

#endif

	STF	c01, [C1 + 0 * SIZE]

	STF	c02, [C1 + 1 * SIZE]

#ifndef LN

	add	C1, 2 * SIZE, C1

#endif

#ifdef RT

	sll	K, ZBASE_SHIFT, TEMP1

	add	AORIG, TEMP1, AORIG

#endif

#if defined(LT) || defined(RN)

	sub	K, KK, TEMP1

	sll	TEMP1, ZBASE_SHIFT, TEMP1

	add	AO, TEMP1, AO

	add	BO, TEMP1, BO

#endif

#ifdef LT

	add	KK, 1, KK

#endif

#ifdef LN

	sub	KK, 1, KK

#endif

	add	I, -1, I

	cmp	I, 0

	bg,pt	%icc, .LL32

	nop

#ifdef LN

	sll	K, ZBASE_SHIFT, TEMP1

	add	B, TEMP1, B

#endif

#if defined(LT) || defined(RN)

	mov	BO, B

#endif

#ifdef RN

	add	KK, 1, KK

#endif

#ifdef RT

	sub	KK, 1, KK

#endif

	.align 4

.LL20:

	and	N, 2, J

	cmp	J, 0

	ble,pn	%icc, .LL30

	nop

#ifdef RT

	sll	K, ZBASE_SHIFT + 1, TEMP1

	sub	B, TEMP1, B

#endif

#ifndef RT

	mov	C,  C1

	add	C,  LDC, C2

	add	C2, LDC, C

#else

	sub	C,  LDC, C2

	sub	C2, LDC, C1

	sub	C2, LDC, C

#endif

#ifdef LN

	add	M, OFFSET, KK

#endif

#ifdef LT

	mov	OFFSET, KK

#endif

#if defined(LN) || defined(RT)

	mov	A, AORIG

#else

	mov	A, AO

#endif

	mov	M, I

	.align 4

.LL22:

#if defined(LT) || defined(RN)

	mov	B, BO

#else

#ifdef LN

	sll	K,  ZBASE_SHIFT, TEMP1

	sub	AORIG, TEMP1, AORIG

#endif

	sll	KK, ZBASE_SHIFT + 0, TEMP1

	sll	KK, ZBASE_SHIFT + 1, TEMP2

	add	AORIG, TEMP1, AO

	add	B,     TEMP2, BO

#endif

	LDF	[AO +  0 * SIZE], a1

	LDF	[AO +  1 * SIZE], a2

	LDF	[BO +  0 * SIZE], b1

	LDF	[BO +  1 * SIZE], b2

	LDF	[BO +  2 * SIZE], b3

	LDF	[BO +  3 * SIZE], b4

	LDF	[BO +  4 * SIZE], b5

	FCLR	(cc01)

	LDF	[BO +  5 * SIZE], b6

	FCLR	(cc02)

	LDF	[BO +  6 * SIZE], b7

	FCLR	(cc03)

	LDF	[BO +  7 * SIZE], b8

	FCLR	(cc04)

	LDF	[BO +  8 * SIZE], b9

	FCLR	(cc05)

	prefetch [C1 + 2 * SIZE], 3

	FCLR	(cc06)

	prefetch [C2 + 2 * SIZE], 3

	FCLR	(cc07)

#if defined(LT) || defined(RN)

	sra	KK, 2, L

#else

	sub	K, KK, L

	sra	L,  2, L

#endif

	cmp	L,  0

	ble,pn	%icc, .LL25

	FCLR	(cc08)

	.align 4

.LL23:

	FMADD1	(aa1, bb1, cc01, cc01)

	LDF	[AO +  2 * SIZE], a3

	FMADD2	(aa2, bb1, cc02, cc02)

	LDF	[AO +  3 * SIZE], a4

	FMADD3	(aa1, bb2, cc03, cc03)

	LDF	[BO + 16 * SIZE], b1

	FMADD4	(aa2, bb2, cc04, cc04)

	LDF	[BO +  9 * SIZE], b2

	FMADD1	(aa1, bb3, cc05, cc05)

	prefetch [AO + (APREFETCHSIZE +  0) * SIZE], APREFETCH_CATEGORY

	FMADD2	(aa2, bb3, cc06, cc06)

	add	L, -1, L

	FMADD3	(aa1, bb4, cc07, cc07)

	LDF	[BO + 10 * SIZE], b3

	FMADD4	(aa2, bb4, cc08, cc08)

	LDF	[BO + 11 * SIZE], b4

	FMADD1	(aa3, bb5, cc01, cc01)

	LDF	[AO +  4 * SIZE], a1

	FMADD2	(aa4, bb5, cc02, cc02)

	LDF	[AO +  5 * SIZE], a2

	FMADD3	(aa3, bb6, cc03, cc03)

	LDF	[BO + 12 * SIZE], b5

	FMADD4	(aa4, bb6, cc04, cc04)

	LDF	[BO + 13 * SIZE], b6

	FMADD1	(aa3, bb7, cc05, cc05)

	cmp	L, 0

	FMADD2	(aa4, bb7, cc06, cc06)

	add	AO,  8 * SIZE, AO

	FMADD3	(aa3, bb8, cc07, cc07)

	LDF	[BO + 14 * SIZE], b7

	FMADD4	(aa4, bb8, cc08, cc08)

	LDF	[BO + 15 * SIZE], b8

	FMADD1	(aa1, bb9, cc01, cc01)

	LDF	[AO -  2 * SIZE], a3

	FMADD2	(aa2, bb9, cc02, cc02)

	LDF	[AO -  1 * SIZE], a4

	FMADD3	(aa1, bb2, cc03, cc03)

	LDF	[BO + 24 * SIZE], b9

	FMADD4	(aa2, bb2, cc04, cc04)

	LDF	[BO + 17 * SIZE], b2

	FMADD1	(aa1, bb3, cc05, cc05)

	add	BO, 16 * SIZE, BO

	FMADD2	(aa2, bb3, cc06, cc06)

	nop

	FMADD3	(aa1, bb4, cc07, cc07)

	LDF	[BO +  2 * SIZE], b3

	FMADD4	(aa2, bb4, cc08, cc08)

	LDF	[BO +  3 * SIZE], b4

	FMADD1	(aa3, bb5, cc01, cc01)

	LDF	[AO +  0 * SIZE], a1

	FMADD2	(aa4, bb5, cc02, cc02)

	LDF	[AO +  1 * SIZE], a2

	FMADD3	(aa3, bb6, cc03, cc03)

	LDF	[BO +  4 * SIZE], b5

	FMADD4	(aa4, bb6, cc04, cc04)

	LDF	[BO +  5 * SIZE], b6

	FMADD1	(aa3, bb7, cc05, cc05)

	nop

	FMADD2	(aa4, bb7, cc06, cc06)

	LDF	[BO +  6 * SIZE], b7

	FMADD3	(aa3, bb8, cc07, cc07)

	FMADD4	(aa4, bb8, cc08, cc08)

	bg,pt	%icc, .LL23

	LDF	[BO +  7 * SIZE], b8

	.align 4

.LL25:

#if defined(LT) || defined(RN)

	and	KK, 3, L

#else

	sub	K, KK, L

	and	L,  3, L

#endif

	cmp	L,  0

	ble,a,pn %icc, .LL28

	nop

	.align 4

.LL27:

	FMADD1	(aa1, bb1, cc01, cc01)

	add	L, -1, L

	FMADD2	(aa2, bb1, cc02, cc02)

	LDF	[BO + 4 * SIZE], b1

	FMADD3	(aa1, bb2, cc03, cc03)

	add	AO, 2 * SIZE, AO

	FMADD4	(aa2, bb2, cc04, cc04)

	LDF	[BO + 5 * SIZE], b2

	FMADD1	(aa1, bb3, cc05, cc05)

	cmp	L, 0

	FMADD2	(aa2, bb3, cc06, cc06)

	LDF	[BO + 6 * SIZE], b3

	FMADD3	(aa1, bb4, cc07, cc07)

	LDF	[AO + 0 * SIZE], a1

	FMADD4	(aa2, bb4, cc08, cc08)

	LDF	[AO + 1 * SIZE], a2

	LDF	[BO + 7 * SIZE], b4

	bg,pt	%icc, .LL27

	add	BO, 4 * SIZE, BO

	.align 4

.LL28:

	FADD	c01, c04, c01

	FADD	c02, c03, c02

	FADD	c05, c08, c05

	FADD	c06, c07, c06

#if defined(LN) || defined(RT)

#ifdef LN

	sub	KK, 1, TEMP1

#else

	sub	KK, 2, TEMP1

#endif

	sll	TEMP1, ZBASE_SHIFT + 0, TEMP2

	sll	TEMP1, ZBASE_SHIFT + 1, TEMP1

	add	AORIG, TEMP2, AO

	add	B,     TEMP1, BO

#endif

#if defined(LN) || defined(LT)

	LDF	[BO +  0 * SIZE], a1

	LDF	[BO +  1 * SIZE], a2

	LDF	[BO +  2 * SIZE], a3

	LDF	[BO +  3 * SIZE], a4

#else

	LDF	[AO +  0 * SIZE], a1

	LDF	[AO +  1 * SIZE], a2

	LDF	[AO +  2 * SIZE], a3

	LDF	[AO +  3 * SIZE], a4

#endif

	FSUB	a1, c01, c01

	FSUB	a2, c02, c02

	FSUB	a3, c05, c05

	FSUB	a4, c06, c06

#if defined(LN) || defined(LT)

	LDF	[AO +  0 * SIZE], a1

	LDF	[AO +  1 * SIZE], a2

	FMUL	a1, c01, b1

	FMUL	a2, c01, b2

	FMUL	a1, c05, b3

	FMUL	a2, c05, b4

#ifndef CONJ

	FNMSUB	(aa2, cc02, bb1, cc01)

	FMADD	(aa1, cc02, bb2, cc02)

	FNMSUB	(aa2, cc06, bb3, cc05)

	FMADD	(aa1, cc06, bb4, cc06)

#else

	FMADD	(aa2, cc02, bb1, cc01)

	FMSUB	(aa1, cc02, bb2, cc02)

	FMADD	(aa2, cc06, bb3, cc05)

	FMSUB	(aa1, cc06, bb4, cc06)

#endif

#endif

#ifdef RN

	LDF	[BO +  0 * SIZE], b1

	LDF	[BO +  1 * SIZE], b2

	LDF	[BO +  2 * SIZE], b3

	LDF	[BO +  3 * SIZE], b4

	FMUL	b1, c01, a1

	FMUL	b2, c01, a2

#ifndef CONJ

	FNMSUB	(bb2, cc02, aa1, cc01)

	FMADD	(bb1, cc02, aa2, cc02)

#else

	FMADD	(bb2, cc02, aa1, cc01)

	FMSUB	(bb1, cc02, aa2, cc02)

#endif

	FNMSUB	(bb3, cc01, cc05, cc05)

	FNMSUB	(bb3, cc02, cc06, cc06)

#ifndef CONJ

	FMADD	(bb4, cc02, cc05, cc05)

	FNMSUB	(bb4, cc01, cc06, cc06)

#else

	FNMSUB	(bb4, cc02, cc05, cc05)

	FMADD	(bb4, cc01, cc06, cc06)

#endif

	LDF	[BO +  6 * SIZE], b1

	LDF	[BO +  7 * SIZE], b2

	FMUL	b1, c05, a1

	FMUL	b2, c05, a2

#ifndef CONJ

	FNMSUB	(bb2, cc06, aa1, cc05)

	FMADD	(bb1, cc06, aa2, cc06)

#else

	FMADD	(bb2, cc06, aa1, cc05)

	FMSUB	(bb1, cc06, aa2, cc06)

#endif

#endif

#ifdef RT

	LDF	[BO +  6 * SIZE], b1

	LDF	[BO +  7 * SIZE], b2

	LDF	[BO +  4 * SIZE], b3

	LDF	[BO +  5 * SIZE], b4

	FMUL	b1, c05, a1

	FMUL	b2, c05, a2

#ifndef CONJ

	FNMSUB	(bb2, cc06, aa1, cc05)

	FMADD	(bb1, cc06, aa2, cc06)

#else

	FMADD	(bb2, cc06, aa1, cc05)

	FMSUB	(bb1, cc06, aa2, cc06)

#endif

	FNMSUB	(bb3, cc05, cc01, cc01)