/*********************************************************************/

/* Copyright 2005-2010 The University of Texas at Austin.           */

/* All rights reserved.                                              */

/*                                                                   */

/* Redistribution and use in source and binary forms, with or        */

/* without modification, are permitted provided that the following   */

/* conditions are met:                                               */

/*                                                                   */

/*   1. Redistributions of source code must retain the above         */

/*      copyright notice, this list of conditions and the following  */

/*      disclaimer.                                                  */

/*                                                                   */

/*   2. Redistributions in binary form must reproduce the above      */

/*      copyright notice, this list of conditions and the following  */

/*      disclaimer in the documentation and/or other materials       */

/*      provided with the distribution.                              */

/*                                                                   */

/*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

/*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

/*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

/*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

/*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

/*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

/*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

/*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

/*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

/*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

/*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

/*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

/*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

/*    POSSIBILITY OF SUCH DAMAGE.                                    */

/*                                                                   */

/* The views and conclusions contained in the software and           */

/* documentation are those of the authors and should not be          */

/* interpreted as representing official policies, either expressed   */

/* or implied, of The University of Texas at Austin.                 */

/*********************************************************************/

#define ASSEMBLER

#include "common.h"

#define APREFETCHSIZE 24

#define APREFETCH_CATEGORY 0

#define M	%i0

#define N	%i1

#define K	%i2

#define A	%i5

#define B	%i3

#define C	%i4

#define LDC	%o0

#define AO	%o1

#define BO	%o2

#define I	%o3

#define J	%o4

#define L	%o5

#define BB	%o7

#define C1	%l0

#define C2	%l1

#define C3	%l2

#define C4	%l3

#define OFFSET	%l4

#define	KK	%l5

#define TEMP1	%l6

#define TEMP2	%l7

#ifdef DOUBLE

#define c01	%f0

#define c02	%f2

#define c03	%f4

#define c04	%f6

#define c05	%f8

#define c06	%f10

#define c07	%f12

#define c08	%f14

#define c09	%f16

#define c10	%f18

#define c11	%f20

#define c12	%f22

#define c13	%f24

#define c14	%f26

#define c15	%f28

#define c16	%f30

#define a1	%f32

#define a2	%f34

#define a3	%f36

#define a4	%f38

#define a5	%f40

#define b1	%f42

#define b2	%f44

#define b3	%f46

#define b4	%f48

#define b5	%f50

#define b6	%f52

#define b7	%f54

#define b8	%f56

#define b9	%f58

#define ALPHA_R	%f60

#define ALPHA_I	%f62

#define cc01	0

#define cc02	2

#define cc03	4

#define cc04	6

#define cc05	8

#define cc06	10

#define cc07	12

#define cc08	14

#define cc09	16

#define cc10	18

#define cc11	20

#define cc12	22

#define cc13	24

#define cc14	26

#define cc15	28

#define cc16	30

#define aa1	 1

#define aa2	 3

#define aa3	 5

#define aa4	 7

#define aa5	 9

#define bb1	11

#define bb2	13

#define bb3	15

#define bb4	17

#define bb5	19

#define bb6	21

#define bb7	23

#define bb8	25

#define bb9	27

#define alpha_r	29

#define alpha_i	31

#else

#define c01	%f0

#define c02	%f1

#define c03	%f2

#define c04	%f3

#define c05	%f4

#define c06	%f5

#define c07	%f6

#define c08	%f7

#define c09	%f8

#define c10	%f9

#define c11	%f10

#define c12	%f11

#define c13	%f12

#define c14	%f13

#define c15	%f14

#define c16	%f15

#define a1	%f16

#define a2	%f17

#define a3	%f18

#define a4	%f19

#define a5	%f20

#define b1	%f21

#define b2	%f22

#define b3	%f23

#define b4	%f24

#define b5	%f25

#define b6	%f26

#define b7	%f27

#define b8	%f28

#define b9	%f29

#define ALPHA_R	%f30

#define ALPHA_I	%f31

#define cc01	0

#define cc02	1

#define cc03	2

#define cc04	3

#define cc05	4

#define cc06	5

#define cc07	6

#define cc08	7

#define cc09	8

#define cc10	9

#define cc11	10

#define cc12	11

#define cc13	12

#define cc14	13

#define cc15	14

#define cc16	15

#define aa1	16

#define aa2	17

#define aa3	18

#define aa4	19

#define aa5	20

#define bb1	21

#define bb2	22

#define bb3	23

#define bb4	24

#define bb5	25

#define bb6	26

#define bb7	27

#define bb8	28

#define bb9	29

#define alpha_r	30

#define alpha_i	31

#endif

#if   defined(NN) || defined(NT) || defined(TN) || defined(TT)

#define FMADD1	FMADD

#define FMADD2	FMADD

#define FMADD3	FMADD

#define FMADD4	FNMSUB

#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)

#define FMADD1	FMADD

#define FMADD2	FMADD

#define FMADD3	FNMSUB

#define FMADD4	FMADD

#elif defined(RN) || defined(RT) || defined(CN) || defined(CT)

#define FMADD1	FMADD

#define FMADD2	FNMSUB

#define FMADD3	FMADD

#define FMADD4	FMADD

#else

#define FMADD1	FMADD

#define FMADD2	FNMSUB

#define FMADD3	FNMSUB

#define FMADD4	FNMSUB

#endif

        .register %g2, #scratch

        .register %g3, #scratch

	PROLOGUE

	SAVESP

#ifndef __64BIT__

#ifdef DOUBLE

	st	%i3, [%sp + STACK_START + 16]

	st	%i4, [%sp + STACK_START + 20]

	st	%i5, [%sp + STACK_START + 24]

	ld	[%sp + STACK_START + 32], A

	ld	[%sp + STACK_START + 36], B

	ld	[%sp + STACK_START + 40], C

	ld	[%sp + STACK_START + 44], LDC

#ifdef TRMMKERNEL

	ld	[%sp + STACK_START + 48], OFFSET

#endif

	ldd	[%sp + STACK_START + 16], ALPHA_R

	ldd	[%sp + STACK_START + 24], ALPHA_I

#else

	st	%i3, [%sp + STACK_START + 16]

	st	%i4, [%sp + STACK_START + 20]

	ld	[%sp + STACK_START + 28], B

	ld	[%sp + STACK_START + 32], C

	ld	[%sp + STACK_START + 36], LDC

#ifdef TRMMKERNEL

	ld	[%sp + STACK_START + 40], OFFSET

#endif

	ld	[%sp + STACK_START + 16], ALPHA_R

	ld	[%sp + STACK_START + 20], ALPHA_I

#endif

#else

	ldx	[%sp + STACK_START + 56], B

	ldx	[%sp + STACK_START + 64], C

	ldx	[%sp + STACK_START + 72], LDC

#ifdef TRMMKERNEL

	ldx	[%sp + STACK_START + 80], OFFSET

#endif

#ifdef DOUBLE

	FMOV	%f6, ALPHA_R

	FMOV	%f8, ALPHA_I

#else

	FMOV	%f7, ALPHA_R

	FMOV	%f9, ALPHA_I

#endif

#endif

#if defined(TRMMKERNEL) && !defined(LEFT)

	neg	OFFSET, KK

#endif

	cmp	M, 0

	ble,pn	%icc, .LL999

	nop

	sra	N, 2, J

	cmp	J, 0

	ble,pn	%icc, .LL20

	sll	LDC, ZBASE_SHIFT, LDC

.LL11:

	mov	C,  C1

	add	C,  LDC, C2

	add	C2, LDC, C3

	add	C3, LDC, C4

	add	C4,  LDC, C

	sll	K, ZBASE_SHIFT + 2, BB

#if defined(TRMMKERNEL) &&  defined(LEFT)

	mov	OFFSET, KK

#endif

	mov	A, AO

	mov	M, I

	add	B, BB, BB

	.align 4

.LL12:

	prefetch [BB +  0 * SIZE], 1

#if !defined(TRMMKERNEL) || (defined(TRMMKERNEL) && ((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))))

	mov	B, BO

#else

	sll	KK, ZBASE_SHIFT + 0, TEMP1

	sll	KK, ZBASE_SHIFT + 2, TEMP2

	add	AO, TEMP1, AO

	add	B,  TEMP2, BO

#endif

	LDF	[AO +  0 * SIZE], a1

	FCLR	(cc01)

	LDF	[AO +  1 * SIZE], a2

	FCLR	(cc05)

	LDF	[AO +  8 * SIZE], a5

	FCLR	(cc09)

	LDF	[BO +  0 * SIZE], b1

	FCLR	(cc13)

	LDF	[BO +  1 * SIZE], b2

	FCLR	(cc02)

	LDF	[BO +  2 * SIZE], b3

	FCLR	(cc06)

	LDF	[BO +  3 * SIZE], b4

	FCLR	(cc10)

	LDF	[BO +  4 * SIZE], b5

	FCLR	(cc14)

	LDF	[BO +  5 * SIZE], b6

	FCLR	(cc03)

	LDF	[BO +  6 * SIZE], b7

	FCLR	(cc07)

	LDF	[BO +  7 * SIZE], b8

	FCLR	(cc11)

	LDF	[BO +  8 * SIZE], b9

	FCLR	(cc15)

	prefetch [C1 + 1 * SIZE], 3

	FCLR	(cc04)

	prefetch [C2 + 2 * SIZE], 3

	FCLR	(cc08)

	prefetch [C3 + 1 * SIZE], 3

	FCLR	(cc12)

	prefetch [C4 + 2 * SIZE], 3

	FCLR	(cc16)

#ifndef TRMMKERNEL

	sra	K,  3, L

#else

#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

	sub	K, KK, L

#elif defined(LEFT)

	add	KK, 1, L

#else

	add	KK, 4, L

#endif

	sra	L,  3, L

#endif

	cmp	L,  0

	ble,pn	%icc, .LL15

	add	 BB, 32 * SIZE, BB

	.align 4

.LL13:

	FMADD1	(aa1, bb1, cc01, cc01)

	FMADD2	(aa2, bb1, cc02, cc02)

	FMADD3	(aa1, bb2, cc03, cc03)

	FMADD4	(aa2, bb2, cc04, cc04)

	FMADD1	(aa1, bb3, cc05, cc05)

	LDF	[BO + 16 * SIZE], b1

	FMADD2	(aa2, bb3, cc06, cc06)

	LDF	[BO +  9 * SIZE], b2

	FMADD3	(aa1, bb4, cc07, cc07)

	LDF	[BO + 10 * SIZE], b3

	FMADD4	(aa2, bb4, cc08, cc08)

	LDF	[BO + 11 * SIZE], b4

	FMADD1	(aa1, bb5, cc09, cc09)

	LDF	[AO +  2 * SIZE], a3

	FMADD2	(aa2, bb5, cc10, cc10)

	LDF	[AO +  3 * SIZE], a4

	FMADD3	(aa1, bb6, cc11, cc11)

	prefetch [AO + (APREFETCHSIZE +  0) * SIZE], APREFETCH_CATEGORY

	FMADD4	(aa2, bb6, cc12, cc12)

	nop

	FMADD1	(aa1, bb7, cc13, cc13)

	LDF	[BO + 12 * SIZE], b5

	FMADD2	(aa2, bb7, cc14, cc14)

	LDF	[BO + 13 * SIZE], b6

	FMADD3	(aa1, bb8, cc15, cc15)

	LDF	[BO + 14 * SIZE], b7

	FMADD4	(aa2, bb8, cc16, cc16)

	LDF	[BO + 15 * SIZE], b8

	FMADD1	(aa3, bb9, cc01, cc01)

	FMADD2	(aa4, bb9, cc02, cc02)

	FMADD3	(aa3, bb2, cc03, cc03)

	FMADD4	(aa4, bb2, cc04, cc04)

	FMADD1	(aa3, bb3, cc05, cc05)

	LDF	[BO + 24 * SIZE], b9

	FMADD2	(aa4, bb3, cc06, cc06)

	LDF	[BO + 17 * SIZE], b2

	FMADD3	(aa3, bb4, cc07, cc07)

	LDF	[BO + 18 * SIZE], b3

	FMADD4	(aa4, bb4, cc08, cc08)

	LDF	[BO + 19 * SIZE], b4

	FMADD1	(aa3, bb5, cc09, cc09)

	LDF	[AO +  4 * SIZE], a1

	FMADD2	(aa4, bb5, cc10, cc10)

	LDF	[AO +  5 * SIZE], a2

	FMADD3	(aa3, bb6, cc11, cc11)

	add	L, -1, L

	FMADD4	(aa4, bb6, cc12, cc12)

	nop

	FMADD1	(aa3, bb7, cc13, cc13)

	LDF	[BO + 20 * SIZE], b5

	FMADD2	(aa4, bb7, cc14, cc14)

	LDF	[BO + 21 * SIZE], b6

	FMADD3	(aa3, bb8, cc15, cc15)

	LDF	[BO + 22 * SIZE], b7

	FMADD4	(aa4, bb8, cc16, cc16)

	LDF	[BO + 23 * SIZE], b8

	FMADD1	(aa1, bb1, cc01, cc01)

	FMADD2	(aa2, bb1, cc02, cc02)

	FMADD3	(aa1, bb2, cc03, cc03)

	FMADD4	(aa2, bb2, cc04, cc04)

	FMADD1	(aa1, bb3, cc05, cc05)

	LDF	[BO + 32 * SIZE], b1

	FMADD2	(aa2, bb3, cc06, cc06)

	LDF	[BO + 25 * SIZE], b2

	FMADD3	(aa1, bb4, cc07, cc07)

	LDF	[BO + 26 * SIZE], b3

	FMADD4	(aa2, bb4, cc08, cc08)

	LDF	[BO + 27 * SIZE], b4

	FMADD1	(aa1, bb5, cc09, cc09)

	LDF	[AO +  6 * SIZE], a3

	FMADD2	(aa2, bb5, cc10, cc10)

	LDF	[AO +  7 * SIZE], a4

	FMADD3	(aa1, bb6, cc11, cc11)

	nop

	FMADD4	(aa2, bb6, cc12, cc12)

	nop

	FMADD1	(aa1, bb7, cc13, cc13)

	LDF	[BO + 28 * SIZE], b5

	FMADD2	(aa2, bb7, cc14, cc14)

	LDF	[BO + 29 * SIZE], b6

	FMADD3	(aa1, bb8, cc15, cc15)

	LDF	[BO + 30 * SIZE], b7

	FMADD4	(aa2, bb8, cc16, cc16)

	LDF	[BO + 31 * SIZE], b8

	FMADD1	(aa3, bb9, cc01, cc01)

	FMADD2	(aa4, bb9, cc02, cc02)

	FMADD3	(aa3, bb2, cc03, cc03)

	FMADD4	(aa4, bb2, cc04, cc04)

	FMADD1	(aa3, bb3, cc05, cc05)

	LDF	[BO + 40 * SIZE], b9

	FMADD2	(aa4, bb3, cc06, cc06)

	LDF	[BO + 33 * SIZE], b2

	FMADD3	(aa3, bb4, cc07, cc07)

	LDF	[BO + 34 * SIZE], b3

	FMADD4	(aa4, bb4, cc08, cc08)

	LDF	[BO + 35 * SIZE], b4

	FMADD1	(aa3, bb5, cc09, cc09)

	LDF	[AO + 16 * SIZE], a1  /****/

	FMADD2	(aa4, bb5, cc10, cc10)

	LDF	[AO +  9 * SIZE], a2

	FMADD3	(aa3, bb6, cc11, cc11)

	nop

	FMADD4	(aa4, bb6, cc12, cc12)

	nop

	FMADD1	(aa3, bb7, cc13, cc13)

	LDF	[BO + 36 * SIZE], b5

	FMADD2	(aa4, bb7, cc14, cc14)

	LDF	[BO + 37 * SIZE], b6

	FMADD3	(aa3, bb8, cc15, cc15)

	LDF	[BO + 38 * SIZE], b7

	FMADD4	(aa4, bb8, cc16, cc16)

	LDF	[BO + 39 * SIZE], b8

	FMADD1	(aa5, bb1, cc01, cc01)

	FMADD2	(aa2, bb1, cc02, cc02)

	FMADD3	(aa5, bb2, cc03, cc03)

	FMADD4	(aa2, bb2, cc04, cc04)

	FMADD1	(aa5, bb3, cc05, cc05)

	LDF	[BO + 48 * SIZE], b1

	FMADD2	(aa2, bb3, cc06, cc06)

	LDF	[BO + 41 * SIZE], b2

	FMADD3	(aa5, bb4, cc07, cc07)

	LDF	[BO + 42 * SIZE], b3

	FMADD4	(aa2, bb4, cc08, cc08)

	LDF	[BO + 43 * SIZE], b4

	FMADD1	(aa5, bb5, cc09, cc09)

	LDF	[AO + 10 * SIZE], a3

	FMADD2	(aa2, bb5, cc10, cc10)

	LDF	[AO + 11 * SIZE], a4

	FMADD3	(aa5, bb6, cc11, cc11)

	prefetch [AO + (APREFETCHSIZE +  8) * SIZE], APREFETCH_CATEGORY

	FMADD4	(aa2, bb6, cc12, cc12)

	nop

	FMADD1	(aa5, bb7, cc13, cc13)

	LDF	[BO + 44 * SIZE], b5

	FMADD2	(aa2, bb7, cc14, cc14)

	LDF	[BO + 45 * SIZE], b6

	FMADD3	(aa5, bb8, cc15, cc15)

	LDF	[BO + 46 * SIZE], b7

	FMADD4	(aa2, bb8, cc16, cc16)

	LDF	[BO + 47 * SIZE], b8

	FMADD1	(aa3, bb9, cc01, cc01)

	FMADD2	(aa4, bb9, cc02, cc02)

	FMADD3	(aa3, bb2, cc03, cc03)

	FMADD4	(aa4, bb2, cc04, cc04)

	FMADD1	(aa3, bb3, cc05, cc05)

	LDF	[BO + 56 * SIZE], b9

	FMADD2	(aa4, bb3, cc06, cc06)

	LDF	[BO + 49 * SIZE], b2

	FMADD3	(aa3, bb4, cc07, cc07)

	LDF	[BO + 50 * SIZE], b3

	FMADD4	(aa4, bb4, cc08, cc08)

	LDF	[BO + 51 * SIZE], b4

	FMADD1	(aa3, bb5, cc09, cc09)

	LDF	[AO + 12 * SIZE], a5

	FMADD2	(aa4, bb5, cc10, cc10)

	LDF	[AO + 13 * SIZE], a2

	FMADD3	(aa3, bb6, cc11, cc11)

	cmp	L, 0

	FMADD4	(aa4, bb6, cc12, cc12)

	nop

	FMADD1	(aa3, bb7, cc13, cc13)

	LDF	[BO + 52 * SIZE], b5

	FMADD2	(aa4, bb7, cc14, cc14)

	LDF	[BO + 53 * SIZE], b6

	FMADD3	(aa3, bb8, cc15, cc15)

	LDF	[BO + 54 * SIZE], b7

	FMADD4	(aa4, bb8, cc16, cc16)

	LDF	[BO + 55 * SIZE], b8

	FMADD1	(aa5, bb1, cc01, cc01)

	FMADD2	(aa2, bb1, cc02, cc02)

	FMADD3	(aa5, bb2, cc03, cc03)

	FMADD4	(aa2, bb2, cc04, cc04)

	FMADD1	(aa5, bb3, cc05, cc05)

	LDF	[BO + 64 * SIZE], b1

	FMADD2	(aa2, bb3, cc06, cc06)

	LDF	[BO + 57 * SIZE], b2

	FMADD3	(aa5, bb4, cc07, cc07)

	LDF	[BO + 58 * SIZE], b3

	FMADD4	(aa2, bb4, cc08, cc08)

	LDF	[BO + 59 * SIZE], b4

	FMADD1	(aa5, bb5, cc09, cc09)

	LDF	[AO + 14 * SIZE], a3

	FMADD2	(aa2, bb5, cc10, cc10)

	LDF	[AO + 15 * SIZE], a4

	FMADD3	(aa5, bb6, cc11, cc11)

	add	BO, 64 * SIZE, BO

	FMADD4	(aa2, bb6, cc12, cc12)

	add	AO, 16 * SIZE, AO

	FMADD1	(aa5, bb7, cc13, cc13)

	LDF	[BO -  4 * SIZE], b5

	FMADD2	(aa2, bb7, cc14, cc14)

	LDF	[BO -  3 * SIZE], b6

	FMADD3	(aa5, bb8, cc15, cc15)

	LDF	[BO -  2 * SIZE], b7

	FMADD4	(aa2, bb8, cc16, cc16)

	LDF	[BO -  1 * SIZE], b8

	FMADD1	(aa3, bb9, cc01, cc01)

	FMADD2	(aa4, bb9, cc02, cc02)

	FMADD3	(aa3, bb2, cc03, cc03)

	FMADD4	(aa4, bb2, cc04, cc04)

	FMADD1	(aa3, bb3, cc05, cc05)

	LDF	[BO +  8 * SIZE], b9

	FMADD2	(aa4, bb3, cc06, cc06)

	LDF	[BO +  1 * SIZE], b2

	FMADD3	(aa3, bb4, cc07, cc07)

	LDF	[BO +  2 * SIZE], b3

	FMADD4	(aa4, bb4, cc08, cc08)

	LDF	[BO +  3 * SIZE], b4

	FMADD1	(aa3, bb5, cc09, cc09)

	LDF	[AO +  8 * SIZE], a5  /****/

	FMADD2	(aa4, bb5, cc10, cc10)

	LDF	[AO +  1 * SIZE], a2

	FMADD3	(aa3, bb6, cc11, cc11)

	FMADD4	(aa4, bb6, cc12, cc12)

	FMADD1	(aa3, bb7, cc13, cc13)

	LDF	[BO +  4 * SIZE], b5

	FMADD2	(aa4, bb7, cc14, cc14)

	LDF	[BO +  5 * SIZE], b6

	FMADD3	(aa3, bb8, cc15, cc15)

	LDF	[BO +  6 * SIZE], b7

	FMADD4	(aa4, bb8, cc16, cc16)

	ble,pn	%icc, .LL15

	LDF	[BO +  7 * SIZE], b8

	FMADD1	(aa1, bb1, cc01, cc01)

	FMADD2	(aa2, bb1, cc02, cc02)

	FMADD3	(aa1, bb2, cc03, cc03)

	FMADD4	(aa2, bb2, cc04, cc04)

	FMADD1	(aa1, bb3, cc05, cc05)

	LDF	[BO + 16 * SIZE], b1

	FMADD2	(aa2, bb3, cc06, cc06)

	LDF	[BO +  9 * SIZE], b2

	FMADD3	(aa1, bb4, cc07, cc07)

	LDF	[BO + 10 * SIZE], b3

	FMADD4	(aa2, bb4, cc08, cc08)

	LDF	[BO + 11 * SIZE], b4

	FMADD1	(aa1, bb5, cc09, cc09)

	LDF	[AO +  2 * SIZE], a3

	FMADD2	(aa2, bb5, cc10, cc10)

	LDF	[AO +  3 * SIZE], a4

	FMADD3	(aa1, bb6, cc11, cc11)

	prefetch [AO + (APREFETCHSIZE +  0) * SIZE], APREFETCH_CATEGORY

	FMADD4	(aa2, bb6, cc12, cc12)

	nop

	FMADD1	(aa1, bb7, cc13, cc13)

	LDF	[BO + 12 * SIZE], b5

	FMADD2	(aa2, bb7, cc14, cc14)

	LDF	[BO + 13 * SIZE], b6

	FMADD3	(aa1, bb8, cc15, cc15)

	LDF	[BO + 14 * SIZE], b7

	FMADD4	(aa2, bb8, cc16, cc16)

	LDF	[BO + 15 * SIZE], b8

	FMADD1	(aa3, bb9, cc01, cc01)

	FMADD2	(aa4, bb9, cc02, cc02)

	FMADD3	(aa3, bb2, cc03, cc03)

	FMADD4	(aa4, bb2, cc04, cc04)

	FMADD1	(aa3, bb3, cc05, cc05)

	LDF	[BO + 24 * SIZE], b9

	FMADD2	(aa4, bb3, cc06, cc06)

	LDF	[BO + 17 * SIZE], b2

	FMADD3	(aa3, bb4, cc07, cc07)

	LDF	[BO + 18 * SIZE], b3

	FMADD4	(aa4, bb4, cc08, cc08)

	LDF	[BO + 19 * SIZE], b4

	FMADD1	(aa3, bb5, cc09, cc09)

	LDF	[AO +  4 * SIZE], a1

	FMADD2	(aa4, bb5, cc10, cc10)

	LDF	[AO +  5 * SIZE], a2

	FMADD3	(aa3, bb6, cc11, cc11)

	add	L, -1, L

	FMADD4	(aa4, bb6, cc12, cc12)

	nop

	FMADD1	(aa3, bb7, cc13, cc13)

	LDF	[BO + 20 * SIZE], b5

	FMADD2	(aa4, bb7, cc14, cc14)

	LDF	[BO + 21 * SIZE], b6

	FMADD3	(aa3, bb8, cc15, cc15)

	LDF	[BO + 22 * SIZE], b7

	FMADD4	(aa4, bb8, cc16, cc16)

	LDF	[BO + 23 * SIZE], b8

	FMADD1	(aa1, bb1, cc01, cc01)

	FMADD2	(aa2, bb1, cc02, cc02)

	FMADD3	(aa1, bb2, cc03, cc03)

	FMADD4	(aa2, bb2, cc04, cc04)

	FMADD1	(aa1, bb3, cc05, cc05)

	LDF	[BO + 32 * SIZE], b1

	FMADD2	(aa2, bb3, cc06, cc06)

	LDF	[BO + 25 * SIZE], b2

	FMADD3	(aa1, bb4, cc07, cc07)

	LDF	[BO + 26 * SIZE], b3

	FMADD4	(aa2, bb4, cc08, cc08)

	LDF	[BO + 27 * SIZE], b4

	FMADD1	(aa1, bb5, cc09, cc09)

	LDF	[AO +  6 * SIZE], a3

	FMADD2	(aa2, bb5, cc10, cc10)

	LDF	[AO +  7 * SIZE], a4

	FMADD3	(aa1, bb6, cc11, cc11)

	nop

	FMADD4	(aa2, bb6, cc12, cc12)

	nop

	FMADD1	(aa1, bb7, cc13, cc13)

	LDF	[BO + 28 * SIZE], b5

	FMADD2	(aa2, bb7, cc14, cc14)

	LDF	[BO + 29 * SIZE], b6

	FMADD3	(aa1, bb8, cc15, cc15)

	LDF	[BO + 30 * SIZE], b7

	FMADD4	(aa2, bb8, cc16, cc16)

	LDF	[BO + 31 * SIZE], b8

	FMADD1	(aa3, bb9, cc01, cc01)

	FMADD2	(aa4, bb9, cc02, cc02)

	FMADD3	(aa3, bb2, cc03, cc03)

	FMADD4	(aa4, bb2, cc04, cc04)

	FMADD1	(aa3, bb3, cc05, cc05)

	LDF	[BO + 40 * SIZE], b9

	FMADD2	(aa4, bb3, cc06, cc06)

	LDF	[BO + 33 * SIZE], b2

	FMADD3	(aa3, bb4, cc07, cc07)

	LDF	[BO + 34 * SIZE], b3

	FMADD4	(aa4, bb4, cc08, cc08)

	LDF	[BO + 35 * SIZE], b4

	FMADD1	(aa3, bb5, cc09, cc09)

	LDF	[AO + 16 * SIZE], a1  /****/

	FMADD2	(aa4, bb5, cc10, cc10)

	LDF	[AO +  9 * SIZE], a2

	FMADD3	(aa3, bb6, cc11, cc11)

	nop

	FMADD4	(aa4, bb6, cc12, cc12)

	nop

	FMADD1	(aa3, bb7, cc13, cc13)

	LDF	[BO + 36 * SIZE], b5

	FMADD2	(aa4, bb7, cc14, cc14)

	LDF	[BO + 37 * SIZE], b6

	FMADD3	(aa3, bb8, cc15, cc15)

	LDF	[BO + 38 * SIZE], b7

	FMADD4	(aa4, bb8, cc16, cc16)

	LDF	[BO + 39 * SIZE], b8

	FMADD1	(aa5, bb1, cc01, cc01)

	FMADD2	(aa2, bb1, cc02, cc02)

	FMADD3	(aa5, bb2, cc03, cc03)

	FMADD4	(aa2, bb2, cc04, cc04)

	FMADD1	(aa5, bb3, cc05, cc05)

	LDF	[BO + 48 * SIZE], b1

	FMADD2	(aa2, bb3, cc06, cc06)

	LDF	[BO + 41 * SIZE], b2

	FMADD3	(aa5, bb4, cc07, cc07)

	LDF	[BO + 42 * SIZE], b3

	FMADD4	(aa2, bb4, cc08, cc08)

	LDF	[BO + 43 * SIZE], b4

	FMADD1	(aa5, bb5, cc09, cc09)

	LDF	[AO + 10 * SIZE], a3

	FMADD2	(aa2, bb5, cc10, cc10)

	LDF	[AO + 11 * SIZE], a4

	FMADD3	(aa5, bb6, cc11, cc11)

	prefetch [AO + (APREFETCHSIZE +  8) * SIZE], APREFETCH_CATEGORY

	FMADD4	(aa2, bb6, cc12, cc12)

	nop

	FMADD1	(aa5, bb7, cc13, cc13)

	LDF	[BO + 44 * SIZE], b5

	FMADD2	(aa2, bb7, cc14, cc14)

	LDF	[BO + 45 * SIZE], b6

	FMADD3	(aa5, bb8, cc15, cc15)

	LDF	[BO + 46 * SIZE], b7

	FMADD4	(aa2, bb8, cc16, cc16)

	LDF	[BO + 47 * SIZE], b8

	FMADD1	(aa3, bb9, cc01, cc01)

	FMADD2	(aa4, bb9, cc02, cc02)

	FMADD3	(aa3, bb2, cc03, cc03)

	FMADD4	(aa4, bb2, cc04, cc04)

	FMADD1	(aa3, bb3, cc05, cc05)

	LDF	[BO + 56 * SIZE], b9

	FMADD2	(aa4, bb3, cc06, cc06)

	LDF	[BO + 49 * SIZE], b2

	FMADD3	(aa3, bb4, cc07, cc07)

	LDF	[BO + 50 * SIZE], b3

	FMADD4	(aa4, bb4, cc08, cc08)

	LDF	[BO + 51 * SIZE], b4

	FMADD1	(aa3, bb5, cc09, cc09)

	LDF	[AO + 12 * SIZE], a5

	FMADD2	(aa4, bb5, cc10, cc10)

	LDF	[AO + 13 * SIZE], a2

	FMADD3	(aa3, bb6, cc11, cc11)

	cmp	L, 0

	FMADD4	(aa4, bb6, cc12, cc12)

	nop

	FMADD1	(aa3, bb7, cc13, cc13)

	LDF	[BO + 52 * SIZE], b5

	FMADD2	(aa4, bb7, cc14, cc14)

	LDF	[BO + 53 * SIZE], b6

	FMADD3	(aa3, bb8, cc15, cc15)

	LDF	[BO + 54 * SIZE], b7

	FMADD4	(aa4, bb8, cc16, cc16)

	LDF	[BO + 55 * SIZE], b8