/*********************************************************************/

/* Copyright 2009, 2010 The University of Texas at Austin.           */

/* All rights reserved.                                              */

/*                                                                   */

/* Redistribution and use in source and binary forms, with or        */

/* without modification, are permitted provided that the following   */

/* conditions are met:                                               */

/*                                                                   */

/*   1. Redistributions of source code must retain the above         */

/*      copyright notice, this list of conditions and the following  */

/*      disclaimer.                                                  */

/*                                                                   */

/*   2. Redistributions in binary form must reproduce the above      */

/*      copyright notice, this list of conditions and the following  */

/*      disclaimer in the documentation and/or other materials       */

/*      provided with the distribution.                              */

/*                                                                   */

/*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

/*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

/*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

/*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

/*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

/*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

/*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

/*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

/*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

/*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

/*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

/*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

/*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

/*    POSSIBILITY OF SUCH DAMAGE.                                    */

/*                                                                   */

/* The views and conclusions contained in the software and           */

/* documentation are those of the authors and should not be          */

/* interpreted as representing official policies, either expressed   */

/* or implied, of The University of Texas at Austin.                 */

/*********************************************************************/

#define ASSEMBLER

#include "common.h"

#define	M	r3

#define	N	r4

#define	A	r5

#define	LDA	r6

#define B	r7

#define AO1	r8

#define AO2	r9

#define AO3	r10

#define AO4	r11

#define J	r12

#define PREA	r14

#define PREB1	r15

#define B1	r16

#define B2	r17

#define B3	r18

#define M4	r19

#define c01	f0

#define c02	f1

#define c03	f2

#define c04	f3

#define c05	f4

#define c06	f5

#define c07	f6

#define c08	f7

#define c09	f8

#define c10	f9

#define c11	f10

#define c12	f11

#define c13	f12

#define c14	f13

#define c15	f14

#define c16	f15

#define STACKSIZE 64

#ifdef CELL

#define PREFETCHSIZE   16

#define PREFETCHWSIZE  48

#endif

#ifdef PPC970

#define PREFETCHSIZE   16

#define PREFETCHWSIZE  48

#endif

#ifdef PPC440

#define PREFETCHSIZE   16

#define PREFETCHWSIZE  48

#endif

#ifdef POWER4

#define PREFETCHSIZE   16

#define PREFETCHWSIZE  48

#endif

#ifdef POWER5

#define PREFETCHSIZE   16

#define PREFETCHWSIZE  48

#endif

#ifdef POWER6

#define PREFETCHSIZE   16

#define PREFETCHWSIZE  48

#endif

#ifdef PPCG4

#define PREFETCHSIZE   16

#define PREFETCHWSIZE  48

#endif

	PROLOGUE

	PROFCODE

	addi	SP, SP, -STACKSIZE

	li	r0, 0

	stfd	f14,    0(SP)

	stfd	f15,    8(SP)

#ifdef __64BIT__

	std	r14,   16(SP)

	std	r15,   24(SP)

	std	r16,   32(SP)

	std	r17,   40(SP)

	std	r18,   48(SP)

	std	r19,   56(SP)

#else

	stw	r14,   16(SP)

	stw	r15,   20(SP)

	stw	r16,   24(SP)

	stw	r17,   28(SP)

	stw	r18,   32(SP)

	stw	r19,   36(SP)

#endif

	slwi	LDA, LDA, BASE_SHIFT

	slwi	M4, M, 2 + BASE_SHIFT

	li	PREA,  -4

	li	PREB1, -2

	and	B2, N, PREA

	and	B3, N, PREB1

	mullw	B2, B2, M

	mullw	B3, B3, M

	slwi	B2, B2, BASE_SHIFT

	slwi	B3, B3, BASE_SHIFT

	add	B2, B2, B

	add	B3, B3, B

	li	PREA,  PREFETCHSIZE * SIZE

	li	PREB1, (PREFETCHWSIZE +  0) * SIZE

	cmpwi	cr0, M, 0

	ble-	LL(999)

	cmpwi	cr0, N, 0

	ble-	LL(999)

	srawi.	J,  M,  2

	ble	LL(20)

	.align 4

LL(10):

	mr	AO1, A

	add	AO2, A,   LDA

	add	AO3, AO2, LDA

	add	AO4, AO3, LDA

	add	A,   AO4, LDA

	mr	B1, B

	addi	B, B, 16 * SIZE

	srawi.	r0,  N,  2

	mtspr	CTR, r0

	ble	LL(13)

	.align 4

LL(12):

	LFD	c01,   0 * SIZE(AO1)

	LFD	c02,   1 * SIZE(AO1)

	LFD	c03,   2 * SIZE(AO1)

	LFD	c04,   3 * SIZE(AO1)

	LFD	c05,   0 * SIZE(AO2)

	LFD	c06,   1 * SIZE(AO2)

	LFD	c07,   2 * SIZE(AO2)

	LFD	c08,   3 * SIZE(AO2)

	LFD	c09,   0 * SIZE(AO3)

	LFD	c10,   1 * SIZE(AO3)

	LFD	c11,   2 * SIZE(AO3)

	LFD	c12,   3 * SIZE(AO3)

	LFD	c13,   0 * SIZE(AO4)

	LFD	c14,   1 * SIZE(AO4)

	LFD	c15,   2 * SIZE(AO4)

	LFD	c16,   3 * SIZE(AO4)

	STFD	c01,   0 * SIZE(B1)

	STFD	c02,   1 * SIZE(B1)

	STFD	c03,   2 * SIZE(B1)

	STFD	c04,   3 * SIZE(B1)

	STFD	c05,   4 * SIZE(B1)

	STFD	c06,   5 * SIZE(B1)

	STFD	c07,   6 * SIZE(B1)

	STFD	c08,   7 * SIZE(B1)

	STFD	c09,   8 * SIZE(B1)

	STFD	c10,   9 * SIZE(B1)

	STFD	c11,  10 * SIZE(B1)

	STFD	c12,  11 * SIZE(B1)

	STFD	c13,  12 * SIZE(B1)

	STFD	c14,  13 * SIZE(B1)

	STFD	c15,  14 * SIZE(B1)

	STFD	c16,  15 * SIZE(B1)

#ifdef POWER6

	dcbtst	PREA, AO1

	dcbtst	PREA, AO2

	dcbtst	PREA, AO3

	dcbtst	PREA, AO4

#else

	dcbt	PREA, AO1

	dcbt	PREA, AO2

	dcbt	PREA, AO3

	dcbt	PREA, AO4

#endif

	dcbtst	PREB1, B

	addi	AO1, AO1,  4 * SIZE

	addi	AO2, AO2,  4 * SIZE

	addi	AO3, AO3,  4 * SIZE

	addi	AO4, AO4,  4 * SIZE

	add	B1,  B1,   M4

	bdnz	LL(12)

	.align 4

LL(13):

	andi.	r0,  N,  2

	ble	LL(14)

	LFD	c01,   0 * SIZE(AO1)

	LFD	c02,   1 * SIZE(AO1)

	LFD	c03,   0 * SIZE(AO2)

	LFD	c04,   1 * SIZE(AO2)

	LFD	c05,   0 * SIZE(AO3)

	LFD	c06,   1 * SIZE(AO3)

	LFD	c07,   0 * SIZE(AO4)

	LFD	c08,   1 * SIZE(AO4)

	STFD	c01,   0 * SIZE(B2)

	STFD	c02,   1 * SIZE(B2)

	STFD	c03,   2 * SIZE(B2)

	STFD	c04,   3 * SIZE(B2)

	STFD	c05,   4 * SIZE(B2)

	STFD	c06,   5 * SIZE(B2)

	STFD	c07,   6 * SIZE(B2)

	STFD	c08,   7 * SIZE(B2)

	addi	AO1, AO1,  2 * SIZE

	addi	AO2, AO2,  2 * SIZE

	addi	AO3, AO3,  2 * SIZE

	addi	AO4, AO4,  2 * SIZE

	addi	B2,  B2,   8 * SIZE

	.align 4

LL(14):

	andi.	r0,  N,  1

	ble	LL(17)

	LFD	c01,   0 * SIZE(AO1)

	LFD	c02,   0 * SIZE(AO2)

	LFD	c03,   0 * SIZE(AO3)

	LFD	c04,   0 * SIZE(AO4)

	STFD	c01,   0 * SIZE(B3)

	STFD	c02,   1 * SIZE(B3)

	STFD	c03,   2 * SIZE(B3)

	STFD	c04,   3 * SIZE(B3)

	addi	B3,  B3,  4 * SIZE

	.align 4

LL(17):

	addic.	J, J, -1

	bgt	LL(10)

	.align 4

LL(20):

	andi.	J,  M,  2

	ble	LL(30)

	mr	AO1, A

	add	AO2, A,   LDA

	add	A,   AO2, LDA

	mr	B1, B

	addi	B, B, 8 * SIZE

	srawi.	r0,  N,  2

	mtspr	CTR, r0

	ble	LL(23)

	.align 4

LL(22):

	LFD	c01,   0 * SIZE(AO1)

	LFD	c02,   1 * SIZE(AO1)

	LFD	c03,   2 * SIZE(AO1)

	LFD	c04,   3 * SIZE(AO1)

	LFD	c05,   0 * SIZE(AO2)

	LFD	c06,   1 * SIZE(AO2)

	LFD	c07,   2 * SIZE(AO2)

	LFD	c08,   3 * SIZE(AO2)

	STFD	c01,   0 * SIZE(B1)

	STFD	c02,   1 * SIZE(B1)

	STFD	c03,   2 * SIZE(B1)

	STFD	c04,   3 * SIZE(B1)

	STFD	c05,   4 * SIZE(B1)

	STFD	c06,   5 * SIZE(B1)

	STFD	c07,   6 * SIZE(B1)

	STFD	c08,   7 * SIZE(B1)

	addi	AO1, AO1,  4 * SIZE

	addi	AO2, AO2,  4 * SIZE

	add	B1,   B1, M4

	bdnz	LL(22)

	.align 4

LL(23):

	andi.	r0,  N,  2

	ble	LL(24)

	LFD	c01,   0 * SIZE(AO1)

	LFD	c02,   1 * SIZE(AO1)

	LFD	c03,   0 * SIZE(AO2)

	LFD	c04,   1 * SIZE(AO2)

	STFD	c01,   0 * SIZE(B2)

	STFD	c02,   1 * SIZE(B2)

	STFD	c03,   2 * SIZE(B2)

	STFD	c04,   3 * SIZE(B2)

	addi	AO1, AO1,  2 * SIZE

	addi	AO2, AO2,  2 * SIZE

	addi	B2,  B2,   4 * SIZE

	.align 4

LL(24):

	andi.	r0,  N,  1

	ble	LL(30)

	LFD	c01,   0 * SIZE(AO1)

	LFD	c02,   0 * SIZE(AO2)

	STFD	c01,   0 * SIZE(B3)

	STFD	c02,   1 * SIZE(B3)

	addi	B3,  B3,  2 * SIZE

	.align 4

LL(30):

	andi.	J,  M,  1

	ble	LL(999)

	mr	AO1, A

	mr	B1, B

	srawi.	r0,  N,  2

	mtspr	CTR, r0

	ble	LL(33)

	.align 4

LL(32):

	LFD	c01,   0 * SIZE(AO1)

	LFD	c02,   1 * SIZE(AO1)

	LFD	c03,   2 * SIZE(AO1)

	LFD	c04,   3 * SIZE(AO1)

	STFD	c01,   0 * SIZE(B1)

	STFD	c02,   1 * SIZE(B1)

	STFD	c03,   2 * SIZE(B1)

	STFD	c04,   3 * SIZE(B1)

	addi	AO1, AO1,  4 * SIZE

	add	B1,   B1,  M4

	bdnz	LL(32)

	.align 4

LL(33):

	andi.	r0,  N,  2

	ble	LL(34)

	LFD	c01,   0 * SIZE(AO1)

	LFD	c02,   1 * SIZE(AO1)

	STFD	c01,   0 * SIZE(B2)

	STFD	c02,   1 * SIZE(B2)

	addi	AO1, AO1,  2 * SIZE

	addi	B2,  B2,   2 * SIZE

	.align 4

LL(34):

	andi.	r0,  N,  1

	ble	LL(999)

	LFD	c01,   0 * SIZE(AO1)

	STFD	c01,   0 * SIZE(B3)

	.align 4

LL(999):

	li	r3, 0

	lfd	f14,    0(SP)

	lfd	f15,    8(SP)

#ifdef __64BIT__

	ld	r14,   16(SP)

	ld	r15,   24(SP)

	ld	r16,   32(SP)

	ld	r17,   40(SP)

	ld	r18,   48(SP)

	ld	r19,   56(SP)

#else

	lwz	r14,   16(SP)

	lwz	r15,   20(SP)

	lwz	r16,   24(SP)

	lwz	r17,   28(SP)

	lwz	r18,   32(SP)

	lwz	r19,   36(SP)

#endif

	addi	SP, SP, STACKSIZE

	blr

	EPILOGUE