Tree - bw/opentoonz - Cool Bug Repo

bw / opentoonz

Blame thirdparty/openblas/xianyi-OpenBLAS-e6e87a2/kernel/power/trsm_kernel_RT.S

Blob Raw

kusano	2b45e8	`/*********************************************************************/`
kusano	2b45e8	`/* Copyright 2009, 2010 The University of Texas at Austin. */`
kusano	2b45e8	`/* All rights reserved. */`
kusano	2b45e8	`/* */`
kusano	2b45e8	`/* Redistribution and use in source and binary forms, with or */`
kusano	2b45e8	`/* without modification, are permitted provided that the following */`
kusano	2b45e8	`/* conditions are met: */`
kusano	2b45e8	`/* */`
kusano	2b45e8	`/* 1. Redistributions of source code must retain the above */`
kusano	2b45e8	`/* copyright notice, this list of conditions and the following */`
kusano	2b45e8	`/* disclaimer. */`
kusano	2b45e8	`/* */`
kusano	2b45e8	`/* 2. Redistributions in binary form must reproduce the above */`
kusano	2b45e8	`/* copyright notice, this list of conditions and the following */`
kusano	2b45e8	`/* disclaimer in the documentation and/or other materials */`
kusano	2b45e8	`/* provided with the distribution. */`
kusano	2b45e8	`/* */`
kusano	2b45e8	`/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */`
kusano	2b45e8	/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
kusano	2b45e8	`/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */`
kusano	2b45e8	`/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */`
kusano	2b45e8	`/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */`
kusano	2b45e8	`/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */`
kusano	2b45e8	`/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */`
kusano	2b45e8	`/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */`
kusano	2b45e8	`/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */`
kusano	2b45e8	`/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */`
kusano	2b45e8	`/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */`
kusano	2b45e8	`/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */`
kusano	2b45e8	`/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */`
kusano	2b45e8	`/* POSSIBILITY OF SUCH DAMAGE. */`
kusano	2b45e8	`/* */`
kusano	2b45e8	`/* The views and conclusions contained in the software and */`
kusano	2b45e8	`/* documentation are those of the authors and should not be */`
kusano	2b45e8	`/* interpreted as representing official policies, either expressed */`
kusano	2b45e8	`/* or implied, of The University of Texas at Austin. */`
kusano	2b45e8	`/*********************************************************************/`
kusano	2b45e8
kusano	2b45e8	`#define ASSEMBLER`
kusano	2b45e8	`#include "common.h"`
kusano	2b45e8
kusano	2b45e8	`#ifndef __64BIT__`
kusano	2b45e8	`#define LOAD lwz`
kusano	2b45e8	`#else`
kusano	2b45e8	`#define LOAD ld`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef __64BIT__`
kusano	2b45e8	`#define STACKSIZE 320`
kusano	2b45e8	`#define ALPHA 296(SP)`
kusano	2b45e8	`#define FZERO 304(SP)`
kusano	2b45e8	`#else`
kusano	2b45e8	`#define STACKSIZE 240`
kusano	2b45e8	`#define ALPHA 224(SP)`
kusano	2b45e8	`#define FZERO 232(SP)`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#define M r3`
kusano	2b45e8	`#define N r4`
kusano	2b45e8	`#define K r5`
kusano	2b45e8
kusano	2b45e8	`#ifdef linux`
kusano	2b45e8	`#ifndef __64BIT__`
kusano	2b45e8	`#define A r6`
kusano	2b45e8	`#define B r7`
kusano	2b45e8	`#define C r8`
kusano	2b45e8	`#define LDC r9`
kusano	2b45e8	`#define OFFSET r10`
kusano	2b45e8	`#else`
kusano	2b45e8	`#define A r7`
kusano	2b45e8	`#define B r8`
kusano	2b45e8	`#define C r9`
kusano	2b45e8	`#define LDC r10`
kusano	2b45e8	`#define OFFSET r6`
kusano	2b45e8	`#endif`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(_AIX) \|\| defined(__APPLE__)`
kusano	2b45e8	`#if !defined(__64BIT__) && defined(DOUBLE)`
kusano	2b45e8	`#define A r8`
kusano	2b45e8	`#define B r9`
kusano	2b45e8	`#define C r10`
kusano	2b45e8	`#define LDC r7`
kusano	2b45e8	`#define OFFSET r6`
kusano	2b45e8	`#else`
kusano	2b45e8	`#define A r7`
kusano	2b45e8	`#define B r8`
kusano	2b45e8	`#define C r9`
kusano	2b45e8	`#define LDC r10`
kusano	2b45e8	`#define OFFSET r6`
kusano	2b45e8	`#endif`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#define AORIG r18`
kusano	2b45e8	`#define TEMP r19`
kusano	2b45e8	`#define KK r20`
kusano	2b45e8	`#define I r21`
kusano	2b45e8	`#define J r22`
kusano	2b45e8	`#define AO r23`
kusano	2b45e8	`#define BO r24`
kusano	2b45e8	`#define CO1 r25`
kusano	2b45e8	`#define CO2 r26`
kusano	2b45e8	`#define CO3 r27`
kusano	2b45e8	`#define CO4 r28`
kusano	2b45e8
kusano	2b45e8	`#define PREA r29`
kusano	2b45e8	`#define PREB r30`
kusano	2b45e8	`#define PREC r31`
kusano	2b45e8
kusano	2b45e8	`#ifndef NEEDPARAM`
kusano	2b45e8
kusano	2b45e8	`PROLOGUE`
kusano	2b45e8	`PROFCODE`
kusano	2b45e8
kusano	2b45e8	`addi SP, SP, -STACKSIZE`
kusano	2b45e8	`li r0, 0`
kusano	2b45e8
kusano	2b45e8	`stfd f14, 0(SP)`
kusano	2b45e8	`stfd f15, 8(SP)`
kusano	2b45e8	`stfd f16, 16(SP)`
kusano	2b45e8	`stfd f17, 24(SP)`
kusano	2b45e8
kusano	2b45e8	`stfd f18, 32(SP)`
kusano	2b45e8	`stfd f19, 40(SP)`
kusano	2b45e8	`stfd f20, 48(SP)`
kusano	2b45e8	`stfd f21, 56(SP)`
kusano	2b45e8
kusano	2b45e8	`stfd f22, 64(SP)`
kusano	2b45e8	`stfd f23, 72(SP)`
kusano	2b45e8	`stfd f24, 80(SP)`
kusano	2b45e8	`stfd f25, 88(SP)`
kusano	2b45e8
kusano	2b45e8	`stfd f26, 96(SP)`
kusano	2b45e8	`stfd f27, 104(SP)`
kusano	2b45e8	`stfd f28, 112(SP)`
kusano	2b45e8	`stfd f29, 120(SP)`
kusano	2b45e8
kusano	2b45e8	`stfd f30, 128(SP)`
kusano	2b45e8	`stfd f31, 136(SP)`
kusano	2b45e8
kusano	2b45e8	`#ifdef __64BIT__`
kusano	2b45e8	`std r31, 144(SP)`
kusano	2b45e8	`std r30, 152(SP)`
kusano	2b45e8	`std r29, 160(SP)`
kusano	2b45e8	`std r28, 168(SP)`
kusano	2b45e8	`std r27, 176(SP)`
kusano	2b45e8	`std r26, 184(SP)`
kusano	2b45e8	`std r25, 192(SP)`
kusano	2b45e8	`std r24, 200(SP)`
kusano	2b45e8	`std r23, 208(SP)`
kusano	2b45e8	`std r22, 216(SP)`
kusano	2b45e8	`std r21, 224(SP)`
kusano	2b45e8	`std r20, 232(SP)`
kusano	2b45e8	`std r19, 240(SP)`
kusano	2b45e8	`std r18, 248(SP)`
kusano	2b45e8	`#else`
kusano	2b45e8	`stw r31, 144(SP)`
kusano	2b45e8	`stw r30, 148(SP)`
kusano	2b45e8	`stw r29, 152(SP)`
kusano	2b45e8	`stw r28, 156(SP)`
kusano	2b45e8	`stw r27, 160(SP)`
kusano	2b45e8	`stw r26, 164(SP)`
kusano	2b45e8	`stw r25, 168(SP)`
kusano	2b45e8	`stw r24, 172(SP)`
kusano	2b45e8	`stw r23, 176(SP)`
kusano	2b45e8	`stw r22, 180(SP)`
kusano	2b45e8	`stw r21, 184(SP)`
kusano	2b45e8	`stw r20, 188(SP)`
kusano	2b45e8	`stw r19, 192(SP)`
kusano	2b45e8	`stw r18, 196(SP)`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`stw r0, FZERO`
kusano	2b45e8
kusano	2b45e8	`#if defined(_AIX) \|\| defined(__APPLE__)`
kusano	2b45e8	`#if !defined(__64BIT__) && defined(DOUBLE)`
kusano	2b45e8	`lwz LDC, 56 + STACKSIZE(SP)`
kusano	2b45e8	`#endif`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`slwi LDC, LDC, BASE_SHIFT`
kusano	2b45e8
kusano	2b45e8	`#if defined(linux) && defined(__64BIT__)`
kusano	2b45e8	`ld OFFSET, 112 + STACKSIZE(SP)`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(_AIX) \|\| defined(__APPLE__)`
kusano	2b45e8	`#ifdef __64BIT__`
kusano	2b45e8	`ld OFFSET, 112 + STACKSIZE(SP)`
kusano	2b45e8	`#else`
kusano	2b45e8	`#ifdef DOUBLE`
kusano	2b45e8	`lwz OFFSET, 60 + STACKSIZE(SP)`
kusano	2b45e8	`#else`
kusano	2b45e8	`lwz OFFSET, 56 + STACKSIZE(SP)`
kusano	2b45e8	`#endif`
kusano	2b45e8	`#endif`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`mullw r0, M, K`
kusano	2b45e8	`slwi r0, r0, BASE_SHIFT`
kusano	2b45e8	`add A, A, r0`
kusano	2b45e8
kusano	2b45e8	`slwi r0, M, BASE_SHIFT`
kusano	2b45e8	`add C, C, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RN`
kusano	2b45e8	`neg KK, OFFSET`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RT`
kusano	2b45e8	`mullw r0, N, K`
kusano	2b45e8	`slwi r0, r0, BASE_SHIFT`
kusano	2b45e8	`add B, B, r0`
kusano	2b45e8
kusano	2b45e8	`mullw r0, N, LDC`
kusano	2b45e8	`add C, C, r0`
kusano	2b45e8
kusano	2b45e8	`sub KK, N, OFFSET`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`cmpwi cr0, M, 0`
kusano	2b45e8	`ble LL(999)`
kusano	2b45e8	`cmpwi cr0, N, 0`
kusano	2b45e8	`ble LL(999)`
kusano	2b45e8	`cmpwi cr0, K, 0`
kusano	2b45e8	`ble LL(999)`
kusano	2b45e8
kusano	2b45e8	`#ifndef PREFETCHTEST`
kusano	2b45e8	`#if defined(TRSMKERNEL) && defined(LN)`
kusano	2b45e8	`/* Direction is special */`
kusano	2b45e8	`#ifdef PPC970`
kusano	2b45e8	`li PREC, -4 * SIZE`
kusano	2b45e8	`#endif`
kusano	2b45e8	`#ifdef POWER4`
kusano	2b45e8	`li PREC, -4 * SIZE`
kusano	2b45e8	`#endif`
kusano	2b45e8	`#ifdef POWER5`
kusano	2b45e8	`li PREC, -4 * SIZE`
kusano	2b45e8	`#endif`
kusano	2b45e8	`#else`
kusano	2b45e8	`/* Normal prefetch */`
kusano	2b45e8	`#ifdef PPC970`
kusano	2b45e8	`li PREC, 4 * SIZE`
kusano	2b45e8	`#endif`
kusano	2b45e8	`#ifdef POWER4`
kusano	2b45e8	`li PREC, 4 * SIZE /* is 12 best? */`
kusano	2b45e8	`#endif`
kusano	2b45e8	`#ifdef POWER5`
kusano	2b45e8	`li PREC, 3 * SIZE`
kusano	2b45e8	`#endif`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#else`
kusano	2b45e8
kusano	2b45e8	`#ifdef linux`
kusano	2b45e8	`#ifndef __64BIT__`
kusano	2b45e8	`mr PREA, r10`
kusano	2b45e8	`lwz PREB, 8 + STACKSIZE(SP)`
kusano	2b45e8	`lwz PREC, 12 + STACKSIZE(SP)`
kusano	2b45e8	`#else`
kusano	2b45e8	`ld PREA, 112 + STACKSIZE(SP)`
kusano	2b45e8	`ld PREB, 120 + STACKSIZE(SP)`
kusano	2b45e8	`ld PREC, 128 + STACKSIZE(SP)`
kusano	2b45e8	`#endif`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(_AIX) \|\| defined(__APPLE__)`
kusano	2b45e8	`#ifdef __64BIT__`
kusano	2b45e8	`ld PREA, 112 + STACKSIZE(SP)`
kusano	2b45e8	`ld PREB, 120 + STACKSIZE(SP)`
kusano	2b45e8	`ld PREC, 128 + STACKSIZE(SP)`
kusano	2b45e8	`#else`
kusano	2b45e8	`#ifdef DOUBLE`
kusano	2b45e8	`lwz PREA, 60 + STACKSIZE(SP)`
kusano	2b45e8	`lwz PREB, 64 + STACKSIZE(SP)`
kusano	2b45e8	`lwz PREC, 68 + STACKSIZE(SP)`
kusano	2b45e8	`#else`
kusano	2b45e8	`lwz PREA, 56 + STACKSIZE(SP)`
kusano	2b45e8	`lwz PREB, 60 + STACKSIZE(SP)`
kusano	2b45e8	`lwz PREC, 64 + STACKSIZE(SP)`
kusano	2b45e8	`#endif`
kusano	2b45e8	`#endif`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifndef PREFETCHTEST`
kusano	2b45e8	`#ifdef PPC970`
kusano	2b45e8	`#ifdef ALLOC_HUGETLB`
kusano	2b45e8	`li PREA, (16 * 5 * SIZE \| 1)`
kusano	2b45e8	`li PREB, (16 * 5 * SIZE \| 3)`
kusano	2b45e8	`#else`
kusano	2b45e8	`li PREA, (16 * 14 * SIZE \| 1)`
kusano	2b45e8	`li PREB, (16 * 8 * SIZE \| 3)`
kusano	2b45e8	`#endif`
kusano	2b45e8	`#endif`
kusano	2b45e8	`#ifdef POWER4`
kusano	2b45e8	`#ifdef ALLOC_HUGETLB`
kusano	2b45e8	`li PREA, (16 * 1 * SIZE + 16)`
kusano	2b45e8	`li PREB, (16 * 1 * SIZE + 16)`
kusano	2b45e8	`#else`
kusano	2b45e8	`li PREA, (16 * 2 * SIZE + 16)`
kusano	2b45e8	`li PREB, (16 * 2 * SIZE + 16)`
kusano	2b45e8	`#endif`
kusano	2b45e8	`#endif`
kusano	2b45e8	`#ifdef POWER5`
kusano	2b45e8	`#ifdef ALLOC_HUGETLB`
kusano	2b45e8	`li PREA, (16 * 7 * SIZE \| 1)`
kusano	2b45e8	`li PREB, (16 * 7 * SIZE \| 3)`
kusano	2b45e8	`#else`
kusano	2b45e8	`li PREA, (16 * 12 * SIZE \| 1)`
kusano	2b45e8	`li PREB, (16 * 6 * SIZE \| 3)`
kusano	2b45e8	`#endif`
kusano	2b45e8	`#endif`
kusano	2b45e8	`#endif`
kusano	2b45e8	`lfs f0, FZERO`
kusano	2b45e8
kusano	2b45e8	`LL(70):`
kusano	2b45e8	`andi. J, N, 1`
kusano	2b45e8	`ble LL(40)`
kusano	2b45e8
kusano	2b45e8	`#ifdef RT`
kusano	2b45e8	`slwi r0, K, 0 + BASE_SHIFT`
kusano	2b45e8	`sub B, B, r0`
kusano	2b45e8
kusano	2b45e8	`sub C, C, LDC`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`mr CO1, C`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`add KK, M, OFFSET`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LT`
kusano	2b45e8	`mr KK, OFFSET`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`fmr f1, f0`
kusano	2b45e8	`fmr f2, f0`
kusano	2b45e8	`fmr f3, f0`
kusano	2b45e8
kusano	2b45e8	`srawi. I, M, 2`
kusano	2b45e8	`#if defined(LN) \|\| defined(RT)`
kusano	2b45e8	`mr AORIG, A`
kusano	2b45e8	`#else`
kusano	2b45e8	`mr AO, A`
kusano	2b45e8	`#endif`
kusano	2b45e8	`#ifndef RT`
kusano	2b45e8	`add C, CO1, LDC`
kusano	2b45e8	`#endif`
kusano	2b45e8	`ble LL(80)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(71):`
kusano	2b45e8	`#if defined(LT) \|\| defined(RN)`
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 0 * SIZE(B)`
kusano	2b45e8	`LFD f21, 1 * SIZE(B)`
kusano	2b45e8	`LFD f22, 2 * SIZE(B)`
kusano	2b45e8	`LFD f23, 3 * SIZE(B)`
kusano	2b45e8
kusano	2b45e8	`dcbt CO1, PREC`
kusano	2b45e8
kusano	2b45e8	`srawi. r0, KK, 2`
kusano	2b45e8	`mtspr CTR, r0`
kusano	2b45e8	`mr BO, B`
kusano	2b45e8	`#else`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`slwi r0, K, 2 + BASE_SHIFT`
kusano	2b45e8	`sub AORIG, AORIG, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`slwi r0, KK, 2 + BASE_SHIFT`
kusano	2b45e8	`slwi TEMP, KK, 0 + BASE_SHIFT`
kusano	2b45e8	`add AO, AORIG, r0`
kusano	2b45e8	`add BO, B, TEMP`
kusano	2b45e8
kusano	2b45e8	`sub TEMP, K, KK`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 0 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 1 * SIZE(BO)`
kusano	2b45e8	`LFD f22, 2 * SIZE(BO)`
kusano	2b45e8	`LFD f23, 3 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`dcbt CO1, PREC`
kusano	2b45e8
kusano	2b45e8	`srawi. r0, TEMP, 2`
kusano	2b45e8	`mtspr CTR, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8	`ble LL(75)`
kusano	2b45e8	`.align 5`
kusano	2b45e8
kusano	2b45e8	`LL(72):`
kusano	2b45e8	`FMADD f0, f16, f20, f0`
kusano	2b45e8	`FMADD f1, f17, f20, f1`
kusano	2b45e8	`FMADD f2, f18, f20, f2`
kusano	2b45e8	`FMADD f3, f19, f20, f3`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 4 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 5 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 6 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 7 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMADD f0, f16, f21, f0`
kusano	2b45e8	`FMADD f1, f17, f21, f1`
kusano	2b45e8	`FMADD f2, f18, f21, f2`
kusano	2b45e8	`FMADD f3, f19, f21, f3`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 8 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 9 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 10 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 11 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMADD f0, f16, f22, f0`
kusano	2b45e8	`FMADD f1, f17, f22, f1`
kusano	2b45e8	`FMADD f2, f18, f22, f2`
kusano	2b45e8	`FMADD f3, f19, f22, f3`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 12 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 13 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 14 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 15 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMADD f0, f16, f23, f0`
kusano	2b45e8	`FMADD f1, f17, f23, f1`
kusano	2b45e8	`FMADD f2, f18, f23, f2`
kusano	2b45e8	`FMADD f3, f19, f23, f3`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 16 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 17 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 18 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 19 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 4 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 5 * SIZE(BO)`
kusano	2b45e8	`LFD f22, 6 * SIZE(BO)`
kusano	2b45e8	`LFD f23, 7 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`addi AO, AO, 16 * SIZE`
kusano	2b45e8	`addi BO, BO, 4 * SIZE`
kusano	2b45e8	`DCBT(BO, PREB)`
kusano	2b45e8	`bdnz LL(72)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(75):`
kusano	2b45e8	`#if defined(LT) \|\| defined(RN)`
kusano	2b45e8	`andi. r0, KK, 3`
kusano	2b45e8	`#else`
kusano	2b45e8	`andi. r0, TEMP, 3`
kusano	2b45e8	`#endif`
kusano	2b45e8	`mtspr CTR, r0`
kusano	2b45e8	`ble+ LL(78)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(76):`
kusano	2b45e8	`FMADD f0, f16, f20, f0`
kusano	2b45e8	`FMADD f1, f17, f20, f1`
kusano	2b45e8	`FMADD f2, f18, f20, f2`
kusano	2b45e8	`FMADD f3, f19, f20, f3`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 4 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 5 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 6 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 7 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 1 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`addi BO, BO, 1 * SIZE`
kusano	2b45e8	`addi AO, AO, 4 * SIZE`
kusano	2b45e8	`bdnz LL(76)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(78):`
kusano	2b45e8	`#if defined(LN) \|\| defined(RT)`
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`subi r0, KK, 4`
kusano	2b45e8	`#else`
kusano	2b45e8	`subi r0, KK, 1`
kusano	2b45e8	`#endif`
kusano	2b45e8	`slwi TEMP, r0, 2 + BASE_SHIFT`
kusano	2b45e8	`slwi r0, r0, 0 + BASE_SHIFT`
kusano	2b45e8	`add AO, AORIG, TEMP`
kusano	2b45e8	`add BO, B, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(LN) \|\| defined(LT)`
kusano	2b45e8	`LFD f16, 0 * SIZE(BO)`
kusano	2b45e8	`LFD f20, 1 * SIZE(BO)`
kusano	2b45e8	`LFD f24, 2 * SIZE(BO)`
kusano	2b45e8	`LFD f28, 3 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FSUB f0, f16, f0`
kusano	2b45e8	`FSUB f1, f20, f1`
kusano	2b45e8	`FSUB f2, f24, f2`
kusano	2b45e8	`FSUB f3, f28, f3`
kusano	2b45e8	`#else`
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FSUB f0, f16, f0`
kusano	2b45e8	`FSUB f1, f17, f1`
kusano	2b45e8	`FSUB f2, f18, f2`
kusano	2b45e8	`FSUB f3, f19, f3`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`LFD f16, 15 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 14 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 13 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 12 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f3, f16, f3`
kusano	2b45e8	`FNMSUB f2, f17, f3, f2`
kusano	2b45e8	`FNMSUB f1, f18, f3, f1`
kusano	2b45e8	`FNMSUB f0, f19, f3, f0`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 10 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 9 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 8 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 5 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 4 * SIZE(AO)`
kusano	2b45e8	`LFD f21, 0 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f2, f16, f2`
kusano	2b45e8	`FNMSUB f1, f17, f2, f1`
kusano	2b45e8	`FNMSUB f0, f18, f2, f0`
kusano	2b45e8
kusano	2b45e8	`FMUL f1, f19, f1`
kusano	2b45e8	`FNMSUB f0, f20, f1, f0`
kusano	2b45e8	`FMUL f0, f21, f0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LT`
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f0, f16, f0`
kusano	2b45e8	`FNMSUB f1, f17, f0, f1`
kusano	2b45e8	`FNMSUB f2, f18, f0, f2`
kusano	2b45e8	`FNMSUB f3, f19, f0, f3`
kusano	2b45e8
kusano	2b45e8	`LFD f17, 5 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 6 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 7 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f1, f17, f1`
kusano	2b45e8	`FNMSUB f2, f18, f1, f2`
kusano	2b45e8	`FNMSUB f3, f19, f1, f3`
kusano	2b45e8
kusano	2b45e8	`LFD f18, 10 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 11 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f2, f18, f2`
kusano	2b45e8	`FNMSUB f3, f19, f2, f3`
kusano	2b45e8
kusano	2b45e8	`LFD f19, 15 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f3, f19, f3`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RN`
kusano	2b45e8	`LFD f16, 0 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f0, f16, f0`
kusano	2b45e8	`FMUL f1, f16, f1`
kusano	2b45e8	`FMUL f2, f16, f2`
kusano	2b45e8	`FMUL f3, f16, f3`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RT`
kusano	2b45e8	`LFD f21, 0 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f0, f21, f0`
kusano	2b45e8	`FMUL f1, f21, f1`
kusano	2b45e8	`FMUL f2, f21, f2`
kusano	2b45e8	`FMUL f3, f21, f3`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`subi CO1, CO1, 4 * SIZE`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(LN) \|\| defined(LT)`
kusano	2b45e8	`STFD f0, 0 * SIZE(BO)`
kusano	2b45e8	`STFD f1, 1 * SIZE(BO)`
kusano	2b45e8	`STFD f2, 2 * SIZE(BO)`
kusano	2b45e8	`STFD f3, 3 * SIZE(BO)`
kusano	2b45e8	`#else`
kusano	2b45e8	`STFD f0, 0 * SIZE(AO)`
kusano	2b45e8	`STFD f1, 1 * SIZE(AO)`
kusano	2b45e8	`STFD f2, 2 * SIZE(AO)`
kusano	2b45e8	`STFD f3, 3 * SIZE(AO)`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`STFD f0, 0 * SIZE(CO1)`
kusano	2b45e8	`STFD f1, 1 * SIZE(CO1)`
kusano	2b45e8	`STFD f2, 2 * SIZE(CO1)`
kusano	2b45e8	`STFD f3, 3 * SIZE(CO1)`
kusano	2b45e8
kusano	2b45e8	`lfs f0, FZERO`
kusano	2b45e8	`fmr f1, f0`
kusano	2b45e8	`fmr f2, f0`
kusano	2b45e8	`fmr f3, f0`
kusano	2b45e8
kusano	2b45e8	`#ifndef LN`
kusano	2b45e8	`addi CO1, CO1, 4 * SIZE`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RT`
kusano	2b45e8	`slwi r0, K, 2 + BASE_SHIFT`
kusano	2b45e8	`add AORIG, AORIG, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(LT) \|\| defined(RN)`
kusano	2b45e8	`sub TEMP, K, KK`
kusano	2b45e8	`slwi r0, TEMP, 2 + BASE_SHIFT`
kusano	2b45e8	`slwi TEMP, TEMP, 0 + BASE_SHIFT`
kusano	2b45e8	`add AO, AO, r0`
kusano	2b45e8	`add BO, BO, TEMP`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`subi KK, KK, 4`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LT`
kusano	2b45e8	`addi KK, KK, 4`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`addic. I, I, -1`
kusano	2b45e8	`bgt+ LL(71)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(80):`
kusano	2b45e8	`andi. I, M, 2`
kusano	2b45e8	`ble LL(90)`
kusano	2b45e8
kusano	2b45e8	`#if defined(LT) \|\| defined(RN)`
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 0 * SIZE(B)`
kusano	2b45e8	`LFD f21, 1 * SIZE(B)`
kusano	2b45e8	`LFD f22, 2 * SIZE(B)`
kusano	2b45e8	`LFD f23, 3 * SIZE(B)`
kusano	2b45e8
kusano	2b45e8	`srawi. r0, KK, 2`
kusano	2b45e8	`mtspr CTR, r0`
kusano	2b45e8	`mr BO, B`
kusano	2b45e8	`#else`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`slwi r0, K, 1 + BASE_SHIFT`
kusano	2b45e8	`sub AORIG, AORIG, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`slwi r0, KK, 1 + BASE_SHIFT`
kusano	2b45e8	`slwi TEMP, KK, 0 + BASE_SHIFT`
kusano	2b45e8	`add AO, AORIG, r0`
kusano	2b45e8	`add BO, B, TEMP`
kusano	2b45e8
kusano	2b45e8	`sub TEMP, K, KK`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 0 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 1 * SIZE(BO)`
kusano	2b45e8	`LFD f22, 2 * SIZE(BO)`
kusano	2b45e8	`LFD f23, 3 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`srawi. r0, TEMP, 2`
kusano	2b45e8	`mtspr CTR, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8	`ble LL(85)`
kusano	2b45e8	`.align 5`
kusano	2b45e8
kusano	2b45e8	`LL(82):`
kusano	2b45e8	`FMADD f0, f16, f20, f0`
kusano	2b45e8	`FMADD f1, f17, f20, f1`
kusano	2b45e8	`FMADD f2, f18, f21, f2`
kusano	2b45e8	`FMADD f3, f19, f21, f3`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 4 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 5 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 6 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 7 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMADD f0, f16, f22, f0`
kusano	2b45e8	`FMADD f1, f17, f22, f1`
kusano	2b45e8	`FMADD f2, f18, f23, f2`
kusano	2b45e8	`FMADD f3, f19, f23, f3`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 8 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 9 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 10 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 11 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 4 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 5 * SIZE(BO)`
kusano	2b45e8	`LFD f22, 6 * SIZE(BO)`
kusano	2b45e8	`LFD f23, 7 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`addi AO, AO, 8 * SIZE`
kusano	2b45e8	`addi BO, BO, 4 * SIZE`
kusano	2b45e8	`DCBT(BO, PREB)`
kusano	2b45e8	`bdnz LL(82)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(85):`
kusano	2b45e8	`#if defined(LT) \|\| defined(RN)`
kusano	2b45e8	`andi. r0, KK, 3`
kusano	2b45e8	`#else`
kusano	2b45e8	`andi. r0, TEMP, 3`
kusano	2b45e8	`#endif`
kusano	2b45e8	`mtspr CTR, r0`
kusano	2b45e8	`ble+ LL(88)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(86):`
kusano	2b45e8	`FMADD f0, f16, f20, f0`
kusano	2b45e8	`FMADD f1, f17, f20, f1`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 3 * SIZE(AO)`
kusano	2b45e8	`LFD f20, 1 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`addi BO, BO, 1 * SIZE`
kusano	2b45e8	`addi AO, AO, 2 * SIZE`
kusano	2b45e8	`bdnz LL(86)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(88):`
kusano	2b45e8	`FADD f0, f2, f0`
kusano	2b45e8	`FADD f1, f3, f1`
kusano	2b45e8
kusano	2b45e8	`#if defined(LN) \|\| defined(RT)`
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`subi r0, KK, 2`
kusano	2b45e8	`#else`
kusano	2b45e8	`subi r0, KK, 1`
kusano	2b45e8	`#endif`
kusano	2b45e8	`slwi TEMP, r0, 1 + BASE_SHIFT`
kusano	2b45e8	`slwi r0, r0, 0 + BASE_SHIFT`
kusano	2b45e8	`add AO, AORIG, TEMP`
kusano	2b45e8	`add BO, B, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(LN) \|\| defined(LT)`
kusano	2b45e8	`LFD f16, 0 * SIZE(BO)`
kusano	2b45e8	`LFD f20, 1 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FSUB f0, f16, f0`
kusano	2b45e8	`FSUB f1, f20, f1`
kusano	2b45e8	`#else`
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FSUB f0, f16, f0`
kusano	2b45e8	`FSUB f1, f17, f1`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`LFD f19, 3 * SIZE(AO)`
kusano	2b45e8	`LFD f20, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f21, 0 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f1, f19, f1`
kusano	2b45e8	`FNMSUB f0, f20, f1, f0`
kusano	2b45e8	`FMUL f0, f21, f0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LT`
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f0, f16, f0`
kusano	2b45e8	`FNMSUB f1, f17, f0, f1`
kusano	2b45e8
kusano	2b45e8	`LFD f17, 3 * SIZE(AO)`
kusano	2b45e8	`FMUL f1, f17, f1`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RN`
kusano	2b45e8	`LFD f16, 0 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f0, f16, f0`
kusano	2b45e8	`FMUL f1, f16, f1`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RT`
kusano	2b45e8	`LFD f21, 0 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f0, f21, f0`
kusano	2b45e8	`FMUL f1, f21, f1`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`subi CO1, CO1, 2 * SIZE`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(LN) \|\| defined(LT)`
kusano	2b45e8	`STFD f0, 0 * SIZE(BO)`
kusano	2b45e8	`STFD f1, 1 * SIZE(BO)`
kusano	2b45e8	`#else`
kusano	2b45e8	`STFD f0, 0 * SIZE(AO)`
kusano	2b45e8	`STFD f1, 1 * SIZE(AO)`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`STFD f0, 0 * SIZE(CO1)`
kusano	2b45e8	`STFD f1, 1 * SIZE(CO1)`
kusano	2b45e8
kusano	2b45e8	`lfs f0, FZERO`
kusano	2b45e8	`fmr f1, f0`
kusano	2b45e8	`fmr f2, f0`
kusano	2b45e8	`fmr f3, f0`
kusano	2b45e8
kusano	2b45e8	`#ifndef LN`
kusano	2b45e8	`addi CO1, CO1, 2 * SIZE`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RT`
kusano	2b45e8	`slwi r0, K, 1 + BASE_SHIFT`
kusano	2b45e8	`add AORIG, AORIG, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(LT) \|\| defined(RN)`
kusano	2b45e8	`sub TEMP, K, KK`
kusano	2b45e8	`slwi r0, TEMP, 1 + BASE_SHIFT`
kusano	2b45e8	`slwi TEMP, TEMP, 0 + BASE_SHIFT`
kusano	2b45e8	`add AO, AO, r0`
kusano	2b45e8	`add BO, BO, TEMP`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`subi KK, KK, 2`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LT`
kusano	2b45e8	`addi KK, KK, 2`
kusano	2b45e8	`#endif`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(90):`
kusano	2b45e8	`andi. I, M, 1`
kusano	2b45e8	`ble LL(99)`
kusano	2b45e8
kusano	2b45e8	`#if defined(LT) \|\| defined(RN)`
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 0 * SIZE(B)`
kusano	2b45e8	`LFD f21, 1 * SIZE(B)`
kusano	2b45e8	`LFD f22, 2 * SIZE(B)`
kusano	2b45e8	`LFD f23, 3 * SIZE(B)`
kusano	2b45e8
kusano	2b45e8	`srawi. r0, KK, 3`
kusano	2b45e8	`mtspr CTR, r0`
kusano	2b45e8	`mr BO, B`
kusano	2b45e8	`#else`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`slwi r0, K, BASE_SHIFT`
kusano	2b45e8	`sub AORIG, AORIG, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`slwi r0, KK, 0 + BASE_SHIFT`
kusano	2b45e8	`slwi TEMP, KK, 0 + BASE_SHIFT`
kusano	2b45e8	`add AO, AORIG, r0`
kusano	2b45e8	`add BO, B, TEMP`
kusano	2b45e8
kusano	2b45e8	`sub TEMP, K, KK`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 0 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 1 * SIZE(BO)`
kusano	2b45e8	`LFD f22, 2 * SIZE(BO)`
kusano	2b45e8	`LFD f23, 3 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`srawi. r0, TEMP, 3`
kusano	2b45e8	`mtspr CTR, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8	`ble LL(95)`
kusano	2b45e8	`.align 5`
kusano	2b45e8
kusano	2b45e8	`LL(92):`
kusano	2b45e8	`FMADD f0, f16, f20, f0`
kusano	2b45e8	`FMADD f1, f17, f21, f1`
kusano	2b45e8	`FMADD f2, f18, f22, f2`
kusano	2b45e8	`FMADD f3, f19, f23, f3`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 4 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 5 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 6 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 7 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 4 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 5 * SIZE(BO)`
kusano	2b45e8	`LFD f22, 6 * SIZE(BO)`
kusano	2b45e8	`LFD f23, 7 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMADD f0, f16, f20, f0`
kusano	2b45e8	`FMADD f1, f17, f21, f1`
kusano	2b45e8	`FMADD f2, f18, f22, f2`
kusano	2b45e8	`FMADD f3, f19, f23, f3`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 8 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 9 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 10 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 11 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 8 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 9 * SIZE(BO)`
kusano	2b45e8	`LFD f22, 10 * SIZE(BO)`
kusano	2b45e8	`LFD f23, 11 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`addi AO, AO, 8 * SIZE`
kusano	2b45e8	`addi BO, BO, 8 * SIZE`
kusano	2b45e8	`bdnz LL(92)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(95):`
kusano	2b45e8	`#if defined(LT) \|\| defined(RN)`
kusano	2b45e8	`andi. r0, KK, 7`
kusano	2b45e8	`#else`
kusano	2b45e8	`andi. r0, TEMP, 7`
kusano	2b45e8	`#endif`
kusano	2b45e8	`mtspr CTR, r0`
kusano	2b45e8	`ble+ LL(98)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(96):`
kusano	2b45e8	`FMADD f0, f16, f20, f0`
kusano	2b45e8	`LFD f16, 1 * SIZE(AO)`
kusano	2b45e8	`LFD f20, 1 * SIZE(BO)`
kusano	2b45e8	`addi BO, BO, 1 * SIZE`
kusano	2b45e8	`addi AO, AO, 1 * SIZE`
kusano	2b45e8	`bdnz LL(96)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(98):`
kusano	2b45e8	`FADD f0, f1, f0`
kusano	2b45e8	`FADD f2, f3, f2`
kusano	2b45e8	`FADD f0, f2, f0`
kusano	2b45e8
kusano	2b45e8	`#if defined(LN) \|\| defined(RT)`
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`subi r0, KK, 1`
kusano	2b45e8	`#else`
kusano	2b45e8	`subi r0, KK, 1`
kusano	2b45e8	`#endif`
kusano	2b45e8	`slwi TEMP, r0, 0 + BASE_SHIFT`
kusano	2b45e8	`slwi r0, r0, 0 + BASE_SHIFT`
kusano	2b45e8	`add AO, AORIG, TEMP`
kusano	2b45e8	`add BO, B, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(LN) \|\| defined(LT)`
kusano	2b45e8	`LFD f16, 0 * SIZE(BO)`
kusano	2b45e8	`FSUB f0, f16, f0`
kusano	2b45e8	`#else`
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`FSUB f0, f16, f0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`LFD f21, 0 * SIZE(AO)`
kusano	2b45e8	`FMUL f0, f21, f0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LT`
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`FMUL f0, f16, f0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RN`
kusano	2b45e8	`LFD f16, 0 * SIZE(BO)`
kusano	2b45e8	`FMUL f0, f16, f0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RT`
kusano	2b45e8	`LFD f21, 0 * SIZE(BO)`
kusano	2b45e8	`FMUL f0, f21, f0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`subi CO1, CO1, 1 * SIZE`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(LN) \|\| defined(LT)`
kusano	2b45e8	`STFD f0, 0 * SIZE(BO)`
kusano	2b45e8	`#else`
kusano	2b45e8	`STFD f0, 0 * SIZE(AO)`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`STFD f0, 0 * SIZE(CO1)`
kusano	2b45e8
kusano	2b45e8	`lfs f0, FZERO`
kusano	2b45e8
kusano	2b45e8	`#ifndef LN`
kusano	2b45e8	`addi CO1, CO1, 1 * SIZE`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RT`
kusano	2b45e8	`slwi r0, K, 0 + BASE_SHIFT`
kusano	2b45e8	`add AORIG, AORIG, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(LT) \|\| defined(RN)`
kusano	2b45e8	`sub TEMP, K, KK`
kusano	2b45e8	`slwi r0, TEMP, 0 + BASE_SHIFT`
kusano	2b45e8	`slwi TEMP, TEMP, 0 + BASE_SHIFT`
kusano	2b45e8	`add AO, AO, r0`
kusano	2b45e8	`add BO, BO, TEMP`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`subi KK, KK, 1`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LT`
kusano	2b45e8	`addi KK, KK, 1`
kusano	2b45e8	`#endif`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(99):`
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`slwi r0, K, 0 + BASE_SHIFT`
kusano	2b45e8	`add B, B, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(LT) \|\| defined(RN)`
kusano	2b45e8	`mr B, BO`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RN`
kusano	2b45e8	`addi KK, KK, 1`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RT`
kusano	2b45e8	`subi KK, KK, 1`
kusano	2b45e8	`#endif`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(40):`
kusano	2b45e8	`andi. J, N, 2`
kusano	2b45e8	`ble LL(09)`
kusano	2b45e8
kusano	2b45e8	`#ifdef RT`
kusano	2b45e8	`slwi r0, K, 1 + BASE_SHIFT`
kusano	2b45e8	`sub B, B, r0`
kusano	2b45e8
kusano	2b45e8	`slwi r0, LDC, 1`
kusano	2b45e8	`sub C, C, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`mr CO1, C`
kusano	2b45e8	`add CO2, C, LDC`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`add KK, M, OFFSET`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LT`
kusano	2b45e8	`mr KK, OFFSET`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`fmr f1, f0`
kusano	2b45e8	`fmr f2, f0`
kusano	2b45e8	`fmr f3, f0`
kusano	2b45e8	`fmr f4, f0`
kusano	2b45e8	`fmr f5, f0`
kusano	2b45e8	`fmr f6, f0`
kusano	2b45e8	`fmr f7, f0`
kusano	2b45e8
kusano	2b45e8	`srawi. I, M, 2`
kusano	2b45e8	`#if defined(LN) \|\| defined(RT)`
kusano	2b45e8	`mr AORIG, A`
kusano	2b45e8	`#else`
kusano	2b45e8	`mr AO, A`
kusano	2b45e8	`#endif`
kusano	2b45e8	`#ifndef RT`
kusano	2b45e8	`add C, CO2, LDC`
kusano	2b45e8	`#endif`
kusano	2b45e8	`ble LL(50)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(41):`
kusano	2b45e8	`#if defined(LT) \|\| defined(RN)`
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 0 * SIZE(B)`
kusano	2b45e8	`LFD f21, 1 * SIZE(B)`
kusano	2b45e8	`LFD f22, 2 * SIZE(B)`
kusano	2b45e8	`LFD f23, 3 * SIZE(B)`
kusano	2b45e8
kusano	2b45e8	`dcbt CO1, PREC`
kusano	2b45e8	`dcbt CO2, PREC`
kusano	2b45e8
kusano	2b45e8	`srawi. r0, KK, 2`
kusano	2b45e8	`mtspr CTR, r0`
kusano	2b45e8	`mr BO, B`
kusano	2b45e8	`#else`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`slwi r0, K, 2 + BASE_SHIFT`
kusano	2b45e8	`sub AORIG, AORIG, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`slwi r0, KK, 2 + BASE_SHIFT`
kusano	2b45e8	`slwi TEMP, KK, 1 + BASE_SHIFT`
kusano	2b45e8	`add AO, AORIG, r0`
kusano	2b45e8	`add BO, B, TEMP`
kusano	2b45e8
kusano	2b45e8	`sub TEMP, K, KK`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 0 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 1 * SIZE(BO)`
kusano	2b45e8	`LFD f22, 2 * SIZE(BO)`
kusano	2b45e8	`LFD f23, 3 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`dcbt CO1, PREC`
kusano	2b45e8	`dcbt CO2, PREC`
kusano	2b45e8
kusano	2b45e8	`srawi. r0, TEMP, 2`
kusano	2b45e8	`mtspr CTR, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8	`ble LL(45)`
kusano	2b45e8	`.align 5`
kusano	2b45e8
kusano	2b45e8	`LL(42):`
kusano	2b45e8	`FMADD f0, f16, f20, f0`
kusano	2b45e8	`FMADD f1, f17, f20, f1`
kusano	2b45e8	`FMADD f2, f18, f20, f2`
kusano	2b45e8	`FMADD f3, f19, f20, f3`
kusano	2b45e8
kusano	2b45e8	`FMADD f4, f16, f21, f4`
kusano	2b45e8	`FMADD f5, f17, f21, f5`
kusano	2b45e8	`FMADD f6, f18, f21, f6`
kusano	2b45e8	`FMADD f7, f19, f21, f7`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 4 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 5 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 6 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 7 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMADD f0, f16, f22, f0`
kusano	2b45e8	`FMADD f1, f17, f22, f1`
kusano	2b45e8	`FMADD f2, f18, f22, f2`
kusano	2b45e8	`FMADD f3, f19, f22, f3`
kusano	2b45e8
kusano	2b45e8	`FMADD f4, f16, f23, f4`
kusano	2b45e8	`FMADD f5, f17, f23, f5`
kusano	2b45e8	`FMADD f6, f18, f23, f6`
kusano	2b45e8	`FMADD f7, f19, f23, f7`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 8 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 9 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 10 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 11 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 4 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 5 * SIZE(BO)`
kusano	2b45e8	`LFD f22, 6 * SIZE(BO)`
kusano	2b45e8	`LFD f23, 7 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMADD f0, f16, f20, f0`
kusano	2b45e8	`FMADD f1, f17, f20, f1`
kusano	2b45e8	`FMADD f2, f18, f20, f2`
kusano	2b45e8	`FMADD f3, f19, f20, f3`
kusano	2b45e8
kusano	2b45e8	`FMADD f4, f16, f21, f4`
kusano	2b45e8	`FMADD f5, f17, f21, f5`
kusano	2b45e8	`FMADD f6, f18, f21, f6`
kusano	2b45e8	`FMADD f7, f19, f21, f7`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 12 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 13 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 14 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 15 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMADD f0, f16, f22, f0`
kusano	2b45e8	`FMADD f1, f17, f22, f1`
kusano	2b45e8	`FMADD f2, f18, f22, f2`
kusano	2b45e8	`FMADD f3, f19, f22, f3`
kusano	2b45e8
kusano	2b45e8	`FMADD f4, f16, f23, f4`
kusano	2b45e8	`FMADD f5, f17, f23, f5`
kusano	2b45e8	`FMADD f6, f18, f23, f6`
kusano	2b45e8	`FMADD f7, f19, f23, f7`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 16 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 17 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 18 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 19 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 8 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 9 * SIZE(BO)`
kusano	2b45e8	`LFD f22, 10 * SIZE(BO)`
kusano	2b45e8	`LFD f23, 11 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`addi AO, AO, 16 * SIZE`
kusano	2b45e8	`addi BO, BO, 8 * SIZE`
kusano	2b45e8	`DCBT(BO, PREB)`
kusano	2b45e8	`bdnz LL(42)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(45):`
kusano	2b45e8	`#if defined(LT) \|\| defined(RN)`
kusano	2b45e8	`andi. r0, KK, 3`
kusano	2b45e8	`#else`
kusano	2b45e8	`andi. r0, TEMP, 3`
kusano	2b45e8	`#endif`
kusano	2b45e8	`mtspr CTR, r0`
kusano	2b45e8	`ble+ LL(48)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(46):`
kusano	2b45e8	`FMADD f0, f16, f20, f0`
kusano	2b45e8	`FMADD f1, f17, f20, f1`
kusano	2b45e8	`FMADD f2, f18, f20, f2`
kusano	2b45e8	`FMADD f3, f19, f20, f3`
kusano	2b45e8
kusano	2b45e8	`FMADD f4, f16, f21, f4`
kusano	2b45e8	`FMADD f5, f17, f21, f5`
kusano	2b45e8	`FMADD f6, f18, f21, f6`
kusano	2b45e8	`FMADD f7, f19, f21, f7`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 4 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 5 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 6 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 7 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 2 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 3 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`addi BO, BO, 2 * SIZE`
kusano	2b45e8	`addi AO, AO, 4 * SIZE`
kusano	2b45e8	`bdnz LL(46)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(48):`
kusano	2b45e8	`#if defined(LN) \|\| defined(RT)`
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`subi r0, KK, 4`
kusano	2b45e8	`#else`
kusano	2b45e8	`subi r0, KK, 2`
kusano	2b45e8	`#endif`
kusano	2b45e8	`slwi TEMP, r0, 2 + BASE_SHIFT`
kusano	2b45e8	`slwi r0, r0, 1 + BASE_SHIFT`
kusano	2b45e8	`add AO, AORIG, TEMP`
kusano	2b45e8	`add BO, B, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(LN) \|\| defined(LT)`
kusano	2b45e8	`LFD f16, 0 * SIZE(BO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(BO)`
kusano	2b45e8	`LFD f20, 2 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 3 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`LFD f24, 4 * SIZE(BO)`
kusano	2b45e8	`LFD f25, 5 * SIZE(BO)`
kusano	2b45e8	`LFD f28, 6 * SIZE(BO)`
kusano	2b45e8	`LFD f29, 7 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FSUB f0, f16, f0`
kusano	2b45e8	`FSUB f4, f17, f4`
kusano	2b45e8	`FSUB f1, f20, f1`
kusano	2b45e8	`FSUB f5, f21, f5`
kusano	2b45e8
kusano	2b45e8	`FSUB f2, f24, f2`
kusano	2b45e8	`FSUB f6, f25, f6`
kusano	2b45e8	`FSUB f3, f28, f3`
kusano	2b45e8	`FSUB f7, f29, f7`
kusano	2b45e8	`#else`
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 4 * SIZE(AO)`
kusano	2b45e8	`LFD f21, 5 * SIZE(AO)`
kusano	2b45e8	`LFD f22, 6 * SIZE(AO)`
kusano	2b45e8	`LFD f23, 7 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FSUB f0, f16, f0`
kusano	2b45e8	`FSUB f1, f17, f1`
kusano	2b45e8	`FSUB f2, f18, f2`
kusano	2b45e8	`FSUB f3, f19, f3`
kusano	2b45e8
kusano	2b45e8	`FSUB f4, f20, f4`
kusano	2b45e8	`FSUB f5, f21, f5`
kusano	2b45e8	`FSUB f6, f22, f6`
kusano	2b45e8	`FSUB f7, f23, f7`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`LFD f16, 15 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 14 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 13 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 12 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f3, f16, f3`
kusano	2b45e8	`FMUL f7, f16, f7`
kusano	2b45e8	`FNMSUB f2, f17, f3, f2`
kusano	2b45e8	`FNMSUB f6, f17, f7, f6`
kusano	2b45e8	`FNMSUB f1, f18, f3, f1`
kusano	2b45e8	`FNMSUB f5, f18, f7, f5`
kusano	2b45e8	`FNMSUB f0, f19, f3, f0`
kusano	2b45e8	`FNMSUB f4, f19, f7, f4`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 10 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 9 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 8 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 5 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 4 * SIZE(AO)`
kusano	2b45e8	`LFD f21, 0 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f2, f16, f2`
kusano	2b45e8	`FMUL f6, f16, f6`
kusano	2b45e8	`FNMSUB f1, f17, f2, f1`
kusano	2b45e8	`FNMSUB f5, f17, f6, f5`
kusano	2b45e8	`FNMSUB f0, f18, f2, f0`
kusano	2b45e8	`FNMSUB f4, f18, f6, f4`
kusano	2b45e8
kusano	2b45e8	`FMUL f1, f19, f1`
kusano	2b45e8	`FMUL f5, f19, f5`
kusano	2b45e8	`FNMSUB f0, f20, f1, f0`
kusano	2b45e8	`FNMSUB f4, f20, f5, f4`
kusano	2b45e8	`FMUL f0, f21, f0`
kusano	2b45e8	`FMUL f4, f21, f4`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LT`
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f0, f16, f0`
kusano	2b45e8	`FMUL f4, f16, f4`
kusano	2b45e8	`FNMSUB f1, f17, f0, f1`
kusano	2b45e8	`FNMSUB f5, f17, f4, f5`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f2, f18, f0, f2`
kusano	2b45e8	`FNMSUB f6, f18, f4, f6`
kusano	2b45e8	`FNMSUB f3, f19, f0, f3`
kusano	2b45e8	`FNMSUB f7, f19, f4, f7`
kusano	2b45e8
kusano	2b45e8	`LFD f17, 5 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 6 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 7 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f1, f17, f1`
kusano	2b45e8	`FMUL f5, f17, f5`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f2, f18, f1, f2`
kusano	2b45e8	`FNMSUB f6, f18, f5, f6`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f3, f19, f1, f3`
kusano	2b45e8	`FNMSUB f7, f19, f5, f7`
kusano	2b45e8
kusano	2b45e8	`LFD f18, 10 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 11 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f2, f18, f2`
kusano	2b45e8	`FMUL f6, f18, f6`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f3, f19, f2, f3`
kusano	2b45e8	`FNMSUB f7, f19, f6, f7`
kusano	2b45e8
kusano	2b45e8	`LFD f19, 15 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f3, f19, f3`
kusano	2b45e8	`FMUL f7, f19, f7`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RN`
kusano	2b45e8	`LFD f16, 0 * SIZE(BO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(BO)`
kusano	2b45e8	`LFD f18, 3 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f0, f16, f0`
kusano	2b45e8	`FMUL f1, f16, f1`
kusano	2b45e8	`FMUL f2, f16, f2`
kusano	2b45e8	`FMUL f3, f16, f3`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f4, f17, f0, f4`
kusano	2b45e8	`FNMSUB f5, f17, f1, f5`
kusano	2b45e8	`FNMSUB f6, f17, f2, f6`
kusano	2b45e8	`FNMSUB f7, f17, f3, f7`
kusano	2b45e8
kusano	2b45e8	`FMUL f4, f18, f4`
kusano	2b45e8	`FMUL f5, f18, f5`
kusano	2b45e8	`FMUL f6, f18, f6`
kusano	2b45e8	`FMUL f7, f18, f7`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RT`
kusano	2b45e8	`LFD f19, 3 * SIZE(BO)`
kusano	2b45e8	`LFD f20, 2 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 0 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f4, f19, f4`
kusano	2b45e8	`FMUL f5, f19, f5`
kusano	2b45e8	`FMUL f6, f19, f6`
kusano	2b45e8	`FMUL f7, f19, f7`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f0, f20, f4, f0`
kusano	2b45e8	`FNMSUB f1, f20, f5, f1`
kusano	2b45e8	`FNMSUB f2, f20, f6, f2`
kusano	2b45e8	`FNMSUB f3, f20, f7, f3`
kusano	2b45e8
kusano	2b45e8	`FMUL f0, f21, f0`
kusano	2b45e8	`FMUL f1, f21, f1`
kusano	2b45e8	`FMUL f2, f21, f2`
kusano	2b45e8	`FMUL f3, f21, f3`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`subi CO1, CO1, 4 * SIZE`
kusano	2b45e8	`subi CO2, CO2, 4 * SIZE`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(LN) \|\| defined(LT)`
kusano	2b45e8	`STFD f0, 0 * SIZE(BO)`
kusano	2b45e8	`STFD f4, 1 * SIZE(BO)`
kusano	2b45e8	`STFD f1, 2 * SIZE(BO)`
kusano	2b45e8	`STFD f5, 3 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`STFD f2, 4 * SIZE(BO)`
kusano	2b45e8	`STFD f6, 5 * SIZE(BO)`
kusano	2b45e8	`STFD f3, 6 * SIZE(BO)`
kusano	2b45e8	`STFD f7, 7 * SIZE(BO)`
kusano	2b45e8	`#else`
kusano	2b45e8	`STFD f0, 0 * SIZE(AO)`
kusano	2b45e8	`STFD f1, 1 * SIZE(AO)`
kusano	2b45e8	`STFD f2, 2 * SIZE(AO)`
kusano	2b45e8	`STFD f3, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`STFD f4, 4 * SIZE(AO)`
kusano	2b45e8	`STFD f5, 5 * SIZE(AO)`
kusano	2b45e8	`STFD f6, 6 * SIZE(AO)`
kusano	2b45e8	`STFD f7, 7 * SIZE(AO)`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`STFD f0, 0 * SIZE(CO1)`
kusano	2b45e8	`STFD f1, 1 * SIZE(CO1)`
kusano	2b45e8	`STFD f2, 2 * SIZE(CO1)`
kusano	2b45e8	`STFD f3, 3 * SIZE(CO1)`
kusano	2b45e8
kusano	2b45e8	`STFD f4, 0 * SIZE(CO2)`
kusano	2b45e8	`STFD f5, 1 * SIZE(CO2)`
kusano	2b45e8	`STFD f6, 2 * SIZE(CO2)`
kusano	2b45e8	`STFD f7, 3 * SIZE(CO2)`
kusano	2b45e8
kusano	2b45e8	`lfs f0, FZERO`
kusano	2b45e8	`fmr f1, f0`
kusano	2b45e8	`fmr f2, f0`
kusano	2b45e8	`fmr f3, f0`
kusano	2b45e8
kusano	2b45e8	`fmr f4, f0`
kusano	2b45e8	`fmr f5, f0`
kusano	2b45e8	`fmr f6, f0`
kusano	2b45e8	`fmr f7, f0`
kusano	2b45e8
kusano	2b45e8	`#ifndef LN`
kusano	2b45e8	`addi CO1, CO1, 4 * SIZE`
kusano	2b45e8	`addi CO2, CO2, 4 * SIZE`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RT`
kusano	2b45e8	`slwi r0, K, 2 + BASE_SHIFT`
kusano	2b45e8	`add AORIG, AORIG, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(LT) \|\| defined(RN)`
kusano	2b45e8	`sub TEMP, K, KK`
kusano	2b45e8	`slwi r0, TEMP, 2 + BASE_SHIFT`
kusano	2b45e8	`slwi TEMP, TEMP, 1 + BASE_SHIFT`
kusano	2b45e8	`add AO, AO, r0`
kusano	2b45e8	`add BO, BO, TEMP`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`subi KK, KK, 4`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LT`
kusano	2b45e8	`addi KK, KK, 4`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`addic. I, I, -1`
kusano	2b45e8	`bgt+ LL(41)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(50):`
kusano	2b45e8	`andi. I, M, 2`
kusano	2b45e8	`ble LL(60)`
kusano	2b45e8
kusano	2b45e8	`#if defined(LT) \|\| defined(RN)`
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 0 * SIZE(B)`
kusano	2b45e8	`LFD f21, 1 * SIZE(B)`
kusano	2b45e8	`LFD f22, 2 * SIZE(B)`
kusano	2b45e8	`LFD f23, 3 * SIZE(B)`
kusano	2b45e8
kusano	2b45e8	`LFD f24, 4 * SIZE(B)`
kusano	2b45e8	`LFD f25, 5 * SIZE(B)`
kusano	2b45e8	`LFD f26, 6 * SIZE(B)`
kusano	2b45e8	`LFD f27, 7 * SIZE(B)`
kusano	2b45e8
kusano	2b45e8	`srawi. r0, KK, 2`
kusano	2b45e8	`mtspr CTR, r0`
kusano	2b45e8	`mr BO, B`
kusano	2b45e8	`#else`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`slwi r0, K, 1 + BASE_SHIFT`
kusano	2b45e8	`sub AORIG, AORIG, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`slwi r0, KK, 1 + BASE_SHIFT`
kusano	2b45e8	`slwi TEMP, KK, 1 + BASE_SHIFT`
kusano	2b45e8	`add AO, AORIG, r0`
kusano	2b45e8	`add BO, B, TEMP`
kusano	2b45e8
kusano	2b45e8	`sub TEMP, K, KK`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 0 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 1 * SIZE(BO)`
kusano	2b45e8	`LFD f22, 2 * SIZE(BO)`
kusano	2b45e8	`LFD f23, 3 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`LFD f24, 4 * SIZE(BO)`
kusano	2b45e8	`LFD f25, 5 * SIZE(BO)`
kusano	2b45e8	`LFD f26, 6 * SIZE(BO)`
kusano	2b45e8	`LFD f27, 7 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`srawi. r0, TEMP, 2`
kusano	2b45e8	`mtspr CTR, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8	`ble LL(55)`
kusano	2b45e8	`.align 5`
kusano	2b45e8
kusano	2b45e8	`LL(52):`
kusano	2b45e8	`FMADD f0, f16, f20, f0`
kusano	2b45e8	`FMADD f1, f17, f20, f1`
kusano	2b45e8	`FMADD f2, f16, f21, f2`
kusano	2b45e8	`FMADD f3, f17, f21, f3`
kusano	2b45e8
kusano	2b45e8	`FMADD f4, f18, f22, f4`
kusano	2b45e8	`FMADD f5, f19, f22, f5`
kusano	2b45e8	`FMADD f6, f18, f23, f6`
kusano	2b45e8	`FMADD f7, f19, f23, f7`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 4 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 5 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 6 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 7 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 8 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 9 * SIZE(BO)`
kusano	2b45e8	`LFD f22, 10 * SIZE(BO)`
kusano	2b45e8	`LFD f23, 11 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMADD f0, f16, f24, f0`
kusano	2b45e8	`FMADD f1, f17, f24, f1`
kusano	2b45e8	`FMADD f2, f16, f25, f2`
kusano	2b45e8	`FMADD f3, f17, f25, f3`
kusano	2b45e8
kusano	2b45e8	`FMADD f4, f18, f26, f4`
kusano	2b45e8	`FMADD f5, f19, f26, f5`
kusano	2b45e8	`FMADD f6, f18, f27, f6`
kusano	2b45e8	`FMADD f7, f19, f27, f7`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 8 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 9 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 10 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 11 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f24, 12 * SIZE(BO)`
kusano	2b45e8	`LFD f25, 13 * SIZE(BO)`
kusano	2b45e8	`LFD f26, 14 * SIZE(BO)`
kusano	2b45e8	`LFD f27, 15 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`addi AO, AO, 8 * SIZE`
kusano	2b45e8	`addi BO, BO, 8 * SIZE`
kusano	2b45e8	`DCBT(BO, PREB)`
kusano	2b45e8	`bdnz LL(52)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(55):`
kusano	2b45e8	`#if defined(LT) \|\| defined(RN)`
kusano	2b45e8	`andi. r0, KK, 3`
kusano	2b45e8	`#else`
kusano	2b45e8	`andi. r0, TEMP, 3`
kusano	2b45e8	`#endif`
kusano	2b45e8	`mtspr CTR, r0`
kusano	2b45e8	`ble+ LL(58)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(56):`
kusano	2b45e8	`FMADD f0, f16, f20, f0`
kusano	2b45e8	`FMADD f1, f17, f20, f1`
kusano	2b45e8	`FMADD f2, f16, f21, f2`
kusano	2b45e8	`FMADD f3, f17, f21, f3`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 3 * SIZE(AO)`
kusano	2b45e8	`LFD f20, 2 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 3 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`addi BO, BO, 2 * SIZE`
kusano	2b45e8	`addi AO, AO, 2 * SIZE`
kusano	2b45e8	`bdnz LL(56)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(58):`
kusano	2b45e8	`FADD f0, f4, f0`
kusano	2b45e8	`FADD f1, f5, f1`
kusano	2b45e8	`FADD f2, f6, f2`
kusano	2b45e8	`FADD f3, f7, f3`
kusano	2b45e8
kusano	2b45e8	`#if defined(LN) \|\| defined(RT)`
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`subi r0, KK, 2`
kusano	2b45e8	`#else`
kusano	2b45e8	`subi r0, KK, 2`
kusano	2b45e8	`#endif`
kusano	2b45e8	`slwi TEMP, r0, 1 + BASE_SHIFT`
kusano	2b45e8	`slwi r0, r0, 1 + BASE_SHIFT`
kusano	2b45e8	`add AO, AORIG, TEMP`
kusano	2b45e8	`add BO, B, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(LN) \|\| defined(LT)`
kusano	2b45e8	`LFD f16, 0 * SIZE(BO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(BO)`
kusano	2b45e8	`LFD f20, 2 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 3 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FSUB f0, f16, f0`
kusano	2b45e8	`FSUB f2, f17, f2`
kusano	2b45e8	`FSUB f1, f20, f1`
kusano	2b45e8	`FSUB f3, f21, f3`
kusano	2b45e8	`#else`
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(AO)`
kusano	2b45e8	`LFD f20, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f21, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FSUB f0, f16, f0`
kusano	2b45e8	`FSUB f1, f17, f1`
kusano	2b45e8	`FSUB f2, f20, f2`
kusano	2b45e8	`FSUB f3, f21, f3`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`LFD f19, 3 * SIZE(AO)`
kusano	2b45e8	`LFD f20, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f21, 0 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f1, f19, f1`
kusano	2b45e8	`FMUL f3, f19, f3`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f0, f20, f1, f0`
kusano	2b45e8	`FNMSUB f2, f20, f3, f2`
kusano	2b45e8
kusano	2b45e8	`FMUL f0, f21, f0`
kusano	2b45e8	`FMUL f2, f21, f2`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LT`
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f0, f16, f0`
kusano	2b45e8	`FMUL f2, f16, f2`
kusano	2b45e8	`FNMSUB f1, f17, f0, f1`
kusano	2b45e8	`FNMSUB f3, f17, f2, f3`
kusano	2b45e8
kusano	2b45e8	`LFD f17, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f1, f17, f1`
kusano	2b45e8	`FMUL f3, f17, f3`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RN`
kusano	2b45e8	`LFD f16, 0 * SIZE(BO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(BO)`
kusano	2b45e8	`LFD f18, 3 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f0, f16, f0`
kusano	2b45e8	`FMUL f1, f16, f1`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f2, f17, f0, f2`
kusano	2b45e8	`FNMSUB f3, f17, f1, f3`
kusano	2b45e8	`FMUL f2, f18, f2`
kusano	2b45e8	`FMUL f3, f18, f3`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RT`
kusano	2b45e8	`LFD f19, 3 * SIZE(BO)`
kusano	2b45e8	`LFD f20, 2 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 0 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f2, f19, f2`
kusano	2b45e8	`FMUL f3, f19, f3`
kusano	2b45e8	`FNMSUB f0, f20, f2, f0`
kusano	2b45e8	`FNMSUB f1, f20, f3, f1`
kusano	2b45e8	`FMUL f0, f21, f0`
kusano	2b45e8	`FMUL f1, f21, f1`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`subi CO1, CO1, 2 * SIZE`
kusano	2b45e8	`subi CO2, CO2, 2 * SIZE`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(LN) \|\| defined(LT)`
kusano	2b45e8	`STFD f0, 0 * SIZE(BO)`
kusano	2b45e8	`STFD f2, 1 * SIZE(BO)`
kusano	2b45e8	`STFD f1, 2 * SIZE(BO)`
kusano	2b45e8	`STFD f3, 3 * SIZE(BO)`
kusano	2b45e8	`#else`
kusano	2b45e8	`STFD f0, 0 * SIZE(AO)`
kusano	2b45e8	`STFD f1, 1 * SIZE(AO)`
kusano	2b45e8	`STFD f2, 2 * SIZE(AO)`
kusano	2b45e8	`STFD f3, 3 * SIZE(AO)`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`STFD f0, 0 * SIZE(CO1)`
kusano	2b45e8	`STFD f1, 1 * SIZE(CO1)`
kusano	2b45e8	`STFD f2, 0 * SIZE(CO2)`
kusano	2b45e8	`STFD f3, 1 * SIZE(CO2)`
kusano	2b45e8
kusano	2b45e8	`lfs f0, FZERO`
kusano	2b45e8	`fmr f1, f0`
kusano	2b45e8	`fmr f2, f0`
kusano	2b45e8	`fmr f3, f0`
kusano	2b45e8
kusano	2b45e8	`fmr f4, f0`
kusano	2b45e8	`fmr f5, f0`
kusano	2b45e8	`fmr f6, f0`
kusano	2b45e8	`fmr f7, f0`
kusano	2b45e8
kusano	2b45e8	`#ifndef LN`
kusano	2b45e8	`addi CO1, CO1, 2 * SIZE`
kusano	2b45e8	`addi CO2, CO2, 2 * SIZE`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RT`
kusano	2b45e8	`slwi r0, K, 1 + BASE_SHIFT`
kusano	2b45e8	`add AORIG, AORIG, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(LT) \|\| defined(RN)`
kusano	2b45e8	`sub TEMP, K, KK`
kusano	2b45e8	`slwi r0, TEMP, 1 + BASE_SHIFT`
kusano	2b45e8	`slwi TEMP, TEMP, 1 + BASE_SHIFT`
kusano	2b45e8	`add AO, AO, r0`
kusano	2b45e8	`add BO, BO, TEMP`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`subi KK, KK, 2`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LT`
kusano	2b45e8	`addi KK, KK, 2`
kusano	2b45e8	`#endif`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(60):`
kusano	2b45e8	`andi. I, M, 1`
kusano	2b45e8	`ble LL(69)`
kusano	2b45e8
kusano	2b45e8	`#if defined(LT) \|\| defined(RN)`
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 0 * SIZE(B)`
kusano	2b45e8	`LFD f21, 1 * SIZE(B)`
kusano	2b45e8	`LFD f22, 2 * SIZE(B)`
kusano	2b45e8	`LFD f23, 3 * SIZE(B)`
kusano	2b45e8
kusano	2b45e8	`LFD f24, 4 * SIZE(B)`
kusano	2b45e8	`LFD f25, 5 * SIZE(B)`
kusano	2b45e8	`LFD f26, 6 * SIZE(B)`
kusano	2b45e8	`LFD f27, 7 * SIZE(B)`
kusano	2b45e8
kusano	2b45e8	`srawi. r0, KK, 2`
kusano	2b45e8	`mtspr CTR, r0`
kusano	2b45e8	`mr BO, B`
kusano	2b45e8	`#else`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`slwi r0, K, BASE_SHIFT`
kusano	2b45e8	`sub AORIG, AORIG, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`slwi r0, KK, 0 + BASE_SHIFT`
kusano	2b45e8	`slwi TEMP, KK, 1 + BASE_SHIFT`
kusano	2b45e8	`add AO, AORIG, r0`
kusano	2b45e8	`add BO, B, TEMP`
kusano	2b45e8
kusano	2b45e8	`sub TEMP, K, KK`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 0 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 1 * SIZE(BO)`
kusano	2b45e8	`LFD f22, 2 * SIZE(BO)`
kusano	2b45e8	`LFD f23, 3 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`LFD f24, 4 * SIZE(BO)`
kusano	2b45e8	`LFD f25, 5 * SIZE(BO)`
kusano	2b45e8	`LFD f26, 6 * SIZE(BO)`
kusano	2b45e8	`LFD f27, 7 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`srawi. r0, TEMP, 2`
kusano	2b45e8	`mtspr CTR, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8	`ble LL(65)`
kusano	2b45e8	`.align 5`
kusano	2b45e8
kusano	2b45e8	`LL(62):`
kusano	2b45e8	`FMADD f0, f16, f20, f0`
kusano	2b45e8	`FMADD f1, f16, f21, f1`
kusano	2b45e8	`FMADD f2, f17, f22, f2`
kusano	2b45e8	`FMADD f3, f17, f23, f3`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 8 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 9 * SIZE(BO)`
kusano	2b45e8	`LFD f22, 10 * SIZE(BO)`
kusano	2b45e8	`LFD f23, 11 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMADD f0, f18, f24, f0`
kusano	2b45e8	`FMADD f1, f18, f25, f1`
kusano	2b45e8	`FMADD f2, f19, f26, f2`
kusano	2b45e8	`FMADD f3, f19, f27, f3`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 4 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 5 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 6 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 7 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f24, 12 * SIZE(BO)`
kusano	2b45e8	`LFD f25, 13 * SIZE(BO)`
kusano	2b45e8	`LFD f26, 14 * SIZE(BO)`
kusano	2b45e8	`LFD f27, 15 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`addi AO, AO, 4 * SIZE`
kusano	2b45e8	`addi BO, BO, 8 * SIZE`
kusano	2b45e8	`bdnz LL(62)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(65):`
kusano	2b45e8	`#if defined(LT) \|\| defined(RN)`
kusano	2b45e8	`andi. r0, KK, 3`
kusano	2b45e8	`#else`
kusano	2b45e8	`andi. r0, TEMP, 3`
kusano	2b45e8	`#endif`
kusano	2b45e8	`mtspr CTR, r0`
kusano	2b45e8	`ble+ LL(68)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(66):`
kusano	2b45e8	`FMADD f0, f16, f20, f0`
kusano	2b45e8	`FMADD f1, f16, f21, f1`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 1 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 2 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 3 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`addi BO, BO, 2 * SIZE`
kusano	2b45e8	`addi AO, AO, 1 * SIZE`
kusano	2b45e8	`bdnz LL(66)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(68):`
kusano	2b45e8	`FADD f0, f2, f0`
kusano	2b45e8	`FADD f1, f3, f1`
kusano	2b45e8
kusano	2b45e8	`#if defined(LN) \|\| defined(RT)`
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`subi r0, KK, 1`
kusano	2b45e8	`#else`
kusano	2b45e8	`subi r0, KK, 2`
kusano	2b45e8	`#endif`
kusano	2b45e8	`slwi TEMP, r0, 0 + BASE_SHIFT`
kusano	2b45e8	`slwi r0, r0, 1 + BASE_SHIFT`
kusano	2b45e8	`add AO, AORIG, TEMP`
kusano	2b45e8	`add BO, B, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(LN) \|\| defined(LT)`
kusano	2b45e8	`LFD f16, 0 * SIZE(BO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FSUB f0, f16, f0`
kusano	2b45e8	`FSUB f1, f17, f1`
kusano	2b45e8	`#else`
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`LFD f20, 1 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FSUB f0, f16, f0`
kusano	2b45e8	`FSUB f1, f20, f1`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`LFD f21, 0 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f0, f21, f0`
kusano	2b45e8	`FMUL f1, f21, f1`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LT`
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f0, f16, f0`
kusano	2b45e8	`FMUL f1, f16, f1`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RN`
kusano	2b45e8	`LFD f16, 0 * SIZE(BO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(BO)`
kusano	2b45e8	`LFD f18, 3 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f0, f16, f0`
kusano	2b45e8	`FNMSUB f1, f17, f0, f1`
kusano	2b45e8	`FMUL f1, f18, f1`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RT`
kusano	2b45e8	`LFD f19, 3 * SIZE(BO)`
kusano	2b45e8	`LFD f20, 2 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 0 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f1, f19, f1`
kusano	2b45e8	`FNMSUB f0, f20, f1, f0`
kusano	2b45e8	`FMUL f0, f21, f0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`subi CO1, CO1, 1 * SIZE`
kusano	2b45e8	`subi CO2, CO2, 1 * SIZE`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(LN) \|\| defined(LT)`
kusano	2b45e8	`STFD f0, 0 * SIZE(BO)`
kusano	2b45e8	`STFD f1, 1 * SIZE(BO)`
kusano	2b45e8	`#else`
kusano	2b45e8	`STFD f0, 0 * SIZE(AO)`
kusano	2b45e8	`STFD f1, 1 * SIZE(AO)`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`STFD f0, 0 * SIZE(CO1)`
kusano	2b45e8	`STFD f1, 0 * SIZE(CO2)`
kusano	2b45e8
kusano	2b45e8	`lfs f0, FZERO`
kusano	2b45e8	`fmr f1, f0`
kusano	2b45e8	`fmr f4, f0`
kusano	2b45e8	`fmr f5, f0`
kusano	2b45e8
kusano	2b45e8	`#ifndef LN`
kusano	2b45e8	`addi CO1, CO1, 1 * SIZE`
kusano	2b45e8	`addi CO2, CO2, 1 * SIZE`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RT`
kusano	2b45e8	`slwi r0, K, 0 + BASE_SHIFT`
kusano	2b45e8	`add AORIG, AORIG, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(LT) \|\| defined(RN)`
kusano	2b45e8	`sub TEMP, K, KK`
kusano	2b45e8	`slwi r0, TEMP, 0 + BASE_SHIFT`
kusano	2b45e8	`slwi TEMP, TEMP, 1 + BASE_SHIFT`
kusano	2b45e8	`add AO, AO, r0`
kusano	2b45e8	`add BO, BO, TEMP`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`subi KK, KK, 1`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LT`
kusano	2b45e8	`addi KK, KK, 1`
kusano	2b45e8	`#endif`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(69):`
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`slwi r0, K, 1 + BASE_SHIFT`
kusano	2b45e8	`add B, B, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(LT) \|\| defined(RN)`
kusano	2b45e8	`mr B, BO`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RN`
kusano	2b45e8	`addi KK, KK, 2`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RT`
kusano	2b45e8	`subi KK, KK, 2`
kusano	2b45e8	`#endif`
kusano	2b45e8	`lfs f0, FZERO`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(09):`
kusano	2b45e8	`srawi. J, N, 2`
kusano	2b45e8	`ble LL(999)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(10):`
kusano	2b45e8
kusano	2b45e8	`#ifdef RT`
kusano	2b45e8	`slwi r0, K, 2 + BASE_SHIFT`
kusano	2b45e8	`sub B, B, r0`
kusano	2b45e8
kusano	2b45e8	`slwi r0, LDC, 2`
kusano	2b45e8	`sub C, C, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`mr CO1, C`
kusano	2b45e8	`add CO2, C, LDC`
kusano	2b45e8	`add CO3, CO2, LDC`
kusano	2b45e8	`add CO4, CO3, LDC`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`add KK, M, OFFSET`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LT`
kusano	2b45e8	`mr KK, OFFSET`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`fmr f1, f0`
kusano	2b45e8	`fmr f2, f0`
kusano	2b45e8	`fmr f3, f0`
kusano	2b45e8	`fmr f4, f0`
kusano	2b45e8	`fmr f5, f0`
kusano	2b45e8	`fmr f6, f0`
kusano	2b45e8	`fmr f7, f0`
kusano	2b45e8	`fmr f8, f0`
kusano	2b45e8	`fmr f9, f0`
kusano	2b45e8	`fmr f10, f0`
kusano	2b45e8	`fmr f11, f0`
kusano	2b45e8	`fmr f12, f0`
kusano	2b45e8	`fmr f13, f0`
kusano	2b45e8	`fmr f14, f0`
kusano	2b45e8	`fmr f15, f0`
kusano	2b45e8
kusano	2b45e8	`srawi. I, M, 2`
kusano	2b45e8
kusano	2b45e8	`#if defined(LN) \|\| defined(RT)`
kusano	2b45e8	`mr AORIG, A`
kusano	2b45e8	`#else`
kusano	2b45e8	`mr AO, A`
kusano	2b45e8	`#endif`
kusano	2b45e8	`#ifndef RT`
kusano	2b45e8	`add C, CO4, LDC`
kusano	2b45e8	`#endif`
kusano	2b45e8	`ble LL(20)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(11):`
kusano	2b45e8	`#if defined(LT) \|\| defined(RN)`
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 0 * SIZE(B)`
kusano	2b45e8	`LFD f21, 1 * SIZE(B)`
kusano	2b45e8	`LFD f22, 2 * SIZE(B)`
kusano	2b45e8	`LFD f23, 3 * SIZE(B)`
kusano	2b45e8
kusano	2b45e8	`dcbt CO1, PREC`
kusano	2b45e8	`dcbt CO2, PREC`
kusano	2b45e8	`dcbt CO3, PREC`
kusano	2b45e8	`dcbt CO4, PREC`
kusano	2b45e8
kusano	2b45e8	`srawi. r0, KK, 2`
kusano	2b45e8	`mtspr CTR, r0`
kusano	2b45e8	`mr BO, B`
kusano	2b45e8	`#else`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`slwi r0, K, 2 + BASE_SHIFT`
kusano	2b45e8	`sub AORIG, AORIG, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`slwi TEMP, KK, 2 + BASE_SHIFT`
kusano	2b45e8	`add AO, AORIG, TEMP`
kusano	2b45e8	`add BO, B, TEMP`
kusano	2b45e8
kusano	2b45e8	`sub TEMP, K, KK`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 0 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 1 * SIZE(BO)`
kusano	2b45e8	`LFD f22, 2 * SIZE(BO)`
kusano	2b45e8	`LFD f23, 3 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`dcbt CO1, PREC`
kusano	2b45e8	`dcbt CO2, PREC`
kusano	2b45e8	`dcbt CO3, PREC`
kusano	2b45e8	`dcbt CO4, PREC`
kusano	2b45e8
kusano	2b45e8	`srawi. r0, TEMP, 2`
kusano	2b45e8	`mtspr CTR, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8	`ble LL(15)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(12):`
kusano	2b45e8	`FMADD f0, f16, f20, f0`
kusano	2b45e8	`FMADD f5, f17, f21, f5`
kusano	2b45e8	`FMADD f10, f18, f22, f10`
kusano	2b45e8	`FMADD f15, f19, f23, f15`
kusano	2b45e8
kusano	2b45e8	`LFD f28, 4 * SIZE(BO)`
kusano	2b45e8	`LFD f29, 5 * SIZE(BO)`
kusano	2b45e8	`LFD f30, 6 * SIZE(BO)`
kusano	2b45e8	`LFD f31, 7 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMADD f1, f17, f20, f1`
kusano	2b45e8	`FMADD f2, f18, f20, f2`
kusano	2b45e8	`FMADD f3, f19, f20, f3`
kusano	2b45e8	`FMADD f4, f16, f21, f4`
kusano	2b45e8
kusano	2b45e8	`LFD f24, 4 * SIZE(AO)`
kusano	2b45e8	`LFD f25, 5 * SIZE(AO)`
kusano	2b45e8	`LFD f26, 6 * SIZE(AO)`
kusano	2b45e8	`LFD f27, 7 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMADD f6, f18, f21, f6`
kusano	2b45e8	`FMADD f7, f19, f21, f7`
kusano	2b45e8	`FMADD f8, f16, f22, f8`
kusano	2b45e8	`FMADD f9, f17, f22, f9`
kusano	2b45e8
kusano	2b45e8	`FMADD f11, f19, f22, f11`
kusano	2b45e8	`FMADD f12, f16, f23, f12`
kusano	2b45e8	`FMADD f13, f17, f23, f13`
kusano	2b45e8	`FMADD f14, f18, f23, f14`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 8 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 9 * SIZE(BO)`
kusano	2b45e8	`LFD f22, 10 * SIZE(BO)`
kusano	2b45e8	`LFD f23, 11 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMADD f0, f24, f28, f0`
kusano	2b45e8	`FMADD f5, f25, f29, f5`
kusano	2b45e8	`FMADD f10, f26, f30, f10`
kusano	2b45e8	`FMADD f15, f27, f31, f15`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 8 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 9 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 10 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 11 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMADD f1, f25, f28, f1`
kusano	2b45e8	`FMADD f2, f26, f28, f2`
kusano	2b45e8	`FMADD f3, f27, f28, f3`
kusano	2b45e8	`FMADD f4, f24, f29, f4`
kusano	2b45e8
kusano	2b45e8	`FMADD f6, f26, f29, f6`
kusano	2b45e8	`FMADD f7, f27, f29, f7`
kusano	2b45e8	`FMADD f8, f24, f30, f8`
kusano	2b45e8	`FMADD f9, f25, f30, f9`
kusano	2b45e8
kusano	2b45e8	`FMADD f11, f27, f30, f11`
kusano	2b45e8	`FMADD f12, f24, f31, f12`
kusano	2b45e8	`FMADD f13, f25, f31, f13`
kusano	2b45e8	`FMADD f14, f26, f31, f14`
kusano	2b45e8
kusano	2b45e8	`LFD f28, 12 * SIZE(BO)`
kusano	2b45e8	`LFD f29, 13 * SIZE(BO)`
kusano	2b45e8	`LFD f30, 14 * SIZE(BO)`
kusano	2b45e8	`LFD f31, 15 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMADD f0, f16, f20, f0`
kusano	2b45e8	`FMADD f5, f17, f21, f5`
kusano	2b45e8	`FMADD f10, f18, f22, f10`
kusano	2b45e8	`FMADD f15, f19, f23, f15`
kusano	2b45e8
kusano	2b45e8	`LFD f24, 12 * SIZE(AO)`
kusano	2b45e8	`LFD f25, 13 * SIZE(AO)`
kusano	2b45e8	`LFD f26, 14 * SIZE(AO)`
kusano	2b45e8	`LFD f27, 15 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMADD f1, f17, f20, f1`
kusano	2b45e8	`FMADD f2, f18, f20, f2`
kusano	2b45e8	`FMADD f3, f19, f20, f3`
kusano	2b45e8	`FMADD f4, f16, f21, f4`
kusano	2b45e8
kusano	2b45e8	`FMADD f6, f18, f21, f6`
kusano	2b45e8	`FMADD f7, f19, f21, f7`
kusano	2b45e8	`FMADD f8, f16, f22, f8`
kusano	2b45e8	`FMADD f9, f17, f22, f9`
kusano	2b45e8
kusano	2b45e8	`FMADD f11, f19, f22, f11`
kusano	2b45e8	`FMADD f12, f16, f23, f12`
kusano	2b45e8	`FMADD f13, f17, f23, f13`
kusano	2b45e8	`FMADD f14, f18, f23, f14`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 16 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 17 * SIZE(BO)`
kusano	2b45e8	`LFD f22, 18 * SIZE(BO)`
kusano	2b45e8	`LFD f23, 19 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMADD f0, f24, f28, f0`
kusano	2b45e8	`FMADD f5, f25, f29, f5`
kusano	2b45e8	`FMADD f10, f26, f30, f10`
kusano	2b45e8	`FMADD f15, f27, f31, f15`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 16 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 17 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 18 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 19 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMADD f1, f25, f28, f1`
kusano	2b45e8	`FMADD f2, f26, f28, f2`
kusano	2b45e8	`FMADD f3, f27, f28, f3`
kusano	2b45e8	`FMADD f4, f24, f29, f4`
kusano	2b45e8
kusano	2b45e8	`FMADD f6, f26, f29, f6`
kusano	2b45e8	`FMADD f7, f27, f29, f7`
kusano	2b45e8	`FMADD f8, f24, f30, f8`
kusano	2b45e8	`FMADD f9, f25, f30, f9`
kusano	2b45e8
kusano	2b45e8	`FMADD f11, f27, f30, f11`
kusano	2b45e8	`FMADD f12, f24, f31, f12`
kusano	2b45e8	`FMADD f13, f25, f31, f13`
kusano	2b45e8	`FMADD f14, f26, f31, f14`
kusano	2b45e8
kusano	2b45e8	`addi AO, AO, 16 * SIZE`
kusano	2b45e8	`addi BO, BO, 16 * SIZE`
kusano	2b45e8
kusano	2b45e8	`#ifdef PPC970`
kusano	2b45e8	`#ifndef ALLOC_HUGETLB`
kusano	2b45e8	`DCBT(AO, PREA)`
kusano	2b45e8	`#endif`
kusano	2b45e8	`DCBT(BO, PREB)`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef POWER4`
kusano	2b45e8	`#ifndef ALLOC_HUGETLB`
kusano	2b45e8	`DCBT(AO, PREA)`
kusano	2b45e8	`#endif`
kusano	2b45e8	`DCBT(BO, PREB)`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef POWER5`
kusano	2b45e8	`DCBT(AO, PREA)`
kusano	2b45e8	`DCBT(BO, PREB)`
kusano	2b45e8	`#endif`
kusano	2b45e8	`bdnz LL(12)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(15):`
kusano	2b45e8	`#if defined(LT) \|\| defined(RN)`
kusano	2b45e8	`andi. r0, KK, 3`
kusano	2b45e8	`#else`
kusano	2b45e8	`andi. r0, TEMP, 3`
kusano	2b45e8	`#endif`
kusano	2b45e8	`mtspr CTR, r0`
kusano	2b45e8	`ble+ LL(18)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(16):`
kusano	2b45e8	`FMADD f0, f16, f20, f0`
kusano	2b45e8	`FMADD f5, f17, f21, f5`
kusano	2b45e8	`FMADD f10, f18, f22, f10`
kusano	2b45e8	`FMADD f15, f19, f23, f15`
kusano	2b45e8
kusano	2b45e8	`FMADD f1, f17, f20, f1`
kusano	2b45e8	`FMADD f2, f18, f20, f2`
kusano	2b45e8	`FMADD f3, f19, f20, f3`
kusano	2b45e8	`FMADD f4, f16, f21, f4`
kusano	2b45e8
kusano	2b45e8	`FMADD f6, f18, f21, f6`
kusano	2b45e8	`FMADD f7, f19, f21, f7`
kusano	2b45e8	`FMADD f8, f16, f22, f8`
kusano	2b45e8	`FMADD f9, f17, f22, f9`
kusano	2b45e8
kusano	2b45e8	`FMADD f11, f19, f22, f11`
kusano	2b45e8	`FMADD f12, f16, f23, f12`
kusano	2b45e8	`FMADD f13, f17, f23, f13`
kusano	2b45e8	`FMADD f14, f18, f23, f14`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 4 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 5 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 6 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 7 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 4 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 5 * SIZE(BO)`
kusano	2b45e8	`LFD f22, 6 * SIZE(BO)`
kusano	2b45e8	`LFD f23, 7 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`addi BO, BO, 4 * SIZE`
kusano	2b45e8	`addi AO, AO, 4 * SIZE`
kusano	2b45e8	`bdnz LL(16)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(18):`
kusano	2b45e8	`#if defined(LN) \|\| defined(RT)`
kusano	2b45e8	`subi r0, KK, 4`
kusano	2b45e8	`slwi r0, r0, 2 + BASE_SHIFT`
kusano	2b45e8	`add AO, AORIG, r0`
kusano	2b45e8	`add BO, B, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(LN) \|\| defined(LT)`
kusano	2b45e8	`LFD f16, 0 * SIZE(BO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(BO)`
kusano	2b45e8	`LFD f18, 2 * SIZE(BO)`
kusano	2b45e8	`LFD f19, 3 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 4 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 5 * SIZE(BO)`
kusano	2b45e8	`LFD f22, 6 * SIZE(BO)`
kusano	2b45e8	`LFD f23, 7 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`LFD f24, 8 * SIZE(BO)`
kusano	2b45e8	`LFD f25, 9 * SIZE(BO)`
kusano	2b45e8	`LFD f26, 10 * SIZE(BO)`
kusano	2b45e8	`LFD f27, 11 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`LFD f28, 12 * SIZE(BO)`
kusano	2b45e8	`LFD f29, 13 * SIZE(BO)`
kusano	2b45e8	`LFD f30, 14 * SIZE(BO)`
kusano	2b45e8	`LFD f31, 15 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FSUB f0, f16, f0`
kusano	2b45e8	`FSUB f4, f17, f4`
kusano	2b45e8	`FSUB f8, f18, f8`
kusano	2b45e8	`FSUB f12, f19, f12`
kusano	2b45e8
kusano	2b45e8	`FSUB f1, f20, f1`
kusano	2b45e8	`FSUB f5, f21, f5`
kusano	2b45e8	`FSUB f9, f22, f9`
kusano	2b45e8	`FSUB f13, f23, f13`
kusano	2b45e8
kusano	2b45e8	`FSUB f2, f24, f2`
kusano	2b45e8	`FSUB f6, f25, f6`
kusano	2b45e8	`FSUB f10, f26, f10`
kusano	2b45e8	`FSUB f14, f27, f14`
kusano	2b45e8
kusano	2b45e8	`FSUB f3, f28, f3`
kusano	2b45e8	`FSUB f7, f29, f7`
kusano	2b45e8	`FSUB f11, f30, f11`
kusano	2b45e8	`FSUB f15, f31, f15`
kusano	2b45e8	`#else`
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 4 * SIZE(AO)`
kusano	2b45e8	`LFD f21, 5 * SIZE(AO)`
kusano	2b45e8	`LFD f22, 6 * SIZE(AO)`
kusano	2b45e8	`LFD f23, 7 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f24, 8 * SIZE(AO)`
kusano	2b45e8	`LFD f25, 9 * SIZE(AO)`
kusano	2b45e8	`LFD f26, 10 * SIZE(AO)`
kusano	2b45e8	`LFD f27, 11 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f28, 12 * SIZE(AO)`
kusano	2b45e8	`LFD f29, 13 * SIZE(AO)`
kusano	2b45e8	`LFD f30, 14 * SIZE(AO)`
kusano	2b45e8	`LFD f31, 15 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FSUB f0, f16, f0`
kusano	2b45e8	`FSUB f1, f17, f1`
kusano	2b45e8	`FSUB f2, f18, f2`
kusano	2b45e8	`FSUB f3, f19, f3`
kusano	2b45e8
kusano	2b45e8	`FSUB f4, f20, f4`
kusano	2b45e8	`FSUB f5, f21, f5`
kusano	2b45e8	`FSUB f6, f22, f6`
kusano	2b45e8	`FSUB f7, f23, f7`
kusano	2b45e8
kusano	2b45e8	`FSUB f8, f24, f8`
kusano	2b45e8	`FSUB f9, f25, f9`
kusano	2b45e8	`FSUB f10, f26, f10`
kusano	2b45e8	`FSUB f11, f27, f11`
kusano	2b45e8
kusano	2b45e8	`FSUB f12, f28, f12`
kusano	2b45e8	`FSUB f13, f29, f13`
kusano	2b45e8	`FSUB f14, f30, f14`
kusano	2b45e8	`FSUB f15, f31, f15`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`LFD f16, 15 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 14 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 13 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 12 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f3, f16, f3`
kusano	2b45e8	`FMUL f7, f16, f7`
kusano	2b45e8	`FMUL f11, f16, f11`
kusano	2b45e8	`FMUL f15, f16, f15`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f2, f17, f3, f2`
kusano	2b45e8	`FNMSUB f6, f17, f7, f6`
kusano	2b45e8	`FNMSUB f10, f17, f11, f10`
kusano	2b45e8	`FNMSUB f14, f17, f15, f14`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f1, f18, f3, f1`
kusano	2b45e8	`FNMSUB f5, f18, f7, f5`
kusano	2b45e8	`FNMSUB f9, f18, f11, f9`
kusano	2b45e8	`FNMSUB f13, f18, f15, f13`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f0, f19, f3, f0`
kusano	2b45e8	`FNMSUB f4, f19, f7, f4`
kusano	2b45e8	`FNMSUB f8, f19, f11, f8`
kusano	2b45e8	`FNMSUB f12, f19, f15, f12`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 10 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 9 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 8 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 5 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f2, f16, f2`
kusano	2b45e8	`FMUL f6, f16, f6`
kusano	2b45e8	`FMUL f10, f16, f10`
kusano	2b45e8	`FMUL f14, f16, f14`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 4 * SIZE(AO)`
kusano	2b45e8	`LFD f21, 0 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f1, f17, f2, f1`
kusano	2b45e8	`FNMSUB f5, f17, f6, f5`
kusano	2b45e8	`FNMSUB f9, f17, f10, f9`
kusano	2b45e8	`FNMSUB f13, f17, f14, f13`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f0, f18, f2, f0`
kusano	2b45e8	`FNMSUB f4, f18, f6, f4`
kusano	2b45e8	`FNMSUB f8, f18, f10, f8`
kusano	2b45e8	`FNMSUB f12, f18, f14, f12`
kusano	2b45e8
kusano	2b45e8	`FMUL f1, f19, f1`
kusano	2b45e8	`FMUL f5, f19, f5`
kusano	2b45e8	`FMUL f9, f19, f9`
kusano	2b45e8	`FMUL f13, f19, f13`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f0, f20, f1, f0`
kusano	2b45e8	`FNMSUB f4, f20, f5, f4`
kusano	2b45e8	`FNMSUB f8, f20, f9, f8`
kusano	2b45e8	`FNMSUB f12, f20, f13, f12`
kusano	2b45e8
kusano	2b45e8	`FMUL f0, f21, f0`
kusano	2b45e8	`FMUL f4, f21, f4`
kusano	2b45e8	`FMUL f8, f21, f8`
kusano	2b45e8	`FMUL f12, f21, f12`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LT`
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f0, f16, f0`
kusano	2b45e8	`FMUL f4, f16, f4`
kusano	2b45e8	`FMUL f8, f16, f8`
kusano	2b45e8	`FMUL f12, f16, f12`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f1, f17, f0, f1`
kusano	2b45e8	`FNMSUB f5, f17, f4, f5`
kusano	2b45e8	`FNMSUB f9, f17, f8, f9`
kusano	2b45e8	`FNMSUB f13, f17, f12, f13`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f2, f18, f0, f2`
kusano	2b45e8	`FNMSUB f6, f18, f4, f6`
kusano	2b45e8	`FNMSUB f10, f18, f8, f10`
kusano	2b45e8	`FNMSUB f14, f18, f12, f14`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f3, f19, f0, f3`
kusano	2b45e8	`FNMSUB f7, f19, f4, f7`
kusano	2b45e8	`FNMSUB f11, f19, f8, f11`
kusano	2b45e8	`FNMSUB f15, f19, f12, f15`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 5 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 6 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 7 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 10 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f1, f16, f1`
kusano	2b45e8	`FMUL f5, f16, f5`
kusano	2b45e8	`FMUL f9, f16, f9`
kusano	2b45e8	`FMUL f13, f16, f13`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 11 * SIZE(AO)`
kusano	2b45e8	`LFD f21, 15 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f2, f17, f1, f2`
kusano	2b45e8	`FNMSUB f6, f17, f5, f6`
kusano	2b45e8	`FNMSUB f10, f17, f9, f10`
kusano	2b45e8	`FNMSUB f14, f17, f13, f14`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f3, f18, f1, f3`
kusano	2b45e8	`FNMSUB f7, f18, f5, f7`
kusano	2b45e8	`FNMSUB f11, f18, f9, f11`
kusano	2b45e8	`FNMSUB f15, f18, f13, f15`
kusano	2b45e8
kusano	2b45e8	`FMUL f2, f19, f2`
kusano	2b45e8	`FMUL f6, f19, f6`
kusano	2b45e8	`FMUL f10, f19, f10`
kusano	2b45e8	`FMUL f14, f19, f14`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f3, f20, f2, f3`
kusano	2b45e8	`FNMSUB f7, f20, f6, f7`
kusano	2b45e8	`FNMSUB f11, f20, f10, f11`
kusano	2b45e8	`FNMSUB f15, f20, f14, f15`
kusano	2b45e8
kusano	2b45e8	`FMUL f3, f21, f3`
kusano	2b45e8	`FMUL f7, f21, f7`
kusano	2b45e8	`FMUL f11, f21, f11`
kusano	2b45e8	`FMUL f15, f21, f15`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RN`
kusano	2b45e8	`LFD f16, 0 * SIZE(BO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(BO)`
kusano	2b45e8	`LFD f18, 2 * SIZE(BO)`
kusano	2b45e8	`LFD f19, 3 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f0, f16, f0`
kusano	2b45e8	`FMUL f1, f16, f1`
kusano	2b45e8	`FMUL f2, f16, f2`
kusano	2b45e8	`FMUL f3, f16, f3`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f4, f17, f0, f4`
kusano	2b45e8	`FNMSUB f5, f17, f1, f5`
kusano	2b45e8	`FNMSUB f6, f17, f2, f6`
kusano	2b45e8	`FNMSUB f7, f17, f3, f7`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f8, f18, f0, f8`
kusano	2b45e8	`FNMSUB f9, f18, f1, f9`
kusano	2b45e8	`FNMSUB f10, f18, f2, f10`
kusano	2b45e8	`FNMSUB f11, f18, f3, f11`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f12, f19, f0, f12`
kusano	2b45e8	`FNMSUB f13, f19, f1, f13`
kusano	2b45e8	`FNMSUB f14, f19, f2, f14`
kusano	2b45e8	`FNMSUB f15, f19, f3, f15`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 5 * SIZE(BO)`
kusano	2b45e8	`LFD f17, 6 * SIZE(BO)`
kusano	2b45e8	`LFD f18, 7 * SIZE(BO)`
kusano	2b45e8	`LFD f19, 10 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f4, f16, f4`
kusano	2b45e8	`FMUL f5, f16, f5`
kusano	2b45e8	`FMUL f6, f16, f6`
kusano	2b45e8	`FMUL f7, f16, f7`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 11 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 15 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f8, f17, f4, f8`
kusano	2b45e8	`FNMSUB f9, f17, f5, f9`
kusano	2b45e8	`FNMSUB f10, f17, f6, f10`
kusano	2b45e8	`FNMSUB f11, f17, f7, f11`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f12, f18, f4, f12`
kusano	2b45e8	`FNMSUB f13, f18, f5, f13`
kusano	2b45e8	`FNMSUB f14, f18, f6, f14`
kusano	2b45e8	`FNMSUB f15, f18, f7, f15`
kusano	2b45e8
kusano	2b45e8	`FMUL f8, f19, f8`
kusano	2b45e8	`FMUL f9, f19, f9`
kusano	2b45e8	`FMUL f10, f19, f10`
kusano	2b45e8	`FMUL f11, f19, f11`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f12, f20, f8, f12`
kusano	2b45e8	`FNMSUB f13, f20, f9, f13`
kusano	2b45e8	`FNMSUB f14, f20, f10, f14`
kusano	2b45e8	`FNMSUB f15, f20, f11, f15`
kusano	2b45e8
kusano	2b45e8	`FMUL f12, f21, f12`
kusano	2b45e8	`FMUL f13, f21, f13`
kusano	2b45e8	`FMUL f14, f21, f14`
kusano	2b45e8	`FMUL f15, f21, f15`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RT`
kusano	2b45e8	`LFD f16, 15 * SIZE(BO)`
kusano	2b45e8	`LFD f17, 14 * SIZE(BO)`
kusano	2b45e8	`LFD f18, 13 * SIZE(BO)`
kusano	2b45e8	`LFD f19, 12 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f12, f16, f12`
kusano	2b45e8	`FMUL f13, f16, f13`
kusano	2b45e8	`FMUL f14, f16, f14`
kusano	2b45e8	`FMUL f15, f16, f15`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f8, f17, f12, f8`
kusano	2b45e8	`FNMSUB f9, f17, f13, f9`
kusano	2b45e8	`FNMSUB f10, f17, f14, f10`
kusano	2b45e8	`FNMSUB f11, f17, f15, f11`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f4, f18, f12, f4`
kusano	2b45e8	`FNMSUB f5, f18, f13, f5`
kusano	2b45e8	`FNMSUB f6, f18, f14, f6`
kusano	2b45e8	`FNMSUB f7, f18, f15, f7`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f0, f19, f12, f0`
kusano	2b45e8	`FNMSUB f1, f19, f13, f1`
kusano	2b45e8	`FNMSUB f2, f19, f14, f2`
kusano	2b45e8	`FNMSUB f3, f19, f15, f3`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 10 * SIZE(BO)`
kusano	2b45e8	`LFD f17, 9 * SIZE(BO)`
kusano	2b45e8	`LFD f18, 8 * SIZE(BO)`
kusano	2b45e8	`LFD f19, 5 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f8, f16, f8`
kusano	2b45e8	`FMUL f9, f16, f9`
kusano	2b45e8	`FMUL f10, f16, f10`
kusano	2b45e8	`FMUL f11, f16, f11`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 4 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 0 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f4, f17, f8, f4`
kusano	2b45e8	`FNMSUB f5, f17, f9, f5`
kusano	2b45e8	`FNMSUB f6, f17, f10, f6`
kusano	2b45e8	`FNMSUB f7, f17, f11, f7`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f0, f18, f8, f0`
kusano	2b45e8	`FNMSUB f1, f18, f9, f1`
kusano	2b45e8	`FNMSUB f2, f18, f10, f2`
kusano	2b45e8	`FNMSUB f3, f18, f11, f3`
kusano	2b45e8
kusano	2b45e8	`FMUL f4, f19, f4`
kusano	2b45e8	`FMUL f5, f19, f5`
kusano	2b45e8	`FMUL f6, f19, f6`
kusano	2b45e8	`FMUL f7, f19, f7`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f0, f20, f4, f0`
kusano	2b45e8	`FNMSUB f1, f20, f5, f1`
kusano	2b45e8	`FNMSUB f2, f20, f6, f2`
kusano	2b45e8	`FNMSUB f3, f20, f7, f3`
kusano	2b45e8
kusano	2b45e8	`FMUL f0, f21, f0`
kusano	2b45e8	`FMUL f1, f21, f1`
kusano	2b45e8	`FMUL f2, f21, f2`
kusano	2b45e8	`FMUL f3, f21, f3`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`subi CO1, CO1, 4 * SIZE`
kusano	2b45e8	`subi CO2, CO2, 4 * SIZE`
kusano	2b45e8	`subi CO3, CO3, 4 * SIZE`
kusano	2b45e8	`subi CO4, CO4, 4 * SIZE`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(LN) \|\| defined(LT)`
kusano	2b45e8	`STFD f0, 0 * SIZE(BO)`
kusano	2b45e8	`STFD f4, 1 * SIZE(BO)`
kusano	2b45e8	`STFD f8, 2 * SIZE(BO)`
kusano	2b45e8	`STFD f12, 3 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`STFD f1, 4 * SIZE(BO)`
kusano	2b45e8	`STFD f5, 5 * SIZE(BO)`
kusano	2b45e8	`STFD f9, 6 * SIZE(BO)`
kusano	2b45e8	`STFD f13, 7 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`STFD f2, 8 * SIZE(BO)`
kusano	2b45e8	`STFD f6, 9 * SIZE(BO)`
kusano	2b45e8	`STFD f10, 10 * SIZE(BO)`
kusano	2b45e8	`STFD f14, 11 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`STFD f3, 12 * SIZE(BO)`
kusano	2b45e8	`STFD f7, 13 * SIZE(BO)`
kusano	2b45e8	`STFD f11, 14 * SIZE(BO)`
kusano	2b45e8	`STFD f15, 15 * SIZE(BO)`
kusano	2b45e8	`#else`
kusano	2b45e8	`STFD f0, 0 * SIZE(AO)`
kusano	2b45e8	`STFD f1, 1 * SIZE(AO)`
kusano	2b45e8	`STFD f2, 2 * SIZE(AO)`
kusano	2b45e8	`STFD f3, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`STFD f4, 4 * SIZE(AO)`
kusano	2b45e8	`STFD f5, 5 * SIZE(AO)`
kusano	2b45e8	`STFD f6, 6 * SIZE(AO)`
kusano	2b45e8	`STFD f7, 7 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`STFD f8, 8 * SIZE(AO)`
kusano	2b45e8	`STFD f9, 9 * SIZE(AO)`
kusano	2b45e8	`STFD f10, 10 * SIZE(AO)`
kusano	2b45e8	`STFD f11, 11 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`STFD f12, 12 * SIZE(AO)`
kusano	2b45e8	`STFD f13, 13 * SIZE(AO)`
kusano	2b45e8	`STFD f14, 14 * SIZE(AO)`
kusano	2b45e8	`STFD f15, 15 * SIZE(AO)`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`STFD f0, 0 * SIZE(CO1)`
kusano	2b45e8	`STFD f1, 1 * SIZE(CO1)`
kusano	2b45e8	`STFD f2, 2 * SIZE(CO1)`
kusano	2b45e8	`STFD f3, 3 * SIZE(CO1)`
kusano	2b45e8
kusano	2b45e8	`STFD f4, 0 * SIZE(CO2)`
kusano	2b45e8	`STFD f5, 1 * SIZE(CO2)`
kusano	2b45e8	`STFD f6, 2 * SIZE(CO2)`
kusano	2b45e8	`STFD f7, 3 * SIZE(CO2)`
kusano	2b45e8
kusano	2b45e8	`STFD f8, 0 * SIZE(CO3)`
kusano	2b45e8	`STFD f9, 1 * SIZE(CO3)`
kusano	2b45e8	`STFD f10, 2 * SIZE(CO3)`
kusano	2b45e8	`STFD f11, 3 * SIZE(CO3)`
kusano	2b45e8
kusano	2b45e8	`STFD f12, 0 * SIZE(CO4)`
kusano	2b45e8	`STFD f13, 1 * SIZE(CO4)`
kusano	2b45e8	`STFD f14, 2 * SIZE(CO4)`
kusano	2b45e8	`STFD f15, 3 * SIZE(CO4)`
kusano	2b45e8
kusano	2b45e8	`lfs f0, FZERO`
kusano	2b45e8	`fmr f1, f0`
kusano	2b45e8	`fmr f2, f0`
kusano	2b45e8	`fmr f3, f0`
kusano	2b45e8
kusano	2b45e8	`fmr f4, f0`
kusano	2b45e8	`fmr f5, f0`
kusano	2b45e8	`fmr f6, f0`
kusano	2b45e8	`fmr f7, f0`
kusano	2b45e8
kusano	2b45e8	`fmr f8, f0`
kusano	2b45e8	`fmr f9, f0`
kusano	2b45e8	`fmr f10, f0`
kusano	2b45e8	`fmr f11, f0`
kusano	2b45e8
kusano	2b45e8	`fmr f12, f0`
kusano	2b45e8	`fmr f13, f0`
kusano	2b45e8	`fmr f14, f0`
kusano	2b45e8	`fmr f15, f0`
kusano	2b45e8
kusano	2b45e8	`#ifndef LN`
kusano	2b45e8	`addi CO1, CO1, 4 * SIZE`
kusano	2b45e8	`addi CO2, CO2, 4 * SIZE`
kusano	2b45e8	`addi CO3, CO3, 4 * SIZE`
kusano	2b45e8	`addi CO4, CO4, 4 * SIZE`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RT`
kusano	2b45e8	`slwi r0, K, 2 + BASE_SHIFT`
kusano	2b45e8	`add AORIG, AORIG, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(LT) \|\| defined(RN)`
kusano	2b45e8	`sub TEMP, K, KK`
kusano	2b45e8	`slwi TEMP, TEMP, 2 + BASE_SHIFT`
kusano	2b45e8	`add AO, AO, TEMP`
kusano	2b45e8	`add BO, BO, TEMP`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LT`
kusano	2b45e8	`addi KK, KK, 4`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`subi KK, KK, 4`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`addic. I, I, -1`
kusano	2b45e8	`bgt+ LL(11)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(20):`
kusano	2b45e8	`andi. I, M, 2`
kusano	2b45e8	`ble LL(30)`
kusano	2b45e8
kusano	2b45e8	`#if defined(LT) \|\| defined(RN)`
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 0 * SIZE(B)`
kusano	2b45e8	`LFD f21, 1 * SIZE(B)`
kusano	2b45e8	`LFD f22, 2 * SIZE(B)`
kusano	2b45e8	`LFD f23, 3 * SIZE(B)`
kusano	2b45e8
kusano	2b45e8	`LFD f24, 4 * SIZE(B)`
kusano	2b45e8	`LFD f25, 5 * SIZE(B)`
kusano	2b45e8	`LFD f26, 6 * SIZE(B)`
kusano	2b45e8	`LFD f27, 7 * SIZE(B)`
kusano	2b45e8
kusano	2b45e8	`srawi. r0, KK, 2`
kusano	2b45e8	`mtspr CTR, r0`
kusano	2b45e8	`mr BO, B`
kusano	2b45e8	`#else`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`slwi r0, K, 1 + BASE_SHIFT`
kusano	2b45e8	`sub AORIG, AORIG, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`slwi r0, KK, 1 + BASE_SHIFT`
kusano	2b45e8	`slwi TEMP, KK, 2 + BASE_SHIFT`
kusano	2b45e8	`add AO, AORIG, r0`
kusano	2b45e8	`add BO, B, TEMP`
kusano	2b45e8
kusano	2b45e8	`sub TEMP, K, KK`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 0 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 1 * SIZE(BO)`
kusano	2b45e8	`LFD f22, 2 * SIZE(BO)`
kusano	2b45e8	`LFD f23, 3 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`LFD f24, 4 * SIZE(BO)`
kusano	2b45e8	`LFD f25, 5 * SIZE(BO)`
kusano	2b45e8	`LFD f26, 6 * SIZE(BO)`
kusano	2b45e8	`LFD f27, 7 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`srawi. r0, TEMP, 2`
kusano	2b45e8	`mtspr CTR, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8	`ble LL(25)`
kusano	2b45e8	`.align 5`
kusano	2b45e8
kusano	2b45e8	`LL(22):`
kusano	2b45e8	`FMADD f0, f16, f20, f0`
kusano	2b45e8	`FMADD f1, f17, f20, f1`
kusano	2b45e8	`FMADD f4, f16, f21, f4`
kusano	2b45e8	`FMADD f5, f17, f21, f5`
kusano	2b45e8
kusano	2b45e8	`FMADD f8, f16, f22, f8`
kusano	2b45e8	`FMADD f9, f17, f22, f9`
kusano	2b45e8	`FMADD f12, f16, f23, f12`
kusano	2b45e8	`FMADD f13, f17, f23, f13`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 8 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 9 * SIZE(BO)`
kusano	2b45e8	`LFD f22, 10 * SIZE(BO)`
kusano	2b45e8	`LFD f23, 11 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMADD f2, f18, f24, f2`
kusano	2b45e8	`FMADD f3, f19, f24, f3`
kusano	2b45e8	`FMADD f6, f18, f25, f6`
kusano	2b45e8	`FMADD f7, f19, f25, f7`
kusano	2b45e8
kusano	2b45e8	`FMADD f10, f18, f26, f10`
kusano	2b45e8	`FMADD f11, f19, f26, f11`
kusano	2b45e8	`FMADD f14, f18, f27, f14`
kusano	2b45e8	`FMADD f15, f19, f27, f15`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 4 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 5 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 6 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 7 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMADD f0, f16, f20, f0`
kusano	2b45e8	`FMADD f1, f17, f20, f1`
kusano	2b45e8	`FMADD f4, f16, f21, f4`
kusano	2b45e8	`FMADD f5, f17, f21, f5`
kusano	2b45e8
kusano	2b45e8	`LFD f24, 12 * SIZE(BO)`
kusano	2b45e8	`LFD f25, 13 * SIZE(BO)`
kusano	2b45e8	`LFD f26, 14 * SIZE(BO)`
kusano	2b45e8	`LFD f27, 15 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMADD f8, f16, f22, f8`
kusano	2b45e8	`FMADD f9, f17, f22, f9`
kusano	2b45e8	`FMADD f12, f16, f23, f12`
kusano	2b45e8	`FMADD f13, f17, f23, f13`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 16 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 17 * SIZE(BO)`
kusano	2b45e8	`LFD f22, 18 * SIZE(BO)`
kusano	2b45e8	`LFD f23, 19 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMADD f2, f18, f24, f2`
kusano	2b45e8	`FMADD f3, f19, f24, f3`
kusano	2b45e8	`FMADD f6, f18, f25, f6`
kusano	2b45e8	`FMADD f7, f19, f25, f7`
kusano	2b45e8
kusano	2b45e8	`FMADD f10, f18, f26, f10`
kusano	2b45e8	`FMADD f11, f19, f26, f11`
kusano	2b45e8	`FMADD f14, f18, f27, f14`
kusano	2b45e8	`FMADD f15, f19, f27, f15`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 8 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 9 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 10 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 11 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f24, 20 * SIZE(BO)`
kusano	2b45e8	`LFD f25, 21 * SIZE(BO)`
kusano	2b45e8	`LFD f26, 22 * SIZE(BO)`
kusano	2b45e8	`LFD f27, 23 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`addi AO, AO, 8 * SIZE`
kusano	2b45e8	`addi BO, BO, 16 * SIZE`
kusano	2b45e8	`DCBT(BO, PREB)`
kusano	2b45e8	`bdnz LL(22)`
kusano	2b45e8
kusano	2b45e8	`fadd f0, f2, f0`
kusano	2b45e8	`fadd f1, f3, f1`
kusano	2b45e8	`fadd f4, f6, f4`
kusano	2b45e8	`fadd f5, f7, f5`
kusano	2b45e8	`fadd f8, f10, f8`
kusano	2b45e8	`fadd f9, f11, f9`
kusano	2b45e8	`fadd f12, f14, f12`
kusano	2b45e8	`fadd f13, f15, f13`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(25):`
kusano	2b45e8	`#if defined(LT) \|\| defined(RN)`
kusano	2b45e8	`andi. r0, KK, 3`
kusano	2b45e8	`#else`
kusano	2b45e8	`andi. r0, TEMP, 3`
kusano	2b45e8	`#endif`
kusano	2b45e8	`mtspr CTR, r0`
kusano	2b45e8	`ble+ LL(28)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(26):`
kusano	2b45e8	`FMADD f0, f16, f20, f0`
kusano	2b45e8	`FMADD f1, f17, f20, f1`
kusano	2b45e8	`FMADD f4, f16, f21, f4`
kusano	2b45e8	`FMADD f5, f17, f21, f5`
kusano	2b45e8
kusano	2b45e8	`FMADD f8, f16, f22, f8`
kusano	2b45e8	`FMADD f9, f17, f22, f9`
kusano	2b45e8	`FMADD f12, f16, f23, f12`
kusano	2b45e8	`FMADD f13, f17, f23, f13`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 4 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 5 * SIZE(BO)`
kusano	2b45e8	`LFD f22, 6 * SIZE(BO)`
kusano	2b45e8	`LFD f23, 7 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`addi BO, BO, 4 * SIZE`
kusano	2b45e8	`addi AO, AO, 2 * SIZE`
kusano	2b45e8	`bdnz LL(26)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(28):`
kusano	2b45e8	`#if defined(LN) \|\| defined(RT)`
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`subi r0, KK, 2`
kusano	2b45e8	`#else`
kusano	2b45e8	`subi r0, KK, 4`
kusano	2b45e8	`#endif`
kusano	2b45e8	`slwi TEMP, r0, 1 + BASE_SHIFT`
kusano	2b45e8	`slwi r0, r0, 2 + BASE_SHIFT`
kusano	2b45e8	`add AO, AORIG, TEMP`
kusano	2b45e8	`add BO, B, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(LN) \|\| defined(LT)`
kusano	2b45e8	`LFD f16, 0 * SIZE(BO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(BO)`
kusano	2b45e8	`LFD f18, 2 * SIZE(BO)`
kusano	2b45e8	`LFD f19, 3 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 4 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 5 * SIZE(BO)`
kusano	2b45e8	`LFD f22, 6 * SIZE(BO)`
kusano	2b45e8	`LFD f23, 7 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FSUB f0, f16, f0`
kusano	2b45e8	`FSUB f4, f17, f4`
kusano	2b45e8	`FSUB f8, f18, f8`
kusano	2b45e8	`FSUB f12, f19, f12`
kusano	2b45e8
kusano	2b45e8	`FSUB f1, f20, f1`
kusano	2b45e8	`FSUB f5, f21, f5`
kusano	2b45e8	`FSUB f9, f22, f9`
kusano	2b45e8	`FSUB f13, f23, f13`
kusano	2b45e8	`#else`
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(AO)`
kusano	2b45e8	`LFD f20, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f21, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f24, 4 * SIZE(AO)`
kusano	2b45e8	`LFD f25, 5 * SIZE(AO)`
kusano	2b45e8	`LFD f28, 6 * SIZE(AO)`
kusano	2b45e8	`LFD f29, 7 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FSUB f0, f16, f0`
kusano	2b45e8	`FSUB f1, f17, f1`
kusano	2b45e8	`FSUB f4, f20, f4`
kusano	2b45e8	`FSUB f5, f21, f5`
kusano	2b45e8
kusano	2b45e8	`FSUB f8, f24, f8`
kusano	2b45e8	`FSUB f9, f25, f9`
kusano	2b45e8	`FSUB f12, f28, f12`
kusano	2b45e8	`FSUB f13, f29, f13`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`LFD f19, 3 * SIZE(AO)`
kusano	2b45e8	`LFD f20, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f21, 0 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f1, f19, f1`
kusano	2b45e8	`FMUL f5, f19, f5`
kusano	2b45e8	`FMUL f9, f19, f9`
kusano	2b45e8	`FMUL f13, f19, f13`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f0, f20, f1, f0`
kusano	2b45e8	`FNMSUB f4, f20, f5, f4`
kusano	2b45e8	`FNMSUB f8, f20, f9, f8`
kusano	2b45e8	`FNMSUB f12, f20, f13, f12`
kusano	2b45e8
kusano	2b45e8	`FMUL f0, f21, f0`
kusano	2b45e8	`FMUL f4, f21, f4`
kusano	2b45e8	`FMUL f8, f21, f8`
kusano	2b45e8	`FMUL f12, f21, f12`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LT`
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f0, f16, f0`
kusano	2b45e8	`FMUL f4, f16, f4`
kusano	2b45e8	`FMUL f8, f16, f8`
kusano	2b45e8	`FMUL f12, f16, f12`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f1, f17, f0, f1`
kusano	2b45e8	`FNMSUB f5, f17, f4, f5`
kusano	2b45e8	`FNMSUB f9, f17, f8, f9`
kusano	2b45e8	`FNMSUB f13, f17, f12, f13`
kusano	2b45e8
kusano	2b45e8	`LFD f17, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f1, f17, f1`
kusano	2b45e8	`FMUL f5, f17, f5`
kusano	2b45e8	`FMUL f9, f17, f9`
kusano	2b45e8	`FMUL f13, f17, f13`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RN`
kusano	2b45e8	`LFD f16, 0 * SIZE(BO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(BO)`
kusano	2b45e8	`LFD f18, 2 * SIZE(BO)`
kusano	2b45e8	`LFD f19, 3 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f0, f16, f0`
kusano	2b45e8	`FMUL f1, f16, f1`
kusano	2b45e8	`FNMSUB f4, f17, f0, f4`
kusano	2b45e8	`FNMSUB f5, f17, f1, f5`
kusano	2b45e8	`FNMSUB f8, f18, f0, f8`
kusano	2b45e8	`FNMSUB f9, f18, f1, f9`
kusano	2b45e8	`FNMSUB f12, f19, f0, f12`
kusano	2b45e8	`FNMSUB f13, f19, f1, f13`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 5 * SIZE(BO)`
kusano	2b45e8	`LFD f17, 6 * SIZE(BO)`
kusano	2b45e8	`LFD f18, 7 * SIZE(BO)`
kusano	2b45e8	`LFD f19, 10 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 11 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 15 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f4, f16, f4`
kusano	2b45e8	`FMUL f5, f16, f5`
kusano	2b45e8	`FNMSUB f8, f17, f4, f8`
kusano	2b45e8	`FNMSUB f9, f17, f5, f9`
kusano	2b45e8	`FNMSUB f12, f18, f4, f12`
kusano	2b45e8	`FNMSUB f13, f18, f5, f13`
kusano	2b45e8
kusano	2b45e8	`FMUL f8, f19, f8`
kusano	2b45e8	`FMUL f9, f19, f9`
kusano	2b45e8	`FNMSUB f12, f20, f8, f12`
kusano	2b45e8	`FNMSUB f13, f20, f9, f13`
kusano	2b45e8	`FMUL f12, f21, f12`
kusano	2b45e8	`FMUL f13, f21, f13`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RT`
kusano	2b45e8	`LFD f16, 15 * SIZE(BO)`
kusano	2b45e8	`LFD f17, 14 * SIZE(BO)`
kusano	2b45e8	`LFD f18, 13 * SIZE(BO)`
kusano	2b45e8	`LFD f19, 12 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f12, f16, f12`
kusano	2b45e8	`FMUL f13, f16, f13`
kusano	2b45e8	`FNMSUB f8, f17, f12, f8`
kusano	2b45e8	`FNMSUB f9, f17, f13, f9`
kusano	2b45e8	`FNMSUB f4, f18, f12, f4`
kusano	2b45e8	`FNMSUB f5, f18, f13, f5`
kusano	2b45e8	`FNMSUB f0, f19, f12, f0`
kusano	2b45e8	`FNMSUB f1, f19, f13, f1`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 10 * SIZE(BO)`
kusano	2b45e8	`LFD f17, 9 * SIZE(BO)`
kusano	2b45e8	`LFD f18, 8 * SIZE(BO)`
kusano	2b45e8	`LFD f19, 5 * SIZE(BO)`
kusano	2b45e8	`LFD f20, 4 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 0 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f8, f16, f8`
kusano	2b45e8	`FMUL f9, f16, f9`
kusano	2b45e8	`FNMSUB f4, f17, f8, f4`
kusano	2b45e8	`FNMSUB f5, f17, f9, f5`
kusano	2b45e8	`FNMSUB f0, f18, f8, f0`
kusano	2b45e8	`FNMSUB f1, f18, f9, f1`
kusano	2b45e8
kusano	2b45e8	`FMUL f4, f19, f4`
kusano	2b45e8	`FMUL f5, f19, f5`
kusano	2b45e8	`FNMSUB f0, f20, f4, f0`
kusano	2b45e8	`FNMSUB f1, f20, f5, f1`
kusano	2b45e8
kusano	2b45e8	`FMUL f0, f21, f0`
kusano	2b45e8	`FMUL f1, f21, f1`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`subi CO1, CO1, 2 * SIZE`
kusano	2b45e8	`subi CO2, CO2, 2 * SIZE`
kusano	2b45e8	`subi CO3, CO3, 2 * SIZE`
kusano	2b45e8	`subi CO4, CO4, 2 * SIZE`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(LN) \|\| defined(LT)`
kusano	2b45e8	`STFD f0, 0 * SIZE(BO)`
kusano	2b45e8	`STFD f4, 1 * SIZE(BO)`
kusano	2b45e8	`STFD f8, 2 * SIZE(BO)`
kusano	2b45e8	`STFD f12, 3 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`STFD f1, 4 * SIZE(BO)`
kusano	2b45e8	`STFD f5, 5 * SIZE(BO)`
kusano	2b45e8	`STFD f9, 6 * SIZE(BO)`
kusano	2b45e8	`STFD f13, 7 * SIZE(BO)`
kusano	2b45e8	`#else`
kusano	2b45e8	`STFD f0, 0 * SIZE(AO)`
kusano	2b45e8	`STFD f1, 1 * SIZE(AO)`
kusano	2b45e8	`STFD f4, 2 * SIZE(AO)`
kusano	2b45e8	`STFD f5, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`STFD f8, 4 * SIZE(AO)`
kusano	2b45e8	`STFD f9, 5 * SIZE(AO)`
kusano	2b45e8	`STFD f12, 6 * SIZE(AO)`
kusano	2b45e8	`STFD f13, 7 * SIZE(AO)`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`STFD f0, 0 * SIZE(CO1)`
kusano	2b45e8	`STFD f1, 1 * SIZE(CO1)`
kusano	2b45e8	`STFD f4, 0 * SIZE(CO2)`
kusano	2b45e8	`STFD f5, 1 * SIZE(CO2)`
kusano	2b45e8
kusano	2b45e8	`STFD f8, 0 * SIZE(CO3)`
kusano	2b45e8	`STFD f9, 1 * SIZE(CO3)`
kusano	2b45e8	`STFD f12, 0 * SIZE(CO4)`
kusano	2b45e8	`STFD f13, 1 * SIZE(CO4)`
kusano	2b45e8
kusano	2b45e8	`lfs f0, FZERO`
kusano	2b45e8	`fmr f1, f0`
kusano	2b45e8	`fmr f2, f0`
kusano	2b45e8	`fmr f3, f0`
kusano	2b45e8
kusano	2b45e8	`fmr f4, f0`
kusano	2b45e8	`fmr f5, f0`
kusano	2b45e8	`fmr f6, f0`
kusano	2b45e8	`fmr f7, f0`
kusano	2b45e8
kusano	2b45e8	`fmr f8, f0`
kusano	2b45e8	`fmr f9, f0`
kusano	2b45e8	`fmr f10, f0`
kusano	2b45e8	`fmr f11, f0`
kusano	2b45e8
kusano	2b45e8	`fmr f12, f0`
kusano	2b45e8	`fmr f13, f0`
kusano	2b45e8	`fmr f14, f0`
kusano	2b45e8	`fmr f15, f0`
kusano	2b45e8
kusano	2b45e8	`#ifndef LN`
kusano	2b45e8	`addi CO1, CO1, 2 * SIZE`
kusano	2b45e8	`addi CO2, CO2, 2 * SIZE`
kusano	2b45e8	`addi CO3, CO3, 2 * SIZE`
kusano	2b45e8	`addi CO4, CO4, 2 * SIZE`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RT`
kusano	2b45e8	`slwi r0, K, 1 + BASE_SHIFT`
kusano	2b45e8	`add AORIG, AORIG, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(LT) \|\| defined(RN)`
kusano	2b45e8	`sub TEMP, K, KK`
kusano	2b45e8	`slwi r0, TEMP, 1 + BASE_SHIFT`
kusano	2b45e8	`slwi TEMP, TEMP, 2 + BASE_SHIFT`
kusano	2b45e8	`add AO, AO, r0`
kusano	2b45e8	`add BO, BO, TEMP`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`subi KK, KK, 2`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LT`
kusano	2b45e8	`addi KK, KK, 2`
kusano	2b45e8	`#endif`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(30):`
kusano	2b45e8	`andi. I, M, 1`
kusano	2b45e8	`ble LL(39)`
kusano	2b45e8
kusano	2b45e8	`#if defined(LT) \|\| defined(RN)`
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 0 * SIZE(B)`
kusano	2b45e8	`LFD f21, 1 * SIZE(B)`
kusano	2b45e8	`LFD f22, 2 * SIZE(B)`
kusano	2b45e8	`LFD f23, 3 * SIZE(B)`
kusano	2b45e8
kusano	2b45e8	`LFD f24, 4 * SIZE(B)`
kusano	2b45e8	`LFD f25, 5 * SIZE(B)`
kusano	2b45e8	`LFD f26, 6 * SIZE(B)`
kusano	2b45e8	`LFD f27, 7 * SIZE(B)`
kusano	2b45e8
kusano	2b45e8	`srawi. r0, KK, 2`
kusano	2b45e8	`mtspr CTR, r0`
kusano	2b45e8	`mr BO, B`
kusano	2b45e8	`#else`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`slwi r0, K, BASE_SHIFT`
kusano	2b45e8	`sub AORIG, AORIG, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`slwi r0, KK, 0 + BASE_SHIFT`
kusano	2b45e8	`slwi TEMP, KK, 2 + BASE_SHIFT`
kusano	2b45e8	`add AO, AORIG, r0`
kusano	2b45e8	`add BO, B, TEMP`
kusano	2b45e8
kusano	2b45e8	`sub TEMP, K, KK`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 0 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 1 * SIZE(BO)`
kusano	2b45e8	`LFD f22, 2 * SIZE(BO)`
kusano	2b45e8	`LFD f23, 3 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`LFD f24, 4 * SIZE(BO)`
kusano	2b45e8	`LFD f25, 5 * SIZE(BO)`
kusano	2b45e8	`LFD f26, 6 * SIZE(BO)`
kusano	2b45e8	`LFD f27, 7 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`srawi. r0, TEMP, 2`
kusano	2b45e8	`mtspr CTR, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8	`ble LL(35)`
kusano	2b45e8	`.align 5`
kusano	2b45e8
kusano	2b45e8	`LL(32):`
kusano	2b45e8	`FMADD f0, f16, f20, f0`
kusano	2b45e8	`FMADD f4, f16, f21, f4`
kusano	2b45e8	`FMADD f8, f16, f22, f8`
kusano	2b45e8	`FMADD f12, f16, f23, f12`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 8 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 9 * SIZE(BO)`
kusano	2b45e8	`LFD f22, 10 * SIZE(BO)`
kusano	2b45e8	`LFD f23, 11 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMADD f1, f17, f24, f1`
kusano	2b45e8	`FMADD f5, f17, f25, f5`
kusano	2b45e8	`FMADD f9, f17, f26, f9`
kusano	2b45e8	`FMADD f13, f17, f27, f13`
kusano	2b45e8
kusano	2b45e8	`LFD f24, 12 * SIZE(BO)`
kusano	2b45e8	`LFD f25, 13 * SIZE(BO)`
kusano	2b45e8	`LFD f26, 14 * SIZE(BO)`
kusano	2b45e8	`LFD f27, 15 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMADD f0, f18, f20, f0`
kusano	2b45e8	`FMADD f4, f18, f21, f4`
kusano	2b45e8	`FMADD f8, f18, f22, f8`
kusano	2b45e8	`FMADD f12, f18, f23, f12`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 16 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 17 * SIZE(BO)`
kusano	2b45e8	`LFD f22, 18 * SIZE(BO)`
kusano	2b45e8	`LFD f23, 19 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMADD f1, f19, f24, f1`
kusano	2b45e8	`FMADD f5, f19, f25, f5`
kusano	2b45e8	`FMADD f9, f19, f26, f9`
kusano	2b45e8	`FMADD f13, f19, f27, f13`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 4 * SIZE(AO)`
kusano	2b45e8	`LFD f17, 5 * SIZE(AO)`
kusano	2b45e8	`LFD f18, 6 * SIZE(AO)`
kusano	2b45e8	`LFD f19, 7 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f24, 20 * SIZE(BO)`
kusano	2b45e8	`LFD f25, 21 * SIZE(BO)`
kusano	2b45e8	`LFD f26, 22 * SIZE(BO)`
kusano	2b45e8	`LFD f27, 23 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`addi AO, AO, 4 * SIZE`
kusano	2b45e8	`addi BO, BO, 16 * SIZE`
kusano	2b45e8	`DCBT(BO, PREB)`
kusano	2b45e8	`bdnz LL(32)`
kusano	2b45e8
kusano	2b45e8	`fadd f0, f1, f0`
kusano	2b45e8	`fadd f4, f5, f4`
kusano	2b45e8	`fadd f8, f9, f8`
kusano	2b45e8	`fadd f12, f13, f12`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(35):`
kusano	2b45e8	`#if defined(LT) \|\| defined(RN)`
kusano	2b45e8	`andi. r0, KK, 3`
kusano	2b45e8	`#else`
kusano	2b45e8	`andi. r0, TEMP, 3`
kusano	2b45e8	`#endif`
kusano	2b45e8	`mtspr CTR, r0`
kusano	2b45e8	`ble+ LL(38)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(36):`
kusano	2b45e8	`FMADD f0, f16, f20, f0`
kusano	2b45e8	`FMADD f4, f16, f21, f4`
kusano	2b45e8	`FMADD f8, f16, f22, f8`
kusano	2b45e8	`FMADD f12, f16, f23, f12`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 1 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 4 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 5 * SIZE(BO)`
kusano	2b45e8	`LFD f22, 6 * SIZE(BO)`
kusano	2b45e8	`LFD f23, 7 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`addi BO, BO, 4 * SIZE`
kusano	2b45e8	`addi AO, AO, 1 * SIZE`
kusano	2b45e8	`bdnz LL(36)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(38):`
kusano	2b45e8	`#if defined(LN) \|\| defined(RT)`
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`subi r0, KK, 1`
kusano	2b45e8	`#else`
kusano	2b45e8	`subi r0, KK, 4`
kusano	2b45e8	`#endif`
kusano	2b45e8	`slwi TEMP, r0, 0 + BASE_SHIFT`
kusano	2b45e8	`slwi r0, r0, 2 + BASE_SHIFT`
kusano	2b45e8	`add AO, AORIG, TEMP`
kusano	2b45e8	`add BO, B, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(LN) \|\| defined(LT)`
kusano	2b45e8	`LFD f16, 0 * SIZE(BO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(BO)`
kusano	2b45e8	`LFD f18, 2 * SIZE(BO)`
kusano	2b45e8	`LFD f19, 3 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FSUB f0, f16, f0`
kusano	2b45e8	`FSUB f4, f17, f4`
kusano	2b45e8	`FSUB f8, f18, f8`
kusano	2b45e8	`FSUB f12, f19, f12`
kusano	2b45e8	`#else`
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8	`LFD f20, 1 * SIZE(AO)`
kusano	2b45e8	`LFD f24, 2 * SIZE(AO)`
kusano	2b45e8	`LFD f28, 3 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FSUB f0, f16, f0`
kusano	2b45e8	`FSUB f4, f20, f4`
kusano	2b45e8	`FSUB f8, f24, f8`
kusano	2b45e8	`FSUB f12, f28, f12`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`LFD f21, 0 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f0, f21, f0`
kusano	2b45e8	`FMUL f4, f21, f4`
kusano	2b45e8	`FMUL f8, f21, f8`
kusano	2b45e8	`FMUL f12, f21, f12`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LT`
kusano	2b45e8	`LFD f16, 0 * SIZE(AO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f0, f16, f0`
kusano	2b45e8	`FMUL f4, f16, f4`
kusano	2b45e8	`FMUL f8, f16, f8`
kusano	2b45e8	`FMUL f12, f16, f12`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RN`
kusano	2b45e8	`LFD f16, 0 * SIZE(BO)`
kusano	2b45e8	`LFD f17, 1 * SIZE(BO)`
kusano	2b45e8	`LFD f18, 2 * SIZE(BO)`
kusano	2b45e8	`LFD f19, 3 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f0, f16, f0`
kusano	2b45e8	`FNMSUB f4, f17, f0, f4`
kusano	2b45e8	`FNMSUB f8, f18, f0, f8`
kusano	2b45e8	`FNMSUB f12, f19, f0, f12`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 5 * SIZE(BO)`
kusano	2b45e8	`LFD f17, 6 * SIZE(BO)`
kusano	2b45e8	`LFD f18, 7 * SIZE(BO)`
kusano	2b45e8	`LFD f19, 10 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 11 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 15 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f4, f16, f4`
kusano	2b45e8	`FNMSUB f8, f17, f4, f8`
kusano	2b45e8	`FNMSUB f12, f18, f4, f12`
kusano	2b45e8	`FMUL f8, f19, f8`
kusano	2b45e8	`FNMSUB f12, f20, f8, f12`
kusano	2b45e8	`FMUL f12, f21, f12`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RT`
kusano	2b45e8	`LFD f16, 15 * SIZE(BO)`
kusano	2b45e8	`LFD f17, 14 * SIZE(BO)`
kusano	2b45e8	`LFD f18, 13 * SIZE(BO)`
kusano	2b45e8	`LFD f19, 12 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f12, f16, f12`
kusano	2b45e8	`FNMSUB f8, f17, f12, f8`
kusano	2b45e8	`FNMSUB f4, f18, f12, f4`
kusano	2b45e8	`FNMSUB f0, f19, f12, f0`
kusano	2b45e8
kusano	2b45e8	`LFD f16, 10 * SIZE(BO)`
kusano	2b45e8	`LFD f17, 9 * SIZE(BO)`
kusano	2b45e8	`LFD f18, 8 * SIZE(BO)`
kusano	2b45e8	`LFD f19, 5 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FMUL f8, f16, f8`
kusano	2b45e8
kusano	2b45e8	`LFD f20, 4 * SIZE(BO)`
kusano	2b45e8	`LFD f21, 0 * SIZE(BO)`
kusano	2b45e8
kusano	2b45e8	`FNMSUB f4, f17, f8, f4`
kusano	2b45e8	`FNMSUB f0, f18, f8, f0`
kusano	2b45e8
kusano	2b45e8	`FMUL f4, f19, f4`
kusano	2b45e8	`FNMSUB f0, f20, f4, f0`
kusano	2b45e8	`FMUL f0, f21, f0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`subi CO1, CO1, 1 * SIZE`
kusano	2b45e8	`subi CO2, CO2, 1 * SIZE`
kusano	2b45e8	`subi CO3, CO3, 1 * SIZE`
kusano	2b45e8	`subi CO4, CO4, 1 * SIZE`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(LN) \|\| defined(LT)`
kusano	2b45e8	`STFD f0, 0 * SIZE(BO)`
kusano	2b45e8	`STFD f4, 1 * SIZE(BO)`
kusano	2b45e8	`STFD f8, 2 * SIZE(BO)`
kusano	2b45e8	`STFD f12, 3 * SIZE(BO)`
kusano	2b45e8	`#else`
kusano	2b45e8	`STFD f0, 0 * SIZE(AO)`
kusano	2b45e8	`STFD f4, 1 * SIZE(AO)`
kusano	2b45e8	`STFD f8, 2 * SIZE(AO)`
kusano	2b45e8	`STFD f12, 3 * SIZE(AO)`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`STFD f0, 0 * SIZE(CO1)`
kusano	2b45e8	`STFD f4, 0 * SIZE(CO2)`
kusano	2b45e8	`STFD f8, 0 * SIZE(CO3)`
kusano	2b45e8	`STFD f12, 0 * SIZE(CO4)`
kusano	2b45e8
kusano	2b45e8	`lfs f0, FZERO`
kusano	2b45e8	`fmr f1, f0`
kusano	2b45e8	`fmr f4, f0`
kusano	2b45e8	`fmr f5, f0`
kusano	2b45e8
kusano	2b45e8	`fmr f8, f0`
kusano	2b45e8	`fmr f9, f0`
kusano	2b45e8	`fmr f12, f0`
kusano	2b45e8	`fmr f13, f0`
kusano	2b45e8
kusano	2b45e8	`#ifndef LN`
kusano	2b45e8	`addi CO1, CO1, 1 * SIZE`
kusano	2b45e8	`addi CO2, CO2, 1 * SIZE`
kusano	2b45e8	`addi CO3, CO3, 1 * SIZE`
kusano	2b45e8	`addi CO4, CO4, 1 * SIZE`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RT`
kusano	2b45e8	`slwi r0, K, 0 + BASE_SHIFT`
kusano	2b45e8	`add AORIG, AORIG, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(LT) \|\| defined(RN)`
kusano	2b45e8	`sub TEMP, K, KK`
kusano	2b45e8	`slwi r0, TEMP, 0 + BASE_SHIFT`
kusano	2b45e8	`slwi TEMP, TEMP, 2 + BASE_SHIFT`
kusano	2b45e8	`add AO, AO, r0`
kusano	2b45e8	`add BO, BO, TEMP`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`subi KK, KK, 1`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef LT`
kusano	2b45e8	`addi KK, KK, 1`
kusano	2b45e8	`#endif`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(39):`
kusano	2b45e8	`#ifdef LN`
kusano	2b45e8	`slwi r0, K, 2 + BASE_SHIFT`
kusano	2b45e8	`add B, B, r0`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#if defined(LT) \|\| defined(RN)`
kusano	2b45e8	`mr B, BO`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RN`
kusano	2b45e8	`addi KK, KK, 4`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`#ifdef RT`
kusano	2b45e8	`subi KK, KK, 4`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`addic. J, J, -1`
kusano	2b45e8	`lfs f0, FZERO`
kusano	2b45e8	`bgt LL(10)`
kusano	2b45e8	`.align 4`
kusano	2b45e8
kusano	2b45e8	`LL(999):`
kusano	2b45e8	`addi r3, 0, 0`
kusano	2b45e8
kusano	2b45e8	`lfd f14, 0(SP)`
kusano	2b45e8	`lfd f15, 8(SP)`
kusano	2b45e8	`lfd f16, 16(SP)`
kusano	2b45e8	`lfd f17, 24(SP)`
kusano	2b45e8
kusano	2b45e8	`lfd f18, 32(SP)`
kusano	2b45e8	`lfd f19, 40(SP)`
kusano	2b45e8	`lfd f20, 48(SP)`
kusano	2b45e8	`lfd f21, 56(SP)`
kusano	2b45e8
kusano	2b45e8	`lfd f22, 64(SP)`
kusano	2b45e8	`lfd f23, 72(SP)`
kusano	2b45e8	`lfd f24, 80(SP)`
kusano	2b45e8	`lfd f25, 88(SP)`
kusano	2b45e8
kusano	2b45e8	`lfd f26, 96(SP)`
kusano	2b45e8	`lfd f27, 104(SP)`
kusano	2b45e8	`lfd f28, 112(SP)`
kusano	2b45e8	`lfd f29, 120(SP)`
kusano	2b45e8
kusano	2b45e8	`lfd f30, 128(SP)`
kusano	2b45e8	`lfd f31, 136(SP)`
kusano	2b45e8
kusano	2b45e8	`#ifdef __64BIT__`
kusano	2b45e8	`ld r31, 144(SP)`
kusano	2b45e8	`ld r30, 152(SP)`
kusano	2b45e8	`ld r29, 160(SP)`
kusano	2b45e8	`ld r28, 168(SP)`
kusano	2b45e8	`ld r27, 176(SP)`
kusano	2b45e8	`ld r26, 184(SP)`
kusano	2b45e8	`ld r25, 192(SP)`
kusano	2b45e8	`ld r24, 200(SP)`
kusano	2b45e8	`ld r23, 208(SP)`
kusano	2b45e8	`ld r22, 216(SP)`
kusano	2b45e8	`ld r21, 224(SP)`
kusano	2b45e8	`ld r20, 232(SP)`
kusano	2b45e8	`ld r19, 240(SP)`
kusano	2b45e8	`ld r18, 248(SP)`
kusano	2b45e8	`#else`
kusano	2b45e8	`lwz r31, 144(SP)`
kusano	2b45e8	`lwz r30, 148(SP)`
kusano	2b45e8	`lwz r29, 152(SP)`
kusano	2b45e8	`lwz r28, 156(SP)`
kusano	2b45e8	`lwz r27, 160(SP)`
kusano	2b45e8	`lwz r26, 164(SP)`
kusano	2b45e8	`lwz r25, 168(SP)`
kusano	2b45e8	`lwz r24, 172(SP)`
kusano	2b45e8	`lwz r23, 176(SP)`
kusano	2b45e8	`lwz r22, 180(SP)`
kusano	2b45e8	`lwz r21, 184(SP)`
kusano	2b45e8	`lwz r20, 188(SP)`
kusano	2b45e8	`lwz r19, 192(SP)`
kusano	2b45e8	`lwz r18, 196(SP)`
kusano	2b45e8	`#endif`
kusano	2b45e8
kusano	2b45e8	`addi SP, SP, STACKSIZE`
kusano	2b45e8
kusano	2b45e8	`blr`
kusano	2b45e8
kusano	2b45e8	`EPILOGUE`
kusano	2b45e8	`#endif`

bw / opentoonz

Source Code

Blame thirdparty/openblas/xianyi-OpenBLAS-e6e87a2/kernel/power/trsm_kernel_RT.S