/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/
#define ASSEMBLER
#include "common.h"
#define PREFETCHSIZE (8 * 16)
#define CPREFETCHSIZE 7
#define CPREFETCH lfetch.excl.nt2
#define M r32
#define N r33
#define K r34
#define A r38
#define B r39
#define C r36
#define LDC r37
#define I r15
#define J r16
#define AOFFSET r17
#define BOFFSET r18
#define TEMP r19
#define L r20
#define C1 r21
#define C2 r22
#define C3 r23
#define C4 r24
#define C5 r25
#define C6 r26
#define C7 r27
#define C8 r28
#define C9 loc0
#define C10 loc1
#define C11 loc2
#define C12 loc3
#define C13 loc4
#define C14 loc5
#define C15 loc6
#define C16 loc7
#define PREA r8
#define PREB r9
#define PREC r10
#define SP r12
#define ARLC r29
#define PR r30
#define ARPFS r31
#define ALPHA f8
#define AORIG loc8
#define KK loc9
#define KK8 loc10
#define OFFSET loc11
PROLOGUE
.prologue
PROFCODE
{ .mmi
.save ar.pfs, ARPFS
#ifdef TRMMKERNEL
alloc ARPFS = ar.pfs, 8, 16, 0, 0
#else
alloc ARPFS = ar.pfs, 8, 8, 0, 0
#endif
adds r14 = 16, SP
mov ARLC = ar.lc
}
{ .mmi
adds r8 = -16 * 16, SP
adds r9 = -15 * 16, SP
adds SP = -16 * 16, SP
}
;;
stf.spill [r8] = f16, 32
stf.spill [r9] = f17, 32
mov PR = pr
;;
stf.spill [r8] = f18, 32
stf.spill [r9] = f19, 32
;;
stf.spill [r8] = f20, 32
stf.spill [r9] = f21, 32
shr J = N, 3
;;
stf.spill [r8] = f22, 32
stf.spill [r9] = f23, 32
mov AOFFSET = A
;;
stf.spill [r8] = f24, 32
stf.spill [r9] = f25, 32
cmp.ge p6, p0 = 0, J
;;
stf.spill [r8] = f26, 32
stf.spill [r9] = f27, 32
;;
stf.spill [r8] = f28, 32
stf.spill [r9] = f29, 32
;;
stf.spill [r8] = f30
stf.spill [r9] = f31
ld8 C = [r14], 8
;;
ld8 LDC = [r14], 8
;;
shladd LDC = LDC, BASE_SHIFT, r0
;;
#ifndef TRMMKERNEL
(p6) br.cond.dpnt .L050
.body
;;
#else
.body
;;
ld8 OFFSET = [r14], 8
;;
#if defined(TRMMKERNEL) && !defined(LEFT)
;;
sub KK = r0, OFFSET
#endif
(p6) br.cond.dpnt .L050
;;
#endif
.align 32
.L010:
{ .mfi
adds J = -1, J
mov f64 = f0
shr I = M, 3
}
{ .mfi
mov C1 = C // coffset1 = c + 0 * ldc
mov f72 = f0
}
;;
{ .mmf
cmp.eq p6, p7 = 0, I
#if defined(TRMMKERNEL) && defined(LEFT)
mov KK = OFFSET
#else
nop __LINE__
#endif
mov f80 = f0
}
{ .mmf
add C2 = LDC, C // coffset2 = c + 1 * ldc
shladd C3 = LDC, 1, C // coffset3 = c + 2 * ldc
mov f88 = f0
}
;;
{ .mmf
shladd C5 = LDC, 2, C // coffset5 = c + 4 * ldc
shladd C = LDC, 3, C // coffset += 8 * ldc
mov f96 = f0
}
{ .mmf
shladd C4 = LDC, 1, C2 // coffset4 = c + 3 * ldc
shladd C6 = LDC, 2, C2 // coffset6 = c + 5 * ldc
mov f104 = f0
}
;;
{ .mfi
shladd C7 = LDC, 2, C3 // coffset7 = c + 6 * ldc
mov f112 = f0
#ifdef TRMMKERNEL
shladd KK8 = KK, BASE_SHIFT, r0
#else
nop __LINE__
#endif
}{ .mfb
sub C8 = C, LDC // coffset8 = c + 7 * ldc
mov f120 = f0
(p6) br.cond.dpnt .L020
}
;;
.align 16
.L011:
#if !defined(TRMMKERNEL) || \
defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
mov BOFFSET = B
;;
{ .mfb
LDFD f48 = [BOFFSET], SIZE
mov f65 = f0
nop __LINE__
}
;;
{ .mfb
LDFD f49 = [BOFFSET], SIZE
mov f73 = f0
nop __LINE__
}
;;
#else
{ .mfi
shladd BOFFSET = KK8, 3, B
mov f65 = f0
shladd AOFFSET = KK8, 3, AOFFSET
}
;;
LDFD f48 = [BOFFSET], SIZE
;;
{ .mfi
LDFD f49 = [BOFFSET], SIZE
mov f73 = f0
nop __LINE__
}
;;
#endif
LDFD f32 = [AOFFSET], SIZE
LDFD f50 = [BOFFSET], SIZE
;;
{ .mfb
LDFD f33 = [AOFFSET], SIZE
mov f81 = f0
nop __LINE__
}
{ .mfb
LDFD f51 = [BOFFSET], SIZE
mov f89 = f0
nop __LINE__
}
;;
LDFD f52 = [BOFFSET], SIZE
;;
{ .mmf
LDFD f53 = [BOFFSET], SIZE
setf.d f97 = r0
mov f105 = f0
}
{ .mfb
setf.d f113 = r0
mov f121 = f0
nop __LINE__
}
;;
LDFD f54 = [BOFFSET], SIZE
;;
{ .mmf
LDFD f55 = [BOFFSET], SIZE
setf.d f66 = r0
mov f74 = f0
}
{ .mfb
setf.d f82 = r0
mov f90 = f0
nop __LINE__
}
;;
LDFD f34 = [AOFFSET], SIZE
;;
{ .mmf
LDFD f35 = [AOFFSET], SIZE
setf.d f98 = r0
mov f106 = f0
}
{ .mfb
setf.d f114 = r0
mov f122 = f0
nop __LINE__
}
;;
LDFD f36 = [AOFFSET], SIZE
;;
{ .mmf
LDFD f37 = [AOFFSET], SIZE
setf.d f67 = r0
mov f75 = f0
}
{ .mfi
setf.d f83 = r0
mov f91 = f0
#ifndef TRMMKERNEL
nop __LINE__
#else
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub L = K, KK
#elif defined(LEFT)
adds L = 8, KK
#else
adds L = 8, KK
#endif
#endif
}
;;
LDFD f38 = [AOFFSET], SIZE
;;
{ .mmf
LDFD f39 = [AOFFSET], SIZE
setf.d f99 = r0
mov f107 = f0
}
{ .mfi
setf.d f115 = r0
mov f123 = f0
adds PREC = CPREFETCHSIZE * SIZE, C1
}
;;
{ .mmf
CPREFETCH [PREC], LDC
setf.d f68 = r0
mov f76 = f0
}
{ .mfi
setf.d f84 = r0
mov f92 = f0
#ifndef TRMMKERNEL
adds L = 1, K
#else
adds L = 1, L
#endif
}
;;
{ .mmf
CPREFETCH [PREC], LDC
setf.d f100 = r0
mov f108 = f0
}
{ .mfi
setf.d f116 = r0
mov f124 = f0
adds PREA = (PREFETCHSIZE + 8) * SIZE, AOFFSET
}
;;
{ .mmf
CPREFETCH [PREC], LDC
setf.d f69 = r0
mov f77 = f0
}
{ .mfi
setf.d f85 = r0
mov f93 = f0
adds PREB = (PREFETCHSIZE - 8) * SIZE, BOFFSET
}
;;
{ .mmf
CPREFETCH [PREC], LDC
setf.d f101 = r0
mov f109 = f0
}
{ .mfi
setf.d f117 = r0
mov f125 = f0
tbit.z p12, p0 = L, 0
}
;;
{ .mmf
CPREFETCH [PREC], LDC
setf.d f70 = r0
mov f78 = f0
}
{ .mfi
setf.d f86 = r0
mov f94 = f0
shr L = L, 1
}
;;
{ .mmf
CPREFETCH [PREC], LDC
setf.d f102 = r0
mov f110 = f0
}
{ .mfi
setf.d f118 = r0
mov f126 = f0
adds L = -1, L
}
;;
{ .mmf
CPREFETCH [PREC], LDC
setf.d f71 = r0
mov f79 = f0
}
{ .mfi
setf.d f87 = r0
mov f95 = f0
mov ar.lc = L
}
;;
{ .mmf
CPREFETCH [PREC]
setf.d f103 = r0
mov f111 = f0
}
{ .mfi
setf.d f119 = r0
mov f127 = f0
cmp.eq p3, p0 = r0, r0
}
;;
.align 16
.L012:
/* 1 */
{ .mfi
lfetch.fault.nt1 [PREA], 8 * SIZE
FMA f64 = f32, f48, f64 // A1 * B1
nop __LINE__
}
{ .mfb
(p12) cmp.ne p3, p0 = 0, L
FMA f72 = f32, f49, f72 // A1 * B2
nop __LINE__
}
;;
/* 2 */
{ .mfi
lfetch.nt1 [PREB], 8 * SIZE
FMA f80 = f32, f50, f80 // A1 * B3
nop __LINE__
}
{ .mfb
cmp.ne p4, p5 = 0, L
FMA f88 = f32, f51, f88 // A1 * B4
nop __LINE__
}
;;
/* 3 */
{ .mfb
(p3) LDFD f40 = [AOFFSET], SIZE
FMA f96 = f32, f52, f96 // A1 * B5
nop __LINE__
}
{ .mfb
adds C9 = 4 * SIZE, C1
FMA f104 = f32, f53, f104 // A1 * B6
nop __LINE__
}
;;
/* 4 */
{ .mfi
(p3) LDFD f56 = [BOFFSET], SIZE
FMA f112 = f32, f54, f112 // A1 * B7
adds C10 = 4 * SIZE, C2
}
{ .mfb
(p3) LDFD f41 = [AOFFSET], SIZE
FMA f120 = f32, f55, f120 // A1 * B8
nop __LINE__
}
;;
/* 5 */
{ .mfi
(p3) LDFD f57 = [BOFFSET], SIZE
FMA f65 = f33, f48, f65 // A2 * B1
adds C11 = 4 * SIZE, C3
}
{ .mfb
(p3) LDFD f42 = [AOFFSET], SIZE
FMA f73 = f33, f49, f73 // A2 * B2
nop __LINE__
}
;;
/* 6 */
{ .mfi
(p3) LDFD f58 = [BOFFSET], SIZE
FMA f81 = f33, f50, f81 // A2 * B3
adds C12 = 4 * SIZE, C4
}
{ .mfb
(p3) LDFD f43 = [AOFFSET], SIZE
FMA f89 = f33, f51, f89 // A2 * B4
nop __LINE__
}
;;
/* 7 */
{ .mfi
(p3) LDFD f59 = [BOFFSET], SIZE
FMA f97 = f33, f52, f97 // A2 * B5
adds C13 = 4 * SIZE, C5
}
{ .mfb
(p3) LDFD f44 = [AOFFSET], SIZE
FMA f105 = f33, f53, f105 // A2 * B6
nop __LINE__
}
;;
/* 8 */
{ .mfi
(p3) LDFD f60 = [BOFFSET], SIZE
FMA f113 = f33, f54, f113 // A2 * B7
adds C14 = 4 * SIZE, C6
}
{ .mfb
(p3) LDFD f45 = [AOFFSET], SIZE
FMA f121 = f33, f55, f121 // A2 * B8
nop __LINE__
}
;;
/* 9 */
{ .mfi
(p3) LDFD f61 = [BOFFSET], SIZE
FMA f66 = f34, f48, f66 // A3 * B1
adds C15 = 4 * SIZE, C7
}
{ .mfb
(p3) LDFD f46 = [AOFFSET], SIZE
FMA f74 = f34, f49, f74 // A3 * B2
nop __LINE__
}
;;
/* 10 */
{ .mfi
(p3) LDFD f62 = [BOFFSET], SIZE
FMA f82 = f34, f50, f82 // A3 * B3
adds C16 = 4 * SIZE, C8
}
{ .mfb
(p3) LDFD f47 = [AOFFSET], SIZE
FMA f90 = f34, f51, f90 // A3 * B4
nop __LINE__
}
;;
/* 11 */
{ .mfb
(p3) LDFD f63 = [BOFFSET], SIZE
FMA f98 = f34, f52, f98 // A3 * B5
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f106 = f34, f53, f106 // A3 * B6
nop __LINE__
}
;;
/* 12 */
{ .mfb
nop __LINE__
FMA f114 = f34, f54, f114 // A3 * B7
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f122 = f34, f55, f122 // A3 * B8
nop __LINE__
}
;;
/* 13 */
{ .mfb
nop __LINE__
FMA f67 = f35, f48, f67 // A4 * B1
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f75 = f35, f49, f75 // A4 * B2
nop __LINE__
}
;;
/* 14 */
{ .mfb
nop __LINE__
FMA f83 = f35, f50, f83 // A4 * B3
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f91 = f35, f51, f91 // A4 * B4
nop __LINE__
}
;;
/* 15 */
{ .mfb
nop __LINE__
FMA f99 = f35, f52, f99 // A4 * B5
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f107 = f35, f53, f107 // A4 * B6
nop __LINE__
}
;;
/* 16 */
{ .mfb
nop __LINE__
FMA f115 = f35, f54, f115 // A4 * B7
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f123 = f35, f55, f123 // A4 * B8
nop __LINE__
}
;;
/* 17 */
{ .mfb
nop __LINE__
FMA f68 = f36, f48, f68 // A5 * B1
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f76 = f36, f49, f76 // A5 * B2
nop __LINE__
}
;;
/* 18 */
{ .mfb
nop __LINE__
FMA f84 = f36, f50, f84 // A5 * B3
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f92 = f36, f51, f92 // A5 * B4
nop __LINE__
}
;;
/* 19 */
{ .mfb
nop __LINE__
FMA f100 = f36, f52, f100 // A5 * B5
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f108 = f36, f53, f108 // A5 * B6
nop __LINE__
}
;;
/* 20 */
{ .mfb
nop __LINE__
FMA f116 = f36, f54, f116 // A5 * B7
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f124 = f36, f55, f124 // A5 * B8
nop __LINE__
}
;;
/* 21 */
{ .mfb
nop __LINE__
FMA f69 = f37, f48, f69 // A6 * B1
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f77 = f37, f49, f77 // A6 * B2
nop __LINE__
}
;;
/* 22 */
{ .mfb
nop __LINE__
FMA f85 = f37, f50, f85 // A6 * B3
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f93 = f37, f51, f93 // A6 * B4
nop __LINE__
}
;;
/* 23 */
{ .mfb
nop __LINE__
FMA f101 = f37, f52, f101 // A6 * B5
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f109 = f37, f53, f109 // A6 * B6
nop __LINE__
}
;;
/* 24 */
{ .mfb
nop __LINE__
FMA f117 = f37, f54, f117 // A6 * B7
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f125 = f37, f55, f125 // A6 * B8
nop __LINE__
}
;;
/* 25 */
{ .mfb
nop __LINE__
FMA f70 = f38, f48, f70 // A7 * B1
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f78 = f38, f49, f78 // A7 * B2
nop __LINE__
}
;;
/* 26 */
{ .mfb
nop __LINE__
FMA f86 = f38, f50, f86 // A7 * B3
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f94 = f38, f51, f94 // A7 * B4
nop __LINE__
}
;;
/* 27 */
{ .mfb
nop __LINE__
FMA f102 = f38, f52, f102 // A7 * B5
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f110 = f38, f53, f110 // A7 * B6
nop __LINE__
}
;;
/* 28 */
{ .mfb
nop __LINE__
FMA f118 = f38, f54, f118 // A7 * B7
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f126 = f38, f55, f126 // A7 * B8
nop __LINE__
}
;;
/* 29 */
{ .mfb
(p4) LDFD f32 = [AOFFSET], SIZE
FMA f71 = f39, f48, f71 // A8 * B1
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f79 = f39, f49, f79 // A8 * B2
nop __LINE__
}
;;
/* 30 */
{ .mfb
(p4) LDFD f33 = [AOFFSET], SIZE
FMA f87 = f39, f50, f87 // A8 * B3
nop __LINE__
}
{ .mfb
(p4) LDFD f48 = [BOFFSET], SIZE
FMA f95 = f39, f51, f95 // A8 * B4
nop __LINE__
}
;;
/* 31 */
{ .mfb
(p4) LDFD f34 = [AOFFSET], SIZE
FMA f103 = f39, f52, f103 // A8 * B5
nop __LINE__
}
{ .mfb
(p4) LDFD f49 = [BOFFSET], SIZE
FMA f111 = f39, f53, f111 // A8 * B6
nop __LINE__
}
;;
/* 32 */
{ .mfb
lfetch.fault.nt1 [PREA], 8 * SIZE
FMA f119 = f39, f54, f119 // A8 * B7
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f127 = f39, f55, f127 // A8 * B8
nop __LINE__
}
;;
/* 33 */
{ .mfb
lfetch.nt1 [PREB], 8 * SIZE
(p3) FMA f64 = f40, f56, f64 // A1 * B1
nop __LINE__
}
{ .mfb
nop __LINE__
(p3) FMA f72 = f40, f57, f72 // A1 * B2
nop __LINE__
}
;;
/* 34 */
{ .mfb
(p4) LDFD f35 = [AOFFSET], SIZE
(p3) FMA f80 = f40, f58, f80 // A1 * B3
nop __LINE__
}
{ .mfb
(p4) LDFD f50 = [BOFFSET], SIZE
(p3) FMA f88 = f40, f59, f88 // A1 * B4
nop __LINE__
}
;;
/* 35 */
{ .mfb
(p4) LDFD f36 = [AOFFSET], SIZE
(p3) FMA f96 = f40, f60, f96 // A1 * B5
nop __LINE__
}
{ .mfb
(p4) LDFD f51 = [BOFFSET], SIZE
(p3) FMA f104 = f40, f61, f104 // A1 * B6
nop __LINE__
}
;;
/* 36 */
{ .mfb
(p4) LDFD f37 = [AOFFSET], SIZE
(p3) FMA f112 = f40, f62, f112 // A1 * B7
nop __LINE__
}
{ .mfb
(p4) LDFD f52 = [BOFFSET], SIZE
(p3) FMA f120 = f40, f63, f120 // A1 * B8
nop __LINE__
}
;;
/* 37 */
{ .mfb
(p4) LDFD f38 = [AOFFSET], SIZE
(p3) FMA f65 = f41, f56, f65 // A2 * B1
nop __LINE__
}
{ .mfb
(p4) LDFD f53 = [BOFFSET], SIZE
(p3) FMA f73 = f41, f57, f73 // A2 * B2
nop __LINE__
}
;;
/* 38 */
{ .mfb
(p4) LDFD f39 = [AOFFSET], SIZE
(p3) FMA f81 = f41, f58, f81 // A2 * B3
nop __LINE__
}
{ .mfb
(p4) LDFD f54 = [BOFFSET], SIZE
(p3) FMA f89 = f41, f59, f89 // A2 * B4
nop __LINE__
}
;;
/* 39 */
{ .mfb
(p4) LDFD f55 = [BOFFSET], SIZE
(p3) FMA f97 = f41, f60, f97 // A2 * B5
nop __LINE__
}
{ .mfb
nop __LINE__
(p3) FMA f105 = f41, f61, f105 // A2 * B6
nop __LINE__
}
;;
/* 40 */
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f6 = [C1 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f113 = f41, f62, f113 // A2 * B7
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f7 = [C9 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f121 = f41, f63, f121 // A2 * B8
nop __LINE__
}
;;
/* 41 */
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f10 = [C1 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f66 = f42, f56, f66 // A3 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f11 = [C9 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f74 = f42, f57, f74 // A3 * B2
nop __LINE__
}
;;
/* 42 */
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f12 = [C1 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f82 = f42, f58, f82 // A3 * B3
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f13 = [C9 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f90 = f42, f59, f90 // A3 * B4
nop __LINE__
}
;;
/* 43 */
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f14 = [C1 ], -3 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f98 = f42, f60, f98 // A3 * B5
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f15 = [C9 ], -3 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f106 = f42, f61, f106 // A3 * B6
nop __LINE__
}
;;
/* 44 */
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f16 = [C2 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f114 = f42, f62, f114 // A3 * B7
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f17 = [C10], SIZE
#else
nop __LINE__
#endif
(p3) FMA f122 = f42, f63, f122 // A3 * B8
nop __LINE__
}
;;
/* 45 */
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f18 = [C2 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f67 = f43, f56, f67 // A4 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f19 = [C10], SIZE
#else
nop __LINE__
#endif
(p3) FMA f75 = f43, f57, f75 // A4 * B2
nop __LINE__
}
;;
/* 46 */
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f20 = [C2 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f83 = f43, f58, f83 // A4 * B3
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f21 = [C10], SIZE
#else
nop __LINE__
#endif
(p3) FMA f91 = f43, f59, f91 // A4 * B4
nop __LINE__
}
;;
/* 47 */
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f22 = [C2 ], -3 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f99 = f43, f60, f99 // A4 * B5
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f23 = [C10], -3 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f107 = f43, f61, f107 // A4 * B6
nop __LINE__
}
;;
/* 48 */
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f24 = [C3 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f115 = f43, f62, f115 // A4 * B7
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f25 = [C11], SIZE
#else
nop __LINE__
#endif
(p3) FMA f123 = f43, f63, f123 // A4 * B8
nop __LINE__
}
;;
/* 49 */
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f26 = [C3 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f68 = f44, f56, f68 // A5 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f27 = [C11], SIZE
#else
nop __LINE__
#endif
(p3) FMA f76 = f44, f57, f76 // A5 * B2
nop __LINE__
}
;;
/* 50 */
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f28 = [C3 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f84 = f44, f58, f84 // A5 * B3
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f29 = [C11], SIZE
#else
nop __LINE__
#endif
(p3) FMA f92 = f44, f59, f92 // A5 * B4
nop __LINE__
}
;;
/* 51 */
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f30 = [C3 ], -3 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f100 = f44, f60, f100 // A5 * B5
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f31 = [C11], -3 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f108 = f44, f61, f108 // A5 * B6
nop __LINE__
}
;;
/* 52 */
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f32 = [C4 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f116 = f44, f62, f116 // A5 * B7
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f33 = [C12], SIZE
#else
nop __LINE__
#endif
(p3) FMA f124 = f44, f63, f124 // A5 * B8
nop __LINE__
}
;;
/* 53 */
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f34 = [C4 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f69 = f45, f56, f69 // A6 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f35 = [C12], SIZE
#else
nop __LINE__
#endif
(p3) FMA f77 = f45, f57, f77 // A6 * B2
nop __LINE__
}
;;
/* 54 */
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f36 = [C4 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f85 = f45, f58, f85 // A6 * B3
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f37 = [C12], SIZE
#else
nop __LINE__
#endif
(p3) FMA f93 = f45, f59, f93 // A6 * B4
nop __LINE__
}
;;
/* 55 */
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f38 = [C4 ], -3 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f101 = f45, f60, f101 // A6 * B5
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f39 = [C12], -3 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f109 = f45, f61, f109 // A6 * B6
nop __LINE__
}
;;
/* 56 */
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f48 = [C5 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f117 = f45, f62, f117 // A6 * B7
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f49 = [C13], SIZE
#else
nop __LINE__
#endif
(p3) FMA f125 = f45, f63, f125 // A6 * B8
nop __LINE__
}
;;
/* 57 */
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f50 = [C5 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f70 = f46, f56, f70 // A7 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f51 = [C13], SIZE
#else
nop __LINE__
#endif
(p3) FMA f78 = f46, f57, f78 // A7 * B2
nop __LINE__
}
;;
/* 58 */
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f52 = [C5 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f86 = f46, f58, f86 // A7 * B3
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f53 = [C13], SIZE
#else
nop __LINE__
#endif
(p3) FMA f94 = f46, f59, f94 // A7 * B4
nop __LINE__
}
;;
/* 59 */
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f54 = [C5 ], -3 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f102 = f46, f60, f102 // A7 * B5
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f55 = [C13], -3 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f110 = f46, f61, f110 // A7 * B6
nop __LINE__
}
;;
/* 60 */
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f40 = [C6 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f118 = f46, f62, f118 // A7 * B7
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f41 = [C14], SIZE
#else
nop __LINE__
#endif
(p3) FMA f126 = f46, f63, f126 // A7 * B8
nop __LINE__
}
;;
/* 61 */
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f42 = [C6 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f71 = f47, f56, f71 // A8 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f43 = [C14], SIZE
#else
nop __LINE__
#endif
(p3) FMA f79 = f47, f57, f79 // A8 * B2
nop __LINE__
}
;;
/* 62 */
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f44 = [C6 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f87 = f47, f58, f87 // A8 * B3
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f45 = [C14], SIZE
#else
nop __LINE__
#endif
(p3) FMA f95 = f47, f59, f95 // A8 * B4
nop __LINE__
}
;;
/* 63 */
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f59 = [C6 ], -3 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f103 = f47, f60, f103 // A8 * B5
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f60 = [C14], -3 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f111 = f47, f61, f111 // A8 * B6
nop __LINE__
}
;;
/* 64 */
{ .mfi
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f61 = [C7 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f119 = f47, f62, f119 // A8 * B7
adds L = -1, L
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f62 = [C15], SIZE
#else
nop __LINE__
#endif
(p3) FMA f127 = f47, f63, f127 // A8 * B8
br.cloop.sptk.few .L012
}
;;
.L013:
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
{ .mfi
(p5) LDFD f63 = [C7 ], SIZE
FMA f64 = ALPHA, f64, f6
cmp.ne p6, p0 = 1, I
}
{ .mfb
(p5) LDFD f6 = [C15], SIZE
FMA f68 = ALPHA, f68, f7
nop __LINE__
}
;;
{ .mfi
(p5) LDFD f7 = [C7 ], SIZE
FMA f65 = ALPHA, f65, f10
adds I = -1, I
}
{ .mfb
(p5) LDFD f10 = [C15], SIZE
FMA f69 = ALPHA, f69, f11
nop __LINE__
}
;;
{ .mfb
(p5) LDFD f11 = [C7 ], -3 * SIZE
FMA f66 = ALPHA, f66, f12
nop __LINE__
}
{ .mfb
(p5) LDFD f12 = [C15], -3 * SIZE
FMA f70 = ALPHA, f70, f13
nop __LINE__
}
;;
{ .mfb
LDFD f13 = [C8 ], SIZE
FMA f67 = ALPHA, f67, f14
nop __LINE__
}
{ .mfb
LDFD f14 = [C16], SIZE
FMA f71 = ALPHA, f71, f15
nop __LINE__
}
;;
{ .mmf
STFD [C1 ] = f64, SIZE
STFD [C9 ] = f68, SIZE
FMA f72 = ALPHA, f72, f16
}
{ .mmf
LDFD f15 = [C8 ], SIZE
LDFD f16 = [C16], SIZE
FMA f76 = ALPHA, f76, f17
}
;;
{ .mmf
STFD [C1 ] = f65, SIZE
STFD [C9 ] = f69, SIZE
FMA f73 = ALPHA, f73, f18
}
{ .mmf
LDFD f17 = [C8 ], SIZE
LDFD f18 = [C16], SIZE
FMA f77 = ALPHA, f77, f19
}
;;
{ .mmf
STFD [C1 ] = f66, SIZE
STFD [C9 ] = f70, SIZE
FMA f74 = ALPHA, f74, f20
}
{ .mmf
LDFD f19 = [C8 ], -3 * SIZE
LDFD f20 = [C16], -3 * SIZE
FMA f78 = ALPHA, f78, f21
}
;;
{ .mfb
STFD [C1 ] = f67, 5 * SIZE
FMA f75 = ALPHA, f75, f22
nop __LINE__
}
{ .mfb
STFD [C9 ] = f71, 5 * SIZE
FMA f79 = ALPHA, f79, f23
nop __LINE__
}
;;
{ .mfb
STFD [C2 ] = f72, SIZE
FMA f80 = ALPHA, f80, f24
nop __LINE__
}
{ .mfb
STFD [C10] = f76, SIZE
FMA f84 = ALPHA, f84, f25
nop __LINE__
}
;;
{ .mfb
STFD [C2 ] = f73, SIZE
FMA f81 = ALPHA, f81, f26
nop __LINE__
}
{ .mfb
STFD [C10] = f77, SIZE
FMA f85 = ALPHA, f85, f27
nop __LINE__
}
;;
{ .mfb
STFD [C2 ] = f74, SIZE
FMA f82 = ALPHA, f82, f28
nop __LINE__
}
{ .mfb
STFD [C10] = f78, SIZE
FMA f86 = ALPHA, f86, f29
nop __LINE__
}
;;
{ .mfb
STFD [C2 ] = f75, 5 * SIZE
FMA f83 = ALPHA, f83, f30
nop __LINE__
}
{ .mfb
STFD [C10] = f79, 5 * SIZE
FMA f87 = ALPHA, f87, f31
nop __LINE__
}
;;
{ .mfb
STFD [C3 ] = f80, SIZE
FMA f88 = ALPHA, f88, f32
nop __LINE__
}
{ .mfb
STFD [C11] = f84, SIZE
FMA f92 = ALPHA, f92, f33
nop __LINE__
}
;;
{ .mfb
STFD [C3 ] = f81, SIZE
FMA f89 = ALPHA, f89, f34
nop __LINE__
}
{ .mfb
STFD [C11] = f85, SIZE
FMA f93 = ALPHA, f93, f35
nop __LINE__
}
;;
{ .mfb
STFD [C3 ] = f82, SIZE
FMA f90 = ALPHA, f90, f36
nop __LINE__
}
{ .mfb
STFD [C11] = f86, SIZE
FMA f94 = ALPHA, f94, f37
nop __LINE__
}
;;
{ .mfb
STFD [C3 ] = f83, 5 * SIZE
FMA f91 = ALPHA, f91, f38
nop __LINE__
}
{ .mfb
STFD [C11] = f87, 5 * SIZE
FMA f95 = ALPHA, f95, f39
nop __LINE__
}
;;
{ .mfb
STFD [C4 ] = f88, SIZE
FMA f96 = ALPHA, f96, f48
nop __LINE__
}
{ .mfb
STFD [C12] = f92, SIZE
FMA f100 = ALPHA, f100, f49
nop __LINE__
}
;;
{ .mfb
STFD [C4 ] = f89, SIZE
FMA f97 = ALPHA, f97, f50
nop __LINE__
}
{ .mfb
STFD [C12] = f93, SIZE
FMA f101 = ALPHA, f101, f51
nop __LINE__
}
;;
{ .mfb
STFD [C4 ] = f90, SIZE
FMA f98 = ALPHA, f98, f52
nop __LINE__
}
{ .mfb
STFD [C12] = f94, SIZE
FMA f102 = ALPHA, f102, f53
nop __LINE__
}
;;
{ .mfb
STFD [C4 ] = f91, 5 * SIZE
FMA f99 = ALPHA, f99, f54
nop __LINE__
}
{ .mfb
STFD [C12] = f95, 5 * SIZE
FMA f103 = ALPHA, f103, f55
nop __LINE__
}
;;
{ .mfb
STFD [C5 ] = f96, SIZE
FMA f104 = ALPHA, f104, f40
nop __LINE__
}
{ .mfb
STFD [C13] = f100, SIZE
FMA f108 = ALPHA, f108, f41
nop __LINE__
}
;;
{ .mfb
STFD [C5 ] = f97, SIZE
FMA f105 = ALPHA, f105, f42
nop __LINE__
}
{ .mfb
STFD [C13] = f101, SIZE
FMA f109 = ALPHA, f109, f43
nop __LINE__
}
;;
{ .mfb
STFD [C5 ] = f98, SIZE
FMA f106 = ALPHA, f106, f44
nop __LINE__
}
{ .mfb
STFD [C13] = f102, SIZE
FMA f110 = ALPHA, f110, f45
nop __LINE__
}
;;
{ .mfb
STFD [C5 ] = f99, 5 * SIZE
FMA f107 = ALPHA, f107, f59
nop __LINE__
}
{ .mfb
STFD [C13] = f103, 5 * SIZE
FMA f111 = ALPHA, f111, f60
nop __LINE__
}
;;
{ .mfb
STFD [C6 ] = f104, SIZE
FMA f112 = ALPHA, f112, f61
nop __LINE__
}
{ .mfb
STFD [C14] = f108, SIZE
FMA f116 = ALPHA, f116, f62
nop __LINE__
}
;;
{ .mfb
STFD [C6 ] = f105, SIZE
FMA f113 = ALPHA, f113, f63
nop __LINE__
}
{ .mfb
STFD [C14] = f109, SIZE
FMA f117 = ALPHA, f117, f6
nop __LINE__
}
;;
{ .mfb
STFD [C6 ] = f106, SIZE
FMA f114 = ALPHA, f114, f7
nop __LINE__
}
{ .mfb
STFD [C14] = f110, SIZE
FMA f118 = ALPHA, f118, f10
nop __LINE__
}
;;
{ .mfb
STFD [C6 ] = f107, 5 * SIZE
FMA f115 = ALPHA, f115, f11
nop __LINE__
}
{ .mfb
STFD [C14] = f111, 5 * SIZE
FMA f119 = ALPHA, f119, f12
nop __LINE__
}
;;
{ .mfb
STFD [C7 ] = f112, SIZE
FMA f120 = ALPHA, f120, f13
nop __LINE__
}
{ .mfb
STFD [C15] = f116, SIZE
FMA f124 = ALPHA, f124, f14
nop __LINE__
}
;;
{ .mfb
STFD [C7 ] = f113, SIZE
FMA f121 = ALPHA, f121, f15
nop __LINE__
}
{ .mfb
STFD [C15] = f117, SIZE
FMA f125 = ALPHA, f125, f16
nop __LINE__
}
;;
{ .mfb
STFD [C7 ] = f114, SIZE
FMA f122 = ALPHA, f122, f17
nop __LINE__
}
{ .mfb
STFD [C15] = f118, SIZE
FMA f126 = ALPHA, f126, f18
nop __LINE__
}
;;
{ .mfb
STFD [C7 ] = f115, 5 * SIZE
FMA f123 = ALPHA, f123, f19
nop __LINE__
}
{ .mfb
STFD [C15] = f119, 5 * SIZE
FMA f127 = ALPHA, f127, f20
nop __LINE__
}
;;
{ .mfb
STFD [C8 ] = f120, SIZE
mov f64 = f0
nop __LINE__
}
{ .mfb
STFD [C16] = f124, SIZE
mov f72 = f0
nop __LINE__
}
;;
{ .mfi
STFD [C8 ] = f121, SIZE
mov f80 = f0
nop __LINE__
}
{ .mfb
STFD [C16] = f125, SIZE
mov f88 = f0
nop __LINE__
}
;;
{ .mfi
STFD [C8 ] = f122, SIZE
mov f96 = f0
nop __LINE__
}
{ .mfb
STFD [C16] = f126, SIZE
mov f104 = f0
nop __LINE__
}
;;
{ .mfi
STFD [C8 ] = f123, 5 * SIZE
mov f112 = f0
nop __LINE__
}
{ .mfb
STFD [C16] = f127, 5 * SIZE
mov f120 = f0
(p6) br.cond.dptk .L011
}
;;
#else
{ .mfi
nop __LINE__
FMPY f64 = ALPHA, f64
cmp.ne p6, p0 = 1, I
}
{ .mfb
nop __LINE__
FMPY f68 = ALPHA, f68
nop __LINE__
}
;;
{ .mfi
nop __LINE__
FMPY f65 = ALPHA, f65
adds I = -1, I
}
{ .mfb
nop __LINE__
FMPY f69 = ALPHA, f69
nop __LINE__
}
;;
{ .mfb
nop __LINE__
FMPY f66 = ALPHA, f66
nop __LINE__
}
{ .mfb
nop __LINE__
FMPY f70 = ALPHA, f70
nop __LINE__
}
;;
{ .mfb
nop __LINE__
FMPY f67 = ALPHA, f67
nop __LINE__
}
{ .mfb
nop __LINE__
FMPY f71 = ALPHA, f71
nop __LINE__
}
;;
{ .mmf
STFD [C1 ] = f64, SIZE
STFD [C9 ] = f68, SIZE
FMPY f72 = ALPHA, f72
}
{ .mmf
nop __LINE__
nop __LINE__
FMPY f76 = ALPHA, f76
}
;;
{ .mmf
STFD [C1 ] = f65, SIZE
STFD [C9 ] = f69, SIZE
FMPY f73 = ALPHA, f73
}
{ .mmf
nop __LINE__
nop __LINE__
FMPY f77 = ALPHA, f77
}
;;
{ .mmf
STFD [C1 ] = f66, SIZE
STFD [C9 ] = f70, SIZE
FMPY f74 = ALPHA, f74
}
{ .mmf
nop __LINE__
nop __LINE__
FMPY f78 = ALPHA, f78
}
;;
{ .mfb
STFD [C1 ] = f67, 5 * SIZE
FMPY f75 = ALPHA, f75
nop __LINE__
}
{ .mfb
STFD [C9 ] = f71, 5 * SIZE
FMPY f79 = ALPHA, f79
nop __LINE__
}
;;
{ .mfb
STFD [C2 ] = f72, SIZE
FMPY f80 = ALPHA, f80
nop __LINE__
}
{ .mfb
STFD [C10] = f76, SIZE
FMPY f84 = ALPHA, f84
nop __LINE__
}
;;
{ .mfb
STFD [C2 ] = f73, SIZE
FMPY f81 = ALPHA, f81
nop __LINE__
}
{ .mfb
STFD [C10] = f77, SIZE
FMPY f85 = ALPHA, f85
nop __LINE__
}
;;
{ .mfb
STFD [C2 ] = f74, SIZE
FMPY f82 = ALPHA, f82
nop __LINE__
}
{ .mfb
STFD [C10] = f78, SIZE
FMPY f86 = ALPHA, f86
nop __LINE__
}
;;
{ .mfb
STFD [C2 ] = f75, 5 * SIZE
FMPY f83 = ALPHA, f83
nop __LINE__
}
{ .mfb
STFD [C10] = f79, 5 * SIZE
FMPY f87 = ALPHA, f87
nop __LINE__
}
;;
{ .mfb
STFD [C3 ] = f80, SIZE
FMPY f88 = ALPHA, f88
nop __LINE__
}
{ .mfb
STFD [C11] = f84, SIZE
FMPY f92 = ALPHA, f92
nop __LINE__
}
;;
{ .mfb
STFD [C3 ] = f81, SIZE
FMPY f89 = ALPHA, f89
nop __LINE__
}
{ .mfb
STFD [C11] = f85, SIZE
FMPY f93 = ALPHA, f93
nop __LINE__
}
;;
{ .mfb
STFD [C3 ] = f82, SIZE
FMPY f90 = ALPHA, f90
nop __LINE__
}
{ .mfb
STFD [C11] = f86, SIZE
FMPY f94 = ALPHA, f94
nop __LINE__
}
;;
{ .mfb
STFD [C3 ] = f83, 5 * SIZE
FMPY f91 = ALPHA, f91
nop __LINE__
}
{ .mfb
STFD [C11] = f87, 5 * SIZE
FMPY f95 = ALPHA, f95
nop __LINE__
}
;;
{ .mfb
STFD [C4 ] = f88, SIZE
FMPY f96 = ALPHA, f96
nop __LINE__
}
{ .mfb
STFD [C12] = f92, SIZE
FMPY f100 = ALPHA, f100
nop __LINE__
}
;;
{ .mfb
STFD [C4 ] = f89, SIZE
FMPY f97 = ALPHA, f97
nop __LINE__
}
{ .mfb
STFD [C12] = f93, SIZE
FMPY f101 = ALPHA, f101
nop __LINE__
}
;;
{ .mfb
STFD [C4 ] = f90, SIZE
FMPY f98 = ALPHA, f98
nop __LINE__
}
{ .mfb
STFD [C12] = f94, SIZE
FMPY f102 = ALPHA, f102
nop __LINE__
}
;;
{ .mfb
STFD [C4 ] = f91, 5 * SIZE
FMPY f99 = ALPHA, f99
nop __LINE__
}
{ .mfb
STFD [C12] = f95, 5 * SIZE
FMPY f103 = ALPHA, f103
nop __LINE__
}
;;
{ .mfb
STFD [C5 ] = f96, SIZE
FMPY f104 = ALPHA, f104
nop __LINE__
}
{ .mfb
STFD [C13] = f100, SIZE
FMPY f108 = ALPHA, f108
nop __LINE__
}
;;
{ .mfb
STFD [C5 ] = f97, SIZE
FMPY f105 = ALPHA, f105
nop __LINE__
}
{ .mfb
STFD [C13] = f101, SIZE
FMPY f109 = ALPHA, f109
nop __LINE__
}
;;
{ .mfb
STFD [C5 ] = f98, SIZE
FMPY f106 = ALPHA, f106
nop __LINE__
}
{ .mfb
STFD [C13] = f102, SIZE
FMPY f110 = ALPHA, f110
nop __LINE__
}
;;
{ .mfb
STFD [C5 ] = f99, 5 * SIZE
FMPY f107 = ALPHA, f107
nop __LINE__
}
{ .mfb
STFD [C13] = f103, 5 * SIZE
FMPY f111 = ALPHA, f111
nop __LINE__
}
;;
{ .mfb
STFD [C6 ] = f104, SIZE
FMPY f112 = ALPHA, f112
nop __LINE__
}
{ .mfb
STFD [C14] = f108, SIZE
FMPY f116 = ALPHA, f116
nop __LINE__
}
;;
{ .mfb
STFD [C6 ] = f105, SIZE
FMPY f113 = ALPHA, f113
nop __LINE__
}
{ .mfb
STFD [C14] = f109, SIZE
FMPY f117 = ALPHA, f117
nop __LINE__
}
;;
{ .mfb
STFD [C6 ] = f106, SIZE
FMPY f114 = ALPHA, f114
nop __LINE__
}
{ .mfb
STFD [C14] = f110, SIZE
FMPY f118 = ALPHA, f118
nop __LINE__
}
;;
{ .mfb
STFD [C6 ] = f107, 5 * SIZE
FMPY f115 = ALPHA, f115
nop __LINE__
}
{ .mfb
STFD [C14] = f111, 5 * SIZE
FMPY f119 = ALPHA, f119
nop __LINE__
}
;;
{ .mfb
STFD [C7 ] = f112, SIZE
FMPY f120 = ALPHA, f120
nop __LINE__
}
{ .mfb
STFD [C15] = f116, SIZE
FMPY f124 = ALPHA, f124
nop __LINE__
}
;;
{ .mfb
STFD [C7 ] = f113, SIZE
FMPY f121 = ALPHA, f121
nop __LINE__
}
{ .mfb
STFD [C15] = f117, SIZE
FMPY f125 = ALPHA, f125
nop __LINE__
}
;;
{ .mfi
STFD [C7 ] = f114, SIZE
FMPY f122 = ALPHA, f122
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
sub L = K, KK
#else
nop __LINE__
#endif
}
{ .mfb
STFD [C15] = f118, SIZE
FMPY f126 = ALPHA, f126
nop __LINE__
}
;;
{ .mfi
STFD [C7 ] = f115, 5 * SIZE
FMPY f123 = ALPHA, f123
#if defined(TRMMKERNEL) && (defined(LEFT) && defined(TRANSA))
adds L = -8, L
#else
nop __LINE__
#endif
}
{ .mfi
STFD [C15] = f119, 5 * SIZE
FMPY f127 = ALPHA, f127
#if defined(TRMMKERNEL) && (!defined(LEFT) && !defined(TRANSA))
adds L = -8, L
#else
nop __LINE__
#endif
}
;;
{ .mfi
STFD [C8 ] = f120, SIZE
mov f64 = f0
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd KK8 = L, BASE_SHIFT, r0
#else
nop __LINE__
#endif
}
{ .mfb
STFD [C16] = f124, SIZE
mov f72 = f0
nop __LINE__
}
;;
{ .mfi
STFD [C8 ] = f121, SIZE
mov f80 = f0
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd AOFFSET = KK8, 3, AOFFSET
#else
nop __LINE__
#endif
}
{ .mfi
STFD [C16] = f125, SIZE
mov f88 = f0
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd BOFFSET = KK8, 3, BOFFSET
#else
nop __LINE__
#endif
}
;;
{ .mfi
STFD [C8 ] = f122, SIZE
mov f96 = f0
#if defined(TRMMKERNEL) && defined(LEFT)
adds KK = 8, KK
#else
nop __LINE__
#endif
}
{ .mfb
STFD [C16] = f126, SIZE
mov f104 = f0
nop __LINE__
}
;;
{ .mfi
STFD [C8 ] = f123, 5 * SIZE
mov f112 = f0
#ifdef TRMMKERNEL
shladd KK8 = KK, BASE_SHIFT, r0
#else
nop __LINE__
#endif
}
{ .mfb
STFD [C16] = f127, 5 * SIZE
mov f120 = f0
(p6) br.cond.dptk .L011
}
;;
#endif
.L020:
#if 0
{ .mfi
cmp.eq p3, p0 = r0, r0
mov f89 = f0
tbit.z p6, p7 = M, 2
}
{ .mfb
#ifndef TRMMKERNEL
nop __LINE__
#else
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub L = K, KK
#elif defined(LEFT)
adds L = 4, KK
#else
adds L = 8, KK
#endif
#endif
mov f81 = f0
(p6) br.cond.dptk .L030
}
;;
#if !defined(TRMMKERNEL) || \
defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
{ .mfi
LDFPD f48, f49 = [B]
mov f65 = f0
nop __LINE__
}
{ .mfi
adds BOFFSET = 2 * SIZE, B
mov f73 = f0
adds PREA = (PREFETCHSIZE + 0) * SIZE, AOFFSET
}
;;
#else
{ .mfi
shladd BOFFSET = KK8, 3, B
mov f65 = f0
shladd AOFFSET = KK8, 2, AOFFSET
}
;;
{ .mfi
LDFPD f48, f49 = [BOFFSET], 2 * SIZE
mov f73 = f0
adds PREA = (PREFETCHSIZE + 0) * SIZE, AOFFSET
}
;;
#endif
{ .mmf
LDFPD f32, f33 = [AOFFSET], 2 * SIZE
setf.d f97 = r0
mov f105 = f0
}
{ .mfi
setf.d f113 = r0
mov f121 = f0
#ifndef TRMMKERNEL
adds L = 1, K
#else
adds L = 1, L
#endif
}
;;
{ .mmf
LDFPD f50, f51 = [BOFFSET], 2 * SIZE
setf.d f66 = r0
mov f74 = f0
}
{ .mfi
setf.d f82 = r0
mov f90 = f0
tbit.z p12, p0 = L, 0
}
;;
{ .mmf
LDFPD f52, f53 = [BOFFSET], 2 * SIZE
setf.d f98 = r0
mov f106 = f0
}
{ .mfi
setf.d f114 = r0
mov f122 = f0
shr L = L, 1
}
;;
{ .mfi
LDFPD f54, f55 = [BOFFSET], 2 * SIZE
mov f75 = f0
adds L = -1, L
}
{ .mmf
setf.d f67 = r0
setf.d f83 = r0
mov f91 = f0
}
;;
{ .mfi
LDFPD f34, f35 = [AOFFSET], 2 * SIZE
mov f107 = f0
mov ar.lc = L
}
{ .mmf
setf.d f99 = r0
setf.d f115 = r0
mov f123 = f0
}
;;
.align 32
.L022:
{ .mfi
lfetch.nt1 [PREA], 16 * SIZE
FMA f64 = f32, f48, f64 // A1 * B1
adds PREB = (PREFETCHSIZE + 0) * SIZE, BOFFSET
}
{ .mfi
nop __LINE__
FMA f72 = f32, f49, f72 // A1 * B2
(p12) cmp.ne p3, p0 = 0, L
}
;;
{ .mfi
lfetch.nt1 [PREB], 16 * SIZE
FMA f80 = f32, f50, f80 // A1 * B3
cmp.ne p4, p5 = 0, L
}
{ .mfb
nop __LINE__
FMA f88 = f32, f51, f88 // A1 * B4
nop __LINE__
}
;;
{ .mfi
(p3) LDFPD f40, f41 = [AOFFSET], 2 * SIZE
FMA f96 = f32, f52, f96 // A1 * B5
(p5) adds C9 = 2 * SIZE, C1
}
{ .mfi
nop __LINE__
FMA f104 = f32, f53, f104 // A1 * B6
(p5) adds C10 = 2 * SIZE, C2
}
;;
{ .mfi
(p3) LDFPD f56, f57 = [BOFFSET], 2 * SIZE
FMA f112 = f32, f54, f112 // A1 * B7
(p5) adds C11 = 2 * SIZE, C3
}
{ .mfi
nop __LINE__
FMA f120 = f32, f55, f120 // A1 * B8
(p5) adds C12 = 2 * SIZE, C4
}
;;
{ .mfi
(p3) LDFPD f58, f59 = [BOFFSET], 2 * SIZE
FMA f65 = f33, f48, f65 // A2 * B1
(p5) adds C13 = 2 * SIZE, C5
}
{ .mfi
nop __LINE__
FMA f73 = f33, f49, f73 // A2 * B2
(p5) adds C14 = 2 * SIZE, C6
}
;;
{ .mfi
(p3) LDFPD f60, f61 = [BOFFSET], 2 * SIZE
FMA f81 = f33, f50, f81 // A2 * B3
(p5) adds C15 = 2 * SIZE, C7
}
{ .mfi
nop __LINE__
FMA f89 = f33, f51, f89 // A2 * B4
(p5) adds C16 = 2 * SIZE, C8
}
;;
{ .mfb
(p3) LDFPD f62, f63 = [BOFFSET], 2 * SIZE
FMA f97 = f33, f52, f97 // A2 * B5
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f105 = f33, f53, f105 // A2 * B6
nop __LINE__
}
;;
{ .mfb
(p3) LDFPD f42, f43 = [AOFFSET], 2 * SIZE
FMA f113 = f33, f54, f113 // A2 * B7
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f121 = f33, f55, f121 // A2 * B8
nop __LINE__
}
;;
{ .mfb
nop __LINE__
FMA f66 = f34, f48, f66 // A3 * B1
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f74 = f34, f49, f74 // A3 * B2
nop __LINE__
}
;;
{ .mfb
nop __LINE__
FMA f82 = f34, f50, f82 // A3 * B3
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f90 = f34, f51, f90 // A3 * B4
nop __LINE__
}
;;
{ .mfb
nop __LINE__
FMA f98 = f34, f52, f98 // A3 * B5
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f106 = f34, f53, f106 // A3 * B6
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f114 = f34, f54, f114 // A3 * B7
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f122 = f34, f55, f122 // A3 * B8
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f67 = f35, f48, f67 // A4 * B1
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f75 = f35, f49, f75 // A4 * B2
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f83 = f35, f50, f83 // A4 * B3
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f91 = f35, f51, f91 // A4 * B4
nop __LINE__
}
{ .mfb
(p4) LDFPD f32, f33 = [AOFFSET], 2 * SIZE
FMA f99 = f35, f52, f99 // A4 * B5
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f107 = f35, f53, f107 // A4 * B6
nop __LINE__
}
{ .mfb
(p4) LDFPD f48, f49 = [BOFFSET], 2 * SIZE
FMA f115 = f35, f54, f115 // A4 * B7
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f123 = f35, f55, f123 // A4 * B8
nop __LINE__
}
;;
{ .mfb
(p4) LDFPD f50, f51 = [BOFFSET], 2 * SIZE
(p3) FMA f64 = f40, f56, f64 // A1 * B1
nop __LINE__
}
{ .mfb
nop __LINE__
(p3) FMA f72 = f40, f57, f72 // A1 * B2
nop __LINE__
}
;;
{ .mfb
(p4) LDFPD f52, f53 = [BOFFSET], 2 * SIZE
(p3) FMA f80 = f40, f58, f80 // A1 * B3
nop __LINE__
}
{ .mfb
nop __LINE__
(p3) FMA f88 = f40, f59, f88 // A1 * B4
nop __LINE__
}
;;
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f68 = [C1 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f96 = f40, f60, f96 // A1 * B5
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f70 = [C9 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f104 = f40, f61, f104 // A1 * B6
nop __LINE__
}
;;
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f69 = [C1 ], -1 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f112 = f40, f62, f112 // A1 * B7
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f71 = [C9 ], -1 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f120 = f40, f63, f120 // A1 * B8
nop __LINE__
}
;;
{ .mfb
(p4) LDFPD f54, f55 = [BOFFSET], 2 * SIZE
(p3) FMA f65 = f41, f56, f65 // A2 * B1
nop __LINE__
}
{ .mfb
(p3) FMA f73 = f41, f57, f73 // A2 * B2
nop __LINE__
}
{ .mfb
(p4) LDFPD f34, f35 = [AOFFSET], 2 * SIZE
(p3) FMA f81 = f41, f58, f81 // A2 * B3
nop __LINE__
}
{ .mfb
(p3) FMA f89 = f41, f59, f89 // A2 * B4
nop __LINE__
}
;;
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f76 = [C2 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f97 = f41, f60, f97 // A2 * B5
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f78 = [C10], SIZE
#else
nop __LINE__
#endif
(p3) FMA f105 = f41, f61, f105 // A2 * B6
nop __LINE__
}
;;
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f77 = [C2 ], -1 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f113 = f41, f62, f113 // A2 * B7
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f79 = [C10], -1 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f121 = f41, f63, f121 // A2 * B8
nop __LINE__
}
;;
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f84 = [C3 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f66 = f42, f56, f66 // A3 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f86 = [C11], SIZE
#else
nop __LINE__
#endif
(p3) FMA f74 = f42, f57, f74 // A3 * B2
nop __LINE__
}
;;
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f85 = [C3 ], -1 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f82 = f42, f58, f82 // A3 * B3
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f87 = [C11], -1 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f90 = f42, f59, f90 // A3 * B4
nop __LINE__
}
;;
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f92 = [C4 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f98 = f42, f60, f98 // A3 * B5
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f94 = [C12], SIZE
#else
nop __LINE__
#endif
(p3) FMA f106 = f42, f61, f106 // A3 * B6
nop __LINE__
}
;;
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f93 = [C4 ], -1 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f114 = f42, f62, f114 // A3 * B7
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f95 = [C12], -1 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f122 = f42, f63, f122 // A3 * B8
nop __LINE__
}
;;
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f100 = [C5 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f67 = f43, f56, f67 // A4 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f102 = [C13], SIZE
#else
nop __LINE__
#endif
(p3) FMA f75 = f43, f57, f75 // A4 * B2
nop __LINE__
}
;;
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f101 = [C5 ], -1 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f83 = f43, f58, f83 // A4 * B3
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f103 = [C13], -1 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f91 = f43, f59, f91 // A4 * B4
nop __LINE__
}
;;
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f108 = [C6 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f99 = f43, f60, f99 // A4 * B5
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f110 = [C14], SIZE
#else
nop __LINE__
#endif
(p3) FMA f107 = f43, f61, f107 // A4 * B6
nop __LINE__
}
;;
{ .mfi
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f109 = [C6 ], -1 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f115 = f43, f62, f115 // A4 * B7
adds L = -1, L
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f111 = [C14], -1 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f123 = f43, f63, f123 // A4 * B8
br.cloop.sptk.few .L022
}
;;
.L028:
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
{ .mfb
LDFD f116 = [C7 ], SIZE
FMA f64 = ALPHA, f64, f68
nop __LINE__
}
{ .mfb
LDFD f118 = [C15], SIZE
FMA f66 = ALPHA, f66, f70
nop __LINE__
}
;;
{ .mfb
LDFD f117 = [C7 ], -1 * SIZE
FMA f65 = ALPHA, f65, f69
nop __LINE__
}
{ .mfb
LDFD f119 = [C15], -1 * SIZE
FMA f67 = ALPHA, f67, f71
nop __LINE__
}
;;
{ .mfb
LDFD f124 = [C8], SIZE
FMA f72 = ALPHA, f72, f76
nop __LINE__
}
{ .mfb
LDFD f126 = [C16], SIZE
FMA f74 = ALPHA, f74, f78
nop __LINE__
}
;;
{ .mfb
LDFD f125 = [C8], -1 * SIZE
FMA f73 = ALPHA, f73, f77
nop __LINE__
}
{ .mfb
LDFD f127 = [C16], -1 * SIZE
FMA f75 = ALPHA, f75, f79
nop __LINE__
}
;;
{ .mfb
STFD [C1 ] = f64, SIZE
FMA f80 = ALPHA, f80, f84
nop __LINE__
}
{ .mfb
STFD [C9 ] = f66, SIZE
FMA f82 = ALPHA, f82, f86
nop __LINE__
}
;;
{ .mfb
STFD [C1 ] = f65, 3 * SIZE
FMA f81 = ALPHA, f81, f85
nop __LINE__
}
{ .mfb
STFD [C9 ] = f67, 3 * SIZE
FMA f83 = ALPHA, f83, f87
nop __LINE__
}
;;
{ .mfb
STFD [C2 ] = f72, SIZE
FMA f88 = ALPHA, f88, f92
nop __LINE__
}
{ .mfb
STFD [C10] = f74, SIZE
FMA f90 = ALPHA, f90, f94
nop __LINE__
}
;;
{ .mfb
STFD [C2 ] = f73, 3 * SIZE
FMA f89 = ALPHA, f89, f93
nop __LINE__
}
{ .mfb
STFD [C10] = f75, 3 * SIZE
FMA f91 = ALPHA, f91, f95
nop __LINE__
}
;;
{ .mfb
STFD [C3 ] = f80, SIZE
FMA f96 = ALPHA, f96, f100
nop __LINE__
}
{ .mfb
STFD [C11] = f82, SIZE
FMA f98 = ALPHA, f98, f102
nop __LINE__
}
;;
{ .mfb
STFD [C3 ] = f81, 3 * SIZE
FMA f97 = ALPHA, f97, f101
nop __LINE__
}
{ .mfb
STFD [C11] = f83, 3 * SIZE
FMA f99 = ALPHA, f99, f103
nop __LINE__
}
;;
{ .mfb
STFD [C4 ] = f88, SIZE
FMA f104 = ALPHA, f104, f108
nop __LINE__
}
{ .mfb
STFD [C12] = f90, SIZE
FMA f106 = ALPHA, f106, f110
nop __LINE__
}
;;
{ .mfb
STFD [C4 ] = f89, 3 * SIZE
FMA f105 = ALPHA, f105, f109
nop __LINE__
}
{ .mfb
STFD [C12] = f91, 3 * SIZE
FMA f107 = ALPHA, f107, f111
nop __LINE__
}
;;
{ .mfb
STFD [C5 ] = f96, SIZE
FMA f112 = ALPHA, f112, f116
nop __LINE__
}
{ .mfb
STFD [C13] = f98, SIZE
FMA f114 = ALPHA, f114, f118
nop __LINE__
}
;;
{ .mfb
STFD [C5 ] = f97, 3 * SIZE
FMA f113 = ALPHA, f113, f117
nop __LINE__
}
{ .mfb
STFD [C13] = f99, 3 * SIZE
FMA f115 = ALPHA, f115, f119
nop __LINE__
}
;;
{ .mfb
STFD [C6 ] = f104, SIZE
FMA f120 = ALPHA, f120, f124
nop __LINE__
}
{ .mfb
STFD [C14] = f106, SIZE
FMA f122 = ALPHA, f122, f126
nop __LINE__
}
;;
{ .mfb
STFD [C6 ] = f105, 3 * SIZE
FMA f121 = ALPHA, f121, f125
nop __LINE__
}
{ .mfb
STFD [C14] = f107, 3 * SIZE
FMA f123 = ALPHA, f123, f127
nop __LINE__
}
;;
{ .mfb
STFD [C7 ] = f112, SIZE
mov f64 = f0
nop __LINE__
}
{ .mfb
STFD [C15] = f114, SIZE
mov f72 = f0
nop __LINE__
}
;;
{ .mfb
STFD [C7 ] = f113, 3 * SIZE
mov f80 = f0
nop __LINE__
}
{ .mfb
STFD [C15] = f115, 3 * SIZE
mov f88 = f0
nop __LINE__
}
;;
{ .mfb
STFD [C8 ] = f120, SIZE
mov f96 = f0
nop __LINE__
}
{ .mfb
STFD [C16] = f122, SIZE
mov f104 = f0
nop __LINE__
}
;;
{ .mfb
STFD [C8 ] = f121, 3 * SIZE
mov f112 = f0
nop __LINE__
}
{ .mfb
STFD [C16] = f123, 3 * SIZE
mov f120 = f0
nop __LINE__
}
;;
#else
{ .mfb
FMPY f64 = ALPHA, f64
nop __LINE__
}
{ .mfb
FMPY f66 = ALPHA, f66
nop __LINE__
}
;;
{ .mfb
FMPY f65 = ALPHA, f65
nop __LINE__
}
{ .mfb
FMPY f67 = ALPHA, f67
nop __LINE__
}
;;
{ .mfb
FMPY f72 = ALPHA, f72
nop __LINE__
}
{ .mfb
FMPY f74 = ALPHA, f74
nop __LINE__
}
;;
{ .mfb
FMPY f73 = ALPHA, f73
nop __LINE__
}
{ .mfb
FMPY f75 = ALPHA, f75
nop __LINE__
}
;;
{ .mfb
STFD [C1 ] = f64, SIZE
FMPY f80 = ALPHA, f80
nop __LINE__
}
{ .mfb
STFD [C9 ] = f66, SIZE
FMPY f82 = ALPHA, f82
nop __LINE__
}
;;
{ .mfb
STFD [C1 ] = f65, 3 * SIZE
FMPY f81 = ALPHA, f81
nop __LINE__
}
{ .mfb
STFD [C9 ] = f67, 3 * SIZE
FMPY f83 = ALPHA, f83
nop __LINE__
}
;;
{ .mfb
STFD [C2 ] = f72, SIZE
FMPY f88 = ALPHA, f88
nop __LINE__
}
{ .mfb
STFD [C10] = f74, SIZE
FMPY f90 = ALPHA, f90
nop __LINE__
}
;;
{ .mfb
STFD [C2 ] = f73, 3 * SIZE
FMPY f89 = ALPHA, f89
nop __LINE__
}
{ .mfb
STFD [C10] = f75, 3 * SIZE
FMPY f91 = ALPHA, f91
nop __LINE__
}
;;
{ .mfb
STFD [C3 ] = f80, SIZE
FMPY f96 = ALPHA, f96
nop __LINE__
}
{ .mfb
STFD [C11] = f82, SIZE
FMPY f98 = ALPHA, f98
nop __LINE__
}
;;
{ .mfb
STFD [C3 ] = f81, 3 * SIZE
FMPY f97 = ALPHA, f97
nop __LINE__
}
{ .mfb
STFD [C11] = f83, 3 * SIZE
FMPY f99 = ALPHA, f99
nop __LINE__
}
;;
{ .mfb
STFD [C4 ] = f88, SIZE
FMPY f104 = ALPHA, f104
nop __LINE__
}
{ .mfb
STFD [C12] = f90, SIZE
FMPY f106 = ALPHA, f106
nop __LINE__
}
;;
{ .mfb
STFD [C4 ] = f89, 3 * SIZE
FMPY f105 = ALPHA, f105
nop __LINE__
}
{ .mfb
STFD [C12] = f91, 3 * SIZE
FMPY f107 = ALPHA, f107
nop __LINE__
}
;;
{ .mfb
STFD [C5 ] = f96, SIZE
FMPY f112 = ALPHA, f112
nop __LINE__
}
{ .mfb
STFD [C13] = f98, SIZE
FMPY f114 = ALPHA, f114
nop __LINE__
}
;;
{ .mfb
STFD [C5 ] = f97, 3 * SIZE
FMPY f113 = ALPHA, f113
nop __LINE__
}
{ .mfb
STFD [C13] = f99, 3 * SIZE
FMPY f115 = ALPHA, f115
nop __LINE__
}
;;
{ .mfi
STFD [C6 ] = f104, SIZE
FMPY f120 = ALPHA, f120
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
sub L = K, KK
#else
nop __LINE__
#endif
}
{ .mfb
STFD [C14] = f106, SIZE
FMPY f122 = ALPHA, f122
nop __LINE__
}
;;
{ .mfi
STFD [C6 ] = f105, 3 * SIZE
FMPY f121 = ALPHA, f121
#if defined(TRMMKERNEL) && (defined(LEFT) && defined(TRANSA))
adds L = -4, L
#else
nop __LINE__
#endif
}
{ .mfi
STFD [C14] = f107, 3 * SIZE
FMPY f123 = ALPHA, f123
#if defined(TRMMKERNEL) && (!defined(LEFT) && !defined(TRANSA))
adds L = -8, L
#else
nop __LINE__
#endif
}
;;
{ .mfi
STFD [C7 ] = f112, SIZE
mov f64 = f0
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd KK8 = L, BASE_SHIFT, r0
#else
nop __LINE__
#endif
}
{ .mfb
STFD [C15] = f114, SIZE
mov f72 = f0
nop __LINE__
}
;;
{ .mfi
STFD [C7 ] = f113, 3 * SIZE
mov f80 = f0
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd AOFFSET = KK8, 2, AOFFSET
#else
nop __LINE__
#endif
}
{ .mfi
STFD [C15] = f115, 3 * SIZE
mov f88 = f0
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd BOFFSET = KK8, 3, BOFFSET
#else
nop __LINE__
#endif
}
;;
{ .mfi
STFD [C8 ] = f120, SIZE
mov f96 = f0
#if defined(TRMMKERNEL) && defined(LEFT)
adds KK = 4, KK
#else
nop __LINE__
#endif
}
{ .mfb
STFD [C16] = f122, SIZE
mov f104 = f0
nop __LINE__
}
;;
{ .mfi
STFD [C8 ] = f121, 3 * SIZE
mov f112 = f0
#ifdef TRMMKERNEL
shladd KK8 = KK, BASE_SHIFT, r0
#else
nop __LINE__
#endif
}
{ .mfb
STFD [C16] = f123, 3 * SIZE
mov f120 = f0
nop __LINE__
}
;;
#endif
.align 32
.L030:
{ .mib
#ifndef TRMMKERNEL
nop __LINE__
#else
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub L = K, KK
#elif defined(LEFT)
adds L = 2, KK
#else
adds L = 8, KK
#endif
#endif
tbit.z p6, p7 = M, 1
(p6) br.cond.dptk .L040
}
;;
#if !defined(TRMMKERNEL) || \
defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
{ .mfi
LDFPD f48, f49 = [B]
mov f65 = f0
nop __LINE__
}
{ .mfi
adds BOFFSET = 2 * SIZE, B
mov f73 = f0
#ifndef TRMMKERNEL
adds L = 1, K
#else
adds L = 1, L
#endif
}
#else
{ .mmf
shladd BOFFSET = KK8, 3, B
shladd AOFFSET = KK8, 1, AOFFSET
mov f65 = f0
}
;;
{ .mfi
LDFPD f48, f49 = [BOFFSET], 2 * SIZE
mov f73 = f0
#ifndef TRMMKERNEL
adds L = 1, K
#else
adds L = 1, L
#endif
}
#endif
;;
{ .mfi
LDFPD f50, f51 = [BOFFSET], 2 * SIZE
mov f81 = f0
tbit.z p12, p0 = L, 0
}
{ .mfi
(p7) LDFPD f32, f33 = [AOFFSET], 2 * SIZE
mov f89 = f0
shr L = L, 1
}
;;
{ .mfi
LDFPD f52, f53 = [BOFFSET], 2 * SIZE
mov f97 = f0
adds L = -1, L
}
{ .mfi
nop __LINE__
mov f105 = f0
adds PREA = (PREFETCHSIZE + 0) * SIZE, AOFFSET
}
;;
{ .mfi
adds PREB = (PREFETCHSIZE + 0) * SIZE, BOFFSET
mov f113 = f0
mov ar.lc = L
}
{ .mfi
LDFPD f54, f55 = [BOFFSET], 2 * SIZE
mov f121 = f0
cmp.eq p3, p0 = r0, r0
}
;;
.align 32
.L032:
{ .mfb
lfetch.nt1 [PREA], 4 * SIZE
FMA f64 = f32, f48, f64 // A1 * B1
nop __LINE__
}
{ .mfi
nop __LINE__
FMA f72 = f32, f49, f72 // A1 * B2
(p12) cmp.ne p3, p0 = 0, L
}
;;
{ .mfi
lfetch.nt1 [PREB], 16 * SIZE
FMA f80 = f32, f50, f80 // A1 * B3
cmp.ne p4, p5 = 0, L
}
{ .mfb
nop __LINE__
FMA f88 = f32, f51, f88 // A1 * B4
nop __LINE__
}
;;
{ .mfb
(p3) LDFPD f56, f57 = [BOFFSET], 2 * SIZE
FMA f96 = f32, f52, f96 // A1 * B5
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f104 = f32, f53, f104 // A1 * B6
nop __LINE__
}
;;
{ .mfb
(p3) LDFPD f40, f41 = [AOFFSET], 2 * SIZE
FMA f112 = f32, f54, f112 // A1 * B7
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f120 = f32, f55, f120 // A1 * B8
nop __LINE__
}
;;
{ .mfb
(p3) LDFPD f58, f59 = [BOFFSET], 2 * SIZE
FMA f65 = f33, f48, f65 // A2 * B1
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f73 = f33, f49, f73 // A2 * B2
nop __LINE__
}
;;
{ .mfb
(p3) LDFPD f60, f61 = [BOFFSET], 2 * SIZE
FMA f81 = f33, f50, f81 // A2 * B3
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f89 = f33, f51, f89 // A2 * B4
nop __LINE__
}
;;
{ .mfb
(p3) LDFPD f62, f63 = [BOFFSET], 2 * SIZE
FMA f97 = f33, f52, f97 // A2 * B5
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f105 = f33, f53, f105 // A2 * B6
nop __LINE__
}
;;
{ .mfb
nop __LINE__
FMA f113 = f33, f54, f113 // A2 * B7
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f121 = f33, f55, f121 // A2 * B8
nop __LINE__
}
;;
{ .mfb
(p4) LDFPD f32, f33 = [AOFFSET], 2 * SIZE
(p3) FMA f64 = f40, f56, f64 // A1 * B1
nop __LINE__
}
{ .mfb
(p4) LDFPD f48, f49 = [BOFFSET], 2 * SIZE
(p3) FMA f72 = f40, f57, f72 // A1 * B2
nop __LINE__
}
;;
{ .mfb
(p4) LDFPD f50, f51 = [BOFFSET], 2 * SIZE
(p3) FMA f80 = f40, f58, f80 // A1 * B3
nop __LINE__
}
{ .mfb
nop __LINE__
(p3) FMA f88 = f40, f59, f88 // A1 * B4
nop __LINE__
}
;;
{ .mfb
(p4) LDFPD f52, f53 = [BOFFSET], 2 * SIZE
(p3) FMA f96 = f40, f60, f96 // A1 * B5
nop __LINE__
}
{ .mfb
nop __LINE__
(p3) FMA f104 = f40, f61, f104 // A1 * B6
nop __LINE__
}
;;
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f68 = [C1], SIZE
#else
nop __LINE__
#endif
(p3) FMA f112 = f40, f62, f112 // A1 * B7
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f76 = [C2], SIZE
#else
nop __LINE__
#endif
(p3) FMA f120 = f40, f63, f120 // A1 * B8
nop __LINE__
}
;;
{ .mfb
(p4) LDFPD f54, f55 = [BOFFSET], 2 * SIZE
(p3) FMA f65 = f41, f56, f65 // A2 * B1
nop __LINE__
}
{ .mfb
(p3) FMA f73 = f41, f57, f73 // A2 * B2
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f69 = [C1], -1 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f81 = f41, f58, f81 // A2 * B3
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f77 = [C2], -1 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f89 = f41, f59, f89 // A2 * B4
nop __LINE__
}
;;
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f84 = [C3], SIZE
#else
nop __LINE__
#endif
(p3) FMA f97 = f41, f60, f97 // A2 * B5
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f92 = [C4], SIZE
#else
nop __LINE__
#endif
(p3) FMA f105 = f41, f61, f105 // A2 * B6
nop __LINE__
}
;;
{ .mfi
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f85 = [C3], -1 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f113 = f41, f62, f113 // A2 * B7
adds L = -1, L
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f93 = [C4], -1 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f121 = f41, f63, f121 // A2 * B8
br.cloop.sptk.few .L032
}
;;
.L038:
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
{ .mfb
LDFD f100 = [C5], SIZE
FMA f64 = ALPHA, f64, f68
nop __LINE__
}
{ .mfb
LDFD f108 = [C6], SIZE
FMA f65 = ALPHA, f65, f69
nop __LINE__
}
;;
{ .mfb
LDFD f101 = [C5], -1 * SIZE
FMA f72 = ALPHA, f72, f76
nop __LINE__
}
{ .mfb
LDFD f109 = [C6], -1 * SIZE
FMA f73 = ALPHA, f73, f77
nop __LINE__
}
;;
{ .mfb
LDFD f116 = [C7], SIZE
FMA f80 = ALPHA, f80, f84
nop __LINE__
}
{ .mfb
LDFD f124 = [C8], SIZE
FMA f81 = ALPHA, f81, f85
nop __LINE__
}
;;
{ .mfb
LDFD f117 = [C7], -1 * SIZE
FMA f88 = ALPHA, f88, f92
nop __LINE__
}
{ .mfb
LDFD f125 = [C8], -1 * SIZE
FMA f89 = ALPHA, f89, f93
nop __LINE__
}
;;
{ .mfb
STFD [C1 ] = f64, SIZE
FMA f96 = ALPHA, f96, f100
nop __LINE__
}
{ .mfb
STFD [C2 ] = f72, SIZE
FMA f104 = ALPHA, f104, f108
nop __LINE__
}
;;
{ .mfb
STFD [C1 ] = f65, SIZE
FMA f97 = ALPHA, f97, f101
nop __LINE__
}
{ .mfb
STFD [C2 ] = f73, SIZE
FMA f105 = ALPHA, f105, f109
nop __LINE__
}
;;
{ .mfb
STFD [C3 ] = f80, SIZE
FMA f112 = ALPHA, f112, f116
nop __LINE__
}
{ .mfb
STFD [C4 ] = f88, SIZE
FMA f120 = ALPHA, f120, f124
nop __LINE__
}
;;
{ .mfb
STFD [C3 ] = f81, SIZE
FMA f113 = ALPHA, f113, f117
nop __LINE__
}
{ .mfb
STFD [C4 ] = f89, SIZE
FMA f121 = ALPHA, f121, f125
nop __LINE__
}
;;
{ .mfb
STFD [C5 ] = f96, SIZE
mov f64 = f0
nop __LINE__
}
{ .mfb
STFD [C6 ] = f104, SIZE
mov f72 = f0
nop __LINE__
}
;;
{ .mfb
STFD [C5 ] = f97, SIZE
mov f80 = f0
nop __LINE__
}
{ .mfb
STFD [C6 ] = f105, SIZE
mov f88 = f0
nop __LINE__
}
;;
{ .mfb
STFD [C7 ] = f112, SIZE
mov f96 = f0
nop __LINE__
}
{ .mfb
STFD [C8 ] = f120, SIZE
mov f104 = f0
nop __LINE__
}
;;
{ .mfb
STFD [C7 ] = f113, SIZE
mov f112 = f0
nop __LINE__
}
{ .mfb
STFD [C8 ] = f121, SIZE
mov f120 = f0
nop __LINE__
}
;;
#else
{ .mfb
nop __LINE__
FMPY f64 = ALPHA, f64
nop __LINE__
}
{ .mfb
nop __LINE__
FMPY f65 = ALPHA, f65
nop __LINE__
}
;;
{ .mfb
nop __LINE__
FMPY f72 = ALPHA, f72
nop __LINE__
}
{ .mfb
nop __LINE__
FMPY f73 = ALPHA, f73
nop __LINE__
}
;;
{ .mfb
nop __LINE__
FMPY f80 = ALPHA, f80
nop __LINE__
}
{ .mfb
nop __LINE__
FMPY f81 = ALPHA, f81
nop __LINE__
}
;;
{ .mfb
nop __LINE__
FMPY f88 = ALPHA, f88
nop __LINE__
}
{ .mfb
nop __LINE__
FMPY f89 = ALPHA, f89
nop __LINE__
}
;;
{ .mfb
STFD [C1 ] = f64, SIZE
FMPY f96 = ALPHA, f96
nop __LINE__
}
{ .mfb
STFD [C2 ] = f72, SIZE
FMPY f104 = ALPHA, f104
nop __LINE__
}
;;
{ .mfb
STFD [C1 ] = f65, SIZE
FMPY f97 = ALPHA, f97
nop __LINE__
}
{ .mfb
STFD [C2 ] = f73, SIZE
FMPY f105 = ALPHA, f105
nop __LINE__
}
;;
{ .mfi
STFD [C3 ] = f80, SIZE
FMPY f112 = ALPHA, f112
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
sub L = K, KK
#else
nop __LINE__
#endif
}
{ .mfb
STFD [C4 ] = f88, SIZE
FMPY f120 = ALPHA, f120
nop __LINE__
}
;;
{ .mfi
STFD [C3 ] = f81, SIZE
FMPY f113 = ALPHA, f113
#if defined(TRMMKERNEL) && (defined(LEFT) && defined(TRANSA))
adds L = -2, L
#else
nop __LINE__
#endif
}
{ .mfi
STFD [C4 ] = f89, SIZE
FMPY f121 = ALPHA, f121
#if defined(TRMMKERNEL) && (!defined(LEFT) && !defined(TRANSA))
adds L = -8, L
#else
nop __LINE__
#endif
}
;;
{ .mfi
STFD [C5 ] = f96, SIZE
mov f64 = f0
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd KK8 = L, BASE_SHIFT, r0
#else
nop __LINE__
#endif
}
{ .mfb
STFD [C6 ] = f104, SIZE
mov f72 = f0
nop __LINE__
}
;;
{ .mfi
STFD [C5 ] = f97, SIZE
mov f80 = f0
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd AOFFSET = KK8, 1, AOFFSET
#else
nop __LINE__
#endif
}
{ .mfi
STFD [C6 ] = f105, SIZE
mov f88 = f0
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd BOFFSET = KK8, 3, BOFFSET
#else
nop __LINE__
#endif
}
;;
{ .mfi
STFD [C7 ] = f112, SIZE
mov f96 = f0
#if defined(TRMMKERNEL) && defined(LEFT)
adds KK = 2, KK
#else
nop __LINE__
#endif
}
{ .mfb
STFD [C8 ] = f120, SIZE
mov f104 = f0
nop __LINE__
}
;;
{ .mfi
STFD [C7 ] = f113, SIZE
mov f112 = f0
#ifdef TRMMKERNEL
shladd KK8 = KK, BASE_SHIFT, r0
#else
nop __LINE__
#endif
}
{ .mfb
STFD [C8 ] = f121, SIZE
mov f120 = f0
nop __LINE__
}
;;
#endif
.align 32
.L040:
{ .mib
#ifndef TRMMKERNEL
nop __LINE__
#else
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub L = K, KK
#elif defined(LEFT)
adds L = 1, KK
#else
adds L = 8, KK
#endif
#endif
tbit.z p6, p7 = M, 0
(p6) br.cond.dptk .L049
}
;;
#if !defined(TRMMKERNEL) || \
defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
{ .mmi
LDFPD f48, f49 = [B]
adds BOFFSET = 2 * SIZE, B
#ifndef TRMMKERNEL
adds L = 1, K
#else
adds L = 1, L
#endif
}
#else
{ .mmi
shladd BOFFSET = KK8, 3, B
add AOFFSET = KK8, AOFFSET
nop __LINE__
}
;;
{ .mmi
LDFPD f48, f49 = [BOFFSET], 2 * SIZE
nop __LINE__
#ifndef TRMMKERNEL
adds L = 1, K
#else
adds L = 1, L
#endif
}
#endif
;;
{ .mii
LDFPD f50, f51 = [BOFFSET], 2 * SIZE
tbit.z p12, p0 = L, 0
shr L = L, 1
}
;;
{ .mmi
LDFPD f52, f53 = [BOFFSET], 2 * SIZE
LDFD f32 = [AOFFSET], 1 * SIZE
adds L = -1, L
}
;;
{ .mmi
adds PREB = (PREFETCHSIZE + 0) * SIZE, BOFFSET
cmp.eq p3, p0 = r0, r0
mov ar.lc = L
}
{ .mmi
LDFPD f54, f55 = [BOFFSET], 2 * SIZE
adds PREA = (PREFETCHSIZE + 0) * SIZE, AOFFSET
nop __LINE__
}
;;
.align 32
.L042:
{ .mfb
lfetch.nt1 [PREB], 16 * SIZE
FMA f64 = f32, f48, f64 // A1 * B1
nop __LINE__
}
{ .mfb
(p12) cmp.ne p3, p0 = 0, L
FMA f72 = f32, f49, f72 // A1 * B2
nop __LINE__
}
;;
{ .mfi
(p3) LDFD f40 = [AOFFSET], 1 * SIZE
FMA f80 = f32, f50, f80 // A1 * B3
cmp.ne p4, p5 = 0, L
}
{ .mfb
(p3) LDFPD f56, f57 = [BOFFSET], 2 * SIZE
FMA f88 = f32, f51, f88 // A1 * B4
nop __LINE__
}
;;
{ .mfb
(p3) LDFPD f58, f59 = [BOFFSET], 2 * SIZE
FMA f96 = f32, f52, f96 // A1 * B5
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f68 = [C1]
#else
nop __LINE__
#endif
FMA f104 = f32, f53, f104 // A1 * B6
nop __LINE__
}
;;
{ .mfb
(p3) LDFPD f60, f61 = [BOFFSET], 2 * SIZE
FMA f112 = f32, f54, f112 // A1 * B7
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f76 = [C2]
#else
nop __LINE__
#endif
FMA f120 = f32, f55, f120 // A1 * B8
nop __LINE__
}
;;
{ .mfb
(p4) LDFD f32 = [AOFFSET], 1 * SIZE
(p3) FMA f64 = f40, f56, f64 // A1 * B1
nop __LINE__
}
{ .mfb
(p3) LDFPD f62, f63 = [BOFFSET], 2 * SIZE
(p3) FMA f72 = f40, f57, f72 // A1 * B2
nop __LINE__
}
;;
{ .mfb
(p4) LDFPD f48, f49 = [BOFFSET], 2 * SIZE
(p3) FMA f80 = f40, f58, f80 // A1 * B3
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f84 = [C3]
#else
nop __LINE__
#endif
(p3) FMA f88 = f40, f59, f88 // A1 * B4
nop __LINE__
}
;;
{ .mfb
(p4) LDFPD f50, f51 = [BOFFSET], 2 * SIZE
(p3) FMA f96 = f40, f60, f96 // A1 * B5
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f92 = [C4]
#else
nop __LINE__
#endif
(p3) FMA f104 = f40, f61, f104 // A1 * B6
nop __LINE__
}
;;
{ .mfi
(p4) LDFPD f52, f53 = [BOFFSET], 2 * SIZE
(p3) FMA f112 = f40, f62, f112 // A1 * B7
adds L = -1, L
}
{ .mmb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f100 = [C5]
(p5) LDFD f108 = [C6]
#else
nop __LINE__
nop __LINE__
#endif
nop __LINE__
}
;;
{ .mfb
(p4) LDFPD f54, f55 = [BOFFSET], 2 * SIZE
(p3) FMA f120 = f40, f63, f120 // A1 * B8
nop __LINE__
}
{ .mmb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f116 = [C7]
(p5) LDFD f124 = [C8]
#else
nop __LINE__
nop __LINE__
#endif
br.cloop.sptk.few .L042
}
;;
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
FMA f64 = ALPHA, f64, f68
FMA f72 = ALPHA, f72, f76
FMA f80 = ALPHA, f80, f84
FMA f88 = ALPHA, f88, f92
FMA f96 = ALPHA, f96, f100
FMA f104 = ALPHA, f104, f108
FMA f112 = ALPHA, f112, f116
FMA f120 = ALPHA, f120, f124
;;
STFD [C1 ] = f64, SIZE
mov f64 = f0
STFD [C2 ] = f72, SIZE
mov f72 = f0
;;
STFD [C3 ] = f80, SIZE
mov f80 = f0
STFD [C4 ] = f88, SIZE
mov f88 = f0
;;
STFD [C5 ] = f96, SIZE
mov f96 = f0
STFD [C6 ] = f104, SIZE
mov f104 = f0
;;
STFD [C7 ] = f112, SIZE
mov f112 = f0
STFD [C8 ] = f120, SIZE
mov f120 = f0
;;
#else
FMPY f64 = ALPHA, f64
FMPY f72 = ALPHA, f72
FMPY f80 = ALPHA, f80
FMPY f88 = ALPHA, f88
{ .mfi
FMPY f96 = ALPHA, f96
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
sub L = K, KK
#else
nop __LINE__
#endif
}
{ .mfi
nop __LINE__
FMPY f104 = ALPHA, f104
nop __LINE__
}
;;
{ .mfi
nop __LINE__
FMPY f112 = ALPHA, f112
#if defined(TRMMKERNEL) && (defined(LEFT) && defined(TRANSA))
adds L = -1, L
#else
nop __LINE__
#endif
}
{ .mfi
nop __LINE__
FMPY f120 = ALPHA, f120
#if defined(TRMMKERNEL) && (!defined(LEFT) && !defined(TRANSA))
adds L = -8, L
#else
nop __LINE__
#endif
}
;;
{ .mfi
STFD [C1 ] = f64, SIZE
mov f64 = f0
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd KK8 = L, BASE_SHIFT, r0
#else
nop __LINE__
#endif
}
{ .mfi
STFD [C2 ] = f72, SIZE
mov f72 = f0
nop __LINE__
}
;;
{ .mfi
STFD [C3 ] = f80, SIZE
mov f80 = f0
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
add AOFFSET = KK8, AOFFSET
#else
nop __LINE__
#endif
}
{ .mfi
STFD [C4 ] = f88, SIZE
mov f88 = f0
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd BOFFSET = KK8, 3, BOFFSET
#else
nop __LINE__
#endif
}
;;
{ .mfi
STFD [C5 ] = f96, SIZE
mov f96 = f0
#if defined(TRMMKERNEL) && defined(LEFT)
adds KK = 1, KK
#else
nop __LINE__
#endif
}
{ .mfi
STFD [C6 ] = f104, SIZE
mov f104 = f0
nop __LINE__
}
;;
{ .mfi
STFD [C7 ] = f112, SIZE
mov f112 = f0
#ifdef TRMMKERNEL
shladd KK8 = KK, BASE_SHIFT, r0
#else
nop __LINE__
#endif
}
{ .mfi
STFD [C8 ] = f120, SIZE
mov f120 = f0
nop __LINE__
}
;;
#endif
.align 32
#endif
.L049:
{ .mmi
mov B = BOFFSET
mov AOFFSET = A
#if defined(TRMMKERNEL) && !defined(LEFT)
adds KK = 8, KK
#else
nop __LINE__
#endif
}
;;
{ .mmb
nop __LINE__
cmp.lt p6, p0 = 0, J
(p6) br.cond.dptk .L010
}
;;
.align 32
.L050:
{ .mfi
mov C1 = C
mov f64 = f0
tbit.z p6, p0 = N, 2
}
{ .mfi
add C2 = LDC, C
mov f72 = f0
shr I = M, 3
}
;;
{ .mfi
shladd C3 = LDC, 1, C
mov f80 = f0
nop __LINE__
}
{ .mfb
mov AOFFSET = A
mov f88 = f0
(p6) br.cond.dpnt .L090
}
;;
#if 0
{ .mfi
cmp.eq p6, p7 = 0, I
mov f65 = f0
#if defined(TRMMKERNEL) && defined(LEFT)
mov KK = OFFSET
#else
nop __LINE__
#endif
}
{ .mfi
shladd C4 = LDC, 1, C2
mov f73 = f0
nop __LINE__
}
;;
{ .mfi
nop __LINE__
mov f81 = f0
#ifdef TRMMKERNEL
shladd KK8 = KK, BASE_SHIFT, r0
#else
nop __LINE__
#endif
}
{ .mfb
shladd C = LDC, 2, C
mov f89 = f0
(p6) br.cond.dpnt .L060
}
;;
.align 32
.L052:
#if !defined(TRMMKERNEL) || \
defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
{ .mfb
LDFPD f48, f49 = [B]
mov f66 = f0
nop __LINE__
}
{ .mfb
adds BOFFSET = 2 * SIZE, B
mov f74 = f0
nop __LINE__
}
;;
#else
{ .mfi
shladd BOFFSET = KK8, 2, B
mov f66 = f0
shladd AOFFSET = KK8, 3, AOFFSET
}
;;
{ .mfi
LDFPD f48, f49 = [BOFFSET], 2 * SIZE
mov f74 = f0
nop __LINE__
}
;;
#endif
;;
{ .mfi
LDFPD f32, f33 = [AOFFSET], 2 * SIZE
mov f82 = f0
#ifndef TRMMKERNEL
nop __LINE__
#else
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub L = K, KK
#elif defined(LEFT)
adds L = 8, KK
#else
adds L = 4, KK
#endif
#endif
}
{ .mfi
setf.d f84 = r0
mov f90 = f0
nop __LINE__
}
;;
{ .mfi
LDFPD f50, f51 = [BOFFSET], 2 * SIZE
mov f67 = f0
adds PREC = CPREFETCHSIZE * SIZE, C1
}
{ .mfi
LDFPD f34, f35 = [AOFFSET], 2 * SIZE
mov f75 = f0
#ifndef TRMMKERNEL
adds L = 1, K
#else
adds L = 1, L
#endif
}
;;
{ .mfi
LDFPD f36, f37 = [AOFFSET], 2 * SIZE
mov f83 = f0
tbit.z p12, p0 = L, 0
}
{ .mfi
setf.d f91 = r0
mov f68 = f0
adds PREB = (PREFETCHSIZE + 0) * SIZE, BOFFSET
}
;;
{ .mfi
CPREFETCH [PREC], LDC
mov f76 = f0
adds PREA = (PREFETCHSIZE + 8) * SIZE, AOFFSET
}
{ .mfi
LDFPD f38, f39 = [AOFFSET], 2 * SIZE
mov f92 = f0
cmp.eq p3, p0 = r0, r0
}
;;
{ .mfi
CPREFETCH [PREC], LDC
mov f69 = f0
shr L = L, 1
}
{ .mmf
setf.d f77 = r0
setf.d f85 = r0
mov f93 = f0
}
;;
{ .mfi
CPREFETCH [PREC], LDC
mov f70 = f0
adds L = -1, L
}
{ .mmf
setf.d f78 = r0
setf.d f86 = r0
mov f94 = f0
}
;;
{ .mfi
CPREFETCH [PREC]
mov f71 = f0
mov ar.lc = L
}
{ .mmf
setf.d f79 = r0
setf.d f87 = r0
mov f95 = f0
}
;;
.align 32
.L053:
{ .mfb
lfetch.nt1 [PREA], 16 * SIZE
FMA f64 = f32, f48, f64 // A1 * B1
nop __LINE__
}
{ .mfi
nop __LINE__
FMA f72 = f32, f49, f72 // A1 * B2
(p12) cmp.ne p3, p0 = 0, L
}
;;
{ .mfi
lfetch.nt1 [PREB], 8 * SIZE
FMA f80 = f32, f50, f80 // A1 * B3
cmp.ne p4, p5 = 0, L
}
{ .mfi
nop __LINE__
FMA f88 = f32, f51, f88 // A1 * B4
adds C9 = 4 * SIZE, C1
}
;;
{ .mfi
(p3) LDFPD f40, f41 = [AOFFSET], 2 * SIZE
FMA f65 = f33, f48, f65 // A2 * B1
adds C10 = 4 * SIZE, C2
}
{ .mfi
nop __LINE__
FMA f73 = f33, f49, f73 // A2 * B2
adds C11 = 4 * SIZE, C3
}
;;
{ .mfi
(p3) LDFPD f56, f57 = [BOFFSET], 2 * SIZE
FMA f81 = f33, f50, f81 // A2 * B3
adds C12 = 4 * SIZE, C4
}
{ .mfb
nop __LINE__
FMA f89 = f33, f51, f89 // A2 * B4
nop __LINE__
}
;;
{ .mfb
(p3) LDFPD f58, f59 = [BOFFSET], 2 * SIZE
FMA f66 = f34, f48, f66 // A3 * B1
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f74 = f34, f49, f74 // A3 * B2
nop __LINE__
}
;;
{ .mfb
(p3) LDFPD f42, f43 = [AOFFSET], 2 * SIZE
FMA f82 = f34, f50, f82 // A3 * B3
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f90 = f34, f51, f90 // A3 * B4
nop __LINE__
}
;;
{ .mfb
(p3) LDFPD f44, f45 = [AOFFSET], 2 * SIZE
FMA f67 = f35, f48, f67 // A4 * B1
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f75 = f35, f49, f75 // A4 * B2
nop __LINE__
}
;;
{ .mfb
(p3) LDFPD f46, f47 = [AOFFSET], 2 * SIZE
FMA f83 = f35, f50, f83 // A4 * B3
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f91 = f35, f51, f91 // A4 * B4
nop __LINE__
}
;;
{ .mfb
nop __LINE__
FMA f68 = f36, f48, f68 // A5 * B1
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f76 = f36, f49, f76 // A5 * B2
nop __LINE__
}
;;
{ .mfb
nop __LINE__
FMA f84 = f36, f50, f84 // A5 * B3
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f92 = f36, f51, f92 // A5 * B4
nop __LINE__
}
;;
{ .mfb
nop __LINE__
FMA f69 = f37, f48, f69 // A6 * B1
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f77 = f37, f49, f77 // A6 * B2
nop __LINE__
}
;;
{ .mfb
nop __LINE__
FMA f85 = f37, f50, f85 // A6 * B3
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f93 = f37, f51, f93 // A6 * B4
nop __LINE__
}
;;
{ .mfb
nop __LINE__
FMA f70 = f38, f48, f70 // A7 * B1
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f78 = f38, f49, f78 // A7 * B2
nop __LINE__
}
;;
{ .mfb
nop __LINE__
FMA f86 = f38, f50, f86 // A7 * B3
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f94 = f38, f51, f94 // A7 * B4
nop __LINE__
}
;;
{ .mfb
(p4) LDFPD f32, f33 = [AOFFSET], 2 * SIZE
FMA f71 = f39, f48, f71 // A8 * B1
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f79 = f39, f49, f79 // A8 * B2
nop __LINE__
}
;;
{ .mfb
(p4) LDFPD f48, f49 = [BOFFSET], 2 * SIZE
FMA f87 = f39, f50, f87 // A8 * B3
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f95 = f39, f51, f95 // A8 * B4
nop __LINE__
}
;;
{ .mfb
(p4) LDFPD f50, f51 = [BOFFSET], 2 * SIZE
(p3) FMA f64 = f40, f56, f64 // A1 * B1
nop __LINE__
}
{ .mfb
nop __LINE__
(p3) FMA f72 = f40, f57, f72 // A1 * B2
nop __LINE__
}
;;
{ .mfb
(p4) LDFPD f34, f35 = [AOFFSET], 2 * SIZE
(p3) FMA f80 = f40, f58, f80 // A1 * B3
nop __LINE__
}
{ .mfb
nop __LINE__
(p3) FMA f88 = f40, f59, f88 // A1 * B4
nop __LINE__
}
;;
{ .mfb
(p4) LDFPD f36, f37 = [AOFFSET], 2 * SIZE
(p3) FMA f65 = f41, f56, f65 // A2 * B1
nop __LINE__
}
{ .mfb
nop __LINE__
(p3) FMA f73 = f41, f57, f73 // A2 * B2
nop __LINE__
}
;;
{ .mfb
(p4) LDFPD f38, f39 = [AOFFSET], 2 * SIZE
(p3) FMA f81 = f41, f58, f81 // A2 * B3
nop __LINE__
}
{ .mfb
nop __LINE__
(p3) FMA f89 = f41, f59, f89 // A2 * B4
nop __LINE__
}
;;
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f96 = [C1 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f66 = f42, f56, f66 // A3 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f97 = [C9 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f74 = f42, f57, f74 // A3 * B2
nop __LINE__
}
;;
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f98 = [C1 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f82 = f42, f58, f82 // A3 * B3
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f99 = [C9 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f90 = f42, f59, f90 // A3 * B4
nop __LINE__
}
;;
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f100 = [C1 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f67 = f43, f56, f67 // A4 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f101 = [C9 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f75 = f43, f57, f75 // A4 * B2
nop __LINE__
}
;;
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f102 = [C1 ], -3 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f83 = f43, f58, f83 // A4 * B3
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f103 = [C9 ], -3 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f91 = f43, f59, f91 // A4 * B4
nop __LINE__
}
;;
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f104 = [C2 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f68 = f44, f56, f68 // A5 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f105 = [C10], SIZE
#else
nop __LINE__
#endif
(p3) FMA f76 = f44, f57, f76 // A5 * B2
nop __LINE__
}
;;
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f106 = [C2 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f84 = f44, f58, f84 // A5 * B3
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f107 = [C10], SIZE
#else
nop __LINE__
#endif
(p3) FMA f92 = f44, f59, f92 // A5 * B4
nop __LINE__
}
;;
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f108 = [C2 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f69 = f45, f56, f69 // A6 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f109 = [C10], SIZE
#else
nop __LINE__
#endif
(p3) FMA f77 = f45, f57, f77 // A6 * B2
nop __LINE__
}
;;
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f110 = [C2 ], -3 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f85 = f45, f58, f85 // A6 * B3
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f111 = [C10], -3 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f93 = f45, f59, f93 // A6 * B4
nop __LINE__
}
;;
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f112 = [C3 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f70 = f46, f56, f70 // A7 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f113 = [C11], SIZE
#else
nop __LINE__
#endif
(p3) FMA f78 = f46, f57, f78 // A7 * B2
nop __LINE__
}
;;
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f114 = [C3 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f86 = f46, f58, f86 // A7 * B3
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f115 = [C11], SIZE
#else
nop __LINE__
#endif
(p3) FMA f94 = f46, f59, f94 // A7 * B4
nop __LINE__
}
;;
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f116 = [C3 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f71 = f47, f56, f71 // A8 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f117 = [C11], SIZE
#else
nop __LINE__
#endif
(p3) FMA f79 = f47, f57, f79 // A8 * B2
nop __LINE__
}
;;
{ .mfi
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f118 = [C3 ], -3 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f87 = f47, f58, f87 // A8 * B3
adds L = -1, L
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f119 = [C11], -3 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f95 = f47, f59, f95 // A8 * B4
br.cloop.sptk.few .L053
}
;;
.align 32
.L058:
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
{ .mfi
LDFD f120 = [C4 ], SIZE
FMA f64 = ALPHA, f64, f96
cmp.ne p6, p0 = 1, I
}
{ .mfb
LDFD f121 = [C12], SIZE
FMA f68 = ALPHA, f68, f97
nop __LINE__
}
;;
{ .mfi
LDFD f122 = [C4 ], SIZE
FMA f65 = ALPHA, f65, f98
adds I = -1, I
}
{ .mfb
LDFD f123 = [C12], SIZE
FMA f69 = ALPHA, f69, f99
nop __LINE__
}
;;
{ .mfb
LDFD f124 = [C4 ], SIZE
FMA f66 = ALPHA, f66, f100
nop __LINE__
}
{ .mfb
LDFD f125 = [C12], SIZE
FMA f70 = ALPHA, f70, f101
nop __LINE__
}
;;
{ .mfb
LDFD f126 = [C4 ], -3 * SIZE
FMA f67 = ALPHA, f67, f102
nop __LINE__
}
{ .mfb
LDFD f127 = [C12], -3 * SIZE
FMA f71 = ALPHA, f71, f103
nop __LINE__
}
;;
{ .mfb
STFD [C1 ] = f64, SIZE
FMA f72 = ALPHA, f72, f104
nop __LINE__
}
{ .mfb
STFD [C9 ] = f68, SIZE
FMA f76 = ALPHA, f76, f105
nop __LINE__
}
;;
{ .mfb
STFD [C1 ] = f65, SIZE
FMA f73 = ALPHA, f73, f106
nop __LINE__
}
{ .mfb
STFD [C9 ] = f69, SIZE
FMA f77 = ALPHA, f77, f107
nop __LINE__
}
;;
{ .mfb
STFD [C1 ] = f66, SIZE
FMA f74 = ALPHA, f74, f108
nop __LINE__
}
{ .mfb
STFD [C9 ] = f70, SIZE
FMA f78 = ALPHA, f78, f109
nop __LINE__
}
;;
{ .mfb
STFD [C1 ] = f67, 5 * SIZE
FMA f75 = ALPHA, f75, f110
nop __LINE__
}
{ .mfb
STFD [C9 ] = f71, 5 * SIZE
FMA f79 = ALPHA, f79, f111
nop __LINE__
}
;;
{ .mfb
STFD [C2 ] = f72, SIZE
FMA f80 = ALPHA, f80, f112
nop __LINE__
}
{ .mfb
STFD [C10] = f76, SIZE
FMA f84 = ALPHA, f84, f113
nop __LINE__
}
;;
{ .mfb
STFD [C2 ] = f73, SIZE
FMA f81 = ALPHA, f81, f114
nop __LINE__
}
{ .mfb
STFD [C10] = f77, SIZE
FMA f85 = ALPHA, f85, f115
nop __LINE__
}
;;
{ .mfb
STFD [C2 ] = f74, SIZE
FMA f82 = ALPHA, f82, f116
nop __LINE__
}
{ .mfb
STFD [C10] = f78, SIZE
FMA f86 = ALPHA, f86, f117
nop __LINE__
}
;;
{ .mfb
STFD [C2 ] = f75, 5 * SIZE
FMA f83 = ALPHA, f83, f118
nop __LINE__
}
{ .mfb
STFD [C10] = f79, 5 * SIZE
FMA f87 = ALPHA, f87, f119
nop __LINE__
}
;;
{ .mfb
STFD [C3 ] = f80, SIZE
FMA f88 = ALPHA, f88, f120
nop __LINE__
}
{ .mfb
STFD [C11] = f84, SIZE
FMA f92 = ALPHA, f92, f121
nop __LINE__
}
;;
{ .mfb
STFD [C3 ] = f81, SIZE
FMA f89 = ALPHA, f89, f122
nop __LINE__
}
{ .mfb
STFD [C11] = f85, SIZE
FMA f93 = ALPHA, f93, f123
nop __LINE__
}
;;
{ .mfb
STFD [C3 ] = f82, SIZE
FMA f90 = ALPHA, f90, f124
nop __LINE__
}
{ .mfb
STFD [C11] = f86, SIZE
FMA f94 = ALPHA, f94, f125
nop __LINE__
}
;;
{ .mfb
STFD [C3 ] = f83, 5 * SIZE
FMA f91 = ALPHA, f91, f126
nop __LINE__
}
{ .mfb
STFD [C11] = f87, 5 * SIZE
FMA f95 = ALPHA, f95, f127
nop __LINE__
}
;;
{ .mfb
STFD [C4 ] = f88, SIZE
mov f64 = f0
nop __LINE__
}
{ .mfb
STFD [C12] = f92, SIZE
mov f72 = f0
nop __LINE__
}
;;
{ .mfb
STFD [C4 ] = f89, SIZE
mov f80 = f0
nop __LINE__
}
{ .mfb
STFD [C12] = f93, SIZE
mov f88 = f0
nop __LINE__
}
;;
{ .mfb
STFD [C4 ] = f90, SIZE
mov f65 = f0
nop __LINE__
}
{ .mfb
STFD [C12] = f94, SIZE
mov f73 = f0
nop __LINE__
}
;;
{ .mfb
STFD [C4 ] = f91, 5 * SIZE
mov f81 = f0
nop __LINE__
}
{ .mfb
STFD [C12] = f95, 5 * SIZE
mov f89 = f0
(p6) br.cond.dptk .L052
}
;;
#else
{ .mfi
nop __LINE__
FMPY f64 = ALPHA, f64
cmp.ne p6, p0 = 1, I
}
{ .mfb
nop __LINE__
FMPY f68 = ALPHA, f68
nop __LINE__
}
;;
{ .mfi
nop __LINE__
FMPY f65 = ALPHA, f65
adds I = -1, I
}
{ .mfb
nop __LINE__
FMPY f69 = ALPHA, f69
nop __LINE__
}
;;
{ .mfb
nop __LINE__
FMPY f66 = ALPHA, f66
nop __LINE__
}
{ .mfb
nop __LINE__
FMPY f70 = ALPHA, f70
nop __LINE__
}
;;
{ .mfb
nop __LINE__
FMPY f67 = ALPHA, f67
nop __LINE__
}
{ .mfb
nop __LINE__
FMPY f71 = ALPHA, f71
nop __LINE__
}
;;
{ .mfb
STFD [C1 ] = f64, SIZE
FMPY f72 = ALPHA, f72
nop __LINE__
}
{ .mfb
STFD [C9 ] = f68, SIZE
FMPY f76 = ALPHA, f76
nop __LINE__
}
;;
{ .mfb
STFD [C1 ] = f65, SIZE
FMPY f73 = ALPHA, f73
nop __LINE__
}
{ .mfb
STFD [C9 ] = f69, SIZE
FMPY f77 = ALPHA, f77
nop __LINE__
}
;;
{ .mfb
STFD [C1 ] = f66, SIZE
FMPY f74 = ALPHA, f74
nop __LINE__
}
{ .mfb
STFD [C9 ] = f70, SIZE
FMPY f78 = ALPHA, f78
nop __LINE__
}
;;
{ .mfb
STFD [C1 ] = f67, 5 * SIZE
FMPY f75 = ALPHA, f75
nop __LINE__
}
{ .mfb
STFD [C9 ] = f71, 5 * SIZE
FMPY f79 = ALPHA, f79
nop __LINE__
}
;;
{ .mfb
STFD [C2 ] = f72, SIZE
FMPY f80 = ALPHA, f80
nop __LINE__
}
{ .mfb
STFD [C10] = f76, SIZE
FMPY f84 = ALPHA, f84
nop __LINE__
}
;;
{ .mfb
STFD [C2 ] = f73, SIZE
FMPY f81 = ALPHA, f81
nop __LINE__
}
{ .mfb
STFD [C10] = f77, SIZE
FMPY f85 = ALPHA, f85
nop __LINE__
}
;;
{ .mfb
STFD [C2 ] = f74, SIZE
FMPY f82 = ALPHA, f82
nop __LINE__
}
{ .mfb
STFD [C10] = f78, SIZE
FMPY f86 = ALPHA, f86
nop __LINE__
}
;;
{ .mfb
STFD [C2 ] = f75, 5 * SIZE
FMPY f83 = ALPHA, f83
nop __LINE__
}
{ .mfb
STFD [C10] = f79, 5 * SIZE
FMPY f87 = ALPHA, f87
nop __LINE__
}
;;
{ .mfb
STFD [C3 ] = f80, SIZE
FMPY f88 = ALPHA, f88
nop __LINE__
}
{ .mfb
STFD [C11] = f84, SIZE
FMPY f92 = ALPHA, f92
nop __LINE__
}
;;
{ .mfb
STFD [C3 ] = f81, SIZE
FMPY f89 = ALPHA, f89
nop __LINE__
}
{ .mfb
STFD [C11] = f85, SIZE
FMPY f93 = ALPHA, f93
nop __LINE__
}
;;
{ .mfi
STFD [C3 ] = f82, SIZE
FMPY f90 = ALPHA, f90
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
sub L = K, KK
#else
nop __LINE__
#endif
}
{ .mfb
STFD [C11] = f86, SIZE
FMPY f94 = ALPHA, f94
nop __LINE__
}
;;
{ .mfi
STFD [C3 ] = f83, 5 * SIZE
FMPY f91 = ALPHA, f91
#if defined(TRMMKERNEL) && (defined(LEFT) && defined(TRANSA))
adds L = -8, L
#else
nop __LINE__
#endif
}
{ .mfi
STFD [C11] = f87, 5 * SIZE
FMPY f95 = ALPHA, f95
#if defined(TRMMKERNEL) && (!defined(LEFT) && !defined(TRANSA))
adds L = -4, L
#else
nop __LINE__
#endif
}
;;
{ .mfi
STFD [C4 ] = f88, SIZE
mov f64 = f0
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd KK8 = L, BASE_SHIFT, r0
#else
nop __LINE__
#endif
}
{ .mfb
STFD [C12] = f92, SIZE
mov f72 = f0
nop __LINE__
}
;;
{ .mfi
STFD [C4 ] = f89, SIZE
mov f80 = f0
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd AOFFSET = KK8, 3, AOFFSET
#else
nop __LINE__
#endif
}
{ .mfi
STFD [C12] = f93, SIZE
mov f88 = f0
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd BOFFSET = KK8, 2, BOFFSET
#else
nop __LINE__
#endif
}
;;
{ .mfi
STFD [C4 ] = f90, SIZE
mov f65 = f0
#if defined(TRMMKERNEL) && defined(LEFT)
adds KK = 8, KK
#else
nop __LINE__
#endif
}
{ .mfb
STFD [C12] = f94, SIZE
mov f73 = f0
nop __LINE__
}
;;
{ .mfi
STFD [C4 ] = f91, 5 * SIZE
mov f81 = f0
#ifdef TRMMKERNEL
shladd KK8 = KK, BASE_SHIFT, r0
#else
nop __LINE__
#endif
}
{ .mfb
STFD [C12] = f95, 5 * SIZE
mov f89 = f0
(p6) br.cond.dptk .L052
}
;;
#endif
.align 32
.L060:
{ .mfi
nop __LINE__
mov f66 = f0
tbit.z p6, p7 = M, 2
}
{ .mfb
#ifndef TRMMKERNEL
nop __LINE__
#else
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub L = K, KK
#elif defined(LEFT)
adds L = 4, KK
#else
adds L = 4, KK
#endif
#endif
mov f74 = f0
(p6) br.cond.dptk .L070
}
;;
#if !defined(TRMMKERNEL) || \
defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
{ .mfb
LDFPD f48, f49 = [B]
mov f82 = f0
nop __LINE__
}
{ .mfi
adds BOFFSET = 2 * SIZE, B
mov f90 = f0
#ifndef TRMMKERNEL
adds L = 1, K
#else
adds L = 1, L
#endif
}
;;
#else
{ .mfi
shladd BOFFSET = KK8, 2, B
mov f82 = f0
shladd AOFFSET = KK8, 2, AOFFSET
}
;;
{ .mfi
LDFPD f48, f49 = [BOFFSET], 2 * SIZE
mov f90 = f0
#ifndef TRMMKERNEL
adds L = 1, K
#else
adds L = 1, L
#endif
}
;;
#endif
;;
{ .mii
LDFPD f32, f33 = [AOFFSET], 2 * SIZE
tbit.z p12, p0 = L, 0
shr L = L, 1
}
;;
{ .mfi
LDFPD f34, f35 = [AOFFSET], 2 * SIZE
mov f67 = f0
adds L = -1, L
}
{ .mfi
adds PREB = (PREFETCHSIZE + 0) * SIZE, BOFFSET
mov f75 = f0
nop __LINE__
}
;;
{ .mfi
LDFPD f50, f51 = [BOFFSET], 2 * SIZE
mov f83 = f0
mov ar.lc = L
}
{ .mfi
adds PREA = (PREFETCHSIZE + 0) * SIZE, AOFFSET
mov f91 = f0
cmp.eq p3, p0 = r0, r0
}
;;
.align 32
.L062:
{ .mfi
lfetch.nt1 [PREA], 8 * SIZE
FMA f64 = f32, f48, f64 // A1 * B1
cmp.ne p4, p5 = 0, L
}
{ .mfi
nop __LINE__
FMA f72 = f32, f49, f72 // A1 * B2
(p12) cmp.ne p3, p0 = 0, L
}
;;
{ .mfi
lfetch.nt1 [PREB], 8 * SIZE
FMA f80 = f32, f50, f80 // A1 * B3
(p5) adds C9 = 2 * SIZE, C1
}
{ .mfi
nop __LINE__
FMA f88 = f32, f51, f88 // A1 * B4
(p5) adds C10 = 2 * SIZE, C2
}
;;
{ .mfi
(p3) LDFPD f56, f57 = [BOFFSET], 2 * SIZE
FMA f65 = f33, f48, f65 // A2 * B1
(p5) adds C11 = 2 * SIZE, C3
}
{ .mfi
nop __LINE__
FMA f73 = f33, f49, f73 // A2 * B2
(p5) adds C12 = 2 * SIZE, C4
}
;;
{ .mfb
(p3) LDFPD f40, f41 = [AOFFSET], 2 * SIZE
FMA f81 = f33, f50, f81 // A2 * B3
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f89 = f33, f51, f89 // A2 * B4
nop __LINE__
}
;;
{ .mfb
(p3) LDFPD f58, f59 = [BOFFSET], 2 * SIZE
FMA f66 = f34, f48, f66 // A3 * B1
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f74 = f34, f49, f74 // A3 * B2
nop __LINE__
}
;;
{ .mfb
(p3) LDFPD f42, f43 = [AOFFSET], 2 * SIZE
FMA f82 = f34, f50, f82 // A3 * B3
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f90 = f34, f51, f90 // A3 * B4
nop __LINE__
}
;;
{ .mfb
(p4) LDFPD f32, f33 = [AOFFSET], 2 * SIZE
FMA f67 = f35, f48, f67 // A4 * B1
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f68 = [C1 ], SIZE
#else
nop __LINE__
#endif
FMA f75 = f35, f49, f75 // A4 * B2
nop __LINE__
}
{ .mfb
(p4) LDFPD f48, f49 = [BOFFSET], 2 * SIZE
FMA f83 = f35, f50, f83 // A4 * B3
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f70 = [C9 ], SIZE
#else
nop __LINE__
#endif
FMA f91 = f35, f51, f91 // A4 * B4
nop __LINE__
}
;;
{ .mfb
(p4) LDFPD f50, f51 = [BOFFSET], 2 * SIZE
(p3) FMA f64 = f40, f56, f64 // A1 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f69 = [C1 ], -1 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f72 = f40, f57, f72 // A1 * B2
nop __LINE__
}
;;
{ .mfb
(p4) LDFPD f34, f35 = [AOFFSET], 2 * SIZE
(p3) FMA f80 = f40, f58, f80 // A1 * B3
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f71 = [C9 ], -1 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f88 = f40, f59, f88 // A1 * B4
nop __LINE__
}
;;
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f76 = [C2 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f65 = f41, f56, f65 // A2 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f78 = [C10], SIZE
#else
nop __LINE__
#endif
(p3) FMA f73 = f41, f57, f73 // A2 * B2
nop __LINE__
}
;;
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f77 = [C2 ], -1 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f81 = f41, f58, f81 // A2 * B3
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f79 = [C10], -1 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f89 = f41, f59, f89 // A2 * B4
nop __LINE__
}
;;
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f84 = [C3 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f66 = f42, f56, f66 // A3 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f86 = [C11], SIZE
#else
nop __LINE__
#endif
(p3) FMA f74 = f42, f57, f74 // A3 * B2
nop __LINE__
}
;;
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f85 = [C3 ], -1 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f82 = f42, f58, f82 // A3 * B3
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f87 = [C11], -1 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f90 = f42, f59, f90 // A3 * B4
nop __LINE__
}
;;
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f92 = [C4 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f67 = f43, f56, f67 // A4 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f94 = [C12], SIZE
#else
nop __LINE__
#endif
(p3) FMA f75 = f43, f57, f75 // A4 * B2
nop __LINE__
}
;;
{ .mfi
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f93 = [C4 ], -1 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f83 = f43, f58, f83 // A4 * B3
adds L = -1, L
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f95 = [C12], -1 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f91 = f43, f59, f91 // A4 * B4
br.cloop.sptk.few .L062
}
;;
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
FMA f64 = ALPHA, f64, f68
FMA f66 = ALPHA, f66, f70
FMA f65 = ALPHA, f65, f69
FMA f67 = ALPHA, f67, f71
FMA f72 = ALPHA, f72, f76
FMA f74 = ALPHA, f74, f78
FMA f73 = ALPHA, f73, f77
FMA f75 = ALPHA, f75, f79
;;
{ .mfb
STFD [C1 ] = f64, SIZE
FMA f80 = ALPHA, f80, f84
nop __LINE__
}
{ .mfb
STFD [C9 ] = f66, SIZE
FMA f82 = ALPHA, f82, f86
nop __LINE__
}
;;
{ .mfb
STFD [C1 ] = f65, 3 * SIZE
FMA f81 = ALPHA, f81, f85
nop __LINE__
}
{ .mfb
STFD [C9 ] = f67, 3 * SIZE
FMA f83 = ALPHA, f83, f87
nop __LINE__
}
;;
{ .mfb
STFD [C2 ] = f72, SIZE
FMA f88 = ALPHA, f88, f92
nop __LINE__
}
{ .mfb
STFD [C10] = f74, SIZE
FMA f90 = ALPHA, f90, f94
nop __LINE__
}
;;
{ .mfb
STFD [C2 ] = f73, 3 * SIZE
FMA f89 = ALPHA, f89, f93
nop __LINE__
}
{ .mfb
STFD [C10] = f75, 3 * SIZE
FMA f91 = ALPHA, f91, f95
nop __LINE__
}
;;
{ .mfb
STFD [C3 ] = f80, SIZE
mov f80 = f0
nop __LINE__
}
{ .mfb
STFD [C11] = f82, SIZE
mov f64 = f0
nop __LINE__
}
;;
{ .mfb
STFD [C3 ] = f81, 3 * SIZE
mov f81 = f0
nop __LINE__
}
{ .mfb
STFD [C11] = f83, 3 * SIZE
mov f72 = f0
nop __LINE__
}
;;
{ .mfi
STFD [C4 ] = f88, SIZE
mov f88 = f0
adds L = 1, K
}
{ .mfb
STFD [C12] = f90, SIZE
mov f65 = f0
nop __LINE__
}
;;
{ .mfi
STFD [C4 ] = f89, 3 * SIZE
mov f89 = f0
shr L = L, 1
}
{ .mfb
STFD [C12] = f91, 3 * SIZE
mov f73 = f0
nop __LINE__
}
;;
#else
FMPY f64 = ALPHA, f64
FMPY f66 = ALPHA, f66
FMPY f65 = ALPHA, f65
FMPY f67 = ALPHA, f67
FMPY f72 = ALPHA, f72
FMPY f74 = ALPHA, f74
FMPY f73 = ALPHA, f73
FMPY f75 = ALPHA, f75
;;
{ .mfb
STFD [C1 ] = f64, SIZE
FMPY f80 = ALPHA, f80
nop __LINE__
}
{ .mfb
STFD [C9 ] = f66, SIZE
FMPY f82 = ALPHA, f82
nop __LINE__
}
;;
{ .mfb
STFD [C1 ] = f65, 3 * SIZE
FMPY f81 = ALPHA, f81
nop __LINE__
}
{ .mfb
STFD [C9 ] = f67, 3 * SIZE
FMPY f83 = ALPHA, f83
nop __LINE__
}
;;
{ .mfi
STFD [C2 ] = f72, SIZE
FMPY f88 = ALPHA, f88
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
sub L = K, KK
#else
nop __LINE__
#endif
}
{ .mfb
STFD [C10] = f74, SIZE
FMPY f90 = ALPHA, f90
nop __LINE__
}
;;
{ .mfi
STFD [C2 ] = f73, 3 * SIZE
FMPY f89 = ALPHA, f89
#if defined(TRMMKERNEL) && (defined(LEFT) && defined(TRANSA))
adds L = -4, L
#else
nop __LINE__
#endif
}
{ .mfi
STFD [C10] = f75, 3 * SIZE
FMPY f91 = ALPHA, f91
#if defined(TRMMKERNEL) && (!defined(LEFT) && !defined(TRANSA))
adds L = -4, L
#else
nop __LINE__
#endif
}
;;
{ .mfi
STFD [C3 ] = f80, SIZE
mov f80 = f0
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd KK8 = L, BASE_SHIFT, r0
#else
nop __LINE__
#endif
}
{ .mfb
STFD [C11] = f82, SIZE
mov f64 = f0
nop __LINE__
}
;;
{ .mfi
STFD [C3 ] = f81, 3 * SIZE
mov f81 = f0
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd AOFFSET = KK8, 2, AOFFSET
#else
nop __LINE__
#endif
}
{ .mfi
STFD [C11] = f83, 3 * SIZE
mov f72 = f0
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd BOFFSET = KK8, 2, BOFFSET
#else
nop __LINE__
#endif
}
;;
{ .mfi
STFD [C4 ] = f88, SIZE
mov f88 = f0
#if defined(TRMMKERNEL) && defined(LEFT)
adds KK = 4, KK
#else
nop __LINE__
#endif
}
{ .mfb
STFD [C12] = f90, SIZE
mov f65 = f0
nop __LINE__
}
;;
{ .mfi
STFD [C4 ] = f89, 3 * SIZE
mov f89 = f0
#ifdef TRMMKERNEL
shladd KK8 = KK, BASE_SHIFT, r0
#else
nop __LINE__
#endif
}
{ .mfb
STFD [C12] = f91, 3 * SIZE
mov f73 = f0
nop __LINE__
}
;;
#endif
.align 32
.L070:
{ .mib
#ifndef TRMMKERNEL
nop __LINE__
#else
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub L = K, KK
#elif defined(LEFT)
adds L = 2, KK
#else
adds L = 4, KK
#endif
#endif
tbit.z p6,p7 = M, 1
(p6) br.cond.dptk .L080
}
;;
#if !defined(TRMMKERNEL) || \
defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
{ .mmi
LDFPD f48, f49 = [B]
adds BOFFSET = 2 * SIZE, B
#ifndef TRMMKERNEL
adds L = 1, K
#else
adds L = 1, L
#endif
}
;;
#else
{ .mmi
shladd BOFFSET = KK8, 2, B
shladd AOFFSET = KK8, 1, AOFFSET
nop __LINE__
}
;;
{ .mmi
LDFPD f48, f49 = [BOFFSET], 2 * SIZE
#ifndef TRMMKERNEL
adds L = 1, K
#else
adds L = 1, L
#endif
nop __LINE__
}
;;
#endif
{ .mii
cmp.eq p3, p0 = r0, r0
tbit.z p12, p0 = L, 0
shr L = L, 1
}
;;
{ .mmi
(p7) LDFPD f32, f33 = [AOFFSET], 2 * SIZE
adds L = -1, L
adds PREB = (PREFETCHSIZE + 0) * SIZE, BOFFSET
}
;;
{ .mmi
LDFPD f50, f51 = [BOFFSET], 2 * SIZE
adds PREA = (PREFETCHSIZE + 0) * SIZE, AOFFSET
mov ar.lc = L
}
;;
.align 32
.L072:
{ .mfb
lfetch.nt1 [PREA], 4 * SIZE
FMA f64 = f32, f48, f64 // A1 * B1
nop __LINE__
}
{ .mfi
nop __LINE__
FMA f72 = f32, f49, f72 // A1 * B2
(p12) cmp.ne p3, p0 = 0, L
}
;;
{ .mfi
lfetch.nt1 [PREB], 8 * SIZE
FMA f80 = f32, f50, f80 // A1 * B3
cmp.ne p4, p5 = 0, L
}
{ .mfb
nop __LINE__
FMA f88 = f32, f51, f88 // A1 * B4
nop __LINE__
}
;;
{ .mfi
(p3) LDFPD f40, f41 = [AOFFSET], 2 * SIZE
FMA f65 = f33, f48, f65 // A2 * B1
}
{ .mfi
nop __LINE__
FMA f73 = f33, f49, f73 // A2 * B2
}
;;
{ .mfi
(p3) LDFPD f56, f57 = [BOFFSET], 2 * SIZE
FMA f81 = f33, f50, f81 // A2 * B3
}
{ .mmf
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f68 = [C1 ], SIZE
(p5) LDFD f76 = [C2 ], SIZE
#else
nop __LINE__
nop __LINE__
#endif
FMA f89 = f33, f51, f89 // A2 * B4
}
;;
{ .mfb
(p3) LDFPD f58, f59 = [BOFFSET], 2 * SIZE
(p3) FMA f64 = f40, f56, f64 // A1 * B1
nop __LINE__
}
{ .mmf
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f69 = [C1 ], -1 * SIZE
(p5) LDFD f77 = [C2 ], -1 * SIZE
#else
nop __LINE__
nop __LINE__
#endif
(p3) FMA f72 = f40, f57, f72 // A1 * B2
}
;;
{ .mfb
(p4) LDFPD f32, f33 = [AOFFSET], 2 * SIZE
(p3) FMA f80 = f40, f58, f80 // A1 * B3
nop __LINE__
}
{ .mmf
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f84 = [C3 ], SIZE
(p5) LDFD f92 = [C4 ], SIZE
#else
nop __LINE__
nop __LINE__
#endif
(p3) FMA f88 = f40, f59, f88 // A1 * B4
}
;;
{ .mfb
(p4) LDFPD f48, f49 = [BOFFSET], 2 * SIZE
(p3) FMA f65 = f41, f56, f65 // A2 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f85 = [C3 ], -1 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f73 = f41, f57, f73 // A2 * B2
nop __LINE__
}
;;
{ .mfi
(p4) LDFPD f50, f51 = [BOFFSET], 2 * SIZE
(p3) FMA f81 = f41, f58, f81 // A2 * B3
adds L = -1, L
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f93 = [C4 ], -1 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f89 = f41, f59, f89 // A2 * B4
br.cloop.sptk.few .L072
}
;;
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
FMA f64 = ALPHA, f64, f68
FMA f65 = ALPHA, f65, f69
FMA f72 = ALPHA, f72, f76
FMA f73 = ALPHA, f73, f77
FMA f80 = ALPHA, f80, f84
FMA f81 = ALPHA, f81, f85
FMA f88 = ALPHA, f88, f92
FMA f89 = ALPHA, f89, f93
;;
{ .mfb
STFD [C1 ] = f64, SIZE
mov f64 = f0
nop __LINE__
}
{ .mfb
STFD [C2 ] = f72, SIZE
mov f72 = f0
nop __LINE__
}
;;
{ .mmi
STFD [C1 ] = f65, SIZE
STFD [C2 ] = f73, SIZE
nop __LINE__
}
;;
{ .mfi
STFD [C3 ] = f80, SIZE
mov f80 = f0
adds L = 1, K
}
{ .mfb
STFD [C4 ] = f88, SIZE
mov f88 = f0
nop __LINE__
}
;;
{ .mmi
STFD [C3 ] = f81, SIZE
STFD [C4 ] = f89, SIZE
shr L = L, 1
}
;;
#else
FMPY f64 = ALPHA, f64
FMPY f65 = ALPHA, f65
;;
{ .mfi
nop __LINE__
FMPY f72 = ALPHA, f72
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
sub L = K, KK
#else
nop __LINE__
#endif
}
{ .mfi
nop __LINE__
FMPY f73 = ALPHA, f73
nop __LINE__
}
;;
{ .mfi
FMPY f80 = ALPHA, f80
#if defined(TRMMKERNEL) && (defined(LEFT) && defined(TRANSA))
adds L = -2, L
#else
nop __LINE__
#endif
}
{ .mfi
FMPY f81 = ALPHA, f81
#if defined(TRMMKERNEL) && (!defined(LEFT) && !defined(TRANSA))
adds L = -4, L
#else
nop __LINE__
#endif
}
;;
{ .mfi
nop __LINE__
FMPY f88 = ALPHA, f88
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd KK8 = L, BASE_SHIFT, r0
#else
nop __LINE__
#endif
}
{ .mfi
nop __LINE__
FMPY f89 = ALPHA, f89
nop __LINE__
}
;;
{ .mfi
STFD [C1 ] = f64, SIZE
mov f64 = f0
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd AOFFSET = KK8, 1, AOFFSET
#else
nop __LINE__
#endif
}
{ .mfb
STFD [C2 ] = f72, SIZE
mov f72 = f0
nop __LINE__
}
;;
{ .mmi
STFD [C1 ] = f65, SIZE
STFD [C2 ] = f73, SIZE
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd BOFFSET = KK8, 2, BOFFSET
#else
nop __LINE__
#endif
}
;;
{ .mfi
STFD [C3 ] = f80, SIZE
mov f80 = f0
#if defined(TRMMKERNEL) && defined(LEFT)
adds KK = 2, KK
#else
nop __LINE__
#endif
}
{ .mfb
STFD [C4 ] = f88, SIZE
mov f88 = f0
nop __LINE__
}
;;
{ .mmi
STFD [C3 ] = f81, SIZE
STFD [C4 ] = f89, SIZE
#ifdef TRMMKERNEL
shladd KK8 = KK, BASE_SHIFT, r0
#else
nop __LINE__
#endif
}
;;
#endif
.align 32
.L080:
{ .mib
#ifndef TRMMKERNEL
nop __LINE__
#else
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub L = K, KK
#elif defined(LEFT)
adds L = 1, KK
#else
adds L = 4, KK
#endif
#endif
tbit.z p6,p7 = M, 0
(p6) br.cond.dptk .L089
}
;;
#if !defined(TRMMKERNEL) || \
defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
{ .mmi
LDFPD f48, f49 = [B]
adds BOFFSET = 2 * SIZE, B
#ifndef TRMMKERNEL
adds L = 1, K
#else
adds L = 1, L
#endif
}
;;
#else
{ .mmi
shladd BOFFSET = KK8, 2, B
add AOFFSET = KK8, AOFFSET
nop __LINE__
}
;;
{ .mmi
LDFPD f48, f49 = [BOFFSET], 2 * SIZE
#ifndef TRMMKERNEL
adds L = 1, K
#else
adds L = 1, L
#endif
nop __LINE__
}
;;
#endif
{ .mii
LDFD f32 = [AOFFSET], 1 * SIZE
tbit.z p12, p0 = L, 0
shr L = L, 1
}
;;
{ .mmi
nop __LINE__
nop __LINE__
adds L = -1, L
}
;;
{ .mmi
LDFPD f50, f51 = [BOFFSET], 2 * SIZE
cmp.eq p3, p0 = r0, r0
mov ar.lc = L
}
;;
.align 32
.L082:
{ .mfb
cmp.ne p4, p5 = 0, L
FMA f64 = f32, f48, f64 // A1 * B1
nop __LINE__
}
{ .mfi
(p12) cmp.ne p3, p0 = 0, L
FMA f72 = f32, f49, f72 // A1 * B2
nop __LINE__
}
;;
{ .mfb
(p3) LDFPD f56, f57 = [BOFFSET], 2 * SIZE
FMA f80 = f32, f50, f80 // A1 * B3
nop __LINE__
}
{ .mfb
(p3) LDFD f40 = [AOFFSET], 1 * SIZE
FMA f88 = f32, f51, f88 // A1 * B4
nop __LINE__
}
;;
{ .mfb
(p3) LDFPD f58, f59 = [BOFFSET], 2 * SIZE
(p3) FMA f64 = f40, f56, f64 // A1 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f68 = [C1]
#else
nop __LINE__
#endif
(p3) FMA f72 = f40, f57, f72 // A1 * B2
nop __LINE__
}
;;
{ .mmf
(p4) LDFPD f48, f49 = [BOFFSET], 2 * SIZE
(p4) LDFD f32 = [AOFFSET], 1 * SIZE
(p3) FMA f80 = f40, f58, f80 // A1 * B3
}
{ .mmf
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f76 = [C2]
(p5) LDFD f84 = [C3]
#else
nop __LINE__
nop __LINE__
#endif
(p3) FMA f88 = f40, f59, f88 // A1 * B4
}
;;
{ .mib
(p4) LDFPD f50, f51 = [BOFFSET], 2 * SIZE
nop __LINE__
nop __LINE__
}
{ .mmb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f92 = [C4]
#else
nop __LINE__
#endif
adds L = -1, L
br.cloop.sptk.few .L082
}
;;
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
FMA f64 = ALPHA, f64, f68
FMA f72 = ALPHA, f72, f76
FMA f80 = ALPHA, f80, f84
FMA f88 = ALPHA, f88, f92
;;
STFD [C1 ] = f64, SIZE
STFD [C2 ] = f72, SIZE
STFD [C3 ] = f80, SIZE
STFD [C4 ] = f88, SIZE
;;
#else
{ .mfi
nop __LINE__
FMPY f64 = ALPHA, f64
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
sub L = K, KK
#else
nop __LINE__
#endif
}
{ .mfi
nop __LINE__
FMPY f72 = ALPHA, f72
nop __LINE__
}
;;
{ .mfi
FMPY f80 = ALPHA, f80
#if defined(TRMMKERNEL) && (defined(LEFT) && defined(TRANSA))
adds L = -1, L
#else
nop __LINE__
#endif
}
{ .mfi
FMPY f88 = ALPHA, f88
#if defined(TRMMKERNEL) && (!defined(LEFT) && !defined(TRANSA))
adds L = -4, L
#else
nop __LINE__
#endif
}
;;
{ .mmi
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd KK8 = L, BASE_SHIFT, r0
#else
nop __LINE__
#endif
;;
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
add AOFFSET = KK8, AOFFSET
#else
nop __LINE__
#endif
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd BOFFSET = KK8, 2, BOFFSET
#else
nop __LINE__
#endif
}
;;
{ .mmi
STFD [C1 ] = f64, SIZE
STFD [C2 ] = f72, SIZE
#if defined(TRMMKERNEL) && defined(LEFT)
adds KK = 1, KK
#else
nop __LINE__
#endif
}
;;
{ .mmi
STFD [C3 ] = f80, SIZE
STFD [C4 ] = f88, SIZE
#ifdef TRMMKERNEL
shladd KK8 = KK, BASE_SHIFT, r0
#else
nop __LINE__
#endif
}
;;
#endif
.align 32
.L089:
{ .mmi
mov B = BOFFSET
mov AOFFSET = A
#if defined(TRMMKERNEL) && !defined(LEFT)
adds KK = 4, KK
#else
nop __LINE__
#endif
}
;;
.align 16
#endif
.L090:
{ .mfi
mov C1 = C
mov f64 = f0
tbit.z p6, p0 = N, 1
}
{ .mfi
add C2 = LDC, C
mov f72 = f0
shr I = M, 3
}
;;
{ .mfi
setf.d f66 = r0
mov f65 = f0
#if defined(TRMMKERNEL) && defined(LEFT)
mov KK = OFFSET
#else
nop __LINE__
#endif
}
{ .mfb
mov AOFFSET = A
mov f73 = f0
(p6) br.cond.dpnt .L130
}
;;
#if 0
{ .mfi
#ifdef TRMMKERNEL
shladd KK8 = KK, BASE_SHIFT, r0
#else
nop __LINE__
#endif
mov f67 = f0
shladd C = LDC, 1, C
}
{ .mfb
cmp.eq p6, p7 = 0, I
mov f74 = f0
(p6) br.cond.dpnt .L100
}
;;
.align 32
.L092:
#if !defined(TRMMKERNEL) || \
defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
{ .mfb
LDFPD f48, f49 = [B]
mov f68 = f0
nop __LINE__
}
{ .mfb
adds BOFFSET = 2 * SIZE, B
mov f79 = f0
nop __LINE__
}
;;
#else
{ .mfi
shladd BOFFSET = KK8, 1, B
mov f68 = f0
shladd AOFFSET = KK8, 3, AOFFSET
}
;;
{ .mfi
LDFPD f48, f49 = [BOFFSET], 2 * SIZE
mov f79 = f0
nop __LINE__
}
;;
#endif
{ .mfi
LDFPD f32, f33 = [AOFFSET], 2 * SIZE
mov f75 = f0
#ifndef TRMMKERNEL
nop __LINE__
#else
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub L = K, KK
#elif defined(LEFT)
adds L = 8, KK
#else
adds L = 2, KK
#endif
#endif
}
;;
{ .mfi
adds PREC = CPREFETCHSIZE * SIZE, C1
mov f76 = f0
#ifndef TRMMKERNEL
adds L = 1, K
#else
adds L = 1, L
#endif
}
;;
{ .mfi
LDFPD f34, f35 = [AOFFSET], 2 * SIZE
mov f69 = f0
tbit.z p12, p0 = L, 0
}
{ .mfi
cmp.eq p3, p0 = r0, r0
mov f77 = f0
shr L = L, 1
}
;;
{ .mfi
adds PREA = (PREFETCHSIZE + 0) * SIZE, AOFFSET
adds L = -1, L
}
{ .mmf
LDFPD f36, f37 = [AOFFSET], 2 * SIZE
CPREFETCH [PREC], LDC
mov f70 = f0
}
;;
{ .mfi
LDFPD f38, f39 = [AOFFSET], 2 * SIZE
mov f78 = f0
mov ar.lc = L
}
{ .mfi
CPREFETCH [PREC]
mov f71 = f0
adds PREB = (PREFETCHSIZE + 0) * SIZE, BOFFSET
}
;;
.align 32
.L093:
/* 1 */
{ .mfi
lfetch.nt1 [PREA], 16 * SIZE
FMA f64 = f32, f48, f64 // A1 * B1
cmp.ne p4, p5 = 0, L
}
{ .mfi
nop __LINE__
FMA f72 = f32, f49, f72 // A1 * B2
(p12) cmp.ne p3, p0 = 0, L
}
;;
{ .mfi
lfetch.nt1 [PREB], 4 * SIZE
FMA f65 = f33, f48, f65 // A2 * B1
adds C9 = 4 * SIZE, C1
}
{ .mfi
nop __LINE__
FMA f73 = f33, f49, f73 // A2 * B2
adds C10 = 4 * SIZE, C2
}
;;
{ .mfi
(p3) LDFPD f56, f57 = [BOFFSET], 2 * SIZE
FMA f66 = f34, f48, f66 // A3 * B1
adds C11 = 4 * SIZE, C3
}
{ .mfi
nop __LINE__
FMA f74 = f34, f49, f74 // A3 * B2
adds C12 = 4 * SIZE, C4
}
;;
{ .mfb
(p3) LDFPD f40, f41 = [AOFFSET], 2 * SIZE
FMA f67 = f35, f48, f67 // A4 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f96 = [C1 ], SIZE
#else
nop __LINE__
#endif
FMA f75 = f35, f49, f75 // A4 * B2
nop __LINE__
}
;;
{ .mfb
(p3) LDFPD f42, f43 = [AOFFSET], 2 * SIZE
FMA f68 = f36, f48, f68 // A5 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f97 = [C9 ], SIZE
#else
nop __LINE__
#endif
FMA f76 = f36, f49, f76 // A5 * B2
nop __LINE__
}
;;
{ .mfb
(p3) LDFPD f44, f45 = [AOFFSET], 2 * SIZE
FMA f69 = f37, f48, f69 // A6 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f98 = [C1 ], SIZE
#else
nop __LINE__
#endif
FMA f77 = f37, f49, f77 // A6 * B2
nop __LINE__
}
;;
{ .mfb
(p3) LDFPD f46, f47 = [AOFFSET], 2 * SIZE
FMA f70 = f38, f48, f70 // A7 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f99 = [C9 ], SIZE
#else
nop __LINE__
#endif
FMA f78 = f38, f49, f78 // A7 * B2
nop __LINE__
}
;;
{ .mfb
(p4) LDFPD f32, f33 = [AOFFSET], 2 * SIZE
FMA f71 = f39, f48, f71 // A8 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f100 = [C1 ], SIZE
#else
nop __LINE__
#endif
FMA f79 = f39, f49, f79 // A8 * B2
nop __LINE__
}
;;
{ .mfb
(p4) LDFPD f48, f49 = [BOFFSET], 2 * SIZE
(p3) FMA f64 = f40, f56, f64 // A1 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f101 = [C9 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f72 = f40, f57, f72 // A1 * B2
nop __LINE__
}
;;
{ .mfb
(p4) LDFPD f34, f35 = [AOFFSET], 2 * SIZE
(p3) FMA f65 = f41, f56, f65 // A2 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f102 = [C1 ], -3 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f73 = f41, f57, f73 // A2 * B2
nop __LINE__
}
;;
{ .mfb
(p4) LDFPD f36, f37 = [AOFFSET], 2 * SIZE
(p3) FMA f66 = f42, f56, f66 // A3 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f103 = [C9 ], -3 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f74 = f42, f57, f74 // A3 * B2
nop __LINE__
}
;;
{ .mfb
(p4) LDFPD f38, f39 = [AOFFSET], 2 * SIZE
(p3) FMA f67 = f43, f56, f67 // A4 * B1
nop __LINE__
}
{ .mfb
nop __LINE__
(p3) FMA f75 = f43, f57, f75 // A4 * B2
nop __LINE__
}
;;
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f104 = [C2 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f68 = f44, f56, f68 // A5 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f105 = [C10], SIZE
#else
nop __LINE__
#endif
(p3) FMA f76 = f44, f57, f76 // A5 * B2
nop __LINE__
}
;;
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f106 = [C2 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f69 = f45, f56, f69 // A6 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f107 = [C10], SIZE
#else
nop __LINE__
#endif
(p3) FMA f77 = f45, f57, f77 // A6 * B2
nop __LINE__
}
;;
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f108 = [C2 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f70 = f46, f56, f70 // A7 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f109 = [C10], SIZE
#else
nop __LINE__
#endif
(p3) FMA f78 = f46, f57, f78 // A7 * B2
nop __LINE__
}
;;
{ .mfi
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f110 = [C2 ], -3 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f71 = f47, f56, f71 // A8 * B1
adds L = -1, L
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f111 = [C10], -3 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f79 = f47, f57, f79 // A8 * B2
br.cloop.sptk.few .L093
}
;;
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
{ .mfi
nop __LINE__
FMA f64 = ALPHA, f64, f96
cmp.ne p6, p0 = 1, I
}
{ .mfb
nop __LINE__
FMA f68 = ALPHA, f68, f97
nop __LINE__
}
;;
{ .mfi
nop __LINE__
FMA f65 = ALPHA, f65, f98
adds I = -1, I
}
{ .mfb
nop __LINE__
FMA f69 = ALPHA, f69, f99
nop __LINE__
}
;;
{ .mfi
nop __LINE__
FMA f66 = ALPHA, f66, f100
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f70 = ALPHA, f70, f101
nop __LINE__
}
;;
{ .mfb
nop __LINE__
FMA f67 = ALPHA, f67, f102
nop __LINE__
}
{ .mfb
nop __LINE__
FMA f71 = ALPHA, f71, f103
nop __LINE__
}
;;
{ .mfb
STFD [C1 ] = f64, SIZE
FMA f72 = ALPHA, f72, f104
nop __LINE__
}
{ .mfb
STFD [C9 ] = f68, SIZE
FMA f76 = ALPHA, f76, f105
nop __LINE__
}
;;
{ .mfb
STFD [C1 ] = f65, SIZE
FMA f73 = ALPHA, f73, f106
nop __LINE__
}
{ .mfb
STFD [C9 ] = f69, SIZE
FMA f77 = ALPHA, f77, f107
nop __LINE__
}
;;
{ .mfb
STFD [C1 ] = f66, SIZE
FMA f74 = ALPHA, f74, f108
nop __LINE__
}
{ .mfb
STFD [C9 ] = f70, SIZE
FMA f78 = ALPHA, f78, f109
nop __LINE__
}
;;
{ .mfb
STFD [C1 ] = f67, 5 * SIZE
FMA f75 = ALPHA, f75, f110
nop __LINE__
}
{ .mfb
STFD [C9 ] = f71, 5 * SIZE
FMA f79 = ALPHA, f79, f111
nop __LINE__
}
;;
{ .mfb
STFD [C2 ] = f72, SIZE
mov f64 = f0
nop __LINE__
}
{ .mfb
STFD [C10] = f76, SIZE
mov f72 = f0
nop __LINE__
}
;;
{ .mfb
STFD [C2 ] = f73, SIZE
mov f65 = f0
nop __LINE__
}
{ .mfb
STFD [C10] = f77, SIZE
mov f73 = f0
nop __LINE__
}
;;
{ .mfb
STFD [C2 ] = f74, SIZE
mov f66 = f0
nop __LINE__
}
{ .mfb
STFD [C10] = f78, SIZE
mov f74 = f0
nop __LINE__
}
;;
{ .mfb
STFD [C2 ] = f75, 5 * SIZE
mov f67 = f0
nop __LINE__
}
{ .mfb
STFD [C10] = f79, 5 * SIZE
(p6) br.cond.dptk .L092
}
;;
#else
{ .mfi
nop __LINE__
FMPY f64 = ALPHA, f64
cmp.ne p6, p0 = 1, I
}
{ .mfb
nop __LINE__
FMPY f68 = ALPHA, f68
nop __LINE__
}
;;
{ .mfi
nop __LINE__
FMPY f65 = ALPHA, f65
adds I = -1, I
}
{ .mfb
nop __LINE__
FMPY f69 = ALPHA, f69
nop __LINE__
}
;;
{ .mfi
nop __LINE__
FMPY f66 = ALPHA, f66
nop __LINE__
}
{ .mfb
nop __LINE__
FMPY f70 = ALPHA, f70
nop __LINE__
}
;;
{ .mfb
nop __LINE__
FMPY f67 = ALPHA, f67
nop __LINE__
}
{ .mfb
nop __LINE__
FMPY f71 = ALPHA, f71
nop __LINE__
}
;;
{ .mfb
STFD [C1 ] = f64, SIZE
FMPY f72 = ALPHA, f72
nop __LINE__
}
{ .mfb
STFD [C9 ] = f68, SIZE
FMPY f76 = ALPHA, f76
nop __LINE__
}
;;
{ .mfb
STFD [C1 ] = f65, SIZE
FMPY f73 = ALPHA, f73
nop __LINE__
}
{ .mfb
STFD [C9 ] = f69, SIZE
FMPY f77 = ALPHA, f77
nop __LINE__
}
;;
{ .mfi
STFD [C1 ] = f66, SIZE
FMPY f74 = ALPHA, f74
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
sub L = K, KK
#else
nop __LINE__
#endif
}
{ .mfb
STFD [C9 ] = f70, SIZE
FMPY f78 = ALPHA, f78
nop __LINE__
}
;;
{ .mfi
STFD [C1 ] = f67, 5 * SIZE
FMPY f75 = ALPHA, f75
#if defined(TRMMKERNEL) && (defined(LEFT) && defined(TRANSA))
adds L = -8, L
#else
nop __LINE__
#endif
}
{ .mfi
STFD [C9 ] = f71, 5 * SIZE
FMPY f79 = ALPHA, f79
#if defined(TRMMKERNEL) && (!defined(LEFT) && !defined(TRANSA))
adds L = -2, L
#else
nop __LINE__
#endif
}
;;
{ .mfi
STFD [C2 ] = f72, SIZE
mov f64 = f0
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd KK8 = L, BASE_SHIFT, r0
#else
nop __LINE__
#endif
}
{ .mfb
STFD [C10] = f76, SIZE
mov f72 = f0
nop __LINE__
}
;;
{ .mfi
STFD [C2 ] = f73, SIZE
mov f65 = f0
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd AOFFSET = KK8, 3, AOFFSET
#else
nop __LINE__
#endif
}
{ .mfi
STFD [C10] = f77, SIZE
mov f73 = f0
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd BOFFSET = KK8, 1, BOFFSET
#else
nop __LINE__
#endif
}
;;
{ .mfi
STFD [C2 ] = f74, SIZE
mov f66 = f0
#if defined(TRMMKERNEL) && defined(LEFT)
adds KK = 8, KK
#else
nop __LINE__
#endif
}
{ .mfb
STFD [C10] = f78, SIZE
mov f74 = f0
nop __LINE__
}
;;
{ .mfi
STFD [C2 ] = f75, 5 * SIZE
mov f67 = f0
#ifdef TRMMKERNEL
shladd KK8 = KK, BASE_SHIFT, r0
#else
nop __LINE__
#endif
}
{ .mib
STFD [C10] = f79, 5 * SIZE
nop __LINE__
(p6) br.cond.dptk .L092
}
;;
#endif
.align 32
.L100:
{ .mib
#ifndef TRMMKERNEL
nop __LINE__
#else
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub L = K, KK
#elif defined(LEFT)
adds L = 4, KK
#else
adds L = 2, KK
#endif
#endif
tbit.z p6, p7 = M, 2
(p6) br.cond.dptk .L110
}
;;
#if !defined(TRMMKERNEL) || \
defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
{ .mmf
LDFPD f48, f49 = [B]
adds BOFFSET = 2 * SIZE, B
mov f75 = f0
}
{ .mii
nop __LINE__
#ifndef TRMMKERNEL
adds L = 1, K
#else
adds L = 1, L
#endif
}
;;
#else
{ .mfi
shladd BOFFSET = KK8, 1, B
mov f75 = f0
shladd AOFFSET = KK8, 2, AOFFSET
}
;;
{ .mmi
LDFPD f48, f49 = [BOFFSET], 2 * SIZE
nop __LINE__
adds L = 1, L
}
;;
#endif
;;
{ .mii
adds PREA = (PREFETCHSIZE + 0) * SIZE, AOFFSET
tbit.z p12, p0 = L, 0
shr L = L, 1
}
;;
{ .mmi
LDFPD f32, f33 = [AOFFSET], 2 * SIZE
nop __LINE__
adds L = -1, L
}
;;
{ .mmi
LDFPD f34, f35 = [AOFFSET], 2 * SIZE
cmp.eq p3, p0 = r0, r0
mov ar.lc = L
}
;;
.align 32
.L102:
{ .mfi
lfetch.nt1 [PREA], 8 * SIZE
FMA f64 = f32, f48, f64 // A1 * B1
cmp.ne p4, p5 = 0, L
}
{ .mfi
adds PREB = (PREFETCHSIZE + 0) * SIZE, BOFFSET
FMA f72 = f32, f49, f72 // A1 * B2
(p12) cmp.ne p3, p0 = 0, L
}
;;
{ .mfi
lfetch.nt1 [PREB], 4 * SIZE
FMA f65 = f33, f48, f65 // A2 * B1
adds C9 = 2 * SIZE, C1
}
{ .mfi
nop __LINE__
FMA f73 = f33, f49, f73 // A2 * B2
adds C10 = 2 * SIZE, C2
}
;;
{ .mfb
(p3) LDFPD f56, f57 = [BOFFSET], 2 * SIZE
FMA f66 = f34, f48, f66 // A3 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f68 = [C1 ], SIZE
#else
nop __LINE__
#endif
FMA f74 = f34, f49, f74 // A3 * B2
nop __LINE__
}
;;
{ .mfb
(p3) LDFPD f40, f41 = [AOFFSET], 2 * SIZE
FMA f67 = f35, f48, f67 // A4 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f70 = [C9 ], SIZE
#else
nop __LINE__
#endif
FMA f75 = f35, f49, f75 // A4 * B2
nop __LINE__
}
;;
{ .mfb
(p3) LDFPD f42, f43 = [AOFFSET], 2 * SIZE
(p3) FMA f64 = f40, f56, f64 // A1 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f69 = [C1 ], -1 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f72 = f40, f57, f72 // A1 * B2
nop __LINE__
}
;;
{ .mfb
(p4) LDFPD f32, f33 = [AOFFSET], 2 * SIZE
(p3) FMA f65 = f41, f56, f65 // A2 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f71 = [C9 ], -1 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f73 = f41, f57, f73 // A2 * B2
nop __LINE__
}
;;
{ .mfb
(p4) LDFPD f48, f49 = [BOFFSET], 2 * SIZE
(p3) FMA f66 = f42, f56, f66 // A3 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f76 = [C2 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f74 = f42, f57, f74 // A3 * B2
nop __LINE__
}
;;
{ .mfi
(p4) LDFPD f34, f35 = [AOFFSET], 2 * SIZE
(p3) FMA f67 = f43, f56, f67 // A4 * B1
adds L = -1, L
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f78 = [C10], SIZE
#else
nop __LINE__
#endif
(p3) FMA f75 = f43, f57, f75 // A4 * B2
br.cloop.sptk.few .L102
}
;;
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
{ .mfb
LDFD f77 = [C2 ], -1 * SIZE
FMA f64 = ALPHA, f64, f68
nop __LINE__
}
{ .mfb
LDFD f79 = [C10], -1 * SIZE
FMA f66 = ALPHA, f66, f70
nop __LINE__
}
;;
FMA f65 = ALPHA, f65, f69
adds L = 1, K
FMA f67 = ALPHA, f67, f71
;;
FMA f72 = ALPHA, f72, f76
shr L = L, 1
FMA f74 = ALPHA, f74, f78
FMA f73 = ALPHA, f73, f77
FMA f75 = ALPHA, f75, f79
;;
{ .mmf
STFD [C1 ] = f64, SIZE
STFD [C9 ] = f66, SIZE
mov f64 = f0
}
;;
{ .mmf
STFD [C1 ] = f65, 3 * SIZE
STFD [C9 ] = f67, 3 * SIZE
mov f65 = f0
}
;;
{ .mmf
STFD [C2 ] = f72, SIZE
STFD [C10] = f74, SIZE
mov f72 = f0
}
;;
{ .mmf
STFD [C2 ] = f73, 3 * SIZE
STFD [C10] = f75, 3 * SIZE
mov f73 = f0
}
;;
#else
{ .mfb
nop __LINE__
FMPY f64 = ALPHA, f64
nop __LINE__
}
{ .mfb
nop __LINE__
FMPY f66 = ALPHA, f66
nop __LINE__
}
;;
FMPY f65 = ALPHA, f65
FMPY f67 = ALPHA, f67
;;
{ .mfi
nop __LINE__
FMPY f72 = ALPHA, f72
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
sub L = K, KK
#else
nop __LINE__
#endif
}
{ .mfi
nop __LINE__
FMPY f74 = ALPHA, f74
nop __LINE__
}
;;
{ .mfi
nop __LINE__
FMPY f73 = ALPHA, f73
#if defined(TRMMKERNEL) && (defined(LEFT) && defined(TRANSA))
adds L = -4, L
#else
nop __LINE__
#endif
}
{ .mfi
nop __LINE__
FMPY f75 = ALPHA, f75
#if defined(TRMMKERNEL) && (!defined(LEFT) && !defined(TRANSA))
adds L = -2, L
#else
nop __LINE__
#endif
}
;;
{ .mfi
STFD [C1 ] = f64, SIZE
mov f64 = f0
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd KK8 = L, BASE_SHIFT, r0
#else
nop __LINE__
#endif
}
{ .mmi
STFD [C9 ] = f66, SIZE
nop __LINE__
nop __LINE__
}
;;
{ .mfi
STFD [C1 ] = f65, 3 * SIZE
mov f65 = f0
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd AOFFSET = KK8, 2, AOFFSET
#else
nop __LINE__
#endif
}
{ .mmi
STFD [C9 ] = f67, 3 * SIZE
nop __LINE__
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd BOFFSET = KK8, 1, BOFFSET
#else
nop __LINE__
#endif
}
;;
{ .mfi
STFD [C2 ] = f72, SIZE
mov f72 = f0
#if defined(TRMMKERNEL) && defined(LEFT)
adds KK = 4, KK
#else
nop __LINE__
#endif
}
{ .mmi
STFD [C10] = f74, SIZE
nop __LINE__
nop __LINE__
}
;;
{ .mfi
STFD [C2 ] = f73, 3 * SIZE
mov f73 = f0
#ifdef TRMMKERNEL
shladd KK8 = KK, BASE_SHIFT, r0
#else
nop __LINE__
#endif
}
{ .mib
STFD [C10] = f75, 3 * SIZE
nop __LINE__
nop __LINE__
}
;;
#endif
.align 32
.L110:
{ .mib
#ifndef TRMMKERNEL
nop __LINE__
#else
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub L = K, KK
#elif defined(LEFT)
adds L = 2, KK
#else
adds L = 2, KK
#endif
#endif
tbit.z p6, p7 = M, 1
(p6) br.cond.dptk .L120
}
;;
#if !defined(TRMMKERNEL) || \
defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
{ .mmi
LDFPD f48, f49 = [B]
adds BOFFSET = 2 * SIZE, B
#ifndef TRMMKERNEL
adds L = 1, K
#else
adds L = 1, L
#endif
}
;;
#else
{ .mmi
shladd BOFFSET = KK8, 1, B
shladd AOFFSET = KK8, 1, AOFFSET
}
;;
{ .mmi
LDFPD f48, f49 = [BOFFSET], 2 * SIZE
nop __LINE__
adds L = 1, L
}
;;
#endif
;;
{ .mii
adds PREA = (PREFETCHSIZE + 0) * SIZE, AOFFSET
tbit.z p12, p0 = L, 0
shr L = L, 1
}
;;
{ .mmi
LDFPD f32, f33 = [AOFFSET], 2 * SIZE
nop __LINE__
adds L = -1, L
}
;;
{ .mmi
cmp.eq p3, p0 = r0, r0
adds PREB = (PREFETCHSIZE + 0) * SIZE, BOFFSET
mov ar.lc = L
}
;;
.align 32
.L112:
{ .mfi
lfetch.nt1 [PREA], 4 * SIZE
FMA f64 = f32, f48, f64 // A1 * B1
cmp.ne p4, p5 = 0, L
}
{ .mfi
lfetch.nt1 [PREB], 4 * SIZE
FMA f72 = f32, f49, f72 // A1 * B2
(p12) cmp.ne p3, p0 = 0, L
}
;;
{ .mmf
(p3) LDFPD f40, f41 = [AOFFSET], 2 * SIZE
(p3) LDFPD f56, f57 = [BOFFSET], 2 * SIZE
FMA f65 = f33, f48, f65 // A2 * B1
}
{ .mmf
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f68 = [C1 ], SIZE
(p5) LDFD f76 = [C2 ], SIZE
#else
nop __LINE__
nop __LINE__
#endif
FMA f73 = f33, f49, f73 // A2 * B2
}
;;
{ .mfb
(p4) LDFPD f32, f33 = [AOFFSET], 2 * SIZE
(p3) FMA f64 = f40, f56, f64 // A1 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f69 = [C1 ], -1 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f72 = f40, f57, f72 // A1 * B2
nop __LINE__
}
;;
{ .mfi
(p4) LDFPD f48, f49 = [BOFFSET], 2 * SIZE
(p3) FMA f65 = f41, f56, f65 // A2 * B1
adds L = -1, L
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f77 = [C2 ], -1 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f73 = f41, f57, f73 // A2 * B2
br.cloop.sptk.few .L112
}
;;
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
FMA f64 = ALPHA, f64, f68
FMA f65 = ALPHA, f65, f69
FMA f72 = ALPHA, f72, f76
FMA f73 = ALPHA, f73, f77
;;
{ .mfi
STFD [C1 ] = f64, SIZE
mov f64 = f0
nop __LINE__
}
{ .mfb
STFD [C2 ] = f72, SIZE
mov f72 = f0
nop __LINE__
}
;;
{ .mfi
STFD [C1 ] = f65, SIZE
mov f65 = f0
nop __LINE__
}
{ .mfb
STFD [C2 ] = f73, SIZE
mov f73 = f0
nop __LINE__
}
;;
#else
{ .mfi
nop __LINE__
FMPY f64 = ALPHA, f64
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
sub L = K, KK
#else
nop __LINE__
#endif
}
{ .mfi
nop __LINE__
FMPY f65 = ALPHA, f65
nop __LINE__
}
;;
{ .mfi
nop __LINE__
FMPY f72 = ALPHA, f72
#if defined(TRMMKERNEL) && (defined(LEFT) && defined(TRANSA))
adds L = -2, L
#else
nop __LINE__
#endif
}
{ .mfi
nop __LINE__
FMPY f73 = ALPHA, f73
#if defined(TRMMKERNEL) && (!defined(LEFT) && !defined(TRANSA))
adds L = -2, L
#else
nop __LINE__
#endif
}
;;
{ .mmi
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd KK8 = L, BASE_SHIFT, r0
#else
nop __LINE__
#endif
;;
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd AOFFSET = KK8, 1, AOFFSET
#else
nop __LINE__
#endif
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd BOFFSET = KK8, 1, BOFFSET
#else
nop __LINE__
#endif
}
;;
{ .mfi
STFD [C1 ] = f64, SIZE
mov f64 = f0
#if defined(TRMMKERNEL) && defined(LEFT)
adds KK = 2, KK
#else
nop __LINE__
#endif
}
{ .mfb
STFD [C2 ] = f72, SIZE
mov f72 = f0
nop __LINE__
}
;;
{ .mfi
STFD [C1 ] = f65, SIZE
mov f65 = f0
#ifdef TRMMKERNEL
shladd KK8 = KK, BASE_SHIFT, r0
#else
nop __LINE__
#endif
}
{ .mfb
STFD [C2 ] = f73, SIZE
mov f73 = f0
nop __LINE__
}
;;
#endif
.align 32
.L120:
{ .mib
#ifndef TRMMKERNEL
nop __LINE__
#else
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub L = K, KK
#elif defined(LEFT)
adds L = 1, KK
#else
adds L = 2, KK
#endif
#endif
tbit.z p6, p7 = M, 0
(p6) br.cond.dptk .L129
}
;;
#if !defined(TRMMKERNEL) || \
defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
{ .mmi
LDFPD f48, f49 = [B]
adds BOFFSET = 2 * SIZE, B
#ifndef TRMMKERNEL
adds L = 1, K
#else
adds L = 1, L
#endif
}
;;
#else
{ .mmi
shladd BOFFSET = KK8, 1, B
add AOFFSET = KK8, AOFFSET
}
;;
{ .mmi
LDFPD f48, f49 = [BOFFSET], 2 * SIZE
nop __LINE__
adds L = 1, L
}
;;
#endif
{ .mii
nop __LINE__
tbit.z p12, p0 = L, 0
shr L = L, 1
}
;;
{ .mmi
LDFD f32 = [AOFFSET], 1 * SIZE
nop __LINE__
adds L = -1, L
}
;;
{ .mmi
cmp.eq p3, p0 = r0, r0
nop __LINE__
mov ar.lc = L
}
;;
.align 32
.L122:
{ .mfi
FMA f64 = f32, f48, f64 // A1 * B1
cmp.ne p4, p5 = 0, L
}
{ .mfi
nop __LINE__
FMA f72 = f32, f49, f72 // A1 * B2
(p12) cmp.ne p3, p0 = 0, L
}
;;
{ .mmi
(p3) LDFPD f56, f57 = [BOFFSET], 2 * SIZE
(p3) LDFD f40 = [AOFFSET], 1 * SIZE
nop __LINE__
}
{ .mmi
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f68 = [C1]
(p5) LDFD f76 = [C2]
#else
nop __LINE__
nop __LINE__
#endif
nop __LINE__
}
;;
{ .mfi
(p4) LDFPD f48, f49 = [BOFFSET], 2 * SIZE
(p3) FMA f64 = f40, f56, f64 // A1 * B1
adds L = -1, L
}
{ .mfb
(p4) LDFD f32 = [AOFFSET], 1 * SIZE
(p3) FMA f72 = f40, f57, f72 // A1 * B2
br.cloop.sptk.few .L122
}
;;
.L128:
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
FMA f64 = ALPHA, f64, f68
FMA f72 = ALPHA, f72, f76
;;
{ .mfi
STFD [C1 ] = f64
mov f64 = f0
}
{ .mfb
STFD [C2 ] = f72
mov f72 = f0
}
;;
#else
{ .mfi
nop __LINE__
FMPY f64 = ALPHA, f64
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
sub L = K, KK
#else
nop __LINE__
#endif
}
{ .mfi
nop __LINE__
FMPY f72 = ALPHA, f72
nop __LINE__
}
;;
{ .mmi
nop __LINE__
#if defined(TRMMKERNEL) && (defined(LEFT) && defined(TRANSA))
adds L = -1, L
#else
nop __LINE__
#endif
#if defined(TRMMKERNEL) && (!defined(LEFT) && !defined(TRANSA))
adds L = -2, L
#else
nop __LINE__
#endif
}
;;
{ .mmi
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd KK8 = L, BASE_SHIFT, r0
#else
nop __LINE__
#endif
;;
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
add AOFFSET = KK8, AOFFSET
#else
nop __LINE__
#endif
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd BOFFSET = KK8, 1, BOFFSET
#else
nop __LINE__
#endif
}
;;
#if defined(TRMMKERNEL) && defined(LEFT)
adds KK = 1, KK
#else
nop __LINE__
#endif
;;
{ .mfi
STFD [C1 ] = f64
mov f64 = f0
#ifdef TRMMKERNEL
shladd KK8 = KK, BASE_SHIFT, r0
#else
nop __LINE__
#endif
}
{ .mfb
STFD [C2 ] = f72
mov f72 = f0
}
;;
#endif
.align 32
.L129:
{ .mmi
mov B = BOFFSET
mov AOFFSET = A
#if defined(TRMMKERNEL) && !defined(LEFT)
adds KK = 2, KK
#else
nop __LINE__
#endif
}
;;
.align 16
#endif
.L130:
{ .mfi
#if defined(TRMMKERNEL) && defined(LEFT)
mov KK = OFFSET
#else
nop __LINE__
#endif
mov f64 = f0
tbit.z p6, p0 = N, 0
}
{ .mib
mov AOFFSET = A
shr I = M, 3
(p6) br.cond.dpnt .L999
}
;;
#if 0
{ .mfi
mov C1 = C
mov f65 = f0
#ifdef TRMMKERNEL
shladd KK8 = KK, BASE_SHIFT, r0
#else
nop __LINE__
#endif
}
;;
{ .mfi
nop __LINE__
mov f66 = f0
nop __LINE__
}
{ .mfb
cmp.eq p7, p0 = 0, I
mov f67 = f0
(p7) br.cond.dpnt .L140
}
;;
.align 32
.L132:
#if !defined(TRMMKERNEL) || \
defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
{ .mfb
LDFD f48 = [B]
mov f68 = f0
nop __LINE__
}
{ .mfi
adds BOFFSET = 1 * SIZE, B
mov f69 = f0
#ifndef TRMMKERNEL
nop __LINE__
#else
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub L = K, KK
#elif defined(LEFT)
adds L = 8, KK
#else
adds L = 1, KK
#endif
#endif
}
;;
#else
{ .mfi
add BOFFSET = KK8, B
mov f68 = f0
shladd AOFFSET = KK8, 3, AOFFSET
}
;;
{ .mfi
LDFD f48 = [BOFFSET], 1 * SIZE
mov f69 = f0
#ifndef TRMMKERNEL
nop __LINE__
#else
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub L = K, KK
#elif defined(LEFT)
adds L = 8, KK
#else
adds L = 1, KK
#endif
#endif
}
;;
#endif
{ .mfi
LDFPD f32, f33 = [AOFFSET], 2 * SIZE
mov f70 = f0
#ifndef TRMMKERNEL
adds L = 1, K
#else
adds L = 1, L
#endif
}
;;
{ .mii
LDFPD f34, f35 = [AOFFSET], 2 * SIZE
tbit.z p12, p0 = L, 0
shr L = L, 1
}
;;
{ .mfi
LDFPD f36, f37 = [AOFFSET], 2 * SIZE
mov f71 = f0
adds L = -1, L
}
;;
{ .mmi
LDFPD f38, f39 = [AOFFSET], 2 * SIZE
adds PREC = CPREFETCHSIZE * SIZE, C1
cmp.eq p3, p0 = r0, r0
}
;;
{ .mmi
CPREFETCH [PREC]
adds PREA = (PREFETCHSIZE + 0) * SIZE, AOFFSET
mov ar.lc = L
}
;;
.align 32
.L133:
{ .mfi
lfetch.nt1 [PREA], 16 * SIZE
FMA f64 = f32, f48, f64 // A1 * B1
cmp.ne p4, p5 = 0, L
}
{ .mfi
adds PREB = (PREFETCHSIZE + 0) * SIZE, BOFFSET
FMA f65 = f33, f48, f65 // A2 * B1
(p12) cmp.ne p3, p0 = 0, L
}
;;
{ .mfi
(p3) LDFPD f40, f41 = [AOFFSET], 2 * SIZE
FMA f66 = f34, f48, f66 // A3 * B1
adds C9 = 4 * SIZE, C1
}
{ .mmf
(p3) LDFD f56 = [BOFFSET], 1 * SIZE
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f6 = [C1 ], SIZE
#else
nop __LINE__
#endif
FMA f67 = f35, f48, f67 // A4 * B1
}
;;
{ .mfb
(p3) LDFPD f42, f43 = [AOFFSET], 2 * SIZE
FMA f68 = f36, f48, f68 // A5 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f7 = [C9 ], SIZE
#else
nop __LINE__
#endif
FMA f69 = f37, f48, f69 // A6 * B1
nop __LINE__
}
;;
{ .mfb
(p3) LDFPD f44, f45 = [AOFFSET], 2 * SIZE
FMA f70 = f38, f48, f70 // A7 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f10 = [C1 ], SIZE
#else
nop __LINE__
#endif
FMA f71 = f39, f48, f71 // A8 * B1
nop __LINE__
}
;;
{ .mfb
(p3) LDFPD f46, f47 = [AOFFSET], 2 * SIZE
(p3) FMA f64 = f40, f56, f64 // A1 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f11 = [C9 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f65 = f41, f56, f65 // A2 * B1
nop __LINE__
}
;;
{ .mfb
(p4) LDFPD f32, f33 = [AOFFSET], 2 * SIZE
(p3) FMA f66 = f42, f56, f66 // A3 * B1
nop __LINE__
}
{ .mmf
(p4) LDFD f48 = [BOFFSET], 1 * SIZE
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f12 = [C1 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f67 = f43, f56, f67 // A4 * B1
}
;;
{ .mfb
(p4) LDFPD f34, f35 = [AOFFSET], 2 * SIZE
(p3) FMA f68 = f44, f56, f68 // A5 * B1
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f13 = [C9 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f69 = f45, f56, f69 // A6 * B1
nop __LINE__
}
;;
{ .mfi
(p4) LDFPD f36, f37 = [AOFFSET], 2 * SIZE
(p3) FMA f70 = f46, f56, f70 // A7 * B1
adds L = -1, L
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f14 = [C1 ], -3 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f71 = f47, f56, f71 // A8 * B1
nop __LINE__
}
;;
{ .mfb
(p4) LDFPD f38, f39 = [AOFFSET], 2 * SIZE
nop __LINE__
nop __LINE__
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f15 = [C9 ], -3 * SIZE
#else
nop __LINE__
#endif
nop __LINE__
br.cloop.sptk.few .L133
}
;;
.L138:
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
{ .mfi
FMA f64 = ALPHA, f64, f6
cmp.ne p6, p0 = 1, I
}
{ .mfb
FMA f68 = ALPHA, f68, f7
}
;;
{ .mfi
FMA f65 = ALPHA, f65, f10
adds I = -1, I
}
{ .mfb
FMA f69 = ALPHA, f69, f11
}
;;
{ .mfi
FMA f66 = ALPHA, f66, f12
}
{ .mfb
FMA f70 = ALPHA, f70, f13
}
;;
{ .mfb
FMA f67 = ALPHA, f67, f14
}
{ .mfb
FMA f71 = ALPHA, f71, f15
}
;;
{ .mmf
STFD [C1 ] = f64, SIZE
STFD [C9 ] = f68, SIZE
mov f64 = f0
}
;;
{ .mmf
STFD [C1 ] = f65, SIZE
STFD [C9 ] = f69, SIZE
mov f65 = f0
}
;;
{ .mmf
STFD [C1 ] = f66, SIZE
STFD [C9 ] = f70, SIZE
mov f66 = f0
}
;;
{ .mmf
STFD [C1 ] = f67, 5 * SIZE
nop __LINE__
mov f67 = f0
}
{ .mmb
STFD [C9 ] = f71, 5 * SIZE
nop __LINE__
(p6) br.cond.dptk .L132
}
;;
#else
{ .mfi
FMPY f64 = ALPHA, f64
cmp.ne p6, p0 = 1, I
}
{ .mfb
FMPY f68 = ALPHA, f68
}
;;
{ .mfi
FMPY f65 = ALPHA, f65
adds I = -1, I
}
{ .mfb
FMPY f69 = ALPHA, f69
}
;;
{ .mfi
FMPY f66 = ALPHA, f66
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
sub L = K, KK
#else
nop __LINE__
#endif
}
{ .mfb
FMPY f70 = ALPHA, f70
}
;;
{ .mfi
FMPY f67 = ALPHA, f67
#if defined(TRMMKERNEL) && (defined(LEFT) && defined(TRANSA))
adds L = -8, L
#else
nop __LINE__
#endif
}
{ .mfi
FMPY f71 = ALPHA, f71
#if defined(TRMMKERNEL) && (!defined(LEFT) && !defined(TRANSA))
adds L = -1, L
#else
nop __LINE__
#endif
}
;;
{ .mfi
STFD [C1 ] = f64, SIZE
mov f64 = f0
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd KK8 = L, BASE_SHIFT, r0
#else
nop __LINE__
#endif
}
{ .mmi
STFD [C9 ] = f68, SIZE
nop __LINE__
nop __LINE__
}
;;
{ .mfi
STFD [C1 ] = f65, SIZE
mov f65 = f0
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd AOFFSET = KK8, 3, AOFFSET
#else
nop __LINE__
#endif
}
{ .mmi
STFD [C9 ] = f69, SIZE
nop __LINE__
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
add BOFFSET = KK8, BOFFSET
#else
nop __LINE__
#endif
}
;;
{ .mfi
STFD [C1 ] = f66, SIZE
mov f66 = f0
#if defined(TRMMKERNEL) && defined(LEFT)
adds KK = 8, KK
#else
nop __LINE__
#endif
}
{ .mmi
STFD [C9 ] = f70, SIZE
nop __LINE__
nop __LINE__
}
;;
{ .mfi
STFD [C1 ] = f67, 5 * SIZE
mov f67 = f0
#ifdef TRMMKERNEL
shladd KK8 = KK, BASE_SHIFT, r0
#else
nop __LINE__
#endif
}
{ .mmb
STFD [C9 ] = f71, 5 * SIZE
nop __LINE__
(p6) br.cond.dptk .L132
}
;;
#endif
.align 32
.L140:
{ .mib
#ifndef TRMMKERNEL
nop __LINE__
#else
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub L = K, KK
#elif defined(LEFT)
adds L = 4, KK
#else
adds L = 1, KK
#endif
#endif
tbit.z p6, p7 = M, 2
(p6) br.cond.dptk .L150
}
;;
#if !defined(TRMMKERNEL) || \
defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
{ .mmi
LDFD f48 = [B]
adds BOFFSET = 1 * SIZE, B
#ifndef TRMMKERNEL
adds L = 1, K
#else
adds L = 1, L
#endif
}
;;
#else
{ .mmi
add BOFFSET = KK8, B
shladd AOFFSET = KK8, 2, AOFFSET
nop __LINE__
}
;;
{ .mmi
LDFD f48 = [BOFFSET], 1 * SIZE
nop __LINE__
#ifndef TRMMKERNEL
adds L = 1, K
#else
adds L = 1, L
#endif
}
;;
#endif
{ .mii
(p7) LDFPD f32, f33 = [AOFFSET], 2 * SIZE
tbit.z p12, p0 = L, 0
shr L = L, 1
}
;;
{ .mmi
LDFPD f34, f35 = [AOFFSET], 2 * SIZE
adds L = -1, L
nop __LINE__
}
;;
{ .mmi
adds PREA = (PREFETCHSIZE + 0) * SIZE, AOFFSET
cmp.eq p3, p0 = r0, r0
mov ar.lc = L
}
;;
.align 32
.L142:
{ .mfi
lfetch.nt1 [PREA], 8 * SIZE
FMA f64 = f32, f48, f64 // A1 * B1
cmp.ne p4, p5 = 0, L
}
{ .mfi
nop __LINE__
FMA f65 = f33, f48, f65 // A2 * B1
(p12) cmp.ne p3, p0 = 0, L
}
;;
{ .mfi
(p3) LDFPD f40, f41 = [AOFFSET], 2 * SIZE
FMA f66 = f34, f48, f66 // A3 * B1
(p5) adds C9 = 2 * SIZE, C1
}
{ .mmf
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f68 = [C1 ], SIZE
#else
nop __LINE__
#endif
(p3) LDFD f56 = [BOFFSET], 1 * SIZE
FMA f67 = f35, f48, f67 // A4 * B1
}
;;
{ .mfi
(p3) LDFPD f42, f43 = [AOFFSET], 2 * SIZE
(p3) FMA f64 = f40, f56, f64 // A1 * B1
(p5) adds C10 = 2 * SIZE, C2
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f70 = [C9 ], SIZE
#else
nop __LINE__
#endif
(p3) FMA f65 = f41, f56, f65 // A2 * B1
nop __LINE__
}
;;
{ .mfb
(p4) LDFPD f32, f33 = [AOFFSET], 2 * SIZE
(p3) FMA f66 = f42, f56, f66 // A3 * B1
nop __LINE__
}
{ .mmf
(p4) LDFD f48 = [BOFFSET], 1 * SIZE
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f69 = [C1 ], -1 * SIZE
#else
nop __LINE__
#endif
(p3) FMA f67 = f43, f56, f67 // A4 * B1
}
;;
{ .mfi
(p4) LDFPD f34, f35 = [AOFFSET], 2 * SIZE
nop __LINE__
adds L = -1, L
}
{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f71 = [C9 ], -1 * SIZE
#else
nop __LINE__
#endif
nop.f 0
br.cloop.sptk.few .L142
}
;;
.L148:
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
FMA f64 = ALPHA, f64, f68
FMA f66 = ALPHA, f66, f70
FMA f65 = ALPHA, f65, f69
FMA f67 = ALPHA, f67, f71
;;
{ .mfi
STFD [C1 ] = f64, SIZE
mov f64 = f0
adds L = 1, K
}
{ .mfb
STFD [C9 ] = f66, SIZE
mov f66 = f0
nop __LINE__
}
;;
{ .mfi
STFD [C1 ] = f65, 3 * SIZE
mov f65 = f0
shr L = L, 1
}
{ .mfb
STFD [C9 ] = f67, 3 * SIZE
mov f67 = f0
nop __LINE__
}
;;
#else
{ .mfi
nop __LINE__
FMPY f64 = ALPHA, f64
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
sub L = K, KK
#else
nop __LINE__
#endif
}
{ .mfi
nop __LINE__
FMPY f66 = ALPHA, f66
nop __LINE__
}
;;
{ .mfi
nop __LINE__
FMPY f65 = ALPHA, f65
#if defined(TRMMKERNEL) && (defined(LEFT) && defined(TRANSA))
adds L = -4, L
#else
nop __LINE__
#endif
}
{ .mfi
nop __LINE__
FMPY f67 = ALPHA, f67
#if defined(TRMMKERNEL) && (!defined(LEFT) && !defined(TRANSA))
adds L = -1, L
#else
nop __LINE__
#endif
}
;;
{ .mmi
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd KK8 = L, BASE_SHIFT, r0
#else
nop __LINE__
#endif
;;
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd AOFFSET = KK8, 2, AOFFSET
#else
nop __LINE__
#endif
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
add BOFFSET = KK8, BOFFSET
#else
nop __LINE__
#endif
}
;;
{ .mfi
STFD [C1 ] = f64, SIZE
mov f64 = f0
#if defined(TRMMKERNEL) && defined(LEFT)
adds KK = 4, KK
#else
nop __LINE__
#endif
}
{ .mfb
STFD [C9 ] = f66, SIZE
mov f66 = f0
nop __LINE__
}
;;
{ .mfi
STFD [C1 ] = f65, 3 * SIZE
mov f65 = f0
#ifdef TRMMKERNEL
shladd KK8 = KK, BASE_SHIFT, r0
#else
nop __LINE__
#endif
}
{ .mfb
STFD [C9 ] = f67, 3 * SIZE
mov f67 = f0
nop __LINE__
}
;;
#endif
.align 32
.L150:
{ .mib
#ifndef TRMMKERNEL
nop __LINE__
#else
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub L = K, KK
#elif defined(LEFT)
adds L = 2, KK
#else
adds L = 1, KK
#endif
#endif
tbit.z p6, p7 = M, 1
(p6) br.cond.dptk .L160
}
;;
#if !defined(TRMMKERNEL) || \
defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
{ .mmi
LDFD f48 = [B]
adds BOFFSET = 1 * SIZE, B
#ifndef TRMMKERNEL
adds L = 1, K
#else
adds L = 1, L
#endif
}
;;
#else
{ .mmi
add BOFFSET = KK8, B
shladd AOFFSET = KK8, 1, AOFFSET
nop __LINE__
}
;;
{ .mmi
LDFD f48 = [BOFFSET], 1 * SIZE
nop __LINE__
#ifndef TRMMKERNEL
adds L = 1, K
#else
adds L = 1, L
#endif
}
;;
#endif
{ .mii
cmp.eq p3, p0 = r0, r0
tbit.z p12, p0 = L, 0
shr L = L, 1
}
;;
{ .mii
LDFPD f32, f33 = [AOFFSET], 2 * SIZE
adds L = -1, L
;;
mov ar.lc = L
}
;;
.align 32
.L152:
{ .mfi
cmp.ne p4, p5 = 0, L
FMA f64 = f32, f48, f64 // A1 * B1
(p12) cmp.ne p3, p0 = 0, L
}
;;
{ .mmf
(p3) LDFD f56 = [BOFFSET], 1 * SIZE
(p3) LDFPD f40, f41 = [AOFFSET], 2 * SIZE
FMA f65 = f33, f48, f65 // A2 * B1
}
;;
{ .mfi
(p4) LDFPD f32, f33 = [AOFFSET], 2 * SIZE
(p3) FMA f64 = f40, f56, f64 // A1 * B1
adds L = -1, L
}
;;
{ .mfb
(p4) LDFD f48 = [BOFFSET], 1 * SIZE
(p3) FMA f65 = f41, f56, f65 // A2 * B1
br.cloop.sptk.few .L152
}
;;
.L158:
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
LDFD f68 = [C1 ], SIZE
;;
LDFD f69 = [C1 ], -1 * SIZE
;;
FMA f64 = ALPHA, f64, f68
FMA f65 = ALPHA, f65, f69
;;
STFD [C1 ] = f64, SIZE
mov f64 = f0
;;
{ .mfi
STFD [C1 ] = f65, SIZE
mov f65 = f0
}
;;
#else
{ .mfi
nop __LINE__
FMPY f64 = ALPHA, f64
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
sub L = K, KK
#else
nop __LINE__
#endif
}
{ .mfi
nop __LINE__
FMPY f65 = ALPHA, f65
nop __LINE__
}
;;
{ .mii
nop __LINE__
#if defined(TRMMKERNEL) && (defined(LEFT) && defined(TRANSA))
adds L = -2, L
#else
nop __LINE__
#endif
#if defined(TRMMKERNEL) && (!defined(LEFT) && !defined(TRANSA))
adds L = -1, L
#else
nop __LINE__
#endif
}
;;
{ .mmi
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd KK8 = L, BASE_SHIFT, r0
#else
nop __LINE__
#endif
;;
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
shladd AOFFSET = KK8, 1, AOFFSET
#else
nop __LINE__
#endif
#if defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
add BOFFSET = KK8, BOFFSET
#else
nop __LINE__
#endif
}
;;
{ .mfi
STFD [C1 ] = f64, SIZE
mov f64 = f0
#if defined(TRMMKERNEL) && defined(LEFT)
adds KK = 2, KK
#else
nop __LINE__
#endif
}
;;
{ .mfi
STFD [C1 ] = f65, SIZE
mov f65 = f0
#ifdef TRMMKERNEL
shladd KK8 = KK, BASE_SHIFT, r0
#else
nop __LINE__
#endif
}
;;
#endif
.align 32
.L160:
{ .mib
#ifndef TRMMKERNEL
nop __LINE__
#else
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub L = K, KK
#elif defined(LEFT)
adds L = 1, KK
#else
adds L = 1, KK
#endif
#endif
tbit.z p6, p7 = M, 0
(p6) br.cond.dptk .L169
}
;;
#if !defined(TRMMKERNEL) || \
defined(TRMMKERNEL) && \
((defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
{ .mmi
LDFD f48 = [B]
adds BOFFSET = 1 * SIZE, B
#ifndef TRMMKERNEL
adds L = 1, K
#else
adds L = 1, L
#endif
}
;;
#else
{ .mmi
add BOFFSET = KK8, B
add AOFFSET = KK8, AOFFSET
nop __LINE__
}
;;
{ .mmi
LDFD f48 = [BOFFSET], 1 * SIZE
nop __LINE__
#ifndef TRMMKERNEL
adds L = 1, K
#else
adds L = 1, L
#endif
}
;;
#endif
;;
{ .mii
LDFD f32 = [AOFFSET], 1 * SIZE
tbit.z p12, p0 = L, 0
shr L = L, 1
}
;;
{ .mii
adds L = -1, L
cmp.eq p3, p0 = r0, r0
;;
mov ar.lc = L
}
;;
.align 32
.L162:
{ .mmf
cmp.ne p4, p5 = 0, L
(p12) cmp.ne p3, p0 = 0, L
FMA f64 = f32, f48, f64 // A1 * B1
}
;;
{ .mmi
(p3) LDFD f56 = [BOFFSET], 1 * SIZE
(p3) LDFD f40 = [AOFFSET], 1 * SIZE
nop __LINE__
}
;;
{ .mmi
(p4) LDFD f32 = [AOFFSET], 1 * SIZE
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
(p5) LDFD f68 = [C1]
#else
nop __LINE__
#endif
adds L = -1, L
}
{ .mfb
(p4) LDFD f48 = [BOFFSET], 1 * SIZE
(p3) FMA f64 = f40, f56, f64 // A1 * B1
br.cloop.sptk.few .L162
}
;;
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
FMA f64 = ALPHA, f64, f68
#else
FMPY f64 = ALPHA, f64
#endif
;;
STFD [C1 ] = f64
;;
.align 32
.L169:
{ .mmi
mov B = BOFFSET
mov AOFFSET = A
#if defined(TRMMKERNEL) && !defined(LEFT)
adds KK = 1, KK
#else
nop __LINE__
#endif
}
;;
.align 16
#endif
.L999:
mov r8 = r0
adds r9 = 1 * 16, SP
;;
ldf.fill f16 = [SP], 32
ldf.fill f17 = [r9], 32
;;
ldf.fill f18 = [SP], 32
ldf.fill f19 = [r9], 32
;;
ldf.fill f20 = [SP], 32
ldf.fill f21 = [r9], 32
;;
ldf.fill f22 = [SP], 32
ldf.fill f23 = [r9], 32
mov ar.lc = ARLC
;;
ldf.fill f24 = [SP], 32
ldf.fill f25 = [r9], 32
mov pr = PR, -1
;;
ldf.fill f26 = [SP], 32
ldf.fill f27 = [r9], 32
mov ar.pfs = ARPFS
;;
ldf.fill f28 = [SP], 32
ldf.fill f29 = [r9], 32
;;
ldf.fill f30 = [SP], 32
ldf.fill f31 = [r9]
br.ret.sptk.many b0
EPILOGUE