1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
|
/* The dense primitive against a naive triple loop, over the shapes the model
* actually uses plus enough odd ones to exercise every edge path.
*
* The reference here is deliberately the most obvious implementation possible:
* the point is to catch a blocking or tail-handling mistake, and a second
* clever implementation would be liable to the same mistakes. */
#include "kernels.h"
#include "test_util.h"
#include <math.h>
#include <stdlib.h>
static unsigned rng_state = 12345u;
static float next_float(void) {
rng_state = rng_state * 1664525u + 1013904223u;
return (float)((rng_state >> 8) & 0xFFFF) / 32768.0f - 1.0f;
}
static void gemm_ref(float *y,
int ldy,
const float *a,
int lda,
const float *w,
const float *bias,
int m,
int n,
int k) {
for (int s = 0; s < m; ++s)
for (int i = 0; i < n; ++i) {
double acc = bias ? bias[i] : 0.0;
for (int j = 0; j < k; ++j)
acc += (double)a[(size_t)s * lda + j] * w[(size_t)i * k + j];
y[(size_t)s * ldy + i] = (float)acc;
}
}
/* Tolerance scales with the window length because that is how many terms the
* two orders can disagree over. */
static int
check(const char *label, const pyseg_kernels *kern, int m, int n, int k, int lda, int with_bias) {
const size_t a_len = (size_t)(m - 1) * lda + k + 8;
float *a = malloc(a_len * sizeof(float));
float *w = malloc((size_t)n * k * sizeof(float));
float *bias = malloc((size_t)n * sizeof(float));
float *got = malloc((size_t)m * n * sizeof(float));
float *want = malloc((size_t)m * n * sizeof(float));
for (size_t i = 0; i < a_len; ++i)
a[i] = next_float();
for (size_t i = 0; i < (size_t)n * k; ++i)
w[i] = next_float();
for (int i = 0; i < n; ++i)
bias[i] = next_float();
kern->gemm_nt(got, n, a, lda, w, with_bias ? bias : NULL, m, n, k);
gemm_ref(want, n, a, lda, w, with_bias ? bias : NULL, m, n, k);
double worst = 0.0;
for (int i = 0; i < m * n; ++i) {
const double d = fabs((double)got[i] - want[i]);
if (d > worst)
worst = d;
}
const double limit = 2e-6 * k;
TEST_ASSERT(worst <= limit, "%s %s m=%d n=%d k=%d lda=%d bias=%d: worst %.3e > %.3e", label,
kern->name, m, n, k, lda, with_bias, worst, limit);
free(a);
free(w);
free(bias);
free(got);
free(want);
return 0;
}
/* The SIMD cell update replaces libm's exponential and hyperbolic tangent with
* polynomial approximations, so it is checked against libm directly rather
* than only through the model. The gate arguments are swept well past the
* range where both saturate, because that range is where a range reduction
* built on assembling an exponent by hand goes wrong. */
static void check_lstm_cell(void) {
enum { HID = 128 };
float rec[4 * HID], in[4 * HID];
float h_scalar[HID], c_scalar[HID], h_simd[HID], c_simd[HID];
for (int trial = 0; trial < 64; ++trial) {
const float span = 1.0f + (float)trial * 3.0f; /* up to about 190 */
for (int i = 0; i < 4 * HID; ++i) {
rec[i] = next_float() * span;
in[i] = next_float() * span;
}
for (int i = 0; i < HID; ++i)
c_scalar[i] = c_simd[i] = next_float() * 8.0f;
pyseg_kernels_scalar()->lstm_cell(h_scalar, c_scalar, rec, in, HID);
pyseg_kernels_simd()->lstm_cell(h_simd, c_simd, rec, in, HID);
double worst = 0.0;
for (int i = 0; i < HID; ++i) {
const double dh = fabs((double)h_scalar[i] - h_simd[i]);
const double dc = fabs((double)c_scalar[i] - c_simd[i]);
if (dh > worst)
worst = dh;
if (dc > worst)
worst = dc;
if (!isfinite(h_simd[i]) || !isfinite(c_simd[i]))
worst = 1.0;
}
TEST_ASSERT(worst <= 1e-6, "lstm_cell over +/-%.0f: worst %.3e", span, worst);
}
}
int main(void) {
const pyseg_kernels *sets[] = {pyseg_kernels_scalar(), pyseg_kernels_simd()};
/* The shapes the forward pass issues, in order. */
static const struct {
const char *label;
int m, n, k, lda, bias;
} model_shapes[] = {
{"sinc", 384, 80, 256, 10, 0}, {"sinc-tail", 231, 80, 256, 10, 0},
{"conv2", 384, 60, 400, 80, 1}, {"conv3", 231, 60, 300, 60, 1},
{"lstm-in-l0", 589, 512, 60, 60, 1}, {"lstm-in-l1", 589, 512, 256, 256, 1},
{"lstm-hh", 1, 512, 128, 0, 1}, {"linear0", 589, 128, 256, 256, 1},
{"linear1", 589, 128, 128, 128, 1}, {"classifier", 589, 7, 128, 128, 1},
};
for (size_t s = 0; s < sizeof sets / sizeof *sets; ++s)
for (size_t i = 0; i < sizeof model_shapes / sizeof *model_shapes; ++i)
check(model_shapes[i].label, sets[s], model_shapes[i].m, model_shapes[i].n,
model_shapes[i].k, model_shapes[i].lda ? model_shapes[i].lda : model_shapes[i].k,
model_shapes[i].bias);
/* Every combination of leftover rows and leftover weight rows. */
for (size_t s = 0; s < sizeof sets / sizeof *sets; ++s)
for (int m = 1; m <= 9; ++m)
for (int n = 1; n <= 13; ++n)
for (int k = 4; k <= 20; k += 4)
check("edge", sets[s], m, n, k, k + 3, m & 1);
/* Panels: a shape larger than one panel and one that straddles a boundary. */
for (size_t s = 0; s < sizeof sets / sizeof *sets; ++s) {
check("panel-exact", sets[s], 2 * PYSEG_MPANEL, 8, 16, 16, 1);
check("panel-split", sets[s], 2 * PYSEG_MPANEL + 3, 9, 12, 17, 1);
}
check_lstm_cell();
return TEST_REPORT();
}
|