aboutsummaryrefslogtreecommitdiff
path: root/src/kernels.h
diff options
context:
space:
mode:
Diffstat (limited to 'src/kernels.h')
-rw-r--r--src/kernels.h137
1 files changed, 67 insertions, 70 deletions
diff --git a/src/kernels.h b/src/kernels.h
index 6f180dd..17c45c0 100644
--- a/src/kernels.h
+++ b/src/kernels.h
@@ -49,27 +49,27 @@ void embed_linear_nobias_bf16(
/* Small-sequence BF16 path: compute two projections with one threaded dispatch. */
void embed_linear_nobias_bf16_pair(float *a,
- float *b,
- const float *x,
- const uint16_t *Wa_bf16,
- const uint16_t *Wb_bf16,
- int seq_len,
- int in_dim,
- int a_dim,
- int b_dim);
+ float *b,
+ const float *x,
+ const uint16_t *Wa_bf16,
+ const uint16_t *Wb_bf16,
+ int seq_len,
+ int in_dim,
+ int a_dim,
+ int b_dim);
/* Small-sequence BF16 path: compute Q/K/V matvecs with one threaded dispatch. */
void embed_linear_nobias_bf16_qkv(float *q,
- float *k,
- float *v,
- const float *x,
- const uint16_t *Wq_bf16,
- const uint16_t *Wk_bf16,
- const uint16_t *Wv_bf16,
- int seq_len,
- int in_dim,
- int q_dim,
- int kv_dim);
+ float *k,
+ float *v,
+ const float *x,
+ const uint16_t *Wq_bf16,
+ const uint16_t *Wk_bf16,
+ const uint16_t *Wv_bf16,
+ int seq_len,
+ int in_dim,
+ int q_dim,
+ int kv_dim);
/* Dot product using the best available local SIMD implementation. */
float embed_dot_f32(const float *a, const float *b, int n);
@@ -92,12 +92,12 @@ void embed_rms_norm_per_head(
* + beta, with mean and biased (population) variance over the hidden axis per
* row. Unlike RMSNorm this subtracts the mean and adds a bias. */
void embed_layer_norm(float *out,
- const float *x,
- const float *gamma,
- const float *beta,
- int seq_len,
- int hidden,
- float eps);
+ const float *x,
+ const float *gamma,
+ const float *beta,
+ int seq_len,
+ int hidden,
+ float eps);
/* ========================================================================
* Activation Functions
@@ -130,57 +130,57 @@ void embed_gelu_tanh_inplace(float *x, int n);
* offsets[b]..offsets[b+1] span.
*/
void embed_bidirectional_gqa_attention_packed(float *out,
- const float *Q,
- const float *K,
- const float *V,
- const int *offsets,
- int batch,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale);
+ const float *Q,
+ const float *K,
+ const float *V,
+ const int *offsets,
+ int batch,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale);
/* Optional reusable scratch for the BLAS-tiled packed attention path. */
size_t embed_bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch);
void embed_bidirectional_gqa_attention_packed_with_scratch(float *out,
- const float *Q,
- const float *K,
- const float *V,
- const int *offsets,
- int batch,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- float *scratch,
- size_t scratch_bytes);
+ const float *Q,
+ const float *K,
+ const float *V,
+ const int *offsets,
+ int batch,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale,
+ float *scratch,
+ size_t scratch_bytes);
/*
* Causal GQA attention over a packed/ragged batch. Query position i attends
* only to positions at or before i within its sequence.
*/
void embed_causal_gqa_attention_packed(float *out,
- const float *Q,
- const float *K,
- const float *V,
- const int *offsets,
- int batch,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale);
+ const float *Q,
+ const float *K,
+ const float *V,
+ const int *offsets,
+ int batch,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale);
void embed_causal_gqa_attention_packed_with_scratch(float *out,
- const float *Q,
- const float *K,
- const float *V,
- const int *offsets,
- int batch,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- float *scratch,
- size_t scratch_bytes);
+ const float *Q,
+ const float *K,
+ const float *V,
+ const int *offsets,
+ int batch,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale,
+ float *scratch,
+ size_t scratch_bytes);
/* ========================================================================
* Position Embeddings
@@ -206,12 +206,9 @@ void embed_apply_rope_neox(
* Threading
* ======================================================================== */
-/* Set number of threads for parallel operations (default: 1).
- * Creates a persistent thread pool. Call before inference. */
-EMBED_API void embed_set_threads(int n);
-
-/* Get number of available CPU cores */
-EMBED_API int embed_get_num_cpus(void);
+/* The persistent worker pool lives in threadpool.{c,h}; included here so kernel
+ * callers keep getting embed_set_threads / embed_get_num_cpus from one header. */
+#include "threadpool.h"
/* Global verbose flag */
EMBED_API extern int embed_verbose;