aboutsummaryrefslogtreecommitdiff
path: root/tools/server/server.h
blob: b8ec7e4cda2dc44bd97bc9810f90e96866fb59cf (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
#ifndef FFWD_SERVER_H
#define FFWD_SERVER_H

/* Public-API export annotation. The libraries are built with
 * -fvisibility=hidden, so only declarations carrying FFWD_API are exported
 * from the shared library; everything else stays internal. */
#ifndef FFWD_API
#    if defined(__GNUC__)
#        define FFWD_API __attribute__((visibility("default")))
#    else
#        define FFWD_API
#    endif
#endif

#include <stdbool.h>

typedef enum {
    FFWD_SERVER_MODEL_STANDARD = 0,
    FFWD_SERVER_MODEL_CONTEXTUAL = 1,
    FFWD_SERVER_MODEL_LATE = 2,
} ffwd_server_model_kind_t;

typedef enum {
    FFWD_SERVER_API_DEFAULT = 0, /* currently OpenAI-compatible float output */
    FFWD_SERVER_API_OPENAI = 1,
    FFWD_SERVER_API_PERPLEXITY = 2,
} ffwd_server_embedding_api_t;

typedef struct {
    const char *id; /* served label; requests match this string exactly */
    const char *path;
    ffwd_server_model_kind_t kind;
    ffwd_server_embedding_api_t api;
    int min_dim;                /* 0 means the loaded model's full hidden size */
    const char *query_instruct; /* NULL means use config_sentence_transformers.json if present */
} ffwd_server_model_spec_t;

typedef struct {
    const ffwd_server_model_spec_t *models;
    int n_models;
    const char *host;
    int port;
    int batch_size;
    int max_batch_tokens;
    int batch_wait_us;
    /* GPU build tuning; ignored by backends that do not use it. */
    int gpu_quantize_bits;
    int gpu_quantize_group_size;
    double memory_utilization; /* fraction of physical memory the GPU preflight
                                  may plan for; 0 = default 0.90, above 1.0
                                  allows overcommit */
    const char *api_key;
} ffwd_server_config_t;

FFWD_API int ffwd_run_server(const ffwd_server_config_t *cfg);

#endif