aboutsummaryrefslogtreecommitdiff
path: root/tools/server/test_schedule.c
blob: 26976ee64255fecd42b958c4dbfe5dce17038060 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
/* test_schedule.c - job queue micro-batching: the batch-wait deadline and the
 * zero-wait fast path through enqueue_job / collect_job_batch. */

#include "server_internal.h"
#include "test_util.h"
#include "util.h" /* nstime */

#include <pthread.h>
#include <stdio.h>
#include <string.h>

static void test_collect_job_batch_deadline(void) {
    http_server s;
    memset(&s, 0, sizeof(s));
    s.batch_size = 32;
    s.batch_wait_us = 1000;
    pthread_mutex_init(&s.mu, NULL);
    pthread_cond_init(&s.cv, NULL);

    uint64_t first_ns = nstime() - 5000000u;
    job first = {.srv = &s, .created_ns = first_ns};
    job before_deadline = {.srv = &s, .created_ns = first_ns + 500000u};
    job after_deadline = {.srv = &s, .created_ns = first_ns + 2000000u};
    snprintf(first.path, sizeof(first.path), "%s", "/v1/embeddings");
    snprintf(before_deadline.path, sizeof(before_deadline.path), "%s", "/v1/embeddings");
    snprintf(after_deadline.path, sizeof(after_deadline.path), "%s", "/v1/embeddings");
    enqueue_job(&first);
    enqueue_job(&before_deadline);
    enqueue_job(&after_deadline);

    job *batch[3] = {0};
    TEST_ASSERT(collect_job_batch(&s, batch, 3) == 2);
    TEST_ASSERT(batch[0] == &first);
    TEST_ASSERT(batch[1] == &before_deadline);
    TEST_ASSERT(s.job_head == &after_deadline);
    TEST_ASSERT(s.job_tail == &after_deadline);

    s.job_head = NULL;
    s.job_tail = NULL;
    pthread_cond_destroy(&s.cv);
    pthread_mutex_destroy(&s.mu);
}

static void test_collect_job_batch_zero_wait(void) {
    http_server s;
    memset(&s, 0, sizeof(s));
    s.batch_size = 32;
    pthread_mutex_init(&s.mu, NULL);
    pthread_cond_init(&s.cv, NULL);

    job first = {.srv = &s, .created_ns = nstime()};
    job second = {.srv = &s, .created_ns = nstime()};
    snprintf(first.path, sizeof(first.path), "%s", "/v1/embeddings");
    snprintf(second.path, sizeof(second.path), "%s", "/v1/embeddings");
    enqueue_job(&first);
    enqueue_job(&second);

    job *batch[2] = {0};
    TEST_ASSERT(collect_job_batch(&s, batch, 2) == 2);
    TEST_ASSERT(batch[0] == &first);
    TEST_ASSERT(batch[1] == &second);
    TEST_ASSERT(s.job_head == NULL);
    TEST_ASSERT(s.job_tail == NULL);

    pthread_cond_destroy(&s.cv);
    pthread_mutex_destroy(&s.mu);
}

static void test_collect_job_batch_token_budget(void) {
    http_server s;
    memset(&s, 0, sizeof(s));
    s.batch_size = 32;
    s.max_batch_tokens = 1000;
    pthread_mutex_init(&s.mu, NULL);
    pthread_cond_init(&s.cv, NULL);

    /* 600 + 300 fit the 1000-token window; the 200-token job must wait for
     * the next window. */
    embedding_request r1 = {.total_tokens = 600};
    embedding_request r2 = {.total_tokens = 300};
    embedding_request r3 = {.total_tokens = 200};
    job a = {.srv = &s, .tokenized = 1, .prep_kind = 1, .prep = &r1};
    job b = {.srv = &s, .tokenized = 1, .prep_kind = 1, .prep = &r2};
    job c = {.srv = &s, .tokenized = 1, .prep_kind = 1, .prep = &r3};
    snprintf(a.path, sizeof(a.path), "%s", "/v1/embeddings");
    snprintf(b.path, sizeof(b.path), "%s", "/v1/embeddings");
    snprintf(c.path, sizeof(c.path), "%s", "/v1/embeddings");
    enqueue_job(&a);
    enqueue_job(&b);
    enqueue_job(&c);

    job *batch[3] = {0};
    TEST_ASSERT(collect_job_batch(&s, batch, 3) == 2);
    TEST_ASSERT(batch[0] == &a);
    TEST_ASSERT(batch[1] == &b);
    TEST_ASSERT(s.job_head == &c);

    s.job_head = NULL;
    s.job_tail = NULL;
    pthread_cond_destroy(&s.cv);
    pthread_mutex_destroy(&s.mu);
}

int main(void) {
    test_collect_job_batch_deadline();
    test_collect_job_batch_zero_wait();
    test_collect_job_batch_token_budget();
    return TEST_REPORT("schedule");
}