From ab50d908ba2eae14929a07341b1b716cde681c1c Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Sat, 30 May 2026 04:07:37 +0200 Subject: server: implement HTTP API with libae integration --- Makefile | 31 +- deps/ae/ae.c | 513 +++++++++++++++++ deps/ae/ae.h | 118 ++++ deps/ae/ae_epoll.c | 144 +++++ deps/ae/ae_kqueue.c | 183 ++++++ deps/ae/ae_select.c | 108 ++++ deps/ae/anet.c | 812 ++++++++++++++++++++++++++ deps/ae/anet.h | 58 ++ deps/ae/config.h | 375 ++++++++++++ deps/ae/fmacros.h | 56 ++ deps/ae/monotonic.c | 247 ++++++++ deps/ae/monotonic.h | 61 ++ deps/ae/redisassert.h | 18 + deps/ae/util.h | 21 + deps/ae/zmalloc.h | 15 + main.c | 183 +++++- pplx_embed.c | 62 ++ pplx_embed.h | 25 + pplx_embed_mlx.c | 156 +++++ pplx_embed_mlx.h | 8 + pplx_server.c | 1524 +++++++++++++++++++++++++++++++++++++++++++++++++ pplx_server.h | 24 + 22 files changed, 4715 insertions(+), 27 deletions(-) create mode 100644 deps/ae/ae.c create mode 100644 deps/ae/ae.h create mode 100644 deps/ae/ae_epoll.c create mode 100644 deps/ae/ae_kqueue.c create mode 100644 deps/ae/ae_select.c create mode 100644 deps/ae/anet.c create mode 100644 deps/ae/anet.h create mode 100644 deps/ae/config.h create mode 100644 deps/ae/fmacros.h create mode 100644 deps/ae/monotonic.c create mode 100644 deps/ae/monotonic.h create mode 100644 deps/ae/redisassert.h create mode 100644 deps/ae/util.h create mode 100644 deps/ae/zmalloc.h create mode 100644 pplx_server.c create mode 100644 pplx_server.h diff --git a/Makefile b/Makefile index bb3815e..9c94cc4 100644 --- a/Makefile +++ b/Makefile @@ -1,17 +1,26 @@ CC = gcc CFLAGS_BASE = -Wall -Wextra -O3 -march=native -ffast-math LDFLAGS = -lm -lpthread +CJSON_CFLAGS ?= $(shell sh -c 'pkg-config --cflags libcjson 2>/dev/null') +CJSON_LDFLAGS ?= $(shell sh -c 'pkg-config --libs libcjson 2>/dev/null') +ifeq ($(strip $(CJSON_LDFLAGS)),) + CJSON_LDFLAGS = -lcjson +endif UNAME_S := $(shell uname -s) # Source files SRCS = pplx_embed.c \ + pplx_server.c \ qwen_asr_kernels.c \ qwen_asr_kernels_generic.c \ qwen_asr_kernels_neon.c \ qwen_asr_kernels_avx.c \ qwen_asr_tokenizer.c \ - qwen_asr_safetensors.c + qwen_asr_safetensors.c \ + deps/ae/ae.c \ + deps/ae/anet.c \ + deps/ae/monotonic.c MLX_SRCS = pplx_embed_mlx.c @@ -40,6 +49,12 @@ help: # BLAS build (Apple Accelerate on macOS, OpenBLAS on Linux) # ============================================================================= ifeq ($(UNAME_S),Darwin) +ifeq ($(strip $(CJSON_CFLAGS)),) + CJSON_CFLAGS = -I/opt/homebrew/include -I/usr/local/include +endif +ifeq ($(strip $(CJSON_LDFLAGS)),-lcjson) + CJSON_LDFLAGS = -L/opt/homebrew/lib -L/usr/local/lib -lcjson +endif blas: CFLAGS = $(CFLAGS_BASE) -DUSE_BLAS -DACCELERATE_NEW_LAPACK blas: LDFLAGS += -framework Accelerate else @@ -84,19 +99,22 @@ debug: # Link # ============================================================================= $(TARGET): $(OBJS) $(EXTRA_OBJS) main.o - $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) + $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) $(CJSON_LDFLAGS) # ============================================================================= # Compile rules # ============================================================================= pplx_embed.o: pplx_embed.c pplx_embed.h qwen_asr_kernels.h qwen_asr_safetensors.h - $(CC) $(CFLAGS) -c -o $@ $< + $(CC) $(CFLAGS) $(CJSON_CFLAGS) -Ideps/ae -c -o $@ $< + +pplx_server.o: pplx_server.c pplx_server.h pplx_embed.h pplx_embed_mlx.h qwen_asr_tokenizer.h deps/ae/ae.h deps/ae/anet.h + $(CC) $(CFLAGS) $(CJSON_CFLAGS) -Ideps/ae -c -o $@ $< pplx_embed_mlx.o: pplx_embed_mlx.c pplx_embed_mlx.h pplx_embed.h qwen_asr_safetensors.h $(CC) $(CFLAGS) -c -o $@ $< -main.o: main.c pplx_embed.h qwen_asr_kernels.h qwen_asr_tokenizer.h - $(CC) $(CFLAGS) -c -o $@ $< +main.o: main.c pplx_embed.h pplx_server.h qwen_asr_kernels.h qwen_asr_tokenizer.h + $(CC) $(CFLAGS) $(CJSON_CFLAGS) -Ideps/ae -c -o $@ $< qwen_asr_kernels.o: qwen_asr_kernels.c qwen_asr_kernels.h qwen_asr_kernels_impl.h $(CC) $(CFLAGS) -c -o $@ $< @@ -116,6 +134,9 @@ qwen_asr_tokenizer.o: qwen_asr_tokenizer.c qwen_asr_tokenizer.h qwen_asr_kernels qwen_asr_safetensors.o: qwen_asr_safetensors.c qwen_asr_safetensors.h $(CC) $(CFLAGS) -c -o $@ $< +deps/ae/%.o: deps/ae/%.c deps/ae/ae.h deps/ae/anet.h deps/ae/monotonic.h + $(CC) $(CFLAGS) -Ideps/ae -c -o $@ $< + # ============================================================================= clean: rm -f $(OBJS) pplx_embed_mlx.o main.o $(TARGET) diff --git a/deps/ae/ae.c b/deps/ae/ae.c new file mode 100644 index 0000000..c07b203 --- /dev/null +++ b/deps/ae/ae.c @@ -0,0 +1,513 @@ +/* A simple event-driven programming library. Originally I wrote this code + * for the Jim's event-loop (Jim is a Tcl interpreter) but later translated + * it in form of a library for easy reuse. + * + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of (a) the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ + +#include "ae.h" +#include "anet.h" +#include "redisassert.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "zmalloc.h" +#include "config.h" + +/* Include the best multiplexing layer supported by this system. + * The following should be ordered by performances, descending. */ +#ifdef HAVE_EVPORT +#include "ae_evport.c" +#else + #ifdef HAVE_EPOLL + #include "ae_epoll.c" + #else + #ifdef HAVE_KQUEUE + #include "ae_kqueue.c" + #else + #include "ae_select.c" + #endif + #endif +#endif + +#define INITIAL_EVENT 1024 +aeEventLoop *aeCreateEventLoop(int setsize) { + aeEventLoop *eventLoop; + int i; + + monotonicInit(); /* just in case the calling app didn't initialize */ + + if ((eventLoop = zmalloc(sizeof(*eventLoop))) == NULL) goto err; + eventLoop->nevents = setsize < INITIAL_EVENT ? setsize : INITIAL_EVENT; + eventLoop->events = zmalloc(sizeof(aeFileEvent)*eventLoop->nevents); + eventLoop->fired = zmalloc(sizeof(aeFiredEvent)*eventLoop->nevents); + if (eventLoop->events == NULL || eventLoop->fired == NULL) goto err; + eventLoop->setsize = setsize; + eventLoop->timeEventHead = NULL; + eventLoop->timeEventNextId = 0; + eventLoop->stop = 0; + eventLoop->maxfd = -1; + eventLoop->beforesleep = NULL; + eventLoop->aftersleep = NULL; + eventLoop->flags = 0; + memset(eventLoop->privdata, 0, sizeof(eventLoop->privdata)); + if (aeApiCreate(eventLoop) == -1) goto err; + /* Events with mask == AE_NONE are not set. So let's initialize the + * vector with it. */ + for (i = 0; i < eventLoop->nevents; i++) + eventLoop->events[i].mask = AE_NONE; + return eventLoop; + +err: + if (eventLoop) { + zfree(eventLoop->events); + zfree(eventLoop->fired); + zfree(eventLoop); + } + return NULL; +} + +/* Return the current set size. */ +int aeGetSetSize(aeEventLoop *eventLoop) { + return eventLoop->setsize; +} + +/* + * Tell the event processing to change the wait timeout as soon as possible. + * + * Note: it just means you turn on/off the global AE_DONT_WAIT. + */ +void aeSetDontWait(aeEventLoop *eventLoop, int noWait) { + if (noWait) + eventLoop->flags |= AE_DONT_WAIT; + else + eventLoop->flags &= ~AE_DONT_WAIT; +} + +/* Resize the maximum set size of the event loop. + * If the requested set size is smaller than the current set size, but + * there is already a file descriptor in use that is >= the requested + * set size minus one, AE_ERR is returned and the operation is not + * performed at all. + * + * Otherwise AE_OK is returned and the operation is successful. */ +int aeResizeSetSize(aeEventLoop *eventLoop, int setsize) { + if (setsize == eventLoop->setsize) return AE_OK; + if (eventLoop->maxfd >= setsize) return AE_ERR; + if (aeApiResize(eventLoop,setsize) == -1) return AE_ERR; + + eventLoop->setsize = setsize; + + /* If the current allocated space is larger than the requested size, + * we need to shrink it to the requested size. */ + if (setsize < eventLoop->nevents) { + eventLoop->events = zrealloc(eventLoop->events,sizeof(aeFileEvent)*setsize); + eventLoop->fired = zrealloc(eventLoop->fired,sizeof(aeFiredEvent)*setsize); + eventLoop->nevents = setsize; + } + return AE_OK; +} + +void aeDeleteEventLoop(aeEventLoop *eventLoop) { + aeApiFree(eventLoop); + zfree(eventLoop->events); + zfree(eventLoop->fired); + + /* Free the time events list. */ + aeTimeEvent *next_te, *te = eventLoop->timeEventHead; + while (te) { + next_te = te->next; + if (te->finalizerProc) + te->finalizerProc(eventLoop, te->clientData); + zfree(te); + te = next_te; + } + zfree(eventLoop); +} + +void aeStop(aeEventLoop *eventLoop) { + eventLoop->stop = 1; +} + +int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask, + aeFileProc *proc, void *clientData) +{ + if (fd >= eventLoop->setsize) { + errno = ERANGE; + return AE_ERR; + } + + /* Resize the events and fired arrays if the file + * descriptor exceeds the current number of events. */ + if (unlikely(fd >= eventLoop->nevents)) { + int newnevents = eventLoop->nevents; + newnevents = (newnevents * 2 > fd + 1) ? newnevents * 2 : fd + 1; + newnevents = (newnevents > eventLoop->setsize) ? eventLoop->setsize : newnevents; + eventLoop->events = zrealloc(eventLoop->events, sizeof(aeFileEvent) * newnevents); + eventLoop->fired = zrealloc(eventLoop->fired, sizeof(aeFiredEvent) * newnevents); + + /* Initialize new slots with an AE_NONE mask */ + for (int i = eventLoop->nevents; i < newnevents; i++) + eventLoop->events[i].mask = AE_NONE; + eventLoop->nevents = newnevents; + } + + aeFileEvent *fe = &eventLoop->events[fd]; + + if (aeApiAddEvent(eventLoop, fd, mask) == -1) + return AE_ERR; + fe->mask |= mask; + if (mask & AE_READABLE) fe->rfileProc = proc; + if (mask & AE_WRITABLE) fe->wfileProc = proc; + fe->clientData = clientData; + if (fd > eventLoop->maxfd) + eventLoop->maxfd = fd; + return AE_OK; +} + +void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask) +{ + if (fd >= eventLoop->setsize) return; + aeFileEvent *fe = &eventLoop->events[fd]; + if (fe->mask == AE_NONE) return; + + /* We want to always remove AE_BARRIER if set when AE_WRITABLE + * is removed. */ + if (mask & AE_WRITABLE) mask |= AE_BARRIER; + + aeApiDelEvent(eventLoop, fd, mask); + fe->mask = fe->mask & (~mask); + if (fd == eventLoop->maxfd && fe->mask == AE_NONE) { + /* Update the max fd */ + int j; + + for (j = eventLoop->maxfd-1; j >= 0; j--) + if (eventLoop->events[j].mask != AE_NONE) break; + eventLoop->maxfd = j; + } +} + +void *aeGetFileClientData(aeEventLoop *eventLoop, int fd) { + if (fd >= eventLoop->setsize) return NULL; + aeFileEvent *fe = &eventLoop->events[fd]; + if (fe->mask == AE_NONE) return NULL; + + return fe->clientData; +} + +int aeGetFileEvents(aeEventLoop *eventLoop, int fd) { + if (fd >= eventLoop->setsize) return 0; + aeFileEvent *fe = &eventLoop->events[fd]; + + return fe->mask; +} + +long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds, + aeTimeProc *proc, void *clientData, + aeEventFinalizerProc *finalizerProc) +{ + long long id = eventLoop->timeEventNextId++; + aeTimeEvent *te; + + te = zmalloc(sizeof(*te)); + if (te == NULL) return AE_ERR; + te->id = id; + te->when = getMonotonicUs() + milliseconds * 1000; + te->timeProc = proc; + te->finalizerProc = finalizerProc; + te->clientData = clientData; + te->prev = NULL; + te->next = eventLoop->timeEventHead; + te->refcount = 0; + if (te->next) + te->next->prev = te; + eventLoop->timeEventHead = te; + return id; +} + +int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id) +{ + aeTimeEvent *te = eventLoop->timeEventHead; + while(te) { + if (te->id == id) { + te->id = AE_DELETED_EVENT_ID; + return AE_OK; + } + te = te->next; + } + return AE_ERR; /* NO event with the specified ID found */ +} + +/* How many microseconds until the first timer should fire. + * If there are no timers, -1 is returned. + * + * Note that's O(N) since time events are unsorted. + * Possible optimizations (not needed by Redis so far, but...): + * 1) Insert the event in order, so that the nearest is just the head. + * Much better but still insertion or deletion of timers is O(N). + * 2) Use a skiplist to have this operation as O(1) and insertion as O(log(N)). + */ +static int64_t usUntilEarliestTimer(aeEventLoop *eventLoop) { + aeTimeEvent *te = eventLoop->timeEventHead; + if (te == NULL) return -1; + + aeTimeEvent *earliest = NULL; + while (te) { + if ((!earliest || te->when < earliest->when) && te->id != AE_DELETED_EVENT_ID) + earliest = te; + te = te->next; + } + + if (earliest == NULL) return -1; + + monotime now = getMonotonicUs(); + return (now >= earliest->when) ? 0 : earliest->when - now; +} + +/* Process time events */ +static int processTimeEvents(aeEventLoop *eventLoop) { + int processed = 0; + aeTimeEvent *te; + long long maxId; + + te = eventLoop->timeEventHead; + maxId = eventLoop->timeEventNextId-1; + monotime now = getMonotonicUs(); + while(te) { + long long id; + + /* Remove events scheduled for deletion. */ + if (te->id == AE_DELETED_EVENT_ID) { + aeTimeEvent *next = te->next; + /* If a reference exists for this timer event, + * don't free it. This is currently incremented + * for recursive timerProc calls */ + if (te->refcount) { + te = next; + continue; + } + if (te->prev) + te->prev->next = te->next; + else + eventLoop->timeEventHead = te->next; + if (te->next) + te->next->prev = te->prev; + if (te->finalizerProc) { + te->finalizerProc(eventLoop, te->clientData); + now = getMonotonicUs(); + } + zfree(te); + te = next; + continue; + } + + /* Make sure we don't process time events created by time events in + * this iteration. Note that this check is currently useless: we always + * add new timers on the head, however if we change the implementation + * detail, this check may be useful again: we keep it here for future + * defense. */ + if (te->id > maxId) { + te = te->next; + continue; + } + + if (te->when <= now) { + int retval; + + id = te->id; + te->refcount++; + retval = te->timeProc(eventLoop, id, te->clientData); + te->refcount--; + processed++; + now = getMonotonicUs(); + if (retval != AE_NOMORE) { + te->when = now + (monotime)retval * 1000; + } else { + te->id = AE_DELETED_EVENT_ID; + } + } + te = te->next; + } + return processed; +} + +/* Process every pending file event, then every pending time event + * (that may be registered by file event callbacks just processed). + * Without special flags the function sleeps until some file event + * fires, or when the next time event occurs (if any). + * + * If flags is 0, the function does nothing and returns. + * if flags has AE_ALL_EVENTS set, all the kind of events are processed. + * if flags has AE_FILE_EVENTS set, file events are processed. + * if flags has AE_TIME_EVENTS set, time events are processed. + * if flags has AE_DONT_WAIT set, the function returns ASAP once all + * the events that can be handled without a wait are processed. + * if flags has AE_CALL_AFTER_SLEEP set, the aftersleep callback is called. + * if flags has AE_CALL_BEFORE_SLEEP set, the beforesleep callback is called. + * + * The function returns the number of events processed. */ +int aeProcessEvents(aeEventLoop *eventLoop, int flags) +{ + int processed = 0, numevents; + + /* Nothing to do? return ASAP */ + if (!(flags & AE_TIME_EVENTS) && !(flags & AE_FILE_EVENTS)) return 0; + + /* Note that we want to call aeApiPoll() even if there are no + * file events to process as long as we want to process time + * events, in order to sleep until the next time event is ready + * to fire. */ + if (eventLoop->maxfd != -1 || + ((flags & AE_TIME_EVENTS) && !(flags & AE_DONT_WAIT))) { + int j; + struct timeval tv, *tvp = NULL; /* NULL means infinite wait. */ + int64_t usUntilTimer; + + if (eventLoop->beforesleep != NULL && (flags & AE_CALL_BEFORE_SLEEP)) + eventLoop->beforesleep(eventLoop); + + /* The eventLoop->flags may be changed inside beforesleep. + * So we should check it after beforesleep be called. At the same time, + * the parameter flags always should have the highest priority. + * That is to say, once the parameter flag is set to AE_DONT_WAIT, + * no matter what value eventLoop->flags is set to, we should ignore it. */ + if ((flags & AE_DONT_WAIT) || (eventLoop->flags & AE_DONT_WAIT)) { + tv.tv_sec = tv.tv_usec = 0; + tvp = &tv; + } else if (flags & AE_TIME_EVENTS) { + usUntilTimer = usUntilEarliestTimer(eventLoop); + if (usUntilTimer >= 0) { + tv.tv_sec = usUntilTimer / 1000000; + tv.tv_usec = usUntilTimer % 1000000; + tvp = &tv; + } + } + /* Call the multiplexing API, will return only on timeout or when + * some event fires. */ + numevents = aeApiPoll(eventLoop, tvp); + + /* Don't process file events if not requested. */ + if (!(flags & AE_FILE_EVENTS)) { + numevents = 0; + } + + /* After sleep callback. */ + if (eventLoop->aftersleep != NULL && flags & AE_CALL_AFTER_SLEEP) + eventLoop->aftersleep(eventLoop); + + for (j = 0; j < numevents; j++) { + int fd = eventLoop->fired[j].fd; + aeFileEvent *fe = &eventLoop->events[fd]; + int mask = eventLoop->fired[j].mask; + int fired = 0; /* Number of events fired for current fd. */ + + /* Normally we execute the readable event first, and the writable + * event later. This is useful as sometimes we may be able + * to serve the reply of a query immediately after processing the + * query. + * + * However if AE_BARRIER is set in the mask, our application is + * asking us to do the reverse: never fire the writable event + * after the readable. In such a case, we invert the calls. + * This is useful when, for instance, we want to do things + * in the beforeSleep() hook, like fsyncing a file to disk, + * before replying to a client. */ + int invert = fe->mask & AE_BARRIER; + + /* Note the "fe->mask & mask & ..." code: maybe an already + * processed event removed an element that fired and we still + * didn't processed, so we check if the event is still valid. + * + * Fire the readable event if the call sequence is not + * inverted. */ + if (!invert && fe->mask & mask & AE_READABLE) { + fe->rfileProc(eventLoop,fd,fe->clientData,mask); + fired++; + fe = &eventLoop->events[fd]; /* Refresh in case of resize. */ + } + + /* Fire the writable event. */ + if (fe->mask & mask & AE_WRITABLE) { + if (!fired || fe->wfileProc != fe->rfileProc) { + fe->wfileProc(eventLoop,fd,fe->clientData,mask); + fired++; + } + } + + /* If we have to invert the call, fire the readable event now + * after the writable one. */ + if (invert) { + fe = &eventLoop->events[fd]; /* Refresh in case of resize. */ + if ((fe->mask & mask & AE_READABLE) && + (!fired || fe->wfileProc != fe->rfileProc)) + { + fe->rfileProc(eventLoop,fd,fe->clientData,mask); + fired++; + } + } + + processed++; + } + } + /* Check time events */ + if (flags & AE_TIME_EVENTS) + processed += processTimeEvents(eventLoop); + + return processed; /* return the number of processed file/time events */ +} + +/* Wait for milliseconds until the given file descriptor becomes + * writable/readable/exception */ +int aeWait(int fd, int mask, long long milliseconds) { + struct pollfd pfd; + int retmask = 0, retval; + + memset(&pfd, 0, sizeof(pfd)); + pfd.fd = fd; + if (mask & AE_READABLE) pfd.events |= POLLIN; + if (mask & AE_WRITABLE) pfd.events |= POLLOUT; + + if ((retval = poll(&pfd, 1, milliseconds))== 1) { + if (pfd.revents & POLLIN) retmask |= AE_READABLE; + if (pfd.revents & POLLOUT) retmask |= AE_WRITABLE; + if (pfd.revents & POLLERR) retmask |= AE_WRITABLE; + if (pfd.revents & POLLHUP) retmask |= AE_WRITABLE; + return retmask; + } else { + return retval; + } +} + +void aeMain(aeEventLoop *eventLoop) { + eventLoop->stop = 0; + while (!eventLoop->stop) { + aeProcessEvents(eventLoop, AE_ALL_EVENTS| + AE_CALL_BEFORE_SLEEP| + AE_CALL_AFTER_SLEEP); + } +} + +char *aeGetApiName(void) { + return aeApiName(); +} + +void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep) { + eventLoop->beforesleep = beforesleep; +} + +void aeSetAfterSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *aftersleep) { + eventLoop->aftersleep = aftersleep; +} diff --git a/deps/ae/ae.h b/deps/ae/ae.h new file mode 100644 index 0000000..996d48b --- /dev/null +++ b/deps/ae/ae.h @@ -0,0 +1,118 @@ +/* A simple event-driven programming library. Originally I wrote this code + * for the Jim's event-loop (Jim is a Tcl interpreter) but later translated + * it in form of a library for easy reuse. + * + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of (a) the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ + +#ifndef __AE_H__ +#define __AE_H__ + +#include "monotonic.h" + +#define AE_OK 0 +#define AE_ERR -1 + +#define AE_NONE 0 /* No events registered. */ +#define AE_READABLE 1 /* Fire when descriptor is readable. */ +#define AE_WRITABLE 2 /* Fire when descriptor is writable. */ +#define AE_BARRIER 4 /* With WRITABLE, never fire the event if the + READABLE event already fired in the same event + loop iteration. Useful when you want to persist + things to disk before sending replies, and want + to do that in a group fashion. */ + +#define AE_FILE_EVENTS (1<<0) +#define AE_TIME_EVENTS (1<<1) +#define AE_ALL_EVENTS (AE_FILE_EVENTS|AE_TIME_EVENTS) +#define AE_DONT_WAIT (1<<2) +#define AE_CALL_BEFORE_SLEEP (1<<3) +#define AE_CALL_AFTER_SLEEP (1<<4) + +#define AE_NOMORE -1 +#define AE_DELETED_EVENT_ID -1 + +/* Macros */ +#define AE_NOTUSED(V) ((void) V) + +struct aeEventLoop; + +/* Types and data structures */ +typedef void aeFileProc(struct aeEventLoop *eventLoop, int fd, void *clientData, int mask); +typedef int aeTimeProc(struct aeEventLoop *eventLoop, long long id, void *clientData); +typedef void aeEventFinalizerProc(struct aeEventLoop *eventLoop, void *clientData); +typedef void aeBeforeSleepProc(struct aeEventLoop *eventLoop); + +/* File event structure */ +typedef struct aeFileEvent { + int mask; /* one of AE_(READABLE|WRITABLE|BARRIER) */ + aeFileProc *rfileProc; + aeFileProc *wfileProc; + void *clientData; +} aeFileEvent; + +/* Time event structure */ +typedef struct aeTimeEvent { + long long id; /* time event identifier. */ + monotime when; + aeTimeProc *timeProc; + aeEventFinalizerProc *finalizerProc; + void *clientData; + struct aeTimeEvent *prev; + struct aeTimeEvent *next; + int refcount; /* refcount to prevent timer events from being + * freed in recursive time event calls. */ +} aeTimeEvent; + +/* A fired event */ +typedef struct aeFiredEvent { + int fd; + int mask; +} aeFiredEvent; + +/* State of an event based program */ +typedef struct aeEventLoop { + int maxfd; /* highest file descriptor currently registered */ + int setsize; /* max number of file descriptors tracked */ + long long timeEventNextId; + int nevents; /* Size of Registered events */ + aeFileEvent *events; /* Registered events */ + aeFiredEvent *fired; /* Fired events */ + aeTimeEvent *timeEventHead; + int stop; + void *apidata; /* This is used for polling API specific data */ + aeBeforeSleepProc *beforesleep; + aeBeforeSleepProc *aftersleep; + int flags; + void *privdata[2]; +} aeEventLoop; + +/* Prototypes */ +aeEventLoop *aeCreateEventLoop(int setsize); +void aeDeleteEventLoop(aeEventLoop *eventLoop); +void aeStop(aeEventLoop *eventLoop); +int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask, + aeFileProc *proc, void *clientData); +void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask); +int aeGetFileEvents(aeEventLoop *eventLoop, int fd); +void *aeGetFileClientData(aeEventLoop *eventLoop, int fd); +long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds, + aeTimeProc *proc, void *clientData, + aeEventFinalizerProc *finalizerProc); +int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id); +int aeProcessEvents(aeEventLoop *eventLoop, int flags); +int aeWait(int fd, int mask, long long milliseconds); +void aeMain(aeEventLoop *eventLoop); +char *aeGetApiName(void); +void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep); +void aeSetAfterSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *aftersleep); +int aeGetSetSize(aeEventLoop *eventLoop); +int aeResizeSetSize(aeEventLoop *eventLoop, int setsize); +void aeSetDontWait(aeEventLoop *eventLoop, int noWait); + +#endif diff --git a/deps/ae/ae_epoll.c b/deps/ae/ae_epoll.c new file mode 100644 index 0000000..751639b --- /dev/null +++ b/deps/ae/ae_epoll.c @@ -0,0 +1,144 @@ +/* Linux epoll(2) based ae.c module + * + * Copyright (c) 2009-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of (a) the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ + +#include + +typedef struct aeApiState +{ + int epfd; + struct epoll_event *events; +} aeApiState; + +static int aeApiCreate(aeEventLoop *eventLoop) +{ + aeApiState *state = zmalloc(sizeof(aeApiState)); + + if (!state) + return -1; + state->events = zmalloc(sizeof(struct epoll_event) * eventLoop->setsize); + if (!state->events) + { + zfree(state); + return -1; + } + state->epfd = epoll_create(1024); /* 1024 is just a hint for the kernel */ + if (state->epfd == -1) + { + zfree(state->events); + zfree(state); + return -1; + } + anetCloexec(state->epfd); + eventLoop->apidata = state; + return 0; +} + +static int aeApiResize(aeEventLoop *eventLoop, int setsize) +{ + aeApiState *state = eventLoop->apidata; + + state->events = zrealloc(state->events, sizeof(struct epoll_event) * setsize); + return 0; +} + +static void aeApiFree(aeEventLoop *eventLoop) +{ + aeApiState *state = eventLoop->apidata; + + close(state->epfd); + zfree(state->events); + zfree(state); +} + +static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) +{ + aeApiState *state = eventLoop->apidata; + struct epoll_event ee = {0}; /* avoid valgrind warning */ + /* If the fd was already monitored for some event, we need a MOD + * operation. Otherwise we need an ADD operation. */ + int op = eventLoop->events[fd].mask == AE_NONE ? EPOLL_CTL_ADD : EPOLL_CTL_MOD; + + ee.events = 0; + mask |= eventLoop->events[fd].mask; /* Merge old events */ + if (mask & AE_READABLE) + ee.events |= EPOLLIN; + if (mask & AE_WRITABLE) + ee.events |= EPOLLOUT; + ee.data.fd = fd; + if (epoll_ctl(state->epfd, op, fd, &ee) == -1) + return -1; + return 0; +} + +static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int delmask) +{ + aeApiState *state = eventLoop->apidata; + struct epoll_event ee = {0}; /* avoid valgrind warning */ + int mask = eventLoop->events[fd].mask & (~delmask); + + ee.events = 0; + if (mask & AE_READABLE) + ee.events |= EPOLLIN; + if (mask & AE_WRITABLE) + ee.events |= EPOLLOUT; + ee.data.fd = fd; + if (mask != AE_NONE) + { + epoll_ctl(state->epfd, EPOLL_CTL_MOD, fd, &ee); + } + else + { + /* Note, Kernel < 2.6.9 requires a non null event pointer even for + * EPOLL_CTL_DEL. */ + epoll_ctl(state->epfd, EPOLL_CTL_DEL, fd, &ee); + } +} + +static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) +{ + aeApiState *state = eventLoop->apidata; + int retval, numevents = 0; + + retval = epoll_wait(state->epfd, state->events, eventLoop->setsize, + tvp ? (tvp->tv_sec * 1000 + (tvp->tv_usec + 999) / 1000) : -1); + if (retval > 0) + { + int j; + + numevents = retval; + for (j = 0; j < numevents; j++) + { + int mask = 0; + struct epoll_event *e = state->events + j; + + if (e->events & EPOLLIN) + mask |= AE_READABLE; + if (e->events & EPOLLOUT) + mask |= AE_WRITABLE; + if (e->events & EPOLLERR) + mask |= AE_WRITABLE | AE_READABLE; + if (e->events & EPOLLHUP) + mask |= AE_WRITABLE | AE_READABLE; + eventLoop->fired[j].fd = e->data.fd; + eventLoop->fired[j].mask = mask; + } + } + else if (retval == -1 && errno != EINTR) + { + panic("aeApiPoll: epoll_wait, %s", strerror(errno)); + } + + return numevents; +} + +static char *aeApiName(void) +{ + return "epoll"; +} diff --git a/deps/ae/ae_kqueue.c b/deps/ae/ae_kqueue.c new file mode 100644 index 0000000..ec10a5e --- /dev/null +++ b/deps/ae/ae_kqueue.c @@ -0,0 +1,183 @@ +/* Kqueue(2)-based ae.c module + * + * Copyright (C) 2009 Harish Mallipeddi - harish.mallipeddi@gmail.com + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + + +#include +#include +#include + +typedef struct aeApiState { + int kqfd; + struct kevent *events; + + /* Events mask for merge read and write event. + * To reduce memory consumption, we use 2 bits to store the mask + * of an event, so that 1 byte will store the mask of 4 events. */ + char *eventsMask; +} aeApiState; + +#define EVENT_MASK_MALLOC_SIZE(sz) (((sz) + 3) / 4) +#define EVENT_MASK_OFFSET(fd) ((fd) % 4 * 2) +#define EVENT_MASK_ENCODE(fd, mask) (((mask) & 0x3) << EVENT_MASK_OFFSET(fd)) + +static inline int getEventMask(const char *eventsMask, int fd) { + return (eventsMask[fd/4] >> EVENT_MASK_OFFSET(fd)) & 0x3; +} + +static inline void addEventMask(char *eventsMask, int fd, int mask) { + eventsMask[fd/4] |= EVENT_MASK_ENCODE(fd, mask); +} + +static inline void resetEventMask(char *eventsMask, int fd) { + eventsMask[fd/4] &= ~EVENT_MASK_ENCODE(fd, 0x3); +} + +static int aeApiCreate(aeEventLoop *eventLoop) { + aeApiState *state = zmalloc(sizeof(aeApiState)); + + if (!state) return -1; + state->events = zmalloc(sizeof(struct kevent)*eventLoop->setsize); + if (!state->events) { + zfree(state); + return -1; + } + state->kqfd = kqueue(); + if (state->kqfd == -1) { + zfree(state->events); + zfree(state); + return -1; + } + anetCloexec(state->kqfd); + state->eventsMask = zmalloc(EVENT_MASK_MALLOC_SIZE(eventLoop->setsize)); + memset(state->eventsMask, 0, EVENT_MASK_MALLOC_SIZE(eventLoop->setsize)); + eventLoop->apidata = state; + return 0; +} + +static int aeApiResize(aeEventLoop *eventLoop, int setsize) { + aeApiState *state = eventLoop->apidata; + + state->events = zrealloc(state->events, sizeof(struct kevent)*setsize); + state->eventsMask = zrealloc(state->eventsMask, EVENT_MASK_MALLOC_SIZE(setsize)); + memset(state->eventsMask, 0, EVENT_MASK_MALLOC_SIZE(setsize)); + return 0; +} + +static void aeApiFree(aeEventLoop *eventLoop) { + aeApiState *state = eventLoop->apidata; + + close(state->kqfd); + zfree(state->events); + zfree(state->eventsMask); + zfree(state); +} + +static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) { + aeApiState *state = eventLoop->apidata; + struct kevent evs[2]; + int nch = 0; + + if (mask & AE_READABLE) EV_SET(evs + nch++, fd, EVFILT_READ, EV_ADD, 0, 0, NULL); + if (mask & AE_WRITABLE) EV_SET(evs + nch++, fd, EVFILT_WRITE, EV_ADD, 0, 0, NULL); + + return kevent(state->kqfd, evs, nch, NULL, 0, NULL); +} + +static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) { + aeApiState *state = eventLoop->apidata; + struct kevent evs[2]; + int nch = 0; + + if (mask & AE_READABLE) EV_SET(evs + nch++, fd, EVFILT_READ, EV_DELETE, 0, 0, NULL); + if (mask & AE_WRITABLE) EV_SET(evs + nch++, fd, EVFILT_WRITE, EV_DELETE, 0, 0, NULL); + + kevent(state->kqfd, evs, nch, NULL, 0, NULL); +} + +static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { + aeApiState *state = eventLoop->apidata; + int retval, numevents = 0; + + if (tvp != NULL) { + struct timespec timeout; + timeout.tv_sec = tvp->tv_sec; + timeout.tv_nsec = tvp->tv_usec * 1000; + retval = kevent(state->kqfd, NULL, 0, state->events, eventLoop->setsize, + &timeout); + } else { + retval = kevent(state->kqfd, NULL, 0, state->events, eventLoop->setsize, + NULL); + } + + if (retval > 0) { + int j; + + /* Normally we execute the read event first and then the write event. + * When the barrier is set, we will do it reverse. + * + * However, under kqueue, read and write events would be separate + * events, which would make it impossible to control the order of + * reads and writes. So we store the event's mask we've got and merge + * the same fd events later. */ + for (j = 0; j < retval; j++) { + struct kevent *e = state->events+j; + int fd = e->ident; + int mask = 0; + + if (e->filter == EVFILT_READ) mask = AE_READABLE; + else if (e->filter == EVFILT_WRITE) mask = AE_WRITABLE; + addEventMask(state->eventsMask, fd, mask); + } + + /* Re-traversal to merge read and write events, and set the fd's mask to + * 0 so that events are not added again when the fd is encountered again. */ + numevents = 0; + for (j = 0; j < retval; j++) { + struct kevent *e = state->events+j; + int fd = e->ident; + int mask = getEventMask(state->eventsMask, fd); + + if (mask) { + eventLoop->fired[numevents].fd = fd; + eventLoop->fired[numevents].mask = mask; + resetEventMask(state->eventsMask, fd); + numevents++; + } + } + } else if (retval == -1 && errno != EINTR) { + panic("aeApiPoll: kevent, %s", strerror(errno)); + } + + return numevents; +} + +static char *aeApiName(void) { + return "kqueue"; +} diff --git a/deps/ae/ae_select.c b/deps/ae/ae_select.c new file mode 100644 index 0000000..f3c7aa7 --- /dev/null +++ b/deps/ae/ae_select.c @@ -0,0 +1,108 @@ +/* Select()-based ae.c module. + * + * Copyright (c) 2009-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of (a) the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ + +#include +#include + +typedef struct aeApiState +{ + fd_set rfds, wfds; + /* We need to have a copy of the fd sets as it's not safe to reuse + * FD sets after select(). */ + fd_set _rfds, _wfds; +} aeApiState; + +static int aeApiCreate(aeEventLoop *eventLoop) +{ + aeApiState *state = zmalloc(sizeof(aeApiState)); + + if (!state) + return -1; + FD_ZERO(&state->rfds); + FD_ZERO(&state->wfds); + eventLoop->apidata = state; + return 0; +} + +static int aeApiResize(aeEventLoop *eventLoop, int setsize) +{ + AE_NOTUSED(eventLoop); + /* Just ensure we have enough room in the fd_set type. */ + if (setsize >= FD_SETSIZE) + return -1; + return 0; +} + +static void aeApiFree(aeEventLoop *eventLoop) +{ + zfree(eventLoop->apidata); +} + +static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) +{ + aeApiState *state = eventLoop->apidata; + + if (mask & AE_READABLE) + FD_SET(fd, &state->rfds); + if (mask & AE_WRITABLE) + FD_SET(fd, &state->wfds); + return 0; +} + +static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) +{ + aeApiState *state = eventLoop->apidata; + + if (mask & AE_READABLE) + FD_CLR(fd, &state->rfds); + if (mask & AE_WRITABLE) + FD_CLR(fd, &state->wfds); +} + +static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) +{ + aeApiState *state = eventLoop->apidata; + int retval, j, numevents = 0; + + memcpy(&state->_rfds, &state->rfds, sizeof(fd_set)); + memcpy(&state->_wfds, &state->wfds, sizeof(fd_set)); + + retval = select(eventLoop->maxfd + 1, + &state->_rfds, &state->_wfds, NULL, tvp); + if (retval > 0) + { + for (j = 0; j <= eventLoop->maxfd; j++) + { + int mask = 0; + aeFileEvent *fe = &eventLoop->events[j]; + + if (fe->mask == AE_NONE) + continue; + if (fe->mask & AE_READABLE && FD_ISSET(j, &state->_rfds)) + mask |= AE_READABLE; + if (fe->mask & AE_WRITABLE && FD_ISSET(j, &state->_wfds)) + mask |= AE_WRITABLE; + eventLoop->fired[numevents].fd = j; + eventLoop->fired[numevents].mask = mask; + numevents++; + } + } + else if (retval == -1 && errno != EINTR) + { + panic("aeApiPoll: select, %s", strerror(errno)); + } + + return numevents; +} + +static char *aeApiName(void) +{ + return "select"; +} diff --git a/deps/ae/anet.c b/deps/ae/anet.c new file mode 100644 index 0000000..8b7b91e --- /dev/null +++ b/deps/ae/anet.c @@ -0,0 +1,812 @@ +/* anet.c -- Basic TCP socket stuff made a bit less boring + * + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of (a) the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ + +#include "fmacros.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "anet.h" +#include "config.h" +#include "util.h" + +#define UNUSED(x) (void)(x) + +static void anetSetError(char *err, const char *fmt, ...) +{ + va_list ap; + + if (!err) return; + va_start(ap, fmt); + vsnprintf(err, ANET_ERR_LEN, fmt, ap); + va_end(ap); +} + +int anetGetError(int fd) { + int sockerr = 0; + socklen_t errlen = sizeof(sockerr); + + if (getsockopt(fd, SOL_SOCKET, SO_ERROR, &sockerr, &errlen) == -1) + sockerr = errno; + return sockerr; +} + +int anetSetBlock(char *err, int fd, int non_block) { + int flags; + + /* Set the socket blocking (if non_block is zero) or non-blocking. + * Note that fcntl(2) for F_GETFL and F_SETFL can't be + * interrupted by a signal. */ + if ((flags = fcntl(fd, F_GETFL)) == -1) { + anetSetError(err, "fcntl(F_GETFL): %s", strerror(errno)); + return ANET_ERR; + } + + /* Check if this flag has been set or unset, if so, + * then there is no need to call fcntl to set/unset it again. */ + if (!!(flags & O_NONBLOCK) == !!non_block) + return ANET_OK; + + if (non_block) + flags |= O_NONBLOCK; + else + flags &= ~O_NONBLOCK; + + if (fcntl(fd, F_SETFL, flags) == -1) { + anetSetError(err, "fcntl(F_SETFL,O_NONBLOCK): %s", strerror(errno)); + return ANET_ERR; + } + return ANET_OK; +} + +int anetNonBlock(char *err, int fd) { + return anetSetBlock(err,fd,1); +} + +int anetBlock(char *err, int fd) { + return anetSetBlock(err,fd,0); +} + +/* Enable the FD_CLOEXEC on the given fd to avoid fd leaks. + * This function should be invoked for fd's on specific places + * where fork + execve system calls are called. */ +int anetCloexec(int fd) { + int r; + int flags; + + do { + r = fcntl(fd, F_GETFD); + } while (r == -1 && errno == EINTR); + + if (r == -1 || (r & FD_CLOEXEC)) + return r; + + flags = r | FD_CLOEXEC; + + do { + r = fcntl(fd, F_SETFD, flags); + } while (r == -1 && errno == EINTR); + + return r; +} + +/* Enable TCP keep-alive mechanism to detect dead peers, + * TCP_KEEPIDLE, TCP_KEEPINTVL and TCP_KEEPCNT will be set accordingly. */ +int anetKeepAlive(char *err, int fd, int interval) +{ + int enabled = 1; + if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &enabled, sizeof(enabled))) + { + anetSetError(err, "setsockopt SO_KEEPALIVE: %s", strerror(errno)); + return ANET_ERR; + } + + int idle; + int intvl; + int cnt; + + /* There are platforms that are expected to support the full mechanism of TCP keep-alive, + * we want the compiler to emit warnings of unused variables if the preprocessor directives + * somehow fail, and other than those platforms, just omit these warnings if they happen. + */ +#if !(defined(_AIX) || defined(__APPLE__) || defined(__DragonFly__) || \ + defined(__FreeBSD__) || defined(__illumos__) || defined(__linux__) || \ + defined(__NetBSD__) || defined(__sun)) + UNUSED(interval); + UNUSED(idle); + UNUSED(intvl); + UNUSED(cnt); +#endif + +#ifdef __sun + /* The implementation of TCP keep-alive on Solaris/SmartOS is a bit unusual + * compared to other Unix-like systems. + * Thus, we need to specialize it on Solaris. + * + * There are two keep-alive mechanisms on Solaris: + * - By default, the first keep-alive probe is sent out after a TCP connection is idle for two hours. + * If the peer does not respond to the probe within eight minutes, the TCP connection is aborted. + * You can alter the interval for sending out the first probe using the socket option TCP_KEEPALIVE_THRESHOLD + * in milliseconds or TCP_KEEPIDLE in seconds. + * The system default is controlled by the TCP ndd parameter tcp_keepalive_interval. The minimum value is ten seconds. + * The maximum is ten days, while the default is two hours. If you receive no response to the probe, + * you can use the TCP_KEEPALIVE_ABORT_THRESHOLD socket option to change the time threshold for aborting a TCP connection. + * The option value is an unsigned integer in milliseconds. The value zero indicates that TCP should never time out and + * abort the connection when probing. The system default is controlled by the TCP ndd parameter tcp_keepalive_abort_interval. + * The default is eight minutes. + * + * - The second implementation is activated if socket option TCP_KEEPINTVL and/or TCP_KEEPCNT are set. + * The time between each consequent probes is set by TCP_KEEPINTVL in seconds. + * The minimum value is ten seconds. The maximum is ten days, while the default is two hours. + * The TCP connection will be aborted after certain amount of probes, which is set by TCP_KEEPCNT, without receiving response. + */ + + idle = interval; + if (idle < 10) idle = 10; // kernel expects at least 10 seconds + if (idle > 10*24*60*60) idle = 10*24*60*60; // kernel expects at most 10 days + + /* `TCP_KEEPIDLE`, `TCP_KEEPINTVL`, and `TCP_KEEPCNT` were not available on Solaris + * until version 11.4, but let's take a chance here. */ +#if defined(TCP_KEEPIDLE) && defined(TCP_KEEPINTVL) && defined(TCP_KEEPCNT) + if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, &idle, sizeof(idle))) { + anetSetError(err, "setsockopt TCP_KEEPIDLE: %s\n", strerror(errno)); + return ANET_ERR; + } + + intvl = idle/3; + if (intvl < 10) intvl = 10; /* kernel expects at least 10 seconds */ + if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPINTVL, &intvl, sizeof(intvl))) { + anetSetError(err, "setsockopt TCP_KEEPINTVL: %s\n", strerror(errno)); + return ANET_ERR; + } + + cnt = 3; + if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPCNT, &cnt, sizeof(cnt))) { + anetSetError(err, "setsockopt TCP_KEEPCNT: %s\n", strerror(errno)); + return ANET_ERR; + } +#else + /* Fall back to the first implementation of tcp-alive mechanism for older Solaris, + * simulate the tcp-alive mechanism on other platforms via `TCP_KEEPALIVE_THRESHOLD` + `TCP_KEEPALIVE_ABORT_THRESHOLD`. + */ + idle *= 1000; // kernel expects milliseconds + if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPALIVE_THRESHOLD, &idle, sizeof(idle))) { + anetSetError(err, "setsockopt TCP_KEEPINTVL: %s\n", strerror(errno)); + return ANET_ERR; + } + + /* Note that the consequent probes will not be sent at equal intervals on Solaris, + * but will be sent using the exponential backoff algorithm. */ + int time_to_abort = idle; + if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPALIVE_ABORT_THRESHOLD, &time_to_abort, sizeof(time_to_abort))) { + anetSetError(err, "setsockopt TCP_KEEPCNT: %s\n", strerror(errno)); + return ANET_ERR; + } +#endif + + return ANET_OK; + +#endif + +#ifdef TCP_KEEPIDLE + /* Default settings are more or less garbage, with the keepalive time + * set to 7200 by default on Linux and other Unix-like systems. + * Modify settings to make the feature actually useful. */ + + /* Send first probe after interval. */ + idle = interval; + if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, &idle, sizeof(idle))) { + anetSetError(err, "setsockopt TCP_KEEPIDLE: %s\n", strerror(errno)); + return ANET_ERR; + } +#elif defined(TCP_KEEPALIVE) + /* Darwin/macOS uses TCP_KEEPALIVE in place of TCP_KEEPIDLE. */ + idle = interval; + if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPALIVE, &idle, sizeof(idle))) { + anetSetError(err, "setsockopt TCP_KEEPALIVE: %s\n", strerror(errno)); + return ANET_ERR; + } +#endif + +#ifdef TCP_KEEPINTVL + /* Send next probes after the specified interval. Note that we set the + * delay as interval / 3, as we send three probes before detecting + * an error (see the next setsockopt call). */ + intvl = interval/3; + if (intvl == 0) intvl = 1; + if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPINTVL, &intvl, sizeof(intvl))) { + anetSetError(err, "setsockopt TCP_KEEPINTVL: %s\n", strerror(errno)); + return ANET_ERR; + } +#endif + +#ifdef TCP_KEEPCNT + /* Consider the socket in error state after three we send three ACK + * probes without getting a reply. */ + cnt = 3; + if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPCNT, &cnt, sizeof(cnt))) { + anetSetError(err, "setsockopt TCP_KEEPCNT: %s\n", strerror(errno)); + return ANET_ERR; + } +#endif + + return ANET_OK; +} + +static int anetSetTcpNoDelay(char *err, int fd, int val) +{ + if (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &val, sizeof(val)) == -1) + { + anetSetError(err, "setsockopt TCP_NODELAY: %s", strerror(errno)); + return ANET_ERR; + } + return ANET_OK; +} + +int anetEnableTcpNoDelay(char *err, int fd) +{ + return anetSetTcpNoDelay(err, fd, 1); +} + +int anetDisableTcpNoDelay(char *err, int fd) +{ + return anetSetTcpNoDelay(err, fd, 0); +} + +/* Set the socket send timeout (SO_SNDTIMEO socket option) to the specified + * number of milliseconds, or disable it if the 'ms' argument is zero. */ +int anetSendTimeout(char *err, int fd, long long ms) { + struct timeval tv; + + tv.tv_sec = ms/1000; + tv.tv_usec = (ms%1000)*1000; + if (setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)) == -1) { + anetSetError(err, "setsockopt SO_SNDTIMEO: %s", strerror(errno)); + return ANET_ERR; + } + return ANET_OK; +} + +/* Set the socket receive timeout (SO_RCVTIMEO socket option) to the specified + * number of milliseconds, or disable it if the 'ms' argument is zero. */ +int anetRecvTimeout(char *err, int fd, long long ms) { + struct timeval tv; + + tv.tv_sec = ms/1000; + tv.tv_usec = (ms%1000)*1000; + if (setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)) == -1) { + anetSetError(err, "setsockopt SO_RCVTIMEO: %s", strerror(errno)); + return ANET_ERR; + } + return ANET_OK; +} + +/* Resolve the hostname "host" and set the string representation of the + * IP address into the buffer pointed by "ipbuf". + * + * If flags is set to ANET_IP_ONLY the function only resolves hostnames + * that are actually already IPv4 or IPv6 addresses. This turns the function + * into a validating / normalizing function. + * + * If the flag ANET_PREFER_IPV4 is set, IPv4 is preferred over IPv6. + * If the flag ANET_PREFER_IPV6 is set, IPv6 is preferred over IPv4. + * */ +int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len, + int flags) +{ + struct addrinfo hints, *info; + int rv; + + memset(&hints,0,sizeof(hints)); + if (flags & ANET_IP_ONLY) hints.ai_flags = AI_NUMERICHOST; + hints.ai_family = AF_UNSPEC; + if (flags & ANET_PREFER_IPV4 && !(flags & ANET_PREFER_IPV6)) { + hints.ai_family = AF_INET; + } else if (flags & ANET_PREFER_IPV6 && !(flags & ANET_PREFER_IPV4)) { + hints.ai_family = AF_INET6; + } + hints.ai_socktype = SOCK_STREAM; /* specify socktype to avoid dups */ + + rv = getaddrinfo(host, NULL, &hints, &info); + if (rv != 0 && hints.ai_family != AF_UNSPEC) { + /* Try the other IP version. */ + hints.ai_family = (hints.ai_family == AF_INET) ? AF_INET6 : AF_INET; + rv = getaddrinfo(host, NULL, &hints, &info); + } + if (rv != 0) { + anetSetError(err, "%s", gai_strerror(rv)); + return ANET_ERR; + } + if (info->ai_family == AF_INET) { + struct sockaddr_in *sa = (struct sockaddr_in *)info->ai_addr; + inet_ntop(AF_INET, &(sa->sin_addr), ipbuf, ipbuf_len); + } else { + struct sockaddr_in6 *sa = (struct sockaddr_in6 *)info->ai_addr; + inet_ntop(AF_INET6, &(sa->sin6_addr), ipbuf, ipbuf_len); + } + + freeaddrinfo(info); + return ANET_OK; +} + +static int anetSetReuseAddr(char *err, int fd) { + int yes = 1; + /* Make sure connection-intensive things like the redis benchmark + * will be able to close/open sockets a zillion of times */ + if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) == -1) { + anetSetError(err, "setsockopt SO_REUSEADDR: %s", strerror(errno)); + return ANET_ERR; + } + return ANET_OK; +} + +static int anetCreateSocket(char *err, int domain) { + int s; + if ((s = socket(domain, SOCK_STREAM, 0)) == -1) { + anetSetError(err, "creating socket: %s", strerror(errno)); + return ANET_ERR; + } + + /* Make sure connection-intensive things like the redis benchmark + * will be able to close/open sockets a zillion of times */ + if (anetSetReuseAddr(err,s) == ANET_ERR) { + close(s); + return ANET_ERR; + } + return s; +} + +#define ANET_CONNECT_NONE 0 +#define ANET_CONNECT_NONBLOCK 1 +#define ANET_CONNECT_BE_BINDING 2 /* Best effort binding. */ +static int anetTcpGenericConnect(char *err, const char *addr, int port, + const char *source_addr, int flags) +{ + int s = ANET_ERR, rv; + char portstr[6]; /* strlen("65535") + 1; */ + struct addrinfo hints, *servinfo, *bservinfo, *p, *b; + + snprintf(portstr,sizeof(portstr),"%d",port); + memset(&hints,0,sizeof(hints)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + + if ((rv = getaddrinfo(addr,portstr,&hints,&servinfo)) != 0) { + anetSetError(err, "%s", gai_strerror(rv)); + return ANET_ERR; + } + for (p = servinfo; p != NULL; p = p->ai_next) { + /* Try to create the socket and to connect it. + * If we fail in the socket() call, or on connect(), we retry with + * the next entry in servinfo. */ + if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == -1) + continue; + if (anetSetReuseAddr(err,s) == ANET_ERR) goto error; + if (flags & ANET_CONNECT_NONBLOCK && anetNonBlock(err,s) != ANET_OK) + goto error; + if (source_addr) { + int bound = 0; + /* Using getaddrinfo saves us from self-determining IPv4 vs IPv6 */ + if ((rv = getaddrinfo(source_addr, NULL, &hints, &bservinfo)) != 0) + { + anetSetError(err, "%s", gai_strerror(rv)); + goto error; + } + for (b = bservinfo; b != NULL; b = b->ai_next) { + if (bind(s,b->ai_addr,b->ai_addrlen) != -1) { + bound = 1; + break; + } + } + freeaddrinfo(bservinfo); + if (!bound) { + anetSetError(err, "bind: %s", strerror(errno)); + goto error; + } + } + if (connect(s,p->ai_addr,p->ai_addrlen) == -1) { + /* If the socket is non-blocking, it is ok for connect() to + * return an EINPROGRESS error here. */ + if (errno == EINPROGRESS && flags & ANET_CONNECT_NONBLOCK) + goto end; + close(s); + s = ANET_ERR; + continue; + } + + /* If we ended an iteration of the for loop without errors, we + * have a connected socket. Let's return to the caller. */ + goto end; + } + if (p == NULL) + anetSetError(err, "creating socket: %s", strerror(errno)); + +error: + if (s != ANET_ERR) { + close(s); + s = ANET_ERR; + } + +end: + freeaddrinfo(servinfo); + + /* Handle best effort binding: if a binding address was used, but it is + * not possible to create a socket, try again without a binding address. */ + if (s == ANET_ERR && source_addr && (flags & ANET_CONNECT_BE_BINDING)) { + return anetTcpGenericConnect(err,addr,port,NULL,flags); + } else { + return s; + } +} + +int anetTcpNonBlockConnect(char *err, const char *addr, int port) +{ + return anetTcpGenericConnect(err,addr,port,NULL,ANET_CONNECT_NONBLOCK); +} + +int anetTcpNonBlockBestEffortBindConnect(char *err, const char *addr, int port, + const char *source_addr) +{ + return anetTcpGenericConnect(err,addr,port,source_addr, + ANET_CONNECT_NONBLOCK|ANET_CONNECT_BE_BINDING); +} + +int anetUnixGenericConnect(char *err, const char *path, int flags) +{ + int s; + struct sockaddr_un sa; + + if ((s = anetCreateSocket(err,AF_LOCAL)) == ANET_ERR) + return ANET_ERR; + + sa.sun_family = AF_LOCAL; + redis_strlcpy(sa.sun_path,path,sizeof(sa.sun_path)); + if (flags & ANET_CONNECT_NONBLOCK) { + if (anetNonBlock(err,s) != ANET_OK) { + close(s); + return ANET_ERR; + } + } + if (connect(s,(struct sockaddr*)&sa,sizeof(sa)) == -1) { + if (errno == EINPROGRESS && + flags & ANET_CONNECT_NONBLOCK) + return s; + + anetSetError(err, "connect: %s", strerror(errno)); + close(s); + return ANET_ERR; + } + return s; +} + +static int anetListen(char *err, int s, struct sockaddr *sa, socklen_t len, int backlog, mode_t perm) { + if (bind(s,sa,len) == -1) { + anetSetError(err, "bind: %s", strerror(errno)); + close(s); + return ANET_ERR; + } + + if (sa->sa_family == AF_LOCAL && perm) + chmod(((struct sockaddr_un *) sa)->sun_path, perm); + + if (listen(s, backlog) == -1) { + anetSetError(err, "listen: %s", strerror(errno)); + close(s); + return ANET_ERR; + } + return ANET_OK; +} + +static int anetV6Only(char *err, int s) { + int yes = 1; + if (setsockopt(s,IPPROTO_IPV6,IPV6_V6ONLY,&yes,sizeof(yes)) == -1) { + anetSetError(err, "setsockopt: %s", strerror(errno)); + return ANET_ERR; + } + return ANET_OK; +} + +static int _anetTcpServer(char *err, int port, char *bindaddr, int af, int backlog) +{ + int s = -1, rv; + char _port[6]; /* strlen("65535") */ + struct addrinfo hints, *servinfo, *p; + + snprintf(_port,6,"%d",port); + memset(&hints,0,sizeof(hints)); + hints.ai_family = af; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_PASSIVE; /* No effect if bindaddr != NULL */ + if (bindaddr && !strcmp("*", bindaddr)) + bindaddr = NULL; + if (af == AF_INET6 && bindaddr && !strcmp("::*", bindaddr)) + bindaddr = NULL; + + if ((rv = getaddrinfo(bindaddr,_port,&hints,&servinfo)) != 0) { + anetSetError(err, "%s", gai_strerror(rv)); + return ANET_ERR; + } + for (p = servinfo; p != NULL; p = p->ai_next) { + if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == -1) + continue; + + if (af == AF_INET6 && anetV6Only(err,s) == ANET_ERR) goto error; + if (anetSetReuseAddr(err,s) == ANET_ERR) goto error; + if (anetListen(err,s,p->ai_addr,p->ai_addrlen,backlog,0) == ANET_ERR) s = ANET_ERR; + goto end; + } + if (p == NULL) { + anetSetError(err, "unable to bind socket, errno: %d", errno); + goto error; + } + +error: + if (s != -1) close(s); + s = ANET_ERR; +end: + freeaddrinfo(servinfo); + return s; +} + +int anetTcpServer(char *err, int port, char *bindaddr, int backlog) +{ + return _anetTcpServer(err, port, bindaddr, AF_INET, backlog); +} + +int anetTcp6Server(char *err, int port, char *bindaddr, int backlog) +{ + return _anetTcpServer(err, port, bindaddr, AF_INET6, backlog); +} + +int anetUnixServer(char *err, char *path, mode_t perm, int backlog) +{ + int s; + struct sockaddr_un sa; + + if (strlen(path) > sizeof(sa.sun_path)-1) { + anetSetError(err,"unix socket path too long (%zu), must be under %zu", strlen(path), sizeof(sa.sun_path)); + return ANET_ERR; + } + if ((s = anetCreateSocket(err,AF_LOCAL)) == ANET_ERR) + return ANET_ERR; + + memset(&sa,0,sizeof(sa)); + sa.sun_family = AF_LOCAL; + redis_strlcpy(sa.sun_path,path,sizeof(sa.sun_path)); + if (anetListen(err,s,(struct sockaddr*)&sa,sizeof(sa),backlog,perm) == ANET_ERR) + return ANET_ERR; + return s; +} + +/* Accept a connection and also make sure the socket is non-blocking, and CLOEXEC. + * returns the new socket FD, or -1 on error. */ +static int anetGenericAccept(char *err, int s, struct sockaddr *sa, socklen_t *len) { + int fd; + do { + /* Use the accept4() call on linux to simultaneously accept and + * set a socket as non-blocking. */ +#ifdef HAVE_ACCEPT4 + fd = accept4(s, sa, len, SOCK_NONBLOCK | SOCK_CLOEXEC); +#else + fd = accept(s,sa,len); +#endif + } while(fd == -1 && errno == EINTR); + if (fd == -1) { + anetSetError(err, "accept: %s", strerror(errno)); + return ANET_ERR; + } +#ifndef HAVE_ACCEPT4 + if (anetCloexec(fd) == -1) { + anetSetError(err, "anetCloexec: %s", strerror(errno)); + close(fd); + return ANET_ERR; + } + if (anetNonBlock(err, fd) != ANET_OK) { + close(fd); + return ANET_ERR; + } +#endif + return fd; +} + +/* Accept a connection and also make sure the socket is non-blocking, and CLOEXEC. + * returns the new socket FD, or -1 on error. */ +int anetTcpAccept(char *err, int serversock, char *ip, size_t ip_len, int *port) { + int fd; + struct sockaddr_storage sa; + socklen_t salen = sizeof(sa); + if ((fd = anetGenericAccept(err,serversock,(struct sockaddr*)&sa,&salen)) == ANET_ERR) + return ANET_ERR; + + if (sa.ss_family == AF_INET) { + struct sockaddr_in *s = (struct sockaddr_in *)&sa; + if (ip) inet_ntop(AF_INET,(void*)&(s->sin_addr),ip,ip_len); + if (port) *port = ntohs(s->sin_port); + } else { + struct sockaddr_in6 *s = (struct sockaddr_in6 *)&sa; + if (ip) inet_ntop(AF_INET6,(void*)&(s->sin6_addr),ip,ip_len); + if (port) *port = ntohs(s->sin6_port); + } + return fd; +} + +/* Accept a connection and also make sure the socket is non-blocking, and CLOEXEC. + * returns the new socket FD, or -1 on error. */ +int anetUnixAccept(char *err, int s) { + int fd; + struct sockaddr_un sa; + socklen_t salen = sizeof(sa); + if ((fd = anetGenericAccept(err,s,(struct sockaddr*)&sa,&salen)) == ANET_ERR) + return ANET_ERR; + + return fd; +} + +int anetFdToString(int fd, char *ip, size_t ip_len, int *port, int remote) { + struct sockaddr_storage sa; + socklen_t salen = sizeof(sa); + + if (remote) { + if (getpeername(fd, (struct sockaddr *)&sa, &salen) == -1) goto error; + } else { + if (getsockname(fd, (struct sockaddr *)&sa, &salen) == -1) goto error; + } + + if (sa.ss_family == AF_INET) { + struct sockaddr_in *s = (struct sockaddr_in *)&sa; + if (ip) { + if (inet_ntop(AF_INET,(void*)&(s->sin_addr),ip,ip_len) == NULL) + goto error; + } + if (port) *port = ntohs(s->sin_port); + } else if (sa.ss_family == AF_INET6) { + struct sockaddr_in6 *s = (struct sockaddr_in6 *)&sa; + if (ip) { + if (inet_ntop(AF_INET6,(void*)&(s->sin6_addr),ip,ip_len) == NULL) + goto error; + } + if (port) *port = ntohs(s->sin6_port); + } else if (sa.ss_family == AF_UNIX) { + if (ip) { + int res = snprintf(ip, ip_len, "/unixsocket"); + if (res < 0 || (unsigned int) res >= ip_len) goto error; + } + if (port) *port = 0; + } else { + goto error; + } + return 0; + +error: + if (ip) { + if (ip_len >= 2) { + ip[0] = '?'; + ip[1] = '\0'; + } else if (ip_len == 1) { + ip[0] = '\0'; + } + } + if (port) *port = 0; + return -1; +} + +/* Create a pipe buffer with given flags for read end and write end. + * Note that it supports the file flags defined by pipe2() and fcntl(F_SETFL), + * and one of the use cases is O_CLOEXEC|O_NONBLOCK. */ +int anetPipe(int fds[2], int read_flags, int write_flags) { + int pipe_flags = 0; +#if defined(__linux__) || defined(__FreeBSD__) + /* When possible, try to leverage pipe2() to apply flags that are common to both ends. + * There is no harm to set O_CLOEXEC to prevent fd leaks. */ + pipe_flags = O_CLOEXEC | (read_flags & write_flags); + if (pipe2(fds, pipe_flags)) { + /* Fail on real failures, and fallback to simple pipe if pipe2 is unsupported. */ + if (errno != ENOSYS && errno != EINVAL) + return -1; + pipe_flags = 0; + } else { + /* If the flags on both ends are identical, no need to do anything else. */ + if ((O_CLOEXEC | read_flags) == (O_CLOEXEC | write_flags)) + return 0; + /* Clear the flags which have already been set using pipe2. */ + read_flags &= ~pipe_flags; + write_flags &= ~pipe_flags; + } +#endif + + /* When we reach here with pipe_flags of 0, it means pipe2 failed (or was not attempted), + * so we try to use pipe. Otherwise, we skip and proceed to set specific flags below. */ + if (pipe_flags == 0 && pipe(fds)) + return -1; + + /* File descriptor flags. + * Currently, only one such flag is defined: FD_CLOEXEC, the close-on-exec flag. */ + if (read_flags & O_CLOEXEC) + if (fcntl(fds[0], F_SETFD, FD_CLOEXEC)) + goto error; + if (write_flags & O_CLOEXEC) + if (fcntl(fds[1], F_SETFD, FD_CLOEXEC)) + goto error; + + /* File status flags after clearing the file descriptor flag O_CLOEXEC. */ + read_flags &= ~O_CLOEXEC; + if (read_flags) + if (fcntl(fds[0], F_SETFL, read_flags)) + goto error; + write_flags &= ~O_CLOEXEC; + if (write_flags) + if (fcntl(fds[1], F_SETFL, write_flags)) + goto error; + + return 0; + +error: + close(fds[0]); + close(fds[1]); + return -1; +} + +int anetSetSockMarkId(char *err, int fd, uint32_t id) { +#ifdef HAVE_SOCKOPTMARKID + if (setsockopt(fd, SOL_SOCKET, SOCKOPTMARKID, (void *)&id, sizeof(id)) == -1) { + anetSetError(err, "setsockopt: %s", strerror(errno)); + return ANET_ERR; + } + return ANET_OK; +#else + UNUSED(fd); + UNUSED(id); + anetSetError(err,"anetSetSockMarkid unsupported on this platform"); + return ANET_OK; +#endif +} + +int anetIsFifo(char *filepath) { + struct stat sb; + if (stat(filepath, &sb) == -1) return 0; + return S_ISFIFO(sb.st_mode); +} + +/* This function must be called after accept4() fails. It returns 1 if 'err' + * indicates accepted connection faced an error, and it's okay to continue + * accepting next connection by calling accept4() again. Other errors either + * indicate programming errors, e.g. calling accept() on a closed fd or indicate + * a resource limit has been reached, e.g. -EMFILE, open fd limit has been + * reached. In the latter case, caller might wait until resources are available. + * See accept4() documentation for details. */ +int anetAcceptFailureNeedsRetry(int err) { + if (err == ECONNABORTED) + return 1; + +#if defined(__linux__) + /* For details, see 'Error Handling' section on + * https://man7.org/linux/man-pages/man2/accept.2.html */ + if (err == ENETDOWN || err == EPROTO || err == ENOPROTOOPT || + err == EHOSTDOWN || err == ENONET || err == EHOSTUNREACH || + err == EOPNOTSUPP || err == ENETUNREACH) + { + return 1; + } +#endif + return 0; +} diff --git a/deps/ae/anet.h b/deps/ae/anet.h new file mode 100644 index 0000000..1d3aec9 --- /dev/null +++ b/deps/ae/anet.h @@ -0,0 +1,58 @@ +/* anet.c -- Basic TCP socket stuff made a bit less boring + * + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of (a) the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ + +#ifndef ANET_H +#define ANET_H + +#include + +#define ANET_OK 0 +#define ANET_ERR -1 +#define ANET_ERR_LEN 256 + +/* Flags used with certain functions. */ +#define ANET_NONE 0 +#define ANET_IP_ONLY (1<<0) +#define ANET_PREFER_IPV4 (1<<1) +#define ANET_PREFER_IPV6 (1<<2) + +#if defined(__sun) || defined(_AIX) +#define AF_LOCAL AF_UNIX +#endif + +#ifdef _AIX +#undef ip_len +#endif + +int anetTcpNonBlockConnect(char *err, const char *addr, int port); +int anetTcpNonBlockBestEffortBindConnect(char *err, const char *addr, int port, const char *source_addr); +int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len, int flags); +int anetTcpServer(char *err, int port, char *bindaddr, int backlog); +int anetTcp6Server(char *err, int port, char *bindaddr, int backlog); +int anetUnixServer(char *err, char *path, mode_t perm, int backlog); +int anetTcpAccept(char *err, int serversock, char *ip, size_t ip_len, int *port); +int anetUnixAccept(char *err, int serversock); +int anetNonBlock(char *err, int fd); +int anetBlock(char *err, int fd); +int anetCloexec(int fd); +int anetEnableTcpNoDelay(char *err, int fd); +int anetDisableTcpNoDelay(char *err, int fd); +int anetSendTimeout(char *err, int fd, long long ms); +int anetRecvTimeout(char *err, int fd, long long ms); +int anetFdToString(int fd, char *ip, size_t ip_len, int *port, int remote); +int anetKeepAlive(char *err, int fd, int interval); +int anetFormatAddr(char *fmt, size_t fmt_len, char *ip, int port); +int anetPipe(int fds[2], int read_flags, int write_flags); +int anetSetSockMarkId(char *err, int fd, uint32_t id); +int anetGetError(int fd); +int anetIsFifo(char *filepath); +int anetAcceptFailureNeedsRetry(int err); + +#endif diff --git a/deps/ae/config.h b/deps/ae/config.h new file mode 100644 index 0000000..a2fdabf --- /dev/null +++ b/deps/ae/config.h @@ -0,0 +1,375 @@ +/* + * Copyright (c) 2009-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of (a) the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ + +#ifndef __CONFIG_H +#define __CONFIG_H + +#include + +#ifdef __APPLE__ +#include // for fcntl(fd, F_FULLFSYNC) +#include +#endif + +#ifdef __linux__ +#include +#include +#endif + +#if defined(__APPLE__) && defined(__MAC_OS_X_VERSION_MAX_ALLOWED) && __MAC_OS_X_VERSION_MAX_ALLOWED >= 1060 +#define MAC_OS_10_6_DETECTED +#endif + +/* Define redis_fstat to fstat or fstat64() */ +#if defined(__APPLE__) && !defined(MAC_OS_10_6_DETECTED) +#define redis_fstat fstat64 +#define redis_stat stat64 +#else +#define redis_fstat fstat +#define redis_stat stat +#endif + +#ifndef CACHE_LINE_SIZE +#if defined(__aarch64__) && defined(__APPLE__) +#define CACHE_LINE_SIZE 128 +#else +#define CACHE_LINE_SIZE 64 +#endif +#endif + +/* Test for proc filesystem */ +#ifdef __linux__ +#define HAVE_PROC_STAT 1 +#define HAVE_PROC_MAPS 1 +#define HAVE_PROC_SMAPS 1 +#define HAVE_PROC_SOMAXCONN 1 +#define HAVE_PROC_OOM_SCORE_ADJ 1 +#define HAVE_EVENT_FD 1 +#endif + +/* Test for task_info() */ +#if defined(__APPLE__) +#define HAVE_TASKINFO 1 +#endif + +/* Test for somaxconn check */ +#if defined(__APPLE__) || defined(__FreeBSD__) +#define HAVE_SYSCTL_KIPC_SOMAXCONN 1 +#elif defined(__OpenBSD__) +#define HAVE_SYSCTL_KERN_SOMAXCONN 1 +#endif + +/* Test for backtrace() */ +#if defined(__APPLE__) || (defined(__linux__) && defined(__GLIBC__)) || \ + defined(__FreeBSD__) || ((defined(__OpenBSD__) || defined(__NetBSD__) || defined(__sun)) && defined(USE_BACKTRACE))\ + || defined(__DragonFly__) || (defined(__UCLIBC__) && defined(__UCLIBC_HAS_BACKTRACE__)) +#define HAVE_BACKTRACE 1 +#endif + +/* MSG_NOSIGNAL. */ +#ifdef __linux__ +#define HAVE_MSG_NOSIGNAL 1 +#if defined(SO_MARK) +#define HAVE_SOCKOPTMARKID 1 +#define SOCKOPTMARKID SO_MARK +#endif +#endif + +/* Test for polling API */ +#ifdef __linux__ +#define HAVE_EPOLL 1 +#endif + +/* Test for accept4() */ +#if defined(__linux__) || (defined(OpenBSD) && OpenBSD >= 201505) || \ + defined(__FreeBSD__) || \ + (defined(__NetBSD_Version__) && __NetBSD_Version__ >= 800000000) || \ + (defined(__DragonFly__) && __DragonFly_version >= 400305) +#define HAVE_ACCEPT4 1 +#endif + +#if (defined(__APPLE__) && defined(MAC_OS_10_6_DETECTED)) || defined(__FreeBSD__) || defined(__OpenBSD__) || defined (__NetBSD__) +#define HAVE_KQUEUE 1 +#endif + +#ifdef __sun +#include +#ifdef _DTRACE_VERSION +#define HAVE_EVPORT 1 +#define HAVE_PSINFO 1 +#endif +#endif + +/* Test for __builtin_prefetch() + * Supported in LLVM since 2.9: https://releases.llvm.org/2.9/docs/ReleaseNotes.html + * Supported in GCC since 3.1 but we use 4.8 given it's too old: https://gcc.gnu.org/gcc-3.1/changes.html. */ +#if defined(__clang__) && (__clang_major__ > 2 || (__clang_major__ == 2 && __clang_minor__ >= 9)) +#define HAS_BUILTIN_PREFETCH 1 +#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) +#define HAS_BUILTIN_PREFETCH 1 +#else +#define HAS_BUILTIN_PREFETCH 0 +#endif + +#if HAS_BUILTIN_PREFETCH +#define redis_prefetch_read(addr) __builtin_prefetch(addr, 0, 3) /* Read with high locality */ +#define redis_prefetch_write(addr) __builtin_prefetch(addr, 1, 3) /* Write with high locality */ +#else +#define redis_prefetch_read(addr) ((void)(addr)) /* No-op if unsupported */ +#define redis_prefetch_write(addr) ((void)(addr)) /* No-op if unsupported */ +#endif + +/* Define redis_fsync to fdatasync() in Linux and fsync() for all the rest */ +#if defined(__linux__) +#define redis_fsync(fd) fdatasync(fd) +#elif defined(__APPLE__) +#define redis_fsync(fd) fcntl(fd, F_FULLFSYNC) +#else +#define redis_fsync(fd) fsync(fd) +#endif + +#if defined(__FreeBSD__) +#if defined(SO_USER_COOKIE) +#define HAVE_SOCKOPTMARKID 1 +#define SOCKOPTMARKID SO_USER_COOKIE +#endif +#endif + +#if defined(__OpenBSD__) +#if defined(SO_RTABLE) +#define HAVE_SOCKOPTMARKID 1 +#define SOCKOPTMARKID SO_RTABLE +#endif +#endif + +#if __GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5) +#define redis_unreachable __builtin_unreachable +#else +#define redis_unreachable abort +#endif + +#if __GNUC__ >= 3 +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) +#else +#define likely(x) (x) +#define unlikely(x) (x) +#endif + +#if defined(__has_attribute) +#if __has_attribute(no_sanitize) +#define REDIS_NO_SANITIZE(sanitizer) __attribute__((no_sanitize(sanitizer))) +#endif +#endif +#if !defined(REDIS_NO_SANITIZE) +#define REDIS_NO_SANITIZE(sanitizer) +#endif + +#if defined(__clang__) +#define REDIS_NO_SANITIZE_MSAN(sanitizer) REDIS_NO_SANITIZE(sanitizer) +#else +#define REDIS_NO_SANITIZE_MSAN(sanitizer) +#endif + +/* Define rdb_fsync_range to sync_file_range() on Linux, otherwise we use + * the plain fsync() call. */ +#if (defined(__linux__) && defined(SYNC_FILE_RANGE_WAIT_BEFORE)) +#define HAVE_SYNC_FILE_RANGE 1 +#define rdb_fsync_range(fd,off,size) sync_file_range(fd,off,size,SYNC_FILE_RANGE_WAIT_BEFORE|SYNC_FILE_RANGE_WRITE) +#elif defined(__APPLE__) +#define rdb_fsync_range(fd,off,size) fcntl(fd, F_FULLFSYNC) +#else +#define rdb_fsync_range(fd,off,size) fsync(fd) +#endif + +/* Check if we can use setproctitle(). + * BSD systems have support for it, we provide an implementation for + * Linux and osx. */ +#if (defined __NetBSD__ || defined __FreeBSD__ || defined __OpenBSD__) +#define USE_SETPROCTITLE +#endif + +#if defined(__HAIKU__) +#define ESOCKTNOSUPPORT 0 +#endif + +#if (defined __linux || defined __APPLE__) +#define USE_SETPROCTITLE +#define INIT_SETPROCTITLE_REPLACEMENT +void spt_init(int argc, char *argv[]); +void setproctitle(const char *fmt, ...); +#endif + +/* Byte ordering detection */ +#include /* This will likely define BYTE_ORDER */ + +#ifndef BYTE_ORDER +#if (BSD >= 199103) +# include +#else +#if defined(linux) || defined(__linux__) +# include +#else +#define LITTLE_ENDIAN 1234 /* least-significant byte first (vax, pc) */ +#define BIG_ENDIAN 4321 /* most-significant byte first (IBM, net) */ +#define PDP_ENDIAN 3412 /* LSB first in word, MSW first in long (pdp)*/ + +#if defined(__i386__) || defined(__x86_64__) || defined(__amd64__) || \ + defined(vax) || defined(ns32000) || defined(sun386) || \ + defined(MIPSEL) || defined(_MIPSEL) || defined(BIT_ZERO_ON_RIGHT) || \ + defined(__alpha__) || defined(__alpha) +#define BYTE_ORDER LITTLE_ENDIAN +#endif + +#if defined(sel) || defined(pyr) || defined(mc68000) || defined(sparc) || \ + defined(is68k) || defined(tahoe) || defined(ibm032) || defined(ibm370) || \ + defined(MIPSEB) || defined(_MIPSEB) || defined(_IBMR2) || defined(DGUX) ||\ + defined(apollo) || defined(__convex__) || defined(_CRAY) || \ + defined(__hppa) || defined(__hp9000) || \ + defined(__hp9000s300) || defined(__hp9000s700) || \ + defined (BIT_ZERO_ON_LEFT) || defined(m68k) || defined(__sparc) +#define BYTE_ORDER BIG_ENDIAN +#endif +#endif /* linux */ +#endif /* BSD */ +#endif /* BYTE_ORDER */ + +/* Sometimes after including an OS-specific header that defines the + * endianness we end with __BYTE_ORDER but not with BYTE_ORDER that is what + * the Redis code uses. In this case let's define everything without the + * underscores. */ +#ifndef BYTE_ORDER +#ifdef __BYTE_ORDER +#if defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN) +#ifndef LITTLE_ENDIAN +#define LITTLE_ENDIAN __LITTLE_ENDIAN +#endif +#ifndef BIG_ENDIAN +#define BIG_ENDIAN __BIG_ENDIAN +#endif +#if (__BYTE_ORDER == __LITTLE_ENDIAN) +#define BYTE_ORDER LITTLE_ENDIAN +#else +#define BYTE_ORDER BIG_ENDIAN +#endif +#endif +#endif +#endif + +#if !defined(BYTE_ORDER) || \ + (BYTE_ORDER != BIG_ENDIAN && BYTE_ORDER != LITTLE_ENDIAN) + /* you must determine what the correct bit order is for + * your compiler - the next line is an intentional error + * which will force your compiles to bomb until you fix + * the above macros. + */ +#error "Undefined or invalid BYTE_ORDER" +#endif + +#if (__i386 || __amd64 || __powerpc__) && __GNUC__ +#define GNUC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) +#if defined(__clang__) +#define HAVE_ATOMIC +#endif +#if (defined(__GLIBC__) && defined(__GLIBC_PREREQ)) +#if (GNUC_VERSION >= 40100 && __GLIBC_PREREQ(2, 6)) +#define HAVE_ATOMIC +#endif +#endif +#endif + +/* Make sure we can test for ARM just checking for __arm__, since sometimes + * __arm is defined but __arm__ is not. */ +#if defined(__arm) && !defined(__arm__) +#define __arm__ +#endif +#if defined (__aarch64__) && !defined(__arm64__) +#define __arm64__ +#endif + +/* Make sure we can test for SPARC just checking for __sparc__. */ +#if defined(__sparc) && !defined(__sparc__) +#define __sparc__ +#endif + +#if defined(__sparc__) || defined(__arm__) +#define USE_ALIGNED_ACCESS +#endif + +/* Define for redis_set_thread_title */ +#ifdef __linux__ +#define redis_set_thread_title(name) pthread_setname_np(pthread_self(), name) +#else +#if (defined __FreeBSD__ || defined __OpenBSD__) +#include +#define redis_set_thread_title(name) pthread_set_name_np(pthread_self(), name) +#elif defined __NetBSD__ +#include +#define redis_set_thread_title(name) pthread_setname_np(pthread_self(), "%s", name) +#elif defined __HAIKU__ +#include +#define redis_set_thread_title(name) rename_thread(find_thread(0), name) +#else +#if (defined __APPLE__ && defined(__MAC_OS_X_VERSION_MAX_ALLOWED) && __MAC_OS_X_VERSION_MAX_ALLOWED >= 1070) +int pthread_setname_np(const char *name); +#include +#define redis_set_thread_title(name) pthread_setname_np(name) +#else +#define redis_set_thread_title(name) +#endif +#endif +#endif + +/* Check if we can use setcpuaffinity(). */ +#if (defined __linux || defined __NetBSD__ || defined __FreeBSD__ || defined __DragonFly__) +#define USE_SETCPUAFFINITY +void setcpuaffinity(const char *cpulist); +#endif + +/* Test for posix_fadvise() */ +#if defined(__linux__) || defined(__FreeBSD__) +#define HAVE_FADVISE +#endif + +#if defined(__x86_64__) && ((defined(__GNUC__) && __GNUC__ > 5) || (defined(__clang__))) + #if defined(__has_attribute) && __has_attribute(target) + #define HAVE_POPCNT + #define ATTRIBUTE_TARGET_POPCNT __attribute__((target("popcnt"))) + #else + #define ATTRIBUTE_TARGET_POPCNT + #endif +#else + #define ATTRIBUTE_TARGET_POPCNT +#endif + +/* Check if we can compile AVX2 code */ +#if defined (__x86_64__) && ((defined(__GNUC__) && __GNUC__ >= 5) || (defined(__clang__) && __clang_major__ >= 4)) +#if defined(__has_attribute) && __has_attribute(target) +#define HAVE_AVX2 +#define ATTRIBUTE_TARGET_AVX2 __attribute__((target("avx2"))) +#define ATTRIBUTE_TARGET_AVX2_POPCOUNT __attribute__((target("avx2,popcnt"))) +#endif +#endif + +/* Check if we can compile AVX512 code */ +#if defined (__x86_64__) && ((defined(__GNUC__) && __GNUC__ >= 5) || (defined(__clang__) && __clang_major__ >= 4)) +#if defined(__has_attribute) && __has_attribute(target) +#define HAVE_AVX512 +#define ATTRIBUTE_TARGET_AVX512_POPCOUNT __attribute__((target("avx512f,avx512vpopcntdq"))) +#endif +#endif + +/* Check for AArch64 (ARM v8) specific optimizations */ +#if defined(__aarch64__) && ((defined(__GNUC__) && __GNUC__ >= 5) || defined(__clang__)) +#if defined(__has_attribute) && __has_attribute(target) +#define HAVE_AARCH64_NEON +#endif +#endif + +#endif diff --git a/deps/ae/fmacros.h b/deps/ae/fmacros.h new file mode 100644 index 0000000..d05230d --- /dev/null +++ b/deps/ae/fmacros.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2009-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of (a) the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ + +#ifndef _REDIS_FMACRO_H +#define _REDIS_FMACRO_H + +#define _BSD_SOURCE + +#if defined(__linux__) +#define _GNU_SOURCE +#define _DEFAULT_SOURCE +#endif + +#if defined(_AIX) +#define _ALL_SOURCE +#endif + +#if defined(__linux__) || defined(__OpenBSD__) +#define _XOPEN_SOURCE 700 +/* + * On NetBSD, _XOPEN_SOURCE undefines _NETBSD_SOURCE and + * thus hides inet_aton etc. + */ +#elif !defined(__NetBSD__) +#define _XOPEN_SOURCE +#endif + +#if defined(__sun) +#define _POSIX_C_SOURCE 199506L +#endif + +#define _LARGEFILE_SOURCE +#define _FILE_OFFSET_BITS 64 + +/* deprecate unsafe functions + * + * NOTE: We do not use the poison pragma since it + * will error on stdlib definitions in files as well*/ +#if (__GNUC__ && __GNUC__ >= 4) && !defined __APPLE__ +int sprintf(char *str, const char *format, ...) __attribute__((deprecated("please avoid use of unsafe C functions. prefer use of snprintf instead"))); +char *strcpy(char *restrict dest, const char *src) __attribute__((deprecated("please avoid use of unsafe C functions. prefer use of redis_strlcpy instead"))); +char *strcat(char *restrict dest, const char *restrict src) __attribute__((deprecated("please avoid use of unsafe C functions. prefer use of redis_strlcat instead"))); +#endif + +#ifdef __linux__ +/* features.h uses the defines above to set feature specific defines. */ +#include +#endif + +#endif diff --git a/deps/ae/monotonic.c b/deps/ae/monotonic.c new file mode 100644 index 0000000..b198fe3 --- /dev/null +++ b/deps/ae/monotonic.c @@ -0,0 +1,247 @@ +#include "monotonic.h" +#include +#include +#include +#include +#include "redisassert.h" +#include + +/* The function pointer for clock retrieval. */ +monotime (*getMonotonicUs)(void) = NULL; + +static char monotonic_info_string[32]; + + +/* Using the processor clock (aka TSC on x86) can provide improved performance + * throughout Redis wherever the monotonic clock is used. The processor clock + * is significantly faster than calling 'clock_getting' (POSIX). While this is + * generally safe on modern systems, this link provides additional information + * about use of the x86 TSC: http://oliveryang.net/2015/09/pitfalls-of-TSC-usage + * + * On ARM aarch64 systems, the hardware clock is enabled by default because the + * ARM Generic Timer is architecturally guaranteed to be available and monotonic + * on all ARMv8-A processors (see the “The Generic Timer in AArch64 state” + * section of the Arm Architecture Reference Manual for Armv8-A). + * + * To use the processor clock on other architectures, either uncomment this line, + * or build with + * CFLAGS="-DUSE_PROCESSOR_CLOCK" +#define USE_PROCESSOR_CLOCK + */ + + +#if defined(USE_PROCESSOR_CLOCK) && defined(__x86_64__) && defined(__linux__) +#include +#include + +static long mono_ticksPerMicrosecond = 0; + +static monotime getMonotonicUs_x86(void) { + return __rdtsc() / mono_ticksPerMicrosecond; +} + +static void monotonicInit_x86linux(void) { + const int bufflen = 256; + char buf[bufflen]; + regex_t cpuGhzRegex, constTscRegex; + const size_t nmatch = 2; + regmatch_t pmatch[nmatch]; + int constantTsc = 0; + int rc; + + /* Determine the number of TSC ticks in a micro-second. This is + * a constant value matching the standard speed of the processor. + * On modern processors, this speed remains constant even though + * the actual clock speed varies dynamically for each core. */ + rc = regcomp(&cpuGhzRegex, "^model name\\s+:.*@ ([0-9.]+)GHz", REG_EXTENDED); + assert(rc == 0); + + /* Also check that the constant_tsc flag is present. (It should be + * unless this is a really old CPU. */ + rc = regcomp(&constTscRegex, "^flags\\s+:.* constant_tsc", REG_EXTENDED); + assert(rc == 0); + + FILE *cpuinfo = fopen("/proc/cpuinfo", "r"); + if (cpuinfo != NULL) { + while (fgets(buf, bufflen, cpuinfo) != NULL) { + if (regexec(&cpuGhzRegex, buf, nmatch, pmatch, 0) == 0) { + buf[pmatch[1].rm_eo] = '\0'; + double ghz = atof(&buf[pmatch[1].rm_so]); + mono_ticksPerMicrosecond = (long)(ghz * 1000); + break; + } + } + while (fgets(buf, bufflen, cpuinfo) != NULL) { + if (regexec(&constTscRegex, buf, nmatch, pmatch, 0) == 0) { + constantTsc = 1; + break; + } + } + + fclose(cpuinfo); + } + regfree(&cpuGhzRegex); + regfree(&constTscRegex); + + if (mono_ticksPerMicrosecond == 0) { + fprintf(stderr, "monotonic: x86 linux, unable to determine clock rate\n"); + return; + } + if (!constantTsc) { + fprintf(stderr, "monotonic: x86 linux, 'constant_tsc' flag not present\n"); + return; + } + + snprintf(monotonic_info_string, sizeof(monotonic_info_string), + "X86 TSC @ %ld ticks/us", mono_ticksPerMicrosecond); + getMonotonicUs = getMonotonicUs_x86; +} +#endif + +#if defined(__aarch64__) +static long mono_ticksPerMicrosecond = 0; + +/* Read the clock value. + * CNTVCT_EL0 is a system counter register, that provides the monotonic + * timestamp as a 64-bit count value. */ +static inline uint64_t __cntvct(void) { + uint64_t virtual_timer_value; + __asm__ volatile("mrs %0, cntvct_el0" : "=r"(virtual_timer_value)); + return virtual_timer_value; +} + +/* Read the Count-timer Frequency. + * CNTFRQ_EL0 is a system counter register that provides the frequency (in Hz) + * needed to convert ticks to microseconds. Together with CNTVCT_EL0, this enables + * high-performance monotonic time measurement without system calls. */ +static inline uint32_t cntfrq_hz(void) { + uint64_t virtual_freq_value; + __asm__ volatile("mrs %0, cntfrq_el0" : "=r"(virtual_freq_value)); + return (uint32_t)virtual_freq_value; /* top 32 bits are reserved */ +} + +static monotime getMonotonicUs_aarch64(void) { + return __cntvct() / mono_ticksPerMicrosecond; +} + +static void monotonicInit_aarch64(void) { + mono_ticksPerMicrosecond = (long)cntfrq_hz() / 1000L / 1000L; + if (mono_ticksPerMicrosecond == 0) { + fprintf(stderr, "monotonic: aarch64, unable to determine clock rate\n"); + return; + } + + snprintf(monotonic_info_string, sizeof(monotonic_info_string), + "ARM CNTVCT @ %ld ticks/us", mono_ticksPerMicrosecond); + getMonotonicUs = getMonotonicUs_aarch64; +} +#endif + + +#if defined(USE_PROCESSOR_CLOCK) && defined(__riscv) && defined(__linux__) +static long mono_ticksPerMicrosecond = 0; + +static inline uint64_t read_mtime(void) { + uint64_t val; + asm volatile("csrr %0, time" : "=r"(val)); + return val; +} + +/* Read RISC-V timebase-frequency, which may be stored as either a 64-bit + * or 32-bit big-endian integer in the device tree. */ +static uint64_t get_timebase_frequency(void) { + uint64_t freq = 0; + FILE *fp = fopen("/proc/device-tree/cpus/timebase-frequency", "rb"); + if (!fp) + return 0; + + uint8_t buf[8] = {0}; + size_t cnt = fread(buf, 1, sizeof(buf), fp); + fclose(fp); + + if (cnt == 8) { + uint64_t be64 = 0; + memcpy(&be64, buf, sizeof(be64)); + /* Convert be64 from big-endian to little-endian. */ + freq = __builtin_bswap64(be64); + } else if (cnt == 4) { + uint32_t be32 = 0; + memcpy(&be32, buf, sizeof(be32)); + /* Convert be32 from big-endian to little-endian. */ + freq = __builtin_bswap32(be32); + } else { + /* Unable to read timebase-frequency. */ + return 0; + } + + return freq; +} + +static monotime getMonotonicUs_riscv(void) { + return read_mtime() / mono_ticksPerMicrosecond; +} + +static void monotonicInit_riscv(void) { + mono_ticksPerMicrosecond = (long)get_timebase_frequency() / 1000L / 1000L; + if (mono_ticksPerMicrosecond == 0) { + fprintf(stderr, "monotonic: riscv, unable to determine clock rate\n"); + return; + } + snprintf(monotonic_info_string, sizeof(monotonic_info_string), + "RISC-V mtime @ %ld ticks/us", mono_ticksPerMicrosecond); + getMonotonicUs = getMonotonicUs_riscv; +} +#endif + +static monotime getMonotonicUs_posix(void) { + /* clock_gettime() is specified in POSIX.1b (1993). Even so, some systems + * did not support this until much later. CLOCK_MONOTONIC is technically + * optional and may not be supported - but it appears to be universal. + * If this is not supported, provide a system-specific alternate version. */ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return ((uint64_t)ts.tv_sec) * 1000000 + ts.tv_nsec / 1000; +} + +static void monotonicInit_posix(void) { + /* Ensure that CLOCK_MONOTONIC is supported. This should be supported + * on any reasonably current OS. If the assertion below fails, provide + * an appropriate alternate implementation. */ + struct timespec ts; + int rc = clock_gettime(CLOCK_MONOTONIC, &ts); + assert(rc == 0); + + snprintf(monotonic_info_string, sizeof(monotonic_info_string), + "POSIX clock_gettime"); + getMonotonicUs = getMonotonicUs_posix; +} + + + +const char * monotonicInit(void) { + #if defined(USE_PROCESSOR_CLOCK) && defined(__x86_64__) && defined(__linux__) + if (getMonotonicUs == NULL) monotonicInit_x86linux(); + #endif + + #if defined(__aarch64__) + if (getMonotonicUs == NULL) monotonicInit_aarch64(); + #endif + + #if defined(USE_PROCESSOR_CLOCK) && defined(__riscv) && defined(__linux__) + if (getMonotonicUs == NULL) monotonicInit_riscv(); + #endif + + if (getMonotonicUs == NULL) monotonicInit_posix(); + + return monotonic_info_string; +} + +const char *monotonicInfoString(void) { + return monotonic_info_string; +} + +monotonic_clock_type monotonicGetType(void) { + if (getMonotonicUs == getMonotonicUs_posix) + return MONOTONIC_CLOCK_POSIX; + return MONOTONIC_CLOCK_HW; +} diff --git a/deps/ae/monotonic.h b/deps/ae/monotonic.h new file mode 100644 index 0000000..b465f90 --- /dev/null +++ b/deps/ae/monotonic.h @@ -0,0 +1,61 @@ +#ifndef __MONOTONIC_H +#define __MONOTONIC_H +/* The monotonic clock is an always increasing clock source. It is unrelated to + * the actual time of day and should only be used for relative timings. The + * monotonic clock is also not guaranteed to be chronologically precise; there + * may be slight skew/shift from a precise clock. + * + * Depending on system architecture, the monotonic time may be able to be + * retrieved much faster than a normal clock source by using an instruction + * counter on the CPU. On x86 architectures (for example), the RDTSC + * instruction is a very fast clock source for this purpose. + */ + +#include "fmacros.h" +#include +#include + +/* A counter in micro-seconds. The 'monotime' type is provided for variables + * holding a monotonic time. This will help distinguish & document that the + * variable is associated with the monotonic clock and should not be confused + * with other types of time.*/ +typedef uint64_t monotime; + +/* Retrieve counter of micro-seconds relative to an arbitrary point in time. */ +extern monotime (*getMonotonicUs)(void); + +typedef enum monotonic_clock_type { + MONOTONIC_CLOCK_POSIX, + MONOTONIC_CLOCK_HW, +} monotonic_clock_type; + +/* Call once at startup to initialize the monotonic clock. Though this only + * needs to be called once, it may be called additional times without impact. + * Returns a printable string indicating the type of clock initialized. + * (The returned string is static and doesn't need to be freed.) */ +const char *monotonicInit(void); + +/* Return a string indicating the type of monotonic clock being used. */ +const char *monotonicInfoString(void); + +/* Return the type of monotonic clock being used. */ +monotonic_clock_type monotonicGetType(void); + +/* Functions to measure elapsed time. Example: + * monotime myTimer; + * elapsedStart(&myTimer); + * while (elapsedMs(myTimer) < 10) {} // loops for 10ms + */ +static inline void elapsedStart(monotime *start_time) { + *start_time = getMonotonicUs(); +} + +static inline uint64_t elapsedUs(monotime start_time) { + return getMonotonicUs() - start_time; +} + +static inline uint64_t elapsedMs(monotime start_time) { + return elapsedUs(start_time) / 1000; +} + +#endif diff --git a/deps/ae/redisassert.h b/deps/ae/redisassert.h new file mode 100644 index 0000000..8abecfb --- /dev/null +++ b/deps/ae/redisassert.h @@ -0,0 +1,18 @@ +/* redisassert.h - simplified assert for standalone ae library */ +#ifndef __REDIS_ASSERT_H__ +#define __REDIS_ASSERT_H__ + +#include +#include +#include + +/* config.h provides redis_unreachable for standalone AE builds. */ + +/* Panic function for error handling */ +#define panic(...) do { \ + fprintf(stderr, "PANIC: " __VA_ARGS__); \ + fprintf(stderr, "\n"); \ + abort(); \ +} while(0) + +#endif diff --git a/deps/ae/util.h b/deps/ae/util.h new file mode 100644 index 0000000..630c4ca --- /dev/null +++ b/deps/ae/util.h @@ -0,0 +1,21 @@ +/* util.h - minimal utilities needed by anet */ +#ifndef __UTIL_H +#define __UTIL_H + +#include + +/* Simple error buffer size */ +#define ANET_ERR_LEN 256 + +/* Safe string copy - simplified version */ +static inline size_t redis_strlcpy(char *dst, const char *src, size_t dsize) { + size_t len = strlen(src); + if (dsize > 0) { + size_t copy_len = (len >= dsize) ? dsize - 1 : len; + memcpy(dst, src, copy_len); + dst[copy_len] = '\0'; + } + return len; +} + +#endif diff --git a/deps/ae/zmalloc.h b/deps/ae/zmalloc.h new file mode 100644 index 0000000..86e9d92 --- /dev/null +++ b/deps/ae/zmalloc.h @@ -0,0 +1,15 @@ +/* zmalloc.h - simplified memory allocation wrapper for ae library + * This is a minimal version for standalone use outside Redis. + */ +#ifndef __ZMALLOC_H +#define __ZMALLOC_H + +#include + +/* Simple wrappers around standard malloc/free */ +#define zmalloc malloc +#define zfree free +#define zrealloc realloc +#define zcalloc calloc + +#endif diff --git a/main.c b/main.c index 2c3a0ae..1eba511 100644 --- a/main.c +++ b/main.c @@ -1,6 +1,7 @@ /* main.c - pplx_embed command-line tool */ #include "pplx_embed.h" +#include "pplx_server.h" #include "qwen_asr_kernels.h" #include "qwen_asr_tokenizer.h" @@ -29,8 +30,17 @@ static void print_usage(const char *prog) "Options:\n" " -d Model directory (required)\n" " --mlx Use Apple MLX GPU backend\n" - " --daemon Read lines from stdin, write JSON embeddings to stdout\n" - " -b Max texts per engine batch (default: all; daemon: 1)\n" + " --stdin Read lines from stdin, write JSON embeddings to stdout\n" + " --serve Serve the Perplexity-compatible HTTP API\n" + " --model ID=DIR\n" + " Load model ID from directory for --serve (repeatable)\n" + " --backend cpu|mlx\n" + " Backend for --serve (default: cpu)\n" + " --host HOST Server bind host (default: 127.0.0.1)\n" + " --port N Server bind port (default: 8000)\n" + " --cors Enable CORS headers in --serve mode\n" + " --api-key K Require Authorization: Bearer K in --serve mode\n" + " -b Max texts per engine batch (default: all; stdin/server: 1)\n" " -t CPU threads (default: all cores)\n" " -e Print raw embeddings (with multiple texts)\n" " -v Verbose (-vv for debug)\n" @@ -40,12 +50,14 @@ static void print_usage(const char *prog) " 1 text arg Print embedding as space-separated floats\n" " 2+ text args Print cosine similarity matrix\n" " no args Batch: read all stdin lines, then similarity matrix\n" - " --daemon Streaming: read stdin lines, write JSON per line\n" + " --stdin Streaming: read stdin lines, write JSON per line\n" + " --serve HTTP server for /v1/embeddings and /v1/contextualizedembeddings\n" "\n" "Examples:\n" " %s -d ./model \"query: what is AI?\"\n" - " %s -d ./model --mlx --daemon < texts.txt\n", - prog, prog, prog); + " %s -d ./model --mlx --stdin < texts.txt\n" + " %s --serve --model pplx-embed-v1-0.6b=./model --port 8000\n", + prog, prog, prog, prog); } /* ======================================================================== @@ -67,6 +79,12 @@ typedef struct { int n_tokens; } token_buf_t; +typedef struct { + pplx_server_model_spec_t *v; + int n; + int cap; +} model_specs_t; + static int engine_embed_batch(engine_t *e, const pplx_input_t *inputs, int batch, float *out_embeddings) { @@ -108,6 +126,45 @@ static void free_tokens(token_buf_t *tokens, int n) free(tokens[i].ids); } +static int append_model_spec(model_specs_t *specs, const char *arg) +{ + const char *eq = strchr(arg, '='); + if (!eq || eq == arg || !eq[1]) { + fprintf(stderr, "--model expects MODEL_ID=DIR\n"); + return -1; + } + if (specs->n == specs->cap) { + int new_cap = specs->cap ? specs->cap * 2 : 4; + void *p = realloc(specs->v, (size_t)new_cap * sizeof(specs->v[0])); + if (!p) return -1; + specs->v = p; + specs->cap = new_cap; + } + size_t id_len = (size_t)(eq - arg); + char *id = (char *)malloc(id_len + 1); + char *path = strdup(eq + 1); + if (id) { + memcpy(id, arg, id_len); + id[id_len] = '\0'; + } + specs->v[specs->n].id = id; + specs->v[specs->n].path = path; + if (!specs->v[specs->n].id || !specs->v[specs->n].path) + return -1; + specs->n++; + return 0; +} + +static void free_model_specs(model_specs_t *specs) +{ + if (!specs) return; + for (int i = 0; i < specs->n; i++) { + free((char *)specs->v[i].id); + free((char *)specs->v[i].path); + } + free(specs->v); +} + /* ======================================================================== * Output helpers * ======================================================================== */ @@ -140,10 +197,10 @@ static void print_embedding_json(const float *emb, int dim, int n_tokens, } /* ======================================================================== - * Daemon mode: read lines from stdin, write JSON to stdout + * Stdin mode: read lines from stdin, write JSON to stdout * ======================================================================== */ -static int process_daemon_batch(engine_t *e, char **lines, int n_lines) +static int process_stdin_batch(engine_t *e, char **lines, int n_lines) { if (n_lines <= 0) return 0; @@ -208,13 +265,13 @@ static int process_daemon_batch(engine_t *e, char **lines, int n_lines) return rc == 0 ? 0 : 1; } -static int run_daemon(engine_t *e, int batch_size) +static int run_stdin(engine_t *e, int batch_size) { char line[65536]; if (batch_size <= 0) batch_size = 1; if (pplx_verbose >= 1) - fprintf(stderr, "daemon: ready, reading from stdin (batch_size=%d)\n", + fprintf(stderr, "stdin: ready, reading from stdin (batch_size=%d)\n", batch_size); char **batch_lines = (char **)calloc((size_t)batch_size, sizeof(char *)); @@ -233,7 +290,7 @@ static int run_daemon(engine_t *e, int batch_size) if (len == 0) continue; if (pplx_verbose >= 1) - fprintf(stderr, "daemon: \"%.*s%s\"\n", + fprintf(stderr, "stdin: \"%.*s%s\"\n", (int)(len > 60 ? 60 : len), line, len > 60 ? "..." : ""); batch_lines[n_batch] = strdup(line); @@ -245,7 +302,7 @@ static int run_daemon(engine_t *e, int batch_size) n_batch++; if (n_batch == batch_size) { - if (process_daemon_batch(e, batch_lines, n_batch) != 0) + if (process_stdin_batch(e, batch_lines, n_batch) != 0) rc = 1; for (int i = 0; i < n_batch; i++) { free(batch_lines[i]); @@ -256,7 +313,7 @@ static int run_daemon(engine_t *e, int batch_size) } if (n_batch > 0) { - if (process_daemon_batch(e, batch_lines, n_batch) != 0) + if (process_stdin_batch(e, batch_lines, n_batch) != 0) rc = 1; for (int i = 0; i < n_batch; i++) free(batch_lines[i]); @@ -264,7 +321,7 @@ static int run_daemon(engine_t *e, int batch_size) free(batch_lines); if (pplx_verbose >= 1) - fprintf(stderr, "daemon: stdin EOF\n"); + fprintf(stderr, "stdin: EOF\n"); return rc; } @@ -406,8 +463,15 @@ int main(int argc, char *argv[]) int print_embs = 0; int verbose = 0; int use_mlx = 0; - int daemon = 0; + int stdin_mode = 0; + int serve_mode = 0; int batch_size = 0; + const char *backend = "cpu"; + const char *host = "127.0.0.1"; + const char *api_key = NULL; + int port = 8000; + int cors = 0; + model_specs_t model_specs = {0}; int arg_start = 1; while (arg_start < argc && argv[arg_start][0] == '-') { @@ -417,31 +481,105 @@ int main(int argc, char *argv[]) else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) { batch_size = atoi(argv[++arg_start]); } + else if (!strcmp(f, "--model")) { + if (append_model_spec(&model_specs, argv[++arg_start]) != 0) return 1; + } + else if (!strcmp(f, "--backend")) { + backend = argv[++arg_start]; + if (!strcmp(backend, "mlx")) { +#ifdef USE_MLX + use_mlx = 1; +#else + fprintf(stderr, "mlx backend not available (build with: make mlx)\n"); + free_model_specs(&model_specs); + return 1; +#endif + } else if (!strcmp(backend, "cpu")) { + use_mlx = 0; + } else { + fprintf(stderr, "invalid --backend: %s\n", backend); + free_model_specs(&model_specs); + return 1; + } + } + else if (!strcmp(f, "--host")) { host = argv[++arg_start]; } + else if (!strcmp(f, "--port")) { port = atoi(argv[++arg_start]); } + else if (!strcmp(f, "--cors")) { cors = 1; } + else if (!strcmp(f, "--api-key")){ api_key = argv[++arg_start]; } else if (!strcmp(f, "-e")) { print_embs = 1; } else if (!strcmp(f, "-v")) { verbose++; } else if (!strcmp(f, "-vv")) { verbose = 2; } - else if (!strcmp(f, "--daemon")){ daemon = 1; } + else if (!strcmp(f, "--stdin")) { stdin_mode = 1; } + else if (!strcmp(f, "--serve")) { serve_mode = 1; } else if (!strcmp(f, "--mlx")) { #ifdef USE_MLX use_mlx = 1; + backend = "mlx"; #else - fprintf(stderr, "--mlx not available (build with: make mlx)\n"); return 1; + fprintf(stderr, "--mlx not available (build with: make mlx)\n"); + free_model_specs(&model_specs); + return 1; #endif } - else if (!strcmp(f, "-h") || !strcmp(f, "--help")) { print_usage(argv[0]); return 0; } + else if (!strcmp(f, "-h") || !strcmp(f, "--help")) { + print_usage(argv[0]); + free_model_specs(&model_specs); + return 0; + } else break; arg_start++; } + pplx_verbose = verbose; + qwen_verbose = verbose; + + if (serve_mode) { + if (stdin_mode) { + fprintf(stderr, "--serve and --stdin are mutually exclusive\n"); + free_model_specs(&model_specs); + return 1; + } + if (model_specs.n == 0) { + fprintf(stderr, "--serve requires at least one --model MODEL_ID=DIR\n"); + print_usage(argv[0]); + free_model_specs(&model_specs); + return 1; + } + if (!use_mlx) { + if (n_threads <= 0) n_threads = qwen_get_num_cpus(); + qwen_set_threads(n_threads); + if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads); + } else if (verbose >= 1) { + fprintf(stderr, "Using MLX GPU backend\n"); + } + pplx_server_config_t scfg = { + .models = model_specs.v, + .n_models = model_specs.n, + .host = host, + .port = port, + .batch_size = batch_size > 0 ? batch_size : 1, + .use_mlx = use_mlx, + .enable_cors = cors, + .api_key = api_key, + }; + int rc = pplx_run_server(&scfg); + free_model_specs(&model_specs); + return rc; + } + + if (model_specs.n != 0) { + fprintf(stderr, "--model is only valid with --serve\n"); + free_model_specs(&model_specs); + return 1; + } + if (!model_dir || !model_dir[0]) { fprintf(stderr, "model directory required (-d )\n"); print_usage(argv[0]); + free_model_specs(&model_specs); return 1; } - pplx_verbose = verbose; - qwen_verbose = verbose; - /* Threads (CPU backend) */ if (!use_mlx) { if (n_threads <= 0) n_threads = qwen_get_num_cpus(); @@ -497,8 +635,8 @@ int main(int argc, char *argv[]) /* Run */ int rc; - if (daemon) - rc = run_daemon(&e, batch_size > 0 ? batch_size : 1); + if (stdin_mode) + rc = run_stdin(&e, batch_size > 0 ? batch_size : 1); else rc = run_batch(&e, argc, argv, arg_start, print_embs, batch_size); @@ -509,5 +647,6 @@ int main(int argc, char *argv[]) if (e.mlx_ctx) pplx_mlx_free(e.mlx_ctx); #endif qwen_tokenizer_free(tok); + free_model_specs(&model_specs); return rc; } diff --git a/pplx_embed.c b/pplx_embed.c index 9260433..e4ea605 100644 --- a/pplx_embed.c +++ b/pplx_embed.c @@ -961,6 +961,68 @@ int pplx_model_forward_into(const pplx_model_t *model, pplx_workspace_t *ws, return 0; } +int pplx_pool_spans(const pplx_config_t *cfg, const float *states, int n_tokens, + const pplx_span_t *spans, int n_spans, + float *out_embeddings) +{ + if (!cfg || !states || n_tokens <= 0 || !spans || n_spans <= 0 || + !out_embeddings) + return -1; + + int hidden = cfg->hidden_size; + if (hidden <= 0) return -1; + + for (int s = 0; s < n_spans; s++) { + int start = spans[s].start; + int len = spans[s].n_tokens; + if (start < 0 || len <= 0 || start > n_tokens || len > n_tokens - start) + return -1; + + float *emb = out_embeddings + (size_t)s * hidden; + memset(emb, 0, (size_t)hidden * sizeof(float)); + + for (int t = 0; t < len; t++) { + const float *row = states + (size_t)(start + t) * hidden; + for (int d = 0; d < hidden; d++) + emb[d] += row[d]; + } + + float inv_len = 1.0f / (float)len; + float norm_sq = 0.0f; + for (int d = 0; d < hidden; d++) { + emb[d] *= inv_len; + norm_sq += emb[d] * emb[d]; + } + + float nv = sqrtf(norm_sq); + if (nv > 1e-8f) { + float inv = 1.0f / nv; + for (int d = 0; d < hidden; d++) + emb[d] *= inv; + } + } + return 0; +} + +int pplx_model_embed_spans(const pplx_model_t *model, pplx_workspace_t *ws, + const int *token_ids, int n_tokens, + const pplx_span_t *spans, int n_spans, + float *out_embeddings) +{ + if (!model || !ws || !token_ids || n_tokens <= 0 || !spans || + n_spans <= 0 || !out_embeddings) + return -1; + + pplx_input_t input = { token_ids, n_tokens }; + int offsets[2] = { 0, n_tokens }; + if (forward_packed_inplace(model, ws, &input, 1, offsets, + n_tokens, n_tokens, 1) != 0) + return -1; + + return pplx_pool_spans(&model->config, ws->x, n_tokens, spans, n_spans, + out_embeddings); +} + float *pplx_model_forward(const pplx_model_t *model, pplx_workspace_t *ws, const int *token_ids, int n_tokens) { diff --git a/pplx_embed.h b/pplx_embed.h index 23ca2a9..ba46639 100644 --- a/pplx_embed.h +++ b/pplx_embed.h @@ -48,6 +48,11 @@ typedef struct { int n_tokens; } pplx_input_t; +typedef struct { + int start; + int n_tokens; +} pplx_span_t; + /* ======================================================================== * API * ======================================================================== */ @@ -136,6 +141,26 @@ int pplx_model_forward_into(const pplx_model_t *model, pplx_workspace_t *ws, const int *token_ids, int n_tokens, float *out_states); +/* + * Pool normalized embeddings from final hidden states. + * + * states is [n_tokens, hidden_size], normally produced by + * pplx_model_forward_into(). Each span is mean pooled and L2 normalized into + * out_embeddings[n_spans, hidden_size]. + */ +int pplx_pool_spans(const pplx_config_t *cfg, const float *states, int n_tokens, + const pplx_span_t *spans, int n_spans, + float *out_embeddings); + +/* + * Run one contextual sequence and pool selected token spans. + * Returns 0 on success, -1 on error. + */ +int pplx_model_embed_spans(const pplx_model_t *model, pplx_workspace_t *ws, + const int *token_ids, int n_tokens, + const pplx_span_t *spans, int n_spans, + float *out_embeddings); + /* * Cosine similarity between two L2-normalized vectors. */ diff --git a/pplx_embed_mlx.c b/pplx_embed_mlx.c index c6991c5..305d235 100644 --- a/pplx_embed_mlx.c +++ b/pplx_embed_mlx.c @@ -484,6 +484,162 @@ int pplx_mlx_embed_into(pplx_mlx_ctx_t *ctx, const int *token_ids, return pplx_mlx_embed_batch(ctx, &input, 1, out_embedding); } +static int pplx_mlx_forward_into(pplx_mlx_ctx_t *ctx, const int *token_ids, + int n_tokens, float *out_states) +{ + if (!ctx || !token_ids || n_tokens <= 0 || !out_states) return -1; + + const pplx_config_t *c = &ctx->config; + mlx_stream S = ctx->stream; + + int hidden = c->hidden_size; + int n_heads = c->n_heads; + int n_kv_heads = c->n_kv_heads; + int head_dim = c->head_dim; + int q_dim = c->q_dim; + + for (int i = 0; i < n_tokens; i++) { + if (token_ids[i] < 0 || token_ids[i] >= c->vocab_size) { + fprintf(stderr, "mlx: invalid token id %d at position %d\n", + token_ids[i], i); + return -1; + } + } + + int ids_shape[] = {1, n_tokens}; + mlx_array ids = mlx_array_new_data((void *)token_ids, ids_shape, 2, MLX_INT32); + if (!arr_ok(ids)) return -1; + + float scale = 1.0f / sqrtf((float)head_dim); + mlx_array null_arr = (mlx_array){0}; + + mlx_array x = mlx_array_new(); + mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S); + mlx_array_free(ids); + + for (int layer = 0; layer < c->n_layers; layer++) { + mlx_layer_t *l = &ctx->layers[layer]; + + mlx_array xn = mlx_array_new(); + mlx_fast_rms_norm(&xn, x, l->input_norm, c->rms_norm_eps, S); + + mlx_array q_flat = linear(xn, l->wq, S); + mlx_array k_flat = linear(xn, l->wk, S); + mlx_array v_flat = linear(xn, l->wv, S); + mlx_array_free(xn); + + int q_shape[] = {1, n_tokens, n_heads, head_dim}; + int k_shape[] = {1, n_tokens, n_kv_heads, head_dim}; + mlx_array q = mlx_array_new(), k = mlx_array_new(), v = mlx_array_new(); + mlx_reshape(&q, q_flat, q_shape, 4, S); + mlx_reshape(&k, k_flat, k_shape, 4, S); + mlx_reshape(&v, v_flat, k_shape, 4, S); + mlx_array_free(q_flat); mlx_array_free(k_flat); mlx_array_free(v_flat); + + mlx_array qn = mlx_array_new(), kn = mlx_array_new(); + mlx_fast_rms_norm(&qn, q, l->q_norm, c->rms_norm_eps, S); + mlx_fast_rms_norm(&kn, k, l->k_norm, c->rms_norm_eps, S); + mlx_array_free(q); mlx_array_free(k); + + int perm[] = {0, 2, 1, 3}; + mlx_array qt = mlx_array_new(), kt = mlx_array_new(), vt = mlx_array_new(); + mlx_transpose_axes(&qt, qn, perm, 4, S); + mlx_transpose_axes(&kt, kn, perm, 4, S); + mlx_transpose_axes(&vt, v, perm, 4, S); + mlx_array_free(qn); mlx_array_free(kn); mlx_array_free(v); + + mlx_optional_float base = {.value = c->rope_theta, .has_value = true}; + mlx_array qr = mlx_array_new(), kr = mlx_array_new(); + mlx_fast_rope(&qr, qt, head_dim, false, base, 1.0f, 0, null_arr, S); + mlx_fast_rope(&kr, kt, head_dim, false, base, 1.0f, 0, null_arr, S); + mlx_array_free(qt); mlx_array_free(kt); + + mlx_array attn = mlx_array_new(); + mlx_fast_scaled_dot_product_attention(&attn, qr, kr, vt, scale, "", + null_arr, null_arr, S); + mlx_array_free(qr); mlx_array_free(kr); mlx_array_free(vt); + + mlx_array attn_t = mlx_array_new(); + mlx_transpose_axes(&attn_t, attn, perm, 4, S); + mlx_array_free(attn); + + int flat_shape[] = {1, n_tokens, q_dim}; + mlx_array attn_flat = mlx_array_new(); + mlx_reshape(&attn_flat, attn_t, flat_shape, 3, S); + mlx_array_free(attn_t); + + mlx_array proj = linear(attn_flat, l->wo, S); + mlx_array_free(attn_flat); + mlx_array x2 = mlx_array_new(); + mlx_add(&x2, x, proj, S); + mlx_array_free(x); mlx_array_free(proj); + x = x2; + + mlx_array xn2 = mlx_array_new(); + mlx_fast_rms_norm(&xn2, x, l->post_attn_norm, c->rms_norm_eps, S); + + mlx_array gate = linear(xn2, l->gate_proj, S); + mlx_array up = linear(xn2, l->up_proj, S); + mlx_array_free(xn2); + + mlx_array gate_sig = mlx_array_new(); + mlx_sigmoid(&gate_sig, gate, S); + mlx_array silu_gate = mlx_array_new(); + mlx_multiply(&silu_gate, gate, gate_sig, S); + mlx_array_free(gate); mlx_array_free(gate_sig); + + mlx_array mid = mlx_array_new(); + mlx_multiply(&mid, silu_gate, up, S); + mlx_array_free(silu_gate); mlx_array_free(up); + + mlx_array ffn = linear(mid, l->down_proj, S); + mlx_array_free(mid); + + mlx_array x3 = mlx_array_new(); + mlx_add(&x3, x, ffn, S); + mlx_array_free(x); mlx_array_free(ffn); + x = x3; + } + + mlx_array x_normed = mlx_array_new(); + mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S); + mlx_array_free(x); + + mlx_array x_f32 = mlx_array_new(); + mlx_astype(&x_f32, x_normed, MLX_FLOAT32, S); + mlx_array_free(x_normed); + mlx_array_eval(x_f32); + + const float *data = mlx_array_data_float32(x_f32); + int rc = -1; + if (data) { + memcpy(out_states, data, (size_t)n_tokens * hidden * sizeof(float)); + rc = 0; + } + mlx_array_free(x_f32); + return rc; +} + +int pplx_mlx_embed_spans(pplx_mlx_ctx_t *ctx, const int *token_ids, + int n_tokens, const pplx_span_t *spans, + int n_spans, float *out_embeddings) +{ + if (!ctx || !token_ids || n_tokens <= 0 || !spans || n_spans <= 0 || + !out_embeddings) + return -1; + + int hidden = ctx->config.hidden_size; + float *states = (float *)malloc((size_t)n_tokens * hidden * sizeof(float)); + if (!states) return -1; + + int rc = pplx_mlx_forward_into(ctx, token_ids, n_tokens, states); + if (rc == 0) + rc = pplx_pool_spans(&ctx->config, states, n_tokens, spans, n_spans, + out_embeddings); + free(states); + return rc; +} + float *pplx_mlx_embed(pplx_mlx_ctx_t *ctx, const int *token_ids, int n_tokens) { if (!ctx || !token_ids || n_tokens <= 0) return NULL; diff --git a/pplx_embed_mlx.h b/pplx_embed_mlx.h index e528367..9c0af9d 100644 --- a/pplx_embed_mlx.h +++ b/pplx_embed_mlx.h @@ -45,6 +45,14 @@ int pplx_mlx_embed_into(pplx_mlx_ctx_t *ctx, const int *token_ids, int pplx_mlx_embed_batch(pplx_mlx_ctx_t *ctx, const pplx_input_t *inputs, int batch, float *out_embeddings); +/* + * Run one contextual sequence and pool selected token spans. + * Returns 0 on success, -1 on error. + */ +int pplx_mlx_embed_spans(pplx_mlx_ctx_t *ctx, const int *token_ids, + int n_tokens, const pplx_span_t *spans, + int n_spans, float *out_embeddings); + /* Get the config (so main.c can read hidden_size etc.) */ const pplx_config_t *pplx_mlx_config(const pplx_mlx_ctx_t *ctx); diff --git a/pplx_server.c b/pplx_server.c new file mode 100644 index 0000000..825d448 --- /dev/null +++ b/pplx_server.c @@ -0,0 +1,1524 @@ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include "pplx_server.h" +#include "pplx_embed.h" +#include "qwen_asr_tokenizer.h" + +#ifdef USE_MLX +#include "pplx_embed_mlx.h" +#endif + +#include "deps/ae/ae.h" +#include "deps/ae/anet.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#define PPLX_HTTP_IO_BUF 8192 +#define PPLX_HTTP_MAX_HEADER (64u * 1024u) +#define PPLX_HTTP_MAX_BODY (64u * 1024u * 1024u) +#define PPLX_HTTP_MAX_PATH 255u +#define PPLX_HTTP_BACKLOG 128 +#define PPLX_HTTP_SETSIZE 10240 +#define PPLX_HTTP_CLIENT_TIMEOUT_MS 30000 +#define PPLX_HTTP_SWEEP_MS 1000 + +#define PPLX_API_MAX_STANDARD_INPUTS 512 +#define PPLX_API_MAX_CONTEXT_DOCS 512 +#define PPLX_API_MAX_CONTEXT_CHUNKS 16000 +#define PPLX_API_MAX_ITEM_TOKENS 32768 +#define PPLX_API_MAX_TOTAL_TOKENS 120000 + +typedef struct { + char *ptr; + size_t len; + size_t cap; +} sbuf; + +static void die_oom(void) { + fprintf(stderr, "pplx-serve: out of memory\n"); + exit(1); +} + +static void *xmalloc(size_t n) { + void *p = malloc(n ? n : 1); + if (!p) die_oom(); + return p; +} + +static void *xcalloc(size_t n, size_t sz) { + void *p = calloc(n ? n : 1, sz ? sz : 1); + if (!p) die_oom(); + return p; +} + +static void *xrealloc(void *p, size_t n) { + p = realloc(p, n ? n : 1); + if (!p) die_oom(); + return p; +} + +static char *xstrdup(const char *s) { + size_t n = strlen(s); + char *p = xmalloc(n + 1); + memcpy(p, s, n + 1); + return p; +} + +static char *xstrndup(const char *s, size_t n) { + char *p = xmalloc(n + 1); + memcpy(p, s, n); + p[n] = '\0'; + return p; +} + +static uint64_t mstime(void) { + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts) == 0) + return (uint64_t)ts.tv_sec * 1000u + (uint64_t)ts.tv_nsec / 1000000u; + struct timeval tv; + gettimeofday(&tv, NULL); + return (uint64_t)tv.tv_sec * 1000u + (uint64_t)tv.tv_usec / 1000u; +} + +static void server_log(const char *fmt, ...) { + char ts[32]; + time_t now = time(NULL); + struct tm tmv; + localtime_r(&now, &tmv); + strftime(ts, sizeof(ts), "%m%d %H:%M:%S", &tmv); + fprintf(stderr, "%s ", ts); + va_list ap; + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + fputc('\n', stderr); +} + +static void sbuf_reserve(sbuf *b, size_t add) { + if (add > SIZE_MAX - b->len - 1) die_oom(); + size_t need = b->len + add + 1; + if (need <= b->cap) return; + size_t cap = b->cap ? b->cap * 2 : 256; + while (cap < need) { + if (cap > SIZE_MAX / 2) die_oom(); + cap *= 2; + } + b->ptr = xrealloc(b->ptr, cap); + b->cap = cap; +} + +static void sbuf_append(sbuf *b, const void *p, size_t n) { + sbuf_reserve(b, n); + memcpy(b->ptr + b->len, p, n); + b->len += n; + b->ptr[b->len] = '\0'; +} + +static void sbuf_putc(sbuf *b, char c) { sbuf_append(b, &c, 1); } +static void sbuf_puts(sbuf *b, const char *s) { sbuf_append(b, s, strlen(s)); } + +static void sbuf_printf(sbuf *b, const char *fmt, ...) { + va_list ap, ap2; + va_start(ap, fmt); + va_copy(ap2, ap); + int n = vsnprintf(NULL, 0, fmt, ap); + va_end(ap); + if (n < 0) die_oom(); + sbuf_reserve(b, (size_t)n); + int n2 = vsnprintf(b->ptr + b->len, b->cap - b->len, fmt, ap2); + va_end(ap2); + if (n2 < 0 || n2 != n) die_oom(); + b->len += (size_t)n; +} + +static void sbuf_free(sbuf *b) { + free(b->ptr); + memset(b, 0, sizeof(*b)); +} + +typedef enum { + MODEL_STD_06, + MODEL_STD_4, + MODEL_CTX_06, + MODEL_CTX_4, + MODEL_UNKNOWN +} model_slot; + +typedef struct { + const char *id; + int is_contextual; + int dim; + double price_per_mtok; +} model_info; + +static const model_info k_models[] = { + {"pplx-embed-v1-0.6b", 0, 1024, 0.004}, + {"pplx-embed-v1-4b", 0, 2560, 0.030}, + {"pplx-embed-context-v1-0.6b", 1, 1024, 0.008}, + {"pplx-embed-context-v1-4b", 1, 2560, 0.050}, +}; + +static model_slot model_slot_for_id(const char *id) { + if (!id) return MODEL_UNKNOWN; + for (int i = 0; i < 4; i++) { + if (!strcmp(id, k_models[i].id)) return (model_slot)i; + } + return MODEL_UNKNOWN; +} + +typedef struct { + const model_info *info; + char *path; + qwen_tokenizer_t *tok; + int newline_n; + int *newline_ids; + pplx_model_t *cpu_model; + pplx_workspace_t *cpu_ws; +#ifdef USE_MLX + pplx_mlx_ctx_t *mlx_ctx; +#endif +} loaded_model; + +typedef struct client client; +typedef struct job job; + +typedef struct { + aeEventLoop *loop; + int listen_fd; + int completion_pipe[2]; + int signal_pipe[2]; + const char *host; + int port; + int batch_size; + int enable_cors; + char *api_key; + loaded_model models[4]; + int use_mlx; + + pthread_t worker; + pthread_mutex_t mu; + pthread_cond_t cv; + int stopping; + job *job_head; + job *job_tail; + job *done_head; + job *done_tail; + + client *clients; + int n_clients; +} http_server; + +typedef struct { + char method[8]; + char path[PPLX_HTTP_MAX_PATH + 1]; + char *body; + size_t body_len; + char *auth; +} http_request; + +struct client { + http_server *srv; + int fd; + int refcount; + int cancelled; + int linked; + uint64_t last_active_ms; + sbuf in; + sbuf out; + size_t sent; + int header_done; + size_t header_len; + size_t content_length; + http_request req; + client *prev; + client *next; +}; + +struct job { + http_server *srv; + client *c; + char method[8]; + char path[PPLX_HTTP_MAX_PATH + 1]; + char *body; + size_t body_len; + char *auth; + int status; + char *content_type; + char *response; + size_t response_len; + job *next; +}; + +static volatile sig_atomic_t g_stop_requested = 0; +static volatile sig_atomic_t g_signal_wfd = -1; + +static void stop_signal_handler(int sig) { + (void)sig; + if (g_stop_requested) _exit(130); + int saved = errno; + g_stop_requested = 1; + int fd = (int)g_signal_wfd; + if (fd >= 0) { + const unsigned char byte = 1; + (void)write(fd, &byte, 1); + } + errno = saved; +} + +static int set_nonblock(int fd) { + int flags = fcntl(fd, F_GETFL, 0); + if (flags < 0) return -1; + return fcntl(fd, F_SETFL, flags | O_NONBLOCK); +} + +static void http_request_free(http_request *r) { + free(r->body); + free(r->auth); + memset(r, 0, sizeof(*r)); +} + +static void client_free(client *c) { + if (!c) return; + http_request_free(&c->req); + sbuf_free(&c->in); + sbuf_free(&c->out); + free(c); +} + +static void client_incref(client *c) { + pthread_mutex_lock(&c->srv->mu); + c->refcount++; + pthread_mutex_unlock(&c->srv->mu); +} + +static void client_decref(client *c) { + int free_now = 0; + pthread_mutex_lock(&c->srv->mu); + if (--c->refcount == 0) free_now = 1; + pthread_mutex_unlock(&c->srv->mu); + if (free_now) client_free(c); +} + +static void client_unlink(client *c) { + http_server *s = c->srv; + if (!c->linked) return; + if (c->prev) c->prev->next = c->next; + else s->clients = c->next; + if (c->next) c->next->prev = c->prev; + c->prev = c->next = NULL; + c->linked = 0; + if (s->n_clients > 0) s->n_clients--; +} + +static void close_client(client *c) { + if (!c || c->cancelled) return; + c->cancelled = 1; + if (c->fd >= 0) { + aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE); + aeDeleteFileEvent(c->srv->loop, c->fd, AE_WRITABLE); + close(c->fd); + c->fd = -1; + } + client_unlink(c); + client_decref(c); +} + +static void append_http_response(client *c, int status, const char *ctype, + const char *body, size_t body_len) { + const char *reason = "OK"; + if (status == 204) reason = "No Content"; + else if (status == 400) reason = "Bad Request"; + else if (status == 401) reason = "Unauthorized"; + else if (status == 404) reason = "Not Found"; + else if (status == 405) reason = "Method Not Allowed"; + else if (status == 422) reason = "Unprocessable Entity"; + else if (status == 503) reason = "Service Unavailable"; + else if (status >= 500) reason = "Internal Server Error"; + + sbuf_printf(&c->out, + "HTTP/1.1 %d %s\r\n" + "Content-Length: %zu\r\n" + "Connection: close\r\n", + status, reason, body_len); + if (ctype) + sbuf_printf(&c->out, "Content-Type: %s\r\n", ctype); + if (c->srv->enable_cors) { + sbuf_puts(&c->out, "Access-Control-Allow-Origin: *\r\n"); + sbuf_puts(&c->out, "Access-Control-Allow-Headers: authorization, content-type\r\n"); + sbuf_puts(&c->out, "Access-Control-Allow-Methods: POST, OPTIONS\r\n"); + } + sbuf_puts(&c->out, "\r\n"); + if (body_len) sbuf_append(&c->out, body, body_len); +} + +static void append_json_error(sbuf *b, const char *message, const char *type) { + sbuf_puts(b, "{\"error\":{\"message\":\""); + for (const char *p = message ? message : ""; *p; p++) { + if (*p == '"' || *p == '\\') { + sbuf_putc(b, '\\'); + sbuf_putc(b, *p); + } else { + sbuf_putc(b, *p); + } + } + sbuf_puts(b, "\",\"type\":\""); + sbuf_puts(b, type ? type : "server_error"); + sbuf_puts(b, "\"}}"); +} + +static char *json_error_body(const char *message, const char *type, size_t *len) { + sbuf b = {0}; + append_json_error(&b, message, type); + *len = b.len; + return b.ptr; +} + +static void job_set_error(job *j, int status, const char *message, const char *type) { + free(j->response); + j->status = status; + j->content_type = "application/json"; + j->response = json_error_body(message, type, &j->response_len); +} + +static bool auth_ok(http_server *s, const char *auth) { + if (!s->api_key || !s->api_key[0]) return true; + if (!auth) return false; + const char prefix[] = "Bearer "; + return !strncmp(auth, prefix, sizeof(prefix) - 1) && + !strcmp(auth + sizeof(prefix) - 1, s->api_key); +} + +static ssize_t find_header_end(const char *p, size_t n) { + for (size_t i = 3; i < n; i++) { + if (p[i - 3] == '\r' && p[i - 2] == '\n' && + p[i - 1] == '\r' && p[i] == '\n') + return (ssize_t)(i + 1); + } + for (size_t i = 1; i < n; i++) { + if (p[i - 1] == '\n' && p[i] == '\n') return (ssize_t)(i + 1); + } + return -1; +} + +static char *header_value_dup(const char *h, size_t n, const char *name) { + size_t name_len = strlen(name); + const char *p = h, *end = h + n; + while (p < end) { + const char *line = p; + while (p < end && *p != '\n') p++; + size_t len = (size_t)(p - line); + if (len && line[len - 1] == '\r') len--; + if (len > name_len && !strncasecmp(line, name, name_len) && + line[name_len] == ':') { + const char *v = line + name_len + 1; + while (v < line + len && isspace((unsigned char)*v)) v++; + const char *e = line + len; + while (e > v && isspace((unsigned char)e[-1])) e--; + return xstrndup(v, (size_t)(e - v)); + } + if (p < end) p++; + } + return NULL; +} + +static int parse_request_headers(client *c) { + char line[512]; + size_t i = 0; + while (i < c->header_len && c->in.ptr[i] != '\n' && i + 1 < sizeof(line)) { + line[i] = c->in.ptr[i]; + i++; + } + line[i] = '\0'; + if (sscanf(line, "%7s %255s", c->req.method, c->req.path) != 2) + return -1; + char *q = strchr(c->req.path, '?'); + if (q) *q = '\0'; + + char *cl = header_value_dup(c->in.ptr, c->header_len, "Content-Length"); + if (cl) { + char *end = NULL; + errno = 0; + unsigned long long v = strtoull(cl, &end, 10); + if (errno || !end || *end || v > PPLX_HTTP_MAX_BODY) { + free(cl); + return -1; + } + c->content_length = (size_t)v; + free(cl); + } + c->req.auth = header_value_dup(c->in.ptr, c->header_len, "Authorization"); + return 0; +} + +static int complete_request(client *c) { + if (!c->header_done) { + ssize_t hend = find_header_end(c->in.ptr, c->in.len); + if (hend < 0) { + if (c->in.len > PPLX_HTTP_MAX_HEADER) return -1; + return 0; + } + c->header_done = 1; + c->header_len = (size_t)hend; + if (parse_request_headers(c) != 0) return -1; + } + if (c->content_length > PPLX_HTTP_MAX_BODY) return -1; + if (c->in.len < c->header_len + c->content_length) return 0; + c->req.body_len = c->content_length; + c->req.body = xmalloc(c->content_length + 1); + memcpy(c->req.body, c->in.ptr + c->header_len, c->content_length); + c->req.body[c->content_length] = '\0'; + return 1; +} + +static void write_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { + (void)loop; (void)mask; + client *c = clientData; + while (c->sent < c->out.len) { + ssize_t n = write(fd, c->out.ptr + c->sent, c->out.len - c->sent); + if (n > 0) { + c->sent += (size_t)n; + c->last_active_ms = mstime(); + continue; + } + if (n < 0 && errno == EINTR) continue; + if (n < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) return; + close_client(c); + return; + } + close_client(c); +} + +static int queue_write(client *c) { + if (!c || c->cancelled || c->fd < 0) return -1; + return aeCreateFileEvent(c->srv->loop, c->fd, AE_WRITABLE, write_cb, c); +} + +static void enqueue_done(job *j) { + http_server *s = j->srv; + pthread_mutex_lock(&s->mu); + if (s->done_tail) s->done_tail->next = j; + else s->done_head = j; + s->done_tail = j; + pthread_mutex_unlock(&s->mu); + const unsigned char byte = 1; + (void)write(s->completion_pipe[1], &byte, 1); +} + +static job *dequeue_job(http_server *s) { + pthread_mutex_lock(&s->mu); + while (!s->job_head && !s->stopping) + pthread_cond_wait(&s->cv, &s->mu); + job *j = s->job_head; + if (j) { + s->job_head = j->next; + if (!s->job_head) s->job_tail = NULL; + j->next = NULL; + } + pthread_mutex_unlock(&s->mu); + return j; +} + +static void enqueue_job(job *j) { + http_server *s = j->srv; + pthread_mutex_lock(&s->mu); + if (s->job_tail) s->job_tail->next = j; + else s->job_head = j; + s->job_tail = j; + pthread_cond_signal(&s->cv); + pthread_mutex_unlock(&s->mu); +} + +static job *pop_done(http_server *s) { + pthread_mutex_lock(&s->mu); + job *j = s->done_head; + if (j) { + s->done_head = j->next; + if (!s->done_head) s->done_tail = NULL; + j->next = NULL; + } + pthread_mutex_unlock(&s->mu); + return j; +} + +static void job_free(job *j) { + if (!j) return; + free(j->body); + free(j->auth); + free(j->response); + free(j); +} + +static void completion_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { + (void)loop; (void)mask; + http_server *s = clientData; + unsigned char tmp[128]; + while (read(fd, tmp, sizeof(tmp)) > 0) {} + + job *j; + while ((j = pop_done(s)) != NULL) { + client *c = j->c; + if (!c->cancelled) { + append_http_response(c, j->status, j->content_type, + j->response ? j->response : "", + j->response ? j->response_len : 0); + if (queue_write(c) != AE_OK) + close_client(c); + } + client_decref(c); + job_free(j); + } +} + +static bool route_is_embedding(const char *method, const char *path) { + return !strcmp(method, "POST") && + (!strcmp(path, "/v1/embeddings") || + !strcmp(path, "/v1/contextualizedembeddings")); +} + +static void dispatch_request(client *c) { + aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE); + + if (!strcmp(c->req.method, "OPTIONS")) { + append_http_response(c, 204, NULL, "", 0); + if (queue_write(c) != AE_OK) close_client(c); + return; + } + + if (!route_is_embedding(c->req.method, c->req.path)) { + size_t len; + char *body = json_error_body("unknown endpoint", "invalid_request_error", &len); + append_http_response(c, 404, "application/json", body, len); + free(body); + if (queue_write(c) != AE_OK) close_client(c); + return; + } + + if (!auth_ok(c->srv, c->req.auth)) { + size_t len; + char *body = json_error_body("missing or invalid bearer token", + "authentication_error", &len); + append_http_response(c, 401, "application/json", body, len); + free(body); + if (queue_write(c) != AE_OK) close_client(c); + return; + } + + job *j = xcalloc(1, sizeof(*j)); + j->srv = c->srv; + j->c = c; + snprintf(j->method, sizeof(j->method), "%s", c->req.method); + snprintf(j->path, sizeof(j->path), "%s", c->req.path); + j->body = xstrndup(c->req.body ? c->req.body : "", c->req.body_len); + j->body_len = c->req.body_len; + j->auth = c->req.auth ? xstrdup(c->req.auth) : NULL; + client_incref(c); + enqueue_job(j); +} + +static void read_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { + (void)loop; (void)mask; + client *c = clientData; + char tmp[PPLX_HTTP_IO_BUF]; + for (;;) { + ssize_t n = read(fd, tmp, sizeof(tmp)); + if (n > 0) { + sbuf_append(&c->in, tmp, (size_t)n); + c->last_active_ms = mstime(); + int done = complete_request(c); + if (done < 0) { + size_t len; + char *body = json_error_body("bad HTTP request", + "invalid_request_error", &len); + append_http_response(c, 400, "application/json", body, len); + free(body); + aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE); + if (queue_write(c) != AE_OK) close_client(c); + return; + } + if (done > 0) { + dispatch_request(c); + return; + } + continue; + } + if (n == 0) { + close_client(c); + return; + } + if (errno == EINTR) continue; + if (errno == EAGAIN || errno == EWOULDBLOCK) return; + close_client(c); + return; + } +} + +static void accept_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { + (void)loop; (void)mask; + http_server *s = clientData; + char cip[64]; + int cport = 0; + for (;;) { + int cfd = anetTcpAccept(NULL, fd, cip, sizeof(cip), &cport); + if (cfd == ANET_ERR) { + if (errno == EAGAIN || errno == EWOULDBLOCK) return; + if (errno == EINTR) continue; + return; + } + set_nonblock(cfd); + client *c = xcalloc(1, sizeof(*c)); + c->srv = s; + c->fd = cfd; + c->refcount = 1; + c->last_active_ms = mstime(); + c->linked = 1; + c->next = s->clients; + if (s->clients) s->clients->prev = c; + s->clients = c; + s->n_clients++; + if (aeCreateFileEvent(s->loop, cfd, AE_READABLE, read_cb, c) != AE_OK) + close_client(c); + } +} + +static int timeout_cb(aeEventLoop *loop, long long id, void *clientData) { + (void)loop; (void)id; + http_server *s = clientData; + uint64_t now = mstime(); + client *c = s->clients; + while (c) { + client *next = c->next; + if (!c->cancelled && + now - c->last_active_ms > PPLX_HTTP_CLIENT_TIMEOUT_MS) + close_client(c); + c = next; + } + return PPLX_HTTP_SWEEP_MS; +} + +static void signal_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { + (void)fd; (void)mask; + http_server *s = clientData; + unsigned char tmp[128]; + while (read(s->signal_pipe[0], tmp, sizeof(tmp)) > 0) {} + aeStop(loop); +} + +static bool ve_add(cJSON *detail, const char *loc_json, + const char *msg, const char *type) { + cJSON *obj = cJSON_CreateObject(); + cJSON *loc = cJSON_Parse(loc_json); + if (!obj || !loc) { + cJSON_Delete(obj); + cJSON_Delete(loc); + return false; + } + cJSON_AddItemToObject(obj, "loc", loc); + cJSON_AddStringToObject(obj, "msg", msg ? msg : ""); + cJSON_AddStringToObject(obj, "type", type ? type : ""); + cJSON_AddItemToArray(detail, obj); + return true; +} + +static void job_set_422(job *j, cJSON *detail) { + cJSON *root = cJSON_CreateObject(); + cJSON_AddItemToObject(root, "detail", cJSON_Duplicate(detail, 1)); + char *s = cJSON_PrintUnformatted(root); + cJSON_Delete(root); + if (!s) { + job_set_error(j, 500, "failed to render validation error", "server_error"); + return; + } + j->status = 422; + j->content_type = "application/json"; + j->response = xstrdup(s); + j->response_len = strlen(j->response); + cJSON_free(s); +} + +static cJSON *parse_json_body(job *j, cJSON *detail) { + if (!j->body || j->body_len == 0) { + ve_add(detail, "[\"body\"]", "body required", "missing"); + return NULL; + } + const char *end = NULL; + cJSON *root = cJSON_ParseWithLengthOpts(j->body, j->body_len, &end, false); + if (!root) { + ve_add(detail, "[\"body\"]", "invalid JSON", "json_invalid"); + return NULL; + } + const char *body_end = j->body + j->body_len; + while (end && end < body_end && isspace((unsigned char)*end)) end++; + if (!end || end != body_end) { + ve_add(detail, "[\"body\"]", "invalid JSON", "json_invalid"); + cJSON_Delete(root); + return NULL; + } + if (!cJSON_IsObject(root)) { + ve_add(detail, "[\"body\"]", "body must be a JSON object", "type_error.object"); + cJSON_Delete(root); + return NULL; + } + return root; +} + +static bool cjson_is_integer(cJSON *item) { + if (!cJSON_IsNumber(item)) return false; + double d = item->valuedouble; + int i = item->valueint; + return d >= (double)INT_MIN && d <= (double)INT_MAX && d == (double)i; +} + +static const char *encoding_from_root(cJSON *root, cJSON *detail) { + cJSON *encoding = cJSON_GetObjectItemCaseSensitive(root, "encoding_format"); + if (!encoding) return "base64_int8"; + if (!cJSON_IsString(encoding) || !encoding->valuestring) { + ve_add(detail, "[\"body\",\"encoding_format\"]", + "encoding_format must be a string", "type_error.string"); + return "base64_int8"; + } + if (strcmp(encoding->valuestring, "base64_int8") && + strcmp(encoding->valuestring, "base64_binary")) { + ve_add(detail, "[\"body\",\"encoding_format\"]", + "value is not a valid enum member; permitted: 'base64_int8', 'base64_binary'", + "enum"); + return "base64_int8"; + } + return encoding->valuestring; +} + +static int dimensions_from_root(cJSON *root, cJSON *detail, int max_dim, + const char *encoding) { + int dims = max_dim; + cJSON *dimensions = cJSON_GetObjectItemCaseSensitive(root, "dimensions"); + if (dimensions) { + if (!cjson_is_integer(dimensions)) { + ve_add(detail, "[\"body\",\"dimensions\"]", "dimensions must be an integer", + "type_error.integer"); + } else { + dims = dimensions->valueint; + if (dims < 128 || dims > max_dim) { + char msg[96]; + snprintf(msg, sizeof(msg), "dimensions must be between 128 and %d", max_dim); + ve_add(detail, "[\"body\",\"dimensions\"]", msg, "value_error.range"); + } + } + } + if (!strcmp(encoding, "base64_binary") && dims % 8 != 0) + ve_add(detail, "[\"body\",\"dimensions\"]", + "dimensions must be divisible by 8 for base64_binary", + "value_error.divisible"); + return dims; +} + +static char *base64_encode(const unsigned char *src, size_t n) { + static const char tbl[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + size_t out_len = ((n + 2) / 3) * 4; + char *out = xmalloc(out_len + 1); + size_t o = 0; + for (size_t i = 0; i < n; i += 3) { + unsigned v = (unsigned)src[i] << 16; + int remain = (int)(n - i); + if (remain > 1) v |= (unsigned)src[i + 1] << 8; + if (remain > 2) v |= (unsigned)src[i + 2]; + out[o++] = tbl[(v >> 18) & 63]; + out[o++] = tbl[(v >> 12) & 63]; + out[o++] = remain > 1 ? tbl[(v >> 6) & 63] : '='; + out[o++] = remain > 2 ? tbl[v & 63] : '='; + } + out[o] = '\0'; + return out; +} + +static char *encode_embedding(const float *emb, int dims, const char *encoding) { + if (!strcmp(encoding, "base64_binary")) { + size_t bytes = (size_t)dims / 8; + unsigned char *bits = xcalloc(bytes, 1); + for (int i = 0; i < dims; i++) { + if (emb[i] >= 0.0f) + bits[(size_t)i >> 3] |= (unsigned char)(1u << (i & 7)); + } + char *out = base64_encode(bits, bytes); + free(bits); + return out; + } + + signed char *q = xmalloc((size_t)dims); + float max_abs = 0.0f; + for (int i = 0; i < dims; i++) { + float a = fabsf(emb[i]); + if (a > max_abs) max_abs = a; + } + float scale = max_abs > 1e-12f ? 127.0f / max_abs : 0.0f; + for (int i = 0; i < dims; i++) { + int v = (int)lrintf(emb[i] * scale); + if (v > 127) v = 127; + if (v < -127) v = -127; + q[i] = (signed char)v; + } + char *out = base64_encode((const unsigned char *)q, (size_t)dims); + free(q); + return out; +} + +typedef struct { + int *ids; + int n_tokens; +} token_buf; + +static void free_token_bufs(token_buf *t, int n) { + if (!t) return; + for (int i = 0; i < n; i++) free(t[i].ids); +} + +static int tokenize_one(loaded_model *m, const char *text, token_buf *out) { + memset(out, 0, sizeof(*out)); + out->ids = qwen_tokenizer_encode(m->tok, text, &out->n_tokens); + if (!out->ids || out->n_tokens <= 0) { + free(out->ids); + memset(out, 0, sizeof(*out)); + return -1; + } + return 0; +} + +static int model_embed_batch(loaded_model *m, const pplx_input_t *inputs, + int batch, float *out) { +#ifdef USE_MLX + if (m->mlx_ctx) + return pplx_mlx_embed_batch(m->mlx_ctx, inputs, batch, out); +#endif + return pplx_model_embed_batch(m->cpu_model, m->cpu_ws, inputs, batch, out); +} + +static int model_embed_spans(loaded_model *m, const int *ids, int n_tokens, + const pplx_span_t *spans, int n_spans, float *out) { +#ifdef USE_MLX + if (m->mlx_ctx) + return pplx_mlx_embed_spans(m->mlx_ctx, ids, n_tokens, spans, n_spans, out); +#endif + return pplx_model_embed_spans(m->cpu_model, m->cpu_ws, ids, n_tokens, + spans, n_spans, out); +} + +static void append_embedding_object(sbuf *b, int index, const char *embedding) { + sbuf_printf(b, "{\"object\":\"embedding\",\"index\":%d,\"embedding\":\"", index); + sbuf_puts(b, embedding); + sbuf_puts(b, "\"}"); +} + +static void set_response_from_buf(job *j, sbuf *b) { + j->status = 200; + j->content_type = "application/json"; + j->response = b->ptr; + j->response_len = b->len; + memset(b, 0, sizeof(*b)); +} + +static void handle_embeddings_job(job *j, cJSON *root, loaded_model *m, + int dims, const char *encoding) { + cJSON *input = cJSON_GetObjectItemCaseSensitive(root, "input"); + int n_inputs = cJSON_IsString(input) ? 1 : cJSON_GetArraySize(input); + token_buf *tokens = xcalloc((size_t)n_inputs, sizeof(*tokens)); + pplx_input_t *inputs = xmalloc((size_t)n_inputs * sizeof(*inputs)); + float *embs = xmalloc((size_t)n_inputs * m->info->dim * sizeof(float)); + char **encoded = xcalloc((size_t)n_inputs, sizeof(*encoded)); + + int total_tokens = 0; + int idx = 0; + if (cJSON_IsString(input)) { + tokenize_one(m, input->valuestring, &tokens[0]); + inputs[0].ids = tokens[0].ids; + inputs[0].n_tokens = tokens[0].n_tokens; + total_tokens = tokens[0].n_tokens; + } else { + cJSON *item; + cJSON_ArrayForEach(item, input) { + tokenize_one(m, item->valuestring, &tokens[idx]); + inputs[idx].ids = tokens[idx].ids; + inputs[idx].n_tokens = tokens[idx].n_tokens; + total_tokens += tokens[idx].n_tokens; + idx++; + } + } + + for (int i = 0; i < n_inputs; i++) { + if (!tokens[i].ids) { + job_set_error(j, 500, "tokenization failed", "server_error"); + goto done; + } + } + int max_batch = j->srv->batch_size > 0 ? j->srv->batch_size : n_inputs; + for (int start = 0; start < n_inputs; start += max_batch) { + int cur = n_inputs - start; + if (cur > max_batch) cur = max_batch; + if (model_embed_batch(m, inputs + start, cur, + embs + (size_t)start * m->info->dim) != 0) { + job_set_error(j, 500, "embedding failed", "server_error"); + goto done; + } + } + for (int i = 0; i < n_inputs; i++) + encoded[i] = encode_embedding(embs + (size_t)i * m->info->dim, dims, encoding); + + sbuf b = {0}; + sbuf_puts(&b, "{\"object\":\"list\",\"data\":["); + for (int i = 0; i < n_inputs; i++) { + if (i) sbuf_putc(&b, ','); + append_embedding_object(&b, i, encoded[i]); + } + double cost = ((double)total_tokens / 1000000.0) * m->info->price_per_mtok; + sbuf_printf(&b, + "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d," + "\"total_tokens\":%d,\"cost\":{\"input_cost\":%.10g," + "\"total_cost\":%.10g,\"currency\":\"USD\"}}}", + m->info->id, total_tokens, total_tokens, cost, cost); + set_response_from_buf(j, &b); + +done: + for (int i = 0; i < n_inputs; i++) free(encoded[i]); + free(encoded); + free(embs); + free(inputs); + free_token_bufs(tokens, n_inputs); + free(tokens); +} + +static void handle_context_job(job *j, cJSON *root, loaded_model *m, + int dims, const char *encoding) { + cJSON *input = cJSON_GetObjectItemCaseSensitive(root, "input"); + int n_docs = cJSON_GetArraySize(input); + int *chunks = xcalloc((size_t)n_docs, sizeof(*chunks)); + char ***encoded = xcalloc((size_t)n_docs, sizeof(*encoded)); + int total_tokens = 0; + + cJSON *doc_arr; + int di = 0; + cJSON_ArrayForEach(doc_arr, input) { + chunks[di] = cJSON_GetArraySize(doc_arr); + encoded[di] = xcalloc((size_t)chunks[di], sizeof(char *)); + token_buf *chunk_tokens = xcalloc((size_t)chunks[di], sizeof(*chunk_tokens)); + pplx_span_t *spans = xcalloc((size_t)chunks[di], sizeof(*spans)); + int doc_tokens = 0; + cJSON *chunk; + int ci = 0; + cJSON_ArrayForEach(chunk, doc_arr) { + if (tokenize_one(m, chunk->valuestring, &chunk_tokens[ci]) != 0) { + job_set_error(j, 500, "tokenization failed", "server_error"); + free_token_bufs(chunk_tokens, chunks[di]); + free(chunk_tokens); free(spans); + goto done; + } + doc_tokens += chunk_tokens[ci].n_tokens; + if (ci + 1 < chunks[di]) doc_tokens += m->newline_n; + ci++; + } + int *doc_ids = xmalloc((size_t)doc_tokens * sizeof(int)); + int pos = 0; + for (ci = 0; ci < chunks[di]; ci++) { + spans[ci].start = pos; + spans[ci].n_tokens = chunk_tokens[ci].n_tokens; + memcpy(doc_ids + pos, chunk_tokens[ci].ids, + (size_t)chunk_tokens[ci].n_tokens * sizeof(int)); + pos += chunk_tokens[ci].n_tokens; + if (ci + 1 < chunks[di] && m->newline_n > 0) { + memcpy(doc_ids + pos, m->newline_ids, + (size_t)m->newline_n * sizeof(int)); + pos += m->newline_n; + } + } + total_tokens += doc_tokens; + float *embs = xmalloc((size_t)chunks[di] * m->info->dim * sizeof(float)); + if (model_embed_spans(m, doc_ids, doc_tokens, spans, chunks[di], embs) != 0) { + free(embs); free(doc_ids); + job_set_error(j, 500, "contextual embedding failed", "server_error"); + free_token_bufs(chunk_tokens, chunks[di]); + free(chunk_tokens); free(spans); + goto done; + } + for (ci = 0; ci < chunks[di]; ci++) + encoded[di][ci] = encode_embedding(embs + (size_t)ci * m->info->dim, + dims, encoding); + free(embs); + free(doc_ids); + free_token_bufs(chunk_tokens, chunks[di]); + free(chunk_tokens); + free(spans); + di++; + } + + sbuf b = {0}; + sbuf_puts(&b, "{\"object\":\"list\",\"data\":["); + for (di = 0; di < n_docs; di++) { + if (di) sbuf_putc(&b, ','); + sbuf_printf(&b, "{\"object\":\"list\",\"index\":%d,\"data\":[", di); + for (int ci = 0; ci < chunks[di]; ci++) { + if (ci) sbuf_putc(&b, ','); + append_embedding_object(&b, ci, encoded[di][ci]); + } + sbuf_puts(&b, "]}"); + } + sbuf_printf(&b, + "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d,\"total_tokens\":%d}}", + m->info->id, total_tokens, total_tokens); + set_response_from_buf(j, &b); + +done: + for (di = 0; di < n_docs; di++) { + if (!encoded[di]) continue; + for (int ci = 0; ci < chunks[di]; ci++) free(encoded[di][ci]); + free(encoded[di]); + } + free(encoded); + free(chunks); +} + +static void validate_text_tokens(cJSON *detail, loaded_model *m, const char *loc, + const char *text, int *total_tokens) { + token_buf t; + if (tokenize_one(m, text, &t) != 0) { + ve_add(detail, loc, "tokenization failed", "value_error.tokenization"); + return; + } + if (t.n_tokens > PPLX_API_MAX_ITEM_TOKENS) + ve_add(detail, loc, "input exceeds 32768 token limit", "value_error.context_length"); + if (*total_tokens <= PPLX_API_MAX_TOTAL_TOKENS) + *total_tokens += t.n_tokens; + free(t.ids); +} + +static int validate_common(cJSON *root, cJSON *detail, int expect_context, + loaded_model **out_model, int *out_dims, + const char **out_encoding, http_server *s) { + cJSON *model_item = cJSON_GetObjectItemCaseSensitive(root, "model"); + const char *model_id = NULL; + if (!model_item) { + ve_add(detail, "[\"body\",\"model\"]", "field required", "missing"); + } else if (!cJSON_IsString(model_item) || !model_item->valuestring) { + ve_add(detail, "[\"body\",\"model\"]", "model must be a string", "type_error.string"); + } else { + model_id = model_item->valuestring; + model_slot slot = model_slot_for_id(model_id); + if (slot == MODEL_UNKNOWN || k_models[slot].is_contextual != expect_context) { + ve_add(detail, "[\"body\",\"model\"]", + "value is not a valid enum member for this endpoint", "enum"); + } else { + loaded_model *m = &s->models[slot]; + if (!m->info) { + *out_model = NULL; + } else { + *out_model = m; + } + } + } + const char *enc = encoding_from_root(root, detail); + int max_dim = model_id && model_slot_for_id(model_id) != MODEL_UNKNOWN + ? k_models[model_slot_for_id(model_id)].dim : 2560; + int dims = dimensions_from_root(root, detail, max_dim, enc); + *out_encoding = enc; + *out_dims = dims; + return model_id && model_slot_for_id(model_id) != MODEL_UNKNOWN && + !s->models[model_slot_for_id(model_id)].info ? 503 : 0; +} + +static void process_embeddings(job *j, cJSON *root, http_server *s) { + cJSON *detail = cJSON_CreateArray(); + loaded_model *m = NULL; + int dims = 0; + const char *encoding = NULL; + int unloaded = validate_common(root, detail, 0, &m, &dims, &encoding, s); + + cJSON *input = cJSON_GetObjectItemCaseSensitive(root, "input"); + int n_inputs = 0; + int total_tokens = 0; + if (!input) { + ve_add(detail, "[\"body\",\"input\"]", "field required", "missing"); + } else if (cJSON_IsString(input)) { + if (!input->valuestring || input->valuestring[0] == '\0') + ve_add(detail, "[\"body\",\"input\"]", "input must not be empty", + "value_error.empty"); + else if (m) + validate_text_tokens(detail, m, "[\"body\",\"input\"]", + input->valuestring, &total_tokens); + n_inputs = 1; + } else if (cJSON_IsArray(input)) { + n_inputs = cJSON_GetArraySize(input); + if (n_inputs < 1) + ve_add(detail, "[\"body\",\"input\"]", "input must contain at least 1 item", + "value_error.list.min_items"); + else if (n_inputs > PPLX_API_MAX_STANDARD_INPUTS) + ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 512 items", + "value_error.list.max_items"); + cJSON *item; + int i = 0; + cJSON_ArrayForEach(item, input) { + char loc[64]; + snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", i); + if (!cJSON_IsString(item)) { + ve_add(detail, loc, "input item must be a string", "type_error.string"); + } else if (!item->valuestring || item->valuestring[0] == '\0') { + ve_add(detail, loc, "input item must not be empty", "value_error.empty"); + } else if (m) { + validate_text_tokens(detail, m, loc, item->valuestring, &total_tokens); + } + i++; + } + } else { + ve_add(detail, "[\"body\",\"input\"]", + "input must be a string or an array of strings", "type_error"); + } + if (total_tokens > PPLX_API_MAX_TOTAL_TOKENS) + ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit", + "value_error.context_length"); + + if (cJSON_GetArraySize(detail) > 0) { + job_set_422(j, detail); + } else if (unloaded) { + job_set_error(j, 503, "requested model is valid but not loaded", + "model_not_loaded"); + } else { + handle_embeddings_job(j, root, m, dims, encoding); + } + cJSON_Delete(detail); +} + +static void process_contextual(job *j, cJSON *root, http_server *s) { + cJSON *detail = cJSON_CreateArray(); + loaded_model *m = NULL; + int dims = 0; + const char *encoding = NULL; + int unloaded = validate_common(root, detail, 1, &m, &dims, &encoding, s); + + cJSON *input = cJSON_GetObjectItemCaseSensitive(root, "input"); + int n_docs = 0, total_chunks = 0, total_tokens = 0; + if (!input) { + ve_add(detail, "[\"body\",\"input\"]", "field required", "missing"); + } else if (!cJSON_IsArray(input)) { + ve_add(detail, "[\"body\",\"input\"]", + "input must be an array of document chunk arrays", "type_error.array"); + } else { + n_docs = cJSON_GetArraySize(input); + if (n_docs < 1) + ve_add(detail, "[\"body\",\"input\"]", "input must contain at least 1 document", + "value_error.list.min_items"); + else if (n_docs > PPLX_API_MAX_CONTEXT_DOCS) + ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 512 documents", + "value_error.list.max_items"); + cJSON *doc_arr; + int di = 0; + cJSON_ArrayForEach(doc_arr, input) { + if (!cJSON_IsArray(doc_arr)) { + char loc[64]; + snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di); + ve_add(detail, loc, "document must be an array of strings", + "type_error.array"); + di++; + continue; + } + int doc_chunks = cJSON_GetArraySize(doc_arr); + total_chunks += doc_chunks; + if (doc_chunks < 1) { + char loc[64]; + snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di); + ve_add(detail, loc, "document must contain at least 1 chunk", + "value_error.list.min_items"); + } + int doc_tokens = 0; + cJSON *chunk; + int ci = 0; + cJSON_ArrayForEach(chunk, doc_arr) { + char loc[80]; + snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d,%d]", di, ci); + if (!cJSON_IsString(chunk)) { + ve_add(detail, loc, "chunk must be a string", "type_error.string"); + } else if (!chunk->valuestring || chunk->valuestring[0] == '\0') { + ve_add(detail, loc, "chunk must not be empty", "value_error.empty"); + } else if (m) { + token_buf t; + if (tokenize_one(m, chunk->valuestring, &t) != 0) { + ve_add(detail, loc, "tokenization failed", "value_error.tokenization"); + } else { + doc_tokens += t.n_tokens; + total_tokens += t.n_tokens; + free(t.ids); + } + } + ci++; + } + if (m && doc_chunks > 1) { + doc_tokens += (doc_chunks - 1) * m->newline_n; + total_tokens += (doc_chunks - 1) * m->newline_n; + } + if (doc_tokens > PPLX_API_MAX_ITEM_TOKENS) { + char loc[64]; + snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di); + ve_add(detail, loc, "document exceeds 32768 token limit", + "value_error.context_length"); + } + di++; + } + if (total_chunks > PPLX_API_MAX_CONTEXT_CHUNKS) + ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 16000 chunks", + "value_error.list.max_items"); + if (total_tokens > PPLX_API_MAX_TOTAL_TOKENS) + ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit", + "value_error.context_length"); + } + + if (cJSON_GetArraySize(detail) > 0) { + job_set_422(j, detail); + } else if (unloaded) { + job_set_error(j, 503, "requested model is valid but not loaded", + "model_not_loaded"); + } else { + handle_context_job(j, root, m, dims, encoding); + } + cJSON_Delete(detail); +} + +static void process_job(job *j) { + cJSON *detail = cJSON_CreateArray(); + cJSON *root = parse_json_body(j, detail); + if (!root) { + job_set_422(j, detail); + cJSON_Delete(detail); + return; + } + cJSON_Delete(detail); + + if (!strcmp(j->path, "/v1/embeddings")) + process_embeddings(j, root, j->srv); + else if (!strcmp(j->path, "/v1/contextualizedembeddings")) + process_contextual(j, root, j->srv); + else + job_set_error(j, 404, "unknown endpoint", "invalid_request_error"); + cJSON_Delete(root); +} + +static void *worker_main(void *arg) { + http_server *s = arg; + for (;;) { + job *j = dequeue_job(s); + if (!j) break; + process_job(j); + enqueue_done(j); + } + return NULL; +} + +static int listen_on(const char *host, int port) { + char portbuf[32]; + snprintf(portbuf, sizeof(portbuf), "%d", port); + char err[ANET_ERR_LEN]; + int fd = anetTcpServer(err, port, (char *)host, PPLX_HTTP_BACKLOG); + (void)portbuf; + if (fd == ANET_ERR) { + server_log("pplx-serve: listen failed on %s:%d: %s", host, port, err); + return -1; + } + set_nonblock(fd); + return fd; +} + +static int load_one_model(http_server *s, model_slot slot, const char *path) { + loaded_model *m = &s->models[slot]; + m->info = &k_models[slot]; + m->path = xstrdup(path); + char vocab_path[1024]; + snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", path); + m->tok = qwen_tokenizer_load(vocab_path); + if (!m->tok) { + server_log("pplx-serve: failed to load tokenizer: %s", vocab_path); + return -1; + } + m->newline_ids = qwen_tokenizer_encode(m->tok, "\n", &m->newline_n); + if (!m->newline_ids || m->newline_n <= 0) { + server_log("pplx-serve: failed to tokenize newline separator"); + return -1; + } +#ifdef USE_MLX + if (s->use_mlx) { + m->mlx_ctx = pplx_mlx_load(path); + if (!m->mlx_ctx) { + server_log("pplx-serve: failed to load MLX model: %s", path); + return -1; + } + if (pplx_mlx_config(m->mlx_ctx)->hidden_size != m->info->dim) { + server_log("pplx-serve: model %s has unexpected dimension %d", + m->info->id, pplx_mlx_config(m->mlx_ctx)->hidden_size); + return -1; + } + return 0; + } +#endif + m->cpu_model = pplx_model_load(path); + if (!m->cpu_model) { + server_log("pplx-serve: failed to load model: %s", path); + return -1; + } + m->cpu_ws = pplx_workspace_new(m->cpu_model); + if (!m->cpu_ws) { + server_log("pplx-serve: failed to allocate workspace: %s", path); + return -1; + } + if (pplx_model_config(m->cpu_model)->hidden_size != m->info->dim) { + server_log("pplx-serve: model %s has unexpected dimension %d", + m->info->id, pplx_model_config(m->cpu_model)->hidden_size); + return -1; + } + return 0; +} + +static void free_models(http_server *s) { + for (int i = 0; i < 4; i++) { + loaded_model *m = &s->models[i]; + free(m->path); + free(m->newline_ids); + if (m->tok) qwen_tokenizer_free(m->tok); + if (m->cpu_ws) pplx_workspace_free(m->cpu_ws); + if (m->cpu_model) pplx_model_free(m->cpu_model); +#ifdef USE_MLX + if (m->mlx_ctx) pplx_mlx_free(m->mlx_ctx); +#endif + } +} + +int pplx_run_server(const pplx_server_config_t *cfg) { + if (!cfg || !cfg->models || cfg->n_models <= 0) { + fprintf(stderr, "pplx-serve: at least one --model MODEL_ID=PATH is required\n"); + return 1; + } + + http_server s; + memset(&s, 0, sizeof(s)); + s.host = cfg->host && cfg->host[0] ? cfg->host : "127.0.0.1"; + s.port = cfg->port > 0 ? cfg->port : 8000; + s.batch_size = cfg->batch_size > 0 ? cfg->batch_size : 1; + s.enable_cors = cfg->enable_cors; + s.use_mlx = cfg->use_mlx; + if (cfg->api_key && cfg->api_key[0]) + s.api_key = xstrdup(cfg->api_key); + else { + const char *env_key = getenv("PPLX_API_KEY"); + if (env_key && env_key[0]) s.api_key = xstrdup(env_key); + } + + pthread_mutex_init(&s.mu, NULL); + pthread_cond_init(&s.cv, NULL); + + for (int i = 0; i < cfg->n_models; i++) { + model_slot slot = model_slot_for_id(cfg->models[i].id); + if (slot == MODEL_UNKNOWN) { + fprintf(stderr, "pplx-serve: unknown model id: %s\n", cfg->models[i].id); + free_models(&s); + return 1; + } + if (load_one_model(&s, slot, cfg->models[i].path) != 0) { + free_models(&s); + return 1; + } + } + + if (pipe(s.completion_pipe) != 0 || pipe(s.signal_pipe) != 0) { + perror("pipe"); + free_models(&s); + return 1; + } + set_nonblock(s.completion_pipe[0]); + set_nonblock(s.completion_pipe[1]); + set_nonblock(s.signal_pipe[0]); + set_nonblock(s.signal_pipe[1]); + + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = stop_signal_handler; + sigemptyset(&sa.sa_mask); + sigaction(SIGINT, &sa, NULL); + sigaction(SIGTERM, &sa, NULL); + signal(SIGPIPE, SIG_IGN); + g_signal_wfd = s.signal_pipe[1]; + + s.loop = aeCreateEventLoop(PPLX_HTTP_SETSIZE); + if (!s.loop) { + fprintf(stderr, "pplx-serve: failed to create event loop\n"); + free_models(&s); + return 1; + } + + if (pthread_create(&s.worker, NULL, worker_main, &s) != 0) { + fprintf(stderr, "pplx-serve: failed to start worker\n"); + aeDeleteEventLoop(s.loop); + free_models(&s); + return 1; + } + + s.listen_fd = listen_on(s.host, s.port); + if (s.listen_fd < 0) { + pthread_mutex_lock(&s.mu); + s.stopping = 1; + pthread_cond_signal(&s.cv); + pthread_mutex_unlock(&s.mu); + pthread_join(s.worker, NULL); + aeDeleteEventLoop(s.loop); + free_models(&s); + return 1; + } + + aeCreateFileEvent(s.loop, s.listen_fd, AE_READABLE, accept_cb, &s); + aeCreateFileEvent(s.loop, s.completion_pipe[0], AE_READABLE, completion_cb, &s); + aeCreateFileEvent(s.loop, s.signal_pipe[0], AE_READABLE, signal_cb, &s); + aeCreateTimeEvent(s.loop, PPLX_HTTP_SWEEP_MS, timeout_cb, &s, NULL); + + server_log("pplx-serve: listening on http://%s:%d", s.host, s.port); + aeMain(s.loop); + + server_log("pplx-serve: shutdown requested"); + if (s.listen_fd >= 0) close(s.listen_fd); + client *c = s.clients; + while (c) { + client *next = c->next; + close_client(c); + c = next; + } + + pthread_mutex_lock(&s.mu); + s.stopping = 1; + pthread_cond_signal(&s.cv); + pthread_mutex_unlock(&s.mu); + pthread_join(s.worker, NULL); + + aeDeleteEventLoop(s.loop); + close(s.completion_pipe[0]); + close(s.completion_pipe[1]); + close(s.signal_pipe[0]); + close(s.signal_pipe[1]); + free_models(&s); + free(s.api_key); + pthread_cond_destroy(&s.cv); + pthread_mutex_destroy(&s.mu); + return 0; +} diff --git a/pplx_server.h b/pplx_server.h new file mode 100644 index 0000000..c46f4f9 --- /dev/null +++ b/pplx_server.h @@ -0,0 +1,24 @@ +#ifndef PPLX_SERVER_H +#define PPLX_SERVER_H + +#include + +typedef struct { + const char *id; + const char *path; +} pplx_server_model_spec_t; + +typedef struct { + const pplx_server_model_spec_t *models; + int n_models; + const char *host; + int port; + int batch_size; + int use_mlx; + int enable_cors; + const char *api_key; +} pplx_server_config_t; + +int pplx_run_server(const pplx_server_config_t *cfg); + +#endif -- cgit v1.3.1