diff --git a/CorRestState.h b/CorRestState.h index f90f7a4..6344e81 100644 --- a/CorRestState.h +++ b/CorRestState.h @@ -9,8 +9,8 @@ #ifndef CORREST_STATE_H_ #define CORREST_STATE_H_ -#include #include +#include // NULL - came in via microhttpd.h until this header stopped including it #include "kalloc/KAlloc.h" #include "kjson/kjson.h" @@ -30,7 +30,15 @@ // typedef struct CorRestState { - struct MHD_Connection* mhdConnection; + // + // The HTTP backend's handle for the connection this request came in on - + // an `struct MHD_Connection*` under libmicrohttpd, a `CorHttpConn*` under the + // built-in server. Opaque HERE on purpose: this header is included by ~2000 + // call sites in the layers above, and naming one server's type in it would + // make every one of them depend on that server being the one in the build. + // The backend that put it here is the only code that casts it back. + // + void* connection; // Allocator: pool-based, bulk-free after request completes KAlloc kalloc; @@ -49,7 +57,7 @@ typedef struct CorRestState // Matched service CorRestService* serviceP; - // Payload accumulation (during MHD body reads) + // Payload accumulation (during the backend's body reads) int payloadBufSize; // Request timing @@ -63,6 +71,21 @@ typedef struct CorRestState // this state; a worker runs corRestProcessRequest off the I/O thread, sets // asyncProcessed, and resumes the connection. asyncNext links the FIFO queue. bool asyncProcessed; + + // + // ...and the SECOND thing a worker does for a request: the post-response + // phase - the deferred notifications, and releasing the arena they are built + // in - once the response is on the wire. + // + // A phase and not a second queue, because it is the same work item at a later + // moment. It is off the I/O thread for the same reason the dispatch is, and + // one reason more: a notification's @context can be one the broker HOSTS + // ITSELF, so this phase can issue a request the broker has to answer. On a + // single-threaded event loop, running it there is a deadlock that resolves + // itself as a timeout - a notification sent ten seconds late with an + // uncompacted body, and nothing in the log saying why. + // + bool asyncFinishing; struct CorRestState* asyncNext; } CorRestState; @@ -74,7 +97,7 @@ typedef struct CorRestState // // Historically `corRest` was a plain __thread object. It is now a __thread // POINTER (corRestP) behind the `corRest` macro, so the state can later be -// relocated off the thread (into MHD per-connection con_cls) without touching +// relocated off the thread (into the backend's per-connection slot) without touching // the ~2000 `corRest.foo` call sites. Until a request handler binds corRestP to // a connection's state, corRestBind() auto-binds it to a per-thread fallback // object — making every access crash-proof and, for now, behaviourally diff --git a/corRestBackend.h b/corRestBackend.h new file mode 100644 index 0000000..0ec1954 --- /dev/null +++ b/corRestBackend.h @@ -0,0 +1,166 @@ +// +// FILE corRestBackend.h +// +// AUTHOR Ken Zangelin +// +// Copyright 2026 Seamware +// SPDX-License-Identifier: Apache-2.0 +// +// The seam between corRest and the HTTP server underneath it. +// +// There are two servers - libmicrohttpd and the built-in epoll one in corHttp - +// and exactly one of them is in a build (COR_HTTP_SERVER). Everything on the +// corRest side of this header is the same either way: the service table, the +// dispatch, the worker pool, the response policy. Everything on the other side +// is a socket and a callback shape. +// +// Not a public header: it is the INTERNAL contract between corRestInit.c and +// corRestBackend.c, and nothing above corRest includes it. +// +#ifndef CORREST_BACKEND_H_ +#define CORREST_BACKEND_H_ + +#include // bool + +#include "corRest/CorRestKeyValue.h" // CorRestKeyValue +#include "corRest/CorRestState.h" // CorRestState + + + +// ----------------------------------------------------------------------------- +// +// What the backend implements +// +// corRestBackendStart - listen on 'port' and serve, and RETURN. +// +// Both backends run their own threads; neither blocks the caller, because +// corRestInit's caller goes on to do other things and then parks. keyPem / +// certPem are the HTTPS server credentials, NULL for plain HTTP - a backend +// without TLS refuses rather than silently serving the port unencrypted. +// +// Returns 0, or -1 with a reason on stderr. +// +extern int corRestBackendStart(unsigned short port, int poolSize, char* keyPem, char* certPem); + + +// +// corRestBackendStop - stop serving; called after the worker pool has drained. +// +extern void corRestBackendStop(void); + + +// +// corRestBackendFinish - the request is over; run its post-response work and +// release it +// +// The backend's, because only it knows what the request state BORROWED and what +// therefore has to be given back. Binds corRestP to stateP, runs the +// post-response hook, releases the arena, frees the state, and unbinds. +// +// Runs on a worker whenever there is a pool (corRestAsyncFinish), on the +// calling thread otherwise. +// +extern void corRestBackendFinish(CorRestState* stateP); + + +// +// corRestBackendResume - a worker has built the response; send it +// +// Called ON THE WORKER THREAD with corRestP still bound to 'stateP', which is +// what lets a backend read corRest.out here. It must not write to the socket +// itself - that belongs to whichever thread owns the connection - and both +// backends therefore only hand the connection back to their event loop. +// +extern void corRestBackendResume(CorRestState* stateP); + + + +// ----------------------------------------------------------------------------- +// +// What corRestInit.c provides to the backend +// +// corRestHttpHeaderAdd - one request header; key/value are BORROWED, so they +// must live as long as the request does. +// corRestUriParamsParse - split + percent-decode corRest.in.urlParams into +// corRest.in.uriParamV. Destroys the buffer it parses. +// corRestUriParamAdd - one already-split, already-decoded parameter. +// +extern void corRestHttpHeaderAdd(const char* key, const char* value); +extern void corRestUriParamsParse(void); +extern void corRestUriParamAdd(char* name, char* value); + + +// +// corRestBodyPolicyCheck - the two answers that are decided before the body +// +// § 6.3.4: POST/PATCH/PUT to an NGSI-LD route must carry Content-Length (411, +// no payload). § 6.3.2: an announced body over the broker's cap is a 413. +// Both are known from the request line and the headers alone, which is why +// they are here and not in the dispatch: a backend that streams the body wants +// to stop reading it, and one that has it already wants to not parse it. +// +// 'clHeader' is the Content-Length header value, NULL when absent. +// +extern void corRestBodyPolicyCheck(const char* url, const char* clHeader); + + +// +// corRestResponseHeaderVBuild - the response headers, in the order they go out +// +// ONE implementation of the header policy - content type, Preference-Applied, +// the service routine's own headers, CORS - for both backends, because the set +// and the ORDER of these headers is compared line by line by several hundred +// functional tests. Two copies of it would be two chances to drift. +// +// Fills 'hv' and returns how many. Keys and values are borrowed from the +// request arena and from static configuration, so they last as long as the +// request state does. +// +extern int corRestResponseHeaderVBuild(CorRestKeyValue* hv, int max); + +#define COR_REST_RESPONSE_HEADERS_MAX 64 + + +// +// corRestAsyncPoolUp / corRestAsyncEnqueue - hand the request to a worker +// +// Two calls and not one, because the connection has to be SUSPENDED BETWEEN +// THEM: a worker can pick the request up and finish it before the enqueue has +// returned, and resuming a connection that was never suspended is an API +// violation in one backend and a lost response in the other. +// +// if (corRestAsyncPoolUp()) { suspend(); corRestAsyncEnqueue(stateP); return; } +// corRestProcessRequest(); // pool down (shutdown, or a server without one) +// +extern bool corRestAsyncPoolUp(void); +extern void corRestAsyncEnqueue(CorRestState* stateP); + + +// +// corRestAsyncFinish - hand the POST-RESPONSE phase to a worker +// +// Returns true when a worker has taken it - the caller must then not touch +// stateP again. False means there is no pool and the caller runs +// corRestBackendFinish itself. +// +// The response has already gone out, so this is not about latency: it is about +// the thread. The phase can issue an HTTP request the broker itself has to +// answer (a notification compacted with an @context the broker hosts), and a +// single-threaded event loop running it would be waiting on itself. +// +extern bool corRestAsyncFinish(CorRestState* stateP); + + +// +// corRestWorkerPoolStart / Stop - one worker per I/O thread +// +extern int corRestWorkerPoolStart(int workers); +extern void corRestWorkerPoolStop(void); + + +// +// corRestProcessRequest - the dispatch itself (public; corRestInit.h) +// +extern void corRestProcessRequest(void); + +#endif // CORREST_BACKEND_H_ diff --git a/corRestBackendBuiltin.c b/corRestBackendBuiltin.c new file mode 100644 index 0000000..88b151a --- /dev/null +++ b/corRestBackendBuiltin.c @@ -0,0 +1,455 @@ +// +// FILE corRestBackendBuiltin.c +// +// AUTHOR Ken Zangelin +// +// Copyright 2026 Seamware +// SPDX-License-Identifier: Apache-2.0 +// +// The corHttp backend - the built-in HTTP server, no libmicrohttpd. +// +// The counterpart of corRestBackendMhd.c, and the same division of labour: this +// file translates one server's shape into corRest's, and decides nothing. What +// a URL means, what a body has to be, which headers go back and in what order +// all live on the other side of corRestBackend.h and are shared with the MHD +// backend byte for byte - which is the whole point, since several hundred +// functional tests compare captured responses line by line and must not be able +// to tell which server produced them. +// +// TWO THINGS MHD DID THAT corHttp DOES NOT, and both are done here: +// +// PERCENT-DECODING. MHD hands over a decoded URL path and decoded query +// parameter values. corHttp decodes nothing - deliberately, because the +// decoding rules differ per parameter in NGSI-LD and the layer that knows +// which is which is this one. So the path is decoded here (after the query +// has been split off, so a `%3F` in a path cannot masquerade as the query +// delimiter) and the query goes through corRestUriParamsParse, the same +// split-and-decode the in-process self-forward path uses. +// +// ...and '+' IN THE QUERY IS A SPACE, which is not what RFC 3986 says and is +// what every HTTP client does. The rule belongs to +// application/x-www-form-urlencoded, but a query string is what that format +// is FOR, so urlencode/quote_plus is what client libraries produce: Python's +// requests - and therefore the ETSI conformance suite - sends +// `q=name=="Eiffel Tower"` on the wire as `q=name%3D%3D%22Eiffel+Tower%22`. +// Read literally, that query asks for a name with a plus in it and matches +// nothing, with a 200 and an empty array to show for it. A value that really +// does contain a plus - the NGSI-LD single-level scope wildcard, +// `scopeQ=/Madrid/%2B/ParqueNorte` - has to be percent-encoded, and then both +// servers agree. +// +// THE BODY LIMIT. MHD delivers a body in chunks and can be told to stop +// mid-stream; corHttp delivers a whole request or none, so "stop reading" has +// to become "do not start". Its cap is set from corRest's § 6.3.2 threshold +// here, and over it corHttp hands the request up WITHOUT its body - which is +// exactly what corRestBodyPolicyCheck needs, since that answers from the +// Content-Length header and not from the bytes. +// +// AND ONE THING THIS BACKEND DOES THAT MHD'S DOES NOT: the request is COPIED +// out of corHttp's read buffer - method, header keys and values, body - rather +// than pointed at. corHttp is zero-copy and the pointers are good for exactly +// as long as the connection stays on this request; the post-response phase runs +// AFTER the response is out and the connection has moved on, on another thread, +// and it reads the request (the tenant, the Via chain) while it builds the +// notifications. Copying is what makes the request state outlive the connection +// it arrived on. MHD needs none of this because MHD owns copies of its own and +// keeps them until NOTIFY_COMPLETED. +// +#include // malloc, free +#include // fprintf +#include // memcpy +#include // pthread_create +#include // clock_gettime + +#include "kalloc/kaAlloc.h" // kaAlloc +#include "kalloc/kaStrdup.h" // kaStrdup +#include "ktrace/kTrace.h" // KT_V + +#include "corHttp/CorHttp.h" // CorHttpServer, CorHttpConn + +#include "corRest/CorRestState.h" // CorRestState, corRest +#include "corRest/corRestStateInit.h" // corRestStateInit, corRestUrlPathNormalize +#include "corRest/corRestHooks.h" // CorRestHook, ... +#include "corRest/corRestUrlValueEncode.h" // corRestUrlValueDecode +#include "corRest/corRestBackend.h" // Own interface + + + +// ----------------------------------------------------------------------------- +// +// The server, and the thread its event loop runs on +// +// corHttpServe() does not return until it is told to stop, and corRestInit's +// caller has other things to do before it parks - so the loop gets a thread of +// its own here, which is also what MHD_USE_SELECT_INTERNALLY does on the other +// side. +// +static CorHttpServer corHttpServer; +static pthread_t corHttpThread; +static bool corHttpRunning = false; + + + +// ----------------------------------------------------------------------------- +// +// Hook globals (defined in corRestHooks.c) +// +extern CorRestUserDataAllocHook corRestUserDataAllocHookF; +extern CorRestUserDataFreeHook corRestUserDataFreeHookF; +extern CorRestHook corRestPostResponseHook; +extern unsigned long long corRestMaxRequestSize; + + + +// ----------------------------------------------------------------------------- +// +// requestResponseFill - move corRest.out onto the connection +// +// Runs on whichever thread built the response - a worker for the async path, +// the loop thread for the inline one - and never touches the socket. Nothing is +// copied: corHttp borrows what it is given, and everything given here lives in +// the request arena, which is released in httpRequestDone, after the bytes are +// out. +// +static void requestResponseFill(CorHttpConn* connP) +{ + CorRestKeyValue headerV[COR_REST_RESPONSE_HEADERS_MAX]; + int headers = corRestResponseHeaderVBuild(headerV, COR_REST_RESPONSE_HEADERS_MAX); + + corHttpResponseStatus(connP, corRest.out.httpStatusCode); + + for (int ix = 0; ix < headers; ix++) + corHttpResponseHeader(connP, headerV[ix].key, headerV[ix].value); + + // + // HEAD gets the length of the body and none of its bytes, and that decision + // is corHttp's (it knows the method) - so the body goes over either way. + // + if ((corRest.out.payload != NULL) && (corRest.out.payloadSize > 0)) + corHttpResponseBody(connP, corRest.out.payload, corRest.out.payloadSize); +} + + + +// ----------------------------------------------------------------------------- +// +// httpRequestCb - one complete request, on the event-loop thread +// +static void httpRequestCb(CorHttpConn* connP) +{ + KT_V("Request: %s %s", connP->method.s, connP->path.s); // one line per request (-v) + + // + // One CorRestState per request, hung on the connection - the same arrangement + // as MHD's con_cls, and for the same reason: the loop may run another + // connection's request between this one's dispatch and its send. + // + CorRestState* stateP = (CorRestState*) malloc(sizeof(CorRestState)); + + if (stateP == NULL) + { + corHttpResponseStatus(connP, 503); + return; + } + + connP->userData = stateP; + corRestP = stateP; + + corRestStateInit(connP, connP->path.s, connP->method.s); + + // The verb string is corHttp's too - copied, like everything else below. + corRest.in.verbString = kaStrdup(&corRest.kalloc, connP->method.s); + + // + // The path arrived percent-ENCODED (MHD would have decoded it before we ever + // saw it). Decoded here, on corRest's own strdup'd copy rather than in the + // read buffer, and AFTER corRestStateInit split off a query that corHttp had + // already separated - so a path containing `%3F` decodes to a '?' inside the + // path, which is what it is, instead of splitting the URL in two. + // + corRestUrlValueDecode(corRest.in.urlPath); + corRestUrlPathNormalize(); + + // Create this connection's application state (e.g. per-conn corNgsild), + // stored in corRest.userData and freed in httpRequestDone. + if (corRestUserDataAllocHookF != NULL) + corRest.userData = corRestUserDataAllocHookF(); + + // Capture request start time — REALTIME for timestamps, MONOTONIC for duration metrics + struct timespec ts; + clock_gettime(CLOCK_REALTIME, &ts); + corRest.requestStartTime = (uint64_t) ts.tv_sec * 1000000000ULL + (uint64_t) ts.tv_nsec; + + struct timespec tsM; + clock_gettime(CLOCK_MONOTONIC, &tsM); + corRest.requestStartTimeMono = (uint64_t) tsM.tv_sec * 1000000000ULL + (uint64_t) tsM.tv_nsec; + + // + // Request headers, COPIED into the request arena - see the note at the top of + // this file. A header that will not fit is dropped rather than borrowed: a + // missing header is a wrong answer, a dangling one is a crash under load + // weeks later. + // + for (int ix = 0; ix < connP->headers; ix++) + { + char* key = kaStrdup(&corRest.kalloc, connP->header[ix].key.s); + char* value = kaStrdup(&corRest.kalloc, connP->header[ix].value.s); + + if ((key != NULL) && (value != NULL)) + corRestHttpHeaderAdd(key, value); + } + + // + // The query string, split and percent-decoded by corRest itself - corHttp + // split it too, but on a copy and without decoding, which is not what the + // layers above expect. Parsed from a copy of our own because the parse is + // destructive and corHttp's raw query is not ours to consume. + // + // corRest.in.urlParams is left NULL afterwards, exactly as it is under MHD: + // it does not survive its own parser (see corRestUriParamsParse), and a + // half-eaten query string is worse than none. + // + if (connP->query.len > 0) + { + char* query = (char*) kaAlloc(&corRest.kalloc, connP->query.len + 1); + + if (query != NULL) + { + memcpy(query, connP->query.s, connP->query.len); + query[connP->query.len] = 0; + + // + // '+' means space - see the note at the top of this file. BEFORE the + // percent-decoding in corRestUriParamsParse and not after, so that a + // `%2B` (a plus the client meant literally) decodes to a plus and stays + // one, instead of being turned into a space by this loop. + // + for (char* p = query; *p != 0; p++) + { + if (*p == '+') + *p = ' '; + } + + corRest.in.urlParams = query; + corRestUriParamsParse(); + corRest.in.urlParams = NULL; + } + } + + corRestBodyPolicyCheck(corRest.in.urlPath, corHttpHeader(connP, "Content-Length")); + + // + // The body, into the request arena. Released with everything else by + // corRestStateRelease, which is why corRestBackendFinish here does not + // free(corRest.in.payload) the way the MHD backend has to - there, the body + // is a malloc'd accumulation buffer. + // + if ((connP->body.s != NULL) && (connP->body.len > 0) && + (corRest.in.contentLengthMissing == false) && (corRest.out.httpStatusCode != 413)) + { + char* body = (char*) kaAlloc(&corRest.kalloc, connP->body.len + 1); + + if (body != NULL) + { + memcpy(body, connP->body.s, connP->body.len); + body[connP->body.len] = 0; + + corRest.in.payload = body; + corRest.in.payloadSize = connP->body.len; + } + } + + if (corRestAsyncPoolUp() == true) + { + // + // Off the event loop: a DB round-trip or a distributed operation would stop + // every other connection for its duration. Suspend BEFORE enqueue - a + // worker can finish before the enqueue call returns. + // + corHttpSuspend(connP); + corRestAsyncEnqueue(stateP); + return; + } + + corRestProcessRequest(); // pool down (shutdown / tests): run inline here + requestResponseFill(connP); +} + + + +// ----------------------------------------------------------------------------- +// +// httpRequestDone - the response is on the wire; the request state can go +// +// corHttp calls this on the LOOP thread after the last byte of the response has +// been written (or when the connection died with a request on it). That is the +// same moment MHD's NOTIFY_COMPLETED fires, and it has to be: the response +// headers and body are borrowed from the arena released at the end of it. +// +// ⚠️ THE WORK ITSELF MUST NOT RUN HERE. The post-response phase dispatches the +// notifications for a write, and a notification is compacted with the +// subscription's @context - which can be one THIS BROKER HOSTS. The loop thread +// would then be waiting for an answer only the loop thread can produce. It does +// not hang forever, which is worse: the download times out, the notification +// goes out ten seconds late with an uncompacted body, and nothing says why. +// +// So it goes to a worker, and the loop returns immediately. Everything the +// phase reads was copied out of the connection in httpRequestCb, so the +// connection is free to take its next request the instant this returns. +// +static void httpRequestDone(CorHttpConn* connP) +{ + CorRestState* stateP = (CorRestState*) connP->userData; + + connP->userData = NULL; // before anything can fail: this must not run twice + stateP->connection = NULL; // the connection is not this request's any more + + if (corRestAsyncFinish(stateP) == true) + return; + + corRestBackendFinish(stateP); // no pool (shutdown / tests): here, then +} + + + +// ----------------------------------------------------------------------------- +// +// corRestBackendFinish - the post-response phase, and the end of the request +// +void corRestBackendFinish(CorRestState* stateP) +{ + corRestP = stateP; + + // Post-response hook BEFORE releasing the arena, so deferred work (the + // notifications for a write, say) can still read what the service routine built. + corRestPostResponseHook(); + + corRestStateRelease(); + + if (corRestUserDataFreeHookF != NULL && corRest.userData != NULL) + corRestUserDataFreeHookF(corRest.userData); + + free(stateP); + + corRestP = NULL; // no dangling pointer to freed state on this thread +} + + + +// ----------------------------------------------------------------------------- +// +// corRestBackendResume - +// +// On the worker thread, with corRestP still bound: build the response, then +// hand the connection back to the loop. corHttpResume does not write to the +// socket - the loop does - which is what keeps two answers off one connection. +// +void corRestBackendResume(CorRestState* stateP) +{ + CorHttpConn* connP = (CorHttpConn*) stateP->connection; + + requestResponseFill(connP); + corHttpResume(connP); +} + + + +// ----------------------------------------------------------------------------- +// +// serveThread - +// +static void* serveThread(void* unused) +{ + corHttpServe(&corHttpServer); + return NULL; +} + + + +// ----------------------------------------------------------------------------- +// +// corRestBackendStart - +// +int corRestBackendStart(unsigned short port, int poolSize, char* keyPem, char* certPem) +{ + if ((keyPem != NULL) || (certPem != NULL)) + { + // + // Refused, not ignored. The alternative is a server that was asked for TLS + // and serves the port in the clear, which nothing downstream would notice + // until a client's traffic was already on the wire unencrypted. + // + fprintf(stderr, "corRestInit: the built-in HTTP server has no TLS - " + "rebuild with COR_HTTP_SERVER=mhd for an HTTPS listener\n"); + return -1; + } + + // + // The connection pool, not a thread pool: this server is one thread and the + // parallelism is corRest's worker pool behind it. Sized off poolSize all the + // same, since that is the caller's statement about how much concurrency to + // expect, and 16 connections per worker is room rather than a limit. + // + int connPoolSize = (poolSize > 0) ? poolSize * 16 : COR_HTTP_CONN_POOL_SIZE; + + if (connPoolSize < COR_HTTP_CONN_POOL_SIZE) + connPoolSize = COR_HTTP_CONN_POOL_SIZE; + + if (corHttpInit(&corHttpServer, port, connPoolSize, httpRequestCb) != CorHttpOk) + { + fprintf(stderr, "corRestInit: the built-in HTTP server failed to listen on port %d\n", port); + return -1; + } + + corHttpServer.doneCb = httpRequestDone; + + // + // The same § 6.3.2 threshold the dispatch enforces, plus room for the headers + // that arrive in the same buffer as the body. + // + // Over it, corHttp delivers the request WITHOUT its body rather than reading + // one it is going to refuse, and corRestBodyPolicyCheck then answers from the + // Content-Length header with the ProblemDetails the spec asks for - the same + // answer MHD's streaming path produces, from the same header, in the same + // words. What the engine's own cap is left holding is a client that LIES + // about its length, which is the one case where there is no announcement to + // believe. + // + if (corRestMaxRequestSize > 0) + { + unsigned long long cap = corRestMaxRequestSize + (64 * 1024); + + // maxRequestSize is an int over there; a --maxRequestSize of a few GiB would + // otherwise wrap and cap every request at a negative number of bytes. + corHttpServer.maxRequestSize = (cap > 0x7fffffffULL) ? 0 : (int) cap; + } + + if (pthread_create(&corHttpThread, NULL, serveThread, NULL) != 0) + { + fprintf(stderr, "corRestInit: the built-in HTTP server's event loop failed to start\n"); + corHttpRelease(&corHttpServer); + return -1; + } + + corHttpRunning = true; + + return 0; +} + + + +// ----------------------------------------------------------------------------- +// +// corRestBackendStop - +// +void corRestBackendStop(void) +{ + if (corHttpRunning == false) + return; + + corHttpStop(&corHttpServer); // a flag; the loop notices within its 1 s timeout + pthread_join(corHttpThread, NULL); + corHttpRelease(&corHttpServer); + + corHttpRunning = false; +} diff --git a/corRestBackendMhd.c b/corRestBackendMhd.c new file mode 100644 index 0000000..58ad84f --- /dev/null +++ b/corRestBackendMhd.c @@ -0,0 +1,400 @@ +// +// FILE corRestBackendMhd.c +// +// AUTHOR Ken Zangelin +// +// Copyright 2026 Seamware +// SPDX-License-Identifier: Apache-2.0 +// +// The libmicrohttpd backend. +// +// Everything here is MHD's shape: its multi-call request callback, its value +// iterators, its response object, its suspend/resume. The request itself - +// what the URL means, what the body has to be, which headers go back and in +// what order - is on the other side of corRestBackend.h and is shared with the +// built-in server. This file translates, and decides nothing. +// +#include // strtoull, malloc, free, realloc +#include // memcpy, strncmp +#include // fprintf +#include // clock_gettime + +#include + +#include "ktrace/kTrace.h" // KT_V + +#include "corRest/CorRestVerb.h" // CorVerbPost, ... +#include "corRest/CorRestState.h" // CorRestState, corRest +#include "corRest/corRestStateInit.h" // corRestStateInit, corRestStateRelease +#include "corRest/corRestHooks.h" // CorRestHook, corRestSetMaxRequestSize +#include "corRest/corRestProblem.h" // COR_REST_ERROR_*, corRestProblem +#include "corRest/corRestBackend.h" // Own interface + + + +// ----------------------------------------------------------------------------- +// +// The daemon +// +static struct MHD_Daemon* mhdDaemon = NULL; + + + +// ----------------------------------------------------------------------------- +// +// Hook globals (defined in corRestHooks.c) +// +extern CorRestUserDataAllocHook corRestUserDataAllocHookF; +extern CorRestUserDataFreeHook corRestUserDataFreeHookF; +extern CorRestHook corRestPostResponseHook; +extern unsigned long long corRestMaxRequestSize; + + + +// ----------------------------------------------------------------------------- +// +// mhdHeaderIterator - MHD callback to collect request headers +// +static enum MHD_Result mhdHeaderIterator +( + void* cls, + enum MHD_ValueKind kind, + const char* key, + const char* value +) +{ + corRestHttpHeaderAdd(key, value); + return MHD_YES; +} + + + +// ----------------------------------------------------------------------------- +// +// mhdUriParamIterator - MHD callback to collect URI query parameters +// +// MHD strips query params from the URL and provides them via MHD_GET_ARGUMENT_KIND, +// already percent-decoded — which is why this hands them straight over rather +// than going through corRestUriParamsParse (the built-in backend's route, since +// that server decodes nothing). +// +static enum MHD_Result mhdUriParamIterator +( + void* cls, + enum MHD_ValueKind kind, + const char* key, + const char* value +) +{ + corRestUriParamAdd((char*) key, (char*) (value ? value : "")); + return MHD_YES; +} + + + +// ----------------------------------------------------------------------------- +// +// mhdConnectionHandler - MHD callback, called for each incoming request +// +// MHD calls this multiple times per request: +// 1. First call: *con_cls == NULL -> init corRest state +// 2. Middle calls: upload_data_size > 0 -> accumulate payload +// 3. Final call: upload_data_size == 0 -> parse, dispatch, render, respond +// +static enum MHD_Result mhdConnectionHandler +( + void* cls, + struct MHD_Connection* connection, + const char* url, + const char* method, + const char* version, + const char* uploadData, + size_t* uploadDataSize, + void** con_cls +) +{ + // --- First call: allocate this connection's per-request state --- + if (*con_cls == NULL) + { + KT_V("Request: %s %s", method, url); // one line per request (verbose mode, -v) + + // Each connection owns its CorRestState (hung on con_cls), so when the + // epoll pool thread interleaves connection B's callbacks between + // connection A's body-read callbacks it can no longer clobber A's state. + // corRestP is bound to it before corRestStateInit (whose memset/init runs + // through the corRest macro). + CorRestState* conP = (CorRestState*) malloc(sizeof(CorRestState)); + if (conP == NULL) + return MHD_NO; + *con_cls = conP; + corRestP = conP; + + corRestStateInit(connection, url, method); + + // Create this connection's application state (e.g. per-conn corNgsild), + // stored in corRest.userData and freed in mhdRequestCompleted. + if (corRestUserDataAllocHookF != NULL) + corRest.userData = corRestUserDataAllocHookF(); + + // Capture request start time — REALTIME for timestamps, MONOTONIC for duration metrics + struct timespec ts; + clock_gettime(CLOCK_REALTIME, &ts); + corRest.requestStartTime = (uint64_t) ts.tv_sec * 1000000000ULL + (uint64_t) ts.tv_nsec; + + struct timespec tsM; + clock_gettime(CLOCK_MONOTONIC, &tsM); + corRest.requestStartTimeMono = (uint64_t) tsM.tv_sec * 1000000000ULL + (uint64_t) tsM.tv_nsec; + + // Collect request headers from MHD + MHD_get_connection_values(connection, MHD_HEADER_KIND, mhdHeaderIterator, NULL); + + // Collect URI query parameters from MHD (already percent-decoded by MHD) + MHD_get_connection_values(connection, MHD_GET_ARGUMENT_KIND, mhdUriParamIterator, NULL); + + corRestBodyPolicyCheck(url, MHD_lookup_connection_value(connection, MHD_HEADER_KIND, "Content-Length")); + + return MHD_YES; + } + + // Subsequent calls (body chunks, final dispatch): rebind corRestP to THIS + // connection's state — another connection's callback may have re-pointed + // corRestP on this pool thread since our last invocation here. + corRestP = (CorRestState*) *con_cls; + + // --- Middle calls: accumulate payload --- + if (*uploadDataSize > 0) + { + // If the first-call check flagged 411/413, drop the bytes. + if (corRest.in.contentLengthMissing || corRest.out.httpStatusCode == 413) + { + *uploadDataSize = 0; + return MHD_YES; + } + + // Streaming size cap — defends against clients that lie in + // Content-Length or use chunked encoding without a length. + if (corRestMaxRequestSize > 0 && + (unsigned long long)(corRest.in.payloadSize + *uploadDataSize) > corRestMaxRequestSize) + { + corRestProblem(413, COR_REST_ERROR_REQUEST_LENGTH, "Request Entity Too Large", + "request body exceeds broker limit of %llu bytes", corRestMaxRequestSize); + *uploadDataSize = 0; + return MHD_YES; + } + + int needed = corRest.in.payloadSize + *uploadDataSize + 1; + + if (needed > corRest.payloadBufSize) + { + int newSize = (needed + 4096) & ~4095; + char* newBuf = (char*) realloc(corRest.in.payload, newSize); + if (newBuf == NULL) + return MHD_NO; + corRest.in.payload = newBuf; + corRest.payloadBufSize = newSize; + } + + memcpy(corRest.in.payload + corRest.in.payloadSize, uploadData, *uploadDataSize); + corRest.in.payloadSize += *uploadDataSize; + corRest.in.payload[corRest.in.payloadSize] = 0; + + *uploadDataSize = 0; + return MHD_YES; + } + + // --- Final call: process (off the I/O thread when the pool is up), respond --- + if (corRest.asyncProcessed == false) + { + if (corRestAsyncPoolUp() == true) + { + // Suspend this connection and hand it to a worker so DB/distop latency + // doesn't block this epoll thread. The worker runs corRestProcessRequest + // and resumes us; MHD then re-invokes this handler with asyncProcessed + // set and we fall through to build + send the response. Suspend BEFORE + // enqueue so a worker can never resume a not-yet-suspended connection. + MHD_suspend_connection(connection); + corRestAsyncEnqueue(corRestP); + return MHD_YES; + } + + corRestProcessRequest(); // pool down (shutdown / tests): run inline here + } + + char* responseBody = (corRest.out.payload != NULL) ? corRest.out.payload : (char*) ""; + int responseBodySize = corRest.out.payloadSize; + + // Send HTTP response + // For HEAD: pass the full body — MHD will set Content-Length correctly + // but suppress the body in the actual response. + struct MHD_Response* response; + + response = MHD_create_response_from_buffer( + responseBodySize, + (void*) responseBody, + MHD_RESPMEM_MUST_COPY + ); + + CorRestKeyValue headerV[COR_REST_RESPONSE_HEADERS_MAX]; + int headers = corRestResponseHeaderVBuild(headerV, COR_REST_RESPONSE_HEADERS_MAX); + + for (int i = 0; i < headers; i++) + MHD_add_response_header(response, headerV[i].key, headerV[i].value); + + enum MHD_Result ret = MHD_queue_response(connection, corRest.out.httpStatusCode, response); + MHD_destroy_response(response); + + return ret; +} + + + +// ----------------------------------------------------------------------------- +// +// corRestBackendFinish - +// +// Runs on the MHD I/O thread that finished the request. NOT handed to a worker +// the way the built-in backend has to hand it: MHD has poolSize I/O threads, so +// a post-response phase that blocks - and it can, on an @context this broker +// hosts itself - stalls one of them and the others keep serving. That is why +// this backend is left exactly as it was. +// +void corRestBackendFinish(CorRestState* stateP) +{ + corRestP = stateP; + + // Run post-response hook BEFORE releasing the per-request arena so the + // hook can still touch arena-allocated data (e.g. deferred notification + // dispatch reading the entity tree built by the service routine). + corRestPostResponseHook(); + + // Free payload buffer (malloc'd during accumulation, not in kalloc) + free(corRest.in.payload); + corRest.in.payload = NULL; + + corRestStateRelease(); + + // Destroy this connection's application state (per-conn corNgsild, ...). + if (corRestUserDataFreeHookF != NULL && corRest.userData != NULL) + corRestUserDataFreeHookF(corRest.userData); + + free(stateP); + + corRestP = NULL; // no dangling pointer to freed state on this thread +} + + + +// ----------------------------------------------------------------------------- +// +// mhdRequestCompleted - MHD callback when a request is fully handled +// +static void mhdRequestCompleted +( + void* cls, + struct MHD_Connection* connection, + void** con_cls, + enum MHD_RequestTerminationCode toe +) +{ + if (*con_cls != NULL) + { + CorRestState* conP = (CorRestState*) *con_cls; + + *con_cls = NULL; + corRestBackendFinish(conP); + } +} + + + +// ----------------------------------------------------------------------------- +// +// corRestBackendResume - +// +// The worker built the response into corRest.out; MHD sends it by re-invoking +// the connection handler, which it does once the connection is resumed (the ITC +// pipe wakes the polling thread). +// +void corRestBackendResume(CorRestState* stateP) +{ + MHD_resume_connection((struct MHD_Connection*) stateP->connection); +} + + + +// ----------------------------------------------------------------------------- +// +// corRestBackendStart - +// +int corRestBackendStart(unsigned short port, int poolSize, char* keyPem, char* certPem) +{ + // MHD_ALLOW_SUSPEND_RESUME (which bundles MHD_USE_ITC) lets the I/O threads + // suspend a connection and hand it to the async worker pool; the worker + // resumes it once the response is built. + // + // When HTTPS server credentials have been set (a TLS test receiver, not the + // broker), MHD_USE_TLS is added together with the in-memory key/cert. + unsigned int flags = MHD_USE_SELECT_INTERNALLY | MHD_USE_EPOLL | MHD_ALLOW_SUSPEND_RESUME; + + if ((keyPem != NULL) && (certPem != NULL)) + mhdDaemon = MHD_start_daemon( + flags | MHD_USE_TLS, + port, + NULL, + NULL, + mhdConnectionHandler, + NULL, + MHD_OPTION_NOTIFY_COMPLETED, + mhdRequestCompleted, + NULL, + MHD_OPTION_THREAD_POOL_SIZE, + (unsigned int) poolSize, + MHD_OPTION_CONNECTION_TIMEOUT, + (unsigned int) 30, + MHD_OPTION_HTTPS_MEM_KEY, + keyPem, + MHD_OPTION_HTTPS_MEM_CERT, + certPem, + MHD_OPTION_END + ); + else + mhdDaemon = MHD_start_daemon( + flags, + port, + NULL, + NULL, + mhdConnectionHandler, + NULL, + MHD_OPTION_NOTIFY_COMPLETED, + mhdRequestCompleted, + NULL, + MHD_OPTION_THREAD_POOL_SIZE, + (unsigned int) poolSize, + MHD_OPTION_CONNECTION_TIMEOUT, + (unsigned int) 30, + MHD_OPTION_END + ); + + if (mhdDaemon == NULL) + { + fprintf(stderr, "corRestInit: MHD_start_daemon failed on port %d\n", port); + return -1; + } + + return 0; +} + + + +// ----------------------------------------------------------------------------- +// +// corRestBackendStop - +// +void corRestBackendStop(void) +{ + if (mhdDaemon != NULL) + { + MHD_stop_daemon(mhdDaemon); + mhdDaemon = NULL; + } +} diff --git a/corRestInit.c b/corRestInit.c index a22138b..d4aa4b2 100644 --- a/corRestInit.c +++ b/corRestInit.c @@ -12,8 +12,6 @@ #include // clock_gettime, CLOCK_MONOTONIC #include // pthread_* (async worker pool) -#include - #include "kalloc/kaAlloc.h" // kaAlloc #include "kalloc/kaStrdup.h" // kaStrdup #include "kjson/kjParse.h" // kjParse @@ -30,6 +28,7 @@ #include "corRest/corRestHooks.h" // CorRestHook, etc. #include "corRest/corRestProblem.h" // COR_REST_ERROR_*, corRestProblem #include "corRest/corRestParamRegistry.h" // corRestParamLookup +#include "corRest/corRestBackend.h" // corRestBackendStart, corRestBackendResume #include "corRest/corRestInit.h" // Own interface #include "corRest/corRestUrlValueEncode.h" // corRestUrlValueDecode @@ -40,7 +39,6 @@ // Globals // CorRestServiceVector corRestServiceV[CorVerbs]; -struct MHD_Daemon* corRestDaemon = NULL; @@ -51,7 +49,8 @@ struct MHD_Daemon* corRestDaemon = NULL; // The broker listens over plain HTTP and never sets these. A test server such // as ftClient that must receive notifications over TLS calls // corRestHttpsServerCredentialsSet() with the key+certificate PEM before -// corRestInit; the MHD daemon is then started with TLS enabled. +// corRestInit; the backend is then started with TLS enabled - and a backend +// that has no TLS refuses to start rather than serving the port in the clear. // static char* httpsServerKey = NULL; static char* httpsServerCert = NULL; @@ -78,6 +77,7 @@ extern CorRestHook corRestPostResponseHook; extern CorRestUserDataAllocHook corRestUserDataAllocHookF; extern CorRestUserDataFreeHook corRestUserDataFreeHookF; extern unsigned long long corRestMaxRequestSize; +extern CorRestCorsConfig corRestCors; @@ -213,9 +213,9 @@ static void servicePrepare(CorRestService* serviceP, CorRestServiceSimplified* s // ----------------------------------------------------------------------------- // -// addUriParam - add URI parameter to dynamic array, growing if needed +// corRestUriParamAdd - add URI parameter to dynamic array, growing if needed // -static void addUriParam(char* name, char* value) +void corRestUriParamAdd(char* name, char* value) { if (corRest.in.uriParamCount >= corRest.in.uriParamSize) { @@ -241,9 +241,15 @@ static void addUriParam(char* name, char* value) // ----------------------------------------------------------------------------- // -// parseUriParams - parse query string into key-value pairs with percent-decoding +// corRestUriParamsParse - split the query string, percent-decoding as it goes +// +// IN PLACE, over corRest.in.urlParams: the buffer comes out with a NUL where +// every '&' and the first '=' of each parameter was, so the raw query string +// does not survive this. Nothing reads it afterwards (purgeEntities rebuilds +// the query from the parsed array for exactly that reason), and the alternative +// - a second copy on every request that carries a query - buys nothing. // -static void parseUriParams(void) +void corRestUriParamsParse(void) { if (corRest.in.urlParams == NULL) return; @@ -285,7 +291,7 @@ static void parseUriParams(void) if (value[0] != '\0') corRestUrlValueDecode(value); - addUriParam(name, value); + corRestUriParamAdd(name, value); } } @@ -293,13 +299,13 @@ static void parseUriParams(void) // ----------------------------------------------------------------------------- // -// addHttpHeader - append a request header to corRest.in, extracting well-knowns +// corRestHttpHeaderAdd - append a request header to corRest.in, extracting well-knowns // -// Shared by the MHD header iterator and the in-process self-forward path. The +// Shared by the HTTP backends and the in-process self-forward path. The // key/value pointers are borrowed (not copied) — the caller keeps them alive // for the request's lifetime. // -static void addHttpHeader(const char* key, const char* value) +void corRestHttpHeaderAdd(const char* key, const char* value) { if (corRest.in.httpHeaderCount >= corRest.in.httpHeaderSize) { @@ -331,71 +337,14 @@ static void addHttpHeader(const char* key, const char* value) -// ----------------------------------------------------------------------------- -// -// mhdHeaderIterator - MHD callback to collect request headers -// -static enum MHD_Result mhdHeaderIterator -( - void* cls, - enum MHD_ValueKind kind, - const char* key, - const char* value -) -{ - addHttpHeader(key, value); - return MHD_YES; -} - - - -// ----------------------------------------------------------------------------- -// -// mhdUriParamIterator - MHD callback to collect URI query parameters -// -// MHD strips query params from the URL and provides them via MHD_GET_ARGUMENT_KIND. -// The values are already percent-decoded by MHD. -// -static enum MHD_Result mhdUriParamIterator -( - void* cls, - enum MHD_ValueKind kind, - const char* key, - const char* value -) -{ - if (corRest.in.uriParamCount >= corRest.in.uriParamSize) - { - int newSize = corRest.in.uriParamSize + COR_REST_KV_GROW_SIZE; - CorRestKeyValue* newV = (CorRestKeyValue*) kaAlloc(&corRest.kalloc, newSize * sizeof(CorRestKeyValue)); - - if (newV == NULL) - return MHD_YES; - - memcpy(newV, corRest.in.uriParamV, corRest.in.uriParamCount * sizeof(CorRestKeyValue)); - - corRest.in.uriParamV = newV; - corRest.in.uriParamSize = newSize; - } - - corRest.in.uriParamV[corRest.in.uriParamCount].key = (char*) key; - corRest.in.uriParamV[corRest.in.uriParamCount].value = (char*) (value ? value : ""); - corRest.in.uriParamV[corRest.in.uriParamCount].bit = 0; // resolved in the allowlist pass - corRest.in.uriParamCount++; - - return MHD_YES; -} - - - // ----------------------------------------------------------------------------- // // corRestProcessRequest - run the request-dispatch core on the bound corRest state // // Connection-free: consumes corRest.in (verb, url, headers, params, requestTree) // and produces corRest.out (status, headers, problem / responseTree) plus the -// final rendered body in corRest.out.payload / payloadSize. The MHD send stays in -// the connection handler. Reusable for an in-process self-forward (a distop +// final rendered body in corRest.out.payload / payloadSize. The send stays in +// the backend's connection handler. Reusable for an in-process self-forward (a distop // whose endpoint is this broker): bind corRestP to an inner state, populate its // .in, call this, read its .out — no socket round-trip. // @@ -825,11 +774,11 @@ int corRestProcessInProcess(CorRestVerb verb, corRest.requestStartTimeMono = outerP->requestStartTimeMono; // URI params (corRestStateInit split a trailing ?query into corRest.in.urlParams) - parseUriParams(); + corRestUriParamsParse(); // Request headers — borrowed; the (outer) arena they live in stays alive for (int i = 0; i < headerCount; i++) - addHttpHeader(headerV[i].key, headerV[i].value); + corRestHttpHeaderAdd(headerV[i].key, headerV[i].value); // Body — copy into the inner arena (freed by corRestStateRelease) if (body != NULL && bodyLen > 0) @@ -900,20 +849,20 @@ int corRestProcessInProcess(CorRestVerb verb, // // Async worker pool (6d) // -// The MHD I/O threads accept, parse headers and accumulate the body, then +// The backend's I/O threads accept, parse headers and accumulate the body, then // suspend the connection and hand its CorRestState to this pool. A worker binds // corRestP to that state and runs the (potentially slow: DB, distops) dispatch -// off the I/O thread, then resumes the connection — MHD re-invokes the handler, -// which sends the already-built response. This decouples per-request processing -// latency from epoll I/O throughput: a slow request no longer blocks the I/O -// thread from servicing every other connection pinned to it. +// off the I/O thread, then resumes the connection, and the backend sends the +// already-built response. This decouples per-request processing latency from +// epoll I/O throughput: a slow request no longer blocks the I/O thread from +// servicing every other connection pinned to it. // // All request state lives in the per-connection CorRestState (arena, kjson, // in/out, userData -> per-conn corNgsild), so a worker needs nothing but // `corRestP = conP` — the I/O thread already ran corRestStateInit. The deferred // notification caches are per-connection too, and still flushed by the -// post-response hook in mhdRequestCompleted, so notification ordering is -// unchanged. +// post-response hook once the response has gone out, so notification ordering +// is unchanged. // static pthread_t* corRestWorkerV = NULL; static int corRestWorkerCount = 0; @@ -959,23 +908,42 @@ static void* corRestWorkerMain(void* unused) corRestQueueTail = NULL; pthread_mutex_unlock(&corRestQueueMtx); + corRestP = conP; + + // + // Second visit: the response has gone out and what is left is the request's + // post-response work. Nothing below applies - there is no connection to + // resume, and after this the state is gone. + // + if (conP->asyncFinishing == true) + { + corRestBackendFinish(conP); + corRestP = NULL; + continue; + } + // Run the dispatch on this worker, bound to the connection's own state. // A self-targeted forward (corRestProcessInProcess) runs synchronously // inside this same call on this same worker thread — never re-enqueued. - corRestP = conP; corRestProcessRequest(); conP->asyncProcessed = true; - corRestP = NULL; // drop the bind; the request leaves this thread - // Hand the connection back to MHD (ITC wakes the polling thread); MHD - // re-invokes mhdConnectionHandler, which sends corRest.out. - MHD_resume_connection(conP->mhdConnection); + // + // Hand the connection back to the backend's event loop. STILL BOUND: the + // built-in backend reads corRest.out here to build the response before it + // wakes the loop, and unbinding first would leave it reading the fallback + // state of this worker thread. After this call the connection may already + // be finished and freed by the loop, so nothing below may touch conP. + // + corRestBackendResume(conP); + + corRestP = NULL; // drop the bind; the request has left this thread } return NULL; } -static int corRestWorkerPoolStart(int workers) +int corRestWorkerPoolStart(int workers) { if (workers < 1) workers = 1; @@ -1005,11 +973,11 @@ static int corRestWorkerPoolStart(int workers) // // corRestWorkerPoolStop - stop new suspensions, drain the queue, join workers // -// Clearing corRestWorkersRun first makes mhdConnectionHandler process inline +// Clearing corRestWorkersRun first makes the backend's handler process inline // again, so no NEW connection is suspended; the workers then drain whatever is // already queued (processing + resuming each) before exiting. After this -// returns there are no suspended connections, so MHD_stop_daemon is safe -// (suspending across MHD_stop_daemon is an API violation). +// returns there are no suspended connections, so stopping the backend is safe +// (resuming across MHD_stop_daemon is an API violation). // void corRestWorkerPoolStop(void) { @@ -1033,170 +1001,101 @@ void corRestWorkerPoolStop(void) // ----------------------------------------------------------------------------- // -// mhdConnectionHandler - MHD callback, called for each incoming request +// corRestAsyncPoolUp / corRestAsyncEnqueue - hand this request to a worker // -// MHD calls this multiple times per request: -// 1. First call: *con_cls == NULL -> init corRest state -// 2. Middle calls: upload_data_size > 0 -> accumulate payload -// 3. Final call: upload_data_size == 0 -> parse, dispatch, render, respond +// Two calls, with the connection suspended between them - see corRestBackend.h. // -static enum MHD_Result mhdConnectionHandler -( - void* cls, - struct MHD_Connection* connection, - const char* url, - const char* method, - const char* version, - const char* uploadData, - size_t* uploadDataSize, - void** con_cls -) +bool corRestAsyncPoolUp(void) { - // --- First call: allocate this connection's per-request state --- - if (*con_cls == NULL) - { - KT_V("Request: %s %s", method, url); // one line per request (verbose mode, -v) - - // Each connection owns its CorRestState (hung on con_cls), so when the - // epoll pool thread interleaves connection B's callbacks between - // connection A's body-read callbacks it can no longer clobber A's state. - // corRestP is bound to it before corRestStateInit (whose memset/init runs - // through the corRest macro). - CorRestState* conP = (CorRestState*) malloc(sizeof(CorRestState)); - if (conP == NULL) - return MHD_NO; - *con_cls = conP; - corRestP = conP; - - corRestStateInit(connection, url, method); - - // Create this connection's application state (e.g. per-conn corNgsild), - // stored in corRest.userData and freed in mhdRequestCompleted. - if (corRestUserDataAllocHookF != NULL) - corRest.userData = corRestUserDataAllocHookF(); - - // Capture request start time — REALTIME for timestamps, MONOTONIC for duration metrics - struct timespec ts; - clock_gettime(CLOCK_REALTIME, &ts); - corRest.requestStartTime = (uint64_t) ts.tv_sec * 1000000000ULL + (uint64_t) ts.tv_nsec; - - struct timespec tsM; - clock_gettime(CLOCK_MONOTONIC, &tsM); - corRest.requestStartTimeMono = (uint64_t) tsM.tv_sec * 1000000000ULL + (uint64_t) tsM.tv_nsec; - - // Collect request headers from MHD - MHD_get_connection_values(connection, MHD_HEADER_KIND, mhdHeaderIterator, NULL); - - // Collect URI query parameters from MHD (already percent-decoded by MHD) - MHD_get_connection_values(connection, MHD_GET_ARGUMENT_KIND, mhdUriParamIterator, NULL); - - // § 6.3.4 — POST / PATCH / PUT to NGSI-LD endpoints must carry - // Content-Length. § 6.3.2 — 413 when the announced body exceeds - // the broker cap. Non-NGSI-LD endpoints (e.g. /admin/*) are - // outside the spec's scope and accept bodyless POSTs. - if ((corRest.in.verb == CorVerbPost || - corRest.in.verb == CorVerbPut || - corRest.in.verb == CorVerbPatch) && - url != NULL && - strncmp(url, "/ngsi-ld/", 9) == 0) - { - const char* clHdr = MHD_lookup_connection_value(connection, MHD_HEADER_KIND, "Content-Length"); - if (clHdr == NULL) - { - // § 6.3.4 — just a 411 status, no body. - corRest.out.httpStatusCode = 411; - corRest.in.contentLengthMissing = true; - } - else if (corRestMaxRequestSize > 0) - { - unsigned long long cl = strtoull(clHdr, NULL, 10); - if (cl > corRestMaxRequestSize) - { - corRestProblem(413, COR_REST_ERROR_REQUEST_LENGTH, "Request Entity Too Large", - "request body of %llu bytes exceeds broker limit of %llu bytes", - cl, corRestMaxRequestSize); - } - } - } + return corRestWorkersRun; +} - return MHD_YES; - } +void corRestAsyncEnqueue(CorRestState* stateP) +{ + corRestWorkerEnqueue(stateP); +} - // Subsequent calls (body chunks, final dispatch): rebind corRestP to THIS - // connection's state — another connection's callback may have re-pointed - // corRestP on this pool thread since our last invocation here. - corRestP = (CorRestState*) *con_cls; - // --- Middle calls: accumulate payload --- - if (*uploadDataSize > 0) - { - // If the first-call check flagged 411/413, drop the bytes. - if (corRest.in.contentLengthMissing || corRest.out.httpStatusCode == 413) - { - *uploadDataSize = 0; - return MHD_YES; - } - // Streaming size cap — defends against clients that lie in - // Content-Length or use chunked encoding without a length. - if (corRestMaxRequestSize > 0 && - (unsigned long long)(corRest.in.payloadSize + *uploadDataSize) > corRestMaxRequestSize) - { - corRestProblem(413, COR_REST_ERROR_REQUEST_LENGTH, "Request Entity Too Large", - "request body exceeds broker limit of %llu bytes", corRestMaxRequestSize); - *uploadDataSize = 0; - return MHD_YES; - } +// ----------------------------------------------------------------------------- +// +// corRestAsyncFinish - the post-response phase, off the I/O thread +// +bool corRestAsyncFinish(CorRestState* stateP) +{ + if (corRestWorkersRun == false) + return false; - int needed = corRest.in.payloadSize + *uploadDataSize + 1; + stateP->asyncFinishing = true; + corRestWorkerEnqueue(stateP); - if (needed > corRest.payloadBufSize) - { - int newSize = (needed + 4096) & ~4095; - char* newBuf = (char*) realloc(corRest.in.payload, newSize); - if (newBuf == NULL) - return MHD_NO; - corRest.in.payload = newBuf; - corRest.payloadBufSize = newSize; - } + return true; +} - memcpy(corRest.in.payload + corRest.in.payloadSize, uploadData, *uploadDataSize); - corRest.in.payloadSize += *uploadDataSize; - corRest.in.payload[corRest.in.payloadSize] = 0; - *uploadDataSize = 0; - return MHD_YES; - } - // --- Final call: process (off the I/O thread when the pool is up), respond --- - if (corRestWorkersRun && !corRest.asyncProcessed) +// ----------------------------------------------------------------------------- +// +// corRestBodyPolicyCheck - +// +// § 6.3.4 — POST / PATCH / PUT to NGSI-LD endpoints must carry Content-Length +// (411 Length Required, and NO payload). § 6.3.2 — 413 when the announced body +// exceeds the broker cap. Non-NGSI-LD endpoints (e.g. /admin/*) are outside the +// spec's scope and accept bodyless POSTs. +// +void corRestBodyPolicyCheck(const char* url, const char* clHeader) +{ + if ((corRest.in.verb != CorVerbPost) && + (corRest.in.verb != CorVerbPut) && + (corRest.in.verb != CorVerbPatch)) + return; + + if ((url == NULL) || (strncmp(url, "/ngsi-ld/", 9) != 0)) + return; + + if (clHeader == NULL) { - // Suspend this connection and hand it to a worker so DB/distop latency - // doesn't block this epoll thread. The worker runs corRestProcessRequest and - // resumes us; MHD then re-invokes this handler with asyncProcessed set and - // we fall through to build + send the response. Suspend BEFORE enqueue so a - // worker can never resume a not-yet-suspended connection. - MHD_suspend_connection(connection); - corRestWorkerEnqueue(corRestP); - return MHD_YES; + // § 6.3.4 — just a 411 status, no body. + corRest.out.httpStatusCode = 411; + corRest.in.contentLengthMissing = true; + return; } - if (!corRest.asyncProcessed) - corRestProcessRequest(); // pool down (shutdown / tests): run inline here + if (corRestMaxRequestSize > 0) + { + unsigned long long cl = strtoull(clHeader, NULL, 10); - char* responseBody = (corRest.out.payload != NULL) ? corRest.out.payload : (char*) ""; - int responseBodySize = corRest.out.payloadSize; + if (cl > corRestMaxRequestSize) + corRestProblem(413, COR_REST_ERROR_REQUEST_LENGTH, "Request Entity Too Large", + "request body of %llu bytes exceeds broker limit of %llu bytes", + cl, corRestMaxRequestSize); + } +} - // Send HTTP response - // For HEAD: pass the full body — MHD will set Content-Length correctly - // but suppress the body in the actual response. - struct MHD_Response* response; - response = MHD_create_response_from_buffer( - responseBodySize, - (void*) responseBody, - MHD_RESPMEM_MUST_COPY - ); + +// ----------------------------------------------------------------------------- +// +// corRestResponseHeaderVBuild - +// +// One implementation of the response-header policy for both backends. The SET +// and the ORDER are pinned by several hundred functional tests that compare +// captured responses line by line, so this is a specification rather than a +// preference - and two copies of it would be two chances to drift. +// +int corRestResponseHeaderVBuild(CorRestKeyValue* hv, int max) +{ + int n = 0; + + #define HDR_ADD(k, v) \ + do { \ + if (n < max) \ + { \ + hv[n].key = (char*) (k); \ + hv[n].value = (char*) (v); \ + n++; \ + } \ + } while (0) // Content-Type policy: // - No body (size 0): omit Content-Type — it describes a body there is none @@ -1208,14 +1107,15 @@ static enum MHD_Result mhdConnectionHandler // TS 104-176 specifies bare media types throughout (§ 6.2.3, § 6.3.3, // § 6.4.7.2 "exactly equal to the media type"); RFC 8259 defines no charset // parameter for application/json — so emit the type verbatim, no charset. - if (responseBodySize > 0) + if (corRest.out.payloadSize > 0) { int code = corRest.out.httpStatusCode; const char* ct = (code == 201 || (code >= 400 && code <= 599)) ? "application/json" : corRest.out.contentType; + if (ct != NULL) - MHD_add_response_header(response, "Content-Type", ct); + HDR_ADD("Content-Type", ct); } // § 6.3.6 Prefer / Preference-Applied: when the client sent a @@ -1230,20 +1130,19 @@ static enum MHD_Result mhdConnectionHandler corRest.in.httpHeaderV[hi].value != NULL && strncasecmp(corRest.in.httpHeaderV[hi].value, "ngsi-ld=", 8) == 0) { - MHD_add_response_header(response, "Preference-Applied", "ngsi-ld=1.9.1"); + HDR_ADD("Preference-Applied", "ngsi-ld=1.9.1"); break; } } - // Add custom response headers + // The service routine's own headers, in the order it added them for (int i = 0; i < corRest.out.headerCount; i++) - MHD_add_response_header(response, corRest.out.headerV[i].key, corRest.out.headerV[i].value); + HDR_ADD(corRest.out.headerV[i].key, corRest.out.headerV[i].value); // CORS headers (if configured) - extern CorRestCorsConfig corRestCors; if (corRestCors.allowOrigin != NULL) { - MHD_add_response_header(response, "Access-Control-Allow-Origin", corRestCors.allowOrigin); + HDR_ADD("Access-Control-Allow-Origin", corRestCors.allowOrigin); if (corRest.in.verb == CorVerbOptions) { @@ -1252,71 +1151,38 @@ static enum MHD_Result mhdConnectionHandler { if (strcmp(corRest.out.headerV[i].key, "Allow") == 0) { - MHD_add_response_header(response, "Access-Control-Allow-Methods", corRest.out.headerV[i].value); + HDR_ADD("Access-Control-Allow-Methods", corRest.out.headerV[i].value); break; } } - const char* ah = corRestCors.allowHeaders ? corRestCors.allowHeaders : "Content-Type, Accept, Link"; - MHD_add_response_header(response, "Access-Control-Allow-Headers", ah); + HDR_ADD("Access-Control-Allow-Headers", + corRestCors.allowHeaders ? corRestCors.allowHeaders : "Content-Type, Accept, Link"); if (corRestCors.maxAge > 0) { - char maxAgeBuf[16]; - snprintf(maxAgeBuf, sizeof(maxAgeBuf), "%d", corRestCors.maxAge); - MHD_add_response_header(response, "Access-Control-Max-Age", maxAgeBuf); + // + // Rendered into the request arena rather than a local buffer: the + // backend sends these AFTER this function has returned, and one of them + // does not copy what it is given. + // + char* maxAgeBuf = (char*) kaAlloc(&corRest.kalloc, 16); + + if (maxAgeBuf != NULL) + { + snprintf(maxAgeBuf, 16, "%d", corRestCors.maxAge); + HDR_ADD("Access-Control-Max-Age", maxAgeBuf); + } } } if (corRestCors.exposeHeaders != NULL) - MHD_add_response_header(response, "Access-Control-Expose-Headers", corRestCors.exposeHeaders); + HDR_ADD("Access-Control-Expose-Headers", corRestCors.exposeHeaders); } - enum MHD_Result ret = MHD_queue_response(connection, corRest.out.httpStatusCode, response); - MHD_destroy_response(response); + #undef HDR_ADD - return ret; -} - - - -// ----------------------------------------------------------------------------- -// -// mhdRequestCompleted - MHD callback when a request is fully handled -// -static void mhdRequestCompleted -( - void* cls, - struct MHD_Connection* connection, - void** con_cls, - enum MHD_RequestTerminationCode toe -) -{ - if (*con_cls != NULL) - { - // Bind to this connection's state before the hook / release touch corRest. - CorRestState* conP = (CorRestState*) *con_cls; - corRestP = conP; - - // Run post-response hook BEFORE releasing the per-request arena so the - // hook can still touch arena-allocated data (e.g. deferred notification - // dispatch reading the entity tree built by the service routine). - corRestPostResponseHook(); - - // Free payload buffer (malloc'd during accumulation, not in kalloc) - free(corRest.in.payload); - corRest.in.payload = NULL; - - corRestStateRelease(); - - // Destroy this connection's application state (per-conn corNgsild, ...). - if (corRestUserDataFreeHookF != NULL && corRest.userData != NULL) - corRestUserDataFreeHookF(corRest.userData); - - free(conP); - *con_cls = NULL; - corRestP = NULL; // no dangling pointer to freed state on this thread - } + return n; } @@ -1458,58 +1324,12 @@ int corRestInit(CorRestServiceSimplified serviceV[], int services, unsigned shor } } - // Start MHD daemon with thread pool. MHD_ALLOW_SUSPEND_RESUME (which bundles - // MHD_USE_ITC) lets the I/O threads suspend a connection and hand it to the - // async worker pool below; the worker resumes it once the response is built. // - // When HTTPS server credentials have been set (a TLS test receiver, not the - // broker), MHD_USE_TLS is added together with the in-memory key/cert. - unsigned int flags = MHD_USE_SELECT_INTERNALLY | MHD_USE_EPOLL | MHD_ALLOW_SUSPEND_RESUME; - - if ((httpsServerKey != NULL) && (httpsServerCert != NULL)) - corRestDaemon = MHD_start_daemon( - flags | MHD_USE_TLS, - port, - NULL, - NULL, - mhdConnectionHandler, - NULL, - MHD_OPTION_NOTIFY_COMPLETED, - mhdRequestCompleted, - NULL, - MHD_OPTION_THREAD_POOL_SIZE, - (unsigned int) poolSize, - MHD_OPTION_CONNECTION_TIMEOUT, - (unsigned int) 30, - MHD_OPTION_HTTPS_MEM_KEY, - httpsServerKey, - MHD_OPTION_HTTPS_MEM_CERT, - httpsServerCert, - MHD_OPTION_END - ); - else - corRestDaemon = MHD_start_daemon( - flags, - port, - NULL, - NULL, - mhdConnectionHandler, - NULL, - MHD_OPTION_NOTIFY_COMPLETED, - mhdRequestCompleted, - NULL, - MHD_OPTION_THREAD_POOL_SIZE, - (unsigned int) poolSize, - MHD_OPTION_CONNECTION_TIMEOUT, - (unsigned int) 30, - MHD_OPTION_END - ); - - if (corRestDaemon == NULL) - { - fprintf(stderr, "corRestInit: MHD_start_daemon failed on port %d\n", port); + // The HTTP server itself - libmicrohttpd or the built-in one, decided at + // compile time by COR_HTTP_SERVER. It starts its own threads and returns. + // + if (corRestBackendStart(port, poolSize, httpsServerKey, httpsServerCert) != 0) return -1; - } // Async worker pool — one worker per I/O thread. corRestWorkersRun stays false // (handler processes inline) if the pool fails to start. diff --git a/corRestInit.h b/corRestInit.h index 9cb30b6..4760d63 100644 --- a/corRestInit.h +++ b/corRestInit.h @@ -32,13 +32,15 @@ // // corRestInit - // -// Initialize the REST library: prepare service lookup tables and start MHD. +// Initialize the REST library: prepare service lookup tables, start the HTTP +// server (libmicrohttpd or the built-in one - COR_HTTP_SERVER, decided at +// compile time) and the worker pool that runs requests off its I/O threads. // // Parameters: // serviceV - flat array of service definitions (each includes its verb) // services - number of entries in serviceV // port - TCP port to listen on -// poolSize - MHD thread pool size +// poolSize - I/O threads, and one dispatch worker each // // GET services automatically get HEAD support (same handler, body suppressed). // OPTIONS is auto-generated for all registered URL paths. @@ -53,9 +55,13 @@ extern int corRestInit(CorRestServiceSimplified serviceV[], int services, unsign // corRestHttpsServerCredentialsSet - serve HTTPS instead of HTTP // // Call before corRestInit with the PEM contents (not file paths) of the private -// key and certificate. The MHD daemon is then started with TLS enabled. NULL -// (the default) keeps the server on plain HTTP. Used by TLS test receivers -// (e.g. ftClient), not by the broker. +// key and certificate; the server is then started with TLS enabled. NULL (the +// default) keeps it on plain HTTP. Used by TLS test receivers (e.g. ftClient), +// not by the broker. +// +// ⚠️ The BUILT-IN server has no TLS, and corRestInit REFUSES to start rather +// than quietly serving the port in the clear. Build with COR_HTTP_SERVER=mhd +// for an HTTPS listener. // extern void corRestHttpsServerCredentialsSet(char* keyPem, char* certPem); @@ -66,8 +72,8 @@ extern void corRestHttpsServerCredentialsSet(char* keyPem, char* certPem); // corRestProcessRequest - run the request-dispatch core on the bound corRest state // // Consumes corRest.in, produces corRest.out (incl. the rendered body in -// corRest.out.payload / payloadSize). Connection-free — the MHD send lives in the -// connection handler. +// corRest.out.payload / payloadSize). Connection-free — the send lives in the +// HTTP backend's connection handler. // extern void corRestProcessRequest(void); diff --git a/corRestStateInit.c b/corRestStateInit.c index 87496fc..c863127 100644 --- a/corRestStateInit.c +++ b/corRestStateInit.c @@ -42,11 +42,11 @@ extern int corRestDefaultPrettySpaces; // // corRestStateInit - // -void corRestStateInit(struct MHD_Connection* connection, const char* url, const char* method) +void corRestStateInit(void* connection, const char* url, const char* method) { memset(&corRest, 0, sizeof(CorRestState)); - corRest.mhdConnection = connection; + corRest.connection = connection; // Initialize kalloc pool (inline buffer first, then malloc-based overflow). // 256KB grow chunks: keeps single-shot renderings of ~1000-entity responses @@ -72,17 +72,7 @@ void corRestStateInit(struct MHD_Connection* connection, const char* url, const corRest.in.urlParams = qMark + 1; } - corRest.in.urlPathLen = strlen(corRest.in.urlPath); - - // Strip a single trailing '/' from the path (except the root "/" itself). - // Routes are registered without a trailing slash; clients that send one - // (e.g. ETSI test suite: POST /ngsi-ld/v1/entities/) would otherwise - // 404 against an exact-match service lookup. - if (corRest.in.urlPathLen > 1 && corRest.in.urlPath[corRest.in.urlPathLen - 1] == '/') - { - corRest.in.urlPath[corRest.in.urlPathLen - 1] = 0; - corRest.in.urlPathLen--; - } + corRestUrlPathNormalize(); // Initialize URI param dynamic array (starts with inline slots) corRest.in.uriParamV = corRest.in.uriParams; @@ -113,6 +103,31 @@ void corRestStateInit(struct MHD_Connection* connection, const char* url, const +// ----------------------------------------------------------------------------- +// +// corRestUrlPathNormalize - +// +// Strip a single trailing '/' from the path (except the root "/" itself). +// Routes are registered without a trailing slash; clients that send one +// (e.g. ETSI test suite: POST /ngsi-ld/v1/entities/) would otherwise +// 404 against an exact-match service lookup. +// +void corRestUrlPathNormalize(void) +{ + if (corRest.in.urlPath == NULL) + return; + + corRest.in.urlPathLen = strlen(corRest.in.urlPath); + + if (corRest.in.urlPathLen > 1 && corRest.in.urlPath[corRest.in.urlPathLen - 1] == '/') + { + corRest.in.urlPath[corRest.in.urlPathLen - 1] = 0; + corRest.in.urlPathLen--; + } +} + + + // ----------------------------------------------------------------------------- // // corRestStateRelease - release per-request resources diff --git a/corRestStateInit.h b/corRestStateInit.h index 515794e..3eef8d3 100644 --- a/corRestStateInit.h +++ b/corRestStateInit.h @@ -9,15 +9,30 @@ #ifndef CORREST_STATE_INIT_H_ #define CORREST_STATE_INIT_H_ -#include - // ----------------------------------------------------------------------------- // // corRestStateInit - initialize the thread-local corRest for a new request // -extern void corRestStateInit(struct MHD_Connection* connection, const char* url, const char* method); +// 'connection' is the HTTP backend's own handle for the connection, stored +// untouched in corRest.connection - see CorRestState.h. NULL for a request that +// arrived on no connection at all (an in-process self-forward). +// +extern void corRestStateInit(void* connection, const char* url, const char* method); + + + +// ----------------------------------------------------------------------------- +// +// corRestUrlPathNormalize - length, and a single trailing '/' removed +// +// Called by corRestStateInit, and AGAIN by a backend that has to percent-decode +// the path itself: decoding changes the length and can uncover the trailing +// slash that was written `%2F`. Idempotent, which is what lets it be called +// twice rather than duplicated. +// +extern void corRestUrlPathNormalize(void); diff --git a/corRestStop.c b/corRestStop.c index edfb568..3718942 100644 --- a/corRestStop.c +++ b/corRestStop.c @@ -7,10 +7,10 @@ // SPDX-License-Identifier: Apache-2.0 // #include // free -#include #include "corRest/CorRestVerb.h" // CorVerbs #include "corRest/CorRestService.h" // CorRestServiceVector +#include "corRest/corRestBackend.h" // corRestBackendStop, corRestWorkerPoolStop @@ -19,8 +19,6 @@ // Globals (defined in corRestInit.c) // extern CorRestServiceVector corRestServiceV[]; -extern struct MHD_Daemon* corRestDaemon; -extern void corRestWorkerPoolStop(void); @@ -30,16 +28,13 @@ extern void corRestWorkerPoolStop(void); // void corRestStop(void) { - // Drain + join the async worker pool BEFORE stopping the daemon: this clears + // Drain + join the async worker pool BEFORE stopping the server: this clears // any suspended connections (resuming across MHD_stop_daemon is an API - // violation) and stops new suspensions. + // violation, and the built-in server would be tearing down a connection a + // worker still holds) and stops new suspensions. corRestWorkerPoolStop(); - if (corRestDaemon != NULL) - { - MHD_stop_daemon(corRestDaemon); - corRestDaemon = NULL; - } + corRestBackendStop(); for (int verb = 0; verb < CorVerbs; verb++) { diff --git a/makefile b/makefile index f9d8eae..3d43301 100644 --- a/makefile +++ b/makefile @@ -51,24 +51,34 @@ DFLAGS = # In CFLAGS and not DFLAGS on purpose - see the note above: a caller who passes # DFLAGS on the command line would drop these along with every other default. # +# The value picks the backend SOURCE FILE as well as the defines: the two +# corRestBackend*.c files are alternatives, and compiling the unused one would +# need the library it exists to avoid. +# COR_HTTP_SERVER ?= mhd ifeq ($(COR_HTTP_SERVER),mhd) - HTTP_SERVER_FLAGS = -DCOR_HTTP_SERVER_MHD=1 -DCOR_HTTP_SERVER_BUILTIN=0 + HTTP_SERVER_FLAGS = -DCOR_HTTP_SERVER_MHD=1 -DCOR_HTTP_SERVER_BUILTIN=0 + HTTP_SERVER_SOURCE = corRestBackendMhd.c + HTTP_SERVER_LIBS = -lmicrohttpd + HTTP_SERVER_ARCHIVE = else ifeq ($(COR_HTTP_SERVER),builtin) + HTTP_SERVER_FLAGS = -DCOR_HTTP_SERVER_MHD=0 -DCOR_HTTP_SERVER_BUILTIN=1 + HTTP_SERVER_SOURCE = corRestBackendBuiltin.c + HTTP_SERVER_LIBS = # - # Refused rather than built: the flags below are right, and the server backend - # they select does not exist in this repo yet, so a build would fail at link - # with a pile of missing MHD symbols and no clue why. The switch says what it - # is waiting for instead. + # corHttp is a sibling repo and a static archive, like every other lib here. + # It is NOT folded into libcorRest.a - an archive cannot contain another one - + # so a consumer linking the built-in flavour links both, which is what + # coraine's CMakeLists does. # - $(error COR_HTTP_SERVER=builtin: the built-in HTTP server is not wired into this library yet - use 'mhd') - HTTP_SERVER_FLAGS = -DCOR_HTTP_SERVER_MHD=0 -DCOR_HTTP_SERVER_BUILTIN=1 + HTTP_SERVER_ARCHIVE = ../corHttp/libcorHttp.a else $(error COR_HTTP_SERVER must be 'mhd' or 'builtin', not '$(COR_HTTP_SERVER)') endif CFLAGS = -O2 -Wall -Werror -Wundef -fPIC -Wno-unused-function -fstack-protector-all $(DFLAGS) $(HTTP_SERVER_FLAGS) $(INCLUDE) -MMD -MP $(EXTRA_CFLAGS) LIB_SOURCES = corRestInit.c \ + $(HTTP_SERVER_SOURCE) \ corMimeType.c \ corRestStop.c \ corRestStateInit.c \ @@ -126,10 +136,10 @@ TEST_SOURCES = corRestTest.c TEST_OBJS = $(addprefix $(OBJDIR)/,$(TEST_SOURCES:.c=.o)) SO_LDFLAGS = -L../kalloc -L../kjson -L../kbase -L../klog -L../ktrace -SO_LIBS = -lkalloc -lkjson -lklog -lktrace -lkbase -lmicrohttpd -lssl -lcrypto -lpthread +SO_LIBS = -lkalloc -lkjson -lklog -lktrace -lkbase $(HTTP_SERVER_ARCHIVE) $(HTTP_SERVER_LIBS) -lssl -lcrypto -lpthread SO_RPATH = -Wl,-rpath,'$$ORIGIN/../kalloc:$$ORIGIN/../kjson:$$ORIGIN/../kbase:$$ORIGIN/../klog:$$ORIGIN/../ktrace' -LIBS = ../kalloc/libkalloc.a ../kjson/libkjson.a ../klog/libklog.a ../ktrace/libktrace.a ../kbase/libkbase.a -lmicrohttpd -lssl -lcrypto -lpthread -lm +LIBS = ../kalloc/libkalloc.a ../kjson/libkjson.a ../klog/libklog.a ../ktrace/libktrace.a ../kbase/libkbase.a $(HTTP_SERVER_ARCHIVE) $(HTTP_SERVER_LIBS) -lssl -lcrypto -lpthread -lm # # Built per flavour, then STAGED to the repo root where every consumer expects @@ -182,7 +192,14 @@ $(LIB): $(OBJDIR)/$(LIB) $(LIB_SO): $(OBJDIR)/$(LIB_SO) @cp -f $< $@ +# +# Removed and rebuilt, never updated in place. `ar r` REPLACES and ADDS but +# never removes, so an archive built once with corRestBackendMhd.o keeps it +# after a switch to the built-in backend - and the link then has two definitions +# of every corRestBackend* function, one of which wants libmicrohttpd. +# $(OBJDIR)/$(LIB): $(LIB_OBJS) + @rm -f $@ ar r $@ $(LIB_OBJS) ranlib $@