From b37057b2b4aed780dbbaf7b4fd7bfd86342a7365 Mon Sep 17 00:00:00 2001 From: kzangeli Date: Sat, 5 Sep 2026 21:45:00 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20two=20HTTP=20servers=20behind=20one=20s?= =?UTF-8?q?eam=20=E2=80=94=20the=20built-in=20backend?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit COR_HTTP_SERVER=builtin now builds, links and serves. libmicrohttpd leaves the process entirely: `ldd coraine` names it under `mhd` and does not under `builtin`, which is the check that the switch took. full build (mhd) 641 tests, 641 passed built-in 640 tests, 640 passed (- the one HTTPS receiver) ETSI, built-in 1046 TPs, 1046 passed THE SPLIT. corRestInit.c was 1523 lines with ~68 of MHD scattered through it. It is now the shared half - the service table, the dispatch, the worker pool, the response policy - and corRestBackend{Mhd,Builtin}.c are the two halves that know a socket. Exactly one is compiled; the makefile picks the SOURCE FILE as well as the -D pair, because compiling the unused one would need the library it exists to avoid. corRestBackend.h is the contract, and it is internal: nothing above corRest includes it. WHAT MOVED INTO SHARED CODE RATHER THAN BEING WRITTEN TWICE. The response headers - content type, Preference-Applied, the service routine's own, CORS - are built ONCE by corRestResponseHeaderVBuild, in the order they go out, because several hundred functional tests compare captured responses line by line and two copies of that policy would be two chances to drift. Same for the § 6.3.4 / § 6.3.2 body checks (corRestBodyPolicyCheck) and the query parser. WHAT THE BUILT-IN BACKEND HAS TO DO THAT MHD DID FOR US: PERCENT-DECODING - MHD hands over a decoded path and decoded query values and corHttp decodes nothing, so the path is decoded here (after the query is split off, so a `%3F` in a path cannot masquerade as the delimiter) and the query goes through corRestUriParamsParse, the same split-and-decode the in-process self-forward already used. INCLUDING '+' MEANS SPACE, which RFC 3986 does not ask for and every HTTP client library produces anyway: Python requests, and so the ETSI suite, puts `q=name=="Eiffel Tower"` on the wire as `q=name%3D%3D%22Eiffel+Tower%22`. Read literally that matches nothing, with a 200 and an empty array to show for it - six ETSI TPs, found exactly that way. THE BODY LIMIT - MHD streams and can be told to stop mid-body; corHttp delivers a whole request or none. Its cap is set from corRest's own threshold and over it corHttp hands the request up WITHOUT its body, which is precisely what corRestBodyPolicyCheck wants: it answers from the Content-Length header, so the ProblemDetails is identical either way. COPIES - corHttp is zero-copy into the connection's read buffer, and the post-response phase runs after the connection has moved on. Method, headers and body are copied into the request arena, which is what lets the request state outlive the connection it arrived on. MHD needs none of this; it owns copies of its own until NOTIFY_COMPLETED. ⭐ THE POST-RESPONSE PHASE IS NOW A WORKER PHASE (corRestAsyncFinish). It was running on the I/O thread, which is fine on MHD's pool of them and is a DEADLOCK on a single event loop: dispatching a notification compacts it with the subscription's @context, that @context can be one THIS BROKER HOSTS, and the loop thread then waits for an answer only the loop thread can produce. It does not hang forever, which is worse - the download times out, the notification goes out TEN SECONDS LATE with an uncompacted body, and nothing in the log says why. Two functests caught it. The MHD backend keeps running it inline, unchanged. Also: corRestStateInit takes a void* connection and CorRestState.h no longer includes microhttpd.h - it is included by ~2000 call sites in the layers above, and naming one server's type in it made every one of them depend on that server being the one in the build. The archive is removed before it is rebuilt, or switching flavours leaves both backends' objects in it and the link finds two definitions of everything. TLS: the built-in server has none, and corRestInit REFUSES to start rather than serving the port in the clear. Only ftClient asks for it (the broker never does), so one functional test carries REQUIRE_HTTPSERVER: mhd. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01TGatXwrHx1CreL49sCuS37 --- CorRestState.h | 31 ++- corRestBackend.h | 166 +++++++++++++ corRestBackendBuiltin.c | 455 ++++++++++++++++++++++++++++++++++++ corRestBackendMhd.c | 400 +++++++++++++++++++++++++++++++ corRestInit.c | 506 +++++++++++++--------------------------- corRestInit.h | 20 +- corRestStateInit.c | 41 ++-- corRestStateInit.h | 21 +- corRestStop.c | 15 +- makefile | 35 ++- 10 files changed, 1301 insertions(+), 389 deletions(-) create mode 100644 corRestBackend.h create mode 100644 corRestBackendBuiltin.c create mode 100644 corRestBackendMhd.c diff --git a/CorRestState.h b/CorRestState.h index f90f7a4..6344e81 100644 --- a/CorRestState.h +++ b/CorRestState.h @@ -9,8 +9,8 @@ #ifndef CORREST_STATE_H_ #define CORREST_STATE_H_ -#include #include +#include // NULL - came in via microhttpd.h until this header stopped including it #include "kalloc/KAlloc.h" #include "kjson/kjson.h" @@ -30,7 +30,15 @@ // typedef struct CorRestState { - struct MHD_Connection* mhdConnection; + // + // The HTTP backend's handle for the connection this request came in on - + // an `struct MHD_Connection*` under libmicrohttpd, a `CorHttpConn*` under the + // built-in server. Opaque HERE on purpose: this header is included by ~2000 + // call sites in the layers above, and naming one server's type in it would + // make every one of them depend on that server being the one in the build. + // The backend that put it here is the only code that casts it back. + // + void* connection; // Allocator: pool-based, bulk-free after request completes KAlloc kalloc; @@ -49,7 +57,7 @@ typedef struct CorRestState // Matched service CorRestService* serviceP; - // Payload accumulation (during MHD body reads) + // Payload accumulation (during the backend's body reads) int payloadBufSize; // Request timing @@ -63,6 +71,21 @@ typedef struct CorRestState // this state; a worker runs corRestProcessRequest off the I/O thread, sets // asyncProcessed, and resumes the connection. asyncNext links the FIFO queue. bool asyncProcessed; + + // + // ...and the SECOND thing a worker does for a request: the post-response + // phase - the deferred notifications, and releasing the arena they are built + // in - once the response is on the wire. + // + // A phase and not a second queue, because it is the same work item at a later + // moment. It is off the I/O thread for the same reason the dispatch is, and + // one reason more: a notification's @context can be one the broker HOSTS + // ITSELF, so this phase can issue a request the broker has to answer. On a + // single-threaded event loop, running it there is a deadlock that resolves + // itself as a timeout - a notification sent ten seconds late with an + // uncompacted body, and nothing in the log saying why. + // + bool asyncFinishing; struct CorRestState* asyncNext; } CorRestState; @@ -74,7 +97,7 @@ typedef struct CorRestState // // Historically `corRest` was a plain __thread object. It is now a __thread // POINTER (corRestP) behind the `corRest` macro, so the state can later be -// relocated off the thread (into MHD per-connection con_cls) without touching +// relocated off the thread (into the backend's per-connection slot) without touching // the ~2000 `corRest.foo` call sites. Until a request handler binds corRestP to // a connection's state, corRestBind() auto-binds it to a per-thread fallback // object — making every access crash-proof and, for now, behaviourally diff --git a/corRestBackend.h b/corRestBackend.h new file mode 100644 index 0000000..0ec1954 --- /dev/null +++ b/corRestBackend.h @@ -0,0 +1,166 @@ +// +// FILE corRestBackend.h +// +// AUTHOR Ken Zangelin +// +// Copyright 2026 Seamware +// SPDX-License-Identifier: Apache-2.0 +// +// The seam between corRest and the HTTP server underneath it. +// +// There are two servers - libmicrohttpd and the built-in epoll one in corHttp - +// and exactly one of them is in a build (COR_HTTP_SERVER). Everything on the +// corRest side of this header is the same either way: the service table, the +// dispatch, the worker pool, the response policy. Everything on the other side +// is a socket and a callback shape. +// +// Not a public header: it is the INTERNAL contract between corRestInit.c and +// corRestBackend.c, and nothing above corRest includes it. +// +#ifndef CORREST_BACKEND_H_ +#define CORREST_BACKEND_H_ + +#include // bool + +#include "corRest/CorRestKeyValue.h" // CorRestKeyValue +#include "corRest/CorRestState.h" // CorRestState + + + +// ----------------------------------------------------------------------------- +// +// What the backend implements +// +// corRestBackendStart - listen on 'port' and serve, and RETURN. +// +// Both backends run their own threads; neither blocks the caller, because +// corRestInit's caller goes on to do other things and then parks. keyPem / +// certPem are the HTTPS server credentials, NULL for plain HTTP - a backend +// without TLS refuses rather than silently serving the port unencrypted. +// +// Returns 0, or -1 with a reason on stderr. +// +extern int corRestBackendStart(unsigned short port, int poolSize, char* keyPem, char* certPem); + + +// +// corRestBackendStop - stop serving; called after the worker pool has drained. +// +extern void corRestBackendStop(void); + + +// +// corRestBackendFinish - the request is over; run its post-response work and +// release it +// +// The backend's, because only it knows what the request state BORROWED and what +// therefore has to be given back. Binds corRestP to stateP, runs the +// post-response hook, releases the arena, frees the state, and unbinds. +// +// Runs on a worker whenever there is a pool (corRestAsyncFinish), on the +// calling thread otherwise. +// +extern void corRestBackendFinish(CorRestState* stateP); + + +// +// corRestBackendResume - a worker has built the response; send it +// +// Called ON THE WORKER THREAD with corRestP still bound to 'stateP', which is +// what lets a backend read corRest.out here. It must not write to the socket +// itself - that belongs to whichever thread owns the connection - and both +// backends therefore only hand the connection back to their event loop. +// +extern void corRestBackendResume(CorRestState* stateP); + + + +// ----------------------------------------------------------------------------- +// +// What corRestInit.c provides to the backend +// +// corRestHttpHeaderAdd - one request header; key/value are BORROWED, so they +// must live as long as the request does. +// corRestUriParamsParse - split + percent-decode corRest.in.urlParams into +// corRest.in.uriParamV. Destroys the buffer it parses. +// corRestUriParamAdd - one already-split, already-decoded parameter. +// +extern void corRestHttpHeaderAdd(const char* key, const char* value); +extern void corRestUriParamsParse(void); +extern void corRestUriParamAdd(char* name, char* value); + + +// +// corRestBodyPolicyCheck - the two answers that are decided before the body +// +// § 6.3.4: POST/PATCH/PUT to an NGSI-LD route must carry Content-Length (411, +// no payload). § 6.3.2: an announced body over the broker's cap is a 413. +// Both are known from the request line and the headers alone, which is why +// they are here and not in the dispatch: a backend that streams the body wants +// to stop reading it, and one that has it already wants to not parse it. +// +// 'clHeader' is the Content-Length header value, NULL when absent. +// +extern void corRestBodyPolicyCheck(const char* url, const char* clHeader); + + +// +// corRestResponseHeaderVBuild - the response headers, in the order they go out +// +// ONE implementation of the header policy - content type, Preference-Applied, +// the service routine's own headers, CORS - for both backends, because the set +// and the ORDER of these headers is compared line by line by several hundred +// functional tests. Two copies of it would be two chances to drift. +// +// Fills 'hv' and returns how many. Keys and values are borrowed from the +// request arena and from static configuration, so they last as long as the +// request state does. +// +extern int corRestResponseHeaderVBuild(CorRestKeyValue* hv, int max); + +#define COR_REST_RESPONSE_HEADERS_MAX 64 + + +// +// corRestAsyncPoolUp / corRestAsyncEnqueue - hand the request to a worker +// +// Two calls and not one, because the connection has to be SUSPENDED BETWEEN +// THEM: a worker can pick the request up and finish it before the enqueue has +// returned, and resuming a connection that was never suspended is an API +// violation in one backend and a lost response in the other. +// +// if (corRestAsyncPoolUp()) { suspend(); corRestAsyncEnqueue(stateP); return; } +// corRestProcessRequest(); // pool down (shutdown, or a server without one) +// +extern bool corRestAsyncPoolUp(void); +extern void corRestAsyncEnqueue(CorRestState* stateP); + + +// +// corRestAsyncFinish - hand the POST-RESPONSE phase to a worker +// +// Returns true when a worker has taken it - the caller must then not touch +// stateP again. False means there is no pool and the caller runs +// corRestBackendFinish itself. +// +// The response has already gone out, so this is not about latency: it is about +// the thread. The phase can issue an HTTP request the broker itself has to +// answer (a notification compacted with an @context the broker hosts), and a +// single-threaded event loop running it would be waiting on itself. +// +extern bool corRestAsyncFinish(CorRestState* stateP); + + +// +// corRestWorkerPoolStart / Stop - one worker per I/O thread +// +extern int corRestWorkerPoolStart(int workers); +extern void corRestWorkerPoolStop(void); + + +// +// corRestProcessRequest - the dispatch itself (public; corRestInit.h) +// +extern void corRestProcessRequest(void); + +#endif // CORREST_BACKEND_H_ diff --git a/corRestBackendBuiltin.c b/corRestBackendBuiltin.c new file mode 100644 index 0000000..88b151a --- /dev/null +++ b/corRestBackendBuiltin.c @@ -0,0 +1,455 @@ +// +// FILE corRestBackendBuiltin.c +// +// AUTHOR Ken Zangelin +// +// Copyright 2026 Seamware +// SPDX-License-Identifier: Apache-2.0 +// +// The corHttp backend - the built-in HTTP server, no libmicrohttpd. +// +// The counterpart of corRestBackendMhd.c, and the same division of labour: this +// file translates one server's shape into corRest's, and decides nothing. What +// a URL means, what a body has to be, which headers go back and in what order +// all live on the other side of corRestBackend.h and are shared with the MHD +// backend byte for byte - which is the whole point, since several hundred +// functional tests compare captured responses line by line and must not be able +// to tell which server produced them. +// +// TWO THINGS MHD DID THAT corHttp DOES NOT, and both are done here: +// +// PERCENT-DECODING. MHD hands over a decoded URL path and decoded query +// parameter values. corHttp decodes nothing - deliberately, because the +// decoding rules differ per parameter in NGSI-LD and the layer that knows +// which is which is this one. So the path is decoded here (after the query +// has been split off, so a `%3F` in a path cannot masquerade as the query +// delimiter) and the query goes through corRestUriParamsParse, the same +// split-and-decode the in-process self-forward path uses. +// +// ...and '+' IN THE QUERY IS A SPACE, which is not what RFC 3986 says and is +// what every HTTP client does. The rule belongs to +// application/x-www-form-urlencoded, but a query string is what that format +// is FOR, so urlencode/quote_plus is what client libraries produce: Python's +// requests - and therefore the ETSI conformance suite - sends +// `q=name=="Eiffel Tower"` on the wire as `q=name%3D%3D%22Eiffel+Tower%22`. +// Read literally, that query asks for a name with a plus in it and matches +// nothing, with a 200 and an empty array to show for it. A value that really +// does contain a plus - the NGSI-LD single-level scope wildcard, +// `scopeQ=/Madrid/%2B/ParqueNorte` - has to be percent-encoded, and then both +// servers agree. +// +// THE BODY LIMIT. MHD delivers a body in chunks and can be told to stop +// mid-stream; corHttp delivers a whole request or none, so "stop reading" has +// to become "do not start". Its cap is set from corRest's § 6.3.2 threshold +// here, and over it corHttp hands the request up WITHOUT its body - which is +// exactly what corRestBodyPolicyCheck needs, since that answers from the +// Content-Length header and not from the bytes. +// +// AND ONE THING THIS BACKEND DOES THAT MHD'S DOES NOT: the request is COPIED +// out of corHttp's read buffer - method, header keys and values, body - rather +// than pointed at. corHttp is zero-copy and the pointers are good for exactly +// as long as the connection stays on this request; the post-response phase runs +// AFTER the response is out and the connection has moved on, on another thread, +// and it reads the request (the tenant, the Via chain) while it builds the +// notifications. Copying is what makes the request state outlive the connection +// it arrived on. MHD needs none of this because MHD owns copies of its own and +// keeps them until NOTIFY_COMPLETED. +// +#include // malloc, free +#include // fprintf +#include // memcpy +#include // pthread_create +#include // clock_gettime + +#include "kalloc/kaAlloc.h" // kaAlloc +#include "kalloc/kaStrdup.h" // kaStrdup +#include "ktrace/kTrace.h" // KT_V + +#include "corHttp/CorHttp.h" // CorHttpServer, CorHttpConn + +#include "corRest/CorRestState.h" // CorRestState, corRest +#include "corRest/corRestStateInit.h" // corRestStateInit, corRestUrlPathNormalize +#include "corRest/corRestHooks.h" // CorRestHook, ... +#include "corRest/corRestUrlValueEncode.h" // corRestUrlValueDecode +#include "corRest/corRestBackend.h" // Own interface + + + +// ----------------------------------------------------------------------------- +// +// The server, and the thread its event loop runs on +// +// corHttpServe() does not return until it is told to stop, and corRestInit's +// caller has other things to do before it parks - so the loop gets a thread of +// its own here, which is also what MHD_USE_SELECT_INTERNALLY does on the other +// side. +// +static CorHttpServer corHttpServer; +static pthread_t corHttpThread; +static bool corHttpRunning = false; + + + +// ----------------------------------------------------------------------------- +// +// Hook globals (defined in corRestHooks.c) +// +extern CorRestUserDataAllocHook corRestUserDataAllocHookF; +extern CorRestUserDataFreeHook corRestUserDataFreeHookF; +extern CorRestHook corRestPostResponseHook; +extern unsigned long long corRestMaxRequestSize; + + + +// ----------------------------------------------------------------------------- +// +// requestResponseFill - move corRest.out onto the connection +// +// Runs on whichever thread built the response - a worker for the async path, +// the loop thread for the inline one - and never touches the socket. Nothing is +// copied: corHttp borrows what it is given, and everything given here lives in +// the request arena, which is released in httpRequestDone, after the bytes are +// out. +// +static void requestResponseFill(CorHttpConn* connP) +{ + CorRestKeyValue headerV[COR_REST_RESPONSE_HEADERS_MAX]; + int headers = corRestResponseHeaderVBuild(headerV, COR_REST_RESPONSE_HEADERS_MAX); + + corHttpResponseStatus(connP, corRest.out.httpStatusCode); + + for (int ix = 0; ix < headers; ix++) + corHttpResponseHeader(connP, headerV[ix].key, headerV[ix].value); + + // + // HEAD gets the length of the body and none of its bytes, and that decision + // is corHttp's (it knows the method) - so the body goes over either way. + // + if ((corRest.out.payload != NULL) && (corRest.out.payloadSize > 0)) + corHttpResponseBody(connP, corRest.out.payload, corRest.out.payloadSize); +} + + + +// ----------------------------------------------------------------------------- +// +// httpRequestCb - one complete request, on the event-loop thread +// +static void httpRequestCb(CorHttpConn* connP) +{ + KT_V("Request: %s %s", connP->method.s, connP->path.s); // one line per request (-v) + + // + // One CorRestState per request, hung on the connection - the same arrangement + // as MHD's con_cls, and for the same reason: the loop may run another + // connection's request between this one's dispatch and its send. + // + CorRestState* stateP = (CorRestState*) malloc(sizeof(CorRestState)); + + if (stateP == NULL) + { + corHttpResponseStatus(connP, 503); + return; + } + + connP->userData = stateP; + corRestP = stateP; + + corRestStateInit(connP, connP->path.s, connP->method.s); + + // The verb string is corHttp's too - copied, like everything else below. + corRest.in.verbString = kaStrdup(&corRest.kalloc, connP->method.s); + + // + // The path arrived percent-ENCODED (MHD would have decoded it before we ever + // saw it). Decoded here, on corRest's own strdup'd copy rather than in the + // read buffer, and AFTER corRestStateInit split off a query that corHttp had + // already separated - so a path containing `%3F` decodes to a '?' inside the + // path, which is what it is, instead of splitting the URL in two. + // + corRestUrlValueDecode(corRest.in.urlPath); + corRestUrlPathNormalize(); + + // Create this connection's application state (e.g. per-conn corNgsild), + // stored in corRest.userData and freed in httpRequestDone. + if (corRestUserDataAllocHookF != NULL) + corRest.userData = corRestUserDataAllocHookF(); + + // Capture request start time — REALTIME for timestamps, MONOTONIC for duration metrics + struct timespec ts; + clock_gettime(CLOCK_REALTIME, &ts); + corRest.requestStartTime = (uint64_t) ts.tv_sec * 1000000000ULL + (uint64_t) ts.tv_nsec; + + struct timespec tsM; + clock_gettime(CLOCK_MONOTONIC, &tsM); + corRest.requestStartTimeMono = (uint64_t) tsM.tv_sec * 1000000000ULL + (uint64_t) tsM.tv_nsec; + + // + // Request headers, COPIED into the request arena - see the note at the top of + // this file. A header that will not fit is dropped rather than borrowed: a + // missing header is a wrong answer, a dangling one is a crash under load + // weeks later. + // + for (int ix = 0; ix < connP->headers; ix++) + { + char* key = kaStrdup(&corRest.kalloc, connP->header[ix].key.s); + char* value = kaStrdup(&corRest.kalloc, connP->header[ix].value.s); + + if ((key != NULL) && (value != NULL)) + corRestHttpHeaderAdd(key, value); + } + + // + // The query string, split and percent-decoded by corRest itself - corHttp + // split it too, but on a copy and without decoding, which is not what the + // layers above expect. Parsed from a copy of our own because the parse is + // destructive and corHttp's raw query is not ours to consume. + // + // corRest.in.urlParams is left NULL afterwards, exactly as it is under MHD: + // it does not survive its own parser (see corRestUriParamsParse), and a + // half-eaten query string is worse than none. + // + if (connP->query.len > 0) + { + char* query = (char*) kaAlloc(&corRest.kalloc, connP->query.len + 1); + + if (query != NULL) + { + memcpy(query, connP->query.s, connP->query.len); + query[connP->query.len] = 0; + + // + // '+' means space - see the note at the top of this file. BEFORE the + // percent-decoding in corRestUriParamsParse and not after, so that a + // `%2B` (a plus the client meant literally) decodes to a plus and stays + // one, instead of being turned into a space by this loop. + // + for (char* p = query; *p != 0; p++) + { + if (*p == '+') + *p = ' '; + } + + corRest.in.urlParams = query; + corRestUriParamsParse(); + corRest.in.urlParams = NULL; + } + } + + corRestBodyPolicyCheck(corRest.in.urlPath, corHttpHeader(connP, "Content-Length")); + + // + // The body, into the request arena. Released with everything else by + // corRestStateRelease, which is why corRestBackendFinish here does not + // free(corRest.in.payload) the way the MHD backend has to - there, the body + // is a malloc'd accumulation buffer. + // + if ((connP->body.s != NULL) && (connP->body.len > 0) && + (corRest.in.contentLengthMissing == false) && (corRest.out.httpStatusCode != 413)) + { + char* body = (char*) kaAlloc(&corRest.kalloc, connP->body.len + 1); + + if (body != NULL) + { + memcpy(body, connP->body.s, connP->body.len); + body[connP->body.len] = 0; + + corRest.in.payload = body; + corRest.in.payloadSize = connP->body.len; + } + } + + if (corRestAsyncPoolUp() == true) + { + // + // Off the event loop: a DB round-trip or a distributed operation would stop + // every other connection for its duration. Suspend BEFORE enqueue - a + // worker can finish before the enqueue call returns. + // + corHttpSuspend(connP); + corRestAsyncEnqueue(stateP); + return; + } + + corRestProcessRequest(); // pool down (shutdown / tests): run inline here + requestResponseFill(connP); +} + + + +// ----------------------------------------------------------------------------- +// +// httpRequestDone - the response is on the wire; the request state can go +// +// corHttp calls this on the LOOP thread after the last byte of the response has +// been written (or when the connection died with a request on it). That is the +// same moment MHD's NOTIFY_COMPLETED fires, and it has to be: the response +// headers and body are borrowed from the arena released at the end of it. +// +// ⚠️ THE WORK ITSELF MUST NOT RUN HERE. The post-response phase dispatches the +// notifications for a write, and a notification is compacted with the +// subscription's @context - which can be one THIS BROKER HOSTS. The loop thread +// would then be waiting for an answer only the loop thread can produce. It does +// not hang forever, which is worse: the download times out, the notification +// goes out ten seconds late with an uncompacted body, and nothing says why. +// +// So it goes to a worker, and the loop returns immediately. Everything the +// phase reads was copied out of the connection in httpRequestCb, so the +// connection is free to take its next request the instant this returns. +// +static void httpRequestDone(CorHttpConn* connP) +{ + CorRestState* stateP = (CorRestState*) connP->userData; + + connP->userData = NULL; // before anything can fail: this must not run twice + stateP->connection = NULL; // the connection is not this request's any more + + if (corRestAsyncFinish(stateP) == true) + return; + + corRestBackendFinish(stateP); // no pool (shutdown / tests): here, then +} + + + +// ----------------------------------------------------------------------------- +// +// corRestBackendFinish - the post-response phase, and the end of the request +// +void corRestBackendFinish(CorRestState* stateP) +{ + corRestP = stateP; + + // Post-response hook BEFORE releasing the arena, so deferred work (the + // notifications for a write, say) can still read what the service routine built. + corRestPostResponseHook(); + + corRestStateRelease(); + + if (corRestUserDataFreeHookF != NULL && corRest.userData != NULL) + corRestUserDataFreeHookF(corRest.userData); + + free(stateP); + + corRestP = NULL; // no dangling pointer to freed state on this thread +} + + + +// ----------------------------------------------------------------------------- +// +// corRestBackendResume - +// +// On the worker thread, with corRestP still bound: build the response, then +// hand the connection back to the loop. corHttpResume does not write to the +// socket - the loop does - which is what keeps two answers off one connection. +// +void corRestBackendResume(CorRestState* stateP) +{ + CorHttpConn* connP = (CorHttpConn*) stateP->connection; + + requestResponseFill(connP); + corHttpResume(connP); +} + + + +// ----------------------------------------------------------------------------- +// +// serveThread - +// +static void* serveThread(void* unused) +{ + corHttpServe(&corHttpServer); + return NULL; +} + + + +// ----------------------------------------------------------------------------- +// +// corRestBackendStart - +// +int corRestBackendStart(unsigned short port, int poolSize, char* keyPem, char* certPem) +{ + if ((keyPem != NULL) || (certPem != NULL)) + { + // + // Refused, not ignored. The alternative is a server that was asked for TLS + // and serves the port in the clear, which nothing downstream would notice + // until a client's traffic was already on the wire unencrypted. + // + fprintf(stderr, "corRestInit: the built-in HTTP server has no TLS - " + "rebuild with COR_HTTP_SERVER=mhd for an HTTPS listener\n"); + return -1; + } + + // + // The connection pool, not a thread pool: this server is one thread and the + // parallelism is corRest's worker pool behind it. Sized off poolSize all the + // same, since that is the caller's statement about how much concurrency to + // expect, and 16 connections per worker is room rather than a limit. + // + int connPoolSize = (poolSize > 0) ? poolSize * 16 : COR_HTTP_CONN_POOL_SIZE; + + if (connPoolSize < COR_HTTP_CONN_POOL_SIZE) + connPoolSize = COR_HTTP_CONN_POOL_SIZE; + + if (corHttpInit(&corHttpServer, port, connPoolSize, httpRequestCb) != CorHttpOk) + { + fprintf(stderr, "corRestInit: the built-in HTTP server failed to listen on port %d\n", port); + return -1; + } + + corHttpServer.doneCb = httpRequestDone; + + // + // The same § 6.3.2 threshold the dispatch enforces, plus room for the headers + // that arrive in the same buffer as the body. + // + // Over it, corHttp delivers the request WITHOUT its body rather than reading + // one it is going to refuse, and corRestBodyPolicyCheck then answers from the + // Content-Length header with the ProblemDetails the spec asks for - the same + // answer MHD's streaming path produces, from the same header, in the same + // words. What the engine's own cap is left holding is a client that LIES + // about its length, which is the one case where there is no announcement to + // believe. + // + if (corRestMaxRequestSize > 0) + { + unsigned long long cap = corRestMaxRequestSize + (64 * 1024); + + // maxRequestSize is an int over there; a --maxRequestSize of a few GiB would + // otherwise wrap and cap every request at a negative number of bytes. + corHttpServer.maxRequestSize = (cap > 0x7fffffffULL) ? 0 : (int) cap; + } + + if (pthread_create(&corHttpThread, NULL, serveThread, NULL) != 0) + { + fprintf(stderr, "corRestInit: the built-in HTTP server's event loop failed to start\n"); + corHttpRelease(&corHttpServer); + return -1; + } + + corHttpRunning = true; + + return 0; +} + + + +// ----------------------------------------------------------------------------- +// +// corRestBackendStop - +// +void corRestBackendStop(void) +{ + if (corHttpRunning == false) + return; + + corHttpStop(&corHttpServer); // a flag; the loop notices within its 1 s timeout + pthread_join(corHttpThread, NULL); + corHttpRelease(&corHttpServer); + + corHttpRunning = false; +} diff --git a/corRestBackendMhd.c b/corRestBackendMhd.c new file mode 100644 index 0000000..58ad84f --- /dev/null +++ b/corRestBackendMhd.c @@ -0,0 +1,400 @@ +// +// FILE corRestBackendMhd.c +// +// AUTHOR Ken Zangelin +// +// Copyright 2026 Seamware +// SPDX-License-Identifier: Apache-2.0 +// +// The libmicrohttpd backend. +// +// Everything here is MHD's shape: its multi-call request callback, its value +// iterators, its response object, its suspend/resume. The request itself - +// what the URL means, what the body has to be, which headers go back and in +// what order - is on the other side of corRestBackend.h and is shared with the +// built-in server. This file translates, and decides nothing. +// +#include // strtoull, malloc, free, realloc +#include // memcpy, strncmp +#include // fprintf +#include // clock_gettime + +#include + +#include "ktrace/kTrace.h" // KT_V + +#include "corRest/CorRestVerb.h" // CorVerbPost, ... +#include "corRest/CorRestState.h" // CorRestState, corRest +#include "corRest/corRestStateInit.h" // corRestStateInit, corRestStateRelease +#include "corRest/corRestHooks.h" // CorRestHook, corRestSetMaxRequestSize +#include "corRest/corRestProblem.h" // COR_REST_ERROR_*, corRestProblem +#include "corRest/corRestBackend.h" // Own interface + + + +// ----------------------------------------------------------------------------- +// +// The daemon +// +static struct MHD_Daemon* mhdDaemon = NULL; + + + +// ----------------------------------------------------------------------------- +// +// Hook globals (defined in corRestHooks.c) +// +extern CorRestUserDataAllocHook corRestUserDataAllocHookF; +extern CorRestUserDataFreeHook corRestUserDataFreeHookF; +extern CorRestHook corRestPostResponseHook; +extern unsigned long long corRestMaxRequestSize; + + + +// ----------------------------------------------------------------------------- +// +// mhdHeaderIterator - MHD callback to collect request headers +// +static enum MHD_Result mhdHeaderIterator +( + void* cls, + enum MHD_ValueKind kind, + const char* key, + const char* value +) +{ + corRestHttpHeaderAdd(key, value); + return MHD_YES; +} + + + +// ----------------------------------------------------------------------------- +// +// mhdUriParamIterator - MHD callback to collect URI query parameters +// +// MHD strips query params from the URL and provides them via MHD_GET_ARGUMENT_KIND, +// already percent-decoded — which is why this hands them straight over rather +// than going through corRestUriParamsParse (the built-in backend's route, since +// that server decodes nothing). +// +static enum MHD_Result mhdUriParamIterator +( + void* cls, + enum MHD_ValueKind kind, + const char* key, + const char* value +) +{ + corRestUriParamAdd((char*) key, (char*) (value ? value : "")); + return MHD_YES; +} + + + +// ----------------------------------------------------------------------------- +// +// mhdConnectionHandler - MHD callback, called for each incoming request +// +// MHD calls this multiple times per request: +// 1. First call: *con_cls == NULL -> init corRest state +// 2. Middle calls: upload_data_size > 0 -> accumulate payload +// 3. Final call: upload_data_size == 0 -> parse, dispatch, render, respond +// +static enum MHD_Result mhdConnectionHandler +( + void* cls, + struct MHD_Connection* connection, + const char* url, + const char* method, + const char* version, + const char* uploadData, + size_t* uploadDataSize, + void** con_cls +) +{ + // --- First call: allocate this connection's per-request state --- + if (*con_cls == NULL) + { + KT_V("Request: %s %s", method, url); // one line per request (verbose mode, -v) + + // Each connection owns its CorRestState (hung on con_cls), so when the + // epoll pool thread interleaves connection B's callbacks between + // connection A's body-read callbacks it can no longer clobber A's state. + // corRestP is bound to it before corRestStateInit (whose memset/init runs + // through the corRest macro). + CorRestState* conP = (CorRestState*) malloc(sizeof(CorRestState)); + if (conP == NULL) + return MHD_NO; + *con_cls = conP; + corRestP = conP; + + corRestStateInit(connection, url, method); + + // Create this connection's application state (e.g. per-conn corNgsild), + // stored in corRest.userData and freed in mhdRequestCompleted. + if (corRestUserDataAllocHookF != NULL) + corRest.userData = corRestUserDataAllocHookF(); + + // Capture request start time — REALTIME for timestamps, MONOTONIC for duration metrics + struct timespec ts; + clock_gettime(CLOCK_REALTIME, &ts); + corRest.requestStartTime = (uint64_t) ts.tv_sec * 1000000000ULL + (uint64_t) ts.tv_nsec; + + struct timespec tsM; + clock_gettime(CLOCK_MONOTONIC, &tsM); + corRest.requestStartTimeMono = (uint64_t) tsM.tv_sec * 1000000000ULL + (uint64_t) tsM.tv_nsec; + + // Collect request headers from MHD + MHD_get_connection_values(connection, MHD_HEADER_KIND, mhdHeaderIterator, NULL); + + // Collect URI query parameters from MHD (already percent-decoded by MHD) + MHD_get_connection_values(connection, MHD_GET_ARGUMENT_KIND, mhdUriParamIterator, NULL); + + corRestBodyPolicyCheck(url, MHD_lookup_connection_value(connection, MHD_HEADER_KIND, "Content-Length")); + + return MHD_YES; + } + + // Subsequent calls (body chunks, final dispatch): rebind corRestP to THIS + // connection's state — another connection's callback may have re-pointed + // corRestP on this pool thread since our last invocation here. + corRestP = (CorRestState*) *con_cls; + + // --- Middle calls: accumulate payload --- + if (*uploadDataSize > 0) + { + // If the first-call check flagged 411/413, drop the bytes. + if (corRest.in.contentLengthMissing || corRest.out.httpStatusCode == 413) + { + *uploadDataSize = 0; + return MHD_YES; + } + + // Streaming size cap — defends against clients that lie in + // Content-Length or use chunked encoding without a length. + if (corRestMaxRequestSize > 0 && + (unsigned long long)(corRest.in.payloadSize + *uploadDataSize) > corRestMaxRequestSize) + { + corRestProblem(413, COR_REST_ERROR_REQUEST_LENGTH, "Request Entity Too Large", + "request body exceeds broker limit of %llu bytes", corRestMaxRequestSize); + *uploadDataSize = 0; + return MHD_YES; + } + + int needed = corRest.in.payloadSize + *uploadDataSize + 1; + + if (needed > corRest.payloadBufSize) + { + int newSize = (needed + 4096) & ~4095; + char* newBuf = (char*) realloc(corRest.in.payload, newSize); + if (newBuf == NULL) + return MHD_NO; + corRest.in.payload = newBuf; + corRest.payloadBufSize = newSize; + } + + memcpy(corRest.in.payload + corRest.in.payloadSize, uploadData, *uploadDataSize); + corRest.in.payloadSize += *uploadDataSize; + corRest.in.payload[corRest.in.payloadSize] = 0; + + *uploadDataSize = 0; + return MHD_YES; + } + + // --- Final call: process (off the I/O thread when the pool is up), respond --- + if (corRest.asyncProcessed == false) + { + if (corRestAsyncPoolUp() == true) + { + // Suspend this connection and hand it to a worker so DB/distop latency + // doesn't block this epoll thread. The worker runs corRestProcessRequest + // and resumes us; MHD then re-invokes this handler with asyncProcessed + // set and we fall through to build + send the response. Suspend BEFORE + // enqueue so a worker can never resume a not-yet-suspended connection. + MHD_suspend_connection(connection); + corRestAsyncEnqueue(corRestP); + return MHD_YES; + } + + corRestProcessRequest(); // pool down (shutdown / tests): run inline here + } + + char* responseBody = (corRest.out.payload != NULL) ? corRest.out.payload : (char*) ""; + int responseBodySize = corRest.out.payloadSize; + + // Send HTTP response + // For HEAD: pass the full body — MHD will set Content-Length correctly + // but suppress the body in the actual response. + struct MHD_Response* response; + + response = MHD_create_response_from_buffer( + responseBodySize, + (void*) responseBody, + MHD_RESPMEM_MUST_COPY + ); + + CorRestKeyValue headerV[COR_REST_RESPONSE_HEADERS_MAX]; + int headers = corRestResponseHeaderVBuild(headerV, COR_REST_RESPONSE_HEADERS_MAX); + + for (int i = 0; i < headers; i++) + MHD_add_response_header(response, headerV[i].key, headerV[i].value); + + enum MHD_Result ret = MHD_queue_response(connection, corRest.out.httpStatusCode, response); + MHD_destroy_response(response); + + return ret; +} + + + +// ----------------------------------------------------------------------------- +// +// corRestBackendFinish - +// +// Runs on the MHD I/O thread that finished the request. NOT handed to a worker +// the way the built-in backend has to hand it: MHD has poolSize I/O threads, so +// a post-response phase that blocks - and it can, on an @context this broker +// hosts itself - stalls one of them and the others keep serving. That is why +// this backend is left exactly as it was. +// +void corRestBackendFinish(CorRestState* stateP) +{ + corRestP = stateP; + + // Run post-response hook BEFORE releasing the per-request arena so the + // hook can still touch arena-allocated data (e.g. deferred notification + // dispatch reading the entity tree built by the service routine). + corRestPostResponseHook(); + + // Free payload buffer (malloc'd during accumulation, not in kalloc) + free(corRest.in.payload); + corRest.in.payload = NULL; + + corRestStateRelease(); + + // Destroy this connection's application state (per-conn corNgsild, ...). + if (corRestUserDataFreeHookF != NULL && corRest.userData != NULL) + corRestUserDataFreeHookF(corRest.userData); + + free(stateP); + + corRestP = NULL; // no dangling pointer to freed state on this thread +} + + + +// ----------------------------------------------------------------------------- +// +// mhdRequestCompleted - MHD callback when a request is fully handled +// +static void mhdRequestCompleted +( + void* cls, + struct MHD_Connection* connection, + void** con_cls, + enum MHD_RequestTerminationCode toe +) +{ + if (*con_cls != NULL) + { + CorRestState* conP = (CorRestState*) *con_cls; + + *con_cls = NULL; + corRestBackendFinish(conP); + } +} + + + +// ----------------------------------------------------------------------------- +// +// corRestBackendResume - +// +// The worker built the response into corRest.out; MHD sends it by re-invoking +// the connection handler, which it does once the connection is resumed (the ITC +// pipe wakes the polling thread). +// +void corRestBackendResume(CorRestState* stateP) +{ + MHD_resume_connection((struct MHD_Connection*) stateP->connection); +} + + + +// ----------------------------------------------------------------------------- +// +// corRestBackendStart - +// +int corRestBackendStart(unsigned short port, int poolSize, char* keyPem, char* certPem) +{ + // MHD_ALLOW_SUSPEND_RESUME (which bundles MHD_USE_ITC) lets the I/O threads + // suspend a connection and hand it to the async worker pool; the worker + // resumes it once the response is built. + // + // When HTTPS server credentials have been set (a TLS test receiver, not the + // broker), MHD_USE_TLS is added together with the in-memory key/cert. + unsigned int flags = MHD_USE_SELECT_INTERNALLY | MHD_USE_EPOLL | MHD_ALLOW_SUSPEND_RESUME; + + if ((keyPem != NULL) && (certPem != NULL)) + mhdDaemon = MHD_start_daemon( + flags | MHD_USE_TLS, + port, + NULL, + NULL, + mhdConnectionHandler, + NULL, + MHD_OPTION_NOTIFY_COMPLETED, + mhdRequestCompleted, + NULL, + MHD_OPTION_THREAD_POOL_SIZE, + (unsigned int) poolSize, + MHD_OPTION_CONNECTION_TIMEOUT, + (unsigned int) 30, + MHD_OPTION_HTTPS_MEM_KEY, + keyPem, + MHD_OPTION_HTTPS_MEM_CERT, + certPem, + MHD_OPTION_END + ); + else + mhdDaemon = MHD_start_daemon( + flags, + port, + NULL, + NULL, + mhdConnectionHandler, + NULL, + MHD_OPTION_NOTIFY_COMPLETED, + mhdRequestCompleted, + NULL, + MHD_OPTION_THREAD_POOL_SIZE, + (unsigned int) poolSize, + MHD_OPTION_CONNECTION_TIMEOUT, + (unsigned int) 30, + MHD_OPTION_END + ); + + if (mhdDaemon == NULL) + { + fprintf(stderr, "corRestInit: MHD_start_daemon failed on port %d\n", port); + return -1; + } + + return 0; +} + + + +// ----------------------------------------------------------------------------- +// +// corRestBackendStop - +// +void corRestBackendStop(void) +{ + if (mhdDaemon != NULL) + { + MHD_stop_daemon(mhdDaemon); + mhdDaemon = NULL; + } +} diff --git a/corRestInit.c b/corRestInit.c index a22138b..d4aa4b2 100644 --- a/corRestInit.c +++ b/corRestInit.c @@ -12,8 +12,6 @@ #include // clock_gettime, CLOCK_MONOTONIC #include // pthread_* (async worker pool) -#include - #include "kalloc/kaAlloc.h" // kaAlloc #include "kalloc/kaStrdup.h" // kaStrdup #include "kjson/kjParse.h" // kjParse @@ -30,6 +28,7 @@ #include "corRest/corRestHooks.h" // CorRestHook, etc. #include "corRest/corRestProblem.h" // COR_REST_ERROR_*, corRestProblem #include "corRest/corRestParamRegistry.h" // corRestParamLookup +#include "corRest/corRestBackend.h" // corRestBackendStart, corRestBackendResume #include "corRest/corRestInit.h" // Own interface #include "corRest/corRestUrlValueEncode.h" // corRestUrlValueDecode @@ -40,7 +39,6 @@ // Globals // CorRestServiceVector corRestServiceV[CorVerbs]; -struct MHD_Daemon* corRestDaemon = NULL; @@ -51,7 +49,8 @@ struct MHD_Daemon* corRestDaemon = NULL; // The broker listens over plain HTTP and never sets these. A test server such // as ftClient that must receive notifications over TLS calls // corRestHttpsServerCredentialsSet() with the key+certificate PEM before -// corRestInit; the MHD daemon is then started with TLS enabled. +// corRestInit; the backend is then started with TLS enabled - and a backend +// that has no TLS refuses to start rather than serving the port in the clear. // static char* httpsServerKey = NULL; static char* httpsServerCert = NULL; @@ -78,6 +77,7 @@ extern CorRestHook corRestPostResponseHook; extern CorRestUserDataAllocHook corRestUserDataAllocHookF; extern CorRestUserDataFreeHook corRestUserDataFreeHookF; extern unsigned long long corRestMaxRequestSize; +extern CorRestCorsConfig corRestCors; @@ -213,9 +213,9 @@ static void servicePrepare(CorRestService* serviceP, CorRestServiceSimplified* s // ----------------------------------------------------------------------------- // -// addUriParam - add URI parameter to dynamic array, growing if needed +// corRestUriParamAdd - add URI parameter to dynamic array, growing if needed // -static void addUriParam(char* name, char* value) +void corRestUriParamAdd(char* name, char* value) { if (corRest.in.uriParamCount >= corRest.in.uriParamSize) { @@ -241,9 +241,15 @@ static void addUriParam(char* name, char* value) // ----------------------------------------------------------------------------- // -// parseUriParams - parse query string into key-value pairs with percent-decoding +// corRestUriParamsParse - split the query string, percent-decoding as it goes +// +// IN PLACE, over corRest.in.urlParams: the buffer comes out with a NUL where +// every '&' and the first '=' of each parameter was, so the raw query string +// does not survive this. Nothing reads it afterwards (purgeEntities rebuilds +// the query from the parsed array for exactly that reason), and the alternative +// - a second copy on every request that carries a query - buys nothing. // -static void parseUriParams(void) +void corRestUriParamsParse(void) { if (corRest.in.urlParams == NULL) return; @@ -285,7 +291,7 @@ static void parseUriParams(void) if (value[0] != '\0') corRestUrlValueDecode(value); - addUriParam(name, value); + corRestUriParamAdd(name, value); } } @@ -293,13 +299,13 @@ static void parseUriParams(void) // ----------------------------------------------------------------------------- // -// addHttpHeader - append a request header to corRest.in, extracting well-knowns +// corRestHttpHeaderAdd - append a request header to corRest.in, extracting well-knowns // -// Shared by the MHD header iterator and the in-process self-forward path. The +// Shared by the HTTP backends and the in-process self-forward path. The // key/value pointers are borrowed (not copied) — the caller keeps them alive // for the request's lifetime. // -static void addHttpHeader(const char* key, const char* value) +void corRestHttpHeaderAdd(const char* key, const char* value) { if (corRest.in.httpHeaderCount >= corRest.in.httpHeaderSize) { @@ -331,71 +337,14 @@ static void addHttpHeader(const char* key, const char* value) -// ----------------------------------------------------------------------------- -// -// mhdHeaderIterator - MHD callback to collect request headers -// -static enum MHD_Result mhdHeaderIterator -( - void* cls, - enum MHD_ValueKind kind, - const char* key, - const char* value -) -{ - addHttpHeader(key, value); - return MHD_YES; -} - - - -// ----------------------------------------------------------------------------- -// -// mhdUriParamIterator - MHD callback to collect URI query parameters -// -// MHD strips query params from the URL and provides them via MHD_GET_ARGUMENT_KIND. -// The values are already percent-decoded by MHD. -// -static enum MHD_Result mhdUriParamIterator -( - void* cls, - enum MHD_ValueKind kind, - const char* key, - const char* value -) -{ - if (corRest.in.uriParamCount >= corRest.in.uriParamSize) - { - int newSize = corRest.in.uriParamSize + COR_REST_KV_GROW_SIZE; - CorRestKeyValue* newV = (CorRestKeyValue*) kaAlloc(&corRest.kalloc, newSize * sizeof(CorRestKeyValue)); - - if (newV == NULL) - return MHD_YES; - - memcpy(newV, corRest.in.uriParamV, corRest.in.uriParamCount * sizeof(CorRestKeyValue)); - - corRest.in.uriParamV = newV; - corRest.in.uriParamSize = newSize; - } - - corRest.in.uriParamV[corRest.in.uriParamCount].key = (char*) key; - corRest.in.uriParamV[corRest.in.uriParamCount].value = (char*) (value ? value : ""); - corRest.in.uriParamV[corRest.in.uriParamCount].bit = 0; // resolved in the allowlist pass - corRest.in.uriParamCount++; - - return MHD_YES; -} - - - // ----------------------------------------------------------------------------- // // corRestProcessRequest - run the request-dispatch core on the bound corRest state // // Connection-free: consumes corRest.in (verb, url, headers, params, requestTree) // and produces corRest.out (status, headers, problem / responseTree) plus the -// final rendered body in corRest.out.payload / payloadSize. The MHD send stays in -// the connection handler. Reusable for an in-process self-forward (a distop +// final rendered body in corRest.out.payload / payloadSize. The send stays in +// the backend's connection handler. Reusable for an in-process self-forward (a distop // whose endpoint is this broker): bind corRestP to an inner state, populate its // .in, call this, read its .out — no socket round-trip. // @@ -825,11 +774,11 @@ int corRestProcessInProcess(CorRestVerb verb, corRest.requestStartTimeMono = outerP->requestStartTimeMono; // URI params (corRestStateInit split a trailing ?query into corRest.in.urlParams) - parseUriParams(); + corRestUriParamsParse(); // Request headers — borrowed; the (outer) arena they live in stays alive for (int i = 0; i < headerCount; i++) - addHttpHeader(headerV[i].key, headerV[i].value); + corRestHttpHeaderAdd(headerV[i].key, headerV[i].value); // Body — copy into the inner arena (freed by corRestStateRelease) if (body != NULL && bodyLen > 0) @@ -900,20 +849,20 @@ int corRestProcessInProcess(CorRestVerb verb, // // Async worker pool (6d) // -// The MHD I/O threads accept, parse headers and accumulate the body, then +// The backend's I/O threads accept, parse headers and accumulate the body, then // suspend the connection and hand its CorRestState to this pool. A worker binds // corRestP to that state and runs the (potentially slow: DB, distops) dispatch -// off the I/O thread, then resumes the connection — MHD re-invokes the handler, -// which sends the already-built response. This decouples per-request processing -// latency from epoll I/O throughput: a slow request no longer blocks the I/O -// thread from servicing every other connection pinned to it. +// off the I/O thread, then resumes the connection, and the backend sends the +// already-built response. This decouples per-request processing latency from +// epoll I/O throughput: a slow request no longer blocks the I/O thread from +// servicing every other connection pinned to it. // // All request state lives in the per-connection CorRestState (arena, kjson, // in/out, userData -> per-conn corNgsild), so a worker needs nothing but // `corRestP = conP` — the I/O thread already ran corRestStateInit. The deferred // notification caches are per-connection too, and still flushed by the -// post-response hook in mhdRequestCompleted, so notification ordering is -// unchanged. +// post-response hook once the response has gone out, so notification ordering +// is unchanged. // static pthread_t* corRestWorkerV = NULL; static int corRestWorkerCount = 0; @@ -959,23 +908,42 @@ static void* corRestWorkerMain(void* unused) corRestQueueTail = NULL; pthread_mutex_unlock(&corRestQueueMtx); + corRestP = conP; + + // + // Second visit: the response has gone out and what is left is the request's + // post-response work. Nothing below applies - there is no connection to + // resume, and after this the state is gone. + // + if (conP->asyncFinishing == true) + { + corRestBackendFinish(conP); + corRestP = NULL; + continue; + } + // Run the dispatch on this worker, bound to the connection's own state. // A self-targeted forward (corRestProcessInProcess) runs synchronously // inside this same call on this same worker thread — never re-enqueued. - corRestP = conP; corRestProcessRequest(); conP->asyncProcessed = true; - corRestP = NULL; // drop the bind; the request leaves this thread - // Hand the connection back to MHD (ITC wakes the polling thread); MHD - // re-invokes mhdConnectionHandler, which sends corRest.out. - MHD_resume_connection(conP->mhdConnection); + // + // Hand the connection back to the backend's event loop. STILL BOUND: the + // built-in backend reads corRest.out here to build the response before it + // wakes the loop, and unbinding first would leave it reading the fallback + // state of this worker thread. After this call the connection may already + // be finished and freed by the loop, so nothing below may touch conP. + // + corRestBackendResume(conP); + + corRestP = NULL; // drop the bind; the request has left this thread } return NULL; } -static int corRestWorkerPoolStart(int workers) +int corRestWorkerPoolStart(int workers) { if (workers < 1) workers = 1; @@ -1005,11 +973,11 @@ static int corRestWorkerPoolStart(int workers) // // corRestWorkerPoolStop - stop new suspensions, drain the queue, join workers // -// Clearing corRestWorkersRun first makes mhdConnectionHandler process inline +// Clearing corRestWorkersRun first makes the backend's handler process inline // again, so no NEW connection is suspended; the workers then drain whatever is // already queued (processing + resuming each) before exiting. After this -// returns there are no suspended connections, so MHD_stop_daemon is safe -// (suspending across MHD_stop_daemon is an API violation). +// returns there are no suspended connections, so stopping the backend is safe +// (resuming across MHD_stop_daemon is an API violation). // void corRestWorkerPoolStop(void) { @@ -1033,170 +1001,101 @@ void corRestWorkerPoolStop(void) // ----------------------------------------------------------------------------- // -// mhdConnectionHandler - MHD callback, called for each incoming request +// corRestAsyncPoolUp / corRestAsyncEnqueue - hand this request to a worker // -// MHD calls this multiple times per request: -// 1. First call: *con_cls == NULL -> init corRest state -// 2. Middle calls: upload_data_size > 0 -> accumulate payload -// 3. Final call: upload_data_size == 0 -> parse, dispatch, render, respond +// Two calls, with the connection suspended between them - see corRestBackend.h. // -static enum MHD_Result mhdConnectionHandler -( - void* cls, - struct MHD_Connection* connection, - const char* url, - const char* method, - const char* version, - const char* uploadData, - size_t* uploadDataSize, - void** con_cls -) +bool corRestAsyncPoolUp(void) { - // --- First call: allocate this connection's per-request state --- - if (*con_cls == NULL) - { - KT_V("Request: %s %s", method, url); // one line per request (verbose mode, -v) - - // Each connection owns its CorRestState (hung on con_cls), so when the - // epoll pool thread interleaves connection B's callbacks between - // connection A's body-read callbacks it can no longer clobber A's state. - // corRestP is bound to it before corRestStateInit (whose memset/init runs - // through the corRest macro). - CorRestState* conP = (CorRestState*) malloc(sizeof(CorRestState)); - if (conP == NULL) - return MHD_NO; - *con_cls = conP; - corRestP = conP; - - corRestStateInit(connection, url, method); - - // Create this connection's application state (e.g. per-conn corNgsild), - // stored in corRest.userData and freed in mhdRequestCompleted. - if (corRestUserDataAllocHookF != NULL) - corRest.userData = corRestUserDataAllocHookF(); - - // Capture request start time — REALTIME for timestamps, MONOTONIC for duration metrics - struct timespec ts; - clock_gettime(CLOCK_REALTIME, &ts); - corRest.requestStartTime = (uint64_t) ts.tv_sec * 1000000000ULL + (uint64_t) ts.tv_nsec; - - struct timespec tsM; - clock_gettime(CLOCK_MONOTONIC, &tsM); - corRest.requestStartTimeMono = (uint64_t) tsM.tv_sec * 1000000000ULL + (uint64_t) tsM.tv_nsec; - - // Collect request headers from MHD - MHD_get_connection_values(connection, MHD_HEADER_KIND, mhdHeaderIterator, NULL); - - // Collect URI query parameters from MHD (already percent-decoded by MHD) - MHD_get_connection_values(connection, MHD_GET_ARGUMENT_KIND, mhdUriParamIterator, NULL); - - // § 6.3.4 — POST / PATCH / PUT to NGSI-LD endpoints must carry - // Content-Length. § 6.3.2 — 413 when the announced body exceeds - // the broker cap. Non-NGSI-LD endpoints (e.g. /admin/*) are - // outside the spec's scope and accept bodyless POSTs. - if ((corRest.in.verb == CorVerbPost || - corRest.in.verb == CorVerbPut || - corRest.in.verb == CorVerbPatch) && - url != NULL && - strncmp(url, "/ngsi-ld/", 9) == 0) - { - const char* clHdr = MHD_lookup_connection_value(connection, MHD_HEADER_KIND, "Content-Length"); - if (clHdr == NULL) - { - // § 6.3.4 — just a 411 status, no body. - corRest.out.httpStatusCode = 411; - corRest.in.contentLengthMissing = true; - } - else if (corRestMaxRequestSize > 0) - { - unsigned long long cl = strtoull(clHdr, NULL, 10); - if (cl > corRestMaxRequestSize) - { - corRestProblem(413, COR_REST_ERROR_REQUEST_LENGTH, "Request Entity Too Large", - "request body of %llu bytes exceeds broker limit of %llu bytes", - cl, corRestMaxRequestSize); - } - } - } + return corRestWorkersRun; +} - return MHD_YES; - } +void corRestAsyncEnqueue(CorRestState* stateP) +{ + corRestWorkerEnqueue(stateP); +} - // Subsequent calls (body chunks, final dispatch): rebind corRestP to THIS - // connection's state — another connection's callback may have re-pointed - // corRestP on this pool thread since our last invocation here. - corRestP = (CorRestState*) *con_cls; - // --- Middle calls: accumulate payload --- - if (*uploadDataSize > 0) - { - // If the first-call check flagged 411/413, drop the bytes. - if (corRest.in.contentLengthMissing || corRest.out.httpStatusCode == 413) - { - *uploadDataSize = 0; - return MHD_YES; - } - // Streaming size cap — defends against clients that lie in - // Content-Length or use chunked encoding without a length. - if (corRestMaxRequestSize > 0 && - (unsigned long long)(corRest.in.payloadSize + *uploadDataSize) > corRestMaxRequestSize) - { - corRestProblem(413, COR_REST_ERROR_REQUEST_LENGTH, "Request Entity Too Large", - "request body exceeds broker limit of %llu bytes", corRestMaxRequestSize); - *uploadDataSize = 0; - return MHD_YES; - } +// ----------------------------------------------------------------------------- +// +// corRestAsyncFinish - the post-response phase, off the I/O thread +// +bool corRestAsyncFinish(CorRestState* stateP) +{ + if (corRestWorkersRun == false) + return false; - int needed = corRest.in.payloadSize + *uploadDataSize + 1; + stateP->asyncFinishing = true; + corRestWorkerEnqueue(stateP); - if (needed > corRest.payloadBufSize) - { - int newSize = (needed + 4096) & ~4095; - char* newBuf = (char*) realloc(corRest.in.payload, newSize); - if (newBuf == NULL) - return MHD_NO; - corRest.in.payload = newBuf; - corRest.payloadBufSize = newSize; - } + return true; +} - memcpy(corRest.in.payload + corRest.in.payloadSize, uploadData, *uploadDataSize); - corRest.in.payloadSize += *uploadDataSize; - corRest.in.payload[corRest.in.payloadSize] = 0; - *uploadDataSize = 0; - return MHD_YES; - } - // --- Final call: process (off the I/O thread when the pool is up), respond --- - if (corRestWorkersRun && !corRest.asyncProcessed) +// ----------------------------------------------------------------------------- +// +// corRestBodyPolicyCheck - +// +// § 6.3.4 — POST / PATCH / PUT to NGSI-LD endpoints must carry Content-Length +// (411 Length Required, and NO payload). § 6.3.2 — 413 when the announced body +// exceeds the broker cap. Non-NGSI-LD endpoints (e.g. /admin/*) are outside the +// spec's scope and accept bodyless POSTs. +// +void corRestBodyPolicyCheck(const char* url, const char* clHeader) +{ + if ((corRest.in.verb != CorVerbPost) && + (corRest.in.verb != CorVerbPut) && + (corRest.in.verb != CorVerbPatch)) + return; + + if ((url == NULL) || (strncmp(url, "/ngsi-ld/", 9) != 0)) + return; + + if (clHeader == NULL) { - // Suspend this connection and hand it to a worker so DB/distop latency - // doesn't block this epoll thread. The worker runs corRestProcessRequest and - // resumes us; MHD then re-invokes this handler with asyncProcessed set and - // we fall through to build + send the response. Suspend BEFORE enqueue so a - // worker can never resume a not-yet-suspended connection. - MHD_suspend_connection(connection); - corRestWorkerEnqueue(corRestP); - return MHD_YES; + // § 6.3.4 — just a 411 status, no body. + corRest.out.httpStatusCode = 411; + corRest.in.contentLengthMissing = true; + return; } - if (!corRest.asyncProcessed) - corRestProcessRequest(); // pool down (shutdown / tests): run inline here + if (corRestMaxRequestSize > 0) + { + unsigned long long cl = strtoull(clHeader, NULL, 10); - char* responseBody = (corRest.out.payload != NULL) ? corRest.out.payload : (char*) ""; - int responseBodySize = corRest.out.payloadSize; + if (cl > corRestMaxRequestSize) + corRestProblem(413, COR_REST_ERROR_REQUEST_LENGTH, "Request Entity Too Large", + "request body of %llu bytes exceeds broker limit of %llu bytes", + cl, corRestMaxRequestSize); + } +} - // Send HTTP response - // For HEAD: pass the full body — MHD will set Content-Length correctly - // but suppress the body in the actual response. - struct MHD_Response* response; - response = MHD_create_response_from_buffer( - responseBodySize, - (void*) responseBody, - MHD_RESPMEM_MUST_COPY - ); + +// ----------------------------------------------------------------------------- +// +// corRestResponseHeaderVBuild - +// +// One implementation of the response-header policy for both backends. The SET +// and the ORDER are pinned by several hundred functional tests that compare +// captured responses line by line, so this is a specification rather than a +// preference - and two copies of it would be two chances to drift. +// +int corRestResponseHeaderVBuild(CorRestKeyValue* hv, int max) +{ + int n = 0; + + #define HDR_ADD(k, v) \ + do { \ + if (n < max) \ + { \ + hv[n].key = (char*) (k); \ + hv[n].value = (char*) (v); \ + n++; \ + } \ + } while (0) // Content-Type policy: // - No body (size 0): omit Content-Type — it describes a body there is none @@ -1208,14 +1107,15 @@ static enum MHD_Result mhdConnectionHandler // TS 104-176 specifies bare media types throughout (§ 6.2.3, § 6.3.3, // § 6.4.7.2 "exactly equal to the media type"); RFC 8259 defines no charset // parameter for application/json — so emit the type verbatim, no charset. - if (responseBodySize > 0) + if (corRest.out.payloadSize > 0) { int code = corRest.out.httpStatusCode; const char* ct = (code == 201 || (code >= 400 && code <= 599)) ? "application/json" : corRest.out.contentType; + if (ct != NULL) - MHD_add_response_header(response, "Content-Type", ct); + HDR_ADD("Content-Type", ct); } // § 6.3.6 Prefer / Preference-Applied: when the client sent a @@ -1230,20 +1130,19 @@ static enum MHD_Result mhdConnectionHandler corRest.in.httpHeaderV[hi].value != NULL && strncasecmp(corRest.in.httpHeaderV[hi].value, "ngsi-ld=", 8) == 0) { - MHD_add_response_header(response, "Preference-Applied", "ngsi-ld=1.9.1"); + HDR_ADD("Preference-Applied", "ngsi-ld=1.9.1"); break; } } - // Add custom response headers + // The service routine's own headers, in the order it added them for (int i = 0; i < corRest.out.headerCount; i++) - MHD_add_response_header(response, corRest.out.headerV[i].key, corRest.out.headerV[i].value); + HDR_ADD(corRest.out.headerV[i].key, corRest.out.headerV[i].value); // CORS headers (if configured) - extern CorRestCorsConfig corRestCors; if (corRestCors.allowOrigin != NULL) { - MHD_add_response_header(response, "Access-Control-Allow-Origin", corRestCors.allowOrigin); + HDR_ADD("Access-Control-Allow-Origin", corRestCors.allowOrigin); if (corRest.in.verb == CorVerbOptions) { @@ -1252,71 +1151,38 @@ static enum MHD_Result mhdConnectionHandler { if (strcmp(corRest.out.headerV[i].key, "Allow") == 0) { - MHD_add_response_header(response, "Access-Control-Allow-Methods", corRest.out.headerV[i].value); + HDR_ADD("Access-Control-Allow-Methods", corRest.out.headerV[i].value); break; } } - const char* ah = corRestCors.allowHeaders ? corRestCors.allowHeaders : "Content-Type, Accept, Link"; - MHD_add_response_header(response, "Access-Control-Allow-Headers", ah); + HDR_ADD("Access-Control-Allow-Headers", + corRestCors.allowHeaders ? corRestCors.allowHeaders : "Content-Type, Accept, Link"); if (corRestCors.maxAge > 0) { - char maxAgeBuf[16]; - snprintf(maxAgeBuf, sizeof(maxAgeBuf), "%d", corRestCors.maxAge); - MHD_add_response_header(response, "Access-Control-Max-Age", maxAgeBuf); + // + // Rendered into the request arena rather than a local buffer: the + // backend sends these AFTER this function has returned, and one of them + // does not copy what it is given. + // + char* maxAgeBuf = (char*) kaAlloc(&corRest.kalloc, 16); + + if (maxAgeBuf != NULL) + { + snprintf(maxAgeBuf, 16, "%d", corRestCors.maxAge); + HDR_ADD("Access-Control-Max-Age", maxAgeBuf); + } } } if (corRestCors.exposeHeaders != NULL) - MHD_add_response_header(response, "Access-Control-Expose-Headers", corRestCors.exposeHeaders); + HDR_ADD("Access-Control-Expose-Headers", corRestCors.exposeHeaders); } - enum MHD_Result ret = MHD_queue_response(connection, corRest.out.httpStatusCode, response); - MHD_destroy_response(response); + #undef HDR_ADD - return ret; -} - - - -// ----------------------------------------------------------------------------- -// -// mhdRequestCompleted - MHD callback when a request is fully handled -// -static void mhdRequestCompleted -( - void* cls, - struct MHD_Connection* connection, - void** con_cls, - enum MHD_RequestTerminationCode toe -) -{ - if (*con_cls != NULL) - { - // Bind to this connection's state before the hook / release touch corRest. - CorRestState* conP = (CorRestState*) *con_cls; - corRestP = conP; - - // Run post-response hook BEFORE releasing the per-request arena so the - // hook can still touch arena-allocated data (e.g. deferred notification - // dispatch reading the entity tree built by the service routine). - corRestPostResponseHook(); - - // Free payload buffer (malloc'd during accumulation, not in kalloc) - free(corRest.in.payload); - corRest.in.payload = NULL; - - corRestStateRelease(); - - // Destroy this connection's application state (per-conn corNgsild, ...). - if (corRestUserDataFreeHookF != NULL && corRest.userData != NULL) - corRestUserDataFreeHookF(corRest.userData); - - free(conP); - *con_cls = NULL; - corRestP = NULL; // no dangling pointer to freed state on this thread - } + return n; } @@ -1458,58 +1324,12 @@ int corRestInit(CorRestServiceSimplified serviceV[], int services, unsigned shor } } - // Start MHD daemon with thread pool. MHD_ALLOW_SUSPEND_RESUME (which bundles - // MHD_USE_ITC) lets the I/O threads suspend a connection and hand it to the - // async worker pool below; the worker resumes it once the response is built. // - // When HTTPS server credentials have been set (a TLS test receiver, not the - // broker), MHD_USE_TLS is added together with the in-memory key/cert. - unsigned int flags = MHD_USE_SELECT_INTERNALLY | MHD_USE_EPOLL | MHD_ALLOW_SUSPEND_RESUME; - - if ((httpsServerKey != NULL) && (httpsServerCert != NULL)) - corRestDaemon = MHD_start_daemon( - flags | MHD_USE_TLS, - port, - NULL, - NULL, - mhdConnectionHandler, - NULL, - MHD_OPTION_NOTIFY_COMPLETED, - mhdRequestCompleted, - NULL, - MHD_OPTION_THREAD_POOL_SIZE, - (unsigned int) poolSize, - MHD_OPTION_CONNECTION_TIMEOUT, - (unsigned int) 30, - MHD_OPTION_HTTPS_MEM_KEY, - httpsServerKey, - MHD_OPTION_HTTPS_MEM_CERT, - httpsServerCert, - MHD_OPTION_END - ); - else - corRestDaemon = MHD_start_daemon( - flags, - port, - NULL, - NULL, - mhdConnectionHandler, - NULL, - MHD_OPTION_NOTIFY_COMPLETED, - mhdRequestCompleted, - NULL, - MHD_OPTION_THREAD_POOL_SIZE, - (unsigned int) poolSize, - MHD_OPTION_CONNECTION_TIMEOUT, - (unsigned int) 30, - MHD_OPTION_END - ); - - if (corRestDaemon == NULL) - { - fprintf(stderr, "corRestInit: MHD_start_daemon failed on port %d\n", port); + // The HTTP server itself - libmicrohttpd or the built-in one, decided at + // compile time by COR_HTTP_SERVER. It starts its own threads and returns. + // + if (corRestBackendStart(port, poolSize, httpsServerKey, httpsServerCert) != 0) return -1; - } // Async worker pool — one worker per I/O thread. corRestWorkersRun stays false // (handler processes inline) if the pool fails to start. diff --git a/corRestInit.h b/corRestInit.h index 9cb30b6..4760d63 100644 --- a/corRestInit.h +++ b/corRestInit.h @@ -32,13 +32,15 @@ // // corRestInit - // -// Initialize the REST library: prepare service lookup tables and start MHD. +// Initialize the REST library: prepare service lookup tables, start the HTTP +// server (libmicrohttpd or the built-in one - COR_HTTP_SERVER, decided at +// compile time) and the worker pool that runs requests off its I/O threads. // // Parameters: // serviceV - flat array of service definitions (each includes its verb) // services - number of entries in serviceV // port - TCP port to listen on -// poolSize - MHD thread pool size +// poolSize - I/O threads, and one dispatch worker each // // GET services automatically get HEAD support (same handler, body suppressed). // OPTIONS is auto-generated for all registered URL paths. @@ -53,9 +55,13 @@ extern int corRestInit(CorRestServiceSimplified serviceV[], int services, unsign // corRestHttpsServerCredentialsSet - serve HTTPS instead of HTTP // // Call before corRestInit with the PEM contents (not file paths) of the private -// key and certificate. The MHD daemon is then started with TLS enabled. NULL -// (the default) keeps the server on plain HTTP. Used by TLS test receivers -// (e.g. ftClient), not by the broker. +// key and certificate; the server is then started with TLS enabled. NULL (the +// default) keeps it on plain HTTP. Used by TLS test receivers (e.g. ftClient), +// not by the broker. +// +// ⚠️ The BUILT-IN server has no TLS, and corRestInit REFUSES to start rather +// than quietly serving the port in the clear. Build with COR_HTTP_SERVER=mhd +// for an HTTPS listener. // extern void corRestHttpsServerCredentialsSet(char* keyPem, char* certPem); @@ -66,8 +72,8 @@ extern void corRestHttpsServerCredentialsSet(char* keyPem, char* certPem); // corRestProcessRequest - run the request-dispatch core on the bound corRest state // // Consumes corRest.in, produces corRest.out (incl. the rendered body in -// corRest.out.payload / payloadSize). Connection-free — the MHD send lives in the -// connection handler. +// corRest.out.payload / payloadSize). Connection-free — the send lives in the +// HTTP backend's connection handler. // extern void corRestProcessRequest(void); diff --git a/corRestStateInit.c b/corRestStateInit.c index 87496fc..c863127 100644 --- a/corRestStateInit.c +++ b/corRestStateInit.c @@ -42,11 +42,11 @@ extern int corRestDefaultPrettySpaces; // // corRestStateInit - // -void corRestStateInit(struct MHD_Connection* connection, const char* url, const char* method) +void corRestStateInit(void* connection, const char* url, const char* method) { memset(&corRest, 0, sizeof(CorRestState)); - corRest.mhdConnection = connection; + corRest.connection = connection; // Initialize kalloc pool (inline buffer first, then malloc-based overflow). // 256KB grow chunks: keeps single-shot renderings of ~1000-entity responses @@ -72,17 +72,7 @@ void corRestStateInit(struct MHD_Connection* connection, const char* url, const corRest.in.urlParams = qMark + 1; } - corRest.in.urlPathLen = strlen(corRest.in.urlPath); - - // Strip a single trailing '/' from the path (except the root "/" itself). - // Routes are registered without a trailing slash; clients that send one - // (e.g. ETSI test suite: POST /ngsi-ld/v1/entities/) would otherwise - // 404 against an exact-match service lookup. - if (corRest.in.urlPathLen > 1 && corRest.in.urlPath[corRest.in.urlPathLen - 1] == '/') - { - corRest.in.urlPath[corRest.in.urlPathLen - 1] = 0; - corRest.in.urlPathLen--; - } + corRestUrlPathNormalize(); // Initialize URI param dynamic array (starts with inline slots) corRest.in.uriParamV = corRest.in.uriParams; @@ -113,6 +103,31 @@ void corRestStateInit(struct MHD_Connection* connection, const char* url, const +// ----------------------------------------------------------------------------- +// +// corRestUrlPathNormalize - +// +// Strip a single trailing '/' from the path (except the root "/" itself). +// Routes are registered without a trailing slash; clients that send one +// (e.g. ETSI test suite: POST /ngsi-ld/v1/entities/) would otherwise +// 404 against an exact-match service lookup. +// +void corRestUrlPathNormalize(void) +{ + if (corRest.in.urlPath == NULL) + return; + + corRest.in.urlPathLen = strlen(corRest.in.urlPath); + + if (corRest.in.urlPathLen > 1 && corRest.in.urlPath[corRest.in.urlPathLen - 1] == '/') + { + corRest.in.urlPath[corRest.in.urlPathLen - 1] = 0; + corRest.in.urlPathLen--; + } +} + + + // ----------------------------------------------------------------------------- // // corRestStateRelease - release per-request resources diff --git a/corRestStateInit.h b/corRestStateInit.h index 515794e..3eef8d3 100644 --- a/corRestStateInit.h +++ b/corRestStateInit.h @@ -9,15 +9,30 @@ #ifndef CORREST_STATE_INIT_H_ #define CORREST_STATE_INIT_H_ -#include - // ----------------------------------------------------------------------------- // // corRestStateInit - initialize the thread-local corRest for a new request // -extern void corRestStateInit(struct MHD_Connection* connection, const char* url, const char* method); +// 'connection' is the HTTP backend's own handle for the connection, stored +// untouched in corRest.connection - see CorRestState.h. NULL for a request that +// arrived on no connection at all (an in-process self-forward). +// +extern void corRestStateInit(void* connection, const char* url, const char* method); + + + +// ----------------------------------------------------------------------------- +// +// corRestUrlPathNormalize - length, and a single trailing '/' removed +// +// Called by corRestStateInit, and AGAIN by a backend that has to percent-decode +// the path itself: decoding changes the length and can uncover the trailing +// slash that was written `%2F`. Idempotent, which is what lets it be called +// twice rather than duplicated. +// +extern void corRestUrlPathNormalize(void); diff --git a/corRestStop.c b/corRestStop.c index edfb568..3718942 100644 --- a/corRestStop.c +++ b/corRestStop.c @@ -7,10 +7,10 @@ // SPDX-License-Identifier: Apache-2.0 // #include // free -#include #include "corRest/CorRestVerb.h" // CorVerbs #include "corRest/CorRestService.h" // CorRestServiceVector +#include "corRest/corRestBackend.h" // corRestBackendStop, corRestWorkerPoolStop @@ -19,8 +19,6 @@ // Globals (defined in corRestInit.c) // extern CorRestServiceVector corRestServiceV[]; -extern struct MHD_Daemon* corRestDaemon; -extern void corRestWorkerPoolStop(void); @@ -30,16 +28,13 @@ extern void corRestWorkerPoolStop(void); // void corRestStop(void) { - // Drain + join the async worker pool BEFORE stopping the daemon: this clears + // Drain + join the async worker pool BEFORE stopping the server: this clears // any suspended connections (resuming across MHD_stop_daemon is an API - // violation) and stops new suspensions. + // violation, and the built-in server would be tearing down a connection a + // worker still holds) and stops new suspensions. corRestWorkerPoolStop(); - if (corRestDaemon != NULL) - { - MHD_stop_daemon(corRestDaemon); - corRestDaemon = NULL; - } + corRestBackendStop(); for (int verb = 0; verb < CorVerbs; verb++) { diff --git a/makefile b/makefile index f9d8eae..3d43301 100644 --- a/makefile +++ b/makefile @@ -51,24 +51,34 @@ DFLAGS = # In CFLAGS and not DFLAGS on purpose - see the note above: a caller who passes # DFLAGS on the command line would drop these along with every other default. # +# The value picks the backend SOURCE FILE as well as the defines: the two +# corRestBackend*.c files are alternatives, and compiling the unused one would +# need the library it exists to avoid. +# COR_HTTP_SERVER ?= mhd ifeq ($(COR_HTTP_SERVER),mhd) - HTTP_SERVER_FLAGS = -DCOR_HTTP_SERVER_MHD=1 -DCOR_HTTP_SERVER_BUILTIN=0 + HTTP_SERVER_FLAGS = -DCOR_HTTP_SERVER_MHD=1 -DCOR_HTTP_SERVER_BUILTIN=0 + HTTP_SERVER_SOURCE = corRestBackendMhd.c + HTTP_SERVER_LIBS = -lmicrohttpd + HTTP_SERVER_ARCHIVE = else ifeq ($(COR_HTTP_SERVER),builtin) + HTTP_SERVER_FLAGS = -DCOR_HTTP_SERVER_MHD=0 -DCOR_HTTP_SERVER_BUILTIN=1 + HTTP_SERVER_SOURCE = corRestBackendBuiltin.c + HTTP_SERVER_LIBS = # - # Refused rather than built: the flags below are right, and the server backend - # they select does not exist in this repo yet, so a build would fail at link - # with a pile of missing MHD symbols and no clue why. The switch says what it - # is waiting for instead. + # corHttp is a sibling repo and a static archive, like every other lib here. + # It is NOT folded into libcorRest.a - an archive cannot contain another one - + # so a consumer linking the built-in flavour links both, which is what + # coraine's CMakeLists does. # - $(error COR_HTTP_SERVER=builtin: the built-in HTTP server is not wired into this library yet - use 'mhd') - HTTP_SERVER_FLAGS = -DCOR_HTTP_SERVER_MHD=0 -DCOR_HTTP_SERVER_BUILTIN=1 + HTTP_SERVER_ARCHIVE = ../corHttp/libcorHttp.a else $(error COR_HTTP_SERVER must be 'mhd' or 'builtin', not '$(COR_HTTP_SERVER)') endif CFLAGS = -O2 -Wall -Werror -Wundef -fPIC -Wno-unused-function -fstack-protector-all $(DFLAGS) $(HTTP_SERVER_FLAGS) $(INCLUDE) -MMD -MP $(EXTRA_CFLAGS) LIB_SOURCES = corRestInit.c \ + $(HTTP_SERVER_SOURCE) \ corMimeType.c \ corRestStop.c \ corRestStateInit.c \ @@ -126,10 +136,10 @@ TEST_SOURCES = corRestTest.c TEST_OBJS = $(addprefix $(OBJDIR)/,$(TEST_SOURCES:.c=.o)) SO_LDFLAGS = -L../kalloc -L../kjson -L../kbase -L../klog -L../ktrace -SO_LIBS = -lkalloc -lkjson -lklog -lktrace -lkbase -lmicrohttpd -lssl -lcrypto -lpthread +SO_LIBS = -lkalloc -lkjson -lklog -lktrace -lkbase $(HTTP_SERVER_ARCHIVE) $(HTTP_SERVER_LIBS) -lssl -lcrypto -lpthread SO_RPATH = -Wl,-rpath,'$$ORIGIN/../kalloc:$$ORIGIN/../kjson:$$ORIGIN/../kbase:$$ORIGIN/../klog:$$ORIGIN/../ktrace' -LIBS = ../kalloc/libkalloc.a ../kjson/libkjson.a ../klog/libklog.a ../ktrace/libktrace.a ../kbase/libkbase.a -lmicrohttpd -lssl -lcrypto -lpthread -lm +LIBS = ../kalloc/libkalloc.a ../kjson/libkjson.a ../klog/libklog.a ../ktrace/libktrace.a ../kbase/libkbase.a $(HTTP_SERVER_ARCHIVE) $(HTTP_SERVER_LIBS) -lssl -lcrypto -lpthread -lm # # Built per flavour, then STAGED to the repo root where every consumer expects @@ -182,7 +192,14 @@ $(LIB): $(OBJDIR)/$(LIB) $(LIB_SO): $(OBJDIR)/$(LIB_SO) @cp -f $< $@ +# +# Removed and rebuilt, never updated in place. `ar r` REPLACES and ADDS but +# never removes, so an archive built once with corRestBackendMhd.o keeps it +# after a switch to the built-in backend - and the link then has two definitions +# of every corRestBackend* function, one of which wants libmicrohttpd. +# $(OBJDIR)/$(LIB): $(LIB_OBJS) + @rm -f $@ ar r $@ $(LIB_OBJS) ranlib $@