From dd4922767aa9fd637feb7114c31c17ae058a2f67 Mon Sep 17 00:00:00 2001 From: subhobhai943 Date: Thu, 27 Aug 2026 02:21:48 +0530 Subject: [PATCH] feat(net,gui,sched): async HTTP client + Web browser app, and the scheduler fix that makes background work possible The visible goal was a Web app: type a URL, see the page, fetched over the TCP stack SUB-OS built. Getting there surfaced -- and forced fixes to -- two deeper problems. Async HTTP client (net/http_client.c) tcp_connect/recv all block, so fetching from the GUI would freeze the compositor. Fetches now run on one long-lived kernel worker thread that services a single-slot mailbox; the requester posts a job and polls its state. Ownership follows the state, and the two hand-off points (worker completion, requester release) run under interrupts-off, so a window can close mid-fetch with neither leak nor use-after-free. do_fetch returns its terminal state rather than publishing it, closing a window where the requester could free the job before finalize read it. Cooperative scheduling for the idle thread (the core fix) The login prompt, the shell and the GUI compositor all run on the pid-0 idle thread, which the scheduler by design never preempts (sched_tick skips pid 0) and never requeues on a yield (weave_on clear). So it could neither be preempted to let the worker run, nor cleanly yield -- the worker was simply starved, and an early attempt to yield from it froze the shell outright. The idle thread now joins the weave rotation once hardware interrupts are live, and the network wait loops (tcp, dns, ping, the worker's idle) yield through a new net_wait() instead of halting the CPU. The compositor spends its frame-budget slack yielding too. Result: a background fetch progresses while the desktop holds 54 FPS. Web app (gui/gui_app_web.c) A text browser: URL bar, Go, and a Raw/Reader toggle. Raw shows the response verbatim; Reader strips markup (dropping and bodies entirely. + if (strncmp(tag, "script", 6) == 0 || strncmp(tag, "style", 5) == 0) { + const char* close = (tag[0] == 's' && tag[1] == 'c') ? "= cap - 1) break; } + out[o++] = ch; + at_line_start = false; + } + + out[o] = '\0'; + return o; +} + +// =========================================================================== +// Line breaking +// =========================================================================== +static void web_build_lines(web_data_t* wd, int cpl) { + wd->nlines = 0; + if (cpl < 8) cpl = 8; + + int n = wd->text_len; + const char* t = wd->text; + int i = 0; + + while (i < n && wd->nlines < WEB_MAX_LINES) { + int start = i; + int last_space = -1; + int col = 0; + + while (i < n && t[i] != '\n' && col < cpl) { + if (t[i] == ' ') last_space = i; + i++; + col++; + } + + int end; + if (i < n && t[i] == '\n') { + end = i; + i++; // consume the newline + } else if (col >= cpl && i < n) { + if (last_space > start) { // wrap at the last space + end = last_space; + i = last_space + 1; + } else { + end = i; // a word longer than a line + } + } else { + end = i; // reached the end of the text + } + + wd->offsets[wd->nlines] = start; + wd->lengths[wd->nlines] = end - start; + wd->nlines++; + } +} + +// Rebuild the processed text and line index if any input changed. +static void web_refresh_cache(web_data_t* wd, int cpl) { + http_fetch_t* j = wd->job; + if (!j || j->state != HTTP_STATE_DONE) { wd->nlines = 0; return; } + + if (wd->cache_src == j->buf && wd->cache_len == j->len && + wd->cache_reader == wd->reader && wd->cache_cpl == cpl) { + return; // still valid + } + + if (wd->reader) { + // Reader view: skip the headers, then strip markup from the body. + const char* body = j->buf; + char* sep = strstr(j->buf, "\r\n\r\n"); + if (sep) body = sep + 4; + int blen = j->len - (int)(body - j->buf); + if (blen < 0) blen = 0; + wd->text_len = html_to_text(body, blen, wd->text, WEB_FETCH_CAP); + } else { + // Raw view: the response verbatim, with tabs normalised. + int n = j->len < WEB_FETCH_CAP - 1 ? j->len : WEB_FETCH_CAP - 1; + for (int k = 0; k < n; k++) { + char c = j->buf[k]; + wd->text[k] = (c == '\t') ? ' ' : c; + } + wd->text[n] = '\0'; + wd->text_len = n; + } + + web_build_lines(wd, cpl); + + wd->cache_src = j->buf; + wd->cache_len = j->len; + wd->cache_reader = wd->reader; + wd->cache_cpl = cpl; + if (wd->scroll > wd->nlines - 1) wd->scroll = 0; +} + +// =========================================================================== +// Fetching +// =========================================================================== +static void web_go(web_data_t* wd) { + if (wd->url[0] == '\0') return; + + // Drop any previous job (releasing a still-running one hands it to the + // worker to free) and invalidate the render cache. + if (wd->job) { http_fetch_release(wd->job); wd->job = NULL; } + wd->cache_src = NULL; + wd->scroll = 0; + + wd->job = http_fetch_start(wd->url, WEB_FETCH_CAP); + // A NULL result (worker busy or bad URL) is reflected in the status line + // drawn from the live job state, so nothing more to do here. +} + +// =========================================================================== +// Paint +// =========================================================================== +static void web_paint(gui_window_t* win) { + web_data_t* wd = (web_data_t*)win->user_data; + if (!wd) return; + + gui_widget_begin(win); + int w = gui_widget_client_width(); + int h = gui_widget_client_height(); + int ox = win->x + 1; + int oy = win->y + GUI_TITLEBAR_HEIGHT + 1; + if (w < 240 || h < 120) { gui_widget_end(); return; } + + // --- toolbar ---------------------------------------------------------- + int go_w = 40, mode_w = 68; + int field_w = w - 20 - go_w - mode_w - 12; + if (gui_textfield(WEB_URL_FIELD, 10, 8, field_w, wd->url, sizeof(wd->url))) { + /* editing; nothing to do until Go */ + } + + // Enter in the focused URL field triggers the fetch: the field leaves + // newline keys unconsumed, so they are still visible here. + const gui_widget_input_t* in = gui_widget_get_input(); + if (in->focus_id == WEB_URL_FIELD && (in->key == '\n' || in->key == '\r')) { + gui_widget_set_focus(0); + web_go(wd); + } + + if (gui_button(2, 10 + field_w + 4, 8, go_w, 20, "Go")) web_go(wd); + + if (gui_button_colored(3, 10 + field_w + 4 + go_w + 4, 8, mode_w, 20, + wd->reader ? "Reader" : "Raw", + wd->reader ? GUI_THEME_SUCCESS : GUI_THEME_ACCENT)) { + wd->reader = !wd->reader; + wd->cache_src = NULL; // force a re-render in the new view + wd->scroll = 0; + } + + // --- status line ------------------------------------------------------ + int bar_y = 34; + char status[128]; + uint32_t status_col = GUI_THEME_TEXT_MUTED; + + if (!wd->job) { + snprintf(status, sizeof(status), "Enter a URL and press Go (e.g. 10.0.2.2:8000/)"); + } else if (wd->job->state == HTTP_STATE_RUNNING) { + // A small spinner off the 10 Hz repaint so a slow fetch looks alive. + static const char frames[] = { '|', '/', '-', '\\' }; + char sp = frames[(ktime_ms() / 120) % 4]; + snprintf(status, sizeof(status), "%c fetching %s ...", sp, wd->job->host); + status_col = GUI_THEME_WARNING; + } else if (wd->job->state == HTTP_STATE_ERROR) { + snprintf(status, sizeof(status), "error: %s", wd->job->err); + status_col = GUI_THEME_DANGER; + } else { + char ips[16]; + ip_to_str(wd->job->ip, ips); + snprintf(status, sizeof(status), "HTTP %d %d bytes %u ms %s:%u (%s)", + wd->job->status_code, wd->job->len, wd->job->elapsed_ms, + wd->job->host, wd->job->port, ips); + status_col = (wd->job->status_code == 200) ? GUI_THEME_SUCCESS : GUI_THEME_TEXT_MAIN; + } + gui_label(10, bar_y, status, status_col); + gui_separator(10, bar_y + 14, w - 20); + + // --- page body -------------------------------------------------------- + int view_x = 10; + int view_y = bar_y + 20; + int view_h = h - view_y - 6; + int line_h = 10; + int cpl = (w - 20 - 12) / GUI_FONT_W; // leave room for the scrollbar + if (cpl < 8) cpl = 8; + int rows = view_h / line_h; + if (rows < 1) rows = 1; + + gui_gfx_fill_rect(ox + view_x, oy + view_y, w - 20, view_h, GUI_THEME_BG_DARK); + gui_gfx_draw_rect(ox + view_x, oy + view_y, w - 20, view_h, GUI_THEME_BORDER); + + web_refresh_cache(wd, cpl); + + if (wd->job && wd->job->state == HTTP_STATE_DONE && wd->nlines > 0) { + if (wd->scroll > wd->nlines - 1) wd->scroll = wd->nlines - 1; + if (wd->scroll < 0) wd->scroll = 0; + + char line[200]; + for (int r = 0; r < rows; r++) { + int idx = wd->scroll + r; + if (idx >= wd->nlines) break; + + int len = wd->lengths[idx]; + if (len > (int)sizeof(line) - 1) len = (int)sizeof(line) - 1; + memcpy(line, wd->text + wd->offsets[idx], len); + line[len] = '\0'; + + // In raw view the header block (up to the blank line) is dimmed so + // the body stands out; reader view is uniform. + uint32_t col = GUI_THEME_TEXT_MAIN; + if (!wd->reader && idx < wd->nlines) { + if (len == 0) col = GUI_THEME_TEXT_DIM; + } + gui_gfx_draw_string(ox + view_x + 6, oy + view_y + 5 + r * line_h, + line, col); + } + + // Scrollbar and keyboard paging. + gui_scrollbar(4, w - 18, view_y, view_h, &wd->scroll, wd->nlines, rows); + + if (in->focus_id != WEB_URL_FIELD) { + if (in->key == KEY_DOWN) wd->scroll += 1; + else if (in->key == KEY_UP) wd->scroll -= 1; + else if (in->key == KEY_PAGE_DOWN) wd->scroll += rows; + else if (in->key == KEY_PAGE_UP) wd->scroll -= rows; + } + } else if (wd->job && wd->job->state == HTTP_STATE_RUNNING) { + gui_gfx_draw_string(ox + view_x + 8, oy + view_y + 10, + "Loading over TCP -- the desktop stays responsive.", + GUI_THEME_TEXT_DIM); + } + + gui_widget_end(); +} + +static void web_event(gui_window_t* win, const gui_event_t* ev) { + gui_widget_feed_event(win, ev); + + if (ev->type == GUI_EVENT_CLOSE && win->user_data) { + web_data_t* wd = (web_data_t*)win->user_data; + if (wd->job) http_fetch_release(wd->job); // safe mid-fetch + if (wd->text) kfree(wd->text); + kfree(wd); + win->user_data = NULL; + } +} + +void gui_app_web_launch(int x, int y, int w, int h) { + gui_window_t* win = gui_wm_create_window("Web", x, y, + (w > 0) ? w : 600, (h > 0) ? h : 420); + if (!win) return; + + web_data_t* wd = (web_data_t*)kzalloc(sizeof(web_data_t)); + if (!wd) { gui_wm_destroy_window(win->id); return; } + + wd->text = (char*)kzalloc(WEB_FETCH_CAP); + if (!wd->text) { kfree(wd); gui_wm_destroy_window(win->id); return; } + + wd->reader = true; + strcpy(wd->url, "10.0.2.2:8000/"); // the host, reachable under QEMU slirp + + win->user_data = wd; + win->paint = web_paint; + win->handle_event = web_event; +} diff --git a/gui/gui_desktop.c b/gui/gui_desktop.c index 78b4440..3ab4f25 100644 --- a/gui/gui_desktop.c +++ b/gui/gui_desktop.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -66,6 +67,7 @@ static const desktop_app_t g_apps[] = { { "Analytics", GUI_ICON_MONITOR, gui_app_analytics_launch,320, 70, 560, 400 }, { "Sorry", GUI_ICON_HEART, gui_app_sorry_launch, 260, 60, 660, 470 }, { "Network", GUI_ICON_NETWORK, gui_app_netmon_launch, 300, 80, 560, 400 }, + { "Web", GUI_ICON_GLOBE, gui_app_web_launch, 280, 70, 600, 420 }, { "About", GUI_ICON_INFO, gui_app_about_launch, 240, 140, 330, 170 }, }; #define APP_COUNT ((int)(sizeof(g_apps) / sizeof(g_apps[0]))) @@ -74,7 +76,7 @@ static const desktop_app_t g_apps[] = { // Indices into g_apps[]: Terminal, Files, Monitor, Kernel Log, KTest, Life, // Rust Lab, Settings, Analytics, Sorry. (Objects/RegEdit live in the start // menu.) Network sits alongside them. -static const int g_desktop_icons[] = { 0, 1, 2, 4, 5, 9, 10, 14, 15, 16, 17 }; +static const int g_desktop_icons[] = { 0, 1, 2, 4, 5, 9, 10, 14, 15, 16, 17, 18 }; #define DESKTOP_ICON_COUNT ((int)(sizeof(g_desktop_icons) / sizeof(g_desktop_icons[0]))) static bool g_start_menu_open = false; @@ -853,9 +855,13 @@ int gui_desktop_run(void) { g_prev_my = my; // --- 5. Pace to the 60 FPS budget ----------------------------------- - uint64_t elapsed_us = ktime_us() - frame_start_us; - if (elapsed_us < FRAME_BUDGET_US) { - ktime_delay_us(FRAME_BUDGET_US - elapsed_us); + // Spend the slack yielding to other kernel threads rather than + // busy-spinning, so a background task -- the async HTTP worker behind + // the Web app above all -- makes progress while the desktop idles + // between frames. When nothing else is runnable each yield returns at + // once and this simply spins out the budget, exactly as before. + while (ktime_us() - frame_start_us < FRAME_BUDGET_US) { + sched_yield(); } // Sample the frame rate roughly twice a second from the monotonic clock. diff --git a/gui/gui_icons.c b/gui/gui_icons.c index 1c03998..8dd5dee 100644 --- a/gui/gui_icons.c +++ b/gui/gui_icons.c @@ -333,12 +333,31 @@ static const char* g_icons[GUI_ICON_COUNT][GUI_ICON_SIZE] = { "0000556666550000", "0000005555000000", "0000000000000000", + }, + // GUI_ICON_GLOBE + { + "0000055555500000", + "0003556555655300", + "0035565556555300", + "0355556555655530", + "0355565556555530", + "0355556555655530", + "3555565556555553", + "3666666666666663", + "3555565556555553", + "0355556555655530", + "0355565556555530", + "0355556555655530", + "0035565556555300", + "0003556555655300", + "0000055555500000", + "0000000000000000", },}; static const char* g_icon_names[GUI_ICON_COUNT] = { "Terminal", "Monitor", "Folder", "Calculator", "Paint", "Info", "Settings", "Tasks", "Log", "Tests", "Clock", "File", "Power", "Warning", - "Heart", "Network" + "Heart", "Network", "Globe" }; static uint8_t hex_index(char c) { diff --git a/include/gui/gui_apps.h b/include/gui/gui_apps.h index 97490f0..b312dd5 100644 --- a/include/gui/gui_apps.h +++ b/include/gui/gui_apps.h @@ -16,5 +16,6 @@ void gui_app_regedit_launch(int x, int y, int w, int h); void gui_app_analytics_launch(int x, int y, int w, int h); void gui_app_sorry_launch(int x, int y, int w, int h); void gui_app_netmon_launch(int x, int y, int w, int h); +void gui_app_web_launch(int x, int y, int w, int h); #endif // _GUI_APPS_H diff --git a/include/gui/gui_icons.h b/include/gui/gui_icons.h index 03141a9..b891655 100644 --- a/include/gui/gui_icons.h +++ b/include/gui/gui_icons.h @@ -27,6 +27,7 @@ typedef enum { GUI_ICON_WARNING, GUI_ICON_HEART, GUI_ICON_NETWORK, + GUI_ICON_GLOBE, GUI_ICON_COUNT } gui_icon_id_t; diff --git a/include/net/http_client.h b/include/net/http_client.h new file mode 100644 index 0000000..e46342d --- /dev/null +++ b/include/net/http_client.h @@ -0,0 +1,65 @@ +#ifndef _NET_HTTP_CLIENT_H +#define _NET_HTTP_CLIENT_H + +#include +#include +#include + +// Asynchronous HTTP/1.0 client. +// +// The TCP client calls (tcp_connect/tcp_send/tcp_recv) all busy-wait against +// the timer tick, so calling them from an interactive context -- the GUI +// compositor above all -- would freeze it for the length of the fetch. This +// module runs every fetch on one long-lived kernel worker thread instead: the +// requester posts a job and polls its state, and the compositor keeps painting +// at full rate while the worker blocks. +// +// Ownership follows the state. While a job is HTTP_STATE_RUNNING the worker +// owns it; once it is DONE or ERROR the requester owns it again. The requester +// hands a job back with http_fetch_release, which frees it now if the fetch has +// finished, or marks it abandoned so the worker frees it when it does -- so a +// window can close mid-fetch without leaking or use-after-free. + +typedef enum { + HTTP_STATE_RUNNING = 0, // worker owns the job + HTTP_STATE_DONE, // completed; buf holds the response + HTTP_STATE_ERROR // failed; err holds a short reason +} http_fetch_state_t; + +typedef struct http_fetch { + // Request, filled in by http_fetch_start from the parsed URL. + char host[128]; + char path[256]; + uint16_t port; + + // Result. `state` is the only field the requester may read while the fetch + // is in flight; everything else is valid once state != RUNNING. + volatile int state; // http_fetch_state_t + volatile bool abandoned; // requester released it mid-flight; worker frees + + char* buf; // response bytes (status line, headers, body) + int len; // bytes stored + int cap; // buffer capacity + + uint32_t ip; // resolved peer address + int status_code; // parsed HTTP status, 0 if not found + uint32_t elapsed_ms; // wall time of the fetch + char err[64]; // human-readable failure reason +} http_fetch_t; + +// Start the worker thread. Call once at boot, after the TCP stack is up. +void http_client_init(void); + +// Begin fetching `url` (forms: "host", "host/path", "host:port/path", with an +// optional "http://" prefix). `cap` bounds the stored response. Returns a job +// to poll, or NULL if the URL is unusable or a fetch is already in flight. +http_fetch_t* http_fetch_start(const char* url, int cap); + +// Give a job back. Frees it if the fetch has finished; otherwise marks it so +// the worker frees it on completion. The pointer must not be used afterwards. +void http_fetch_release(http_fetch_t* f); + +// True while the worker thread is busy with a fetch. +bool http_client_busy(void); + +#endif // _NET_HTTP_CLIENT_H diff --git a/include/net/net.h b/include/net/net.h index cb2f70d..58a84b7 100644 --- a/include/net/net.h +++ b/include/net/net.h @@ -103,6 +103,15 @@ arp_entry_t* net_get_arp_table(int* count_out); uint32_t ip_parse(const char* ip_str); void ip_to_str(uint32_t ip, char* buf); +// Yield the CPU once while blocked in a network wait loop (a handshake, a +// receive, a ping reply). Inbound packets are delivered from the NIC interrupt +// regardless of which task runs, so cooperatively handing the CPU to other +// tasks -- the GUI compositor above all -- keeps the desktop live during a +// fetch instead of freezing it for the duration. When the caller is the only +// runnable task it returns at once, so a shell-context wait still just spins to +// its timeout. +void net_wait(void); + // One's-complement checksum, summed a byte at a time so the loads cannot be // reordered around a caller's store into the buffer being summed, and so no // alignment is assumed of a packed header. Words are read big-endian; diff --git a/include/net/tcp.h b/include/net/tcp.h index a7d4e9d..69156ed 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -139,4 +139,13 @@ size_t tcp_get_connections_count(void); const tcp_conn_t* tcp_get_connection(size_t index); const char* tcp_state_name(uint8_t state); +// Size of the connection table, so callers iterate the right range rather than +// a hardcoded constant. +int tcp_conn_table_size(void); + +// Cumulative count of connections that have reached ESTABLISHED, and the +// high-water mark of simultaneously-live table entries. Diagnostic. +uint64_t tcp_get_total_opened(void); +int tcp_get_peak_connections(void); + #endif // _NET_TCP_H diff --git a/kernel/main.c b/kernel/main.c index 56acb96..17f2d48 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -66,6 +66,7 @@ #include #include #include +#include #include #include #include @@ -285,6 +286,25 @@ static void subos_modular_core_boot(void) { * threads are forced to share the CPU by the tick. Preemption is left armed * afterwards, so CPU-bound kernel threads are time-sliced from here on. */ weave_preempt_selftest(); + + // The async HTTP client runs on its own kernel thread, which idles on hlt + // between fetches. It is started only now, once hardware interrupts are + // live -- earlier, and its first idle would hlt with no timer tick to wake + // it -- and after the deterministic scheduler self-tests, so it never joins + // their fixed thread set. Harmless with no NIC: a fetch then fails at + // connect. It stays dormant until the GUI Web app requests a page. + http_client_init(); + + // The rest of the system -- the login prompt, the shell, the GUI + // compositor -- has run on the pid-0 idle thread, which the scheduler + // neither preempts (sched_tick skips pid 0) nor requeues on a yield + // (weave_on is clear). That was fine while nothing else needed the CPU, but + // now a background worker (the async HTTP client) does. Joining the weave + // rotation makes this thread a first-class participant: a sched_yield from + // the compositor's idle slack requeues it and lets the worker run, and + // control comes straight back on the next rotation. Without this a yield + // here would switch away for good. + sched_join(); #endif // Boot directly into the GUI Desktop unless emergency text mode is requested. diff --git a/net/core/net.c b/net/core/net.c index b50745e..60a445a 100644 --- a/net/core/net.c +++ b/net/core/net.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #define MAX_ARP_ENTRIES 16 @@ -321,7 +322,7 @@ int net_ping(uint32_t target_ip, uint32_t count, uint32_t timeout_ms) { if (wait_ticks == 0) wait_ticks = 100; while (!ping_received && (pit_get_ticks() - t_start < wait_ticks)) { - arch_halt(); + net_wait(); } if (ping_received && ping_id == p_id && ping_seq == seq) { @@ -341,6 +342,15 @@ int net_ping(uint32_t target_ip, uint32_t count, uint32_t timeout_ms) { return received; } +void net_wait(void) { + // A cooperative hand-off rather than a CPU halt: other runnable tasks get + // to run while this one blocks on the wire. If nothing else is runnable the + // scheduler returns immediately, so the enclosing timeout still bounds the + // wait. The NIC interrupt fills the receive path independently of which + // task holds the CPU, so yielding never delays packet delivery. + sched_yield(); +} + void net_init(void) { strcpy(primary_if.name, "eth0"); e1000_get_mac(primary_if.mac); diff --git a/net/http_client.c b/net/http_client.c new file mode 100644 index 0000000..70e66b2 --- /dev/null +++ b/net/http_client.c @@ -0,0 +1,247 @@ +/* + * Asynchronous HTTP/1.0 client for SUB-OS. + * + * One worker thread, created at boot, services a single-slot mailbox. A + * requester parses a URL into a job, drops it in the mailbox, and polls the + * job's state; the worker performs the blocking DNS + TCP fetch and marks the + * job DONE or ERROR. Because the blocking happens on the worker rather than in + * the caller, the GUI compositor keeps running at full frame rate during a + * fetch that may take seconds. + * + * See http_client.h for the ownership rules; the two critical sections that + * enforce them (worker completion and requester release) run with interrupts + * disabled, which on this single-CPU kernel makes the hand-off indivisible. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define HTTP_CONNECT_TIMEOUT_MS 4000 +#define HTTP_RECV_TIMEOUT_MS 3000 + +static task_t* worker_task = NULL; +static http_fetch_t* volatile mailbox = NULL; // job awaiting the worker + +// =========================================================================== +// URL parsing +// +// Accepts host, host/path, host:port/path, with an optional http:// prefix. +// Anything it cannot make sense of leaves the job unusable and is reported as +// an error rather than guessed at. +// =========================================================================== +static void parse_url(const char* url, http_fetch_t* f) { + f->host[0] = '\0'; + strcpy(f->path, "/"); + f->port = 80; + + if (!url) return; + if (strncmp(url, "http://", 7) == 0) url += 7; + else if (strncmp(url, "https://", 8) == 0) url += 8; // no TLS; try plain + + // Host (and optional :port) up to the first '/'. + int i = 0; + char hostport[192]; + while (url[i] && url[i] != '/' && i < (int)sizeof(hostport) - 1) { + hostport[i] = url[i]; + i++; + } + hostport[i] = '\0'; + + const char* rest = url + i; // "" or "/path..." + + // Split host:port. + char* colon = strchr(hostport, ':'); + if (colon) { + *colon = '\0'; + int p = 0; + for (const char* c = colon + 1; *c >= '0' && *c <= '9'; c++) p = p * 10 + (*c - '0'); + if (p > 0 && p < 65536) f->port = (uint16_t)p; + } + strncpy(f->host, hostport, sizeof(f->host) - 1); + f->host[sizeof(f->host) - 1] = '\0'; + + if (rest[0] == '/') { + strncpy(f->path, rest, sizeof(f->path) - 1); + f->path[sizeof(f->path) - 1] = '\0'; + } +} + +// Parse the numeric status out of "HTTP/1.x NNN ...", or 0 if absent. +static int parse_status(const char* buf, int len) { + if (len < 12 || strncmp(buf, "HTTP/", 5) != 0) return 0; + const char* p = buf; + const char* end = buf + len; + while (p < end && *p != ' ') p++; // skip "HTTP/1.x" + while (p < end && *p == ' ') p++; + int code = 0; + while (p < end && *p >= '0' && *p <= '9') code = code * 10 + (*p++ - '0'); + return code; +} + +// =========================================================================== +// The fetch itself, run on the worker thread. +// +// It fills in the result fields but deliberately does NOT touch f->state: +// publishing the terminal state is the finalize step's job, done under +// interrupts-off. If do_fetch flipped the state itself, the requester could +// observe DONE and free the job in the instant before finalize ran, and +// finalize would then read freed memory. Returning the state instead keeps the +// job private to the worker until the atomic hand-off. +// =========================================================================== +static int do_fetch(http_fetch_t* f) { + uint64_t t0 = pit_get_ticks(); + + // Resolve: a dotted quad is taken as-is, otherwise the DNS resolver runs + // (itself bounded, so an unreachable server times out rather than hangs). + f->ip = ip_parse(f->host); + if (f->ip == 0) f->ip = dns_resolve(f->host); + if (f->ip == 0) { + snprintf(f->err, sizeof(f->err), "cannot resolve %s", f->host); + return HTTP_STATE_ERROR; + } + + tcp_conn_t* c = tcp_connect(f->ip, f->port, HTTP_CONNECT_TIMEOUT_MS); + if (!c) { + snprintf(f->err, sizeof(f->err), "connect to :%u refused/timed out", f->port); + return HTTP_STATE_ERROR; + } + + char req[512]; + int rl = snprintf(req, sizeof(req), + "GET %s HTTP/1.0\r\nHost: %s\r\nUser-Agent: SUB-OS/webfetch\r\n" + "Connection: close\r\n\r\n", f->path, f->host); + if (tcp_send(c, req, (uint16_t)rl) < 0) { + snprintf(f->err, sizeof(f->err), "send failed"); + tcp_close(c); + return HTTP_STATE_ERROR; + } + + // Drain the response until the peer closes or the buffer fills. + f->len = 0; + while (f->len < f->cap - 1) { + int n = tcp_recv(c, f->buf + f->len, (uint16_t)(f->cap - 1 - f->len), + HTTP_RECV_TIMEOUT_MS); + if (n <= 0) break; + f->len += n; + } + f->buf[f->len] = '\0'; + tcp_close(c); + + f->status_code = parse_status(f->buf, f->len); + f->elapsed_ms = (uint32_t)((pit_get_ticks() - t0) * 10); + + if (f->len == 0) { + snprintf(f->err, sizeof(f->err), "no data received"); + return HTTP_STATE_ERROR; + } + return HTTP_STATE_DONE; +} + +static void free_job(http_fetch_t* f) { + if (!f) return; + if (f->buf) kfree(f->buf); + kfree(f); +} + +// Publish the terminal state, or free the job if the requester walked away +// while the fetch was in flight. Interrupts are off across the decision so it +// serialises against http_fetch_release: exactly one of the two frees the job, +// and the requester never sees a non-RUNNING state for a job it has released. +static void finalize(http_fetch_t* f, int result) { + unsigned long fl = arch_irq_save(); + bool abandoned = f->abandoned; + if (!abandoned) f->state = result; + arch_irq_restore(fl); + + if (abandoned) free_job(f); +} + +static void http_worker_main(void* arg) { + (void)arg; + for (;;) { + unsigned long fl = arch_irq_save(); + http_fetch_t* job = mailbox; + mailbox = NULL; + arch_irq_restore(fl); + + if (!job) { + // Idle: yield rather than halt. Halting would hold the CPU until a + // timer tick and starve the compositor between ticks; a cooperative + // yield hands the CPU straight back and the two ping-pong cheaply. + net_wait(); + continue; + } + + int result = do_fetch(job); + finalize(job, result); + } +} + +// =========================================================================== +// Public API +// =========================================================================== +void http_client_init(void) { + mailbox = NULL; + worker_task = task_create("http-worker", http_worker_main, NULL, 0); + if (worker_task) { + printk(KERN_INFO "NET: async HTTP client worker online\n"); + } else { + printk(KERN_WARNING "NET: async HTTP client worker failed to start\n"); + } +} + +bool http_client_busy(void) { + return mailbox != NULL; +} + +http_fetch_t* http_fetch_start(const char* url, int cap) { + if (!worker_task || !url || cap < 64) return NULL; + if (mailbox) return NULL; // a fetch is already queued + + http_fetch_t* f = (http_fetch_t*)kzalloc(sizeof(http_fetch_t)); + if (!f) return NULL; + + f->buf = (char*)kzalloc(cap); + if (!f->buf) { kfree(f); return NULL; } + f->cap = cap; + f->state = HTTP_STATE_RUNNING; + + parse_url(url, f); + if (f->host[0] == '\0') { + kfree(f->buf); kfree(f); + return NULL; + } + + // Hand the job to the worker. Re-check the mailbox under interrupts-off so + // two near-simultaneous starts cannot both install a job. + unsigned long fl = arch_irq_save(); + if (mailbox) { arch_irq_restore(fl); kfree(f->buf); kfree(f); return NULL; } + mailbox = f; + arch_irq_restore(fl); + + return f; +} + +void http_fetch_release(http_fetch_t* f) { + if (!f) return; + + unsigned long fl = arch_irq_save(); + if (f->state == HTTP_STATE_RUNNING) { + // The worker still owns it (queued or in flight): let it free the job + // when it finishes rather than pulling the memory out from under it. + f->abandoned = true; + arch_irq_restore(fl); + return; + } + arch_irq_restore(fl); + + free_job(f); +} diff --git a/net/ipv4/dns.c b/net/ipv4/dns.c index 79be4af..285f491 100644 --- a/net/ipv4/dns.c +++ b/net/ipv4/dns.c @@ -163,7 +163,7 @@ uint32_t dns_resolve(const char* hostname) { uint64_t wait_ticks = DNS_TIMEOUT_MS / 10; while (!dns_reply_ready && (pit_get_ticks() - t0 < wait_ticks)) { - arch_halt(); + net_wait(); } if (dns_reply_ready && dns_reply_id == qid && dns_reply_ip != 0) { diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 4cd9322..ff0f1d4 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -35,7 +35,7 @@ #include #include -#define MAX_TCP_CONNS 32 +#define MAX_TCP_CONNS 64 // The timer tick is 100 Hz, so one tick is 10 ms. #define TCP_RTO_TICKS 50 // 500 ms retransmission timeout @@ -43,6 +43,13 @@ #define TCP_CLOSE_TICKS 50 // how long tcp_close waits for the shutdown #define TCP_TIME_WAIT_TICKS 200 // 2 s; a deliberately shortened 2*MSL +// A connection this end closed after serving one request (httpd/sshd) does not +// need the full linger: we sent the FIN, the peer is expected to be a +// short-lived client, and holding the slot for two seconds is what let a burst +// of requests exhaust the table. A brief linger still absorbs a retransmitted +// FIN while freeing the slot roughly ten times sooner. +#define TCP_SVC_TIME_WAIT_TICKS 25 // 250 ms + #define TCP_EPHEMERAL_LO 49152 #define TCP_EPHEMERAL_HI 65535 @@ -60,6 +67,17 @@ typedef struct { static tcp_listener_t tcp_listeners[TCP_MAX_LISTENERS]; +// Cumulative connections that have reached ESTABLISHED, and the high-water mark +// of simultaneously-live table entries. Diagnostic only. +static uint64_t tcp_total_opened = 0; +static int tcp_peak_conns = 0; + +static void tcp_note_peak(void) { + int live = 0; + for (size_t i = 0; i < MAX_TCP_CONNS; i++) if (tcp_connections[i].in_use) live++; + if (live > tcp_peak_conns) tcp_peak_conns = live; +} + static uint16_t tcp_checksum(uint32_t src_ip, uint32_t dst_ip, const tcp_header_t* tcp, const void* payload, uint16_t payload_len) { uint16_t tcp_total_len = (uint16_t)(sizeof(tcp_header_t) + payload_len); @@ -175,10 +193,11 @@ static void reap_time_wait(void) { uint64_t now = pit_get_ticks(); for (size_t i = 0; i < MAX_TCP_CONNS; i++) { tcp_conn_t* c = &tcp_connections[i]; - if (c->in_use && c->state == TCP_STATE_TIME_WAIT && - (now - c->closed_at) >= TCP_TIME_WAIT_TICKS) { - conn_free(c); - } + if (!c->in_use || c->state != TCP_STATE_TIME_WAIT) continue; + + uint64_t linger = (c->role == TCP_ROLE_SERVICE) + ? TCP_SVC_TIME_WAIT_TICKS : TCP_TIME_WAIT_TICKS; + if ((now - c->closed_at) >= linger) conn_free(c); } } @@ -199,6 +218,7 @@ static tcp_conn_t* alloc_conn(void) { if (!tcp_connections[i].in_use) { memset(&tcp_connections[i], 0, sizeof(tcp_conn_t)); tcp_connections[i].in_use = true; + tcp_note_peak(); return &tcp_connections[i]; } } @@ -322,6 +342,7 @@ static void tcp_stream_input(tcp_conn_t* c, uint32_t seq, uint32_t ack, uint8_t c->snd_una = ack; c->ack_num = seq + 1; c->state = TCP_STATE_ESTABLISHED; + tcp_total_opened++; tcp_tx(c, TCP_FLAG_ACK, NULL, 0); } else if (flags & TCP_FLAG_SYN) { // Simultaneous open: both ends sent a SYN at once. @@ -336,6 +357,7 @@ static void tcp_stream_input(tcp_conn_t* c, uint32_t seq, uint32_t ack, uint8_t if ((flags & TCP_FLAG_ACK) && seq_ge(ack, c->iss + 1)) { c->snd_una = ack; c->state = TCP_STATE_ESTABLISHED; + tcp_total_opened++; // A passive open is only complete now: this is the third leg of // the handshake, so the connection can finally be handed out. if (c->role == TCP_ROLE_ACCEPTED) listener_enqueue(c); @@ -439,6 +461,7 @@ static void tcp_server_input(tcp_conn_t* c, uint32_t src_ip, uint16_t src_port, if (c->state == TCP_STATE_SYN_RCVD) { c->state = TCP_STATE_ESTABLISHED; + tcp_total_opened++; if (is_ssh_port && payload_len == 0) { const char* banner = SSH_BANNER; uint16_t blen = (uint16_t)strlen(banner); @@ -596,7 +619,7 @@ tcp_conn_t* tcp_connect(uint32_t dst_ip, uint16_t dst_port, uint32_t timeout_ms) tcp_tx_seq(c, c->iss, TCP_FLAG_SYN, NULL, 0); last = pit_get_ticks(); } - arch_halt(); + net_wait(); } conn_free(c); @@ -629,7 +652,7 @@ int tcp_send(tcp_conn_t* c, const void* data, uint16_t len) { uint64_t t0 = pit_get_ticks(); while ((pit_get_ticks() - t0) < TCP_RTO_TICKS) { if (seq_ge(c->snd_una, seg_end) || c->reset) break; - arch_halt(); + net_wait(); } if (seq_ge(c->snd_una, seg_end)) break; @@ -653,7 +676,7 @@ int tcp_recv(tcp_conn_t* c, void* buf, uint16_t len, uint32_t timeout_ms) { if (c->reset) return -1; if (c->peer_fin) return 0; // stream ended if ((pit_get_ticks() - t0) >= wait) return 0; // nothing arrived - arch_halt(); + net_wait(); } uint16_t before = c->rx_count; @@ -692,7 +715,7 @@ void tcp_close(tcp_conn_t* c) { while ((pit_get_ticks() - t0) < TCP_CLOSE_TICKS) { if (c->reset) break; if (c->state == TCP_STATE_CLOSED || c->state == TCP_STATE_TIME_WAIT) break; - arch_halt(); + net_wait(); } } @@ -746,7 +769,7 @@ tcp_conn_t* tcp_accept(uint16_t port, uint32_t timeout_ms) { while (l->count == 0) { if ((pit_get_ticks() - t0) >= wait) return NULL; - arch_halt(); + net_wait(); } tcp_conn_t* c = l->queue[0]; @@ -821,6 +844,10 @@ const tcp_conn_t* tcp_get_connection(size_t index) { return &tcp_connections[index]; } +int tcp_conn_table_size(void) { return MAX_TCP_CONNS; } +uint64_t tcp_get_total_opened(void) { return tcp_total_opened; } +int tcp_get_peak_connections(void) { return tcp_peak_conns; } + const char* tcp_state_name(uint8_t state) { static const char* const names[] = { "CLOSED", "LISTEN", "SYN_SENT", "SYN_RCVD", "ESTABLISHED", diff --git a/userland/lazybox/lazybox.c b/userland/lazybox/lazybox.c index f6f52ea..eb76c41 100644 --- a/userland/lazybox/lazybox.c +++ b/userland/lazybox/lazybox.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -2226,7 +2227,7 @@ static int applet_netstat(int argc, char** argv) { // Live connections from the TCP engine's own table: inbound ones accepted // by a listener, outbound ones opened by tcp_connect. - for (size_t i = 0; i < 32; i++) { + for (int i = 0; i < tcp_conn_table_size(); i++) { const tcp_conn_t* c = tcp_get_connection(i); if (!c) continue; @@ -2256,7 +2257,10 @@ static int applet_netstat(int argc, char** argv) { udp_get_stats(&urx, &utx, &udrop); printk(ANSI_YELLOW "\nTransport statistics:\n" ANSI_RESET); printk(" udp: %llu datagrams received, %llu sent, %llu dropped\n", urx, utx, udrop); - printk(" tcp: %u active connection(s)\n", (unsigned)tcp_get_connections_count()); + printk(" tcp: %u active, %llu served since boot, peak %d of %d slots\n", + (unsigned)tcp_get_connections_count(), + (unsigned long long)tcp_get_total_opened(), + tcp_get_peak_connections(), tcp_conn_table_size()); return 0; } @@ -2435,6 +2439,55 @@ static int applet_tcpserve(int argc, char** argv) { return 0; } +// Fetch a URL through the ASYNC client: the request runs on the kernel HTTP +// worker thread while this applet just polls, exactly as the GUI Web app does. +// It is the command-line counterpart to that app and the way to exercise the +// worker path from a serial console. +static int applet_webget(int argc, char** argv) { + if (argc < 2) { + printk("usage: webget (e.g. webget 10.0.2.2:8000/)\n"); + return 1; + } + + http_fetch_t* f = http_fetch_start(argv[1], 16384); + if (!f) { + printk(ANSI_BRIGHT_RED "webget: could not start (bad URL, or a fetch is " + "already running)\n" ANSI_RESET); + return 1; + } + + printk("Fetching %s on the HTTP worker thread ...\n", argv[1]); + + // Poll while the worker runs. Yielding hands it the CPU; a wall-clock guard + // stops us waiting forever if something wedges. + uint64_t t0 = pit_get_ticks(); + while (f->state == HTTP_STATE_RUNNING && (pit_get_ticks() - t0) < 1500) { + sched_yield(); + } + + if (f->state == HTTP_STATE_RUNNING) { + printk(ANSI_YELLOW "webget: still running after 15s; giving up on the " + "poll (worker will clean up)\n" ANSI_RESET); + http_fetch_release(f); + return 1; + } + + if (f->state == HTTP_STATE_ERROR) { + printk(ANSI_BRIGHT_RED "webget: %s\n" ANSI_RESET, f->err); + http_fetch_release(f); + return 1; + } + + char ips[16]; + ip_to_str(f->ip, ips); + printk(ANSI_BRIGHT_GREEN "HTTP %d" ANSI_RESET " %d bytes %u ms from %s:%u (%s)\n", + f->status_code, f->len, f->elapsed_ms, f->host, f->port, ips); + printk("---\n%s\n---\n", f->buf); + + http_fetch_release(f); + return 0; +} + // A minimal HTTP/1.0 client over the active-open path. static int applet_httpget(int argc, char** argv) { if (argc < 2) { @@ -2806,6 +2859,7 @@ static const lazybox_applet_t applets[] = { {"sockstat", applet_sockstat, "sockstat", "Socket table & UDP stats", "Network"}, {"tcpconnect", applet_tcpconnect, "tcpconnect [text]", "Open an outbound TCP stream", "Network"}, {"httpget", applet_httpget, "httpget [path] [port]", "Fetch a URL over TCP", "Network"}, + {"webget", applet_webget, "webget ", "Fetch a URL via the async worker", "Network"}, {"tcpserve", applet_tcpserve, "tcpserve ", "Accept one inbound TCP stream", "Network"}, // Storage & Devices