1 // FIXME: if an exception is thrown, we shouldn't necessarily cache...
2 // FIXME: there's some annoying duplication of code in the various versioned mains
3 
4 // FIXME: cgi per-request arena allocator
5 
6 // i need to add a bunch of type templates for validations... mayne @NotNull or NotNull!
7 
8 // FIXME: I might make a cgi proxy class which can change things; the underlying one is still immutable
9 // but the later one can edit and simplify the api. You'd have to use the subclass tho!
10 
11 /*
12 void foo(int f, @("test") string s) {}
13 
14 void main() {
15 	static if(is(typeof(foo) Params == __parameters))
16 		//pragma(msg, __traits(getAttributes, Params[0]));
17 		pragma(msg, __traits(getAttributes, Params[1..2]));
18 	else
19 		pragma(msg, "fail");
20 }
21 */
22 
23 // Note: spawn-fcgi can help with fastcgi on nginx
24 
25 // FIXME: to do: add openssl optionally
26 // make sure embedded_httpd doesn't send two answers if one writes() then dies
27 
28 // future direction: websocket as a separate process that you can sendfile to for an async passoff of those long-lived connections
29 
30 /*
31 	Session manager process: it spawns a new process, passing a
32 	command line argument, to just be a little key/value store
33 	of some serializable struct. On Windows, it CreateProcess.
34 	On Linux, it can just fork or maybe fork/exec. The session
35 	key is in a cookie.
36 
37 	Server-side event process: spawns an async manager. You can
38 	push stuff out to channel ids and the clients listen to it.
39 
40 	websocket process: spawns an async handler. They can talk to
41 	each other or get info from a cgi request.
42 
43 	Tempting to put web.d 2.0 in here. It would:
44 		* map urls and form generation to functions
45 		* have data presentation magic
46 		* do the skeleton stuff like 1.0
47 		* auto-cache generated stuff in files (at least if pure?)
48 		* introspect functions in json for consumers
49 
50 
51 	https://linux.die.net/man/3/posix_spawn
52 */
53 
54 /++
55 	Provides a uniform server-side API for CGI, FastCGI, SCGI, and HTTP web applications.
56 
57 	---
58 	import arsd.cgi;
59 
60 	// Instead of writing your own main(), you should write a function
61 	// that takes a Cgi param, and use mixin GenericMain
62 	// for maximum compatibility with different web servers.
63 	void hello(Cgi cgi) {
64 		cgi.setResponseContentType("text/plain");
65 
66 		if("name" in cgi.get)
67 			cgi.write("Hello, " ~ cgi.get["name"]);
68 		else
69 			cgi.write("Hello, world!");
70 	}
71 
72 	mixin GenericMain!hello;
73 	---
74 
75 	Test on console (works in any interface mode):
76 	$(CONSOLE
77 		$ ./cgi_hello GET / name=whatever
78 	)
79 
80 	If using http version (default on `dub` builds, or on custom builds when passing `-version=embedded_httpd` to dmd):
81 	$(CONSOLE
82 		$ ./cgi_hello --port 8080
83 		# now you can go to http://localhost:8080/?name=whatever
84 	)
85 
86 
87 	Compile_versions:
88 
89 	If you are using `dub`, use:
90 
91 	```sdlang
92 	subConfiguration "arsd-official:cgi" "VALUE_HERE"
93 	```
94 
95 	or to dub.json:
96 
97 	```json
98         	"subConfigurations": {"arsd-official:cgi": "VALUE_HERE"}
99 	```
100 
101 	to change versions. The possible options for `VALUE_HERE` are:
102 
103 	$(LIST
104 		* `embedded_httpd` for the embedded httpd version (built-in web server). This is the default.
105 		* `cgi` for traditional cgi binaries.
106 		* `fastcgi` for FastCGI builds.
107 		* `scgi` for SCGI builds.
108 	)
109 
110 	With dmd, use:
111 
112 	$(TABLE_ROWS
113 
114 		* + Interfaces
115 		  + (mutually exclusive)
116 
117 		* - `-version=plain_cgi`
118 			- The default building the module alone without dub - a traditional, plain CGI executable will be generated.
119 		* - `-version=embedded_httpd`
120 			- A HTTP server will be embedded in the generated executable. This is default when building with dub.
121 		* - `-version=fastcgi`
122 			- A FastCGI executable will be generated.
123 		* - `-version=scgi`
124 			- A SCGI (SimpleCGI) executable will be generated.
125 
126 		* - `-version=embedded_httpd_threads`
127 			- The embedded HTTP server will use a single process with a thread pool. (use instead of plain `embedded_httpd` if you want this specific implementation)
128 		* - `-version=embedded_httpd_processes`
129 			- The embedded HTTP server will use a prefork style process pool. (use instead of plain `embedded_httpd` if you want this specific implementation)
130 		* - `-version=embedded_httpd_processes_accept_after_fork`
131 			- It will call accept() in each child process, after forking. This is currently the only option, though I am experimenting with other ideas. You probably should NOT specify this right now.
132 
133 		* + Tweaks
134 		  + (can be used together with others)
135 
136 		* - `-version=cgi_with_websocket`
137 			- The CGI class has websocket server support.
138 
139 		* - `-version=with_openssl`
140 			- not currently used
141 		* - `-version=cgi_embedded_sessions`
142 			- The session server will be embedded in the cgi.d server process
143 		* - `-version=cgi_session_server_process`
144 			- The session will be provided in a separate process, provided by cgi.d.
145 	)
146 
147 	Compile_and_run:
148 	
149 	For CGI, `dmd yourfile.d cgi.d` then put the executable in your cgi-bin directory.
150 
151 	For FastCGI: `dmd yourfile.d cgi.d -version=fastcgi` and run it. spawn-fcgi helps on nginx. You can put the file in the directory for Apache. On IIS, run it with a port on the command line.
152 
153 	For SCGI: `dmd yourfile.d cgi.d -version=scgi` and run the executable, providing a port number on the command line.
154 
155 	For an embedded HTTP server, run `dmd yourfile.d cgi.d -version=embedded_httpd` and run the generated program. It listens on port 8085 by default. You can change this on the command line with the --port option when running your program.
156 
157 	You can also simulate a request by passing parameters on the command line, like:
158 
159 	$(CONSOLE
160 	./yourprogram GET / name=adr
161 	)
162 
163 	And it will print the result to stdout.
164 
165 	CGI_Setup_tips:
166 
167 	On Apache, you may do `SetHandler cgi-script` in your `.htaccess` file.
168 
169 	Integration_tips:
170 
171 	cgi.d works well with dom.d for generating html. You may also use web.d for other utilities and automatic api wrapping.
172 
173 	dom.d usage:
174 
175 	---
176 		import arsd.cgi;
177 		import arsd.dom;
178 
179 		void hello_dom(Cgi cgi) {
180 			auto document = new Document();
181 
182 			static import std.file;
183 			// parse the file in strict mode, requiring it to be well-formed UTF-8 XHTML
184 			// (You'll appreciate this if you've ever had to deal with a missing </div>
185 			// or something in a php or erb template before that would randomly mess up
186 			// the output in your browser. Just check it and throw an exception early!)
187 			//
188 			// You could also hard-code a template or load one at compile time with an
189 			// import expression, but you might appreciate making it a regular file
190 			// because that means it can be more easily edited by the frontend team and
191 			// they can see their changes without needing to recompile the program.
192 			//
193 			// Note on CTFE: if you do choose to load a static file at compile time,
194 			// you *can* parse it in CTFE using enum, which will cause it to throw at
195 			// compile time, which is kinda cool too. Be careful in modifying that document,
196 			// though, as it will be a static instance. You might want to clone on on demand,
197 			// or perhaps modify it lazily as you print it out. (Try element.tree, it returns
198 			// a range of elements which you could send through std.algorithm functions. But
199 			// since my selector implementation doesn't work on that level yet, you'll find that
200 			// harder to use. Of course, you could make a static list of matching elements and
201 			// then use a simple e is e2 predicate... :) )
202 			document.parseUtf8(std.file.read("your_template.html"), true, true);
203 
204 			// fill in data using DOM functions, so placing it is in the hands of HTML
205 			// and it will be properly encoded as text too.
206 			//
207 			// Plain html templates can't run server side logic, but I think that's a
208 			// good thing - it keeps them simple. You may choose to extend the html,
209 			// but I think it is best to try to stick to standard elements and fill them
210 			// in with requested data with IDs or class names. A further benefit of
211 			// this is the designer can also highlight data based on sources in the CSS.
212 			//
213 			// However, all of dom.d is available, so you can format your data however
214 			// you like. You can do partial templates with innerHTML too, or perhaps better,
215 			// injecting cloned nodes from a partial document.
216 			//
217 			// There's a lot of possibilities.
218 			document["#name"].innerText = cgi.request("name", "default name");
219 
220 			// send the document to the browser. The second argument to `cgi.write`
221 			// indicates that this is all the data at once, enabling a few small
222 			// optimizations.
223 			cgi.write(document.toString(), true);
224 		}
225 	---
226 
227 	Concepts:
228 		Input: [Cgi.get], [Cgi.post], [Cgi.request], [Cgi.files], [Cgi.cookies], [Cgi.pathInfo], [Cgi.requestMethod],
229 		       and HTTP headers ([Cgi.headers], [Cgi.userAgent], [Cgi.referrer], [Cgi.accept], [Cgi.authorization], [Cgi.lastEventId])
230 
231 		Output: [Cgi.write], [Cgi.header], [Cgi.setResponseStatus], [Cgi.setResponseContentType], [Cgi.gzipResponse]
232 
233 		Cookies: [Cgi.setCookie], [Cgi.clearCookie], [Cgi.cookie], [Cgi.cookies]
234 
235 		Caching: [Cgi.setResponseExpires], [Cgi.updateResponseExpires], [Cgi.setCache]
236 
237 		Redirections: [Cgi.setResponseLocation]
238 
239 		Other Information: [Cgi.remoteAddress], [Cgi.https], [Cgi.port], [Cgi.scriptName], [Cgi.requestUri], [Cgi.getCurrentCompleteUri], [Cgi.onRequestBodyDataReceived]
240 
241 		Overriding behavior: [Cgi.handleIncomingDataChunk], [Cgi.prepareForIncomingDataChunks], [Cgi.cleanUpPostDataState]
242 
243 		Installing: Apache, IIS, CGI, FastCGI, SCGI, embedded HTTPD (not recommended for production use)
244 
245 	Guide_for_PHP_users:
246 		If you are coming from PHP, here's a quick guide to help you get started:
247 
248 		$(SIDE_BY_SIDE
249 			$(COLUMN
250 				```php
251 				<?php
252 					$foo = $_POST["foo"];
253 					$bar = $_GET["bar"];
254 					$baz = $_COOKIE["baz"];
255 
256 					$user_ip = $_SERVER["REMOTE_ADDR"];
257 					$host = $_SERVER["HTTP_HOST"];
258 					$path = $_SERVER["PATH_INFO"];
259 
260 					setcookie("baz", "some value");
261 
262 					echo "hello!";
263 				?>
264 				```
265 			)
266 			$(COLUMN
267 				---
268 				import arsd.cgi;
269 				void app(Cgi cgi) {
270 					string foo = cgi.post["foo"];
271 					string bar = cgi.get["bar"];
272 					string baz = cgi.cookies["baz"];
273 
274 					string user_ip = cgi.remoteAddress;
275 					string host = cgi.host;
276 					string path = cgi.pathInfo;
277 
278 					cgi.setCookie("baz", "some value");
279 
280 					cgi.write("hello!");
281 				}
282 
283 				mixin GenericMain!app
284 				---
285 			)
286 		)
287 
288 		$(H3 Array elements)
289 
290 
291 		In PHP, you can give a form element a name like `"something[]"`, and then
292 		`$_POST["something"]` gives an array. In D, you can use whatever name
293 		you want, and access an array of values with the `cgi.getArray["name"]` and
294 		`cgi.postArray["name"]` members.
295 
296 		$(H3 Databases)
297 
298 		PHP has a lot of stuff in its standard library. cgi.d doesn't include most
299 		of these, but the rest of my arsd repository has much of it. For example,
300 		to access a MySQL database, download `database.d` and `mysql.d` from my
301 		github repo, and try this code (assuming, of course, your database is
302 		set up):
303 
304 		---
305 		import arsd.cgi;
306 		import arsd.mysql;
307 
308 		void app(Cgi cgi) {
309 			auto database = new MySql("localhost", "username", "password", "database_name");
310 			foreach(row; mysql.query("SELECT count(id) FROM people"))
311 				cgi.write(row[0] ~ " people in database");
312 		}
313 
314 		mixin GenericMain!app;
315 		---
316 
317 		Similar modules are available for PostgreSQL, Microsoft SQL Server, and SQLite databases,
318 		implementing the same basic interface.
319 
320 	See_Also:
321 
322 	You may also want to see [arsd.dom], [arsd.web], and [arsd.html] for more code for making
323 	web applications.
324 
325 	For working with json, try [arsd.jsvar].
326 	
327 	[arsd.database], [arsd.mysql], [arsd.postgres], [arsd.mssql], and [arsd.sqlite] can help in
328 	accessing databases.
329 
330 	If you are looking to access a web application via HTTP, try [std.net.curl], [arsd.curl], or [arsd.http2].
331 
332 	Copyright:
333 
334 	cgi.d copyright 2008-2019, Adam D. Ruppe. Provided under the Boost Software License.
335 
336 	Yes, this file is almost ten years old, and yes, it is still actively maintained and used.
337 +/
338 module arsd.cgi;
339 
340 static import std.file;
341 
342 // for a single thread, linear request thing, use:
343 // -version=embedded_httpd_threads -version=cgi_no_threads
344 
345 version(Posix) {
346 	version(CRuntime_Musl) {
347 
348 	} else version(minimal) {
349 
350 	} else {
351 		version(GNU) {
352 			// GDC doesn't support static foreach so I had to cheat on it :(
353 		} else {
354 			version=with_breaking_cgi_features;
355 			version=with_sendfd;
356 			version=with_addon_servers;
357 		}
358 	}
359 }
360 
361 // the servers must know about the connections to talk to them; the interfaces are vital
362 version(with_addon_servers)
363 	version=with_addon_servers_connections;
364 
365 version(embedded_httpd) {
366 	version(linux)
367 		version=embedded_httpd_processes;
368 	else {
369 		version=embedded_httpd_threads;
370 	}
371 
372 	/*
373 	version(with_openssl) {
374 		pragma(lib, "crypto");
375 		pragma(lib, "ssl");
376 	}
377 	*/
378 }
379 
380 version(embedded_httpd_processes)
381 	version=embedded_httpd_processes_accept_after_fork; // I am getting much better average performance on this, so just keeping it. But the other way MIGHT help keep the variation down so i wanna keep the code to play with later
382 
383 version(embedded_httpd_threads) {
384 	//  unless the user overrides the default..
385 	version(cgi_session_server_process)
386 		{}
387 	else
388 		version=cgi_embedded_sessions;
389 }
390 version(scgi) {
391 	//  unless the user overrides the default..
392 	version(cgi_session_server_process)
393 		{}
394 	else
395 		version=cgi_embedded_sessions;
396 }
397 
398 // fall back if the other is not defined so we can cleanly version it below
399 version(cgi_embedded_sessions) {}
400 else version=cgi_session_server_process;
401 
402 
403 version=cgi_with_websocket;
404 
405 enum long defaultMaxContentLength = 5_000_000;
406 
407 /*
408 
409 	To do a file download offer in the browser:
410 
411     cgi.setResponseContentType("text/csv");
412     cgi.header("Content-Disposition: attachment; filename=\"customers.csv\"");
413 */
414 
415 // FIXME: the location header is supposed to be an absolute url I guess.
416 
417 // FIXME: would be cool to flush part of a dom document before complete
418 // somehow in here and dom.d.
419 
420 
421 // FIXME: 100 Continue in the nph section? Probably belongs on the
422 // httpd class though.
423 
424 // these are public so you can mixin GenericMain.
425 // FIXME: use a function level import instead!
426 public import std..string;
427 public import std.stdio;
428 public import std.conv;
429 import std.uri;
430 import std.uni;
431 import std.algorithm.comparison;
432 import std.algorithm.searching;
433 import std.exception;
434 import std.base64;
435 static import std.algorithm;
436 import std.datetime;
437 import std.range;
438 
439 import std.process;
440 
441 import std.zlib;
442 
443 
444 T[] consume(T)(T[] range, int count) {
445 	if(count > range.length)
446 		count = range.length;
447 	return range[count..$];
448 }
449 
450 int locationOf(T)(T[] data, string item) {
451 	const(ubyte[]) d = cast(const(ubyte[])) data;
452 	const(ubyte[]) i = cast(const(ubyte[])) item;
453 
454 	// this is a vague sanity check to ensure we aren't getting insanely
455 	// sized input that will infinite loop below. it should never happen;
456 	// even huge file uploads ought to come in smaller individual pieces.
457 	if(d.length > (int.max/2))
458 		throw new Exception("excessive block of input");
459 
460 	for(int a = 0; a < d.length; a++) {
461 		if(a + i.length > d.length)
462 			return -1;
463 		if(d[a..a+i.length] == i)
464 			return a;
465 	}
466 
467 	return -1;
468 }
469 
470 /// If you are doing a custom cgi class, mixing this in can take care of
471 /// the required constructors for you
472 mixin template ForwardCgiConstructors() {
473 	this(long maxContentLength = defaultMaxContentLength,
474 		string[string] env = null,
475 		const(ubyte)[] delegate() readdata = null,
476 		void delegate(const(ubyte)[]) _rawDataOutput = null,
477 		void delegate() _flush = null
478 		) { super(maxContentLength, env, readdata, _rawDataOutput, _flush); }
479 
480 	this(string[] args) { super(args); }
481 
482 	this(
483 		BufferedInputRange inputData,
484 		string address, ushort _port,
485 		int pathInfoStarts = 0,
486 		bool _https = false,
487 		void delegate(const(ubyte)[]) _rawDataOutput = null,
488 		void delegate() _flush = null,
489 		// this pointer tells if the connection is supposed to be closed after we handle this
490 		bool* closeConnection = null)
491 	{
492 		super(inputData, address, _port, pathInfoStarts, _https, _rawDataOutput, _flush, closeConnection);
493 	}
494 
495 	this(BufferedInputRange ir, bool* closeConnection) { super(ir, closeConnection); }
496 }
497 
498 /// thrown when a connection is closed remotely while we waiting on data from it
499 class ConnectionClosedException : Exception {
500 	this(string message, string file = __FILE__, size_t line = __LINE__, Throwable next = null) {
501 		super(message, file, line, next);
502 	}
503 }
504 
505  
506 version(Windows) {
507 // FIXME: ugly hack to solve stdin exception problems on Windows:
508 // reading stdin results in StdioException (Bad file descriptor)
509 // this is probably due to http://d.puremagic.com/issues/show_bug.cgi?id=3425
510 private struct stdin {
511 	struct ByChunk { // Replicates std.stdio.ByChunk
512 	private:
513 		ubyte[] chunk_;
514 	public:
515 		this(size_t size)
516 		in {
517 			assert(size, "size must be larger than 0");
518 		}
519 		body {
520 			chunk_ = new ubyte[](size);
521 			popFront();
522 		}
523 
524 		@property bool empty() const {
525 			return !std.stdio.stdin.isOpen || std.stdio.stdin.eof; // Ugly, but seems to do the job
526 		}
527 		@property nothrow ubyte[] front() {	return chunk_; }
528 		void popFront()	{
529 			enforce(!empty, "Cannot call popFront on empty range");
530 			chunk_ = stdin.rawRead(chunk_);
531 		}
532 	}
533 
534 	import core.sys.windows.windows;
535 static:
536 
537 	static this() {
538 		// Set stdin to binary mode
539 		version(Win64)
540 		_setmode(std.stdio.stdin.fileno(), 0x8000);
541 		else
542 		setmode(std.stdio.stdin.fileno(), 0x8000);
543 	}
544 
545 	T[] rawRead(T)(T[] buf) {
546 		uint bytesRead;
547 		auto result = ReadFile(GetStdHandle(STD_INPUT_HANDLE), buf.ptr, cast(int) (buf.length * T.sizeof), &bytesRead, null);
548 
549 		if (!result) {
550 			auto err = GetLastError();
551 			if (err == 38/*ERROR_HANDLE_EOF*/ || err == 109/*ERROR_BROKEN_PIPE*/) // 'good' errors meaning end of input
552 				return buf[0..0];
553 			// Some other error, throw it
554 
555 			char* buffer;
556 			scope(exit) LocalFree(buffer);
557 
558 			// FORMAT_MESSAGE_ALLOCATE_BUFFER	= 0x00000100
559 			// FORMAT_MESSAGE_FROM_SYSTEM		= 0x00001000
560 			FormatMessageA(0x1100, null, err, 0, cast(char*)&buffer, 256, null);
561 			throw new Exception(to!string(buffer));
562 		}
563 		enforce(!(bytesRead % T.sizeof), "I/O error");
564 		return buf[0..bytesRead / T.sizeof];
565 	}
566 
567 	auto byChunk(size_t sz) { return ByChunk(sz); }
568 }
569 }
570 
571 /// The main interface with the web request
572 class Cgi {
573   public:
574 	/// the methods a request can be
575 	enum RequestMethod { GET, HEAD, POST, PUT, DELETE, // GET and POST are the ones that really work
576 		// these are defined in the standard, but idk if they are useful for anything
577 		OPTIONS, TRACE, CONNECT,
578 		// These seem new, I have only recently seen them
579 		PATCH, MERGE,
580 		// this is an extension for when the method is not specified and you want to assume
581 		CommandLine }
582 
583 
584 	/+
585 	/++
586 		Cgi provides a per-request memory pool
587 
588 	+/
589 	void[] allocateMemory(size_t nBytes) {
590 
591 	}
592 
593 	/// ditto
594 	void[] reallocateMemory(void[] old, size_t nBytes) {
595 
596 	}
597 
598 	/// ditto
599 	void freeMemory(void[] memory) {
600 
601 	}
602 	+/
603 
604 
605 /*
606 	import core.runtime;
607 	auto args = Runtime.args();
608 
609 	we can call the app a few ways:
610 
611 	1) set up the environment variables and call the app (manually simulating CGI)
612 	2) simulate a call automatically:
613 		./app method 'uri'
614 
615 		for example:
616 			./app get /path?arg arg2=something
617 
618 	  Anything on the uri is treated as query string etc
619 
620 	  on get method, further args are appended to the query string (encoded automatically)
621 	  on post method, further args are done as post
622 
623 
624 	  @name means import from file "name". if name == -, it uses stdin
625 	  (so info=@- means set info to the value of stdin)
626 
627 
628 	  Other arguments include:
629 	  	--cookie name=value (these are all concated together)
630 		--header 'X-Something: cool'
631 		--referrer 'something'
632 		--port 80
633 		--remote-address some.ip.address.here
634 		--https yes
635 		--user-agent 'something'
636 		--userpass 'user:pass'
637 		--authorization 'Basic base64encoded_user:pass'
638 		--accept 'content' // FIXME: better example
639 		--last-event-id 'something'
640 		--host 'something.com'
641 
642 	  Non-simulation arguments:
643 	  	--port xxx listening port for non-cgi things (valid for the cgi interfaces)
644 		--listening-host  the ip address the application should listen on, or if you want to use unix domain sockets, it is here you can set them: `--listening-host unix:filename` or, on Linux, `--listening-host abstract:name`.
645 
646 */
647 
648 	/** Initializes it with command line arguments (for easy testing) */
649 	this(string[] args, void delegate(const(ubyte)[]) _rawDataOutput = null) {
650 		rawDataOutput = _rawDataOutput;
651 		// these are all set locally so the loop works
652 		// without triggering errors in dmd 2.064
653 		// we go ahead and set them at the end of it to the this version
654 		int port;
655 		string referrer;
656 		string remoteAddress;
657 		string userAgent;
658 		string authorization;
659 		string origin;
660 		string accept;
661 		string lastEventId;
662 		bool https;
663 		string host;
664 		RequestMethod requestMethod;
665 		string requestUri;
666 		string pathInfo;
667 		string queryString;
668 
669 		bool lookingForMethod;
670 		bool lookingForUri;
671 		string nextArgIs;
672 
673 		string _cookie;
674 		string _queryString;
675 		string[][string] _post;
676 		string[string] _headers;
677 
678 		string[] breakUp(string s) {
679 			string k, v;
680 			auto idx = s.indexOf("=");
681 			if(idx == -1) {
682 				k = s;
683 			} else {
684 				k = s[0 .. idx];
685 				v = s[idx + 1 .. $];
686 			}
687 
688 			return [k, v];
689 		}
690 
691 		lookingForMethod = true;
692 
693 		scriptName = args[0];
694 		scriptFileName = args[0];
695 
696 		environmentVariables = cast(const) environment.toAA;
697 
698 		foreach(arg; args[1 .. $]) {
699 			if(arg.startsWith("--")) {
700 				nextArgIs = arg[2 .. $];
701 			} else if(nextArgIs.length) {
702 				if (nextArgIs == "cookie") {
703 					auto info = breakUp(arg);
704 					if(_cookie.length)
705 						_cookie ~= "; ";
706 					_cookie ~= std.uri.encodeComponent(info[0]) ~ "=" ~ std.uri.encodeComponent(info[1]);
707 				}
708 				else if (nextArgIs == "port") {
709 					port = to!int(arg);
710 				}
711 				else if (nextArgIs == "referrer") {
712 					referrer = arg;
713 				}
714 				else if (nextArgIs == "remote-address") {
715 					remoteAddress = arg;
716 				}
717 				else if (nextArgIs == "user-agent") {
718 					userAgent = arg;
719 				}
720 				else if (nextArgIs == "authorization") {
721 					authorization = arg;
722 				}
723 				else if (nextArgIs == "userpass") {
724 					authorization = "Basic " ~ Base64.encode(cast(immutable(ubyte)[]) (arg)).idup;
725 				}
726 				else if (nextArgIs == "origin") {
727 					origin = arg;
728 				}
729 				else if (nextArgIs == "accept") {
730 					accept = arg;
731 				}
732 				else if (nextArgIs == "last-event-id") {
733 					lastEventId = arg;
734 				}
735 				else if (nextArgIs == "https") {
736 					if(arg == "yes")
737 						https = true;
738 				}
739 				else if (nextArgIs == "header") {
740 					string thing, other;
741 					auto idx = arg.indexOf(":");
742 					if(idx == -1)
743 						throw new Exception("need a colon in a http header");
744 					thing = arg[0 .. idx];
745 					other = arg[idx + 1.. $];
746 					_headers[thing.strip.toLower()] = other.strip;
747 				}
748 				else if (nextArgIs == "host") {
749 					host = arg;
750 				}
751 				// else
752 				// skip, we don't know it but that's ok, it might be used elsewhere so no error
753 
754 				nextArgIs = null;
755 			} else if(lookingForMethod) {
756 				lookingForMethod = false;
757 				lookingForUri = true;
758 
759 				if(arg.asLowerCase().equal("commandline"))
760 					requestMethod = RequestMethod.CommandLine;
761 				else
762 					requestMethod = to!RequestMethod(arg.toUpper());
763 			} else if(lookingForUri) {
764 				lookingForUri = false;
765 
766 				requestUri = arg;
767 
768 				auto idx = arg.indexOf("?");
769 				if(idx == -1)
770 					pathInfo = arg;
771 				else {
772 					pathInfo = arg[0 .. idx];
773 					_queryString = arg[idx + 1 .. $];
774 				}
775 			} else {
776 				// it is an argument of some sort
777 				if(requestMethod == Cgi.RequestMethod.POST || requestMethod == Cgi.RequestMethod.PATCH || requestMethod == Cgi.RequestMethod.PUT) {
778 					auto parts = breakUp(arg);
779 					_post[parts[0]] ~= parts[1];
780 					allPostNamesInOrder ~= parts[0];
781 					allPostValuesInOrder ~= parts[1];
782 				} else {
783 					if(_queryString.length)
784 						_queryString ~= "&";
785 					auto parts = breakUp(arg);
786 					_queryString ~= std.uri.encodeComponent(parts[0]) ~ "=" ~ std.uri.encodeComponent(parts[1]);
787 				}
788 			}
789 		}
790 
791 		acceptsGzip = false;
792 		keepAliveRequested = false;
793 		requestHeaders = cast(immutable) _headers;
794 
795 		cookie = _cookie;
796 		cookiesArray =  getCookieArray();
797 		cookies = keepLastOf(cookiesArray);
798 
799 		queryString = _queryString;
800 		getArray = cast(immutable) decodeVariables(queryString, "&", &allGetNamesInOrder, &allGetValuesInOrder);
801 		get = keepLastOf(getArray);
802 
803 		postArray = cast(immutable) _post;
804 		post = keepLastOf(_post);
805 
806 		// FIXME
807 		filesArray = null;
808 		files = null;
809 
810 		isCalledWithCommandLineArguments = true;
811 
812 		this.port = port;
813 		this.referrer = referrer;
814 		this.remoteAddress = remoteAddress;
815 		this.userAgent = userAgent;
816 		this.authorization = authorization;
817 		this.origin = origin;
818 		this.accept = accept;
819 		this.lastEventId = lastEventId;
820 		this.https = https;
821 		this.host = host;
822 		this.requestMethod = requestMethod;
823 		this.requestUri = requestUri;
824 		this.pathInfo = pathInfo;
825 		this.queryString = queryString;
826 		this.postJson = null;
827 	}
828 
829 	private {
830 		string[] allPostNamesInOrder;
831 		string[] allPostValuesInOrder;
832 		string[] allGetNamesInOrder;
833 		string[] allGetValuesInOrder;
834 	}
835 
836 	CgiConnectionHandle getOutputFileHandle() {
837 		return _outputFileHandle;
838 	}
839 
840 	CgiConnectionHandle _outputFileHandle = INVALID_CGI_CONNECTION_HANDLE;
841 
842 	/** Initializes it using a CGI or CGI-like interface */
843 	this(long maxContentLength = defaultMaxContentLength,
844 		// use this to override the environment variable listing
845 		in string[string] env = null,
846 		// and this should return a chunk of data. return empty when done
847 		const(ubyte)[] delegate() readdata = null,
848 		// finally, use this to do custom output if needed
849 		void delegate(const(ubyte)[]) _rawDataOutput = null,
850 		// to flush teh custom output
851 		void delegate() _flush = null
852 		)
853 	{
854 
855 		// these are all set locally so the loop works
856 		// without triggering errors in dmd 2.064
857 		// we go ahead and set them at the end of it to the this version
858 		int port;
859 		string referrer;
860 		string remoteAddress;
861 		string userAgent;
862 		string authorization;
863 		string origin;
864 		string accept;
865 		string lastEventId;
866 		bool https;
867 		string host;
868 		RequestMethod requestMethod;
869 		string requestUri;
870 		string pathInfo;
871 		string queryString;
872 
873 
874 
875 		isCalledWithCommandLineArguments = false;
876 		rawDataOutput = _rawDataOutput;
877 		flushDelegate = _flush;
878 		auto getenv = delegate string(string var) {
879 			if(env is null)
880 				return std.process.environment.get(var);
881 			auto e = var in env;
882 			if(e is null)
883 				return null;
884 			return *e;
885 		};
886 
887 		environmentVariables = env is null ?
888 			cast(const) environment.toAA :
889 			env;
890 
891 		// fetching all the request headers
892 		string[string] requestHeadersHere;
893 		foreach(k, v; env is null ? cast(const) environment.toAA() : env) {
894 			if(k.startsWith("HTTP_")) {
895 				requestHeadersHere[replace(k["HTTP_".length .. $].toLower(), "_", "-")] = v;
896 			}
897 		}
898 
899 		this.requestHeaders = assumeUnique(requestHeadersHere);
900 
901 		requestUri = getenv("REQUEST_URI");
902 
903 		cookie = getenv("HTTP_COOKIE");
904 		cookiesArray = getCookieArray();
905 		cookies = keepLastOf(cookiesArray);
906 
907 		referrer = getenv("HTTP_REFERER");
908 		userAgent = getenv("HTTP_USER_AGENT");
909 		remoteAddress = getenv("REMOTE_ADDR");
910 		host = getenv("HTTP_HOST");
911 		pathInfo = getenv("PATH_INFO");
912 
913 		queryString = getenv("QUERY_STRING");
914 		scriptName = getenv("SCRIPT_NAME");
915 		{
916 			import core.runtime;
917 			auto sfn = getenv("SCRIPT_FILENAME");
918 			scriptFileName = sfn.length ? sfn : Runtime.args[0];
919 		}
920 
921 		bool iis = false;
922 
923 		// Because IIS doesn't pass requestUri, we simulate it here if it's empty.
924 		if(requestUri.length == 0) {
925 			// IIS sometimes includes the script name as part of the path info - we don't want that
926 			if(pathInfo.length >= scriptName.length && (pathInfo[0 .. scriptName.length] == scriptName))
927 				pathInfo = pathInfo[scriptName.length .. $];
928 
929 			requestUri = scriptName ~ pathInfo ~ (queryString.length ? ("?" ~ queryString) : "");
930 
931 			iis = true; // FIXME HACK - used in byChunk below - see bugzilla 6339
932 
933 			// FIXME: this works for apache and iis... but what about others?
934 		}
935 
936 
937 		auto ugh = decodeVariables(queryString, "&", &allGetNamesInOrder, &allGetValuesInOrder);
938 		getArray = assumeUnique(ugh);
939 		get = keepLastOf(getArray);
940 
941 
942 		// NOTE: on shitpache, you need to specifically forward this
943 		authorization = getenv("HTTP_AUTHORIZATION");
944 		// this is a hack because Apache is a shitload of fuck and
945 		// refuses to send the real header to us. Compatible
946 		// programs should send both the standard and X- versions
947 
948 		// NOTE: if you have access to .htaccess or httpd.conf, you can make this
949 		// unnecessary with mod_rewrite, so it is commented
950 
951 		//if(authorization.length == 0) // if the std is there, use it
952 		//	authorization = getenv("HTTP_X_AUTHORIZATION");
953 
954 		// the REDIRECT_HTTPS check is here because with an Apache hack, the port can become wrong
955 		if(getenv("SERVER_PORT").length && getenv("REDIRECT_HTTPS") != "on")
956 			port = to!int(getenv("SERVER_PORT"));
957 		else
958 			port = 0; // this was probably called from the command line
959 
960 		auto ae = getenv("HTTP_ACCEPT_ENCODING");
961 		if(ae.length && ae.indexOf("gzip") != -1)
962 			acceptsGzip = true;
963 
964 		accept = getenv("HTTP_ACCEPT");
965 		lastEventId = getenv("HTTP_LAST_EVENT_ID");
966 
967 		auto ka = getenv("HTTP_CONNECTION");
968 		if(ka.length && ka.asLowerCase().canFind("keep-alive"))
969 			keepAliveRequested = true;
970 
971 		auto or = getenv("HTTP_ORIGIN");
972 			origin = or;
973 
974 		auto rm = getenv("REQUEST_METHOD");
975 		if(rm.length)
976 			requestMethod = to!RequestMethod(getenv("REQUEST_METHOD"));
977 		else
978 			requestMethod = RequestMethod.CommandLine;
979 
980 						// FIXME: hack on REDIRECT_HTTPS; this is there because the work app uses mod_rewrite which loses the https flag! So I set it with [E=HTTPS=%HTTPS] or whatever but then it gets translated to here so i want it to still work. This is arguably wrong but meh.
981 		https = (getenv("HTTPS") == "on" || getenv("REDIRECT_HTTPS") == "on");
982 
983 		// FIXME: DOCUMENT_ROOT?
984 
985 		// FIXME: what about PUT?
986 		if(requestMethod == RequestMethod.POST || requestMethod == Cgi.RequestMethod.PATCH || requestMethod == Cgi.RequestMethod.PUT) {
987 			version(preserveData) // a hack to make forwarding simpler
988 				immutable(ubyte)[] data;
989 			size_t amountReceived = 0;
990 			auto contentType = getenv("CONTENT_TYPE");
991 
992 			// FIXME: is this ever not going to be set? I guess it depends
993 			// on if the server de-chunks and buffers... seems like it has potential
994 			// to be slow if they did that. The spec says it is always there though.
995 			// And it has worked reliably for me all year in the live environment,
996 			// but some servers might be different.
997 			auto contentLength = to!size_t(getenv("CONTENT_LENGTH"));
998 
999 			immutable originalContentLength = contentLength;
1000 			if(contentLength) {
1001 				if(maxContentLength > 0 && contentLength > maxContentLength) {
1002 					setResponseStatus("413 Request entity too large");
1003 					write("You tried to upload a file that is too large.");
1004 					close();
1005 					throw new Exception("POST too large");
1006 				}
1007 				prepareForIncomingDataChunks(contentType, contentLength);
1008 
1009 
1010 				int processChunk(in ubyte[] chunk) {
1011 					if(chunk.length > contentLength) {
1012 						handleIncomingDataChunk(chunk[0..contentLength]);
1013 						amountReceived += contentLength;
1014 						contentLength = 0;
1015 						return 1;
1016 					} else {
1017 						handleIncomingDataChunk(chunk);
1018 						contentLength -= chunk.length;
1019 						amountReceived += chunk.length;
1020 					}
1021 					if(contentLength == 0)
1022 						return 1;
1023 
1024 					onRequestBodyDataReceived(amountReceived, originalContentLength);
1025 					return 0;
1026 				}
1027 
1028 
1029 				if(readdata is null) {
1030 					foreach(ubyte[] chunk; stdin.byChunk(iis ? contentLength : 4096))
1031 						if(processChunk(chunk))
1032 							break;
1033 				} else {
1034 					// we have a custom data source..
1035 					auto chunk = readdata();
1036 					while(chunk.length) {
1037 						if(processChunk(chunk))
1038 							break;
1039 						chunk = readdata();
1040 					}
1041 				}
1042 
1043 				onRequestBodyDataReceived(amountReceived, originalContentLength);
1044 				postArray = assumeUnique(pps._post);
1045 				filesArray = assumeUnique(pps._files);
1046 				files = keepLastOf(filesArray);
1047 				post = keepLastOf(postArray);
1048 				this.postJson = pps.postJson;
1049 				cleanUpPostDataState();
1050 			}
1051 
1052 			version(preserveData)
1053 				originalPostData = data;
1054 		}
1055 		// fixme: remote_user script name
1056 
1057 
1058 		this.port = port;
1059 		this.referrer = referrer;
1060 		this.remoteAddress = remoteAddress;
1061 		this.userAgent = userAgent;
1062 		this.authorization = authorization;
1063 		this.origin = origin;
1064 		this.accept = accept;
1065 		this.lastEventId = lastEventId;
1066 		this.https = https;
1067 		this.host = host;
1068 		this.requestMethod = requestMethod;
1069 		this.requestUri = requestUri;
1070 		this.pathInfo = pathInfo;
1071 		this.queryString = queryString;
1072 	}
1073 
1074 	/// Cleans up any temporary files. Do not use the object
1075 	/// after calling this.
1076 	///
1077 	/// NOTE: it is called automatically by GenericMain
1078 	// FIXME: this should be called if the constructor fails too, if it has created some garbage...
1079 	void dispose() {
1080 		foreach(file; files) {
1081 			if(!file.contentInMemory)
1082 				if(std.file.exists(file.contentFilename))
1083 					std.file.remove(file.contentFilename);
1084 		}
1085 	}
1086 
1087 	private {
1088 		struct PostParserState {
1089 			string contentType;
1090 			string boundary;
1091 			string localBoundary; // the ones used at the end or something lol
1092 			bool isMultipart;
1093 			bool isJson;
1094 
1095 			ulong expectedLength;
1096 			ulong contentConsumed;
1097 			immutable(ubyte)[] buffer;
1098 
1099 			// multipart parsing state
1100 			int whatDoWeWant;
1101 			bool weHaveAPart;
1102 			string[] thisOnesHeaders;
1103 			immutable(ubyte)[] thisOnesData;
1104 
1105 			string postJson;
1106 
1107 			UploadedFile piece;
1108 			bool isFile = false;
1109 
1110 			size_t memoryCommitted;
1111 
1112 			// do NOT keep mutable references to these anywhere!
1113 			// I assume they are unique in the constructor once we're all done getting data.
1114 			string[][string] _post;
1115 			UploadedFile[][string] _files;
1116 		}
1117 
1118 		PostParserState pps;
1119 	}
1120 
1121 	/// This represents a file the user uploaded via a POST request.
1122 	static struct UploadedFile {
1123 		/// If you want to create one of these structs for yourself from some data,
1124 		/// use this function.
1125 		static UploadedFile fromData(immutable(void)[] data, string name = null) {
1126 			Cgi.UploadedFile f;
1127 			f.filename = name;
1128 			f.content = cast(immutable(ubyte)[]) data;
1129 			f.contentInMemory = true;
1130 			return f;
1131 		}
1132 
1133 		string name; 		/// The name of the form element.
1134 		string filename; 	/// The filename the user set.
1135 		string contentType; 	/// The MIME type the user's browser reported. (Not reliable.)
1136 
1137 		/**
1138 			For small files, cgi.d will buffer the uploaded file in memory, and make it
1139 			directly accessible to you through the content member. I find this very convenient
1140 			and somewhat efficient, since it can avoid hitting the disk entirely. (I
1141 			often want to inspect and modify the file anyway!)
1142 
1143 			I find the file is very large, it is undesirable to eat that much memory just
1144 			for a file buffer. In those cases, if you pass a large enough value for maxContentLength
1145 			to the constructor so they are accepted, cgi.d will write the content to a temporary
1146 			file that you can re-read later.
1147 
1148 			You can override this behavior by subclassing Cgi and overriding the protected
1149 			handlePostChunk method. Note that the object is not initialized when you
1150 			write that method - the http headers are available, but the cgi.post method
1151 			is not. You may parse the file as it streams in using this method.
1152 
1153 
1154 			Anyway, if the file is small enough to be in memory, contentInMemory will be
1155 			set to true, and the content is available in the content member.
1156 
1157 			If not, contentInMemory will be set to false, and the content saved in a file,
1158 			whose name will be available in the contentFilename member.
1159 
1160 
1161 			Tip: if you know you are always dealing with small files, and want the convenience
1162 			of ignoring this member, construct Cgi with a small maxContentLength. Then, if
1163 			a large file comes in, it simply throws an exception (and HTTP error response)
1164 			instead of trying to handle it.
1165 
1166 			The default value of maxContentLength in the constructor is for small files.
1167 		*/
1168 		bool contentInMemory = true; // the default ought to always be true
1169 		immutable(ubyte)[] content; /// The actual content of the file, if contentInMemory == true
1170 		string contentFilename; /// the file where we dumped the content, if contentInMemory == false. Note that if you want to keep it, you MUST move the file, since otherwise it is considered garbage when cgi is disposed.
1171 
1172 		///
1173 		ulong fileSize() {
1174 			if(contentInMemory)
1175 				return content.length;
1176 			import std.file;
1177 			return std.file.getSize(contentFilename);
1178 
1179 		}
1180 
1181 		///
1182 		void writeToFile(string filenameToSaveTo) const {
1183 			import std.file;
1184 			if(contentInMemory)
1185 				std.file.write(filenameToSaveTo, content);
1186 			else
1187 				std.file.rename(contentFilename, filenameToSaveTo);
1188 		}
1189 	}
1190 
1191 	// given a content type and length, decide what we're going to do with the data..
1192 	protected void prepareForIncomingDataChunks(string contentType, ulong contentLength) {
1193 		pps.expectedLength = contentLength;
1194 
1195 		auto terminator = contentType.indexOf(";");
1196 		if(terminator == -1)
1197 			terminator = contentType.length;
1198 
1199 		pps.contentType = contentType[0 .. terminator];
1200 		auto b = contentType[terminator .. $];
1201 		if(b.length) {
1202 			auto idx = b.indexOf("boundary=");
1203 			if(idx != -1) {
1204 				pps.boundary = b[idx + "boundary=".length .. $];
1205 				pps.localBoundary = "\r\n--" ~ pps.boundary;
1206 			}
1207 		}
1208 
1209 		// while a content type SHOULD be sent according to the RFC, it is
1210 		// not required. We're told we SHOULD guess by looking at the content
1211 		// but it seems to me that this only happens when it is urlencoded.
1212 		if(pps.contentType == "application/x-www-form-urlencoded" || pps.contentType == "") {
1213 			pps.isMultipart = false;
1214 		} else if(pps.contentType == "multipart/form-data") {
1215 			pps.isMultipart = true;
1216 			enforce(pps.boundary.length, "no boundary");
1217 		} else if(pps.contentType == "text/plain") {
1218 			pps.isMultipart = false;
1219 			pps.isJson = true; // FIXME: hack, it isn't actually this
1220 		} else if(pps.contentType == "text/xml") { // FIXME: what if we used this as a fallback?
1221 			pps.isMultipart = false;
1222 			pps.isJson = true; // FIXME: hack, it isn't actually this
1223 		} else if(pps.contentType == "application/json") {
1224 			pps.isJson = true;
1225 			pps.isMultipart = false;
1226 		//} else if(pps.contentType == "application/json") {
1227 			//pps.isJson = true;
1228 		} else {
1229 			// FIXME: should set a http error code too
1230 			throw new Exception("unknown request content type: " ~ pps.contentType);
1231 		}
1232 	}
1233 
1234 	// handles streaming POST data. If you handle some other content type, you should
1235 	// override this. If the data isn't the content type you want, you ought to call
1236 	// super.handleIncomingDataChunk so regular forms and files still work.
1237 
1238 	// FIXME: I do some copying in here that I'm pretty sure is unnecessary, and the
1239 	// file stuff I'm sure is inefficient. But, my guess is the real bottleneck is network
1240 	// input anyway, so I'm not going to get too worked up about it right now.
1241 	protected void handleIncomingDataChunk(const(ubyte)[] chunk) {
1242 		if(chunk.length == 0)
1243 			return;
1244 		assert(chunk.length <= 32 * 1024 * 1024); // we use chunk size as a memory constraint thing, so
1245 							// if we're passed big chunks, it might throw unnecessarily.
1246 							// just pass it smaller chunks at a time.
1247 		if(pps.isMultipart) {
1248 			// multipart/form-data
1249 
1250 
1251 			void pieceHasNewContent() {
1252 				// we just grew the piece's buffer. Do we have to switch to file backing?
1253 				if(pps.piece.contentInMemory) {
1254 					if(pps.piece.content.length <= 10 * 1024 * 1024)
1255 						// meh, I'm ok with it.
1256 						return;
1257 					else {
1258 						// this is too big.
1259 						if(!pps.isFile)
1260 							throw new Exception("Request entity too large"); // a variable this big is kinda ridiculous, just reject it.
1261 						else {
1262 							// a file this large is probably acceptable though... let's use a backing file.
1263 							pps.piece.contentInMemory = false;
1264 							// FIXME: say... how do we intend to delete these things? cgi.dispose perhaps.
1265 
1266 							int count = 0;
1267 							pps.piece.contentFilename = getTempDirectory() ~ "arsd_cgi_uploaded_file_" ~ to!string(getUtcTime()) ~ "-" ~ to!string(count);
1268 							// odds are this loop will never be entered, but we want it just in case.
1269 							while(std.file.exists(pps.piece.contentFilename)) {
1270 								count++;
1271 								pps.piece.contentFilename = getTempDirectory() ~ "arsd_cgi_uploaded_file_" ~ to!string(getUtcTime()) ~ "-" ~ to!string(count);
1272 							}
1273 							// I hope this creates the file pretty quickly, or the loop might be useless...
1274 							// FIXME: maybe I should write some kind of custom transaction here.
1275 							std.file.write(pps.piece.contentFilename, pps.piece.content);
1276 
1277 							pps.piece.content = null;
1278 						}
1279 					}
1280 				} else {
1281 					// it's already in a file, so just append it to what we have
1282 					if(pps.piece.content.length) {
1283 						// FIXME: this is surely very inefficient... we'll be calling this by 4kb chunk...
1284 						std.file.append(pps.piece.contentFilename, pps.piece.content);
1285 						pps.piece.content = null;
1286 					}
1287 				}
1288 			}
1289 
1290 
1291 			void commitPart() {
1292 				if(!pps.weHaveAPart)
1293 					return;
1294 
1295 				pieceHasNewContent(); // be sure the new content is handled every time
1296 
1297 				if(pps.isFile) {
1298 					// I'm not sure if other environments put files in post or not...
1299 					// I used to not do it, but I think I should, since it is there...
1300 					pps._post[pps.piece.name] ~= pps.piece.filename;
1301 					pps._files[pps.piece.name] ~= pps.piece;
1302 
1303 					allPostNamesInOrder ~= pps.piece.name;
1304 					allPostValuesInOrder ~= pps.piece.filename;
1305 				} else {
1306 					pps._post[pps.piece.name] ~= cast(string) pps.piece.content;
1307 
1308 					allPostNamesInOrder ~= pps.piece.name;
1309 					allPostValuesInOrder ~= cast(string) pps.piece.content;
1310 				}
1311 
1312 				/*
1313 				stderr.writeln("RECEIVED: ", pps.piece.name, "=", 
1314 					pps.piece.content.length < 1000
1315 					?
1316 					to!string(pps.piece.content)
1317 					:
1318 					"too long");
1319 				*/
1320 
1321 				// FIXME: the limit here
1322 				pps.memoryCommitted += pps.piece.content.length;
1323 
1324 				pps.weHaveAPart = false;
1325 				pps.whatDoWeWant = 1;
1326 				pps.thisOnesHeaders = null;
1327 				pps.thisOnesData = null;
1328 
1329 				pps.piece = UploadedFile.init;
1330 				pps.isFile = false;
1331 			}
1332 
1333 			void acceptChunk() {
1334 				pps.buffer ~= chunk;
1335 				chunk = null; // we've consumed it into the buffer, so keeping it just brings confusion
1336 			}
1337 
1338 			immutable(ubyte)[] consume(size_t howMuch) {
1339 				pps.contentConsumed += howMuch;
1340 				auto ret = pps.buffer[0 .. howMuch];
1341 				pps.buffer = pps.buffer[howMuch .. $];
1342 				return ret;
1343 			}
1344 
1345 			dataConsumptionLoop: do {
1346 			switch(pps.whatDoWeWant) {
1347 				default: assert(0);
1348 				case 0:
1349 					acceptChunk();
1350 					// the format begins with two extra leading dashes, then we should be at the boundary
1351 					if(pps.buffer.length < 2)
1352 						return;
1353 					assert(pps.buffer[0] == '-', "no leading dash");
1354 					consume(1);
1355 					assert(pps.buffer[0] == '-', "no second leading dash");
1356 					consume(1);
1357 
1358 					pps.whatDoWeWant = 1;
1359 					goto case 1;
1360 				/* fallthrough */
1361 				case 1: // looking for headers
1362 					// here, we should be lined up right at the boundary, which is followed by a \r\n
1363 
1364 					// want to keep the buffer under control in case we're under attack
1365 					//stderr.writeln("here once");
1366 					//if(pps.buffer.length + chunk.length > 70 * 1024) // they should be < 1 kb really....
1367 					//	throw new Exception("wtf is up with the huge mime part headers");
1368 
1369 					acceptChunk();
1370 
1371 					if(pps.buffer.length < pps.boundary.length)
1372 						return; // not enough data, since there should always be a boundary here at least
1373 
1374 					if(pps.contentConsumed + pps.boundary.length + 6 == pps.expectedLength) {
1375 						assert(pps.buffer.length == pps.boundary.length + 4 + 2); // --, --, and \r\n
1376 						// we *should* be at the end here!
1377 						assert(pps.buffer[0] == '-');
1378 						consume(1);
1379 						assert(pps.buffer[0] == '-');
1380 						consume(1);
1381 
1382 						// the message is terminated by --BOUNDARY--\r\n (after a \r\n leading to the boundary)
1383 						assert(pps.buffer[0 .. pps.boundary.length] == cast(const(ubyte[])) pps.boundary,
1384 							"not lined up on boundary " ~ pps.boundary);
1385 						consume(pps.boundary.length);
1386 
1387 						assert(pps.buffer[0] == '-');
1388 						consume(1);
1389 						assert(pps.buffer[0] == '-');
1390 						consume(1);
1391 
1392 						assert(pps.buffer[0] == '\r');
1393 						consume(1);
1394 						assert(pps.buffer[0] == '\n');
1395 						consume(1);
1396 
1397 						assert(pps.buffer.length == 0);
1398 						assert(pps.contentConsumed == pps.expectedLength);
1399 						break dataConsumptionLoop; // we're done!
1400 					} else {
1401 						// we're not done yet. We should be lined up on a boundary.
1402 
1403 						// But, we want to ensure the headers are here before we consume anything!
1404 						auto headerEndLocation = locationOf(pps.buffer, "\r\n\r\n");
1405 						if(headerEndLocation == -1)
1406 							return; // they *should* all be here, so we can handle them all at once.
1407 
1408 						assert(pps.buffer[0 .. pps.boundary.length] == cast(const(ubyte[])) pps.boundary,
1409 							"not lined up on boundary " ~ pps.boundary);
1410 
1411 						consume(pps.boundary.length);
1412 						// the boundary is always followed by a \r\n
1413 						assert(pps.buffer[0] == '\r');
1414 						consume(1);
1415 						assert(pps.buffer[0] == '\n');
1416 						consume(1);
1417 					}
1418 
1419 					// re-running since by consuming the boundary, we invalidate the old index.
1420 					auto headerEndLocation = locationOf(pps.buffer, "\r\n\r\n");
1421 					assert(headerEndLocation >= 0, "no header");
1422 					auto thisOnesHeaders = pps.buffer[0..headerEndLocation];
1423 
1424 					consume(headerEndLocation + 4); // The +4 is the \r\n\r\n that caps it off
1425 
1426 					pps.thisOnesHeaders = split(cast(string) thisOnesHeaders, "\r\n");
1427 
1428 					// now we'll parse the headers
1429 					foreach(h; pps.thisOnesHeaders) {
1430 						auto p = h.indexOf(":");
1431 						assert(p != -1, "no colon in header, got " ~ to!string(pps.thisOnesHeaders));
1432 						string hn = h[0..p];
1433 						string hv = h[p+2..$];
1434 
1435 						switch(hn.toLower) {
1436 							default: assert(0);
1437 							case "content-disposition":
1438 								auto info = hv.split("; ");
1439 								foreach(i; info[1..$]) { // skipping the form-data
1440 									auto o = i.split("="); // FIXME
1441 									string pn = o[0];
1442 									string pv = o[1][1..$-1];
1443 
1444 									if(pn == "name") {
1445 										pps.piece.name = pv;
1446 									} else if (pn == "filename") {
1447 										pps.piece.filename = pv;
1448 										pps.isFile = true;
1449 									}
1450 								}
1451 							break;
1452 							case "content-type":
1453 								pps.piece.contentType = hv;
1454 							break;
1455 						}
1456 					}
1457 
1458 					pps.whatDoWeWant++; // move to the next step - the data
1459 				break;
1460 				case 2:
1461 					// when we get here, pps.buffer should contain our first chunk of data
1462 
1463 					if(pps.buffer.length + chunk.length > 8 * 1024 * 1024) // we might buffer quite a bit but not much
1464 						throw new Exception("wtf is up with the huge mime part buffer");
1465 
1466 					acceptChunk();
1467 
1468 					// so the trick is, we want to process all the data up to the boundary,
1469 					// but what if the chunk's end cuts the boundary off? If we're unsure, we
1470 					// want to wait for the next chunk. We start by looking for the whole boundary
1471 					// in the buffer somewhere.
1472 
1473 					auto boundaryLocation = locationOf(pps.buffer, pps.localBoundary);
1474 					// assert(boundaryLocation != -1, "should have seen "~to!string(cast(ubyte[]) pps.localBoundary)~" in " ~ to!string(pps.buffer));
1475 					if(boundaryLocation != -1) {
1476 						// this is easy - we can see it in it's entirety!
1477 
1478 						pps.piece.content ~= consume(boundaryLocation);
1479 
1480 						assert(pps.buffer[0] == '\r');
1481 						consume(1);
1482 						assert(pps.buffer[0] == '\n');
1483 						consume(1);
1484 						assert(pps.buffer[0] == '-');
1485 						consume(1);
1486 						assert(pps.buffer[0] == '-');
1487 						consume(1);
1488 						// the boundary here is always preceded by \r\n--, which is why we used localBoundary instead of boundary to locate it. Cut that off.
1489 						pps.weHaveAPart = true;
1490 						pps.whatDoWeWant = 1; // back to getting headers for the next part
1491 
1492 						commitPart(); // we're done here
1493 					} else {
1494 						// we can't see the whole thing, but what if there's a partial boundary?
1495 
1496 						enforce(pps.localBoundary.length < 128); // the boundary ought to be less than a line...
1497 						assert(pps.localBoundary.length > 1); // should already be sane but just in case
1498 						bool potentialBoundaryFound = false;
1499 
1500 						boundaryCheck: for(int a = 1; a < pps.localBoundary.length; a++) {
1501 							// we grow the boundary a bit each time. If we think it looks the
1502 							// same, better pull another chunk to be sure it's not the end.
1503 							// Starting small because exiting the loop early is desirable, since
1504 							// we're not keeping any ambiguity and 1 / 256 chance of exiting is
1505 							// the best we can do.
1506 							if(a > pps.buffer.length)
1507 								break; // FIXME: is this right?
1508 							assert(a <= pps.buffer.length);
1509 							assert(a > 0);
1510 							if(std.algorithm.endsWith(pps.buffer, pps.localBoundary[0 .. a])) {
1511 								// ok, there *might* be a boundary here, so let's
1512 								// not treat the end as data yet. The rest is good to
1513 								// use though, since if there was a boundary there, we'd
1514 								// have handled it up above after locationOf.
1515 
1516 								pps.piece.content ~= pps.buffer[0 .. $ - a];
1517 								consume(pps.buffer.length - a);
1518 								pieceHasNewContent();
1519 								potentialBoundaryFound = true;
1520 								break boundaryCheck;
1521 							}
1522 						}
1523 
1524 						if(!potentialBoundaryFound) {
1525 							// we can consume the whole thing
1526 							pps.piece.content ~= pps.buffer;
1527 							pieceHasNewContent();
1528 							consume(pps.buffer.length);
1529 						} else {
1530 							// we found a possible boundary, but there was
1531 							// insufficient data to be sure.
1532 							assert(pps.buffer == cast(const(ubyte[])) pps.localBoundary[0 .. pps.buffer.length]);
1533 
1534 							return; // wait for the next chunk.
1535 						}
1536 					}
1537 			}
1538 			} while(pps.buffer.length);
1539 
1540 			// btw all boundaries except the first should have a \r\n before them
1541 		} else {
1542 			// application/x-www-form-urlencoded and application/json
1543 
1544 				// not using maxContentLength because that might be cranked up to allow
1545 				// large file uploads. We can handle them, but a huge post[] isn't any good.
1546 			if(pps.buffer.length + chunk.length > 8 * 1024 * 1024) // surely this is plenty big enough
1547 				throw new Exception("wtf is up with such a gigantic form submission????");
1548 
1549 			pps.buffer ~= chunk;
1550 
1551 			// simple handling, but it works... until someone bombs us with gigabytes of crap at least...
1552 			if(pps.buffer.length == pps.expectedLength) {
1553 				if(pps.isJson)
1554 					pps.postJson = cast(string) pps.buffer;
1555 				else
1556 					pps._post = decodeVariables(cast(string) pps.buffer, "&", &allPostNamesInOrder, &allPostValuesInOrder);
1557 				version(preserveData)
1558 					originalPostData = pps.buffer;
1559 			} else {
1560 				// just for debugging
1561 			}
1562 		}
1563 	}
1564 
1565 	protected void cleanUpPostDataState() {
1566 		pps = PostParserState.init;
1567 	}
1568 
1569 	/// you can override this function to somehow react
1570 	/// to an upload in progress.
1571 	///
1572 	/// Take note that parts of the CGI object is not yet
1573 	/// initialized! Stuff from HTTP headers, including get[], is usable.
1574 	/// But, none of post[] is usable, and you cannot write here. That's
1575 	/// why this method is const - mutating the object won't do much anyway.
1576 	///
1577 	/// My idea here was so you can output a progress bar or
1578 	/// something to a cooperative client (see arsd.rtud for a potential helper)
1579 	///
1580 	/// The default is to do nothing. Subclass cgi and use the 
1581 	/// CustomCgiMain mixin to do something here.
1582 	void onRequestBodyDataReceived(size_t receivedSoFar, size_t totalExpected) const {
1583 		// This space intentionally left blank.
1584 	}
1585 
1586 	/// Initializes the cgi from completely raw HTTP data. The ir must have a Socket source.
1587 	/// *closeConnection will be set to true if you should close the connection after handling this request
1588 	this(BufferedInputRange ir, bool* closeConnection) {
1589 		isCalledWithCommandLineArguments = false;
1590 		import al = std.algorithm;
1591 
1592 		immutable(ubyte)[] data;
1593 
1594 		void rdo(const(ubyte)[] d) {
1595 		//import std.stdio; writeln(d);
1596 			sendAll(ir.source, d);
1597 		}
1598 
1599 		auto ira = ir.source.remoteAddress();
1600 
1601 		// that check for UnixAddress is to work around a Phobos bug
1602 		// see: https://github.com/dlang/phobos/pull/7383
1603 		// but this might be more useful anyway tbh for this case
1604 		version(Posix)
1605 		this(ir, cast(UnixAddress) ira ? "unix:" : ira.toString(), 80 /* FIXME */, 0, false, &rdo, null, closeConnection);
1606 		else
1607 		this(ir, ira.toString(), 80 /* FIXME */, 0, false, &rdo, null, closeConnection);
1608 	}
1609 
1610 	/**
1611 		Initializes it from raw HTTP request data. GenericMain uses this when you compile with -version=embedded_httpd.
1612 
1613 		NOTE: If you are behind a reverse proxy, the values here might not be what you expect.... it will use X-Forwarded-For for remote IP and X-Forwarded-Host for host
1614 
1615 		Params:
1616 			inputData = the incoming data, including headers and other raw http data.
1617 				When the constructor exits, it will leave this range exactly at the start of
1618 				the next request on the connection (if there is one).
1619 
1620 			address = the IP address of the remote user
1621 			_port = the port number of the connection
1622 			pathInfoStarts = the offset into the path component of the http header where the SCRIPT_NAME ends and the PATH_INFO begins.
1623 			_https = if this connection is encrypted (note that the input data must not actually be encrypted)
1624 			_rawDataOutput = delegate to accept response data. It should write to the socket or whatever; Cgi does all the needed processing to speak http.
1625 			_flush = if _rawDataOutput buffers, this delegate should flush the buffer down the wire
1626 			closeConnection = if the request asks to close the connection, *closeConnection == true.
1627 	*/
1628 	this(
1629 		BufferedInputRange inputData,
1630 //		string[] headers, immutable(ubyte)[] data,
1631 		string address, ushort _port,
1632 		int pathInfoStarts = 0, // use this if you know the script name, like if this is in a folder in a bigger web environment
1633 		bool _https = false,
1634 		void delegate(const(ubyte)[]) _rawDataOutput = null,
1635 		void delegate() _flush = null,
1636 		// this pointer tells if the connection is supposed to be closed after we handle this
1637 		bool* closeConnection = null)
1638 	{
1639 		// these are all set locally so the loop works
1640 		// without triggering errors in dmd 2.064
1641 		// we go ahead and set them at the end of it to the this version
1642 		int port;
1643 		string referrer;
1644 		string remoteAddress;
1645 		string userAgent;
1646 		string authorization;
1647 		string origin;
1648 		string accept;
1649 		string lastEventId;
1650 		bool https;
1651 		string host;
1652 		RequestMethod requestMethod;
1653 		string requestUri;
1654 		string pathInfo;
1655 		string queryString;
1656 		string scriptName;
1657 		string[string] get;
1658 		string[][string] getArray;
1659 		bool keepAliveRequested;
1660 		bool acceptsGzip;
1661 		string cookie;
1662 
1663 
1664 
1665 		environmentVariables = cast(const) environment.toAA;
1666 
1667 		idlol = inputData;
1668 
1669 		isCalledWithCommandLineArguments = false;
1670 
1671 		https = _https;
1672 		port = _port;
1673 
1674 		rawDataOutput = _rawDataOutput;
1675 		flushDelegate = _flush;
1676 		nph = true;
1677 
1678 		remoteAddress = address;
1679 
1680 		// streaming parser
1681 		import al = std.algorithm;
1682 
1683 			// FIXME: tis cast is technically wrong, but Phobos deprecated al.indexOf... for some reason.
1684 		auto idx = indexOf(cast(string) inputData.front(), "\r\n\r\n");
1685 		while(idx == -1) {
1686 			inputData.popFront(0);
1687 			idx = indexOf(cast(string) inputData.front(), "\r\n\r\n");
1688 		}
1689 
1690 		assert(idx != -1);
1691 
1692 
1693 		string contentType = "";
1694 		string[string] requestHeadersHere;
1695 
1696 		size_t contentLength;
1697 
1698 		bool isChunked;
1699 
1700 		{
1701 			import core.runtime;
1702 			scriptFileName = Runtime.args[0];
1703 		}
1704 
1705 
1706 		int headerNumber = 0;
1707 		foreach(line; al.splitter(inputData.front()[0 .. idx], "\r\n"))
1708 		if(line.length) {
1709 			headerNumber++;
1710 			auto header = cast(string) line.idup;
1711 			if(headerNumber == 1) {
1712 				// request line
1713 				auto parts = al.splitter(header, " ");
1714 				requestMethod = to!RequestMethod(parts.front);
1715 				parts.popFront();
1716 				requestUri = parts.front;
1717 
1718 				scriptName = requestUri[0 .. pathInfoStarts];
1719 
1720 				auto question = requestUri.indexOf("?");
1721 				if(question == -1) {
1722 					queryString = "";
1723 					// FIXME: double check, this might be wrong since it could be url encoded
1724 					pathInfo = requestUri[pathInfoStarts..$];
1725 				} else {
1726 					queryString = requestUri[question+1..$];
1727 					pathInfo = requestUri[pathInfoStarts..question];
1728 				}
1729 
1730 				auto ugh = decodeVariables(queryString, "&", &allGetNamesInOrder, &allGetValuesInOrder);
1731 				getArray = cast(string[][string]) assumeUnique(ugh);
1732 
1733 				if(header.indexOf("HTTP/1.0") != -1) {
1734 					http10 = true;
1735 					autoBuffer = true;
1736 					if(closeConnection) {
1737 						// on http 1.0, close is assumed (unlike http/1.1 where we assume keep alive)
1738 						*closeConnection = true;
1739 					}
1740 				}
1741 			} else {
1742 				// other header
1743 				auto colon = header.indexOf(":");
1744 				if(colon == -1)
1745 					throw new Exception("HTTP headers should have a colon!");
1746 				string name = header[0..colon].toLower;
1747 				string value = header[colon+2..$]; // skip the colon and the space
1748 
1749 				requestHeadersHere[name] = value;
1750 
1751 				if (name == "accept") {
1752 					accept = value;
1753 				}
1754 				else if (name == "origin") {
1755 					origin = value;
1756 				}
1757 				else if (name == "connection") {
1758 					if(value == "close" && closeConnection)
1759 						*closeConnection = true;
1760 					if(value.asLowerCase().canFind("keep-alive")) {
1761 						keepAliveRequested = true;
1762 
1763 						// on http 1.0, the connection is closed by default,
1764 						// but not if they request keep-alive. then we don't close
1765 						// anymore - undoing the set above
1766 						if(http10 && closeConnection) {
1767 							*closeConnection = false;
1768 						}
1769 					}
1770 				}
1771 				else if (name == "transfer-encoding") {
1772 					if(value == "chunked")
1773 						isChunked = true;
1774 				}
1775 				else if (name == "last-event-id") {
1776 					lastEventId = value;
1777 				}
1778 				else if (name == "authorization") {
1779 					authorization = value;
1780 				}
1781 				else if (name == "content-type") {
1782 					contentType = value;
1783 				}
1784 				else if (name == "content-length") {
1785 					contentLength = to!size_t(value);
1786 				}
1787 				else if (name == "x-forwarded-for") {
1788 					remoteAddress = value;
1789 				}
1790 				else if (name == "x-forwarded-host" || name == "host") {
1791 					if(name != "host" || host is null)
1792 						host = value;
1793 				}
1794 				// FIXME: https://tools.ietf.org/html/rfc7239
1795 				else if (name == "accept-encoding") {
1796 					if(value.indexOf("gzip") != -1)
1797 						acceptsGzip = true;
1798 				}
1799 				else if (name == "user-agent") {
1800 					userAgent = value;
1801 				}
1802 				else if (name == "referer") {
1803 					referrer = value;
1804 				}
1805 				else if (name == "cookie") {
1806 					cookie ~= value;
1807 				}
1808 				// else
1809 				// ignore it
1810 
1811 			}
1812 		}
1813 
1814 		inputData.consume(idx + 4);
1815 		// done
1816 
1817 		requestHeaders = assumeUnique(requestHeadersHere);
1818 
1819 		ByChunkRange dataByChunk;
1820 
1821 		// reading Content-Length type data
1822 		// We need to read up the data we have, and write it out as a chunk.
1823 		if(!isChunked) {
1824 			dataByChunk = byChunk(inputData, contentLength);
1825 		} else {
1826 			// chunked requests happen, but not every day. Since we need to know
1827 			// the content length (for now, maybe that should change), we'll buffer
1828 			// the whole thing here instead of parse streaming. (I think this is what Apache does anyway in cgi modes)
1829 			auto data = dechunk(inputData);
1830 
1831 			// set the range here
1832 			dataByChunk = byChunk(data);
1833 			contentLength = data.length;
1834 		}
1835 
1836 		assert(dataByChunk !is null);
1837 
1838 		if(contentLength) {
1839 			prepareForIncomingDataChunks(contentType, contentLength);
1840 			foreach(dataChunk; dataByChunk) {
1841 				handleIncomingDataChunk(dataChunk);
1842 			}
1843 			postArray = assumeUnique(pps._post);
1844 			filesArray = assumeUnique(pps._files);
1845 			files = keepLastOf(filesArray);
1846 			post = keepLastOf(postArray);
1847 			postJson = pps.postJson;
1848 			cleanUpPostDataState();
1849 		}
1850 
1851 		this.port = port;
1852 		this.referrer = referrer;
1853 		this.remoteAddress = remoteAddress;
1854 		this.userAgent = userAgent;
1855 		this.authorization = authorization;
1856 		this.origin = origin;
1857 		this.accept = accept;
1858 		this.lastEventId = lastEventId;
1859 		this.https = https;
1860 		this.host = host;
1861 		this.requestMethod = requestMethod;
1862 		this.requestUri = requestUri;
1863 		this.pathInfo = pathInfo;
1864 		this.queryString = queryString;
1865 
1866 		this.scriptName = scriptName;
1867 		this.get = keepLastOf(getArray);
1868 		this.getArray = cast(immutable) getArray;
1869 		this.keepAliveRequested = keepAliveRequested;
1870 		this.acceptsGzip = acceptsGzip;
1871 		this.cookie = cookie;
1872 
1873 		cookiesArray = getCookieArray();
1874 		cookies = keepLastOf(cookiesArray);
1875 
1876 	}
1877 	BufferedInputRange idlol;
1878 
1879 	private immutable(string[string]) keepLastOf(in string[][string] arr) {
1880 		string[string] ca;
1881 		foreach(k, v; arr)
1882 			ca[k] = v[$-1];
1883 
1884 		return assumeUnique(ca);
1885 	}
1886 
1887 	// FIXME duplication
1888 	private immutable(UploadedFile[string]) keepLastOf(in UploadedFile[][string] arr) {
1889 		UploadedFile[string] ca;
1890 		foreach(k, v; arr)
1891 			ca[k] = v[$-1];
1892 
1893 		return assumeUnique(ca);
1894 	}
1895 
1896 
1897 	private immutable(string[][string]) getCookieArray() {
1898 		auto forTheLoveOfGod = decodeVariables(cookie, "; ");
1899 		return assumeUnique(forTheLoveOfGod);
1900 	}
1901 
1902 	/// Very simple method to require a basic auth username and password.
1903 	/// If the http request doesn't include the required credentials, it throws a
1904 	/// HTTP 401 error, and an exception.
1905 	///
1906 	/// Note: basic auth does not provide great security, especially over unencrypted HTTP;
1907 	/// the user's credentials are sent in plain text on every request.
1908 	///
1909 	/// If you are using Apache, the HTTP_AUTHORIZATION variable may not be sent to the
1910 	/// application. Either use Apache's built in methods for basic authentication, or add
1911 	/// something along these lines to your server configuration:
1912 	///
1913 	///      RewriteEngine On 
1914 	///      RewriteCond %{HTTP:Authorization} ^(.*) 
1915 	///      RewriteRule ^(.*) - [E=HTTP_AUTHORIZATION:%1]
1916 	///
1917 	/// To ensure the necessary data is available to cgi.d.
1918 	void requireBasicAuth(string user, string pass, string message = null) {
1919 		if(authorization != "Basic " ~ Base64.encode(cast(immutable(ubyte)[]) (user ~ ":" ~ pass))) {
1920 			setResponseStatus("401 Authorization Required");
1921 			header ("WWW-Authenticate: Basic realm=\""~message~"\"");
1922 			close();
1923 			throw new Exception("Not authorized; got " ~ authorization);
1924 		}
1925 	}
1926 
1927 	/// Very simple caching controls - setCache(false) means it will never be cached. Good for rapidly updated or sensitive sites.
1928 	/// setCache(true) means it will always be cached for as long as possible. Best for static content.
1929 	/// Use setResponseExpires and updateResponseExpires for more control
1930 	void setCache(bool allowCaching) {
1931 		noCache = !allowCaching;
1932 	}
1933 
1934 	/// Set to true and use cgi.write(data, true); to send a gzipped response to browsers
1935 	/// who can accept it
1936 	bool gzipResponse;
1937 
1938 	immutable bool acceptsGzip;
1939 	immutable bool keepAliveRequested;
1940 
1941 	/// Set to true if and only if this was initialized with command line arguments
1942 	immutable bool isCalledWithCommandLineArguments;
1943 
1944 	/// This gets a full url for the current request, including port, protocol, host, path, and query
1945 	string getCurrentCompleteUri() const {
1946 		ushort defaultPort = https ? 443 : 80;
1947 
1948 		return format("http%s://%s%s%s",
1949 			https ? "s" : "",
1950 			host,
1951 			(!port || port == defaultPort) ? "" : ":" ~ to!string(port),
1952 			requestUri);
1953 	}
1954 
1955 	/// You can override this if your site base url isn't the same as the script name
1956 	string logicalScriptName() const {
1957 		return scriptName;
1958 	}
1959 
1960 	/// Sets the HTTP status of the response. For example, "404 File Not Found" or "500 Internal Server Error".
1961 	/// It assumes "200 OK", and automatically changes to "302 Found" if you call setResponseLocation().
1962 	/// Note setResponseStatus() must be called *before* you write() any data to the output.
1963 	void setResponseStatus(string status) {
1964 		assert(!outputtedResponseData);
1965 		responseStatus = status;
1966 	}
1967 	private string responseStatus = null;
1968 
1969 	/// Returns true if it is still possible to output headers
1970 	bool canOutputHeaders() {
1971 		return !isClosed && !outputtedResponseData;
1972 	}
1973 
1974 	/// Sets the location header, which the browser will redirect the user to automatically.
1975 	/// Note setResponseLocation() must be called *before* you write() any data to the output.
1976 	/// The optional important argument is used if it's a default suggestion rather than something to insist upon.
1977 	void setResponseLocation(string uri, bool important = true, string status = null) {
1978 		if(!important && isCurrentResponseLocationImportant)
1979 			return; // important redirects always override unimportant ones
1980 
1981 		if(uri is null) {
1982 			responseStatus = "200 OK";
1983 			responseLocation = null;
1984 			isCurrentResponseLocationImportant = important;
1985 			return; // this just cancels the redirect
1986 		}
1987 
1988 		assert(!outputtedResponseData);
1989 		if(status is null)
1990 			responseStatus = "302 Found";
1991 		else
1992 			responseStatus = status;
1993 
1994 		responseLocation = uri.strip;
1995 		isCurrentResponseLocationImportant = important;
1996 	}
1997 	protected string responseLocation = null;
1998 	private bool isCurrentResponseLocationImportant = false;
1999 
2000 	/// Sets the Expires: http header. See also: updateResponseExpires, setPublicCaching
2001 	/// The parameter is in unix_timestamp * 1000. Try setResponseExpires(getUTCtime() + SOME AMOUNT) for normal use.
2002 	/// Note: the when parameter is different than setCookie's expire parameter.
2003 	void setResponseExpires(long when, bool isPublic = false) {
2004 		responseExpires = when;
2005 		setCache(true); // need to enable caching so the date has meaning
2006 
2007 		responseIsPublic = isPublic;
2008 		responseExpiresRelative = false;
2009 	}
2010 
2011 	/// Sets a cache-control max-age header for whenFromNow, in seconds.
2012 	void setResponseExpiresRelative(int whenFromNow, bool isPublic = false) {
2013 		responseExpires = whenFromNow;
2014 		setCache(true); // need to enable caching so the date has meaning
2015 
2016 		responseIsPublic = isPublic;
2017 		responseExpiresRelative = true;
2018 	}
2019 	private long responseExpires = long.min;
2020 	private bool responseIsPublic = false;
2021 	private bool responseExpiresRelative = false;
2022 
2023 	/// This is like setResponseExpires, but it can be called multiple times. The setting most in the past is the one kept.
2024 	/// If you have multiple functions, they all might call updateResponseExpires about their own return value. The program
2025 	/// output as a whole is as cacheable as the least cachable part in the chain.
2026 
2027 	/// setCache(false) always overrides this - it is, by definition, the strictest anti-cache statement available. If your site outputs sensitive user data, you should probably call setCache(false) when you do, to ensure no other functions will cache the content, as it may be a privacy risk.
2028 	/// Conversely, setting here overrides setCache(true), since any expiration date is in the past of infinity.
2029 	void updateResponseExpires(long when, bool isPublic) {
2030 		if(responseExpires == long.min)
2031 			setResponseExpires(when, isPublic);
2032 		else if(when < responseExpires)
2033 			setResponseExpires(when, responseIsPublic && isPublic); // if any part of it is private, it all is
2034 	}
2035 
2036 	/*
2037 	/// Set to true if you want the result to be cached publically - that is, is the content shared?
2038 	/// Should generally be false if the user is logged in. It assumes private cache only.
2039 	/// setCache(true) also turns on public caching, and setCache(false) sets to private.
2040 	void setPublicCaching(bool allowPublicCaches) {
2041 		publicCaching = allowPublicCaches;
2042 	}
2043 	private bool publicCaching = false;
2044 	*/
2045 
2046 	/// Sets an HTTP cookie, automatically encoding the data to the correct string.
2047 	/// expiresIn is how many milliseconds in the future the cookie will expire.
2048 	/// TIP: to make a cookie accessible from subdomains, set the domain to .yourdomain.com.
2049 	/// Note setCookie() must be called *before* you write() any data to the output.
2050 	void setCookie(string name, string data, long expiresIn = 0, string path = null, string domain = null, bool httpOnly = false, bool secure = false) {
2051 		assert(!outputtedResponseData);
2052 		string cookie = std.uri.encodeComponent(name) ~ "=";
2053 		cookie ~= std.uri.encodeComponent(data);
2054 		if(path !is null)
2055 			cookie ~= "; path=" ~ path;
2056 		// FIXME: should I just be using max-age here? (also in cache below)
2057 		if(expiresIn != 0)
2058 			cookie ~= "; expires=" ~ printDate(cast(DateTime) Clock.currTime(UTC()) + dur!"msecs"(expiresIn));
2059 		if(domain !is null)
2060 			cookie ~= "; domain=" ~ domain;
2061 		if(secure == true)
2062 			cookie ~= "; Secure";
2063 		if(httpOnly == true )
2064 			cookie ~= "; HttpOnly";
2065 
2066 		if(auto idx = name in cookieIndexes) {
2067 			responseCookies[*idx] = cookie;
2068 		} else {
2069 			cookieIndexes[name] = responseCookies.length;
2070 			responseCookies ~= cookie;
2071 		}
2072 	}
2073 	private string[] responseCookies;
2074 	private size_t[string] cookieIndexes;
2075 
2076 	/// Clears a previously set cookie with the given name, path, and domain.
2077 	void clearCookie(string name, string path = null, string domain = null) {
2078 		assert(!outputtedResponseData);
2079 		setCookie(name, "", 1, path, domain);
2080 	}
2081 
2082 	/// Sets the content type of the response, for example "text/html" (the default) for HTML, or "image/png" for a PNG image
2083 	void setResponseContentType(string ct) {
2084 		assert(!outputtedResponseData);
2085 		responseContentType = ct;
2086 	}
2087 	private string responseContentType = null;
2088 
2089 	/// Adds a custom header. It should be the name: value, but without any line terminator.
2090 	/// For example: header("X-My-Header: Some value");
2091 	/// Note you should use the specialized functions in this object if possible to avoid
2092 	/// duplicates in the output.
2093 	void header(string h) {
2094 		customHeaders ~= h;
2095 	}
2096 
2097 	private string[] customHeaders;
2098 	private bool websocketMode;
2099 
2100 	void flushHeaders(const(void)[] t, bool isAll = false) {
2101 		string[] hd;
2102 		// Flush the headers
2103 		if(responseStatus !is null) {
2104 			if(nph) {
2105 				if(http10)
2106 					hd ~= "HTTP/1.0 " ~ responseStatus;
2107 				else
2108 					hd ~= "HTTP/1.1 " ~ responseStatus;
2109 			} else
2110 				hd ~= "Status: " ~ responseStatus;
2111 		} else if (nph) {
2112 			if(http10)
2113 				hd ~= "HTTP/1.0 200 OK";
2114 			else
2115 				hd ~= "HTTP/1.1 200 OK";
2116 		}
2117 
2118 		if(websocketMode)
2119 			goto websocket;
2120 
2121 		if(nph) { // we're responsible for setting the date too according to http 1.1
2122 			hd ~= "Date: " ~ printDate(cast(DateTime) Clock.currTime(UTC()));
2123 		}
2124 
2125 		// FIXME: what if the user wants to set his own content-length?
2126 		// The custom header function can do it, so maybe that's best.
2127 		// Or we could reuse the isAll param.
2128 		if(responseLocation !is null) {
2129 			hd ~= "Location: " ~ responseLocation;
2130 		}
2131 		if(!noCache && responseExpires != long.min) { // an explicit expiration date is set
2132 			if(responseExpiresRelative) {
2133 				hd ~= "Cache-Control: "~(responseIsPublic ? "public" : "private")~", max-age="~to!string(responseExpires)~", no-cache=\"set-cookie, set-cookie2\"";
2134 			} else {
2135 				auto expires = SysTime(unixTimeToStdTime(cast(int)(responseExpires / 1000)), UTC());
2136 				hd ~= "Expires: " ~ printDate(
2137 					cast(DateTime) expires);
2138 				// FIXME: assuming everything is private unless you use nocache - generally right for dynamic pages, but not necessarily
2139 				hd ~= "Cache-Control: "~(responseIsPublic ? "public" : "private")~", no-cache=\"set-cookie, set-cookie2\"";
2140 			}
2141 		}
2142 		if(responseCookies !is null && responseCookies.length > 0) {
2143 			foreach(c; responseCookies)
2144 				hd ~= "Set-Cookie: " ~ c;
2145 		}
2146 		if(noCache) { // we specifically do not want caching (this is actually the default)
2147 			hd ~= "Cache-Control: private, no-cache=\"set-cookie\"";
2148 			hd ~= "Expires: 0";
2149 			hd ~= "Pragma: no-cache";
2150 		} else {
2151 			if(responseExpires == long.min) { // caching was enabled, but without a date set - that means assume cache forever
2152 				hd ~= "Cache-Control: public";
2153 				hd ~= "Expires: Tue, 31 Dec 2030 14:00:00 GMT"; // FIXME: should not be more than one year in the future
2154 			}
2155 		}
2156 		if(responseContentType !is null) {
2157 			hd ~= "Content-Type: " ~ responseContentType;
2158 		} else
2159 			hd ~= "Content-Type: text/html; charset=utf-8";
2160 
2161 		if(gzipResponse && acceptsGzip && isAll) { // FIXME: isAll really shouldn't be necessary
2162 			hd ~= "Content-Encoding: gzip";
2163 		}
2164 
2165 
2166 		if(!isAll) {
2167 			if(nph && !http10) {
2168 				hd ~= "Transfer-Encoding: chunked";
2169 				responseChunked = true;
2170 			}
2171 		} else {
2172 			hd ~= "Content-Length: " ~ to!string(t.length);
2173 			if(nph && keepAliveRequested) {
2174 				hd ~= "Connection: Keep-Alive";
2175 			}
2176 		}
2177 
2178 		websocket:
2179 		if(customHeaders !is null)
2180 			hd ~= customHeaders;
2181 
2182 		// FIXME: what about duplicated headers?
2183 
2184 		foreach(h; hd) {
2185 			if(rawDataOutput !is null)
2186 				rawDataOutput(cast(const(ubyte)[]) (h ~ "\r\n"));
2187 			else {
2188 				version(CRuntime_Musl) {
2189 					stdout.rawWrite(h);
2190 					stdout.rawWrite("\n");
2191 				} else {
2192 					writeln(h);
2193 				}
2194 			}
2195 		}
2196 		if(rawDataOutput !is null)
2197 			rawDataOutput(cast(const(ubyte)[]) ("\r\n"));
2198 		else {
2199 			version(CRuntime_Musl) {
2200 				stdout.rawWrite("\n");
2201 			} else {
2202 				writeln("");
2203 			}
2204 		}
2205 
2206 		outputtedResponseData = true;
2207 	}
2208 
2209 	/// Writes the data to the output, flushing headers if they have not yet been sent.
2210 	void write(const(void)[] t, bool isAll = false, bool maybeAutoClose = true) {
2211 		assert(!closed, "Output has already been closed");
2212 
2213 		if(gzipResponse && acceptsGzip && isAll) { // FIXME: isAll really shouldn't be necessary
2214 			// actually gzip the data here
2215 
2216 			auto c = new Compress(HeaderFormat.gzip); // want gzip
2217 
2218 			auto data = c.compress(t);
2219 			data ~= c.flush();
2220 
2221 			// std.file.write("/tmp/last-item", data);
2222 
2223 			t = data;
2224 		}
2225 
2226 		if(!outputtedResponseData && (!autoBuffer || isAll)) {
2227 			flushHeaders(t, isAll);
2228 		}
2229 
2230 		if(requestMethod != RequestMethod.HEAD && t.length > 0) {
2231 			if (autoBuffer) {
2232 				outputBuffer ~= cast(ubyte[]) t;
2233 			}
2234 			if(!autoBuffer || isAll) {
2235 				if(rawDataOutput !is null)
2236 					if(nph && responseChunked) {
2237 						//rawDataOutput(makeChunk(cast(const(ubyte)[]) t));
2238 						// we're making the chunk here instead of in a function
2239 						// to avoid unneeded gc pressure
2240 						rawDataOutput(cast(const(ubyte)[]) toHex(t.length));
2241 						rawDataOutput(cast(const(ubyte)[]) "\r\n");
2242 						rawDataOutput(cast(const(ubyte)[]) t);
2243 						rawDataOutput(cast(const(ubyte)[]) "\r\n");
2244 
2245 
2246 					} else {
2247 						rawDataOutput(cast(const(ubyte)[]) t);
2248 					}
2249 				else
2250 					stdout.rawWrite(t);
2251 			}
2252 		}
2253 
2254 		if(maybeAutoClose && isAll)
2255 			close(); // if you say it is all, that means we're definitely done
2256 				// maybeAutoClose can be false though to avoid this (important if you call from inside close()!
2257 	}
2258 
2259 	void flush() {
2260 		if(rawDataOutput is null)
2261 			stdout.flush();
2262 		else if(flushDelegate !is null)
2263 			flushDelegate();
2264 	}
2265 
2266 	version(autoBuffer)
2267 		bool autoBuffer = true;
2268 	else
2269 		bool autoBuffer = false;
2270 	ubyte[] outputBuffer;
2271 
2272 	/// Flushes the buffers to the network, signifying that you are done.
2273 	/// You should always call this explicitly when you are done outputting data.
2274 	void close() {
2275 		if(closed)
2276 			return; // don't double close
2277 
2278 		if(!outputtedResponseData)
2279 			write("", false, false);
2280 
2281 		// writing auto buffered data
2282 		if(requestMethod != RequestMethod.HEAD && autoBuffer) {
2283 			if(!nph)
2284 				stdout.rawWrite(outputBuffer);
2285 			else
2286 				write(outputBuffer, true, false); // tell it this is everything
2287 		}
2288 
2289 		// closing the last chunk...
2290 		if(nph && rawDataOutput !is null && responseChunked)
2291 			rawDataOutput(cast(const(ubyte)[]) "0\r\n\r\n");
2292 
2293 		if(flushDelegate)
2294 			flushDelegate();
2295 
2296 		closed = true;
2297 	}
2298 
2299 	// Closes without doing anything, shouldn't be used often
2300 	void rawClose() {
2301 		closed = true;
2302 	}
2303 
2304 	/++
2305 		Gets a request variable as a specific type, or the default value of it isn't there
2306 		or isn't convertible to the request type.
2307 		
2308 		Checks both GET and POST variables, preferring the POST variable, if available.
2309 
2310 		A nice trick is using the default value to choose the type:
2311 
2312 		---
2313 			/*
2314 				The return value will match the type of the default.
2315 				Here, I gave 10 as a default, so the return value will
2316 				be an int.
2317 
2318 				If the user-supplied value cannot be converted to the
2319 				requested type, you will get the default value back.
2320 			*/
2321 			int a = cgi.request("number", 10);
2322 
2323 			if(cgi.get["number"] == "11")
2324 				assert(a == 11); // conversion succeeds
2325 
2326 			if("number" !in cgi.get)
2327 				assert(a == 10); // no value means you can't convert - give the default
2328 
2329 			if(cgi.get["number"] == "twelve")
2330 				assert(a == 10); // conversion from string to int would fail, so we get the default
2331 		---
2332 
2333 		You can use an enum as an easy whitelist, too:
2334 
2335 		---
2336 			enum Operations {
2337 				add, remove, query
2338 			}
2339 
2340 			auto op = cgi.request("op", Operations.query);
2341 
2342 			if(cgi.get["op"] == "add")
2343 				assert(op == Operations.add);
2344 			if(cgi.get["op"] == "remove")
2345 				assert(op == Operations.remove);
2346 			if(cgi.get["op"] == "query")
2347 				assert(op == Operations.query);
2348 
2349 			if(cgi.get["op"] == "random string")
2350 				assert(op == Operations.query); // the value can't be converted to the enum, so we get the default
2351 		---
2352 	+/
2353 	T request(T = string)(in string name, in T def = T.init) const nothrow {
2354 		try {
2355 			return
2356 				(name in post) ? to!T(post[name]) :
2357 				(name in get)  ? to!T(get[name]) :
2358 				def;
2359 		} catch(Exception e) { return def; }
2360 	}
2361 
2362 	/// Is the output already closed?
2363 	bool isClosed() const {
2364 		return closed;
2365 	}
2366 
2367 	/++
2368 		Gets a session object associated with the `cgi` request. You can use different type throughout your application.
2369 	+/
2370 	Session!Data getSessionObject(Data)() {
2371 		if(testInProcess !is null) {
2372 			// test mode
2373 			auto obj = testInProcess.getSessionOverride(typeid(typeof(return)));
2374 			if(obj !is null)
2375 				return cast(typeof(return)) obj;
2376 			else {
2377 				auto o = new MockSession!Data();
2378 				testInProcess.setSessionOverride(typeid(typeof(return)), o);
2379 				return o;
2380 			}
2381 		} else {
2382 			// normal operation
2383 			return new BasicDataServerSession!Data(this);
2384 		}
2385 	}
2386 
2387 	// if it is in test mode; triggers mock sessions. Used by CgiTester
2388 	version(with_breaking_cgi_features)
2389 	private CgiTester testInProcess;
2390 
2391 	/* Hooks for redirecting input and output */
2392 	private void delegate(const(ubyte)[]) rawDataOutput = null;
2393 	private void delegate() flushDelegate = null;
2394 
2395 	/* This info is used when handling a more raw HTTP protocol */
2396 	private bool nph;
2397 	private bool http10;
2398 	private bool closed;
2399 	private bool responseChunked = false;
2400 
2401 	version(preserveData) // note: this can eat lots of memory; don't use unless you're sure you need it.
2402 	immutable(ubyte)[] originalPostData;
2403 
2404 	public immutable string postJson;
2405 
2406 	/* Internal state flags */
2407 	private bool outputtedResponseData;
2408 	private bool noCache = true;
2409 
2410 	const(string[string]) environmentVariables;
2411 
2412 	/** What follows is data gotten from the HTTP request. It is all fully immutable,
2413 	    partially because it logically is (your code doesn't change what the user requested...)
2414 	    and partially because I hate how bad programs in PHP change those superglobals to do
2415 	    all kinds of hard to follow ugliness. I don't want that to ever happen in D.
2416 
2417 	    For some of these, you'll want to refer to the http or cgi specs for more details.
2418 	*/
2419 	immutable(string[string]) requestHeaders; /// All the raw headers in the request as name/value pairs. The name is stored as all lower case, but otherwise the same as it is in HTTP; words separated by dashes. For example, "cookie" or "accept-encoding". Many HTTP headers have specialized variables below for more convenience and static name checking; you should generally try to use them.
2420 
2421 	immutable(char[]) host; 	/// The hostname in the request. If one program serves multiple domains, you can use this to differentiate between them.
2422 	immutable(char[]) origin; 	/// The origin header in the request, if present. Some HTML5 cross-domain apis set this and you should check it on those cross domain requests and websockets.
2423 	immutable(char[]) userAgent; 	/// The browser's user-agent string. Can be used to identify the browser.
2424 	immutable(char[]) pathInfo; 	/// This is any stuff sent after your program's name on the url, but before the query string. For example, suppose your program is named "app". If the user goes to site.com/app, pathInfo is empty. But, he can also go to site.com/app/some/sub/path; treating your program like a virtual folder. In this case, pathInfo == "/some/sub/path".
2425 	immutable(char[]) scriptName;   /// The full base path of your program, as seen by the user. If your program is located at site.com/programs/apps, scriptName == "/programs/apps".
2426 	immutable(char[]) scriptFileName;   /// The physical filename of your script
2427 	immutable(char[]) authorization; /// The full authorization string from the header, undigested. Useful for implementing auth schemes such as OAuth 1.0. Note that some web servers do not forward this to the app without taking extra steps. See requireBasicAuth's comment for more info.
2428 	immutable(char[]) accept; 	/// The HTTP accept header is the user agent telling what content types it is willing to accept. This is often */*; they accept everything, so it's not terribly useful. (The similar sounding Accept-Encoding header is handled automatically for chunking and gzipping. Simply set gzipResponse = true and cgi.d handles the details, zipping if the user's browser is willing to accept it.)
2429 	immutable(char[]) lastEventId; 	/// The HTML 5 draft includes an EventSource() object that connects to the server, and remains open to take a stream of events. My arsd.rtud module can help with the server side part of that. The Last-Event-Id http header is defined in the draft to help handle loss of connection. When the browser reconnects to you, it sets this header to the last event id it saw, so you can catch it up. This member has the contents of that header.
2430 
2431 	immutable(RequestMethod) requestMethod; /// The HTTP request verb: GET, POST, etc. It is represented as an enum in cgi.d (which, like many enums, you can convert back to string with std.conv.to()). A HTTP GET is supposed to, according to the spec, not have side effects; a user can GET something over and over again and always have the same result. On all requests, the get[] and getArray[] members may be filled in. The post[] and postArray[] members are only filled in on POST methods.
2432 	immutable(char[]) queryString; 	/// The unparsed content of the request query string - the stuff after the ? in your URL. See get[] and getArray[] for a parse view of it. Sometimes, the unparsed string is useful though if you want a custom format of data up there (probably not a good idea, unless it is really simple, like "?username" perhaps.)
2433 	immutable(char[]) cookie; 	/// The unparsed content of the Cookie: header in the request. See also the cookies[string] member for a parsed view of the data.
2434 	/** The Referer header from the request. (It is misspelled in the HTTP spec, and thus the actual request and cgi specs too, but I spelled the word correctly here because that's sane. The spec's misspelling is an implementation detail.) It contains the site url that referred the user to your program; the site that linked to you, or if you're serving images, the site that has you as an image. Also, if you're in an iframe, the referrer is the site that is framing you.
2435 
2436 	Important note: if the user copy/pastes your url, this is blank, and, just like with all other user data, their browsers can also lie to you. Don't rely on it for real security.
2437 	*/
2438 	immutable(char[]) referrer;
2439 	immutable(char[]) requestUri; 	/// The full url if the current request, excluding the protocol and host. requestUri == scriptName ~ pathInfo ~ (queryString.length ? "?" ~ queryString : "");
2440 
2441 	immutable(char[]) remoteAddress; /// The IP address of the user, as we see it. (Might not match the IP of the user's computer due to things like proxies and NAT.) 
2442 
2443 	immutable bool https; 	/// Was the request encrypted via https?
2444 	immutable int port; 	/// On what TCP port number did the server receive the request?
2445 
2446 	/** Here come the parsed request variables - the things that come close to PHP's _GET, _POST, etc. superglobals in content. */
2447 
2448 	immutable(string[string]) get; 	/// The data from your query string in the url, only showing the last string of each name. If you want to handle multiple values with the same name, use getArray. This only works right if the query string is x-www-form-urlencoded; the default you see on the web with name=value pairs separated by the & character.
2449 	immutable(string[string]) post; /// The data from the request's body, on POST requests. It parses application/x-www-form-urlencoded data (used by most web requests, including typical forms), and multipart/form-data requests (used by file uploads on web forms) into the same container, so you can always access them the same way. It makes no attempt to parse other content types. If you want to accept an XML Post body (for a web api perhaps), you'll need to handle the raw data yourself.
2450 	immutable(string[string]) cookies; /// Separates out the cookie header into individual name/value pairs (which is how you set them!)
2451 
2452 	/**
2453 		Represents user uploaded files.
2454 		
2455 		When making a file upload form, be sure to follow the standard: set method="POST" and enctype="multipart/form-data" in your html <form> tag attributes. The key into this array is the name attribute on your input tag, just like with other post variables. See the comments on the UploadedFile struct for more information about the data inside, including important notes on max size and content location.
2456 	*/
2457 	immutable(UploadedFile[][string]) filesArray;
2458 	immutable(UploadedFile[string]) files;
2459 
2460 	/// Use these if you expect multiple items submitted with the same name. btw, assert(get[name] is getArray[name][$-1); should pass. Same for post and cookies.
2461 	/// the order of the arrays is the order the data arrives
2462 	immutable(string[][string]) getArray; /// like get, but an array of values per name
2463 	immutable(string[][string]) postArray; /// ditto for post
2464 	immutable(string[][string]) cookiesArray; /// ditto for cookies
2465 
2466 	// convenience function for appending to a uri without extra ?
2467 	// matches the name and effect of javascript's location.search property
2468 	string search() const {
2469 		if(queryString.length)
2470 			return "?" ~ queryString;
2471 		return "";
2472 	}
2473 
2474 	// FIXME: what about multiple files with the same name?
2475   private:
2476 	//RequestMethod _requestMethod;
2477 }
2478 
2479 /// use this for testing or other isolated things when you want it to be no-ops
2480 Cgi dummyCgi(Cgi.RequestMethod method = Cgi.RequestMethod.GET, string url = null, in ubyte[] data = null, void delegate(const(ubyte)[]) outputSink = null) {
2481 	// we want to ignore, not use stdout
2482 	if(outputSink is null)
2483 		outputSink = delegate void(const(ubyte)[]) { };
2484 
2485 	string[string] env;
2486 	env["REQUEST_METHOD"] = to!string(method);
2487 	env["CONTENT_LENGTH"] = to!string(data.length);
2488 
2489 	auto cgi = new Cgi(
2490 		0,
2491 		env,
2492 		{ return data; },
2493 		outputSink,
2494 		null);
2495 
2496 	return cgi;
2497 }
2498 
2499 /++
2500 	A helper test class for request handler unittests.
2501 +/
2502 version(with_breaking_cgi_features)
2503 class CgiTester {
2504 	private {
2505 		SessionObject[TypeInfo] mockSessions;
2506 		SessionObject getSessionOverride(TypeInfo ti) {
2507 			if(auto o = ti in mockSessions)
2508 				return *o;
2509 			else
2510 				return null;
2511 		}
2512 		void setSessionOverride(TypeInfo ti, SessionObject so) {
2513 			mockSessions[ti] = so;
2514 		}
2515 	}
2516 
2517 	/++
2518 		Gets (and creates if necessary) a mock session object for this test. Note
2519 		it will be the same one used for any test operations through this CgiTester instance.
2520 	+/
2521 	Session!Data getSessionObject(Data)() {
2522 		auto obj = getSessionOverride(typeid(typeof(return)));
2523 		if(obj !is null)
2524 			return cast(typeof(return)) obj;
2525 		else {
2526 			auto o = new MockSession!Data();
2527 			setSessionOverride(typeid(typeof(return)), o);
2528 			return o;
2529 		}
2530 	}
2531 
2532 	/++
2533 		Pass a reference to your request handler when creating the tester.
2534 	+/
2535 	this(void function(Cgi) requestHandler) {
2536 		this.requestHandler = requestHandler;
2537 	}
2538 
2539 	/++
2540 		You can check response information with these methods after you call the request handler.
2541 	+/
2542 	struct Response {
2543 		int code;
2544 		string[string] headers;
2545 		string responseText;
2546 		ubyte[] responseBody;
2547 	}
2548 
2549 	/++
2550 		Executes a test request on your request handler, and returns the response.
2551 
2552 		Params:
2553 			url = The URL to test. Should be an absolute path, but excluding domain. e.g. `"/test"`.
2554 			args = additional arguments. Same format as cgi's command line handler.
2555 	+/
2556 	Response GET(string url, string[] args = null) {
2557 		return executeTest("GET", url, args);
2558 	}
2559 	/// ditto
2560 	Response POST(string url, string[] args = null) {
2561 		return executeTest("POST", url, args);
2562 	}
2563 
2564 	/// ditto
2565 	Response executeTest(string method, string url, string[] args) {
2566 		ubyte[] outputtedRawData;
2567 		void outputSink(const(ubyte)[] data) {
2568 			outputtedRawData ~= data;
2569 		}
2570 		auto cgi = new Cgi(["test", method, url] ~ args, &outputSink);
2571 		cgi.testInProcess = this;
2572 		scope(exit) cgi.dispose();
2573 
2574 		requestHandler(cgi);
2575 
2576 		cgi.close();
2577 
2578 		Response response;
2579 
2580 		if(outputtedRawData.length) {
2581 			enum LINE = "\r\n";
2582 
2583 			auto idx = outputtedRawData.locationOf(LINE ~ LINE);
2584 			assert(idx != -1, to!string(outputtedRawData));
2585 			auto headers = cast(string) outputtedRawData[0 .. idx];
2586 			response.code = 200;
2587 			while(headers.length) {
2588 				auto i = headers.locationOf(LINE);
2589 				if(i == -1) i = cast(int) headers.length;
2590 
2591 				auto header = headers[0 .. i];
2592 
2593 				auto c = header.locationOf(":");
2594 				if(c != -1) {
2595 					auto name = header[0 .. c];
2596 					auto value = header[c + 2 ..$];
2597 
2598 					if(name == "Status")
2599 						response.code = value[0 .. value.locationOf(" ")].to!int;
2600 
2601 					response.headers[name] = value;
2602 				} else {
2603 					assert(0);
2604 				}
2605 
2606 				if(i != headers.length)
2607 					i += 2;
2608 				headers = headers[i .. $];
2609 			}
2610 			response.responseBody = outputtedRawData[idx + 4 .. $];
2611 			response.responseText = cast(string) response.responseBody;
2612 		}
2613 
2614 		return response;
2615 	}
2616 
2617 	private void function(Cgi) requestHandler;
2618 }
2619 
2620 
2621 // should this be a separate module? Probably, but that's a hassle.
2622 
2623 /// Makes a data:// uri that can be used as links in most newer browsers (IE8+).
2624 string makeDataUrl(string mimeType, in void[] data) {
2625 	auto data64 = Base64.encode(cast(const(ubyte[])) data);
2626 	return "data:" ~ mimeType ~ ";base64," ~ assumeUnique(data64);
2627 }
2628 
2629 // FIXME: I don't think this class correctly decodes/encodes the individual parts
2630 /// Represents a url that can be broken down or built up through properties
2631 struct Uri {
2632 	alias toString this; // blargh idk a url really is a string, but should it be implicit?
2633 
2634 	// scheme//userinfo@host:port/path?query#fragment
2635 
2636 	string scheme; /// e.g. "http" in "http://example.com/"
2637 	string userinfo; /// the username (and possibly a password) in the uri
2638 	string host; /// the domain name
2639 	int port; /// port number, if given. Will be zero if a port was not explicitly given
2640 	string path; /// e.g. "/folder/file.html" in "http://example.com/folder/file.html"
2641 	string query; /// the stuff after the ? in a uri
2642 	string fragment; /// the stuff after the # in a uri.
2643 
2644 	// idk if i want to keep these, since the functions they wrap are used many, many, many times in existing code, so this is either an unnecessary alias or a gratuitous break of compatibility
2645 	// the decode ones need to keep different names anyway because we can't overload on return values...
2646 	static string encode(string s) { return std.uri.encodeComponent(s); }
2647 	static string encode(string[string] s) { return encodeVariables(s); }
2648 	static string encode(string[][string] s) { return encodeVariables(s); }
2649 
2650 	/// Breaks down a uri string to its components
2651 	this(string uri) {
2652 		reparse(uri);
2653 	}
2654 
2655 	private void reparse(string uri) {
2656 		// from RFC 3986
2657 		// the ctRegex triples the compile time and makes ugly errors for no real benefit
2658 		// it was a nice experiment but just not worth it.
2659 		// enum ctr = ctRegex!r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?";
2660 		/*
2661 			Captures:
2662 				0 = whole url
2663 				1 = scheme, with :
2664 				2 = scheme, no :
2665 				3 = authority, with //
2666 				4 = authority, no //
2667 				5 = path
2668 				6 = query string, with ?
2669 				7 = query string, no ?
2670 				8 = anchor, with #
2671 				9 = anchor, no #
2672 		*/
2673 		// Yikes, even regular, non-CT regex is also unacceptably slow to compile. 1.9s on my computer!
2674 		// instead, I will DIY and cut that down to 0.6s on the same computer.
2675 		/*
2676 
2677 				Note that authority is
2678 					user:password@domain:port
2679 				where the user:password@ part is optional, and the :port is optional.
2680 
2681 				Regex translation:
2682 
2683 				Scheme cannot have :, /, ?, or # in it, and must have one or more chars and end in a :. It is optional, but must be first.
2684 				Authority must start with //, but cannot have any other /, ?, or # in it. It is optional.
2685 				Path cannot have any ? or # in it. It is optional.
2686 				Query must start with ? and must not have # in it. It is optional.
2687 				Anchor must start with # and can have anything else in it to end of string. It is optional.
2688 		*/
2689 
2690 		this = Uri.init; // reset all state
2691 
2692 		// empty uri = nothing special
2693 		if(uri.length == 0) {
2694 			return;
2695 		}
2696 
2697 		size_t idx;
2698 
2699 		scheme_loop: foreach(char c; uri[idx .. $]) {
2700 			switch(c) {
2701 				case ':':
2702 				case '/':
2703 				case '?':
2704 				case '#':
2705 					break scheme_loop;
2706 				default:
2707 			}
2708 			idx++;
2709 		}
2710 
2711 		if(idx == 0 && uri[idx] == ':') {
2712 			// this is actually a path! we skip way ahead
2713 			goto path_loop;
2714 		}
2715 
2716 		if(idx == uri.length) {
2717 			// the whole thing is a path, apparently
2718 			path = uri;
2719 			return;
2720 		}
2721 
2722 		if(idx > 0 && uri[idx] == ':') {
2723 			scheme = uri[0 .. idx];
2724 			idx++;
2725 		} else {
2726 			// we need to rewind; it found a / but no :, so the whole thing is prolly a path...
2727 			idx = 0;
2728 		}
2729 
2730 		if(idx + 2 < uri.length && uri[idx .. idx + 2] == "//") {
2731 			// we have an authority....
2732 			idx += 2;
2733 
2734 			auto authority_start = idx;
2735 			authority_loop: foreach(char c; uri[idx .. $]) {
2736 				switch(c) {
2737 					case '/':
2738 					case '?':
2739 					case '#':
2740 						break authority_loop;
2741 					default:
2742 				}
2743 				idx++;
2744 			}
2745 
2746 			auto authority = uri[authority_start .. idx];
2747 
2748 			auto idx2 = authority.indexOf("@");
2749 			if(idx2 != -1) {
2750 				userinfo = authority[0 .. idx2];
2751 				authority = authority[idx2 + 1 .. $];
2752 			}
2753 
2754 			idx2 = authority.indexOf(":");
2755 			if(idx2 == -1) {
2756 				port = 0; // 0 means not specified; we should use the default for the scheme
2757 				host = authority;
2758 			} else {
2759 				host = authority[0 .. idx2];
2760 				port = to!int(authority[idx2 + 1 .. $]);
2761 			}
2762 		}
2763 
2764 		path_loop:
2765 		auto path_start = idx;
2766 		
2767 		foreach(char c; uri[idx .. $]) {
2768 			if(c == '?' || c == '#')
2769 				break;
2770 			idx++;
2771 		}
2772 
2773 		path = uri[path_start .. idx];
2774 
2775 		if(idx == uri.length)
2776 			return; // nothing more to examine...
2777 
2778 		if(uri[idx] == '?') {
2779 			idx++;
2780 			auto query_start = idx;
2781 			foreach(char c; uri[idx .. $]) {
2782 				if(c == '#')
2783 					break;
2784 				idx++;
2785 			}
2786 			query = uri[query_start .. idx];
2787 		}
2788 
2789 		if(idx < uri.length && uri[idx] == '#') {
2790 			idx++;
2791 			fragment = uri[idx .. $];
2792 		}
2793 
2794 		// uriInvalidated = false;
2795 	}
2796 
2797 	private string rebuildUri() const {
2798 		string ret;
2799 		if(scheme.length)
2800 			ret ~= scheme ~ ":";
2801 		if(userinfo.length || host.length)
2802 			ret ~= "//";
2803 		if(userinfo.length)
2804 			ret ~= userinfo ~ "@";
2805 		if(host.length)
2806 			ret ~= host;
2807 		if(port)
2808 			ret ~= ":" ~ to!string(port);
2809 
2810 		ret ~= path;
2811 
2812 		if(query.length)
2813 			ret ~= "?" ~ query;
2814 
2815 		if(fragment.length)
2816 			ret ~= "#" ~ fragment;
2817 
2818 		// uri = ret;
2819 		// uriInvalidated = false;
2820 		return ret;
2821 	}
2822 
2823 	/// Converts the broken down parts back into a complete string
2824 	string toString() const {
2825 		// if(uriInvalidated)
2826 			return rebuildUri();
2827 	}
2828 
2829 	/// Returns a new absolute Uri given a base. It treats this one as
2830 	/// relative where possible, but absolute if not. (If protocol, domain, or
2831 	/// other info is not set, the new one inherits it from the base.)
2832 	///
2833 	/// Browsers use a function like this to figure out links in html.
2834 	Uri basedOn(in Uri baseUrl) const {
2835 		Uri n = this; // copies
2836 		// n.uriInvalidated = true; // make sure we regenerate...
2837 
2838 		// userinfo is not inherited... is this wrong?
2839 
2840 		// if anything is given in the existing url, we don't use the base anymore.
2841 		if(n.scheme.empty) {
2842 			n.scheme = baseUrl.scheme;
2843 			if(n.host.empty) {
2844 				n.host = baseUrl.host;
2845 				if(n.port == 0) {
2846 					n.port = baseUrl.port;
2847 					if(n.path.length > 0 && n.path[0] != '/') {
2848 						auto b = baseUrl.path[0 .. baseUrl.path.lastIndexOf("/") + 1];
2849 						if(b.length == 0)
2850 							b = "/";
2851 						n.path = b ~ n.path;
2852 					} else if(n.path.length == 0) {
2853 						n.path = baseUrl.path;
2854 					}
2855 				}
2856 			}
2857 		}
2858 
2859 		n.removeDots();
2860 
2861 		return n;
2862 	}
2863 
2864 	void removeDots() {
2865 		auto parts = this.path.split("/");
2866 		string[] toKeep;
2867 		foreach(part; parts) {
2868 			if(part == ".") {
2869 				continue;
2870 			} else if(part == "..") {
2871 				toKeep = toKeep[0 .. $-1];
2872 				continue;
2873 			} else {
2874 				toKeep ~= part;
2875 			}
2876 		}
2877 
2878 		this.path = toKeep.join("/");
2879 	}
2880 
2881 	unittest {
2882 		auto uri = Uri("test.html");
2883 		assert(uri.path == "test.html");
2884 		uri = Uri("path/1/lol");
2885 		assert(uri.path == "path/1/lol");
2886 		uri = Uri("http://me@example.com");
2887 		assert(uri.scheme == "http");
2888 		assert(uri.userinfo == "me");
2889 		assert(uri.host == "example.com");
2890 		uri = Uri("http://example.com/#a");
2891 		assert(uri.scheme == "http");
2892 		assert(uri.host == "example.com");
2893 		assert(uri.fragment == "a");
2894 		uri = Uri("#foo");
2895 		assert(uri.fragment == "foo");
2896 		uri = Uri("?lol");
2897 		assert(uri.query == "lol");
2898 		uri = Uri("#foo?lol");
2899 		assert(uri.fragment == "foo?lol");
2900 		uri = Uri("?lol#foo");
2901 		assert(uri.fragment == "foo");
2902 		assert(uri.query == "lol");
2903 	}
2904 
2905 	// This can sometimes be a big pain in the butt for me, so lots of copy/paste here to cover
2906 	// the possibilities.
2907 	unittest {
2908 		auto url = Uri("cool.html"); // checking relative links
2909 
2910 		assert(url.basedOn(Uri("http://test.com/what/test.html")) == "http://test.com/what/cool.html");
2911 		assert(url.basedOn(Uri("https://test.com/what/test.html")) == "https://test.com/what/cool.html");
2912 		assert(url.basedOn(Uri("http://test.com/what/")) == "http://test.com/what/cool.html");
2913 		assert(url.basedOn(Uri("http://test.com/")) == "http://test.com/cool.html");
2914 		assert(url.basedOn(Uri("http://test.com")) == "http://test.com/cool.html");
2915 		assert(url.basedOn(Uri("http://test.com/what/test.html?a=b")) == "http://test.com/what/cool.html");
2916 		assert(url.basedOn(Uri("http://test.com/what/test.html?a=b&c=d")) == "http://test.com/what/cool.html");
2917 		assert(url.basedOn(Uri("http://test.com/what/test.html?a=b&c=d#what")) == "http://test.com/what/cool.html");
2918 		assert(url.basedOn(Uri("http://test.com")) == "http://test.com/cool.html");
2919 
2920 		url = Uri("/something/cool.html"); // same server, different path
2921 		assert(url.basedOn(Uri("http://test.com/what/test.html")) == "http://test.com/something/cool.html");
2922 		assert(url.basedOn(Uri("https://test.com/what/test.html")) == "https://test.com/something/cool.html");
2923 		assert(url.basedOn(Uri("http://test.com/what/")) == "http://test.com/something/cool.html");
2924 		assert(url.basedOn(Uri("http://test.com/")) == "http://test.com/something/cool.html");
2925 		assert(url.basedOn(Uri("http://test.com")) == "http://test.com/something/cool.html");
2926 		assert(url.basedOn(Uri("http://test.com/what/test.html?a=b")) == "http://test.com/something/cool.html");
2927 		assert(url.basedOn(Uri("http://test.com/what/test.html?a=b&c=d")) == "http://test.com/something/cool.html");
2928 		assert(url.basedOn(Uri("http://test.com/what/test.html?a=b&c=d#what")) == "http://test.com/something/cool.html");
2929 		assert(url.basedOn(Uri("http://test.com")) == "http://test.com/something/cool.html");
2930 
2931 		url = Uri("?query=answer"); // same path. server, protocol, and port, just different query string and fragment
2932 		assert(url.basedOn(Uri("http://test.com/what/test.html")) == "http://test.com/what/test.html?query=answer");
2933 		assert(url.basedOn(Uri("https://test.com/what/test.html")) == "https://test.com/what/test.html?query=answer");
2934 		assert(url.basedOn(Uri("http://test.com/what/")) == "http://test.com/what/?query=answer");
2935 		assert(url.basedOn(Uri("http://test.com/")) == "http://test.com/?query=answer");
2936 		assert(url.basedOn(Uri("http://test.com")) == "http://test.com?query=answer");
2937 		assert(url.basedOn(Uri("http://test.com/what/test.html?a=b")) == "http://test.com/what/test.html?query=answer");
2938 		assert(url.basedOn(Uri("http://test.com/what/test.html?a=b&c=d")) == "http://test.com/what/test.html?query=answer");
2939 		assert(url.basedOn(Uri("http://test.com/what/test.html?a=b&c=d#what")) == "http://test.com/what/test.html?query=answer");
2940 		assert(url.basedOn(Uri("http://test.com")) == "http://test.com?query=answer");
2941 
2942 		url = Uri("/test/bar");
2943 		assert(Uri("./").basedOn(url) == "/test/", Uri("./").basedOn(url));
2944 		assert(Uri("../").basedOn(url) == "/");
2945 
2946 		//auto uriBefore = url;
2947 		url = Uri("#anchor"); // everything should remain the same except the anchor
2948 		//uriBefore.anchor = "anchor");
2949 		//assert(url == uriBefore);
2950 
2951 		url = Uri("//example.com"); // same protocol, but different server. the path here should be blank.
2952 
2953 		url = Uri("//example.com/example.html"); // same protocol, but different server and path
2954 
2955 		url = Uri("http://example.com/test.html"); // completely absolute link should never be modified
2956 
2957 		url = Uri("http://example.com"); // completely absolute link should never be modified, even if it has no path
2958 
2959 		// FIXME: add something for port too
2960 	}
2961 
2962 	// these are like javascript's location.search and location.hash
2963 	string search() const {
2964 		return query.length ? ("?" ~ query) : "";
2965 	}
2966 	string hash() const {
2967 		return fragment.length ? ("#" ~ fragment) : "";
2968 	}
2969 }
2970 
2971 
2972 /*
2973 	for session, see web.d
2974 */
2975 
2976 /// breaks down a url encoded string
2977 string[][string] decodeVariables(string data, string separator = "&", string[]* namesInOrder = null, string[]* valuesInOrder = null) {
2978 	auto vars = data.split(separator);
2979 	string[][string] _get;
2980 	foreach(var; vars) {
2981 		auto equal = var.indexOf("=");
2982 		string name;
2983 		string value;
2984 		if(equal == -1) {
2985 			name = decodeComponent(var);
2986 			value = "";
2987 		} else {
2988 			//_get[decodeComponent(var[0..equal])] ~= decodeComponent(var[equal + 1 .. $].replace("+", " "));
2989 			// stupid + -> space conversion.
2990 			name = decodeComponent(var[0..equal].replace("+", " "));
2991 			value = decodeComponent(var[equal + 1 .. $].replace("+", " "));
2992 		}
2993 
2994 		_get[name] ~= value;
2995 		if(namesInOrder)
2996 			(*namesInOrder) ~= name;
2997 		if(valuesInOrder)
2998 			(*valuesInOrder) ~= value;
2999 	}
3000 	return _get;
3001 }
3002 
3003 /// breaks down a url encoded string, but only returns the last value of any array
3004 string[string] decodeVariablesSingle(string data) {
3005 	string[string] va;
3006 	auto varArray = decodeVariables(data);
3007 	foreach(k, v; varArray)
3008 		va[k] = v[$-1];
3009 
3010 	return va;
3011 }
3012 
3013 /// url encodes the whole string
3014 string encodeVariables(in string[string] data) {
3015 	string ret;
3016 
3017 	bool outputted = false;
3018 	foreach(k, v; data) {
3019 		if(outputted)
3020 			ret ~= "&";
3021 		else
3022 			outputted = true;
3023 
3024 		ret ~= std.uri.encodeComponent(k) ~ "=" ~ std.uri.encodeComponent(v);
3025 	}
3026 
3027 	return ret;
3028 }
3029 
3030 /// url encodes a whole string
3031 string encodeVariables(in string[][string] data) {
3032 	string ret;
3033 
3034 	bool outputted = false;
3035 	foreach(k, arr; data) {
3036 		foreach(v; arr) {
3037 			if(outputted)
3038 				ret ~= "&";
3039 			else
3040 				outputted = true;
3041 			ret ~= std.uri.encodeComponent(k) ~ "=" ~ std.uri.encodeComponent(v);
3042 		}
3043 	}
3044 
3045 	return ret;
3046 }
3047 
3048 /// Encodes all but the explicitly unreserved characters per rfc 3986
3049 /// Alphanumeric and -_.~ are the only ones left unencoded
3050 /// name is borrowed from php
3051 string rawurlencode(in char[] data) {
3052 	string ret;
3053 	ret.reserve(data.length * 2);
3054 	foreach(char c; data) {
3055 		if(
3056 			(c >= 'a' && c <= 'z') ||
3057 			(c >= 'A' && c <= 'Z') ||
3058 			(c >= '0' && c <= '9') ||
3059 			c == '-' || c == '_' || c == '.' || c == '~')
3060 		{
3061 			ret ~= c;
3062 		} else {
3063 			ret ~= '%';
3064 			// since we iterate on char, this should give us the octets of the full utf8 string
3065 			ret ~= toHexUpper(c);
3066 		}
3067 	}
3068 
3069 	return ret;
3070 }
3071 
3072 
3073 // http helper functions
3074 
3075 // for chunked responses (which embedded http does whenever possible)
3076 version(none) // this is moved up above to avoid making a copy of the data
3077 const(ubyte)[] makeChunk(const(ubyte)[] data) {
3078 	const(ubyte)[] ret;
3079 
3080 	ret = cast(const(ubyte)[]) toHex(data.length);
3081 	ret ~= cast(const(ubyte)[]) "\r\n";
3082 	ret ~= data;
3083 	ret ~= cast(const(ubyte)[]) "\r\n";
3084 
3085 	return ret;
3086 }
3087 
3088 string toHex(long num) {
3089 	string ret;
3090 	while(num) {
3091 		int v = num % 16;
3092 		num /= 16;
3093 		char d = cast(char) ((v < 10) ? v + '0' : (v-10) + 'a');
3094 		ret ~= d;
3095 	}
3096 
3097 	return to!string(array(ret.retro));
3098 }
3099 
3100 string toHexUpper(long num) {
3101 	string ret;
3102 	while(num) {
3103 		int v = num % 16;
3104 		num /= 16;
3105 		char d = cast(char) ((v < 10) ? v + '0' : (v-10) + 'A');
3106 		ret ~= d;
3107 	}
3108 
3109 	if(ret.length == 1)
3110 		ret ~= "0"; // url encoding requires two digits and that's what this function is used for...
3111 
3112 	return to!string(array(ret.retro));
3113 }
3114 
3115 
3116 // the generic mixins
3117 
3118 /// Use this instead of writing your own main
3119 mixin template GenericMain(alias fun, long maxContentLength = defaultMaxContentLength) {
3120 	mixin CustomCgiMain!(Cgi, fun, maxContentLength);
3121 }
3122 
3123 private string simpleHtmlEncode(string s) {
3124 	return s.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace("\n", "<br />\n");
3125 }
3126 
3127 string messageFromException(Throwable t) {
3128 	string message;
3129 	if(t !is null) {
3130 		debug message = t.toString();
3131 		else  message = "An unexpected error has occurred.";
3132 	} else {
3133 		message = "Unknown error";
3134 	}
3135 	return message;
3136 }
3137 
3138 string plainHttpError(bool isCgi, string type, Throwable t) {
3139 	auto message = messageFromException(t);
3140 	message = simpleHtmlEncode(message);
3141 
3142 	return format("%s %s\r\nContent-Length: %s\r\n\r\n%s",
3143 		isCgi ? "Status:" : "HTTP/1.0",
3144 		type, message.length, message);
3145 }
3146 
3147 // returns true if we were able to recover reasonably
3148 bool handleException(Cgi cgi, Throwable t) {
3149 	if(cgi.isClosed) {
3150 		// if the channel has been explicitly closed, we can't handle it here
3151 		return true;
3152 	}
3153 
3154 	if(cgi.outputtedResponseData) {
3155 		// the headers are sent, but the channel is open... since it closes if all was sent, we can append an error message here.
3156 		return false; // but I don't want to, since I don't know what condition the output is in; I don't want to inject something (nor check the content-type for that matter. So we say it was not a clean handling.
3157 	} else {
3158 		// no headers are sent, we can send a full blown error and recover
3159 		cgi.setCache(false);
3160 		cgi.setResponseContentType("text/html");
3161 		cgi.setResponseLocation(null); // cancel the redirect
3162 		cgi.setResponseStatus("500 Internal Server Error");
3163 		cgi.write(simpleHtmlEncode(messageFromException(t)));
3164 		cgi.close();
3165 		return true;
3166 	}
3167 }
3168 
3169 bool isCgiRequestMethod(string s) {
3170 	s = s.toUpper();
3171 	if(s == "COMMANDLINE")
3172 		return true;
3173 	foreach(member; __traits(allMembers, Cgi.RequestMethod))
3174 		if(s == member)
3175 			return true;
3176 	return false;
3177 }
3178 
3179 /// If you want to use a subclass of Cgi with generic main, use this mixin.
3180 mixin template CustomCgiMain(CustomCgi, alias fun, long maxContentLength = defaultMaxContentLength) if(is(CustomCgi : Cgi)) {
3181 	// kinda hacky - the T... is passed to Cgi's constructor in standard cgi mode, and ignored elsewhere
3182 	void main(string[] args) {
3183 		cgiMainImpl!(fun, CustomCgi, maxContentLength)(args);
3184 	}
3185 }
3186 
3187 version(embedded_httpd_processes)
3188 	int processPoolSize = 8;
3189 
3190 void cgiMainImpl(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMaxContentLength)(string[] args) if(is(CustomCgi : Cgi)) {
3191 	if(args.length > 1) {
3192 		// run the special separate processes if needed
3193 		switch(args[1]) {
3194 			case "--websocket-server":
3195 				version(with_addon_servers)
3196 					runWebsocketServer();
3197 				else
3198 					printf("Add-on servers not compiled in.");
3199 				return;
3200 			case "--session-server":
3201 				version(with_addon_servers)
3202 					runSessionServer();
3203 				else
3204 					printf("Add-on servers not compiled in.");
3205 				return;
3206 			case "--event-server":
3207 				version(with_addon_servers)
3208 					runEventServer();
3209 				else
3210 					printf("Add-on servers not compiled in.");
3211 				return;
3212 			case "--timer-server":
3213 				version(with_addon_servers)
3214 					runTimerServer();
3215 				else
3216 					printf("Add-on servers not compiled in.");
3217 				return;
3218 			case "--timed-jobs":
3219 				import core.demangle;
3220 				version(with_addon_servers_connections)
3221 				foreach(k, v; scheduledJobHandlers)
3222 					writeln(k, "\t", demangle(k));
3223 				return;
3224 			case "--timed-job":
3225 				scheduledJobHandlers[args[2]](args[3 .. $]);
3226 				return;
3227 			default:
3228 				// intentionally blank - do nothing and carry on to run normally
3229 		}
3230 	}
3231 
3232 	// we support command line thing for easy testing everywhere
3233 	// it needs to be called ./app method uri [other args...]
3234 	if(args.length >= 3 && isCgiRequestMethod(args[1])) {
3235 		Cgi cgi = new CustomCgi(args);
3236 		scope(exit) cgi.dispose();
3237 		fun(cgi);
3238 		cgi.close();
3239 		return;
3240 	}
3241 
3242 
3243 	ushort listeningPort(ushort def) {
3244 		bool found = false;
3245 		foreach(arg; args) {
3246 			if(found)
3247 				return to!ushort(arg);
3248 			if(arg == "--port" || arg == "-p" || arg == "/port" || arg == "--listening-port")
3249 				found = true;
3250 		}
3251 		return def;
3252 	}
3253 
3254 	string listeningHost() {
3255 		bool found = false;
3256 		foreach(arg; args) {
3257 			if(found)
3258 				return arg;
3259 			if(arg == "--listening-host" || arg == "-h" || arg == "/listening-host")
3260 				found = true;
3261 		}
3262 		return "";
3263 	}
3264 	version(netman_httpd) {
3265 		import arsd.httpd;
3266 		// what about forwarding the other constructor args?
3267 		// this probably needs a whole redoing...
3268 		serveHttp!CustomCgi(&fun, listeningPort(8080));//5005);
3269 		return;
3270 	} else
3271 	version(embedded_httpd_processes) {
3272 		import core.sys.posix.unistd;
3273 		import core.sys.posix.sys.socket;
3274 		import core.sys.posix.netinet.in_;
3275 		//import std.c.linux.socket;
3276 
3277 		int sock = socket(AF_INET, SOCK_STREAM, 0);
3278 		if(sock == -1)
3279 			throw new Exception("socket");
3280 
3281 		{
3282 			sockaddr_in addr;
3283 			addr.sin_family = AF_INET;
3284 			addr.sin_port = htons(listeningPort(8085));
3285 			auto lh = listeningHost();
3286 			if(lh.length) {
3287 				if(inet_pton(AF_INET, lh.toStringz(), &addr.sin_addr.s_addr) != 1)
3288 					throw new Exception("bad listening host given, please use an IP address.\nExample: --listening-host 127.0.0.1 means listen only on Localhost.\nExample: --listening-host 0.0.0.0 means listen on all interfaces.\nOr you can pass any other single numeric IPv4 address.");
3289 			} else
3290 				addr.sin_addr.s_addr = INADDR_ANY;
3291 
3292 			// HACKISH
3293 			int on = 1;
3294 			setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &on, on.sizeof);
3295 			// end hack
3296 
3297 			
3298 			if(bind(sock, cast(sockaddr*) &addr, addr.sizeof) == -1) {
3299 				close(sock);
3300 				throw new Exception("bind");
3301 			}
3302 
3303 			// FIXME: if this queue is full, it will just ignore it
3304 			// and wait for the client to retransmit it. This is an
3305 			// obnoxious timeout condition there.
3306 			if(sock.listen(128) == -1) {
3307 				close(sock);
3308 				throw new Exception("listen");
3309 			}
3310 		}
3311 
3312 		version(embedded_httpd_processes_accept_after_fork) {} else {
3313 			int pipeReadFd;
3314 			int pipeWriteFd;
3315 
3316 			{
3317 				int[2] pipeFd;
3318 				if(socketpair(AF_UNIX, SOCK_DGRAM, 0, pipeFd)) {
3319 					import core.stdc.errno;
3320 					throw new Exception("pipe failed " ~ to!string(errno));
3321 				}
3322 
3323 				pipeReadFd = pipeFd[0];
3324 				pipeWriteFd = pipeFd[1];
3325 			}
3326 		}
3327 
3328 
3329 		int processCount;
3330 		pid_t newPid;
3331 		reopen:
3332 		while(processCount < processPoolSize) {
3333 			newPid = fork();
3334 			if(newPid == 0) {
3335 				// start serving on the socket
3336 				//ubyte[4096] backingBuffer;
3337 				for(;;) {
3338 					bool closeConnection;
3339 					uint i;
3340 					sockaddr addr;
3341 					i = addr.sizeof;
3342 					version(embedded_httpd_processes_accept_after_fork) {
3343 						int s = accept(sock, &addr, &i);
3344 						int opt = 1;
3345 						import core.sys.posix.netinet.tcp;
3346 						// the Cgi class does internal buffering, so disabling this
3347 						// helps with latency in many cases...
3348 						setsockopt(s, IPPROTO_TCP, TCP_NODELAY, &opt, opt.sizeof);
3349 					} else {
3350 						int s;
3351 						auto readret = read_fd(pipeReadFd, &s, s.sizeof, &s);
3352 						if(readret != s.sizeof) {
3353 							import core.stdc.errno;
3354 							throw new Exception("pipe read failed " ~ to!string(errno));
3355 						}
3356 
3357 						//writeln("process ", getpid(), " got socket ", s);
3358 					}
3359 
3360 					try {
3361 
3362 						if(s == -1)
3363 							throw new Exception("accept");
3364 
3365 						scope(failure) close(s);
3366 						//ubyte[__traits(classInstanceSize, BufferedInputRange)] bufferedRangeContainer;
3367 						auto ir = new BufferedInputRange(s);
3368 						//auto ir = emplace!BufferedInputRange(bufferedRangeContainer, s, backingBuffer);
3369 
3370 						while(!ir.empty) {
3371 							ubyte[__traits(classInstanceSize, CustomCgi)] cgiContainer;
3372 
3373 							Cgi cgi;
3374 							try {
3375 								cgi = new CustomCgi(ir, &closeConnection);
3376 								cgi._outputFileHandle = s;
3377 								// if we have a single process and the browser tries to leave the connection open while concurrently requesting another, it will block everything an deadlock since there's no other server to accept it. By closing after each request in this situation, it tells the browser to serialize for us.
3378 								if(processPoolSize <= 1)
3379 									closeConnection = true;
3380 								//cgi = emplace!CustomCgi(cgiContainer, ir, &closeConnection);
3381 							} catch(Throwable t) {
3382 								// a construction error is either bad code or bad request; bad request is what it should be since this is bug free :P
3383 								// anyway let's kill the connection
3384 								version(CRuntime_Musl) {
3385 									// LockingTextWriter fails here
3386 									// so working around it
3387 									auto estr = t.toString();
3388 									stderr.rawWrite(estr);
3389 									stderr.rawWrite("\n");
3390 								} else
3391 									stderr.writeln(t.toString());
3392 								sendAll(ir.source, plainHttpError(false, "400 Bad Request", t));
3393 								closeConnection = true;
3394 								break;
3395 							}
3396 							assert(cgi !is null);
3397 							scope(exit)
3398 								cgi.dispose();
3399 
3400 							try {
3401 								fun(cgi);
3402 								cgi.close();
3403 								if(cgi.websocketMode)
3404 									closeConnection = true;
3405 							} catch(ConnectionException ce) {
3406 								closeConnection = true;
3407 							} catch(Throwable t) {
3408 								// a processing error can be recovered from
3409 								version(CRuntime_Musl) {
3410 									// LockingTextWriter fails here
3411 									// so working around it
3412 									auto estr = t.toString();
3413 									stderr.rawWrite(estr);
3414 								} else {
3415 									stderr.writeln(t.toString);
3416 								}
3417 								if(!handleException(cgi, t))
3418 									closeConnection = true;
3419 							}
3420 
3421 							if(closeConnection) {
3422 								ir.source.close();
3423 								break;
3424 							} else {
3425 								if(!ir.empty)
3426 									ir.popFront(); // get the next
3427 								else if(ir.sourceClosed) {
3428 									ir.source.close();
3429 								}
3430 							}
3431 						}
3432 
3433 						ir.source.close();
3434 					} catch(Throwable t) {
3435 						version(CRuntime_Musl) {} else
3436 						debug writeln(t);
3437 						// most likely cause is a timeout
3438 					}
3439 				}
3440 			} else {
3441 				processCount++;
3442 			}
3443 		}
3444 
3445 		// the parent should wait for its children...
3446 		if(newPid) {
3447 			import core.sys.posix.sys.wait;
3448 
3449 			version(embedded_httpd_processes_accept_after_fork) {} else {
3450 				import core.sys.posix.sys.select;
3451 				int[] fdQueue;
3452 				while(true) {
3453 					// writeln("select call");
3454 					int nfds = pipeWriteFd;
3455 					if(sock > pipeWriteFd)
3456 						nfds = sock;
3457 					nfds += 1;
3458 					fd_set read_fds;
3459 					fd_set write_fds;
3460 					FD_ZERO(&read_fds);
3461 					FD_ZERO(&write_fds);
3462 					FD_SET(sock, &read_fds);
3463 					if(fdQueue.length)
3464 						FD_SET(pipeWriteFd, &write_fds);
3465 					auto ret = select(nfds, &read_fds, &write_fds, null, null);
3466 					if(ret == -1) {
3467 						import core.stdc.errno;
3468 						if(errno == EINTR)
3469 							goto try_wait;
3470 						else
3471 							throw new Exception("wtf select");
3472 					}
3473 
3474 					int s = -1;
3475 					if(FD_ISSET(sock, &read_fds)) {
3476 						uint i;
3477 						sockaddr addr;
3478 						i = addr.sizeof;
3479 						s = accept(sock, &addr, &i);
3480 						import core.sys.posix.netinet.tcp;
3481 						int opt = 1;
3482 						setsockopt(s, IPPROTO_TCP, TCP_NODELAY, &opt, opt.sizeof);
3483 					}
3484 
3485 					if(FD_ISSET(pipeWriteFd, &write_fds)) {
3486 						if(s == -1 && fdQueue.length) {
3487 							s = fdQueue[0];
3488 							fdQueue = fdQueue[1 .. $]; // FIXME reuse buffer
3489 						}
3490 						write_fd(pipeWriteFd, &s, s.sizeof, s);
3491 						close(s); // we are done with it, let the other process take ownership
3492 					} else
3493 						fdQueue ~= s;
3494 				}
3495 			}
3496 
3497 			try_wait:
3498 
3499 			int status;
3500 			while(-1 != wait(&status)) {
3501 				version(CRuntime_Musl) {} else {
3502 				import std.stdio; writeln("Process died ", status);
3503 				}
3504 				processCount--;
3505 				goto reopen;
3506 			}
3507 			close(sock);
3508 		}
3509 	} else
3510 	version(embedded_httpd_threads) {
3511 		auto manager = new ListeningConnectionManager(listeningHost(), listeningPort(8085), &doThreadHttpConnection!(CustomCgi, fun));
3512 		manager.listen();
3513 	} else
3514 	version(scgi) {
3515 		import std.exception;
3516 		import al = std.algorithm;
3517 		auto manager = new ListeningConnectionManager(listeningHost(), listeningPort(4000), &doThreadScgiConnection!(CustomCgi, fun, maxContentLength));
3518 		manager.listen();
3519 	} else
3520 	version(fastcgi) {
3521 		//         SetHandler fcgid-script
3522 		FCGX_Stream* input, output, error;
3523 		FCGX_ParamArray env;
3524 
3525 
3526 
3527 		const(ubyte)[] getFcgiChunk() {
3528 			const(ubyte)[] ret;
3529 			while(FCGX_HasSeenEOF(input) != -1)
3530 				ret ~= cast(ubyte) FCGX_GetChar(input);
3531 			return ret;
3532 		}
3533 
3534 		void writeFcgi(const(ubyte)[] data) {
3535 			FCGX_PutStr(data.ptr, data.length, output);
3536 		}
3537 
3538 		void doARequest() {
3539 			string[string] fcgienv;
3540 
3541 			for(auto e = env; e !is null && *e !is null; e++) {
3542 				string cur = to!string(*e);
3543 				auto idx = cur.indexOf("=");
3544 				string name, value;
3545 				if(idx == -1)
3546 					name = cur;
3547 				else {
3548 					name = cur[0 .. idx];
3549 					value = cur[idx + 1 .. $];
3550 				}
3551 
3552 				fcgienv[name] = value;
3553 			}
3554 
3555 			void flushFcgi() {
3556 				FCGX_FFlush(output);
3557 			}
3558 
3559 			Cgi cgi;
3560 			try {
3561 				cgi = new CustomCgi(maxContentLength, fcgienv, &getFcgiChunk, &writeFcgi, &flushFcgi);
3562 			} catch(Throwable t) {
3563 				FCGX_PutStr(cast(ubyte*) t.msg.ptr, t.msg.length, error);
3564 				writeFcgi(cast(const(ubyte)[]) plainHttpError(true, "400 Bad Request", t));
3565 				return; //continue;
3566 			}
3567 			assert(cgi !is null);
3568 			scope(exit) cgi.dispose();
3569 			try {
3570 				fun(cgi);
3571 				cgi.close();
3572 			} catch(Throwable t) {
3573 				// log it to the error stream
3574 				FCGX_PutStr(cast(ubyte*) t.msg.ptr, t.msg.length, error);
3575 				// handle it for the user, if we can
3576 				if(!handleException(cgi, t))
3577 					return; // continue;
3578 			}
3579 		}
3580 
3581 		auto lp = listeningPort(0);
3582 		FCGX_Request request;
3583 		if(lp) {
3584 			// if a listening port was specified on the command line, we want to spawn ourself
3585 			// (needed for nginx without spawn-fcgi, e.g. on Windows)
3586 			FCGX_Init();
3587 			auto sock = FCGX_OpenSocket(toStringz(listeningHost() ~ ":" ~ to!string(lp)), 12);
3588 			if(sock < 0)
3589 				throw new Exception("Couldn't listen on the port");
3590 			FCGX_InitRequest(&request, sock, 0);
3591 			while(FCGX_Accept_r(&request) >= 0) {
3592 				input = request.inStream;
3593 				output = request.outStream;
3594 				error = request.errStream;
3595 				env = request.envp;
3596 				doARequest();
3597 			}
3598 		} else {
3599 			// otherwise, assume the httpd is doing it (the case for Apache, IIS, and Lighttpd)
3600 			// using the version with a global variable since we are separate processes anyway
3601 			while(FCGX_Accept(&input, &output, &error, &env) >= 0) {
3602 				doARequest();
3603 			}
3604 		}
3605 	} else {
3606 		// standard CGI is the default version
3607 		Cgi cgi;
3608 		try {
3609 			cgi = new CustomCgi(maxContentLength);
3610 			version(Posix)
3611 				cgi._outputFileHandle = 1; // stdout
3612 			else version(Windows)
3613 				cgi._outputFileHandle = GetStdHandle(STD_OUTPUT_HANDLE);
3614 			else static assert(0);
3615 		} catch(Throwable t) {
3616 			version(CRuntime_Musl) {
3617 				// LockingTextWriter fails here
3618 				// so working around it
3619 				auto s = t.toString();
3620 				stderr.rawWrite(s);
3621 				stdout.rawWrite(plainHttpError(true, "400 Bad Request", t));
3622 			} else {
3623 				stderr.writeln(t.msg);
3624 				// the real http server will probably handle this;
3625 				// most likely, this is a bug in Cgi. But, oh well.
3626 				stdout.write(plainHttpError(true, "400 Bad Request", t));
3627 			}
3628 			return;
3629 		}
3630 		assert(cgi !is null);
3631 		scope(exit) cgi.dispose();
3632 
3633 		try {
3634 			fun(cgi);
3635 			cgi.close();
3636 		} catch (Throwable t) {
3637 			version(CRuntime_Musl) {
3638 				// LockingTextWriter fails here
3639 				// so working around it
3640 				auto s = t.msg;
3641 				stderr.rawWrite(s);
3642 			} else {
3643 				stderr.writeln(t.msg);
3644 			}
3645 			if(!handleException(cgi, t))
3646 				return;
3647 		}
3648 	}
3649 }
3650 
3651 version(embedded_httpd_threads)
3652 void doThreadHttpConnection(CustomCgi, alias fun)(Socket connection) {
3653 	scope(failure) {
3654 		// catch all for other errors
3655 		sendAll(connection, plainHttpError(false, "500 Internal Server Error", null));
3656 		connection.close();
3657 	}
3658 
3659 	bool closeConnection;
3660 	auto ir = new BufferedInputRange(connection);
3661 
3662 	while(!ir.empty) {
3663 
3664 		if(ir.view.length == 0) {
3665 			ir.popFront();
3666 			if(ir.sourceClosed) {
3667 				connection.close();
3668 				break;
3669 			}
3670 		}
3671 
3672 		Cgi cgi;
3673 		try {
3674 			cgi = new CustomCgi(ir, &closeConnection);
3675 			cgi._outputFileHandle = connection.handle;
3676 		} catch(ConnectionException ce) {
3677 			// broken pipe or something, just abort the connection
3678 			closeConnection = true;
3679 			break;
3680 		} catch(Throwable t) {
3681 			// a construction error is either bad code or bad request; bad request is what it should be since this is bug free :P
3682 			// anyway let's kill the connection
3683 			version(CRuntime_Musl) {
3684 				stderr.rawWrite(t.toString());
3685 				stderr.rawWrite("\n");
3686 			} else {
3687 				stderr.writeln(t.toString());
3688 			}
3689 			sendAll(connection, plainHttpError(false, "400 Bad Request", t));
3690 			closeConnection = true;
3691 			break;
3692 		}
3693 		assert(cgi !is null);
3694 		scope(exit)
3695 			cgi.dispose();
3696 
3697 		try {
3698 			fun(cgi);
3699 			cgi.close();
3700 			if(cgi.websocketMode)
3701 				closeConnection = true;
3702 		} catch(ConnectionException ce) {
3703 			// broken pipe or something, just abort the connection
3704 			closeConnection = true;
3705 		} catch(Throwable t) {
3706 			// a processing error can be recovered from
3707 			version(CRuntime_Musl) {} else
3708 			stderr.writeln(t.toString);
3709 			if(!handleException(cgi, t))
3710 				closeConnection = true;
3711 		}
3712 
3713 		if(closeConnection) {
3714 			connection.close();
3715 			break;
3716 		} else {
3717 			if(ir.front.length) {
3718 				ir.popFront(); // we can't just discard the buffer, so get the next bit and keep chugging along
3719 			} else if(ir.sourceClosed) {
3720 				ir.source.close();
3721 			} else {
3722 				continue;
3723 				// break; // this was for a keepalive experiment
3724 			}
3725 		}
3726 	}
3727 
3728 	if(closeConnection)
3729 		connection.close();
3730 
3731 	// I am otherwise NOT closing it here because the parent thread might still be able to make use of the keep-alive connection!
3732 }
3733 
3734 version(scgi)
3735 void doThreadScgiConnection(CustomCgi, alias fun, long maxContentLength)(Socket connection) {
3736 	// and now we can buffer
3737 	scope(failure)
3738 		connection.close();
3739 
3740 	import al = std.algorithm;
3741 
3742 	size_t size;
3743 
3744 	string[string] headers;
3745 
3746 	auto range = new BufferedInputRange(connection);
3747 	more_data:
3748 	auto chunk = range.front();
3749 	// waiting for colon for header length
3750 	auto idx = indexOf(cast(string) chunk, ':');
3751 	if(idx == -1) {
3752 		try {
3753 			range.popFront();
3754 		} catch(Exception e) {
3755 			// it is just closed, no big deal
3756 			connection.close();
3757 			return;
3758 		}
3759 		goto more_data;
3760 	}
3761 
3762 	size = to!size_t(cast(string) chunk[0 .. idx]);
3763 	chunk = range.consume(idx + 1);
3764 	// reading headers
3765 	if(chunk.length < size)
3766 		range.popFront(0, size + 1);
3767 	// we are now guaranteed to have enough
3768 	chunk = range.front();
3769 	assert(chunk.length > size);
3770 
3771 	idx = 0;
3772 	string key;
3773 	string value;
3774 	foreach(part; al.splitter(chunk, '\0')) {
3775 		if(idx & 1) { // odd is value
3776 			value = cast(string)(part.idup);
3777 			headers[key] = value; // commit
3778 		} else
3779 			key = cast(string)(part.idup);
3780 		idx++;
3781 	}
3782 
3783 	enforce(chunk[size] == ','); // the terminator
3784 
3785 	range.consume(size + 1);
3786 	// reading data
3787 	// this will be done by Cgi
3788 
3789 	const(ubyte)[] getScgiChunk() {
3790 		// we are already primed
3791 		auto data = range.front();
3792 		if(data.length == 0 && !range.sourceClosed) {
3793 			range.popFront(0);
3794 			data = range.front();
3795 		} else if (range.sourceClosed)
3796 			range.source.close();
3797 
3798 		return data;
3799 	}
3800 
3801 	void writeScgi(const(ubyte)[] data) {
3802 		sendAll(connection, data);
3803 	}
3804 
3805 	void flushScgi() {
3806 		// I don't *think* I have to do anything....
3807 	}
3808 
3809 	Cgi cgi;
3810 	try {
3811 		cgi = new CustomCgi(maxContentLength, headers, &getScgiChunk, &writeScgi, &flushScgi);
3812 		cgi._outputFileHandle = connection.handle;
3813 	} catch(Throwable t) {
3814 		sendAll(connection, plainHttpError(true, "400 Bad Request", t));
3815 		connection.close();
3816 		return; // this connection is dead
3817 	}
3818 	assert(cgi !is null);
3819 	scope(exit) cgi.dispose();
3820 	try {
3821 		fun(cgi);
3822 		cgi.close();
3823 		connection.close();
3824 	} catch(Throwable t) {
3825 		// no std err
3826 		if(!handleException(cgi, t)) {
3827 			connection.close();
3828 			return;
3829 		}
3830 	}
3831 }
3832 
3833 string printDate(DateTime date) {
3834 	return format(
3835 		"%.3s, %02d %.3s %d %02d:%02d:%02d GMT", // could be UTC too
3836 		to!string(date.dayOfWeek).capitalize,
3837 		date.day,
3838 		to!string(date.month).capitalize,
3839 		date.year,
3840 		date.hour,
3841 		date.minute,
3842 		date.second);
3843 }
3844 
3845 
3846 // Referencing this gigantic typeid seems to remind the compiler
3847 // to actually put the symbol in the object file. I guess the immutable
3848 // assoc array array isn't actually included in druntime
3849 void hackAroundLinkerError() {
3850       stdout.rawWrite(typeid(const(immutable(char)[][])[immutable(char)[]]).toString());
3851       stdout.rawWrite(typeid(immutable(char)[][][immutable(char)[]]).toString());
3852       stdout.rawWrite(typeid(Cgi.UploadedFile[immutable(char)[]]).toString());
3853       stdout.rawWrite(typeid(Cgi.UploadedFile[][immutable(char)[]]).toString());
3854       stdout.rawWrite(typeid(immutable(Cgi.UploadedFile)[immutable(char)[]]).toString());
3855       stdout.rawWrite(typeid(immutable(Cgi.UploadedFile[])[immutable(char)[]]).toString());
3856       stdout.rawWrite(typeid(immutable(char[])[immutable(char)[]]).toString());
3857       // this is getting kinda ridiculous btw. Moving assoc arrays
3858       // to the library is the pain that keeps on coming.
3859 
3860       // eh this broke the build on the work server
3861       // stdout.rawWrite(typeid(immutable(char)[][immutable(string[])]));
3862       stdout.rawWrite(typeid(immutable(string[])[immutable(char)[]]).toString());
3863 }
3864 
3865 
3866 
3867 
3868 
3869 version(fastcgi) {
3870 	pragma(lib, "fcgi");
3871 
3872 	static if(size_t.sizeof == 8) // 64 bit
3873 		alias long c_int;
3874 	else
3875 		alias int c_int;
3876 
3877 	extern(C) {
3878 		struct FCGX_Stream {
3879 			ubyte* rdNext;
3880 			ubyte* wrNext;
3881 			ubyte* stop;
3882 			ubyte* stopUnget;
3883 			c_int isReader;
3884 			c_int isClosed;
3885 			c_int wasFCloseCalled;
3886 			c_int FCGI_errno;
3887 			void* function(FCGX_Stream* stream) fillBuffProc;
3888 			void* function(FCGX_Stream* stream, c_int doClose) emptyBuffProc;
3889 			void* data;
3890 		}
3891 
3892 		// note: this is meant to be opaque, so don't access it directly
3893 		struct FCGX_Request {
3894 			int requestId;
3895 			int role;
3896 			FCGX_Stream* inStream;
3897 			FCGX_Stream* outStream;
3898 			FCGX_Stream* errStream;
3899 			char** envp;
3900 			void* paramsPtr;
3901 			int ipcFd;
3902 			int isBeginProcessed;
3903 			int keepConnection;
3904 			int appStatus;
3905 			int nWriters;
3906 			int flags;
3907 			int listen_sock;
3908 		}
3909 
3910 		int FCGX_InitRequest(FCGX_Request *request, int sock, int flags);
3911 		void FCGX_Init();
3912 
3913 		int FCGX_Accept_r(FCGX_Request *request);
3914 
3915 
3916 		alias char** FCGX_ParamArray;
3917 
3918 		c_int FCGX_Accept(FCGX_Stream** stdin, FCGX_Stream** stdout, FCGX_Stream** stderr, FCGX_ParamArray* envp);
3919 		c_int FCGX_GetChar(FCGX_Stream* stream);
3920 		c_int FCGX_PutStr(const ubyte* str, c_int n, FCGX_Stream* stream);
3921 		int FCGX_HasSeenEOF(FCGX_Stream* stream);
3922 		c_int FCGX_FFlush(FCGX_Stream *stream);
3923 
3924 		int FCGX_OpenSocket(in char*, int);
3925 	}
3926 }
3927 
3928 
3929 /* This might go int a separate module eventually. It is a network input helper class. */
3930 
3931 import std.socket;
3932 
3933 // it is a class primarily for reference semantics
3934 // I might change this interface
3935 /// This is NOT ACTUALLY an input range! It is too different. Historical mistake kinda.
3936 class BufferedInputRange {
3937 	version(Posix)
3938 	this(int source, ubyte[] buffer = null) {
3939 		this(new Socket(cast(socket_t) source, AddressFamily.INET), buffer);
3940 	}
3941 
3942 	this(Socket source, ubyte[] buffer = null) {
3943 		// if they connect but never send stuff to us, we don't want it wasting the process
3944 		// so setting a time out
3945 		source.setOption(SocketOptionLevel.SOCKET, SocketOption.RCVTIMEO, dur!"seconds"(3));
3946 		this.source = source;
3947 		if(buffer is null) {
3948 			underlyingBuffer = new ubyte[4096];
3949 			allowGrowth = true;
3950 		} else {
3951 			underlyingBuffer = buffer;
3952 		}
3953 
3954 		assert(underlyingBuffer.length);
3955 
3956 		// we assume view.ptr is always inside underlyingBuffer
3957 		view = underlyingBuffer[0 .. 0];
3958 
3959 		popFront(); // prime
3960 	}
3961 
3962 	/**
3963 		A slight difference from regular ranges is you can give it the maximum
3964 		number of bytes to consume.
3965 
3966 		IMPORTANT NOTE: the default is to consume nothing, so if you don't call
3967 		consume() yourself and use a regular foreach, it will infinitely loop!
3968 
3969 		The default is to do what a normal range does, and consume the whole buffer
3970 		and wait for additional input.
3971 
3972 		You can also specify 0, to append to the buffer, or any other number
3973 		to remove the front n bytes and wait for more.
3974 	*/
3975 	void popFront(size_t maxBytesToConsume = 0 /*size_t.max*/, size_t minBytesToSettleFor = 0, bool skipConsume = false) {
3976 		if(sourceClosed)
3977 			throw new ConnectionClosedException("can't get any more data from a closed source");
3978 		if(!skipConsume)
3979 			consume(maxBytesToConsume);
3980 
3981 		// we might have to grow the buffer
3982 		if(minBytesToSettleFor > underlyingBuffer.length || view.length == underlyingBuffer.length) {
3983 			if(allowGrowth) {
3984 			//import std.stdio; writeln("growth");
3985 				auto viewStart = view.ptr - underlyingBuffer.ptr;
3986 				size_t growth = 4096;
3987 				// make sure we have enough for what we're being asked for
3988 				if(minBytesToSettleFor > 0 && minBytesToSettleFor - underlyingBuffer.length > growth)
3989 					growth = minBytesToSettleFor - underlyingBuffer.length;
3990 				//import std.stdio; writeln(underlyingBuffer.length, " ", viewStart, " ", view.length, " ", growth,  " ", minBytesToSettleFor, " ", minBytesToSettleFor - underlyingBuffer.length);
3991 				underlyingBuffer.length += growth;
3992 				view = underlyingBuffer[viewStart .. view.length];
3993 			} else
3994 				throw new Exception("No room left in the buffer");
3995 		}
3996 
3997 		do {
3998 			auto freeSpace = underlyingBuffer[view.ptr - underlyingBuffer.ptr + view.length .. $];
3999 			try_again:
4000 			auto ret = source.receive(freeSpace);
4001 			if(ret == Socket.ERROR) {
4002 				if(wouldHaveBlocked()) {
4003 					// gonna treat a timeout here as a close
4004 					sourceClosed = true;
4005 					return;
4006 				}
4007 				version(Posix) {
4008 					import core.stdc.errno;
4009 					if(errno == EINTR || errno == EAGAIN) {
4010 						goto try_again;
4011 					}
4012 				}
4013 				throw new Exception(lastSocketError); // FIXME
4014 			}
4015 			if(ret == 0) {
4016 				sourceClosed = true;
4017 				return;
4018 			}
4019 
4020 			//import std.stdio; writeln(view.ptr); writeln(underlyingBuffer.ptr); writeln(view.length, " ", ret, " = ", view.length + ret);
4021 			view = underlyingBuffer[view.ptr - underlyingBuffer.ptr .. view.length + ret];
4022 		} while(view.length < minBytesToSettleFor);
4023 	}
4024 
4025 	/// Removes n bytes from the front of the buffer, and returns the new buffer slice.
4026 	/// You might want to idup the data you are consuming if you store it, since it may
4027 	/// be overwritten on the new popFront.
4028 	///
4029 	/// You do not need to call this if you always want to wait for more data when you
4030 	/// consume some.
4031 	ubyte[] consume(size_t bytes) {
4032 		//import std.stdio; writeln("consuime ", bytes, "/", view.length);
4033 		view = view[bytes > $ ? $ : bytes .. $];
4034 		if(view.length == 0) {
4035 			view = underlyingBuffer[0 .. 0]; // go ahead and reuse the beginning
4036 			/*
4037 			writeln("HERE");
4038 			popFront(0, 0, true); // try to load more if we can, checks if the source is closed
4039 			writeln(cast(string)front);
4040 			writeln("DONE");
4041 			*/
4042 		}
4043 		return front;
4044 	}
4045 
4046 	bool empty() {
4047 		return sourceClosed && view.length == 0;
4048 	}
4049 
4050 	ubyte[] front() {
4051 		return view;
4052 	}
4053 
4054 	invariant() {
4055 		assert(view.ptr >= underlyingBuffer.ptr);
4056 		// it should never be equal, since if that happens view ought to be empty, and thus reusing the buffer
4057 		assert(view.ptr < underlyingBuffer.ptr + underlyingBuffer.length);
4058 	}
4059 
4060 	ubyte[] underlyingBuffer;
4061 	bool allowGrowth;
4062 	ubyte[] view;
4063 	Socket source;
4064 	bool sourceClosed;
4065 }
4066 
4067 import core.sync.semaphore;
4068 import core.atomic;
4069 
4070 /**
4071 	To use this thing:
4072 
4073 	void handler(Socket s) { do something... }
4074 	auto manager = new ListeningConnectionManager("127.0.0.1", 80, &handler);
4075 	manager.listen();
4076 
4077 	I suggest you use BufferedInputRange(connection) to handle the input. As a packet
4078 	comes in, you will get control. You can just continue; though to fetch more.
4079 
4080 
4081 	FIXME: should I offer an event based async thing like netman did too? Yeah, probably.
4082 */
4083 class ListeningConnectionManager {
4084 	Semaphore semaphore;
4085 	Socket[256] queue;
4086 	shared(ubyte) nextIndexFront;
4087 	ubyte nextIndexBack;
4088 	shared(int) queueLength;
4089 
4090 	void listen() {
4091 		running = true;
4092 		shared(int) loopBroken;
4093 
4094 		version(cgi_no_threads) {
4095 			// NEVER USE THIS
4096 			// it exists only for debugging and other special occasions
4097 
4098 			// the thread mode is faster and less likely to stall the whole
4099 			// thing when a request is slow
4100 			while(!loopBroken && running) {
4101 				auto sn = listener.accept();
4102 				try {
4103 					handler(sn);
4104 				} catch(Exception e) {
4105 					// if a connection goes wrong, we want to just say no, but try to carry on unless it is an Error of some sort (in which case, we'll die. You might want an external helper program to revive the server when it dies)
4106 					sn.close();
4107 				}
4108 			}
4109 		} else {
4110 			semaphore = new Semaphore();
4111 
4112 			ConnectionThread[16] threads;
4113 			foreach(i, ref thread; threads) {
4114 				thread = new ConnectionThread(this, handler, cast(int) i);
4115 				thread.start();
4116 			}
4117 
4118 			/+
4119 			version(linux) {
4120 				import core.sys.linux.epoll;
4121 				epoll_fd = epoll_create1(EPOLL_CLOEXEC);
4122 				if(epoll_fd == -1)
4123 					throw new Exception("epoll_create1 " ~ to!string(errno));
4124 				scope(exit) {
4125 					import core.sys.posix.unistd;
4126 					close(epoll_fd);
4127 				}
4128 
4129 				epoll_event[64] events;
4130 
4131 				epoll_event ev;
4132 				ev.events = EPOLLIN;
4133 				ev.data.fd = listener.handle;
4134 				if(epoll_ctl(epoll_fd, EPOLL_CTL_ADD, listener.handle, &ev) == -1)
4135 					throw new Exception("epoll_ctl " ~ to!string(errno));
4136 			}
4137 			+/
4138 
4139 			while(!loopBroken && running) {
4140 				Socket sn;
4141 
4142 				void accept_new_connection() {
4143 					sn = listener.accept();
4144 					if(tcp) {
4145 						// disable Nagle's algorithm to avoid a 40ms delay when we send/recv
4146 						// on the socket because we do some buffering internally. I think this helps,
4147 						// certainly does for small requests, and I think it does for larger ones too
4148 						sn.setOption(SocketOptionLevel.TCP, SocketOption.TCP_NODELAY, 1);
4149 
4150 						sn.setOption(SocketOptionLevel.SOCKET, SocketOption.RCVTIMEO, dur!"seconds"(10));
4151 					}
4152 				}
4153 
4154 				void existing_connection_new_data() {
4155 					// wait until a slot opens up
4156 					//int waited = 0;
4157 					while(queueLength >= queue.length) {
4158 						Thread.sleep(1.msecs);
4159 						//waited ++;
4160 					}
4161 					//if(waited) {import std.stdio; writeln(waited);}
4162 					synchronized(this) {
4163 						queue[nextIndexBack] = sn;
4164 						nextIndexBack++;
4165 						atomicOp!"+="(queueLength, 1);
4166 					}
4167 					semaphore.notify();
4168 				}
4169 
4170 				bool crash_check() {
4171 					bool hasAnyRunning;
4172 					foreach(thread; threads) {
4173 						if(!thread.isRunning) {
4174 							thread.join();
4175 						} else hasAnyRunning = true;
4176 					}
4177 
4178 					return (!hasAnyRunning);
4179 				}
4180 
4181 
4182 				/+
4183 				version(linux) {
4184 					auto nfds = epoll_wait(epoll_fd, events.ptr, events.length, -1);
4185 					if(nfds == -1) {
4186 						if(errno == EINTR)
4187 							continue;
4188 						throw new Exception("epoll_wait " ~ to!string(errno));
4189 					}
4190 
4191 					foreach(idx; 0 .. nfds) {
4192 						auto flags = events[idx].events;
4193 						auto fd = events[idx].data.fd;
4194 
4195 						if(fd == listener.handle) {
4196 							accept_new_connection();
4197 							existing_connection_new_data();
4198 						} else {
4199 							if(flags & (EPOLLHUP | EPOLLERR | EPOLLRDHUP)) {
4200 								import core.sys.posix.unistd;
4201 								close(fd);
4202 							} else {
4203 								sn = new Socket(cast(socket_t) fd, tcp ? AddressFamily.INET : AddressFamily.UNIX);
4204 								import std.stdio; writeln("existing_connection_new_data");
4205 								existing_connection_new_data();
4206 							}
4207 						}
4208 					}
4209 				} else {
4210 				+/
4211 					accept_new_connection();
4212 					existing_connection_new_data();
4213 				//}
4214 
4215 				if(crash_check())
4216 					break;
4217 			}
4218 
4219 			// FIXME: i typically stop this with ctrl+c which never
4220 			// actually gets here. i need to do a sigint handler.
4221 			if(cleanup)
4222 				cleanup();
4223 		}
4224 	}
4225 
4226 	//version(linux)
4227 		//int epoll_fd;
4228 
4229 	bool tcp;
4230 	void delegate() cleanup;
4231 
4232 	this(string host, ushort port, void function(Socket) handler) {
4233 		this.handler = handler;
4234 
4235 		if(host.startsWith("unix:")) {
4236 			version(Posix) {
4237 				listener = new Socket(AddressFamily.UNIX, SocketType.STREAM);
4238 				string filename = host["unix:".length .. $].idup;
4239 				listener.bind(new UnixAddress(filename));
4240 				cleanup = delegate() {
4241 					import std.file;
4242 					remove(filename);
4243 				};
4244 				tcp = false;
4245 			} else {
4246 				throw new Exception("unix sockets not supported on this system");
4247 			}
4248 		} else if(host.startsWith("abstract:")) {
4249 			version(linux) {
4250 				listener = new Socket(AddressFamily.UNIX, SocketType.STREAM);
4251 				string filename = "\0" ~ host["abstract:".length .. $];
4252 				import std.stdio; stderr.writeln("Listening to abstract unix domain socket: ", host["abstract:".length .. $]);
4253 				listener.bind(new UnixAddress(filename));
4254 				tcp = false;
4255 			} else {
4256 				throw new Exception("abstract unix sockets not supported on this system");
4257 			}
4258 		} else {
4259 			listener = new TcpSocket();
4260 			listener.setOption(SocketOptionLevel.SOCKET, SocketOption.REUSEADDR, true);
4261 			listener.bind(host.length ? parseAddress(host, port) : new InternetAddress(port));
4262 			tcp = true;
4263 		}
4264 
4265 		Thread.getThis.priority = Thread.PRIORITY_MAX;
4266 		listener.listen(128);
4267 	}
4268 
4269 	Socket listener;
4270 	void function(Socket) handler;
4271 
4272 	bool running;
4273 	void quit() {
4274 		running = false;
4275 	}
4276 }
4277 
4278 // helper function to send a lot to a socket. Since this blocks for the buffer (possibly several times), you should probably call it in a separate thread or something.
4279 void sendAll(Socket s, const(void)[] data, string file = __FILE__, size_t line = __LINE__) {
4280 	if(data.length == 0) return;
4281 	ptrdiff_t amount;
4282 	do {
4283 		amount = s.send(data);
4284 		if(amount == Socket.ERROR)
4285 			throw new ConnectionException(s, lastSocketError, file, line);
4286 		assert(amount > 0);
4287 		data = data[amount .. $];
4288 	} while(data.length);
4289 }
4290 
4291 class ConnectionException : Exception {
4292 	Socket socket;
4293 	this(Socket s, string msg, string file = __FILE__, size_t line = __LINE__) {
4294 		this.socket = s;
4295 		super(msg, file, line);
4296 	}
4297 }
4298 
4299 alias void function(Socket) CMT;
4300 
4301 import core.thread;
4302 /+
4303 	cgi.d now uses a hybrid of event i/o and threads at the top level.
4304 
4305 	Top level thread is responsible for accepting sockets and selecting on them.
4306 
4307 	It then indicates to a child that a request is pending, and any random worker
4308 	thread that is free handles it. It goes into blocking mode and handles that
4309 	http request to completion.
4310 
4311 	At that point, it goes back into the waiting queue.
4312 
4313 
4314 	This concept is only implemented on Linux. On all other systems, it still
4315 	uses the worker threads and semaphores (which is perfectly fine for a lot of
4316 	things! Just having a great number of keep-alive connections will break that.)
4317 
4318 
4319 	So the algorithm is:
4320 
4321 	select(accept, event, pending)
4322 		if accept -> send socket to free thread, if any. if not, add socket to queue
4323 		if event -> send the signaling thread a socket from the queue, if not, mark it free
4324 			- event might block until it can be *written* to. it is a fifo sending socket fds!
4325 
4326 	A worker only does one http request at a time, then signals its availability back to the boss.
4327 
4328 	The socket the worker was just doing should be added to the one-off epoll read. If it is closed,
4329 	great, we can get rid of it. Otherwise, it is considered `pending`. The *kernel* manages that; the
4330 	actual FD will not be kept out here.
4331 
4332 	So:
4333 		queue = sockets we know are ready to read now, but no worker thread is available
4334 		idle list = worker threads not doing anything else. they signal back and forth
4335 
4336 	the workers all read off the event fd. This is the semaphore wait
4337 
4338 	the boss waits on accept or other sockets read events (one off! and level triggered). If anything happens wrt ready read,
4339 	it puts it in the queue and writes to the event fd.
4340 
4341 	The child could put the socket back in the epoll thing itself.
4342 
4343 	The child needs to be able to gracefully handle being given a socket that just closed with no work.
4344 +/
4345 class ConnectionThread : Thread {
4346 	this(ListeningConnectionManager lcm, CMT dg, int myThreadNumber) {
4347 		this.lcm = lcm;
4348 		this.dg = dg;
4349 		this.myThreadNumber = myThreadNumber;
4350 		super(&run);
4351 	}
4352 
4353 	void run() {
4354 		while(true) {
4355 			// so if there's a bunch of idle keep-alive connections, it can
4356 			// consume all the worker threads... just sitting there.
4357 			lcm.semaphore.wait();
4358 			Socket socket;
4359 			synchronized(lcm) {
4360 				auto idx = lcm.nextIndexFront;
4361 				socket = lcm.queue[idx];
4362 				lcm.queue[idx] = null;
4363 				atomicOp!"+="(lcm.nextIndexFront, 1);
4364 				atomicOp!"-="(lcm.queueLength, 1);
4365 			}
4366 			try {
4367 			//import std.stdio; writeln(myThreadNumber, " taking it");
4368 				dg(socket);
4369 				/+
4370 				if(socket.isAlive) {
4371 					// process it more later
4372 					version(linux) {
4373 						import core.sys.linux.epoll;
4374 						epoll_event ev;
4375 						ev.events = EPOLLIN | EPOLLONESHOT | EPOLLET;
4376 						ev.data.fd = socket.handle;
4377 						import std.stdio; writeln("adding");
4378 						if(epoll_ctl(lcm.epoll_fd, EPOLL_CTL_ADD, socket.handle, &ev) == -1) {
4379 							if(errno == EEXIST) {
4380 								ev.events = EPOLLIN | EPOLLONESHOT | EPOLLET;
4381 								ev.data.fd = socket.handle;
4382 								if(epoll_ctl(lcm.epoll_fd, EPOLL_CTL_MOD, socket.handle, &ev) == -1)
4383 									throw new Exception("epoll_ctl " ~ to!string(errno));
4384 							} else
4385 								throw new Exception("epoll_ctl " ~ to!string(errno));
4386 						}
4387 						//import std.stdio; writeln("keep alive");
4388 						// writing to this private member is to prevent the GC from closing my precious socket when I'm trying to use it later
4389 						__traits(getMember, socket, "sock") = cast(socket_t) -1;
4390 					} else {
4391 						continue; // hope it times out in a reasonable amount of time...
4392 					}
4393 				}
4394 				+/
4395 			} catch(Throwable e) {
4396 				import std.stdio; stderr.rawWrite(e.toString); stderr.rawWrite("\n");
4397 				socket.close();
4398 			}
4399 		}
4400 	}
4401 
4402 	ListeningConnectionManager lcm;
4403 	CMT dg;
4404 	int myThreadNumber;
4405 }
4406 
4407 /* Done with network helper */
4408 
4409 /* Helpers for doing temporary files. Used both here and in web.d */
4410 
4411 version(Windows) {
4412 	import core.sys.windows.windows;
4413 	extern(Windows) DWORD GetTempPathW(DWORD, LPWSTR);
4414 	alias GetTempPathW GetTempPath;
4415 }
4416 
4417 version(Posix) {
4418 	static import linux = core.sys.posix.unistd;
4419 }
4420 
4421 string getTempDirectory() {
4422 	string path;
4423 	version(Windows) {
4424 		wchar[1024] buffer;
4425 		auto len = GetTempPath(1024, buffer.ptr);
4426 		if(len == 0)
4427 			throw new Exception("couldn't find a temporary path");
4428 
4429 		auto b = buffer[0 .. len];
4430 
4431 		path = to!string(b);
4432 	} else
4433 		path = "/tmp/";
4434 
4435 	return path;
4436 }
4437 
4438 
4439 // I like std.date. These functions help keep my old code and data working with phobos changing.
4440 
4441 long sysTimeToDTime(in SysTime sysTime) {
4442     return convert!("hnsecs", "msecs")(sysTime.stdTime - 621355968000000000L);
4443 }
4444 
4445 long dateTimeToDTime(in DateTime dt) {
4446 	return sysTimeToDTime(cast(SysTime) dt);
4447 }
4448 
4449 long getUtcTime() { // renamed primarily to avoid conflict with std.date itself
4450 	return sysTimeToDTime(Clock.currTime(UTC()));
4451 }
4452 
4453 // NOTE: new SimpleTimeZone(minutes); can perhaps work with the getTimezoneOffset() JS trick
4454 SysTime dTimeToSysTime(long dTime, immutable TimeZone tz = null) {
4455 	immutable hnsecs = convert!("msecs", "hnsecs")(dTime) + 621355968000000000L;
4456 	return SysTime(hnsecs, tz);
4457 }
4458 
4459 
4460 
4461 // this is a helper to read HTTP transfer-encoding: chunked responses
4462 immutable(ubyte[]) dechunk(BufferedInputRange ir) {
4463 	immutable(ubyte)[] ret;
4464 
4465 	another_chunk:
4466 	// If here, we are at the beginning of a chunk.
4467 	auto a = ir.front();
4468 	int chunkSize;
4469 	int loc = locationOf(a, "\r\n");
4470 	while(loc == -1) {
4471 		ir.popFront();
4472 		a = ir.front();
4473 		loc = locationOf(a, "\r\n");
4474 	}
4475 
4476 	string hex;
4477 	hex = "";
4478 	for(int i = 0; i < loc; i++) {
4479 		char c = a[i];
4480 		if(c >= 'A' && c <= 'Z')
4481 			c += 0x20;
4482 		if((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z')) {
4483 			hex ~= c;
4484 		} else {
4485 			break;
4486 		}
4487 	}
4488 
4489 	assert(hex.length);
4490 
4491 	int power = 1;
4492 	int size = 0;
4493 	foreach(cc1; retro(hex)) {
4494 		dchar cc = cc1;
4495 		if(cc >= 'a' && cc <= 'z')
4496 			cc -= 0x20;
4497 		int val = 0;
4498 		if(cc >= '0' && cc <= '9')
4499 			val = cc - '0';
4500 		else
4501 			val = cc - 'A' + 10;
4502 
4503 		size += power * val;
4504 		power *= 16;
4505 	}
4506 
4507 	chunkSize = size;
4508 	assert(size >= 0);
4509 
4510 	if(loc + 2 > a.length) {
4511 		ir.popFront(0, a.length + loc + 2);
4512 		a = ir.front();
4513 	}
4514 
4515 	a = ir.consume(loc + 2);
4516 
4517 	if(chunkSize == 0) { // we're done with the response
4518 		// if we got here, will change must be true....
4519 		more_footers:
4520 		loc = locationOf(a, "\r\n");
4521 		if(loc == -1) {
4522 			ir.popFront();
4523 			a = ir.front;
4524 			goto more_footers;
4525 		} else {
4526 			assert(loc == 0);
4527 			ir.consume(loc + 2);
4528 			goto finish;
4529 		}
4530 	} else {
4531 		// if we got here, will change must be true....
4532 		if(a.length < chunkSize + 2) {
4533 			ir.popFront(0, chunkSize + 2);
4534 			a = ir.front();
4535 		}
4536 
4537 		ret ~= (a[0..chunkSize]);
4538 
4539 		if(!(a.length > chunkSize + 2)) {
4540 			ir.popFront(0, chunkSize + 2);
4541 			a = ir.front();
4542 		}
4543 		assert(a[chunkSize] == 13);
4544 		assert(a[chunkSize+1] == 10);
4545 		a = ir.consume(chunkSize + 2);
4546 		chunkSize = 0;
4547 		goto another_chunk;
4548 	}
4549 
4550 	finish:
4551 	return ret;
4552 }
4553 
4554 // I want to be able to get data from multiple sources the same way...
4555 interface ByChunkRange {
4556 	bool empty();
4557 	void popFront();
4558 	const(ubyte)[] front();
4559 }
4560 
4561 ByChunkRange byChunk(const(ubyte)[] data) {
4562 	return new class ByChunkRange {
4563 		override bool empty() {
4564 			return !data.length;
4565 		}
4566 
4567 		override void popFront() {
4568 			if(data.length > 4096)
4569 				data = data[4096 .. $];
4570 			else
4571 				data = null;
4572 		}
4573 
4574 		override const(ubyte)[] front() {
4575 			return data[0 .. $ > 4096 ? 4096 : $];
4576 		}
4577 	};
4578 }
4579 
4580 ByChunkRange byChunk(BufferedInputRange ir, size_t atMost) {
4581 	const(ubyte)[] f;
4582 
4583 	f = ir.front;
4584 	if(f.length > atMost)
4585 		f = f[0 .. atMost];
4586 
4587 	return new class ByChunkRange {
4588 		override bool empty() {
4589 			return atMost == 0;
4590 		}
4591 
4592 		override const(ubyte)[] front() {
4593 			return f;
4594 		}
4595 
4596 		override void popFront() {
4597 			ir.consume(f.length);
4598 			atMost -= f.length;
4599 			auto a = ir.front();
4600 
4601 			if(a.length <= atMost) {
4602 				f = a;
4603 				atMost -= a.length;
4604 				a = ir.consume(a.length);
4605 				if(atMost != 0)
4606 					ir.popFront();
4607 				if(f.length == 0) {
4608 					f = ir.front();
4609 				}
4610 			} else {
4611 				// we actually have *more* here than we need....
4612 				f = a[0..atMost];
4613 				atMost = 0;
4614 				ir.consume(atMost);
4615 			}
4616 		}
4617 	};
4618 }
4619 
4620 version(cgi_with_websocket) {
4621 	// http://tools.ietf.org/html/rfc6455
4622 
4623 	/**
4624 		WEBSOCKET SUPPORT:
4625 
4626 		Full example:
4627 		---
4628 			import arsd.cgi;
4629 
4630 			void websocketEcho(Cgi cgi) {
4631 				if(cgi.websocketRequested()) {
4632 					if(cgi.origin != "http://arsdnet.net")
4633 						throw new Exception("bad origin");
4634 					auto websocket = cgi.acceptWebsocket();
4635 
4636 					websocket.send("hello");
4637 					websocket.send(" world!");
4638 
4639 					auto msg = websocket.recv();
4640 					while(msg.opcode != WebSocketOpcode.close) {
4641 						if(msg.opcode == WebSocketOpcode.text) {
4642 							websocket.send(msg.textData);
4643 						} else if(msg.opcode == WebSocketOpcode.binary) {
4644 							websocket.send(msg.data);
4645 						}
4646 
4647 						msg = websocket.recv();
4648 					}
4649 
4650 					websocket.close();
4651 				} else assert(0, "i want a web socket!");
4652 			}
4653 
4654 			mixin GenericMain!websocketEcho;
4655 		---
4656 	*/
4657 
4658 	class WebSocket {
4659 		Cgi cgi;
4660 
4661 		private this(Cgi cgi) {
4662 			this.cgi = cgi;
4663 		}
4664 
4665 		// returns true if data available, false if it timed out
4666 		bool recvAvailable(Duration timeout = dur!"msecs"(0)) {
4667 			if(!waitForNextMessageWouldBlock())
4668 				return true;
4669 			if(isDataPending(timeout))
4670 				return true; // this is kinda a lie.
4671 
4672 			return false;
4673 		}
4674 
4675 		public bool lowLevelReceive() {
4676 			auto bfr = cgi.idlol;
4677 			top:
4678 			auto got = bfr.front;
4679 			if(got.length) {
4680 				if(receiveBuffer.length < receiveBufferUsedLength + got.length)
4681 					receiveBuffer.length += receiveBufferUsedLength + got.length;
4682 
4683 				receiveBuffer[receiveBufferUsedLength .. receiveBufferUsedLength + got.length] = got[];
4684 				receiveBufferUsedLength += got.length;
4685 				bfr.consume(got.length);
4686 
4687 				return true;
4688 			}
4689 
4690 			bfr.popFront(0);
4691 			if(bfr.sourceClosed)
4692 				return false;
4693 			goto top;
4694 		}
4695 
4696 
4697 		bool isDataPending(Duration timeout = 0.seconds) {
4698 			Socket socket = cgi.idlol.source;
4699 
4700 			auto check = new SocketSet();
4701 			check.add(socket);
4702 
4703 			auto got = Socket.select(check, null, null, timeout);
4704 			if(got > 0)
4705 				return true;
4706 			return false;
4707 		}
4708 
4709 		// note: this blocks
4710 		WebSocketFrame recv() {
4711 			return waitForNextMessage();
4712 		}
4713 
4714 
4715 
4716 
4717 		private void llclose() {
4718 			cgi.close();
4719 		}
4720 
4721 		private void llsend(ubyte[] data) {
4722 			cgi.write(data);
4723 			cgi.flush();
4724 		}
4725 
4726 		void unregisterActiveSocket(WebSocket) {}
4727 
4728 		/* copy/paste section { */
4729 
4730 		private int readyState_;
4731 		private ubyte[] receiveBuffer;
4732 		private size_t receiveBufferUsedLength;
4733 
4734 		private Config config;
4735 
4736 		enum CONNECTING = 0; /// Socket has been created. The connection is not yet open.
4737 		enum OPEN = 1; /// The connection is open and ready to communicate.
4738 		enum CLOSING = 2; /// The connection is in the process of closing.
4739 		enum CLOSED = 3; /// The connection is closed or couldn't be opened.
4740 
4741 		/++
4742 
4743 		+/
4744 		/// Group: foundational
4745 		static struct Config {
4746 			/++
4747 				These control the size of the receive buffer.
4748 
4749 				It starts at the initial size, will temporarily
4750 				balloon up to the maximum size, and will reuse
4751 				a buffer up to the likely size.
4752 
4753 				Anything larger than the maximum size will cause
4754 				the connection to be aborted and an exception thrown.
4755 				This is to protect you against a peer trying to
4756 				exhaust your memory, while keeping the user-level
4757 				processing simple.
4758 			+/
4759 			size_t initialReceiveBufferSize = 4096;
4760 			size_t likelyReceiveBufferSize = 4096; /// ditto
4761 			size_t maximumReceiveBufferSize = 10 * 1024 * 1024; /// ditto
4762 
4763 			/++
4764 				Maximum combined size of a message.
4765 			+/
4766 			size_t maximumMessageSize = 10 * 1024 * 1024;
4767 
4768 			string[string] cookies; /// Cookies to send with the initial request. cookies[name] = value;
4769 			string origin; /// Origin URL to send with the handshake, if desired.
4770 			string protocol; /// the protocol header, if desired.
4771 
4772 			int pingFrequency = 5000; /// Amount of time (in msecs) of idleness after which to send an automatic ping
4773 		}
4774 
4775 		/++
4776 			Returns one of [CONNECTING], [OPEN], [CLOSING], or [CLOSED].
4777 		+/
4778 		int readyState() {
4779 			return readyState_;
4780 		}
4781 
4782 		/++
4783 			Closes the connection, sending a graceful teardown message to the other side.
4784 		+/
4785 		/// Group: foundational
4786 		void close(int code = 0, string reason = null)
4787 			//in (reason.length < 123)
4788 			in { assert(reason.length < 123); } do
4789 		{
4790 			if(readyState_ != OPEN)
4791 				return; // it cool, we done
4792 			WebSocketFrame wss;
4793 			wss.fin = true;
4794 			wss.opcode = WebSocketOpcode.close;
4795 			wss.data = cast(ubyte[]) reason;
4796 			wss.send(&llsend);
4797 
4798 			readyState_ = CLOSING;
4799 
4800 			llclose();
4801 		}
4802 
4803 		/++
4804 			Sends a ping message to the server. This is done automatically by the library if you set a non-zero [Config.pingFrequency], but you can also send extra pings explicitly as well with this function.
4805 		+/
4806 		/// Group: foundational
4807 		void ping() {
4808 			WebSocketFrame wss;
4809 			wss.fin = true;
4810 			wss.opcode = WebSocketOpcode.ping;
4811 			wss.send(&llsend);
4812 		}
4813 
4814 		// automatically handled....
4815 		void pong() {
4816 			WebSocketFrame wss;
4817 			wss.fin = true;
4818 			wss.opcode = WebSocketOpcode.pong;
4819 			wss.send(&llsend);
4820 		}
4821 
4822 		/++
4823 			Sends a text message through the websocket.
4824 		+/
4825 		/// Group: foundational
4826 		void send(in char[] textData) {
4827 			WebSocketFrame wss;
4828 			wss.fin = true;
4829 			wss.opcode = WebSocketOpcode.text;
4830 			wss.data = cast(ubyte[]) textData;
4831 			wss.send(&llsend);
4832 		}
4833 
4834 		/++
4835 			Sends a binary message through the websocket.
4836 		+/
4837 		/// Group: foundational
4838 		void send(in ubyte[] binaryData) {
4839 			WebSocketFrame wss;
4840 			wss.fin = true;
4841 			wss.opcode = WebSocketOpcode.binary;
4842 			wss.data = cast(ubyte[]) binaryData;
4843 			wss.send(&llsend);
4844 		}
4845 
4846 		/++
4847 			Waits for and returns the next complete message on the socket.
4848 
4849 			Note that the onmessage function is still called, right before
4850 			this returns.
4851 		+/
4852 		/// Group: blocking_api
4853 		public WebSocketFrame waitForNextMessage() {
4854 			do {
4855 				auto m = processOnce();
4856 				if(m.populated)
4857 					return m;
4858 			} while(lowLevelReceive());
4859 
4860 			return WebSocketFrame.init; // FIXME? maybe.
4861 		}
4862 
4863 		/++
4864 			Tells if [waitForNextMessage] would block.
4865 		+/
4866 		/// Group: blocking_api
4867 		public bool waitForNextMessageWouldBlock() {
4868 			checkAgain:
4869 			if(isMessageBuffered())
4870 				return false;
4871 			if(!isDataPending())
4872 				return true;
4873 			while(isDataPending())
4874 				lowLevelReceive();
4875 			goto checkAgain;
4876 		}
4877 
4878 		/++
4879 			Is there a message in the buffer already?
4880 			If `true`, [waitForNextMessage] is guaranteed to return immediately.
4881 			If `false`, check [isDataPending] as the next step.
4882 		+/
4883 		/// Group: blocking_api
4884 		public bool isMessageBuffered() {
4885 			ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength];
4886 			auto s = d;
4887 			if(d.length) {
4888 				auto orig = d;
4889 				auto m = WebSocketFrame.read(d);
4890 				// that's how it indicates that it needs more data
4891 				if(d !is orig)
4892 					return true;
4893 			}
4894 
4895 			return false;
4896 		}
4897 
4898 		private ubyte continuingType;
4899 		private ubyte[] continuingData;
4900 		//private size_t continuingDataLength;
4901 
4902 		private WebSocketFrame processOnce() {
4903 			ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength];
4904 			auto s = d;
4905 			// FIXME: handle continuation frames more efficiently. it should really just reuse the receive buffer.
4906 			WebSocketFrame m;
4907 			if(d.length) {
4908 				auto orig = d;
4909 				m = WebSocketFrame.read(d);
4910 				// that's how it indicates that it needs more data
4911 				if(d is orig)
4912 					return WebSocketFrame.init;
4913 				m.unmaskInPlace();
4914 				switch(m.opcode) {
4915 					case WebSocketOpcode.continuation:
4916 						if(continuingData.length + m.data.length > config.maximumMessageSize)
4917 							throw new Exception("message size exceeded");
4918 
4919 						continuingData ~= m.data;
4920 						if(m.fin) {
4921 							if(ontextmessage)
4922 								ontextmessage(cast(char[]) continuingData);
4923 							if(onbinarymessage)
4924 								onbinarymessage(continuingData);
4925 
4926 							continuingData = null;
4927 						}
4928 					break;
4929 					case WebSocketOpcode.text:
4930 						if(m.fin) {
4931 							if(ontextmessage)
4932 								ontextmessage(m.textData);
4933 						} else {
4934 							continuingType = m.opcode;
4935 							//continuingDataLength = 0;
4936 							continuingData = null;
4937 							continuingData ~= m.data;
4938 						}
4939 					break;
4940 					case WebSocketOpcode.binary:
4941 						if(m.fin) {
4942 							if(onbinarymessage)
4943 								onbinarymessage(m.data);
4944 						} else {
4945 							continuingType = m.opcode;
4946 							//continuingDataLength = 0;
4947 							continuingData = null;
4948 							continuingData ~= m.data;
4949 						}
4950 					break;
4951 					case WebSocketOpcode.close:
4952 						readyState_ = CLOSED;
4953 						if(onclose)
4954 							onclose();
4955 
4956 						unregisterActiveSocket(this);
4957 					break;
4958 					case WebSocketOpcode.ping:
4959 						pong();
4960 					break;
4961 					case WebSocketOpcode.pong:
4962 						// just really references it is still alive, nbd.
4963 					break;
4964 					default: // ignore though i could and perhaps should throw too
4965 				}
4966 			}
4967 			receiveBufferUsedLength -= s.length - d.length;
4968 
4969 			return m;
4970 		}
4971 
4972 		private void autoprocess() {
4973 			// FIXME
4974 			do {
4975 				processOnce();
4976 			} while(lowLevelReceive());
4977 		}
4978 
4979 
4980 		void delegate() onclose; ///
4981 		void delegate() onerror; ///
4982 		void delegate(in char[]) ontextmessage; ///
4983 		void delegate(in ubyte[]) onbinarymessage; ///
4984 		void delegate() onopen; ///
4985 
4986 		/++
4987 
4988 		+/
4989 		/// Group: browser_api
4990 		void onmessage(void delegate(in char[]) dg) {
4991 			ontextmessage = dg;
4992 		}
4993 
4994 		/// ditto
4995 		void onmessage(void delegate(in ubyte[]) dg) {
4996 			onbinarymessage = dg;
4997 		}
4998 
4999 		/* } end copy/paste */
5000 
5001 
5002 	}
5003 
5004 	bool websocketRequested(Cgi cgi) {
5005 		return
5006 			"sec-websocket-key" in cgi.requestHeaders
5007 			&&
5008 			"connection" in cgi.requestHeaders &&
5009 				cgi.requestHeaders["connection"].asLowerCase().canFind("upgrade")
5010 			&&
5011 			"upgrade" in cgi.requestHeaders &&
5012 				cgi.requestHeaders["upgrade"].asLowerCase().equal("websocket")
5013 			;
5014 	}
5015 
5016 	WebSocket acceptWebsocket(Cgi cgi) {
5017 		assert(!cgi.closed);
5018 		assert(!cgi.outputtedResponseData);
5019 		cgi.setResponseStatus("101 Switching Protocols");
5020 		cgi.header("Upgrade: WebSocket");
5021 		cgi.header("Connection: upgrade");
5022 
5023 		string key = cgi.requestHeaders["sec-websocket-key"];
5024 		key ~= "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; // the defined guid from the websocket spec
5025 
5026 		import std.digest.sha;
5027 		auto hash = sha1Of(key);
5028 		auto accept = Base64.encode(hash);
5029 
5030 		cgi.header(("Sec-WebSocket-Accept: " ~ accept).idup);
5031 
5032 		cgi.websocketMode = true;
5033 		cgi.write("");
5034 
5035 		cgi.flush();
5036 
5037 		return new WebSocket(cgi);
5038 	}
5039 
5040 	// FIXME get websocket to work on other modes, not just embedded_httpd
5041 
5042 	/* copy/paste in http2.d { */
5043 	enum WebSocketOpcode : ubyte {
5044 		continuation = 0,
5045 		text = 1,
5046 		binary = 2,
5047 		// 3, 4, 5, 6, 7 RESERVED
5048 		close = 8,
5049 		ping = 9,
5050 		pong = 10,
5051 		// 11,12,13,14,15 RESERVED
5052 	}
5053 
5054 	public struct WebSocketFrame {
5055 		private bool populated;
5056 		bool fin;
5057 		bool rsv1;
5058 		bool rsv2;
5059 		bool rsv3;
5060 		WebSocketOpcode opcode; // 4 bits
5061 		bool masked;
5062 		ubyte lengthIndicator; // don't set this when building one to send
5063 		ulong realLength; // don't use when sending
5064 		ubyte[4] maskingKey; // don't set this when sending
5065 		ubyte[] data;
5066 
5067 		static WebSocketFrame simpleMessage(WebSocketOpcode opcode, void[] data) {
5068 			WebSocketFrame msg;
5069 			msg.fin = true;
5070 			msg.opcode = opcode;
5071 			msg.data = cast(ubyte[]) data;
5072 
5073 			return msg;
5074 		}
5075 
5076 		private void send(scope void delegate(ubyte[]) llsend) {
5077 			ubyte[64] headerScratch;
5078 			int headerScratchPos = 0;
5079 
5080 			realLength = data.length;
5081 
5082 			{
5083 				ubyte b1;
5084 				b1 |= cast(ubyte) opcode;
5085 				b1 |= rsv3 ? (1 << 4) : 0;
5086 				b1 |= rsv2 ? (1 << 5) : 0;
5087 				b1 |= rsv1 ? (1 << 6) : 0;
5088 				b1 |= fin  ? (1 << 7) : 0;
5089 
5090 				headerScratch[0] = b1;
5091 				headerScratchPos++;
5092 			}
5093 
5094 			{
5095 				headerScratchPos++; // we'll set header[1] at the end of this
5096 				auto rlc = realLength;
5097 				ubyte b2;
5098 				b2 |= masked ? (1 << 7) : 0;
5099 
5100 				assert(headerScratchPos == 2);
5101 
5102 				if(realLength > 65535) {
5103 					// use 64 bit length
5104 					b2 |= 0x7f;
5105 
5106 					// FIXME: double check endinaness
5107 					foreach(i; 0 .. 8) {
5108 						headerScratch[2 + 7 - i] = rlc & 0x0ff;
5109 						rlc >>>= 8;
5110 					}
5111 
5112 					headerScratchPos += 8;
5113 				} else if(realLength > 125) {
5114 					// use 16 bit length
5115 					b2 |= 0x7e;
5116 
5117 					// FIXME: double check endinaness
5118 					foreach(i; 0 .. 2) {
5119 						headerScratch[2 + 1 - i] = rlc & 0x0ff;
5120 						rlc >>>= 8;
5121 					}
5122 
5123 					headerScratchPos += 2;
5124 				} else {
5125 					// use 7 bit length
5126 					b2 |= realLength & 0b_0111_1111;
5127 				}
5128 
5129 				headerScratch[1] = b2;
5130 			}
5131 
5132 			//assert(!masked, "masking key not properly implemented");
5133 			if(masked) {
5134 				// FIXME: randomize this
5135 				headerScratch[headerScratchPos .. headerScratchPos + 4] = maskingKey[];
5136 				headerScratchPos += 4;
5137 
5138 				// we'll just mask it in place...
5139 				int keyIdx = 0;
5140 				foreach(i; 0 .. data.length) {
5141 					data[i] = data[i] ^ maskingKey[keyIdx];
5142 					if(keyIdx == 3)
5143 						keyIdx = 0;
5144 					else
5145 						keyIdx++;
5146 				}
5147 			}
5148 
5149 			//writeln("SENDING ", headerScratch[0 .. headerScratchPos], data);
5150 			llsend(headerScratch[0 .. headerScratchPos]);
5151 			llsend(data);
5152 		}
5153 
5154 		static WebSocketFrame read(ref ubyte[] d) {
5155 			WebSocketFrame msg;
5156 
5157 			auto orig = d;
5158 
5159 			WebSocketFrame needsMoreData() {
5160 				d = orig;
5161 				return WebSocketFrame.init;
5162 			}
5163 
5164 			if(d.length < 2)
5165 				return needsMoreData();
5166 
5167 			ubyte b = d[0];
5168 
5169 			msg.populated = true;
5170 
5171 			msg.opcode = cast(WebSocketOpcode) (b & 0x0f);
5172 			b >>= 4;
5173 			msg.rsv3 = b & 0x01;
5174 			b >>= 1;
5175 			msg.rsv2 = b & 0x01;
5176 			b >>= 1;
5177 			msg.rsv1 = b & 0x01;
5178 			b >>= 1;
5179 			msg.fin = b & 0x01;
5180 
5181 			b = d[1];
5182 			msg.masked = (b & 0b1000_0000) ? true : false;
5183 			msg.lengthIndicator = b & 0b0111_1111;
5184 
5185 			d = d[2 .. $];
5186 
5187 			if(msg.lengthIndicator == 0x7e) {
5188 				// 16 bit length
5189 				msg.realLength = 0;
5190 
5191 				if(d.length < 2) return needsMoreData();
5192 
5193 				foreach(i; 0 .. 2) {
5194 					msg.realLength |= d[0] << ((1-i) * 8);
5195 					d = d[1 .. $];
5196 				}
5197 			} else if(msg.lengthIndicator == 0x7f) {
5198 				// 64 bit length
5199 				msg.realLength = 0;
5200 
5201 				if(d.length < 8) return needsMoreData();
5202 
5203 				foreach(i; 0 .. 8) {
5204 					msg.realLength |= d[0] << ((7-i) * 8);
5205 					d = d[1 .. $];
5206 				}
5207 			} else {
5208 				// 7 bit length
5209 				msg.realLength = msg.lengthIndicator;
5210 			}
5211 
5212 			if(msg.masked) {
5213 
5214 				if(d.length < 4) return needsMoreData();
5215 
5216 				msg.maskingKey = d[0 .. 4];
5217 				d = d[4 .. $];
5218 			}
5219 
5220 			if(msg.realLength > d.length) {
5221 				return needsMoreData();
5222 			}
5223 
5224 			msg.data = d[0 .. cast(size_t) msg.realLength];
5225 			d = d[cast(size_t) msg.realLength .. $];
5226 
5227 			return msg;
5228 		}
5229 
5230 		void unmaskInPlace() {
5231 			if(this.masked) {
5232 				int keyIdx = 0;
5233 				foreach(i; 0 .. this.data.length) {
5234 					this.data[i] = this.data[i] ^ this.maskingKey[keyIdx];
5235 					if(keyIdx == 3)
5236 						keyIdx = 0;
5237 					else
5238 						keyIdx++;
5239 				}
5240 			}
5241 		}
5242 
5243 		char[] textData() {
5244 			return cast(char[]) data;
5245 		}
5246 	}
5247 	/* } */
5248 }
5249 
5250 
5251 version(Windows)
5252 {
5253     version(CRuntime_DigitalMars)
5254     {
5255         extern(C) int setmode(int, int) nothrow @nogc;
5256     }
5257     else version(CRuntime_Microsoft)
5258     {
5259         extern(C) int _setmode(int, int) nothrow @nogc;
5260         alias setmode = _setmode;
5261     }
5262     else static assert(0);
5263 }
5264 
5265 version(Posix) {
5266 	version(CRuntime_Musl) {} else {
5267 		import core.sys.posix.unistd;
5268 		private extern(C) int posix_spawn(pid_t*, const char*, void*, void*, const char**, const char**);
5269 	}
5270 }
5271 
5272 
5273 // FIXME: these aren't quite public yet.
5274 //private:
5275 
5276 // template for laziness
5277 void startWebsocketServer()() {
5278 	version(linux) {
5279 		import core.sys.posix.unistd;
5280 		pid_t pid;
5281 		const(char)*[16] args;
5282 		args[0] = "ARSD_CGI_WEBSOCKET_SERVER";
5283 		args[1] = "--websocket-server";
5284 		posix_spawn(&pid, "/proc/self/exe",
5285 			null,
5286 			null,
5287 			args.ptr,
5288 			null // env
5289 		);
5290 	} else version(Windows) {
5291 		wchar[2048] filename;
5292 		auto len = GetModuleFileNameW(null, filename.ptr, cast(DWORD) filename.length);
5293 		if(len == 0 || len == filename.length)
5294 			throw new Exception("could not get process name to start helper server");
5295 
5296 		STARTUPINFOW startupInfo;
5297 		startupInfo.cb = cast(DWORD) startupInfo.sizeof;
5298 		PROCESS_INFORMATION processInfo;
5299 
5300 		// I *MIGHT* need to run it as a new job or a service...
5301 		auto ret = CreateProcessW(
5302 			filename.ptr,
5303 			"--websocket-server"w,
5304 			null, // process attributes
5305 			null, // thread attributes
5306 			false, // inherit handles
5307 			0, // creation flags
5308 			null, // environment
5309 			null, // working directory
5310 			&startupInfo,
5311 			&processInfo
5312 		);
5313 
5314 		if(!ret)
5315 			throw new Exception("create process failed");
5316 
5317 		// when done with those, if we set them
5318 		/*
5319 		CloseHandle(hStdInput);
5320 		CloseHandle(hStdOutput);
5321 		CloseHandle(hStdError);
5322 		*/
5323 
5324 	} else static assert(0, "Websocket server not implemented on this system yet (email me, i can prolly do it if you need it)");
5325 }
5326 
5327 // template for laziness
5328 /*
5329 	The websocket server is a single-process, single-thread, event
5330 	I/O thing. It is passed websockets from other CGI processes
5331 	and is then responsible for handling their messages and responses.
5332 	Note that the CGI process is responsible for websocket setup,
5333 	including authentication, etc.
5334 
5335 	It also gets data sent to it by other processes and is responsible
5336 	for distributing that, as necessary.
5337 */
5338 void runWebsocketServer()() {
5339 	assert(0, "not implemented");
5340 }
5341 
5342 void sendToWebsocketServer(WebSocket ws, string group) {
5343 	assert(0, "not implemented");
5344 }
5345 
5346 void sendToWebsocketServer(string content, string group) {
5347 	assert(0, "not implemented");
5348 }
5349 
5350 
5351 void runEventServer()() {
5352 	runAddonServer("/tmp/arsd_cgi_event_server", new EventSourceServerImplementation());
5353 }
5354 
5355 void runTimerServer()() {
5356 	runAddonServer("/tmp/arsd_scheduled_job_server", new ScheduledJobServerImplementation());
5357 }
5358 
5359 version(Posix) {
5360 	alias LocalServerConnectionHandle = int;
5361 	alias CgiConnectionHandle = int;
5362 	alias SocketConnectionHandle = int;
5363 
5364 	enum INVALID_CGI_CONNECTION_HANDLE = -1;
5365 } else version(Windows) {
5366 	alias LocalServerConnectionHandle = HANDLE;
5367 	version(embedded_httpd_threads) {
5368 		alias CgiConnectionHandle = SOCKET;
5369 		enum INVALID_CGI_CONNECTION_HANDLE = INVALID_SOCKET;
5370 	} else version(fastcgi) {
5371 		alias CgiConnectionHandle = void*; // Doesn't actually work! But I don't want compile to fail pointlessly at this point.
5372 		enum INVALID_CGI_CONNECTION_HANDLE = null;
5373 	} else version(scgi) {
5374 		alias CgiConnectionHandle = SOCKET;
5375 		enum INVALID_CGI_CONNECTION_HANDLE = INVALID_SOCKET;
5376 	} else { /* version(plain_cgi) */
5377 		alias CgiConnectionHandle = HANDLE;
5378 		enum INVALID_CGI_CONNECTION_HANDLE = null;
5379 	}
5380 	alias SocketConnectionHandle = SOCKET;
5381 }
5382 
5383 version(with_addon_servers_connections)
5384 LocalServerConnectionHandle openLocalServerConnection(string name) {
5385 	version(Posix) {
5386 		import core.sys.posix.unistd;
5387 		import core.sys.posix.sys.un;
5388 
5389 		int sock = socket(AF_UNIX, SOCK_STREAM, 0);
5390 		if(sock == -1)
5391 			throw new Exception("socket " ~ to!string(errno));
5392 
5393 		scope(failure)
5394 			close(sock);
5395 
5396 		// add-on server processes are assumed to be local, and thus will
5397 		// use unix domain sockets. Besides, I want to pass sockets to them,
5398 		// so it basically must be local (except for the session server, but meh).
5399 		sockaddr_un addr;
5400 		addr.sun_family = AF_UNIX;
5401 		version(linux) {
5402 			// on linux, we will use the abstract namespace
5403 			addr.sun_path[0] = 0;
5404 			addr.sun_path[1 .. name.length + 1] = cast(typeof(addr.sun_path[])) name[];
5405 		} else {
5406 			// but otherwise, just use a file cuz we must.
5407 			addr.sun_path[0 .. name.length] = cast(typeof(addr.sun_path[])) name[];
5408 		}
5409 
5410 		if(connect(sock, cast(sockaddr*) &addr, addr.sizeof) == -1)
5411 			throw new Exception("connect " ~ to!string(errno));
5412 
5413 		return sock;
5414 	} else version(Windows) {
5415 		return null; // FIXME
5416 	}
5417 }
5418 
5419 version(with_addon_servers_connections)
5420 void closeLocalServerConnection(LocalServerConnectionHandle handle) {
5421 	version(Posix) {
5422 		import core.sys.posix.unistd;
5423 		close(handle);
5424 	} else version(Windows)
5425 		CloseHandle(handle);
5426 }
5427 
5428 void runSessionServer()() {
5429 	runAddonServer("/tmp/arsd_session_server", new BasicDataServerImplementation());
5430 }
5431 
5432 version(Posix)
5433 private void makeNonBlocking(int fd) {
5434 	import core.sys.posix.fcntl;
5435 	auto flags = fcntl(fd, F_GETFL, 0);
5436 	if(flags == -1)
5437 		throw new Exception("fcntl get");
5438 	flags |= O_NONBLOCK;
5439 	auto s = fcntl(fd, F_SETFL, flags);
5440 	if(s == -1)
5441 		throw new Exception("fcntl set");
5442 }
5443 
5444 import core.stdc.errno;
5445 
5446 struct IoOp {
5447 	@disable this();
5448 	@disable this(this);
5449 
5450 	/*
5451 		So we want to be able to eventually handle generic sockets too.
5452 	*/
5453 
5454 	enum Read = 1;
5455 	enum Write = 2;
5456 	enum Accept = 3;
5457 	enum ReadSocketHandle = 4;
5458 
5459 	// Your handler may be called in a different thread than the one that initiated the IO request!
5460 	// It is also possible to have multiple io requests being called simultaneously. Use proper thread safety caution.
5461 	private bool delegate(IoOp*, int) handler; // returns true if you are done and want it to be closed
5462 	private void delegate(IoOp*) closeHandler;
5463 	private void delegate(IoOp*) completeHandler;
5464 	private int internalFd;
5465 	private int operation;
5466 	private int bufferLengthAllocated;
5467 	private int bufferLengthUsed;
5468 	private ubyte[1] internalBuffer; // it can be overallocated!
5469 
5470 	ubyte[] allocatedBuffer() return {
5471 		return internalBuffer.ptr[0 .. bufferLengthAllocated];
5472 	}
5473 
5474 	ubyte[] usedBuffer() return {
5475 		return allocatedBuffer[0 .. bufferLengthUsed];
5476 	}
5477 
5478 	void reset() {
5479 		bufferLengthUsed = 0;
5480 	}
5481 
5482 	int fd() {
5483 		return internalFd;
5484 	}
5485 }
5486 
5487 IoOp* allocateIoOp(int fd, int operation, int bufferSize, bool delegate(IoOp*, int) handler) {
5488 	import core.stdc.stdlib;
5489 
5490 	auto ptr = calloc(IoOp.sizeof + bufferSize, 1);
5491 	if(ptr is null)
5492 		assert(0); // out of memory!
5493 
5494 	auto op = cast(IoOp*) ptr;
5495 
5496 	op.handler = handler;
5497 	op.internalFd = fd;
5498 	op.operation = operation;
5499 	op.bufferLengthAllocated = bufferSize;
5500 	op.bufferLengthUsed = 0;
5501 
5502 	import core.memory;
5503 
5504 	GC.addRoot(ptr);
5505 
5506 	return op;
5507 }
5508 
5509 void freeIoOp(ref IoOp* ptr) {
5510 
5511 	import core.memory;
5512 	GC.removeRoot(ptr);
5513 
5514 	import core.stdc.stdlib;
5515 	free(ptr);
5516 	ptr = null;
5517 }
5518 
5519 version(Posix)
5520 version(with_addon_servers_connections)
5521 void nonBlockingWrite(EventIoServer eis, int connection, const void[] data) {
5522 	import core.sys.posix.unistd;
5523 
5524 	auto ret = write(connection, data.ptr, data.length);
5525 	if(ret != data.length) {
5526 		if(ret == 0 || errno == EPIPE) {
5527 			// the file is closed, remove it
5528 			eis.fileClosed(connection);
5529 		} else
5530 			throw new Exception("alas " ~ to!string(ret) ~ " " ~ to!string(errno)); // FIXME
5531 	}
5532 }
5533 version(Windows)
5534 version(with_addon_servers_connections)
5535 void nonBlockingWrite(EventIoServer eis, int connection, const void[] data) {
5536 	// FIXME
5537 }
5538 
5539 bool isInvalidHandle(CgiConnectionHandle h) {
5540 	return h == INVALID_CGI_CONNECTION_HANDLE;
5541 }
5542 
5543 /+
5544 https://docs.microsoft.com/en-us/windows/desktop/api/winsock2/nf-winsock2-wsarecv
5545 https://support.microsoft.com/en-gb/help/181611/socket-overlapped-i-o-versus-blocking-nonblocking-mode
5546 https://stackoverflow.com/questions/18018489/should-i-use-iocps-or-overlapped-wsasend-receive
5547 https://docs.microsoft.com/en-us/windows/desktop/fileio/i-o-completion-ports
5548 https://docs.microsoft.com/en-us/windows/desktop/fileio/createiocompletionport
5549 https://docs.microsoft.com/en-us/windows/desktop/api/mswsock/nf-mswsock-acceptex
5550 https://docs.microsoft.com/en-us/windows/desktop/Sync/waitable-timer-objects
5551 https://docs.microsoft.com/en-us/windows/desktop/api/synchapi/nf-synchapi-setwaitabletimer
5552 https://docs.microsoft.com/en-us/windows/desktop/Sync/using-a-waitable-timer-with-an-asynchronous-procedure-call
5553 https://docs.microsoft.com/en-us/windows/desktop/api/winsock2/nf-winsock2-wsagetoverlappedresult
5554 
5555 +/
5556 
5557 /++
5558 	You can customize your server by subclassing the appropriate server. Then, register your
5559 	subclass at compile time with the [registerEventIoServer] template, or implement your own
5560 	main function and call it yourself.
5561 	
5562 	$(TIP If you make your subclass a `final class`, there is a slight performance improvement.)
5563 +/
5564 version(with_addon_servers_connections)
5565 interface EventIoServer {
5566 	bool handleLocalConnectionData(IoOp* op, int receivedFd);
5567 	void handleLocalConnectionClose(IoOp* op);
5568 	void handleLocalConnectionComplete(IoOp* op);
5569 	void wait_timeout();
5570 	void fileClosed(int fd);
5571 
5572 	void epoll_fd(int fd);
5573 }
5574 
5575 // the sink should buffer it
5576 private void serialize(T)(scope void delegate(ubyte[]) sink, T t) {
5577 	static if(is(T == struct)) {
5578 		foreach(member; __traits(allMembers, T))
5579 			serialize(sink, __traits(getMember, t, member));
5580 	} else static if(is(T : int)) {
5581 		// no need to think of endianness just because this is only used
5582 		// for local, same-machine stuff anyway. thanks private lol
5583 		sink((cast(ubyte*) &t)[0 .. t.sizeof]);
5584 	} else static if(is(T == string) || is(T : const(ubyte)[])) {
5585 		// these are common enough to optimize
5586 		int len = cast(int) t.length; // want length consistent size tho, in case 32 bit program sends to 64 bit server, etc.
5587 		sink((cast(ubyte*) &len)[0 .. int.sizeof]);
5588 		sink(cast(ubyte[]) t[]);
5589 	} else static if(is(T : A[], A)) {
5590 		// generic array is less optimal but still prolly ok
5591 		int len = cast(int) t.length;
5592 		sink((cast(ubyte*) &len)[0 .. int.sizeof]);
5593 		foreach(item; t)
5594 			serialize(sink, item);
5595 	} else static assert(0, T.stringof);
5596 }
5597 
5598 // all may be stack buffers, so use cautio
5599 private void deserialize(T)(scope ubyte[] delegate(int sz) get, scope void delegate(T) dg) {
5600 	static if(is(T == struct)) {
5601 		T t;
5602 		foreach(member; __traits(allMembers, T))
5603 			deserialize!(typeof(__traits(getMember, T, member)))(get, (mbr) { __traits(getMember, t, member) = mbr; });
5604 		dg(t);
5605 	} else static if(is(T : int)) {
5606 		// no need to think of endianness just because this is only used
5607 		// for local, same-machine stuff anyway. thanks private lol
5608 		T t;
5609 		auto data = get(t.sizeof);
5610 		t = (cast(T[]) data)[0];
5611 		dg(t);
5612 	} else static if(is(T == string) || is(T : const(ubyte)[])) {
5613 		// these are common enough to optimize
5614 		int len;
5615 		auto data = get(len.sizeof);
5616 		len = (cast(int[]) data)[0];
5617 
5618 		/*
5619 		typeof(T[0])[2000] stackBuffer;
5620 		T buffer;
5621 
5622 		if(len < stackBuffer.length)
5623 			buffer = stackBuffer[0 .. len];
5624 		else
5625 			buffer = new T(len);
5626 
5627 		data = get(len * typeof(T[0]).sizeof);
5628 		*/
5629 
5630 		T t = cast(T) get(len * cast(int) typeof(T.init[0]).sizeof);
5631 
5632 		dg(t);
5633 	} else static if(is(T == E[], E)) {
5634 		T t;
5635 		int len;
5636 		auto data = get(len.sizeof);
5637 		len = (cast(int[]) data)[0];
5638 		t.length = len;
5639 		foreach(ref e; t) {
5640 			deserialize!E(get, (ele) { e = ele; });
5641 		}
5642 		dg(t);
5643 	} else static assert(0, T.stringof);
5644 }
5645 
5646 unittest {
5647 	serialize((ubyte[] b) {
5648 		deserialize!int( sz => b[0 .. sz], (t) { assert(t == 1); });
5649 	}, 1);
5650 	serialize((ubyte[] b) {
5651 		deserialize!int( sz => b[0 .. sz], (t) { assert(t == 56674); });
5652 	}, 56674);
5653 	ubyte[1000] buffer;
5654 	int bufferPoint;
5655 	void add(ubyte[] b) {
5656 		buffer[bufferPoint ..  bufferPoint + b.length] = b[];
5657 		bufferPoint += b.length;
5658 	}
5659 	ubyte[] get(int sz) {
5660 		auto b = buffer[bufferPoint .. bufferPoint + sz];
5661 		bufferPoint += sz;
5662 		return b;
5663 	}
5664 	serialize(&add, "test here");
5665 	bufferPoint = 0;
5666 	deserialize!string(&get, (t) { assert(t == "test here"); });
5667 	bufferPoint = 0;
5668 
5669 	struct Foo {
5670 		int a;
5671 		ubyte c;
5672 		string d;
5673 	}
5674 	serialize(&add, Foo(403, 37, "amazing"));
5675 	bufferPoint = 0;
5676 	deserialize!Foo(&get, (t) {
5677 		assert(t.a == 403);
5678 		assert(t.c == 37);
5679 		assert(t.d == "amazing");
5680 	});
5681 	bufferPoint = 0;
5682 }
5683 
5684 /*
5685 	Here's the way the RPC interface works:
5686 
5687 	You define the interface that lists the functions you can call on the remote process.
5688 	The interface may also have static methods for convenience. These forward to a singleton
5689 	instance of an auto-generated class, which actually sends the args over the pipe.
5690 
5691 	An impl class actually implements it. A receiving server deserializes down the pipe and
5692 	calls methods on the class.
5693 
5694 	I went with the interface to get some nice compiler checking and documentation stuff.
5695 
5696 	I could have skipped the interface and just implemented it all from the server class definition
5697 	itself, but then the usage may call the method instead of rpcing it; I just like having the user
5698 	interface and the implementation separate so you aren't tempted to `new impl` to call the methods.
5699 
5700 
5701 	I fiddled with newlines in the mixin string to ensure the assert line numbers matched up to the source code line number. Idk why dmd didn't do this automatically, but it was important to me.
5702 
5703 	Realistically though the bodies would just be
5704 		connection.call(this.mangleof, args...) sooooo.
5705 
5706 	FIXME: overloads aren't supported
5707 */
5708 
5709 /// Base for storing sessions in an array. Exists primarily for internal purposes and you should generally not use this.
5710 interface SessionObject {}
5711 
5712 private immutable void delegate(string[])[string] scheduledJobHandlers;
5713 
5714 version(with_breaking_cgi_features)
5715 mixin(q{
5716 
5717 mixin template ImplementRpcClientInterface(T, string serverPath) {
5718 	static import std.traits;
5719 
5720 	// derivedMembers on an interface seems to give exactly what I want: the virtual functions we need to implement. so I am just going to use it directly without more filtering.
5721 	static foreach(idx, member; __traits(derivedMembers, T)) {
5722 	static if(__traits(isVirtualFunction, __traits(getMember, T, member)))
5723 		mixin( q{
5724 		std.traits.ReturnType!(__traits(getMember, T, member))
5725 		} ~ member ~ q{(std.traits.Parameters!(__traits(getMember, T, member)) params)
5726 		{
5727 			SerializationBuffer buffer;
5728 			auto i = cast(ushort) idx;
5729 			serialize(&buffer.sink, i);
5730 			serialize(&buffer.sink, __traits(getMember, T, member).mangleof);
5731 			foreach(param; params)
5732 				serialize(&buffer.sink, param);
5733 
5734 			auto sendable = buffer.sendable;
5735 
5736 			version(Posix) {{
5737 				auto ret = send(connectionHandle, sendable.ptr, sendable.length, 0);
5738 				assert(ret == sendable.length);
5739 			}} // FIXME Windows impl
5740 
5741 			static if(!is(typeof(return) == void)) {
5742 				// there is a return value; we need to wait for it too
5743 				version(Posix) {
5744 					ubyte[3000] revBuffer;
5745 					auto ret = recv(connectionHandle, revBuffer.ptr, revBuffer.length, 0);
5746 					auto got = revBuffer[0 .. ret];
5747 
5748 					int dataLocation;
5749 					ubyte[] grab(int sz) {
5750 						auto d = got[dataLocation .. dataLocation + sz];
5751 						dataLocation += sz;
5752 						return d;
5753 					}
5754 
5755 					typeof(return) retu;
5756 					deserialize!(typeof(return))(&grab, (a) { retu = a; });
5757 					return retu;
5758 				} else {
5759 					// FIXME Windows impl
5760 					return typeof(return).init;
5761 				}
5762 
5763 			}
5764 		}});
5765 	}
5766 
5767 	private static typeof(this) singletonInstance;
5768 	private LocalServerConnectionHandle connectionHandle;
5769 
5770 	static typeof(this) connection() {
5771 		if(singletonInstance is null) {
5772 			singletonInstance = new typeof(this)();
5773 			singletonInstance.connect();
5774 		}
5775 		return singletonInstance;
5776 	}
5777 
5778 	void connect() {
5779 		connectionHandle = openLocalServerConnection(serverPath);
5780 	}
5781 
5782 	void disconnect() {
5783 		closeLocalServerConnection(connectionHandle);
5784 	}
5785 }
5786 
5787 void dispatchRpcServer(Interface, Class)(Class this_, ubyte[] data, int fd) if(is(Class : Interface)) {
5788 	ushort calledIdx;
5789 	string calledFunction;
5790 
5791 	int dataLocation;
5792 	ubyte[] grab(int sz) {
5793 		auto d = data[dataLocation .. dataLocation + sz];
5794 		dataLocation += sz;
5795 		return d;
5796 	}
5797 
5798 	again:
5799 
5800 	deserialize!ushort(&grab, (a) { calledIdx = a; });
5801 	deserialize!string(&grab, (a) { calledFunction = a; });
5802 
5803 	import std.traits;
5804 
5805 	sw: switch(calledIdx) {
5806 		foreach(idx, memberName; __traits(derivedMembers, Interface))
5807 		static if(__traits(isVirtualFunction, __traits(getMember, Interface, memberName))) {
5808 			case idx:
5809 				assert(calledFunction == __traits(getMember, Interface, memberName).mangleof);
5810 
5811 				Parameters!(__traits(getMember, Interface, memberName)) params;
5812 				foreach(ref param; params)
5813 					deserialize!(typeof(param))(&grab, (a) { param = a; });
5814 
5815 				static if(is(ReturnType!(__traits(getMember, Interface, memberName)) == void)) {
5816 					__traits(getMember, this_, memberName)(params);
5817 				} else {
5818 					auto ret = __traits(getMember, this_, memberName)(params);
5819 					SerializationBuffer buffer;
5820 					serialize(&buffer.sink, ret);
5821 
5822 					auto sendable = buffer.sendable;
5823 
5824 					version(Posix) {
5825 						auto r = send(fd, sendable.ptr, sendable.length, 0);
5826 						assert(r == sendable.length);
5827 					} // FIXME Windows impl
5828 				}
5829 			break sw;
5830 		}
5831 		default: assert(0);
5832 	}
5833 
5834 	if(dataLocation != data.length)
5835 		goto again;
5836 }
5837 
5838 
5839 private struct SerializationBuffer {
5840 	ubyte[2048] bufferBacking;
5841 	int bufferLocation;
5842 	void sink(scope ubyte[] data) {
5843 		bufferBacking[bufferLocation .. bufferLocation + data.length] = data[];
5844 		bufferLocation += data.length;
5845 	}
5846 
5847 	ubyte[] sendable() return {
5848 		return bufferBacking[0 .. bufferLocation];
5849 	}
5850 }
5851 
5852 /*
5853 	FIXME:
5854 		add a version command line arg
5855 		version data in the library
5856 		management gui as external program
5857 
5858 		at server with event_fd for each run
5859 		use .mangleof in the at function name
5860 
5861 		i think the at server will have to:
5862 			pipe args to the child
5863 			collect child output for logging
5864 			get child return value for logging
5865 
5866 			on windows timers work differently. idk how to best combine with the io stuff.
5867 
5868 			will have to have dump and restore too, so i can restart without losing stuff.
5869 */
5870 
5871 /++
5872 	A convenience object for talking to the [BasicDataServer] from a higher level.
5873 	See: [Cgi.getSessionObject].
5874 
5875 	You pass it a `Data` struct describing the data you want saved in the session.
5876 	Then, this class will generate getter and setter properties that allow access
5877 	to that data.
5878 
5879 	Note that each load and store will be done as-accessed; it doesn't front-load
5880 	mutable data nor does it batch updates out of fear of read-modify-write race
5881 	conditions. (In fact, right now it does this for everything, but in the future,
5882 	I might batch load `immutable` members of the Data struct.)
5883 
5884 	At some point in the future, I might also let it do different backends, like
5885 	a client-side cookie store too, but idk.
5886 
5887 	Note that the plain-old-data members of your `Data` struct are wrapped by this
5888 	interface via a static foreach to make property functions.
5889 
5890 	See_Also: [MockSession]
5891 +/
5892 interface Session(Data) : SessionObject {
5893 	@property string sessionId() const;
5894 
5895 	/++
5896 		Starts a new session. Note that a session is also
5897 		implicitly started as soon as you write data to it,
5898 		so if you need to alter these parameters from their
5899 		defaults, be sure to explicitly call this BEFORE doing
5900 		any writes to session data.
5901 
5902 		Params:
5903 			idleLifetime = How long, in seconds, the session
5904 			should remain in memory when not being read from
5905 			or written to. The default is one day.
5906 
5907 			NOT IMPLEMENTED
5908 
5909 			useExtendedLifetimeCookie = The session ID is always
5910 			stored in a HTTP cookie, and by default, that cookie
5911 			is discarded when the user closes their browser.
5912 
5913 			But if you set this to true, it will use a non-perishable
5914 			cookie for the given idleLifetime.
5915 
5916 			NOT IMPLEMENTED
5917 	+/
5918 	void start(int idleLifetime = 2600 * 24, bool useExtendedLifetimeCookie = false);
5919 
5920 	/++
5921 		Regenerates the session ID and updates the associated
5922 		cookie.
5923 
5924 		This is also your chance to change immutable data
5925 		(not yet implemented).
5926 	+/
5927 	void regenerateId();
5928 
5929 	/++
5930 		Terminates this session, deleting all saved data.
5931 	+/
5932 	void terminate();
5933 
5934 	/++
5935 		Plain-old-data members of your `Data` struct are wrapped here via
5936 		the property getters and setters.
5937 
5938 		If the member is a non-string array, it returns a magical array proxy
5939 		object which allows for atomic appends and replaces via overloaded operators.
5940 		You can slice this to get a range representing a $(B const) view of the array.
5941 		This is to protect you against read-modify-write race conditions.
5942 	+/
5943 	static foreach(memberName; __traits(allMembers, Data))
5944 		static if(is(typeof(__traits(getMember, Data, memberName))))
5945 		mixin(q{
5946 			@property inout(typeof(__traits(getMember, Data, memberName))) } ~ memberName ~ q{ () inout;
5947 			@property void } ~ memberName ~ q{ (typeof(__traits(getMember, Data, memberName)) value);
5948 		});
5949 
5950 }
5951 
5952 /++
5953 	An implementation of [Session] that works on real cgi connections utilizing the
5954 	[BasicDataServer].
5955 	
5956 	As opposed to a [MockSession] which is made for testing purposes.
5957 
5958 	You will not construct one of these directly. See [Cgi.getSessionObject] instead.
5959 +/
5960 class BasicDataServerSession(Data) : Session!Data {
5961 	private Cgi cgi;
5962 	private string sessionId_;
5963 
5964 	public @property string sessionId() const {
5965 		return sessionId_;
5966 	}
5967 
5968 	protected @property string sessionId(string s) {
5969 		return this.sessionId_ = s;
5970 	}
5971 
5972 	private this(Cgi cgi) {
5973 		this.cgi = cgi;
5974 		if(auto ptr = "sessionId" in cgi.cookies)
5975 			sessionId = (*ptr).length ? *ptr : null;
5976 	}
5977 
5978 	void start(int idleLifetime = 2600 * 24, bool useExtendedLifetimeCookie = false) {
5979 		assert(sessionId is null);
5980 
5981 		// FIXME: what if there is a session ID cookie, but no corresponding session on the server?
5982 
5983 		import std.random, std.conv;
5984 		sessionId = to!string(uniform(1, long.max));
5985 
5986 		BasicDataServer.connection.createSession(sessionId, idleLifetime);
5987 		setCookie();
5988 	}
5989 
5990 	protected void setCookie() {
5991 		cgi.setCookie(
5992 			"sessionId", sessionId,
5993 			0 /* expiration */,
5994 			"/" /* path */,
5995 			null /* domain */,
5996 			true /* http only */,
5997 			cgi.https /* if the session is started on https, keep it there, otherwise, be flexible */);
5998 	}
5999 
6000 	void regenerateId() {
6001 		if(sessionId is null) {
6002 			start();
6003 			return;
6004 		}
6005 		import std.random, std.conv;
6006 		auto oldSessionId = sessionId;
6007 		sessionId = to!string(uniform(1, long.max));
6008 		BasicDataServer.connection.renameSession(oldSessionId, sessionId);
6009 		setCookie();
6010 	}
6011 
6012 	void terminate() {
6013 		BasicDataServer.connection.destroySession(sessionId);
6014 		sessionId = null;
6015 		setCookie();
6016 	}
6017 
6018 	static foreach(memberName; __traits(allMembers, Data))
6019 		static if(is(typeof(__traits(getMember, Data, memberName))))
6020 		mixin(q{
6021 			@property inout(typeof(__traits(getMember, Data, memberName))) } ~ memberName ~ q{ () inout {
6022 				if(sessionId is null)
6023 					return typeof(return).init;
6024 
6025 				import std.traits;
6026 				auto v = BasicDataServer.connection.getSessionData(sessionId, fullyQualifiedName!Data ~ "." ~ memberName);
6027 				if(v.length == 0)
6028 					return typeof(return).init;
6029 				import std.conv;
6030 				// why this cast? to doesn't like being given an inout argument. so need to do it without that, then
6031 				// we need to return it and that needed the cast. It should be fine since we basically respect constness..
6032 				// basically. Assuming the session is POD this should be fine.
6033 				return cast(typeof(return)) to!(typeof(__traits(getMember, Data, memberName)))(v);
6034 			}
6035 			@property void } ~ memberName ~ q{ (typeof(__traits(getMember, Data, memberName)) value) {
6036 				if(sessionId is null)
6037 					start();
6038 				import std.conv;
6039 				import std.traits;
6040 				BasicDataServer.connection.setSessionData(sessionId, fullyQualifiedName!Data ~ "." ~ memberName, to!string(value));
6041 			}
6042 		});
6043 }
6044 
6045 /++
6046 	A mock object that works like the real session, but doesn't actually interact with any actual database or http connection.
6047 	Simply stores the data in its instance members.
6048 +/
6049 class MockSession(Data) : Session!Data {
6050 	pure {
6051 		@property string sessionId() const { return "mock"; }
6052 		void start(int idleLifetime = 2600 * 24, bool useExtendedLifetimeCookie = false) {}
6053 		void regenerateId() {}
6054 		void terminate() {}
6055 
6056 		private Data store_;
6057 
6058 		static foreach(memberName; __traits(allMembers, Data))
6059 			static if(is(typeof(__traits(getMember, Data, memberName))))
6060 			mixin(q{
6061 				@property inout(typeof(__traits(getMember, Data, memberName))) } ~ memberName ~ q{ () inout {
6062 					return __traits(getMember, store_, memberName);
6063 				}
6064 				@property void } ~ memberName ~ q{ (typeof(__traits(getMember, Data, memberName)) value) {
6065 					__traits(getMember, store_, memberName) = value;
6066 				}
6067 			});
6068 	}
6069 }
6070 
6071 /++
6072 	Direct interface to the basic data add-on server. You can
6073 	typically use [Cgi.getSessionObject] as a more convenient interface.
6074 +/
6075 version(with_addon_servers_connections)
6076 interface BasicDataServer {
6077 	///
6078 	void createSession(string sessionId, int lifetime);
6079 	///
6080 	void renewSession(string sessionId, int lifetime);
6081 	///
6082 	void destroySession(string sessionId);
6083 	///
6084 	void renameSession(string oldSessionId, string newSessionId);
6085 
6086 	///
6087 	void setSessionData(string sessionId, string dataKey, string dataValue);
6088 	///
6089 	string getSessionData(string sessionId, string dataKey);
6090 
6091 	///
6092 	static BasicDataServerConnection connection() {
6093 		return BasicDataServerConnection.connection();
6094 	}
6095 }
6096 
6097 version(with_addon_servers_connections)
6098 class BasicDataServerConnection : BasicDataServer {
6099 	mixin ImplementRpcClientInterface!(BasicDataServer, "/tmp/arsd_session_server");
6100 }
6101 
6102 version(with_addon_servers)
6103 final class BasicDataServerImplementation : BasicDataServer, EventIoServer {
6104 
6105 	void createSession(string sessionId, int lifetime) {
6106 		sessions[sessionId.idup] = Session(lifetime);
6107 	}
6108 	void destroySession(string sessionId) {
6109 		sessions.remove(sessionId);
6110 	}
6111 	void renewSession(string sessionId, int lifetime) {
6112 		sessions[sessionId].lifetime = lifetime;
6113 	}
6114 	void renameSession(string oldSessionId, string newSessionId) {
6115 		sessions[newSessionId.idup] = sessions[oldSessionId];
6116 		sessions.remove(oldSessionId);
6117 	}
6118 	void setSessionData(string sessionId, string dataKey, string dataValue) {
6119 		if(sessionId !in sessions)
6120 			createSession(sessionId, 3600); // FIXME?
6121 		sessions[sessionId].values[dataKey.idup] = dataValue.idup;
6122 	}
6123 	string getSessionData(string sessionId, string dataKey) {
6124 		if(auto session = sessionId in sessions) {
6125 			if(auto data = dataKey in (*session).values)
6126 				return *data;
6127 			else
6128 				return null; // no such data
6129 
6130 		} else {
6131 			return null; // no session
6132 		}
6133 	}
6134 
6135 
6136 	protected:
6137 
6138 	struct Session {
6139 		int lifetime;
6140 
6141 		string[string] values;
6142 	}
6143 
6144 	Session[string] sessions;
6145 
6146 	bool handleLocalConnectionData(IoOp* op, int receivedFd) {
6147 		auto data = op.usedBuffer;
6148 		dispatchRpcServer!BasicDataServer(this, data, op.fd);
6149 		return false;
6150 	}
6151 
6152 	void handleLocalConnectionClose(IoOp* op) {} // doesn't really matter, this is a fairly stateless go
6153 	void handleLocalConnectionComplete(IoOp* op) {} // again, irrelevant
6154 	void wait_timeout() {}
6155 	void fileClosed(int fd) {} // stateless so irrelevant
6156 	void epoll_fd(int fd) {}
6157 }
6158 
6159 /++
6160 	See [schedule] to make one of these. You then call one of the methods here to set it up:
6161 
6162 	---
6163 		schedule!fn(args).at(DateTime(2019, 8, 7, 12, 00, 00)); // run the function at August 7, 2019, 12 noon UTC
6164 		schedule!fn(args).delay(6.seconds); // run it after waiting 6 seconds
6165 		schedule!fn(args).asap(); // run it in the background as soon as the event loop gets around to it
6166 	---
6167 +/
6168 version(with_addon_servers_connections)
6169 struct ScheduledJobHelper {
6170 	private string func;
6171 	private string[] args;
6172 	private bool consumed;
6173 
6174 	private this(string func, string[] args) {
6175 		this.func = func;
6176 		this.args = args;
6177 	}
6178 
6179 	~this() {
6180 		assert(consumed);
6181 	}
6182 
6183 	/++
6184 		Schedules the job to be run at the given time.
6185 	+/
6186 	void at(DateTime when, immutable TimeZone timezone = UTC()) {
6187 		consumed = true;
6188 
6189 		auto conn = ScheduledJobServerConnection.connection;
6190 		import std.file;
6191 		auto st = SysTime(when, timezone);
6192 		auto jobId = conn.scheduleJob(1, cast(int) st.toUnixTime(), thisExePath, func, args);
6193 	}
6194 
6195 	/++
6196 		Schedules the job to run at least after the specified delay.
6197 	+/
6198 	void delay(Duration delay) {
6199 		consumed = true;
6200 
6201 		auto conn = ScheduledJobServerConnection.connection;
6202 		import std.file;
6203 		auto jobId = conn.scheduleJob(0, cast(int) delay.total!"seconds", thisExePath, func, args);
6204 	}
6205 
6206 	/++
6207 		Runs the job in the background ASAP.
6208 
6209 		$(NOTE It may run in a background thread. Don't segfault!)
6210 	+/
6211 	void asap() {
6212 		consumed = true;
6213 
6214 		auto conn = ScheduledJobServerConnection.connection;
6215 		import std.file;
6216 		auto jobId = conn.scheduleJob(0, 1, thisExePath, func, args);
6217 	}
6218 
6219 	/+
6220 	/++
6221 		Schedules the job to recur on the given pattern.
6222 	+/
6223 	void recur(string spec) {
6224 
6225 	}
6226 	+/
6227 }
6228 
6229 /++
6230 	First step to schedule a job on the scheduled job server.
6231 
6232 	The scheduled job needs to be a top-level function that doesn't read any
6233 	variables from outside its arguments because it may be run in a new process,
6234 	without any context existing later.
6235 
6236 	You MUST set details on the returned object to actually do anything!
6237 +/
6238 template schedule(alias fn, T...) if(is(typeof(fn) == function)) {
6239 	///
6240 	ScheduledJobHelper schedule(T args) {
6241 		// this isn't meant to ever be called, but instead just to
6242 		// get the compiler to type check the arguments passed for us
6243 		auto sample = delegate() {
6244 			fn(args);
6245 		};
6246 		string[] sargs;
6247 		foreach(arg; args)
6248 			sargs ~= to!string(arg);
6249 		return ScheduledJobHelper(fn.mangleof, sargs);
6250 	}
6251 
6252 	shared static this() {
6253 		scheduledJobHandlers[fn.mangleof] = delegate(string[] sargs) {
6254 			import std.traits;
6255 			Parameters!fn args;
6256 			foreach(idx, ref arg; args)
6257 				arg = to!(typeof(arg))(sargs[idx]);
6258 			fn(args);
6259 		};
6260 	}
6261 }
6262 
6263 ///
6264 interface ScheduledJobServer {
6265 	/// Use the [schedule] function for a higher-level interface.
6266 	int scheduleJob(int whenIs, int when, string executable, string func, string[] args);
6267 	///
6268 	void cancelJob(int jobId);
6269 }
6270 
6271 version(with_addon_servers_connections)
6272 class ScheduledJobServerConnection : ScheduledJobServer {
6273 	mixin ImplementRpcClientInterface!(ScheduledJobServer, "/tmp/arsd_scheduled_job_server");
6274 }
6275 
6276 version(with_addon_servers)
6277 final class ScheduledJobServerImplementation : ScheduledJobServer, EventIoServer {
6278 	// FIXME: we need to handle SIGCHLD in this somehow
6279 	// whenIs is 0 for relative, 1 for absolute
6280 	protected int scheduleJob(int whenIs, int when, string executable, string func, string[] args) {
6281 		auto nj = nextJobId;
6282 		nextJobId++;
6283 
6284 		version(linux) {
6285 			import core.sys.linux.timerfd;
6286 			import core.sys.linux.epoll;
6287 			import core.sys.posix.unistd;
6288 
6289 
6290 			auto fd = timerfd_create(CLOCK_REALTIME, TFD_NONBLOCK | TFD_CLOEXEC);
6291 			if(fd == -1)
6292 				throw new Exception("fd timer create failed");
6293 
6294 			auto job = Job(executable, func, args, fd, nj);
6295 
6296 			itimerspec value;
6297 			value.it_value.tv_sec = when;
6298 			value.it_value.tv_nsec = 0;
6299 
6300 			value.it_interval.tv_sec = 0;
6301 			value.it_interval.tv_nsec = 0;
6302 
6303 			if(timerfd_settime(fd, whenIs == 1 ? TFD_TIMER_ABSTIME : 0, &value, null) == -1)
6304 				throw new Exception("couldn't set fd timer");
6305 
6306 			auto op = allocateIoOp(fd, IoOp.Read, 16, (IoOp* op, int fd) {
6307 				jobs.remove(nj);
6308 
6309 				spawnProcess([job.executable, "--timed-job", job.func] ~ args);
6310 
6311 				return true;
6312 			});
6313 			scope(failure)
6314 				freeIoOp(op);
6315 
6316 			epoll_event ev;
6317 			ev.events = EPOLLIN | EPOLLET;
6318 			ev.data.ptr = op;
6319 			if(epoll_ctl(epoll_fd, EPOLL_CTL_ADD, fd, &ev) == -1)
6320 				throw new Exception("epoll_ctl " ~ to!string(errno));
6321 
6322 			jobs[nj] = job;
6323 			return nj;
6324 		} else assert(0);
6325 	}
6326 
6327 	protected void cancelJob(int jobId) {
6328 		version(linux) {
6329 			auto job = jobId in jobs;
6330 			if(job is null)
6331 				return;
6332 
6333 			version(linux) {
6334 				import core.sys.linux.timerfd;
6335 				import core.sys.linux.epoll;
6336 				import core.sys.posix.unistd;
6337 				epoll_ctl(epoll_fd, EPOLL_CTL_DEL, job.timerfd, null);
6338 				close(job.timerfd);
6339 			}
6340 		}
6341 		jobs.remove(jobId);
6342 	}
6343 
6344 	int nextJobId = 1;
6345 	static struct Job {
6346 		string executable;
6347 		string func;
6348 		string[] args;
6349 		int timerfd;
6350 		int id;
6351 	}
6352 	Job[int] jobs;
6353 
6354 
6355 	// event io server methods below
6356 
6357 	bool handleLocalConnectionData(IoOp* op, int receivedFd) {
6358 		auto data = op.usedBuffer;
6359 		dispatchRpcServer!ScheduledJobServer(this, data, op.fd);
6360 		return false;
6361 	}
6362 
6363 	void handleLocalConnectionClose(IoOp* op) {} // doesn't really matter, this is a fairly stateless go
6364 	void handleLocalConnectionComplete(IoOp* op) {} // again, irrelevant
6365 	void wait_timeout() {}
6366 	void fileClosed(int fd) {} // stateless so irrelevant
6367 
6368 	int epoll_fd_;
6369 	void epoll_fd(int fd) {this.epoll_fd_ = fd; }
6370 	int epoll_fd() { return epoll_fd_; }
6371 }
6372 
6373 ///
6374 version(with_addon_servers_connections)
6375 interface EventSourceServer {
6376 	/++
6377 		sends this cgi request to the event server so it will be fed events. You should not do anything else with the cgi object after this.
6378 
6379 		$(WARNING This API is extremely unstable. I might change it or remove it without notice.)
6380 
6381 		See_Also:
6382 			[sendEvent]
6383 	+/
6384 	public static void adoptConnection(Cgi cgi, in char[] eventUrl) {
6385 		/*
6386 			If lastEventId is missing or empty, you just get new events as they come.
6387 
6388 			If it is set from something else, it sends all since then (that are still alive)
6389 			down the pipe immediately.
6390 
6391 			The reason it can come from the header is that's what the standard defines for
6392 			browser reconnects. The reason it can come from a query string is just convenience
6393 			in catching up in a user-defined manner.
6394 
6395 			The reason the header overrides the query string is if the browser tries to reconnect,
6396 			it will send the header AND the query (it reconnects to the same url), so we just
6397 			want to do the restart thing.
6398 
6399 			Note that if you ask for "0" as the lastEventId, it will get ALL still living events.
6400 		*/
6401 		string lastEventId = cgi.lastEventId;
6402 		if(lastEventId.length == 0 && "lastEventId" in cgi.get)
6403 			lastEventId = cgi.get["lastEventId"];
6404 
6405 		cgi.setResponseContentType("text/event-stream");
6406 		cgi.write(":\n", false); // to initialize the chunking and send headers before keeping the fd for later
6407 		cgi.flush();
6408 
6409 		cgi.closed = true;
6410 		auto s = openLocalServerConnection("/tmp/arsd_cgi_event_server");
6411 		scope(exit)
6412 			closeLocalServerConnection(s);
6413 
6414 		version(fastcgi)
6415 			throw new Exception("sending fcgi connections not supported");
6416 		else {
6417 			auto fd = cgi.getOutputFileHandle();
6418 			if(isInvalidHandle(fd))
6419 				throw new Exception("bad fd from cgi!");
6420 
6421 			EventSourceServerImplementation.SendableEventConnection sec;
6422 			sec.populate(cgi.responseChunked, eventUrl, lastEventId);
6423 
6424 			version(Posix) {
6425 				auto res = write_fd(s, cast(void*) &sec, sec.sizeof, fd);
6426 				assert(res == sec.sizeof);
6427 			} else version(Windows) {
6428 				// FIXME
6429 			}
6430 		}
6431 	}
6432 
6433 	/++
6434 		Sends an event to the event server, starting it if necessary. The event server will distribute it to any listening clients, and store it for `lifetime` seconds for any later listening clients to catch up later.
6435 
6436 		$(WARNING This API is extremely unstable. I might change it or remove it without notice.)
6437 
6438 		Params:
6439 			url = A string identifying this event "bucket". Listening clients must also connect to this same string. I called it `url` because I envision it being just passed as the url of the request.
6440 			event = the event type string, which is used in the Javascript addEventListener API on EventSource
6441 			data = the event data. Available in JS as `event.data`.
6442 			lifetime = the amount of time to keep this event for replaying on the event server.
6443 
6444 		See_Also:
6445 			[sendEventToEventServer]
6446 	+/
6447 	public static void sendEvent(string url, string event, string data, int lifetime) {
6448 		auto s = openLocalServerConnection("/tmp/arsd_cgi_event_server");
6449 		scope(exit)
6450 			closeLocalServerConnection(s);
6451 
6452 		EventSourceServerImplementation.SendableEvent sev;
6453 		sev.populate(url, event, data, lifetime);
6454 
6455 		version(Posix) {
6456 			auto ret = send(s, &sev, sev.sizeof, 0);
6457 			assert(ret == sev.sizeof);
6458 		} else version(Windows) {
6459 			// FIXME
6460 		}
6461 	}
6462 
6463 	/++
6464 		Messages sent to `url` will also be sent to anyone listening on `forwardUrl`.
6465 
6466 		See_Also: [disconnect]
6467 	+/
6468 	void connect(string url, string forwardUrl);
6469 
6470 	/++
6471 		Disconnects `forwardUrl` from `url`
6472 
6473 		See_Also: [connect]
6474 	+/
6475 	void disconnect(string url, string forwardUrl);
6476 }
6477 
6478 ///
6479 version(with_addon_servers)
6480 final class EventSourceServerImplementation : EventSourceServer, EventIoServer {
6481 
6482 	protected:
6483 
6484 	void connect(string url, string forwardUrl) {
6485 		pipes[url] ~= forwardUrl;
6486 	}
6487 	void disconnect(string url, string forwardUrl) {
6488 		auto t = url in pipes;
6489 		if(t is null)
6490 			return;
6491 		foreach(idx, n; (*t))
6492 			if(n == forwardUrl) {
6493 				(*t)[idx] = (*t)[$-1];
6494 				(*t) = (*t)[0 .. $-1];
6495 				break;
6496 			}
6497 	}
6498 
6499 	bool handleLocalConnectionData(IoOp* op, int receivedFd) {
6500 		if(receivedFd != -1) {
6501 			//writeln("GOT FD ", receivedFd, " -- ", op.usedBuffer);
6502 
6503 			//core.sys.posix.unistd.write(receivedFd, "hello".ptr, 5);
6504 
6505 			SendableEventConnection* got = cast(SendableEventConnection*) op.usedBuffer.ptr;
6506 
6507 			auto url = got.url.idup;
6508 			eventConnectionsByUrl[url] ~= EventConnection(receivedFd, got.responseChunked > 0 ? true : false);
6509 
6510 			// FIXME: catch up on past messages here
6511 		} else {
6512 			auto data = op.usedBuffer;
6513 			auto event = cast(SendableEvent*) data.ptr;
6514 
6515 			if(event.magic == 0xdeadbeef) {
6516 				handleInputEvent(event);
6517 
6518 				if(event.url in pipes)
6519 				foreach(pipe; pipes[event.url]) {
6520 					event.url = pipe;
6521 					handleInputEvent(event);
6522 				}
6523 			} else {
6524 				dispatchRpcServer!EventSourceServer(this, data, op.fd);
6525 			}
6526 		}
6527 		return false;
6528 	}
6529 	void handleLocalConnectionClose(IoOp* op) {}
6530 	void handleLocalConnectionComplete(IoOp* op) {}
6531 
6532 	void wait_timeout() {
6533 		// just keeping alive
6534 		foreach(url, connections; eventConnectionsByUrl)
6535 		foreach(connection; connections)
6536 			if(connection.needsChunking)
6537 				nonBlockingWrite(this, connection.fd, "2\r\n:\n");
6538 			else
6539 				nonBlockingWrite(this, connection.fd, ":\n");
6540 	}
6541 
6542 	void fileClosed(int fd) {
6543 		outer: foreach(url, ref connections; eventConnectionsByUrl) {
6544 			foreach(idx, conn; connections) {
6545 				if(fd == conn.fd) {
6546 					connections[idx] = connections[$-1];
6547 					connections = connections[0 .. $ - 1];
6548 					continue outer;
6549 				}
6550 			}
6551 		}
6552 	}
6553 
6554 	void epoll_fd(int fd) {}
6555 
6556 
6557 	private:
6558 
6559 
6560 	struct SendableEventConnection {
6561 		ubyte responseChunked;
6562 
6563 		int urlLength;
6564 		char[256] urlBuffer = 0;
6565 
6566 		int lastEventIdLength;
6567 		char[32] lastEventIdBuffer = 0;
6568 
6569 		char[] url() return {
6570 			return urlBuffer[0 .. urlLength];
6571 		}
6572 		void url(in char[] u) {
6573 			urlBuffer[0 .. u.length] = u[];
6574 			urlLength = cast(int) u.length;
6575 		}
6576 		char[] lastEventId() return {
6577 			return lastEventIdBuffer[0 .. lastEventIdLength];
6578 		}
6579 		void populate(bool responseChunked, in char[] url, in char[] lastEventId)
6580 		in {
6581 			assert(url.length < this.urlBuffer.length);
6582 			assert(lastEventId.length < this.lastEventIdBuffer.length);
6583 		}
6584 		do {
6585 			this.responseChunked = responseChunked ? 1 : 0;
6586 			this.urlLength = cast(int) url.length;
6587 			this.lastEventIdLength = cast(int) lastEventId.length;
6588 
6589 			this.urlBuffer[0 .. url.length] = url[];
6590 			this.lastEventIdBuffer[0 .. lastEventId.length] = lastEventId[];
6591 		}
6592 	}
6593 
6594 	struct SendableEvent {
6595 		int magic = 0xdeadbeef;
6596 		int urlLength;
6597 		char[256] urlBuffer = 0;
6598 		int typeLength;
6599 		char[32] typeBuffer = 0;
6600 		int messageLength;
6601 		char[2048] messageBuffer = 0;
6602 		int _lifetime;
6603 
6604 		char[] message() return {
6605 			return messageBuffer[0 .. messageLength];
6606 		}
6607 		char[] type() return {
6608 			return typeBuffer[0 .. typeLength];
6609 		}
6610 		char[] url() return {
6611 			return urlBuffer[0 .. urlLength];
6612 		}
6613 		void url(in char[] u) {
6614 			urlBuffer[0 .. u.length] = u[];
6615 			urlLength = cast(int) u.length;
6616 		}
6617 		int lifetime() {
6618 			return _lifetime;
6619 		}
6620 
6621 		///
6622 		void populate(string url, string type, string message, int lifetime)
6623 		in {
6624 			assert(url.length < this.urlBuffer.length);
6625 			assert(type.length < this.typeBuffer.length);
6626 			assert(message.length < this.messageBuffer.length);
6627 		}
6628 		do {
6629 			this.urlLength = cast(int) url.length;
6630 			this.typeLength = cast(int) type.length;
6631 			this.messageLength = cast(int) message.length;
6632 			this._lifetime = lifetime;
6633 
6634 			this.urlBuffer[0 .. url.length] = url[];
6635 			this.typeBuffer[0 .. type.length] = type[];
6636 			this.messageBuffer[0 .. message.length] = message[];
6637 		}
6638 	}
6639 
6640 	struct EventConnection {
6641 		int fd;
6642 		bool needsChunking;
6643 	}
6644 
6645 	private EventConnection[][string] eventConnectionsByUrl;
6646 	private string[][string] pipes;
6647 
6648 	private void handleInputEvent(scope SendableEvent* event) {
6649 		static int eventId;
6650 
6651 		static struct StoredEvent {
6652 			int id;
6653 			string type;
6654 			string message;
6655 			int lifetimeRemaining;
6656 		}
6657 
6658 		StoredEvent[][string] byUrl;
6659 
6660 		int thisId = ++eventId;
6661 
6662 		if(event.lifetime)
6663 			byUrl[event.url.idup] ~= StoredEvent(thisId, event.type.idup, event.message.idup, event.lifetime);
6664 
6665 		auto connectionsPtr = event.url in eventConnectionsByUrl;
6666 		EventConnection[] connections;
6667 		if(connectionsPtr is null)
6668 			return;
6669 		else
6670 			connections = *connectionsPtr;
6671 
6672 		char[4096] buffer;
6673 		char[] formattedMessage;
6674 
6675 		void append(const char[] a) {
6676 			// the 6's here are to leave room for a HTTP chunk header, if it proves necessary
6677 			buffer[6 + formattedMessage.length .. 6 + formattedMessage.length + a.length] = a[];
6678 			formattedMessage = buffer[6 .. 6 + formattedMessage.length + a.length];
6679 		}
6680 
6681 		import std.algorithm.iteration;
6682 
6683 		if(connections.length) {
6684 			append("id: ");
6685 			append(to!string(thisId));
6686 			append("\n");
6687 
6688 			append("event: ");
6689 			append(event.type);
6690 			append("\n");
6691 
6692 			foreach(line; event.message.splitter("\n")) {
6693 				append("data: ");
6694 				append(line);
6695 				append("\n");
6696 			}
6697 
6698 			append("\n");
6699 		}
6700 
6701 		// chunk it for HTTP!
6702 		auto len = toHex(formattedMessage.length);
6703 		buffer[4 .. 6] = "\r\n"[];
6704 		buffer[4 - len.length .. 4] = len[];
6705 
6706 		auto chunkedMessage = buffer[4 - len.length .. 6 + formattedMessage.length];
6707 		// done
6708 
6709 		// FIXME: send back requests when needed
6710 		// FIXME: send a single ":\n" every 15 seconds to keep alive
6711 
6712 		foreach(connection; connections) {
6713 			if(connection.needsChunking)
6714 				nonBlockingWrite(this, connection.fd, chunkedMessage);
6715 			else
6716 				nonBlockingWrite(this, connection.fd, formattedMessage);
6717 		}
6718 	}
6719 }
6720 
6721 void runAddonServer(EIS)(string localListenerName, EIS eis) if(is(EIS : EventIoServer)) {
6722 	version(Posix) {
6723 
6724 		import core.sys.posix.unistd;
6725 		import core.sys.posix.fcntl;
6726 		import core.sys.posix.sys.un;
6727 
6728 		import core.sys.posix.signal;
6729 		signal(SIGPIPE, SIG_IGN);
6730 
6731 		int sock = socket(AF_UNIX, SOCK_STREAM, 0);
6732 		if(sock == -1)
6733 			throw new Exception("socket " ~ to!string(errno));
6734 
6735 		scope(failure)
6736 			close(sock);
6737 
6738 		// add-on server processes are assumed to be local, and thus will
6739 		// use unix domain sockets. Besides, I want to pass sockets to them,
6740 		// so it basically must be local (except for the session server, but meh).
6741 		sockaddr_un addr;
6742 		addr.sun_family = AF_UNIX;
6743 		version(linux) {
6744 			// on linux, we will use the abstract namespace
6745 			addr.sun_path[0] = 0;
6746 			addr.sun_path[1 .. localListenerName.length + 1] = cast(typeof(addr.sun_path[])) localListenerName[];
6747 		} else {
6748 			// but otherwise, just use a file cuz we must.
6749 			addr.sun_path[0 .. localListenerName.length] = cast(typeof(addr.sun_path[])) localListenerName[];
6750 		}
6751 
6752 		if(bind(sock, cast(sockaddr*) &addr, addr.sizeof) == -1)
6753 			throw new Exception("bind " ~ to!string(errno));
6754 
6755 		if(listen(sock, 128) == -1)
6756 			throw new Exception("listen " ~ to!string(errno));
6757 
6758 		makeNonBlocking(sock);
6759 
6760 		version(linux) {
6761 			import core.sys.linux.epoll;
6762 			auto epoll_fd = epoll_create1(EPOLL_CLOEXEC);
6763 			if(epoll_fd == -1)
6764 				throw new Exception("epoll_create1 " ~ to!string(errno));
6765 			scope(failure)
6766 				close(epoll_fd);
6767 		} else {
6768 			import core.sys.posix.poll;
6769 		}
6770 
6771 		version(linux)
6772 		eis.epoll_fd = epoll_fd;
6773 
6774 		auto acceptOp = allocateIoOp(sock, IoOp.Read, 0, null);
6775 		scope(exit)
6776 			freeIoOp(acceptOp);
6777 
6778 		version(linux) {
6779 			epoll_event ev;
6780 			ev.events = EPOLLIN | EPOLLET;
6781 			ev.data.ptr = acceptOp;
6782 			if(epoll_ctl(epoll_fd, EPOLL_CTL_ADD, sock, &ev) == -1)
6783 				throw new Exception("epoll_ctl " ~ to!string(errno));
6784 
6785 			epoll_event[64] events;
6786 		} else {
6787 			pollfd[] pollfds;
6788 			IoOp*[int] ioops;
6789 			pollfds ~= pollfd(sock, POLLIN);
6790 			ioops[sock] = acceptOp;
6791 		}
6792 
6793 		while(true) {
6794 
6795 			// FIXME: it should actually do a timerfd that runs on any thing that hasn't been run recently
6796 
6797 			int timeout_milliseconds = 15000; //  -1; // infinite
6798 			//writeln("waiting for ", name);
6799 
6800 			version(linux) {
6801 				auto nfds = epoll_wait(epoll_fd, events.ptr, events.length, timeout_milliseconds);
6802 				if(nfds == -1) {
6803 					if(errno == EINTR)
6804 						continue;
6805 					throw new Exception("epoll_wait " ~ to!string(errno));
6806 				}
6807 			} else {
6808 				int nfds = poll(pollfds.ptr, cast(int) pollfds.length, timeout_milliseconds);
6809 				size_t lastIdx = 0;
6810 			}
6811 
6812 			if(nfds == 0) {
6813 				eis.wait_timeout();
6814 			}
6815 
6816 			foreach(idx; 0 .. nfds) {
6817 				version(linux) {
6818 					auto flags = events[idx].events;
6819 					auto ioop = cast(IoOp*) events[idx].data.ptr;
6820 				} else {
6821 					IoOp* ioop;
6822 					foreach(tidx, thing; pollfds[lastIdx .. $]) {
6823 						if(thing.revents) {
6824 							ioop = ioops[thing.fd];
6825 							lastIdx += tidx + 1;
6826 							break;
6827 						}
6828 					}
6829 				}
6830 
6831 				//writeln(flags, " ", ioop.fd);
6832 
6833 				void newConnection() {
6834 					// on edge triggering, it is important that we get it all
6835 					while(true) {
6836 						auto size = cast(uint) addr.sizeof;
6837 						auto ns = accept(sock, cast(sockaddr*) &addr, &size);
6838 						if(ns == -1) {
6839 							if(errno == EAGAIN || errno == EWOULDBLOCK) {
6840 								// all done, got it all
6841 								break;
6842 							}
6843 							throw new Exception("accept " ~ to!string(errno));
6844 						}
6845 
6846 						makeNonBlocking(ns);
6847 						auto niop = allocateIoOp(ns, IoOp.ReadSocketHandle, 4096, &eis.handleLocalConnectionData);
6848 						niop.closeHandler = &eis.handleLocalConnectionClose;
6849 						niop.completeHandler = &eis.handleLocalConnectionComplete;
6850 						scope(failure) freeIoOp(niop);
6851 
6852 						version(linux) {
6853 							epoll_event nev;
6854 							nev.events = EPOLLIN | EPOLLET;
6855 							nev.data.ptr = niop;
6856 							if(epoll_ctl(epoll_fd, EPOLL_CTL_ADD, ns, &nev) == -1)
6857 								throw new Exception("epoll_ctl " ~ to!string(errno));
6858 						} else {
6859 							bool found = false;
6860 							foreach(ref pfd; pollfds) {
6861 								if(pfd.fd < 0) {
6862 									pfd.fd = ns;
6863 									found = true;
6864 								}
6865 							}
6866 							if(!found)
6867 								pollfds ~= pollfd(ns, POLLIN);
6868 							ioops[ns] = niop;
6869 						}
6870 					}
6871 				}
6872 
6873 				bool newConnectionCondition() {
6874 					version(linux)
6875 						return ioop.fd == sock && (flags & EPOLLIN);
6876 					else
6877 						return pollfds[idx].fd == sock && (pollfds[idx].revents & POLLIN);
6878 				}
6879 
6880 				if(newConnectionCondition()) {
6881 					newConnection();
6882 				} else if(ioop.operation == IoOp.ReadSocketHandle) {
6883 					while(true) {
6884 						int in_fd;
6885 						auto got = read_fd(ioop.fd, ioop.allocatedBuffer.ptr, ioop.allocatedBuffer.length, &in_fd);
6886 						if(got == -1) {
6887 							if(errno == EAGAIN || errno == EWOULDBLOCK) {
6888 								// all done, got it all
6889 								if(ioop.completeHandler)
6890 									ioop.completeHandler(ioop);
6891 								break;
6892 							}
6893 							throw new Exception("recv " ~ to!string(errno));
6894 						}
6895 
6896 						if(got == 0) {
6897 							if(ioop.closeHandler) {
6898 								ioop.closeHandler(ioop);
6899 								version(linux) {} // nothing needed
6900 								else {
6901 									foreach(ref pfd; pollfds) {
6902 										if(pfd.fd == ioop.fd)
6903 											pfd.fd = -1;
6904 									}
6905 								}
6906 							}
6907 							close(ioop.fd);
6908 							freeIoOp(ioop);
6909 							break;
6910 						}
6911 
6912 						ioop.bufferLengthUsed = cast(int) got;
6913 						ioop.handler(ioop, in_fd);
6914 					}
6915 				} else if(ioop.operation == IoOp.Read) {
6916 					while(true) {
6917 						auto got = read(ioop.fd, ioop.allocatedBuffer.ptr, ioop.allocatedBuffer.length);
6918 						if(got == -1) {
6919 							if(errno == EAGAIN || errno == EWOULDBLOCK) {
6920 								// all done, got it all
6921 								if(ioop.completeHandler)
6922 									ioop.completeHandler(ioop);
6923 								break;
6924 							}
6925 							throw new Exception("recv " ~ to!string(ioop.fd) ~ " errno " ~ to!string(errno));
6926 						}
6927 
6928 						if(got == 0) {
6929 							if(ioop.closeHandler)
6930 								ioop.closeHandler(ioop);
6931 							close(ioop.fd);
6932 							freeIoOp(ioop);
6933 							break;
6934 						}
6935 
6936 						ioop.bufferLengthUsed = cast(int) got;
6937 						if(ioop.handler(ioop, ioop.fd)) {
6938 							close(ioop.fd);
6939 							freeIoOp(ioop);
6940 							break;
6941 						}
6942 					}
6943 				}
6944 
6945 				// EPOLLHUP?
6946 			}
6947 		}
6948 	} else version(Windows) {
6949 
6950 		// set up a named pipe
6951 		// https://msdn.microsoft.com/en-us/library/windows/desktop/ms724251(v=vs.85).aspx
6952 		// https://docs.microsoft.com/en-us/windows/desktop/api/winsock2/nf-winsock2-wsaduplicatesocketw
6953 		// https://docs.microsoft.com/en-us/windows/desktop/api/winbase/nf-winbase-getnamedpipeserverprocessid
6954 
6955 	} else static assert(0);
6956 }
6957 
6958 
6959 version(with_sendfd)
6960 // copied from the web and ported from C
6961 // see https://stackoverflow.com/questions/2358684/can-i-share-a-file-descriptor-to-another-process-on-linux-or-are-they-local-to-t
6962 ssize_t write_fd(int fd, void *ptr, size_t nbytes, int sendfd) {
6963 	msghdr msg;
6964 	iovec[1] iov;
6965 
6966 	version(OSX) {
6967 		//msg.msg_accrights = cast(cattr_t) &sendfd;
6968 		//msg.msg_accrightslen = int.sizeof;
6969 	} else {
6970 		union ControlUnion {
6971 			cmsghdr cm;
6972 			char[CMSG_SPACE(int.sizeof)] control;
6973 		}
6974 
6975 		ControlUnion control_un;
6976 		cmsghdr* cmptr;
6977 
6978 		msg.msg_control = control_un.control.ptr;
6979 		msg.msg_controllen = control_un.control.length;
6980 
6981 		cmptr = CMSG_FIRSTHDR(&msg);
6982 		cmptr.cmsg_len = CMSG_LEN(int.sizeof);
6983 		cmptr.cmsg_level = SOL_SOCKET;
6984 		cmptr.cmsg_type = SCM_RIGHTS;
6985 		*(cast(int *) CMSG_DATA(cmptr)) = sendfd;
6986 	}
6987 
6988 	msg.msg_name = null;
6989 	msg.msg_namelen = 0;
6990 
6991 	iov[0].iov_base = ptr;
6992 	iov[0].iov_len = nbytes;
6993 	msg.msg_iov = iov.ptr;
6994 	msg.msg_iovlen = 1;
6995 
6996 	return sendmsg(fd, &msg, 0);
6997 }
6998 
6999 version(with_sendfd)
7000 // copied from the web and ported from C
7001 ssize_t read_fd(int fd, void *ptr, size_t nbytes, int *recvfd) {
7002 	msghdr msg;
7003 	iovec[1] iov;
7004 	ssize_t n;
7005 	int newfd;
7006 
7007 	version(OSX) {
7008 		//msg.msg_accrights = cast(cattr_t) recvfd;
7009 		//msg.msg_accrightslen = int.sizeof;
7010 	} else {
7011 		union ControlUnion {
7012 			cmsghdr cm;
7013 			char[CMSG_SPACE(int.sizeof)] control;
7014 		}
7015 		ControlUnion control_un;
7016 		cmsghdr* cmptr;
7017 
7018 		msg.msg_control = control_un.control.ptr;
7019 		msg.msg_controllen = control_un.control.length;
7020 	}
7021 
7022 	msg.msg_name = null;
7023 	msg.msg_namelen = 0;
7024 
7025 	iov[0].iov_base = ptr;
7026 	iov[0].iov_len = nbytes;
7027 	msg.msg_iov = iov.ptr;
7028 	msg.msg_iovlen = 1;
7029 
7030 	if ( (n = recvmsg(fd, &msg, 0)) <= 0)
7031 		return n;
7032 
7033 	version(OSX) {
7034 		//if(msg.msg_accrightslen != int.sizeof)
7035 			//*recvfd = -1;
7036 	} else {
7037 		if ( (cmptr = CMSG_FIRSTHDR(&msg)) != null &&
7038 				cmptr.cmsg_len == CMSG_LEN(int.sizeof)) {
7039 			if (cmptr.cmsg_level != SOL_SOCKET)
7040 				throw new Exception("control level != SOL_SOCKET");
7041 			if (cmptr.cmsg_type != SCM_RIGHTS)
7042 				throw new Exception("control type != SCM_RIGHTS");
7043 			*recvfd = *(cast(int *) CMSG_DATA(cmptr));
7044 		} else
7045 			*recvfd = -1;       /* descriptor was not passed */
7046 	}
7047 
7048 	return n;
7049 }
7050 /* end read_fd */
7051 
7052 
7053 /*
7054 	Event source stuff
7055 
7056 	The api is:
7057 
7058 	sendEvent(string url, string type, string data, int timeout = 60*10);
7059 
7060 	attachEventListener(string url, int fd, lastId)
7061 
7062 
7063 	It just sends to all attached listeners, and stores it until the timeout
7064 	for replaying via lastEventId.
7065 */
7066 
7067 /*
7068 	Session process stuff
7069 
7070 	it stores it all. the cgi object has a session object that can grab it
7071 
7072 	session may be done in the same process if possible, there is a version
7073 	switch to choose if you want to override.
7074 */
7075 
7076 struct DispatcherDefinition(alias dispatchHandler, DispatcherDetails = typeof(null)) {// if(is(typeof(dispatchHandler("str", Cgi.init, void) == bool))) { // bool delegate(string urlPrefix, Cgi cgi) dispatchHandler;
7077 	alias handler = dispatchHandler;
7078 	string urlPrefix;
7079 	bool rejectFurther;
7080 	immutable(DispatcherDetails) details;
7081 }
7082 
7083 private string urlify(string name) {
7084 	return beautify(name, '-', true);
7085 }
7086 
7087 private string beautify(string name, char space = ' ', bool allLowerCase = false) {
7088 	if(name == "id")
7089 		return allLowerCase ? name : "ID";
7090 
7091 	char[160] buffer;
7092 	int bufferIndex = 0;
7093 	bool shouldCap = true;
7094 	bool shouldSpace;
7095 	bool lastWasCap;
7096 	foreach(idx, char ch; name) {
7097 		if(bufferIndex == buffer.length) return name; // out of space, just give up, not that important
7098 
7099 		if((ch >= 'A' && ch <= 'Z') || ch == '_') {
7100 			if(lastWasCap) {
7101 				// two caps in a row, don't change. Prolly acronym.
7102 			} else {
7103 				if(idx)
7104 					shouldSpace = true; // new word, add space
7105 			}
7106 
7107 			lastWasCap = true;
7108 		} else {
7109 			lastWasCap = false;
7110 		}
7111 
7112 		if(shouldSpace) {
7113 			buffer[bufferIndex++] = space;
7114 			if(bufferIndex == buffer.length) return name; // out of space, just give up, not that important
7115 			shouldSpace = false;
7116 		}
7117 		if(shouldCap) {
7118 			if(ch >= 'a' && ch <= 'z')
7119 				ch -= 32;
7120 			shouldCap = false;
7121 		}
7122 		if(allLowerCase && ch >= 'A' && ch <= 'Z')
7123 			ch += 32;
7124 		buffer[bufferIndex++] = ch;
7125 	}
7126 	return buffer[0 .. bufferIndex].idup;
7127 }
7128 
7129 /*
7130 string urlFor(alias func)() {
7131 	return __traits(identifier, func);
7132 }
7133 */
7134 
7135 /++
7136 	UDA: The name displayed to the user in auto-generated HTML.
7137 
7138 	Default is `beautify(identifier)`.
7139 +/
7140 struct DisplayName {
7141 	string name;
7142 }
7143 
7144 /++
7145 	UDA: The name used in the URL or web parameter.
7146 
7147 	Default is `urlify(identifier)` for functions and `identifier` for parameters and data members.
7148 +/
7149 struct UrlName {
7150 	string name;
7151 }
7152 
7153 /++
7154 	UDA: default format to respond for this method
7155 +/
7156 struct DefaultFormat { string value; }
7157 
7158 class MissingArgumentException : Exception {
7159 	string functionName;
7160 	string argumentName;
7161 	string argumentType;
7162 
7163 	this(string functionName, string argumentName, string argumentType, string file = __FILE__, size_t line = __LINE__, Throwable next = null) {
7164 		this.functionName = functionName;
7165 		this.argumentName = argumentName;
7166 		this.argumentType = argumentType;
7167 
7168 		super("Missing Argument: " ~ this.argumentName, file, line, next);
7169 	}
7170 }
7171 
7172 /++
7173 	This can be attached to any constructor or function called from the cgi system.
7174 
7175 	If it is present, the function argument can NOT be set from web params, but instead
7176 	is set to the return value of the given `func`.
7177 
7178 	If `func` can take a parameter of type [Cgi], it will be passed the one representing
7179 	the current request. Otherwise, it must take zero arguments.
7180 
7181 	Any params in your function of type `Cgi` are automatically assumed to take the cgi object
7182 	for the connection. Any of type [Session] (with an argument) is	also assumed to come from
7183 	the cgi object.
7184 
7185 	const arguments are also supported.
7186 +/
7187 struct ifCalledFromWeb(alias func) {}
7188 
7189 // it only looks at query params for GET requests, the rest must be in the body for a function argument.
7190 auto callFromCgi(alias method, T)(T dg, Cgi cgi) {
7191 
7192 	// FIXME: any array of structs should also be settable or gettable from csv as well.
7193 
7194 	// FIXME: think more about checkboxes and bools.
7195 
7196 	import std.traits;
7197 
7198 	Parameters!method params;
7199 	alias idents = ParameterIdentifierTuple!method;
7200 	alias defaults = ParameterDefaults!method;
7201 
7202 	const(string)[] names;
7203 	const(string)[] values;
7204 
7205 	// first, check for missing arguments and initialize to defaults if necessary
7206 
7207 	static if(is(typeof(method) P == __parameters))
7208 	foreach(idx, param; P) {{
7209 		// see: mustNotBeSetFromWebParams
7210 		static if(is(param : Cgi)) {
7211 			static assert(!is(param == immutable));
7212 			cast() params[idx] = cgi;
7213 		} else static if(is(param == Session!D, D)) {
7214 			static assert(!is(param == immutable));
7215 			cast() params[idx] = cgi.getSessionObject!D();
7216 		} else {
7217 			bool populated;
7218 			foreach(uda; __traits(getAttributes, P[idx .. idx + 1])) {
7219 				static if(is(uda == ifCalledFromWeb!func, alias func)) {
7220 					static if(is(typeof(func(cgi))))
7221 						params[idx] = func(cgi);
7222 					else
7223 						params[idx] = func();
7224 
7225 					populated = true;
7226 				}
7227 			}
7228 
7229 			if(!populated) {
7230 				static if(__traits(compiles, { params[idx] = param.getAutomaticallyForCgi(cgi); } )) {
7231 					params[idx] = param.getAutomaticallyForCgi(cgi);
7232 					populated = true;
7233 				}
7234 			}
7235 
7236 			if(!populated) {
7237 				auto ident = idents[idx];
7238 				if(cgi.requestMethod == Cgi.RequestMethod.GET) {
7239 					if(ident !in cgi.get) {
7240 						static if(is(defaults[idx] == void)) {
7241 							static if(is(param == bool))
7242 								params[idx] = false;
7243 							else
7244 								throw new MissingArgumentException(__traits(identifier, method), ident, param.stringof);
7245 						} else
7246 							params[idx] = defaults[idx];
7247 					}
7248 				} else {
7249 					if(ident !in cgi.post) {
7250 						static if(is(defaults[idx] == void)) {
7251 							static if(is(param == bool))
7252 								params[idx] = false;
7253 							else
7254 								throw new MissingArgumentException(__traits(identifier, method), ident, param.stringof);
7255 						} else
7256 							params[idx] = defaults[idx];
7257 					}
7258 				}
7259 			}
7260 		}
7261 	}}
7262 
7263 	// second, parse the arguments in order to build up arrays, etc.
7264 
7265 	static bool setVariable(T)(string name, string paramName, T* what, string value) {
7266 		static if(is(T == struct)) {
7267 			if(name == paramName) {
7268 				*what = T.init;
7269 				return true;
7270 			} else {
7271 				// could be a child
7272 				if(name[paramName.length] == '.') {
7273 					paramName = name[paramName.length + 1 .. $];
7274 					name = paramName;
7275 					int p = 0;
7276 					foreach(ch; paramName) {
7277 						if(ch == '.' || ch == '[')
7278 							break;
7279 						p++;
7280 					}
7281 
7282 					// set the child member
7283 					switch(paramName) {
7284 						foreach(idx, memberName; __traits(allMembers, T))
7285 						static if(__traits(compiles, __traits(getMember, T, memberName).offsetof)) {
7286 							// data member!
7287 							case memberName:
7288 								return setVariable(name, paramName, &(__traits(getMember, *what, memberName)), value);
7289 						}
7290 						default:
7291 							// ok, not a member
7292 					}
7293 				}
7294 			}
7295 
7296 			return false;
7297 		} else static if(isSomeString!T || isIntegral!T || isFloatingPoint!T) {
7298 			*what = to!T(value);
7299 			return true;
7300 		} else static if(is(T == bool)) {
7301 			*what = value == "1" || value == "yes" || value == "t" || value == "true" || value == "on";
7302 			return true;
7303 		} else static if(is(T == K[], K)) {
7304 			K tmp;
7305 			if(name == paramName) {
7306 				// direct - set and append
7307 				if(setVariable(name, paramName, &tmp, value)) {
7308 					(*what) ~= tmp;
7309 					return true;
7310 				} else {
7311 					return false;
7312 				}
7313 			} else {
7314 				// child, append to last element
7315 				// FIXME: what about range violations???
7316 				auto ptr = &(*what)[(*what).length - 1];
7317 				return setVariable(name, paramName, ptr, value);
7318 
7319 			}
7320 		} else static if(is(T == V[K], K, V)) {
7321 			// assoc array, name[key] is valid
7322 			if(name == paramName) {
7323 				// no action necessary
7324 				return true;
7325 			} else if(name[paramName.length] == '[') {
7326 				int count = 1;
7327 				auto idx = paramName.length + 1;
7328 				while(idx < name.length && count > 0) {
7329 					if(name[idx] == '[')
7330 						count++;
7331 					else if(name[idx] == ']') {
7332 						count--;
7333 						if(count == 0) break;
7334 					}
7335 					idx++;
7336 				}
7337 				if(idx == name.length)
7338 					return false; // malformed
7339 
7340 				auto insideBrackets = name[paramName.length + 1 .. idx];
7341 				auto afterName = name[idx + 1 .. $];
7342 
7343 				auto k = to!K(insideBrackets);
7344 				V v;
7345 				if(auto ptr = k in *what)
7346 					v = *ptr;
7347 
7348 				name = name[0 .. paramName.length];
7349 				//writeln(name, afterName, " ", paramName);
7350 
7351 				auto ret = setVariable(name ~ afterName, paramName, &v, value);
7352 				if(ret) {
7353 					(*what)[k] = v;
7354 					return true;
7355 				}
7356 			}
7357 
7358 			return false;
7359 		} else {
7360 			static assert(0, "unsupported type for cgi call " ~ T.stringof);
7361 		}
7362 
7363 		//return false;
7364 	}
7365 
7366 	void setArgument(string name, string value) {
7367 		int p;
7368 		foreach(ch; name) {
7369 			if(ch == '.' || ch == '[')
7370 				break;
7371 			p++;
7372 		}
7373 
7374 		auto paramName = name[0 .. p];
7375 
7376 		sw: switch(paramName) {
7377 			static if(is(typeof(method) P == __parameters))
7378 			foreach(idx, param; P) {
7379 				static if(mustNotBeSetFromWebParams!(P[idx], __traits(getAttributes, P[idx .. idx + 1]))) {
7380 					// cannot be set from the outside
7381 				} else {
7382 					case idents[idx]:
7383 						static if(is(param == Cgi.UploadedFile)) {
7384 							params[idx] = cgi.files[name];
7385 						} else {
7386 							setVariable(name, paramName, &params[idx], value);
7387 						}
7388 					break sw;
7389 				}
7390 			}
7391 			default:
7392 				// ignore; not relevant argument
7393 		}
7394 	}
7395 
7396 	if(cgi.requestMethod == Cgi.RequestMethod.GET) {
7397 		names = cgi.allGetNamesInOrder;
7398 		values = cgi.allGetValuesInOrder;
7399 	} else {
7400 		names = cgi.allPostNamesInOrder;
7401 		values = cgi.allPostValuesInOrder;
7402 	}
7403 
7404 	foreach(idx, name; names) {
7405 		setArgument(name, values[idx]);
7406 	}
7407 
7408 	static if(is(ReturnType!method == void)) {
7409 		typeof(null) ret;
7410 		dg(params);
7411 	} else {
7412 		auto ret = dg(params);
7413 	}
7414 
7415 	// FIXME: format return values
7416 	// options are: json, html, csv.
7417 	// also may need to wrap in envelope format: none, html, or json.
7418 	return ret;
7419 }
7420 
7421 private bool mustNotBeSetFromWebParams(T, attrs...)() {
7422 	static if(is(T : const(Cgi))) {
7423 		return true;
7424 	} else static if(is(T : const(Session!D), D)) {
7425 		return true;
7426 	} else static if(__traits(compiles, T.getAutomaticallyForCgi(Cgi.init))) {
7427 		return true;
7428 	} else {
7429 		foreach(uda; attrs)
7430 			static if(is(uda == ifCalledFromWeb!func, alias func))
7431 				return true;
7432 		return false;
7433 	}
7434 }
7435 
7436 private bool hasIfCalledFromWeb(attrs...)() {
7437 	foreach(uda; attrs)
7438 		static if(is(uda == ifCalledFromWeb!func, alias func))
7439 			return true;
7440 	return false;
7441 }
7442 
7443 /+
7444 	Argument conversions: for the most part, it is to!Thing(string).
7445 
7446 	But arrays and structs are a bit different. Arrays come from the cgi array. Thus
7447 	they are passed
7448 
7449 	arr=foo&arr=bar <-- notice the same name.
7450 
7451 	Structs are first declared with an empty thing, then have their members set individually,
7452 	with dot notation. The members are not required, just the initial declaration.
7453 
7454 	struct Foo {
7455 		int a;
7456 		string b;
7457 	}
7458 	void test(Foo foo){}
7459 
7460 	foo&foo.a=5&foo.b=str <-- the first foo declares the arg, the others set the members
7461 
7462 	Arrays of structs use this declaration.
7463 
7464 	void test(Foo[] foo) {}
7465 
7466 	foo&foo.a=5&foo.b=bar&foo&foo.a=9
7467 
7468 	You can use a hidden input field in HTML forms to achieve this. The value of the naked name
7469 	declaration is ignored.
7470 
7471 	Mind that order matters! The declaration MUST come first in the string.
7472 
7473 	Arrays of struct members follow this rule recursively.
7474 
7475 	struct Foo {
7476 		int[] a;
7477 	}
7478 
7479 	foo&foo.a=1&foo.a=2&foo&foo.a=1
7480 
7481 
7482 	Associative arrays are formatted with brackets, after a declaration, like structs:
7483 
7484 	foo&foo[key]=value&foo[other_key]=value
7485 
7486 
7487 	Note: for maximum compatibility with outside code, keep your types simple. Some libraries
7488 	do not support the strict ordering requirements to work with these struct protocols.
7489 
7490 	FIXME: also perhaps accept application/json to better work with outside trash.
7491 
7492 
7493 	Return values are also auto-formatted according to user-requested type:
7494 		for json, it loops over and converts.
7495 		for html, basic types are strings. Arrays are <ol>. Structs are <dl>. Arrays of structs are tables!
7496 +/
7497 
7498 /++
7499 	A web presenter is responsible for rendering things to HTML to be usable
7500 	in a web browser.
7501 
7502 	They are passed as template arguments to the base classes of [WebObject]
7503 
7504 	Responsible for displaying stuff as HTML. You can put this into your own aggregate
7505 	and override it. Use forwarding and specialization to customize it.
7506 
7507 	When you inherit from it, pass your own class as the CRTP argument. This lets the base
7508 	class templates and your overridden templates work with each other.
7509 
7510 	---
7511 	class MyPresenter : WebPresenter!(MyPresenter) {
7512 		@Override
7513 		void presentSuccessfulReturnAsHtml(T : CustomType)(Cgi cgi, T ret) {
7514 			// present the CustomType
7515 		}
7516 		@Override
7517 		void presentSuccessfulReturnAsHtml(T)(Cgi cgi, T ret) {
7518 			// handle everything else via the super class, which will call
7519 			// back to your class when appropriate
7520 			super.presentSuccessfulReturnAsHtml(cgi, ret);
7521 		}
7522 	}
7523 	---
7524 
7525 +/
7526 class WebPresenter(CRTP) {
7527 
7528 	/// A UDA version of the built-in `override`, to be used for static template polymorphism
7529 	/// If you override a plain method, use `override`. If a template, use `@Override`.
7530 	enum Override;
7531 
7532 	string script() {
7533 		return `
7534 		`;
7535 	}
7536 
7537 	string style() {
7538 		return `
7539 			:root {
7540 				--mild-border: #ccc;
7541 				--middle-border: #999;
7542 				--accent-color: #e8e8e8;
7543 				--sidebar-color: #f2f2f2;
7544 			}
7545 		` ~ genericFormStyling() ~ genericSiteStyling();
7546 	}
7547 
7548 	string genericFormStyling() {
7549 		return
7550 q"css
7551 			table.automatic-data-display {
7552 				border-collapse: collapse;
7553 				border: solid 1px var(--mild-border);
7554 			}
7555 
7556 			table.automatic-data-display td {
7557 				vertical-align: top;
7558 				border: solid 1px var(--mild-border);
7559 				padding: 2px 4px;
7560 			}
7561 
7562 			table.automatic-data-display th {
7563 				border: solid 1px var(--mild-border);
7564 				border-bottom: solid 1px var(--middle-border);
7565 				padding: 2px 4px;
7566 			}
7567 
7568 			ol.automatic-data-display {
7569 				margin: 0px;
7570 				list-style-position: inside;
7571 				padding: 0px;
7572 			}
7573 
7574 			.automatic-form {
7575 				max-width: 600px;
7576 			}
7577 
7578 			.form-field {
7579 				margin: 0.5em;
7580 				padding-left: 0.5em;
7581 			}
7582 
7583 			.label-text {
7584 				display: block;
7585 				font-weight: bold;
7586 				margin-left: -0.5em;
7587 			}
7588 
7589 			.add-array-button {
7590 
7591 			}
7592 css";
7593 	}
7594 
7595 	string genericSiteStyling() {
7596 		return
7597 q"css
7598 			* { box-sizing: border-box; }
7599 			html, body { margin: 0px; }
7600 			body {
7601 				font-family: sans-serif;
7602 			}
7603 			header {
7604 				background: var(--accent-color);
7605 				height: 64px;
7606 			}
7607 			footer {
7608 				background: var(--accent-color);
7609 				height: 64px;
7610 			}
7611 			#site-container {
7612 				display: flex;
7613 			}
7614 			main {
7615 				flex: 1 1 auto;
7616 				order: 2;
7617 				min-height: calc(100vh - 64px - 64px);
7618 				padding: 4px;
7619 				padding-left: 1em;
7620 			}
7621 			#sidebar {
7622 				flex: 0 0 16em;
7623 				order: 1;
7624 				background: var(--sidebar-color);
7625 			}
7626 css";
7627 	}
7628 
7629 	import arsd.dom;
7630 	Element htmlContainer() {
7631 		auto document = new Document(q"html
7632 <!DOCTYPE html>
7633 <html>
7634 <head>
7635 	<title>D Application</title>
7636 	<link rel="stylesheet" href="style.css" />
7637 </head>
7638 <body>
7639 	<header></header>
7640 	<div id="site-container">
7641 		<main></main>
7642 		<div id="sidebar"></div>
7643 	</div>
7644 	<footer></footer>
7645 	<script src="script.js"></script>
7646 </body>
7647 </html>
7648 html", true, true);
7649 
7650 		return document.requireSelector("main");
7651 	}
7652 
7653 	/// Renders a response as an HTTP error
7654 	void renderBasicError(Cgi cgi, int httpErrorCode) {
7655 		cgi.setResponseStatus(getHttpCodeText(httpErrorCode));
7656 		auto c = htmlContainer();
7657 		c.innerText = getHttpCodeText(httpErrorCode);
7658 		cgi.setResponseContentType("text/html; charset=utf-8");
7659 		cgi.write(c.parentDocument.toString(), true);
7660 	}
7661 
7662 	/// typeof(null) (which is also used to represent functions returning `void`) do nothing
7663 	/// in the default presenter - allowing the function to have full low-level control over the
7664 	/// response.
7665 	void presentSuccessfulReturnAsHtml(T : typeof(null))(Cgi cgi, T ret) {
7666 		// nothing intentionally!
7667 	}
7668 
7669 	/// Redirections are forwarded to [Cgi.setResponseLocation]
7670 	void presentSuccessfulReturnAsHtml(T : Redirection)(Cgi cgi, T ret) {
7671 		cgi.setResponseLocation(ret.to, true, getHttpCodeText(ret.code));
7672 	}
7673 
7674 	/// Multiple responses deconstruct the algebraic type and forward to the appropriate handler at runtime
7675 	void presentSuccessfulReturnAsHtml(T : MultipleResponses!Types, Types...)(Cgi cgi, T ret) {
7676 		bool outputted = false;
7677 		foreach(index, type; Types) {
7678 			if(ret.contains == index) {
7679 				assert(!outputted);
7680 				outputted = true;
7681 				(cast(CRTP) this).presentSuccessfulReturnAsHtml(cgi, ret.payload[index]);
7682 			}
7683 		}
7684 		if(!outputted)
7685 			assert(0);
7686 	}
7687 
7688 	/// An instance of the [arsd.dom.FileResource] interface has its own content type; assume it is a download of some sort.
7689 	void presentSuccessfulReturnAsHtml(T : FileResource)(Cgi cgi, T ret) {
7690 		cgi.setCache(true); // not necessarily true but meh
7691 		cgi.setResponseContentType(ret.contentType);
7692 		cgi.write(ret.getData(), true);
7693 	}
7694 
7695 	/// And the default handler will call [formatReturnValueAsHtml] and place it inside the [htmlContainer].
7696 	void presentSuccessfulReturnAsHtml(T)(Cgi cgi, T ret) {
7697 		auto container = this.htmlContainer();
7698 		container.appendChild(formatReturnValueAsHtml(ret));
7699 		cgi.write(container.parentDocument.toString(), true);
7700 	}
7701 
7702 	/++
7703 		If you override this, you will need to cast the exception type `t` dynamically,
7704 		but can then use the template arguments here to refer back to the function.
7705 
7706 		`func` is an alias to the method itself, and `dg` is a callable delegate to the same
7707 		method on the live object. You could, in theory, change arguments and retry, but I
7708 		provide that information mostly with the expectation that you will use them to make
7709 		useful forms or richer error messages for the user.
7710 	+/
7711 	void presentExceptionAsHtml(alias func, T)(Cgi cgi, Throwable t, T dg) {
7712 		if(auto mae = cast(MissingArgumentException) t) {
7713 			auto container = this.htmlContainer();
7714 			container.appendChild(Element.make("p", "Argument `" ~ mae.argumentName ~ "` of type `" ~ mae.argumentType ~ "` is missing"));
7715 			container.appendChild(createAutomaticFormForFunction!(func)(dg));
7716 
7717 			cgi.write(container.parentDocument.toString(), true);
7718 		} else {
7719 			auto container = this.htmlContainer();
7720 
7721 			// import std.stdio; writeln(t.toString());
7722 
7723 			container.appendChild(exceptionToElement(t));
7724 
7725 			container.addChild("h4", "GET");
7726 			foreach(k, v; cgi.get) {
7727 				auto deets = container.addChild("details");
7728 				deets.addChild("summary", k);
7729 				deets.addChild("div", v);
7730 			}
7731 
7732 			container.addChild("h4", "POST");
7733 			foreach(k, v; cgi.post) {
7734 				auto deets = container.addChild("details");
7735 				deets.addChild("summary", k);
7736 				deets.addChild("div", v);
7737 			}
7738 
7739 
7740 			if(!cgi.outputtedResponseData)
7741 				cgi.setResponseStatus("500 Internal Server Error");
7742 			cgi.write(container.parentDocument.toString(), true);
7743 		}
7744 	}
7745 
7746 	Element exceptionToElement(Throwable t) {
7747 		auto div = Element.make("div");
7748 		div.addClass("exception-display");
7749 
7750 		div.addChild("p", t.msg);
7751 		div.addChild("p", "Inner code origin: " ~ typeid(t).name ~ "@" ~ t.file ~ ":" ~ to!string(t.line));
7752 
7753 		auto pre = div.addChild("pre");
7754 		string s;
7755 		s = t.toString();
7756 		Element currentBox;
7757 		bool on = false;
7758 		foreach(line; s.splitLines) {
7759 			if(!on && line.startsWith("-----"))
7760 				on = true;
7761 			if(!on) continue;
7762 			if(line.indexOf("arsd/") != -1) {
7763 				if(currentBox is null) {
7764 					currentBox = pre.addChild("details");
7765 					currentBox.addChild("summary", "Framework code");
7766 				}
7767 				currentBox.addChild("span", line ~ "\n");
7768 			} else {
7769 				pre.addChild("span", line ~ "\n");
7770 				currentBox = null;
7771 			}
7772 		}
7773 
7774 		return div;
7775 	}
7776 
7777 	/++
7778 		Returns an element for a particular type
7779 	+/
7780 	Element elementFor(T)(string displayName, string name) {
7781 		import std.traits;
7782 
7783 		auto div = Element.make("div");
7784 		div.addClass("form-field");
7785 
7786 		static if(is(T == struct)) {
7787 			if(displayName !is null)
7788 				div.addChild("span", displayName, "label-text");
7789 			auto fieldset = div.addChild("fieldset");
7790 			fieldset.addChild("legend", beautify(T.stringof)); // FIXME
7791 			fieldset.addChild("input", name);
7792 			foreach(idx, memberName; __traits(allMembers, T))
7793 			static if(__traits(compiles, __traits(getMember, T, memberName).offsetof)) {
7794 				fieldset.appendChild(elementFor!(typeof(__traits(getMember, T, memberName)))(beautify(memberName), name ~ "." ~ memberName));
7795 			}
7796 		} else static if(isSomeString!T || isIntegral!T || isFloatingPoint!T) {
7797 			Element lbl;
7798 			if(displayName !is null) {
7799 				lbl = div.addChild("label");
7800 				lbl.addChild("span", displayName, "label-text");
7801 				lbl.appendText(" ");
7802 			} else {
7803 				lbl = div;
7804 			}
7805 			auto i = lbl.addChild("input", name);
7806 			i.attrs.name = name;
7807 			static if(isSomeString!T)
7808 				i.attrs.type = "text";
7809 			else
7810 				i.attrs.type = "number";
7811 			i.attrs.value = to!string(T.init);
7812 		} else static if(is(T == bool)) {
7813 			Element lbl;
7814 			if(displayName !is null) {
7815 				lbl = div.addChild("label");
7816 				lbl.addChild("span", displayName, "label-text");
7817 				lbl.appendText(" ");
7818 			} else {
7819 				lbl = div;
7820 			}
7821 			auto i = lbl.addChild("input", name);
7822 			i.attrs.type = "checkbox";
7823 			i.attrs.value = "true";
7824 			i.attrs.name = name;
7825 		} else static if(is(T == Cgi.UploadedFile)) {
7826 			Element lbl;
7827 			if(displayName !is null) {
7828 				lbl = div.addChild("label");
7829 				lbl.addChild("span", displayName, "label-text");
7830 				lbl.appendText(" ");
7831 			} else {
7832 				lbl = div;
7833 			}
7834 			auto i = lbl.addChild("input", name);
7835 			i.attrs.name = name;
7836 			i.attrs.type = "file";
7837 		} else static if(is(T == K[], K)) {
7838 			auto templ = div.addChild("template");
7839 			templ.appendChild(elementFor!(K)(null, name));
7840 			if(displayName !is null)
7841 				div.addChild("span", displayName, "label-text");
7842 			auto btn = div.addChild("button");
7843 			btn.addClass("add-array-button");
7844 			btn.attrs.type = "button";
7845 			btn.innerText = "Add";
7846 			btn.attrs.onclick = q{
7847 				var a = document.importNode(this.parentNode.firstChild.content, true);
7848 				this.parentNode.insertBefore(a, this);
7849 			};
7850 		} else static if(is(T == V[K], K, V)) {
7851 			div.innerText = "assoc array not implemented for automatic form at this time";
7852 		} else {
7853 			static assert(0, "unsupported type for cgi call " ~ T.stringof);
7854 		}
7855 
7856 
7857 		return div;
7858 	}
7859 
7860 	/// creates a form for gathering the function's arguments
7861 	Form createAutomaticFormForFunction(alias method, T)(T dg) {
7862 
7863 		auto form = cast(Form) Element.make("form");
7864 
7865 		form.addClass("automatic-form");
7866 
7867 		form.addChild("h3", beautify(__traits(identifier, method)));
7868 
7869 		import std.traits;
7870 
7871 		//Parameters!method params;
7872 		//alias idents = ParameterIdentifierTuple!method;
7873 		//alias defaults = ParameterDefaults!method;
7874 
7875 		static if(is(typeof(method) P == __parameters))
7876 		foreach(idx, _; P) {{
7877 
7878 			alias param = P[idx .. idx + 1];
7879 
7880 			static if(!mustNotBeSetFromWebParams!(param[0], __traits(getAttributes, param))) {
7881 				string displayName = beautify(__traits(identifier, param));
7882 				foreach(attr; __traits(getAttributes, param))
7883 					static if(is(typeof(attr) == DisplayName))
7884 						displayName = attr.name;
7885 				auto i = form.appendChild(elementFor!(param)(displayName, __traits(identifier, param)));
7886 				if(i.querySelector("input[type=file]") !is null)
7887 					form.setAttribute("enctype", "multipart/form-data");
7888 			}
7889 		}}
7890 
7891 		form.addChild("div", Html(`<input type="submit" value="Submit" />`), "submit-button-holder");
7892 
7893 		return form;
7894 	}
7895 
7896 	/// creates a form for gathering object members (for the REST object thing right now)
7897 	Form createAutomaticFormForObject(T)(T obj) {
7898 		auto form = cast(Form) Element.make("form");
7899 
7900 		form.addClass("automatic-form");
7901 
7902 		form.addChild("h3", beautify(__traits(identifier, T)));
7903 
7904 		import std.traits;
7905 
7906 		//Parameters!method params;
7907 		//alias idents = ParameterIdentifierTuple!method;
7908 		//alias defaults = ParameterDefaults!method;
7909 
7910 		foreach(idx, memberName; __traits(derivedMembers, T)) {{
7911 		static if(__traits(compiles, __traits(getMember, obj, memberName).offsetof)) {
7912 			string displayName = beautify(memberName);
7913 			foreach(attr; __traits(getAttributes,  __traits(getMember, T, memberName)))
7914 				static if(is(typeof(attr) == DisplayName))
7915 					displayName = attr.name;
7916 			form.appendChild(elementFor!(typeof(__traits(getMember, T, memberName)))(displayName, memberName));
7917 
7918 			form.setValue(memberName, to!string(__traits(getMember, obj, memberName)));
7919 		}}}
7920 
7921 		form.addChild("div", Html(`<input type="submit" value="Submit" />`), "submit-button-holder");
7922 
7923 		return form;
7924 	}
7925 
7926 	///
7927 	Element formatReturnValueAsHtml(T)(T t) {
7928 		import std.traits;
7929 
7930 		static if(is(T == typeof(null))) {
7931 			return Element.make("span");
7932 		} else static if(is(T : Element)) {
7933 			return t;
7934 		} else static if(is(T == MultipleResponses!Types, Types...)) {
7935 			foreach(index, type; Types) {
7936 				if(t.contains == index)
7937 					return formatReturnValueAsHtml(t.payload[index]);
7938 			}
7939 			assert(0);
7940 		} else static if(isIntegral!T || isSomeString!T || isFloatingPoint!T) {
7941 			return Element.make("span", to!string(t), "automatic-data-display");
7942 		} else static if(is(T == V[K], K, V)) {
7943 			auto dl = Element.make("dl");
7944 			dl.addClass("automatic-data-display");
7945 			foreach(k, v; t) {
7946 				dl.addChild("dt", to!string(k));
7947 				dl.addChild("dd", formatReturnValueAsHtml(v));
7948 			}
7949 			return dl;
7950 		} else static if(is(T == struct)) {
7951 			auto dl = Element.make("dl");
7952 			dl.addClass("automatic-data-display");
7953 
7954 			foreach(idx, memberName; __traits(allMembers, T))
7955 			static if(__traits(compiles, __traits(getMember, T, memberName).offsetof)) {
7956 				dl.addChild("dt", memberName);
7957 				dl.addChild("dt", formatReturnValueAsHtml(__traits(getMember, t, memberName)));
7958 			}
7959 
7960 			return dl;
7961 		} else static if(is(T == bool)) {
7962 			return Element.make("span", t ? "true" : "false", "automatic-data-display");
7963 		} else static if(is(T == E[], E)) {
7964 			static if(is(E : RestObject!Proxy, Proxy)) {
7965 				// treat RestObject similar to struct
7966 				auto table = cast(Table) Element.make("table");
7967 				table.addClass("automatic-data-display");
7968 				string[] names;
7969 				foreach(idx, memberName; __traits(derivedMembers, E))
7970 				static if(__traits(compiles, __traits(getMember, E, memberName).offsetof)) {
7971 					names ~= beautify(memberName);
7972 				}
7973 				table.appendHeaderRow(names);
7974 
7975 				foreach(l; t) {
7976 					auto tr = table.appendRow();
7977 					foreach(idx, memberName; __traits(derivedMembers, E))
7978 					static if(__traits(compiles, __traits(getMember, E, memberName).offsetof)) {
7979 						static if(memberName == "id") {
7980 							string val = to!string(__traits(getMember, l, memberName));
7981 							tr.addChild("td", Element.make("a", val, E.stringof.toLower ~ "s/" ~ val)); // FIXME
7982 						} else {
7983 							tr.addChild("td", formatReturnValueAsHtml(__traits(getMember, l, memberName)));
7984 						}
7985 					}
7986 				}
7987 
7988 				return table;
7989 			} else static if(is(E == struct)) {
7990 				// an array of structs is kinda special in that I like
7991 				// having those formatted as tables.
7992 				auto table = cast(Table) Element.make("table");
7993 				table.addClass("automatic-data-display");
7994 				string[] names;
7995 				foreach(idx, memberName; __traits(allMembers, E))
7996 				static if(__traits(compiles, __traits(getMember, E, memberName).offsetof)) {
7997 					names ~= beautify(memberName);
7998 				}
7999 				table.appendHeaderRow(names);
8000 
8001 				foreach(l; t) {
8002 					auto tr = table.appendRow();
8003 					foreach(idx, memberName; __traits(allMembers, E))
8004 					static if(__traits(compiles, __traits(getMember, E, memberName).offsetof)) {
8005 						tr.addChild("td", formatReturnValueAsHtml(__traits(getMember, l, memberName)));
8006 					}
8007 				}
8008 
8009 				return table;
8010 			} else {
8011 				// otherwise, I will just make a list.
8012 				auto ol = Element.make("ol");
8013 				ol.addClass("automatic-data-display");
8014 				foreach(e; t)
8015 					ol.addChild("li", formatReturnValueAsHtml(e));
8016 				return ol;
8017 			}
8018 		} else static assert(0, "bad return value for cgi call " ~ T.stringof);
8019 
8020 		assert(0);
8021 	}
8022 
8023 }
8024 
8025 /++
8026 	The base class for the [dispatcher] function and object support.
8027 +/
8028 class WebObject {
8029 	//protected Cgi cgi;
8030 
8031 	protected void initialize(Cgi cgi) {
8032 		//this.cgi = cgi;
8033 	}
8034 }
8035 
8036 /++
8037 	Can return one of the given types, decided at runtime. The syntax
8038 	is to declare all the possible types in the return value, then you
8039 	can `return typeof(return)(...value...)` to construct it.
8040 
8041 	It has an auto-generated constructor for each value it can hold.
8042 
8043 	---
8044 	MultipleResponses!(Redirection, string) getData(int how) {
8045 		if(how & 1)
8046 			return typeof(return)(Redirection("http://dpldocs.info/"));
8047 		else
8048 			return typeof(return)("hi there!");
8049 	}
8050 	---
8051 
8052 	If you have lots of returns, you could, inside the function, `alias r = typeof(return);` to shorten it a little.
8053 +/
8054 struct MultipleResponses(T...) {
8055 	private size_t contains;
8056 	private union {
8057 		private T payload;
8058 	}
8059 
8060 	static foreach(index, type; T)
8061 	public this(type t) {
8062 		contains = index;
8063 		payload[index] = t;
8064 	}
8065 
8066 	/++
8067 		This is primarily for testing. It is your way of getting to the response.
8068 
8069 		Let's say you wanted to test that one holding a Redirection and a string actually
8070 		holds a string, by name of "test":
8071 
8072 		---
8073 			auto valueToTest = your_test_function();
8074 
8075 			valueToTest.visit!(
8076 				(Redirection) { assert(0); }, // got a redirection instead of a string, fail the test
8077 				(string s) { assert(s == "test"); } // right value, go ahead and test it.
8078 			);
8079 		---
8080 	+/
8081 	void visit(Handlers...)() {
8082 		template findHandler(type, HandlersToCheck...) {
8083 			static if(HandlersToCheck.length == 0)
8084 				alias findHandler = void;
8085 			else {
8086 				static if(is(typeof(HandlersToCheck[0](type.init))))
8087 					alias findHandler = handler;
8088 				else
8089 					alias findHandler = findHandler!(type, HandlersToCheck[1 .. $]);
8090 			}
8091 		}
8092 		foreach(index, type; T) {
8093 			alias handler = findHandler!(type, Handlers);
8094 			static if(is(handler == void))
8095 				static assert(0, "Type " ~ type.stringof ~ " was not handled by visitor");
8096 			else {
8097 				if(index == contains)
8098 					handler(payload[index]);
8099 			}
8100 		}
8101 	}
8102 
8103 	/+
8104 	auto toArsdJsvar()() {
8105 		import arsd.jsvar;
8106 		return var(null);
8107 	}
8108 	+/
8109 }
8110 
8111 struct RawResponse {
8112 	int code;
8113 	string[] headers;
8114 	const(ubyte)[] responseBody;
8115 }
8116 
8117 /++
8118 	You can return this from [WebObject] subclasses for redirections.
8119 
8120 	(though note the static types means that class must ALWAYS redirect if
8121 	you return this directly. You might want to return [MultipleResponses] if it
8122 	can be conditional)
8123 +/
8124 struct Redirection {
8125 	string to; /// The URL to redirect to.
8126 	int code = 303; /// The HTTP code to retrn.
8127 }
8128 
8129 /++
8130 	Serves a class' methods, as a kind of low-state RPC over the web. To be used with [dispatcher].
8131 
8132 	Usage of this function will add a dependency on [arsd.dom] and [arsd.jsvar] unless you have overriden
8133 	the presenter in the dispatcher.
8134 
8135 	FIXME: explain this better
8136 
8137 	You can overload functions to a limited extent: you can provide a zero-arg and non-zero-arg function,
8138 	and non-zero-arg functions can filter via UDAs for various http methods. Do not attempt other overloads,
8139 	the runtime result of that is undefined.
8140 
8141 	A method is assumed to allow any http method unless it lists some in UDAs, in which case it is limited to only those.
8142 	(this might change, like maybe i will use pure as an indicator GET is ok. idk.)
8143 
8144 	$(WARNING
8145 		---
8146 		// legal in D, undefined runtime behavior with cgi.d, it may call either method
8147 <