1 // Copyright 2013-2022, Adam D. Ruppe. 2 3 // FIXME: websocket proxy support 4 // FIXME: ipv6 support 5 6 // FIXME: headers are supposed to be case insensitive. ugh. 7 8 /++ 9 This is version 2 of my http/1.1 client implementation. 10 11 12 It has no dependencies for basic operation, but does require OpenSSL 13 libraries (or compatible) to be support HTTPS. This dynamically loaded 14 on-demand (meaning it won't be loaded if you don't use it, but if you do 15 use it, the openssl dynamic libraries must be found in the system search path). 16 17 You can compile with `-version=without_openssl` to entirely disable ssl support. 18 19 http2.d, despite its name, does NOT implement HTTP/2.0, but this 20 shouldn't matter for 99.9% of usage, since all servers will continue 21 to support HTTP/1.1 for a very long time. 22 23 History: 24 Automatic `100 Continue` handling was added on September 28, 2021. It doesn't 25 set the Expect header, so it isn't supposed to happen, but plenty of web servers 26 don't follow the standard anyway. 27 +/ 28 module arsd.http2; 29 30 /// 31 unittest { 32 import arsd.http2; 33 34 void main() { 35 auto client = new HttpClient(); 36 37 auto request = client.request(Uri("http://dlang.org/")); 38 auto response = request.waitForCompletion(); 39 40 import std.stdio; 41 writeln(response.contentText); 42 writeln(response.code, " ", response.codeText); 43 writeln(response.contentType); 44 } 45 46 version(arsd_http2_integration_test) main(); // exclude from docs 47 } 48 49 // FIXME: I think I want to disable sigpipe here too. 50 51 import std.uri : encodeComponent; 52 53 debug(arsd_http2_verbose) debug=arsd_http2; 54 55 debug(arsd_http2) import std.stdio : writeln; 56 57 version=arsd_http_internal_implementation; 58 59 version(without_openssl) {} 60 else { 61 version=use_openssl; 62 version=with_openssl; 63 version(older_openssl) {} else 64 version=newer_openssl; 65 } 66 67 version(arsd_http_winhttp_implementation) { 68 pragma(lib, "winhttp") 69 import core.sys.windows.winhttp; 70 // FIXME: alter the dub package file too 71 72 // https://github.com/curl/curl/blob/master/lib/vtls/schannel.c 73 // https://docs.microsoft.com/en-us/windows/win32/secauthn/creating-an-schannel-security-context 74 75 76 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpreaddata 77 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpsendrequest 78 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpopenrequest 79 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpconnect 80 } 81 82 83 84 /++ 85 Demonstrates core functionality, using the [HttpClient], 86 [HttpRequest] (returned by [HttpClient.navigateTo|client.navigateTo]), 87 and [HttpResponse] (returned by [HttpRequest.waitForCompletion|request.waitForCompletion]). 88 89 +/ 90 unittest { 91 import arsd.http2; 92 93 void main() { 94 auto client = new HttpClient(); 95 auto request = client.navigateTo(Uri("http://dlang.org/")); 96 auto response = request.waitForCompletion(); 97 98 string returnedHtml = response.contentText; 99 } 100 } 101 102 // FIXME: multipart encoded file uploads needs implementation 103 // future: do web client api stuff 104 105 private __gshared bool defaultVerifyPeer_ = true; 106 107 void defaultVerifyPeer(bool v) { 108 defaultVerifyPeer_ = v; 109 } 110 111 debug import std.stdio; 112 113 import std.socket; 114 import core.time; 115 116 // FIXME: check Transfer-Encoding: gzip always 117 118 version(with_openssl) { 119 //pragma(lib, "crypto"); 120 //pragma(lib, "ssl"); 121 } 122 123 /+ 124 HttpRequest httpRequest(string method, string url, ubyte[] content, string[string] content) { 125 return null; 126 } 127 +/ 128 129 /** 130 auto request = get("http://arsdnet.net/"); 131 request.send(); 132 133 auto response = get("http://arsdnet.net/").waitForCompletion(); 134 */ 135 HttpRequest get(string url) { 136 auto client = new HttpClient(); 137 auto request = client.navigateTo(Uri(url)); 138 return request; 139 } 140 141 /** 142 Do not forget to call `waitForCompletion()` on the returned object! 143 */ 144 HttpRequest post(string url, string[string] req) { 145 auto client = new HttpClient(); 146 ubyte[] bdata; 147 foreach(k, v; req) { 148 if(bdata.length) 149 bdata ~= cast(ubyte[]) "&"; 150 bdata ~= cast(ubyte[]) encodeComponent(k); 151 bdata ~= cast(ubyte[]) "="; 152 bdata ~= cast(ubyte[]) encodeComponent(v); 153 } 154 auto request = client.request(Uri(url), HttpVerb.POST, bdata, "application/x-www-form-urlencoded"); 155 return request; 156 } 157 158 /// gets the text off a url. basic operation only. 159 string getText(string url) { 160 auto request = get(url); 161 auto response = request.waitForCompletion(); 162 return cast(string) response.content; 163 } 164 165 /+ 166 ubyte[] getBinary(string url, string[string] cookies = null) { 167 auto hr = httpRequest("GET", url, null, cookies); 168 if(hr.code != 200) 169 throw new Exception(format("HTTP answered %d instead of 200 on %s", hr.code, url)); 170 return hr.content; 171 } 172 173 /** 174 Gets a textual document, ignoring headers. Throws on non-text or error. 175 */ 176 string get(string url, string[string] cookies = null) { 177 auto hr = httpRequest("GET", url, null, cookies); 178 if(hr.code != 200) 179 throw new Exception(format("HTTP answered %d instead of 200 on %s", hr.code, url)); 180 if(hr.contentType.indexOf("text/") == -1) 181 throw new Exception(hr.contentType ~ " is bad content for conversion to string"); 182 return cast(string) hr.content; 183 184 } 185 186 static import std.uri; 187 188 string post(string url, string[string] args, string[string] cookies = null) { 189 string content; 190 191 foreach(name, arg; args) { 192 if(content.length) 193 content ~= "&"; 194 content ~= std.uri.encode(name) ~ "=" ~ std.uri.encode(arg); 195 } 196 197 auto hr = httpRequest("POST", url, cast(ubyte[]) content, cookies, ["Content-Type: application/x-www-form-urlencoded"]); 198 if(hr.code != 200) 199 throw new Exception(format("HTTP answered %d instead of 200", hr.code)); 200 if(hr.contentType.indexOf("text/") == -1) 201 throw new Exception(hr.contentType ~ " is bad content for conversion to string"); 202 203 return cast(string) hr.content; 204 } 205 206 +/ 207 208 /// 209 struct HttpResponse { 210 /++ 211 The HTTP response code, if the response was completed, or some value < 100 if it was aborted or failed. 212 213 Code 0 - initial value, nothing happened 214 Code 1 - you called request.abort 215 Code 2 - connection refused 216 Code 3 - connection succeeded, but server disconnected early 217 Code 4 - server sent corrupted response (or this code has a bug and processed it wrong) 218 Code 5 - request timed out 219 220 Code >= 100 - a HTTP response 221 +/ 222 int code; 223 string codeText; /// 224 225 string httpVersion; /// 226 227 string statusLine; /// 228 229 string contentType; /// The *full* content type header. See also [contentTypeMimeType] and [contentTypeCharset]. 230 string location; /// The location header 231 232 /++ 233 234 History: 235 Added December 5, 2020 (version 9.1) 236 +/ 237 bool wasSuccessful() { 238 return code >= 200 && code < 400; 239 } 240 241 /++ 242 Returns the mime type part of the [contentType] header. 243 244 History: 245 Added July 25, 2022 (version 10.9) 246 +/ 247 string contentTypeMimeType() { 248 auto idx = contentType.indexOf(";"); 249 if(idx == -1) 250 return contentType; 251 252 return contentType[0 .. idx].strip; 253 } 254 255 /// the charset out of content type, if present. `null` if not. 256 string contentTypeCharset() { 257 auto idx = contentType.indexOf("charset="); 258 if(idx == -1) 259 return null; 260 auto c = contentType[idx + "charset=".length .. $].strip; 261 if(c.length) 262 return c; 263 return null; 264 } 265 266 /++ 267 Names and values of cookies set in the response. 268 269 History: 270 Prior to July 5, 2021 (dub v10.2), this was a public field instead of a property. I did 271 not consider this a breaking change since the intended use is completely compatible with the 272 property, and it was not actually implemented properly before anyway. 273 +/ 274 @property string[string] cookies() const { 275 string[string] ret; 276 foreach(cookie; cookiesDetails) 277 ret[cookie.name] = cookie.value; 278 return ret; 279 } 280 /++ 281 The full parsed-out information of cookies set in the response. 282 283 History: 284 Added July 5, 2021 (dub v10.2). 285 +/ 286 @property CookieHeader[] cookiesDetails() inout { 287 CookieHeader[] ret; 288 foreach(header; headers) { 289 if(auto content = header.isHttpHeader("set-cookie")) { 290 // format: name=value, value might be double quoted. it MIGHT be url encoded, but im not going to attempt that since the RFC is silent. 291 // then there's optionally ; attr=value after that. attributes need not have a value 292 293 CookieHeader cookie; 294 295 auto remaining = content; 296 297 cookie_name: 298 foreach(idx, ch; remaining) { 299 if(ch == '=') { 300 cookie.name = remaining[0 .. idx].idup_if_needed; 301 remaining = remaining[idx + 1 .. $]; 302 break; 303 } 304 } 305 306 cookie_value: 307 308 { 309 auto idx = remaining.indexOf(";"); 310 if(idx == -1) { 311 cookie.value = remaining.idup_if_needed; 312 remaining = remaining[$..$]; 313 } else { 314 cookie.value = remaining[0 .. idx].idup_if_needed; 315 remaining = remaining[idx + 1 .. $].stripLeft; 316 } 317 318 if(cookie.value.length > 2 && cookie.value[0] == '"' && cookie.value[$-1] == '"') 319 cookie.value = cookie.value[1 .. $ - 1]; 320 } 321 322 cookie_attributes: 323 324 while(remaining.length) { 325 string name; 326 foreach(idx, ch; remaining) { 327 if(ch == '=') { 328 name = remaining[0 .. idx].idup_if_needed; 329 remaining = remaining[idx + 1 .. $]; 330 331 string value; 332 333 foreach(idx2, ch2; remaining) { 334 if(ch2 == ';') { 335 value = remaining[0 .. idx2].idup_if_needed; 336 remaining = remaining[idx2 + 1 .. $].stripLeft; 337 break; 338 } 339 } 340 341 if(value is null) { 342 value = remaining.idup_if_needed; 343 remaining = remaining[$ .. $]; 344 } 345 346 cookie.attributes[name] = value; 347 continue cookie_attributes; 348 } else if(ch == ';') { 349 name = remaining[0 .. idx].idup_if_needed; 350 remaining = remaining[idx + 1 .. $].stripLeft; 351 cookie.attributes[name] = ""; 352 continue cookie_attributes; 353 } 354 } 355 356 if(remaining.length) { 357 cookie.attributes[remaining.idup_if_needed] = ""; 358 remaining = remaining[$..$]; 359 360 } 361 } 362 363 ret ~= cookie; 364 } 365 } 366 return ret; 367 } 368 369 string[] headers; /// Array of all headers returned. 370 string[string] headersHash; /// 371 372 ubyte[] content; /// The raw content returned in the response body. 373 string contentText; /// [content], but casted to string (for convenience) 374 375 alias responseText = contentText; // just cuz I do this so often. 376 //alias body = content; 377 378 /++ 379 returns `new Document(this.contentText)`. Requires [arsd.dom]. 380 +/ 381 auto contentDom()() { 382 import arsd.dom; 383 return new Document(this.contentText); 384 385 } 386 387 /++ 388 returns `var.fromJson(this.contentText)`. Requires [arsd.jsvar]. 389 +/ 390 auto contentJson()() { 391 import arsd.jsvar; 392 return var.fromJson(this.contentText); 393 } 394 395 HttpRequestParameters requestParameters; /// 396 397 LinkHeader[] linksStored; 398 bool linksLazilyParsed; 399 400 HttpResponse deepCopy() const { 401 HttpResponse h = cast(HttpResponse) this; 402 h.headers = h.headers.dup; 403 h.headersHash = h.headersHash.dup; 404 h.content = h.content.dup; 405 h.linksStored = h.linksStored.dup; 406 return h; 407 } 408 409 /// Returns links header sorted by "rel" attribute. 410 /// It returns a new array on each call. 411 LinkHeader[string] linksHash() { 412 auto links = this.links(); 413 LinkHeader[string] ret; 414 foreach(link; links) 415 ret[link.rel] = link; 416 return ret; 417 } 418 419 /// Returns the Link header, parsed. 420 LinkHeader[] links() { 421 if(linksLazilyParsed) 422 return linksStored; 423 linksLazilyParsed = true; 424 LinkHeader[] ret; 425 426 auto hdrPtr = "link" in headersHash; 427 if(hdrPtr is null) 428 return ret; 429 430 auto header = *hdrPtr; 431 432 LinkHeader current; 433 434 while(header.length) { 435 char ch = header[0]; 436 437 if(ch == '<') { 438 // read url 439 header = header[1 .. $]; 440 size_t idx; 441 while(idx < header.length && header[idx] != '>') 442 idx++; 443 current.url = header[0 .. idx]; 444 header = header[idx .. $]; 445 } else if(ch == ';') { 446 // read attribute 447 header = header[1 .. $]; 448 header = header.stripLeft; 449 450 size_t idx; 451 while(idx < header.length && header[idx] != '=') 452 idx++; 453 454 string name = header[0 .. idx]; 455 if(idx + 1 < header.length) 456 header = header[idx + 1 .. $]; 457 else 458 header = header[$ .. $]; 459 460 string value; 461 462 if(header.length && header[0] == '"') { 463 // quoted value 464 header = header[1 .. $]; 465 idx = 0; 466 while(idx < header.length && header[idx] != '\"') 467 idx++; 468 value = header[0 .. idx]; 469 header = header[idx .. $]; 470 471 } else if(header.length) { 472 // unquoted value 473 idx = 0; 474 while(idx < header.length && header[idx] != ',' && header[idx] != ' ' && header[idx] != ';') 475 idx++; 476 477 value = header[0 .. idx]; 478 header = header[idx .. $].stripLeft; 479 } 480 481 name = name.toLower; 482 if(name == "rel") 483 current.rel = value; 484 else 485 current.attributes[name] = value; 486 487 } else if(ch == ',') { 488 // start another 489 ret ~= current; 490 current = LinkHeader.init; 491 } else if(ch == ' ' || ch == '\n' || ch == '\r' || ch == '\t') { 492 // ignore 493 } 494 495 if(header.length) 496 header = header[1 .. $]; 497 } 498 499 ret ~= current; 500 501 linksStored = ret; 502 503 return ret; 504 } 505 } 506 507 /+ 508 headerName MUST be all lower case and NOT have the colon on it 509 510 returns slice of the input thing after the header name 511 +/ 512 private inout(char)[] isHttpHeader(inout(char)[] thing, const(char)[] headerName) { 513 foreach(idx, ch; thing) { 514 if(idx < headerName.length) { 515 if(headerName[idx] == '-' && ch != '-') 516 return null; 517 if((ch | ' ') != headerName[idx]) 518 return null; 519 } else if(idx == headerName.length) { 520 if(ch != ':') 521 return null; 522 } else { 523 return thing[idx .. $].strip; 524 } 525 } 526 return null; 527 } 528 529 private string idup_if_needed(string s) { return s; } 530 private string idup_if_needed(const(char)[] s) { return s.idup; } 531 532 unittest { 533 assert("Cookie: foo=bar".isHttpHeader("cookie") == "foo=bar"); 534 assert("cookie: foo=bar".isHttpHeader("cookie") == "foo=bar"); 535 assert("cOOkie: foo=bar".isHttpHeader("cookie") == "foo=bar"); 536 assert("Set-Cookie: foo=bar".isHttpHeader("set-cookie") == "foo=bar"); 537 assert(!"".isHttpHeader("cookie")); 538 } 539 540 /// 541 struct LinkHeader { 542 string url; /// 543 string rel; /// 544 string[string] attributes; /// like title, rev, media, whatever attributes 545 } 546 547 /++ 548 History: 549 Added July 5, 2021 550 +/ 551 struct CookieHeader { 552 string name; 553 string value; 554 string[string] attributes; 555 } 556 557 import std.string; 558 static import std.algorithm; 559 import std.conv; 560 import std.range; 561 562 563 private AddressFamily family(string unixSocketPath) { 564 if(unixSocketPath.length) 565 return AddressFamily.UNIX; 566 else // FIXME: what about ipv6? 567 return AddressFamily.INET; 568 } 569 570 version(Windows) 571 private class UnixAddress : Address { 572 this(string) { 573 throw new Exception("No unix address support on this system in lib yet :("); 574 } 575 override sockaddr* name() { assert(0); } 576 override const(sockaddr)* name() const { assert(0); } 577 override int nameLen() const { assert(0); } 578 } 579 580 581 // Copy pasta from cgi.d, then stripped down. unix path thing added tho 582 /++ 583 Represents a URI. It offers named access to the components and relative uri resolution, though as a user of the library, you'd mostly just construct it like `Uri("http://example.com/index.html")`. 584 +/ 585 struct Uri { 586 alias toString this; // blargh idk a url really is a string, but should it be implicit? 587 588 // scheme://userinfo@host:port/path?query#fragment 589 590 string scheme; /// e.g. "http" in "http://example.com/" 591 string userinfo; /// the username (and possibly a password) in the uri 592 string host; /// the domain name 593 int port; /// port number, if given. Will be zero if a port was not explicitly given 594 string path; /// e.g. "/folder/file.html" in "http://example.com/folder/file.html" 595 string query; /// the stuff after the ? in a uri 596 string fragment; /// the stuff after the # in a uri. 597 598 /// Breaks down a uri string to its components 599 this(string uri) { 600 size_t lastGoodIndex; 601 foreach(char ch; uri) { 602 if(ch > 127) { 603 break; 604 } 605 lastGoodIndex++; 606 } 607 608 string replacement = uri[0 .. lastGoodIndex]; 609 foreach(char ch; uri[lastGoodIndex .. $]) { 610 if(ch > 127) { 611 // need to percent-encode any non-ascii in it 612 char[3] buffer; 613 buffer[0] = '%'; 614 615 auto first = ch / 16; 616 auto second = ch % 16; 617 first += (first >= 10) ? ('A'-10) : '0'; 618 second += (second >= 10) ? ('A'-10) : '0'; 619 620 buffer[1] = cast(char) first; 621 buffer[2] = cast(char) second; 622 623 replacement ~= buffer[]; 624 } else { 625 replacement ~= ch; 626 } 627 } 628 629 reparse(replacement); 630 } 631 632 /// Returns `port` if set, otherwise if scheme is https 443, otherwise always 80 633 int effectivePort() const @property nothrow pure @safe @nogc { 634 return port != 0 ? port 635 : scheme == "https" ? 443 : 80; 636 } 637 638 private string unixSocketPath = null; 639 /// Indicates it should be accessed through a unix socket instead of regular tcp. Returns new version without modifying this object. 640 Uri viaUnixSocket(string path) const { 641 Uri copy = this; 642 copy.unixSocketPath = path; 643 return copy; 644 } 645 646 /// Goes through a unix socket in the abstract namespace (linux only). Returns new version without modifying this object. 647 version(linux) 648 Uri viaAbstractSocket(string path) const { 649 Uri copy = this; 650 copy.unixSocketPath = "\0" ~ path; 651 return copy; 652 } 653 654 private void reparse(string uri) { 655 // from RFC 3986 656 // the ctRegex triples the compile time and makes ugly errors for no real benefit 657 // it was a nice experiment but just not worth it. 658 // enum ctr = ctRegex!r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?"; 659 /* 660 Captures: 661 0 = whole url 662 1 = scheme, with : 663 2 = scheme, no : 664 3 = authority, with // 665 4 = authority, no // 666 5 = path 667 6 = query string, with ? 668 7 = query string, no ? 669 8 = anchor, with # 670 9 = anchor, no # 671 */ 672 // Yikes, even regular, non-CT regex is also unacceptably slow to compile. 1.9s on my computer! 673 // instead, I will DIY and cut that down to 0.6s on the same computer. 674 /* 675 676 Note that authority is 677 user:password@domain:port 678 where the user:password@ part is optional, and the :port is optional. 679 680 Regex translation: 681 682 Scheme cannot have :, /, ?, or # in it, and must have one or more chars and end in a :. It is optional, but must be first. 683 Authority must start with //, but cannot have any other /, ?, or # in it. It is optional. 684 Path cannot have any ? or # in it. It is optional. 685 Query must start with ? and must not have # in it. It is optional. 686 Anchor must start with # and can have anything else in it to end of string. It is optional. 687 */ 688 689 this = Uri.init; // reset all state 690 691 // empty uri = nothing special 692 if(uri.length == 0) { 693 return; 694 } 695 696 size_t idx; 697 698 scheme_loop: foreach(char c; uri[idx .. $]) { 699 switch(c) { 700 case ':': 701 case '/': 702 case '?': 703 case '#': 704 break scheme_loop; 705 default: 706 } 707 idx++; 708 } 709 710 if(idx == 0 && uri[idx] == ':') { 711 // this is actually a path! we skip way ahead 712 goto path_loop; 713 } 714 715 if(idx == uri.length) { 716 // the whole thing is a path, apparently 717 path = uri; 718 return; 719 } 720 721 if(idx > 0 && uri[idx] == ':') { 722 scheme = uri[0 .. idx]; 723 idx++; 724 } else { 725 // we need to rewind; it found a / but no :, so the whole thing is prolly a path... 726 idx = 0; 727 } 728 729 if(idx + 2 < uri.length && uri[idx .. idx + 2] == "//") { 730 // we have an authority.... 731 idx += 2; 732 733 auto authority_start = idx; 734 authority_loop: foreach(char c; uri[idx .. $]) { 735 switch(c) { 736 case '/': 737 case '?': 738 case '#': 739 break authority_loop; 740 default: 741 } 742 idx++; 743 } 744 745 auto authority = uri[authority_start .. idx]; 746 747 auto idx2 = authority.indexOf("@"); 748 if(idx2 != -1) { 749 userinfo = authority[0 .. idx2]; 750 authority = authority[idx2 + 1 .. $]; 751 } 752 753 if(authority.length && authority[0] == '[') { 754 // ipv6 address special casing 755 idx2 = authority.indexOf(']'); 756 if(idx2 != -1) { 757 auto end = authority[idx2 + 1 .. $]; 758 if(end.length && end[0] == ':') 759 idx2 = idx2 + 1; 760 else 761 idx2 = -1; 762 } 763 } else { 764 idx2 = authority.indexOf(":"); 765 } 766 767 if(idx2 == -1) { 768 port = 0; // 0 means not specified; we should use the default for the scheme 769 host = authority; 770 } else { 771 host = authority[0 .. idx2]; 772 if(idx2 + 1 < authority.length) 773 port = to!int(authority[idx2 + 1 .. $]); 774 else 775 port = 0; 776 } 777 } 778 779 path_loop: 780 auto path_start = idx; 781 782 foreach(char c; uri[idx .. $]) { 783 if(c == '?' || c == '#') 784 break; 785 idx++; 786 } 787 788 path = uri[path_start .. idx]; 789 790 if(idx == uri.length) 791 return; // nothing more to examine... 792 793 if(uri[idx] == '?') { 794 idx++; 795 auto query_start = idx; 796 foreach(char c; uri[idx .. $]) { 797 if(c == '#') 798 break; 799 idx++; 800 } 801 query = uri[query_start .. idx]; 802 } 803 804 if(idx < uri.length && uri[idx] == '#') { 805 idx++; 806 fragment = uri[idx .. $]; 807 } 808 809 // uriInvalidated = false; 810 } 811 812 private string rebuildUri() const { 813 string ret; 814 if(scheme.length) 815 ret ~= scheme ~ ":"; 816 if(userinfo.length || host.length) 817 ret ~= "//"; 818 if(userinfo.length) 819 ret ~= userinfo ~ "@"; 820 if(host.length) 821 ret ~= host; 822 if(port) 823 ret ~= ":" ~ to!string(port); 824 825 ret ~= path; 826 827 if(query.length) 828 ret ~= "?" ~ query; 829 830 if(fragment.length) 831 ret ~= "#" ~ fragment; 832 833 // uri = ret; 834 // uriInvalidated = false; 835 return ret; 836 } 837 838 /// Converts the broken down parts back into a complete string 839 string toString() const { 840 // if(uriInvalidated) 841 return rebuildUri(); 842 } 843 844 /// Returns a new absolute Uri given a base. It treats this one as 845 /// relative where possible, but absolute if not. (If protocol, domain, or 846 /// other info is not set, the new one inherits it from the base.) 847 /// 848 /// Browsers use a function like this to figure out links in html. 849 Uri basedOn(in Uri baseUrl) const { 850 Uri n = this; // copies 851 if(n.scheme == "data") 852 return n; 853 // n.uriInvalidated = true; // make sure we regenerate... 854 855 // userinfo is not inherited... is this wrong? 856 857 // if anything is given in the existing url, we don't use the base anymore. 858 if(n.scheme.empty) { 859 n.scheme = baseUrl.scheme; 860 if(n.host.empty) { 861 n.host = baseUrl.host; 862 if(n.port == 0) { 863 n.port = baseUrl.port; 864 if(n.path.length > 0 && n.path[0] != '/') { 865 auto b = baseUrl.path[0 .. baseUrl.path.lastIndexOf("/") + 1]; 866 if(b.length == 0) 867 b = "/"; 868 n.path = b ~ n.path; 869 } else if(n.path.length == 0) { 870 n.path = baseUrl.path; 871 } 872 } 873 } 874 } 875 876 n.removeDots(); 877 878 // if still basically talking to the same thing, we should inherit the unix path 879 // too since basically the unix path is saying for this service, always use this override. 880 if(n.host == baseUrl.host && n.scheme == baseUrl.scheme && n.port == baseUrl.port) 881 n.unixSocketPath = baseUrl.unixSocketPath; 882 883 return n; 884 } 885 886 /++ 887 Resolves ../ and ./ parts of the path. Used in the implementation of [basedOn] and you could also use it to normalize things. 888 +/ 889 void removeDots() { 890 auto parts = this.path.split("/"); 891 string[] toKeep; 892 foreach(part; parts) { 893 if(part == ".") { 894 continue; 895 } else if(part == "..") { 896 //if(toKeep.length > 1) 897 toKeep = toKeep[0 .. $-1]; 898 //else 899 //toKeep = [""]; 900 continue; 901 } else { 902 //if(toKeep.length && toKeep[$-1].length == 0 && part.length == 0) 903 //continue; // skip a `//` situation 904 toKeep ~= part; 905 } 906 } 907 908 auto path = toKeep.join("/"); 909 if(path.length && path[0] != '/') 910 path = "/" ~ path; 911 912 this.path = path; 913 } 914 } 915 916 /* 917 void main(string args[]) { 918 write(post("http://arsdnet.net/bugs.php", ["test" : "hey", "again" : "what"])); 919 } 920 */ 921 922 /// 923 struct BasicAuth { 924 string username; /// 925 string password; /// 926 } 927 928 class ProxyException : Exception { 929 this(string msg) {super(msg); } 930 } 931 932 /** 933 Represents a HTTP request. You usually create these through a [HttpClient]. 934 935 936 --- 937 auto request = new HttpRequest(); // note that when there's no associated client, some features may not work 938 // normally you'd instead do `new HttpClient(); client.request(...)` 939 // set any properties here 940 941 // synchronous usage 942 auto reply = request.perform(); 943 944 // async usage, type 1: 945 request.send(); 946 request2.send(); 947 948 // wait until the first one is done, with the second one still in-flight 949 auto response = request.waitForCompletion(); 950 951 // async usage, type 2: 952 request.onDataReceived = (HttpRequest hr) { 953 if(hr.state == HttpRequest.State.complete) { 954 // use hr.responseData 955 } 956 }; 957 request.send(); // send, using the callback 958 959 // before terminating, be sure you wait for your requests to finish! 960 961 request.waitForCompletion(); 962 --- 963 */ 964 class HttpRequest { 965 966 /// Automatically follow a redirection? 967 bool followLocation = false; 968 969 /++ 970 Maximum number of redirections to follow (used only if [followLocation] is set to true). Will resolve with an error if a single request has more than this number of redirections. The default value is currently 10, but may change without notice. If you need a specific value, be sure to call this function. 971 972 If you want unlimited redirects, call it with `int.max`. If you set it to 0 but set [followLocation] to `true`, any attempt at redirection will abort the request. To disable automatically following redirection, set [followLocation] to `false` so you can process the 30x code yourself as a completed request. 973 974 History: 975 Added July 27, 2022 (dub v10.9) 976 +/ 977 void setMaximumNumberOfRedirects(int max = 10) { 978 maximumNumberOfRedirectsRemaining = max; 979 } 980 981 private int maximumNumberOfRedirectsRemaining; 982 983 /++ 984 Set to `true` to automatically retain cookies in the associated [HttpClient] from this request. 985 Note that you must have constructed the request from a `HttpClient` or at least passed one into the 986 constructor for this to have any effect. 987 988 Bugs: 989 See [HttpClient.retainCookies] for important caveats. 990 991 History: 992 Added July 5, 2021 (dub v10.2) 993 +/ 994 bool retainCookies = false; 995 996 private HttpClient client; 997 998 this() { 999 } 1000 1001 /// 1002 this(HttpClient client, Uri where, HttpVerb method, ICache cache = null, Duration timeout = 10.seconds, string proxy = null) { 1003 this.client = client; 1004 populateFromInfo(where, method); 1005 setTimeout(timeout); 1006 this.cache = cache; 1007 this.proxy = proxy; 1008 1009 setMaximumNumberOfRedirects(); 1010 } 1011 1012 1013 /// ditto 1014 this(Uri where, HttpVerb method, ICache cache = null, Duration timeout = 10.seconds, string proxy = null) { 1015 this(null, where, method, cache, timeout, proxy); 1016 } 1017 1018 /++ 1019 Sets the timeout from inactivity on the request. This is the amount of time that passes with no send or receive activity on the request before it fails with "request timed out" error. 1020 1021 History: 1022 Added March 31, 2021 1023 +/ 1024 void setTimeout(Duration timeout) { 1025 this.requestParameters.timeoutFromInactivity = timeout; 1026 this.timeoutFromInactivity = MonoTime.currTime + this.requestParameters.timeoutFromInactivity; 1027 } 1028 1029 private MonoTime timeoutFromInactivity; 1030 1031 private Uri where; 1032 1033 private ICache cache; 1034 1035 /++ 1036 Proxy to use for this request. It should be a URL or `null`. 1037 1038 This must be sent before you call [send]. 1039 1040 History: 1041 Added April 12, 2021 (dub v9.5) 1042 +/ 1043 string proxy; 1044 1045 /++ 1046 For https connections, if this is `true`, it will fail to connect if the TLS certificate can not be 1047 verified. Setting this to `false` will skip this check and allow the connection to continue anyway. 1048 1049 When the [HttpRequest] is constructed from a [HttpClient], it will inherit the value from the client 1050 instead of using the `= true` here. You can change this value any time before you call [send] (which 1051 is done implicitly if you call [waitForCompletion]). 1052 1053 History: 1054 Added April 5, 2022 (dub v10.8) 1055 1056 Prior to this, it always used the global (but undocumented) `defaultVerifyPeer` setting, and sometimes 1057 even if it was true, it would skip the verification. Now, it always respects this local setting. 1058 +/ 1059 bool verifyPeer = true; 1060 1061 1062 /// Final url after any redirections 1063 string finalUrl; 1064 1065 void populateFromInfo(Uri where, HttpVerb method) { 1066 auto parts = where.basedOn(this.where); 1067 this.where = parts; 1068 finalUrl = where.toString(); 1069 requestParameters.method = method; 1070 requestParameters.unixSocketPath = where.unixSocketPath; 1071 requestParameters.host = parts.host; 1072 requestParameters.port = cast(ushort) parts.effectivePort; 1073 requestParameters.ssl = parts.scheme == "https"; 1074 requestParameters.uri = parts.path.length ? parts.path : "/"; 1075 if(parts.query.length) { 1076 requestParameters.uri ~= "?"; 1077 requestParameters.uri ~= parts.query; 1078 } 1079 } 1080 1081 ~this() { 1082 } 1083 1084 ubyte[] sendBuffer; 1085 1086 HttpResponse responseData; 1087 private HttpClient parentClient; 1088 1089 size_t bodyBytesSent; 1090 size_t bodyBytesReceived; 1091 1092 State state_; 1093 State state() { return state_; } 1094 State state(State s) { 1095 assert(state_ != State.complete); 1096 return state_ = s; 1097 } 1098 /// Called when data is received. Check the state to see what data is available. 1099 void delegate(HttpRequest) onDataReceived; 1100 1101 enum State { 1102 /// The request has not yet been sent 1103 unsent, 1104 1105 /// The send() method has been called, but no data is 1106 /// sent on the socket yet because the connection is busy. 1107 pendingAvailableConnection, 1108 1109 /// connect has been called, but we're waiting on word of success 1110 connecting, 1111 1112 /// connecting a ssl, needing this 1113 sslConnectPendingRead, 1114 /// ditto 1115 sslConnectPendingWrite, 1116 1117 /// The headers are being sent now 1118 sendingHeaders, 1119 1120 // FIXME: allow Expect: 100-continue and separate the body send 1121 1122 /// The body is being sent now 1123 sendingBody, 1124 1125 /// The request has been sent but we haven't received any response yet 1126 waitingForResponse, 1127 1128 /// We have received some data and are currently receiving headers 1129 readingHeaders, 1130 1131 /// All headers are available but we're still waiting on the body 1132 readingBody, 1133 1134 /// The request is complete. 1135 complete, 1136 1137 /// The request is aborted, either by the abort() method, or as a result of the server disconnecting 1138 aborted 1139 } 1140 1141 /// Sends now and waits for the request to finish, returning the response. 1142 HttpResponse perform() { 1143 send(); 1144 return waitForCompletion(); 1145 } 1146 1147 /// Sends the request asynchronously. 1148 void send() { 1149 sendPrivate(true); 1150 } 1151 1152 private void sendPrivate(bool advance) { 1153 if(state != State.unsent && state != State.aborted) 1154 return; // already sent 1155 1156 if(cache !is null) { 1157 auto res = cache.getCachedResponse(this.requestParameters); 1158 if(res !is null) { 1159 state = State.complete; 1160 responseData = (*res).deepCopy(); 1161 return; 1162 } 1163 } 1164 1165 if(this.where.scheme == "data") { 1166 void error(string content) { 1167 responseData.code = 400; 1168 responseData.codeText = "Bad Request"; 1169 responseData.contentType = "text/plain"; 1170 responseData.content = cast(ubyte[]) content; 1171 responseData.contentText = content; 1172 state = State.complete; 1173 return; 1174 } 1175 1176 auto thing = this.where.path; 1177 // format is: type,data 1178 // type can have ;base64 1179 auto comma = thing.indexOf(","); 1180 if(comma == -1) 1181 return error("Invalid data uri, no comma found"); 1182 1183 auto type = thing[0 .. comma]; 1184 auto data = thing[comma + 1 .. $]; 1185 if(type.length == 0) 1186 type = "text/plain"; 1187 1188 import std.uri; 1189 auto bdata = cast(ubyte[]) decodeComponent(data); 1190 1191 if(type.indexOf(";base64") != -1) { 1192 import std.base64; 1193 try { 1194 bdata = Base64.decode(bdata); 1195 } catch(Exception e) { 1196 return error(e.msg); 1197 } 1198 } 1199 1200 responseData.code = 200; 1201 responseData.codeText = "OK"; 1202 responseData.contentType = type; 1203 responseData.content = bdata; 1204 responseData.contentText = cast(string) responseData.content; 1205 state = State.complete; 1206 return; 1207 } 1208 1209 string headers; 1210 1211 headers ~= to!string(requestParameters.method); 1212 headers ~= " "; 1213 if(proxy.length && !requestParameters.ssl) { 1214 // if we're doing a http proxy, we need to send a complete, absolute uri 1215 // so reconstruct it 1216 headers ~= "http://"; 1217 headers ~= requestParameters.host; 1218 if(requestParameters.port != 80) { 1219 headers ~= ":"; 1220 headers ~= to!string(requestParameters.port); 1221 } 1222 } 1223 1224 headers ~= requestParameters.uri; 1225 1226 if(requestParameters.useHttp11) 1227 headers ~= " HTTP/1.1\r\n"; 1228 else 1229 headers ~= " HTTP/1.0\r\n"; 1230 1231 // the whole authority section is supposed to be there, but curl doesn't send if default port 1232 // so I'll copy what they do 1233 headers ~= "Host: "; 1234 headers ~= requestParameters.host; 1235 if(requestParameters.port != 80 && requestParameters.port != 443) { 1236 headers ~= ":"; 1237 headers ~= to!string(requestParameters.port); 1238 } 1239 headers ~= "\r\n"; 1240 1241 bool specSaysRequestAlwaysHasBody = 1242 requestParameters.method == HttpVerb.POST || 1243 requestParameters.method == HttpVerb.PUT || 1244 requestParameters.method == HttpVerb.PATCH; 1245 1246 if(requestParameters.userAgent.length) 1247 headers ~= "User-Agent: "~requestParameters.userAgent~"\r\n"; 1248 if(requestParameters.contentType.length) 1249 headers ~= "Content-Type: "~requestParameters.contentType~"\r\n"; 1250 if(requestParameters.authorization.length) 1251 headers ~= "Authorization: "~requestParameters.authorization~"\r\n"; 1252 if(requestParameters.bodyData.length || specSaysRequestAlwaysHasBody) 1253 headers ~= "Content-Length: "~to!string(requestParameters.bodyData.length)~"\r\n"; 1254 if(requestParameters.acceptGzip) 1255 headers ~= "Accept-Encoding: gzip\r\n"; 1256 if(requestParameters.keepAlive) 1257 headers ~= "Connection: keep-alive\r\n"; 1258 1259 string cookieHeader; 1260 foreach(name, value; requestParameters.cookies) { 1261 if(cookieHeader is null) 1262 cookieHeader = "Cookie: "; 1263 else 1264 cookieHeader ~= "; "; 1265 cookieHeader ~= name; 1266 cookieHeader ~= "="; 1267 cookieHeader ~= value; 1268 } 1269 1270 if(cookieHeader !is null) { 1271 cookieHeader ~= "\r\n"; 1272 headers ~= cookieHeader; 1273 } 1274 1275 foreach(header; requestParameters.headers) 1276 headers ~= header ~ "\r\n"; 1277 1278 headers ~= "\r\n"; 1279 1280 // FIXME: separate this for 100 continue 1281 sendBuffer = cast(ubyte[]) headers ~ requestParameters.bodyData; 1282 1283 // import std.stdio; writeln("******* ", cast(string) sendBuffer); 1284 1285 responseData = HttpResponse.init; 1286 responseData.requestParameters = requestParameters; 1287 bodyBytesSent = 0; 1288 bodyBytesReceived = 0; 1289 state = State.pendingAvailableConnection; 1290 1291 bool alreadyPending = false; 1292 foreach(req; pending) 1293 if(req is this) { 1294 alreadyPending = true; 1295 break; 1296 } 1297 if(!alreadyPending) { 1298 pending ~= this; 1299 } 1300 1301 if(advance) 1302 HttpRequest.advanceConnections(requestParameters.timeoutFromInactivity); 1303 } 1304 1305 1306 /// Waits for the request to finish or timeout, whichever comes first. 1307 HttpResponse waitForCompletion() { 1308 while(state != State.aborted && state != State.complete) { 1309 if(state == State.unsent) { 1310 send(); 1311 continue; 1312 } 1313 if(auto err = HttpRequest.advanceConnections(requestParameters.timeoutFromInactivity)) { 1314 switch(err) { 1315 case 1: throw new Exception("HttpRequest.advanceConnections returned 1: all connections timed out"); 1316 case 2: throw new Exception("HttpRequest.advanceConnections returned 2: nothing to do"); 1317 case 3: continue; // EINTR 1318 default: throw new Exception("HttpRequest.advanceConnections got err " ~ to!string(err)); 1319 } 1320 } 1321 } 1322 1323 if(state == State.complete && responseData.code >= 200) 1324 if(cache !is null) 1325 cache.cacheResponse(this.requestParameters, this.responseData); 1326 1327 return responseData; 1328 } 1329 1330 /// Aborts this request. 1331 void abort() { 1332 this.state = State.aborted; 1333 this.responseData.code = 1; 1334 this.responseData.codeText = "request.abort called"; 1335 // the actual cancellation happens in the event loop 1336 } 1337 1338 HttpRequestParameters requestParameters; /// 1339 1340 version(arsd_http_winhttp_implementation) { 1341 public static void resetInternals() { 1342 1343 } 1344 1345 static assert(0, "implementation not finished"); 1346 } 1347 1348 1349 version(arsd_http_internal_implementation) { 1350 1351 /++ 1352 Changes the limit of number of open, inactive sockets. Reusing connections can provide a significant 1353 performance improvement, but the operating system can also impose a global limit on the number of open 1354 sockets and/or files that you don't want to run into. This lets you choose a balance right for you. 1355 1356 1357 When the total number of cached, inactive sockets approaches this maximum, it will check for ones closed by the 1358 server first. If there are none already closed by the server, it will select sockets at random from its connection 1359 cache and close them to make room for the new ones. 1360 1361 Please note: 1362 1363 $(LIST 1364 * there is always a limit of six open sockets per domain, per the common practice suggested by the http standard 1365 * the limit given here is thread-local. If you run multiple http clients/requests from multiple threads, don't set this too high or you might bump into the global limit from the OS. 1366 * setting this too low can waste connections because the server might close them, but they will never be garbage collected since my current implementation won't check for dead connections except when it thinks it is running close to the limit. 1367 ) 1368 1369 Setting it just right for your use case may provide an up to 10x performance boost. 1370 1371 This implementation is subject to change. If it does, I'll document it, but may not bump the version number. 1372 1373 History: 1374 Added August 10, 2022 (dub v10.9) 1375 +/ 1376 static void setConnectionCacheSize(int max = 32) { 1377 connectionCacheSize = max; 1378 } 1379 1380 private static { 1381 // we manage the actual connections. When a request is made on a particular 1382 // host, we try to reuse connections. We may open more than one connection per 1383 // host to do parallel requests. 1384 // 1385 // The key is the *domain name* and the port. Multiple domains on the same address will have separate connections. 1386 Socket[][string] socketsPerHost; 1387 1388 // only one request can be active on a given socket (at least HTTP < 2.0) so this is that 1389 HttpRequest[Socket] activeRequestOnSocket; 1390 HttpRequest[] pending; // and these are the requests that are waiting 1391 1392 int cachedSockets; 1393 int connectionCacheSize = 32; 1394 1395 /+ 1396 This is a somewhat expensive, but essential operation. If it isn't used in a heavy 1397 application, you'll risk running out of file descriptors. 1398 +/ 1399 void cleanOldSockets() { 1400 static struct CloseCandidate { 1401 string key; 1402 Socket socket; 1403 } 1404 1405 CloseCandidate[36] closeCandidates; 1406 int closeCandidatesPosition; 1407 1408 outer: foreach(key, sockets; socketsPerHost) { 1409 foreach(socket; sockets) { 1410 if(socket in activeRequestOnSocket) 1411 continue; // it is still in use; we can't close it 1412 1413 closeCandidates[closeCandidatesPosition++] = CloseCandidate(key, socket); 1414 if(closeCandidatesPosition == closeCandidates.length) 1415 break outer; 1416 } 1417 } 1418 1419 auto cc = closeCandidates[0 .. closeCandidatesPosition]; 1420 1421 if(cc.length == 0) 1422 return; // no candidates to even examine 1423 1424 // has the server closed any of these? if so, we also close and drop them 1425 static SocketSet readSet = null; 1426 if(readSet is null) 1427 readSet = new SocketSet(); 1428 readSet.reset(); 1429 1430 foreach(candidate; cc) { 1431 readSet.add(candidate.socket); 1432 } 1433 1434 int closeCount; 1435 1436 auto got = Socket.select(readSet, null, null, 0.msecs /* timeout, want it small since we just checking for eof */); 1437 if(got > 0) { 1438 foreach(ref candidate; cc) { 1439 if(readSet.isSet(candidate.socket)) { 1440 // if we can read when it isn't in use, that means eof; the 1441 // server closed it. 1442 candidate.socket.close(); 1443 loseSocketByKey(candidate.key, candidate.socket); 1444 closeCount++; 1445 } 1446 } 1447 debug(arsd_http2) writeln(closeCount, " from inactivity"); 1448 } else { 1449 // and if not, of the remaining ones, close a few just at random to bring us back beneath the arbitrary limit. 1450 1451 while(cc.length > 0 && (cachedSockets - closeCount) > connectionCacheSize) { 1452 import std.random; 1453 auto idx = uniform(0, cc.length); 1454 1455 cc[idx].socket.close(); 1456 loseSocketByKey(cc[idx].key, cc[idx].socket); 1457 1458 cc[idx] = cc[$ - 1]; 1459 cc = cc[0 .. $-1]; 1460 closeCount++; 1461 } 1462 debug(arsd_http2) writeln(closeCount, " from randomness"); 1463 } 1464 1465 cachedSockets -= closeCount; 1466 } 1467 1468 void loseSocketByKey(string key, Socket s) { 1469 if(auto list = key in socketsPerHost) { 1470 for(int a = 0; a < (*list).length; a++) { 1471 if((*list)[a] is s) { 1472 1473 for(int b = a; b < (*list).length - 1; b++) 1474 (*list)[b] = (*list)[b+1]; 1475 (*list) = (*list)[0 .. $-1]; 1476 break; 1477 } 1478 } 1479 } 1480 } 1481 1482 void loseSocket(string host, ushort port, bool ssl, Socket s) { 1483 import std.string; 1484 auto key = format("http%s://%s:%s", ssl ? "s" : "", host, port); 1485 1486 loseSocketByKey(key, s); 1487 } 1488 1489 Socket getOpenSocketOnHost(string proxy, string host, ushort port, bool ssl, string unixSocketPath, bool verifyPeer) { 1490 Socket openNewConnection() { 1491 Socket socket; 1492 if(ssl) { 1493 version(with_openssl) { 1494 loadOpenSsl(); 1495 socket = new SslClientSocket(family(unixSocketPath), SocketType.STREAM, host, verifyPeer); 1496 socket.blocking = false; 1497 } else 1498 throw new Exception("SSL not compiled in"); 1499 } else { 1500 socket = new Socket(family(unixSocketPath), SocketType.STREAM); 1501 socket.blocking = false; 1502 } 1503 1504 // FIXME: connect timeout? 1505 if(unixSocketPath) { 1506 import std.stdio; writeln(cast(ubyte[]) unixSocketPath); 1507 socket.connect(new UnixAddress(unixSocketPath)); 1508 } else { 1509 // FIXME: i should prolly do ipv6 if available too. 1510 if(host.length == 0) // this could arguably also be an in contract since it is user error, but the exception is good enough 1511 throw new Exception("No host given for request"); 1512 if(proxy.length) { 1513 if(proxy.indexOf("//") == -1) 1514 proxy = "http://" ~ proxy; 1515 auto proxyurl = Uri(proxy); 1516 1517 //auto proxyhttps = proxyurl.scheme == "https"; 1518 enum proxyhttps = false; // this isn't properly implemented and might never be necessary anyway so meh 1519 1520 // the precise types here are important to help with overload 1521 // resolution of the devirtualized call! 1522 Address pa = new InternetAddress(proxyurl.host, proxyurl.port ? cast(ushort) proxyurl.port : 80); 1523 1524 debug(arsd_http2) writeln("using proxy ", pa.toString()); 1525 1526 if(proxyhttps) { 1527 socket.connect(pa); 1528 } else { 1529 // the proxy never actually starts TLS, but if the request is tls then we need to CONNECT then upgrade the connection 1530 // using the parent class functions let us bypass the encryption 1531 socket.Socket.connect(pa); 1532 } 1533 1534 socket.blocking = true; // FIXME total hack to simplify the code here since it isn't really using the event loop yet 1535 1536 string message; 1537 if(ssl) { 1538 auto hostName = host ~ ":" ~ to!string(port); 1539 message = "CONNECT " ~ hostName ~ " HTTP/1.1\r\n"; 1540 message ~= "Host: " ~ hostName ~ "\r\n"; 1541 if(proxyurl.userinfo.length) { 1542 import std.base64; 1543 message ~= "Proxy-Authorization: Basic " ~ Base64.encode(cast(ubyte[]) proxyurl.userinfo) ~ "\r\n"; 1544 } 1545 message ~= "\r\n"; 1546 1547 // FIXME: what if proxy times out? should be reasonably fast too. 1548 if(proxyhttps) { 1549 socket.send(message, SocketFlags.NONE); 1550 } else { 1551 socket.Socket.send(message, SocketFlags.NONE); 1552 } 1553 1554 ubyte[1024] recvBuffer; 1555 // and last time 1556 ptrdiff_t rcvGot; 1557 if(proxyhttps) { 1558 rcvGot = socket.receive(recvBuffer[], SocketFlags.NONE); 1559 // bool verifyPeer = true; 1560 //(cast(OpenSslSocket)socket).freeSsl(); 1561 //(cast(OpenSslSocket)socket).initSsl(verifyPeer, host); 1562 } else { 1563 rcvGot = socket.Socket.receive(recvBuffer[], SocketFlags.NONE); 1564 } 1565 1566 if(rcvGot == -1) 1567 throw new ProxyException("proxy receive error"); 1568 auto got = cast(string) recvBuffer[0 .. rcvGot]; 1569 auto expect = "HTTP/1.1 200"; 1570 if(got.length < expect.length || (got[0 .. expect.length] != expect && got[0 .. expect.length] != "HTTP/1.0 200")) 1571 throw new ProxyException("Proxy rejected request: " ~ got[0 .. expect.length <= got.length ? expect.length : got.length]); 1572 1573 if(proxyhttps) { 1574 //(cast(OpenSslSocket)socket).do_ssl_connect(); 1575 } else { 1576 (cast(OpenSslSocket)socket).do_ssl_connect(); 1577 } 1578 } else { 1579 } 1580 } else { 1581 socket.connect(new InternetAddress(host, port)); 1582 } 1583 } 1584 1585 debug(arsd_http2) writeln("opening to ", host, ":", port, " ", cast(void*) socket, " ssl=", ssl); 1586 assert(socket.handle() !is socket_t.init); 1587 return socket; 1588 } 1589 1590 // import std.stdio; writeln(cachedSockets); 1591 if(cachedSockets > connectionCacheSize) 1592 cleanOldSockets(); 1593 1594 import std.string; 1595 auto key = format("http%s://%s:%s", ssl ? "s" : "", host, port); 1596 1597 if(auto hostListing = key in socketsPerHost) { 1598 // try to find an available socket that is already open 1599 foreach(socket; *hostListing) { 1600 if(socket !in activeRequestOnSocket) { 1601 // let's see if it has closed since we last tried 1602 // e.g. a server timeout or something. If so, we need 1603 // to lose this one and immediately open a new one. 1604 static SocketSet readSet = null; 1605 if(readSet is null) 1606 readSet = new SocketSet(); 1607 readSet.reset(); 1608 assert(socket !is null); 1609 assert(socket.handle() !is socket_t.init, socket is null ? "null" : socket.toString()); 1610 readSet.add(socket); 1611 auto got = Socket.select(readSet, null, null, 0.msecs /* timeout, want it small since we just checking for eof */); 1612 if(got > 0) { 1613 // we can read something off this... but there aren't 1614 // any active requests. Assume it is EOF and open a new one 1615 1616 socket.close(); 1617 loseSocket(host, port, ssl, socket); 1618 goto openNew; 1619 } 1620 cachedSockets--; 1621 return socket; 1622 } 1623 } 1624 1625 // if not too many already open, go ahead and do a new one 1626 if((*hostListing).length < 6) { 1627 auto socket = openNewConnection(); 1628 (*hostListing) ~= socket; 1629 return socket; 1630 } else 1631 return null; // too many, you'll have to wait 1632 } 1633 1634 openNew: 1635 1636 auto socket = openNewConnection(); 1637 socketsPerHost[key] ~= socket; 1638 return socket; 1639 } 1640 1641 SocketSet readSet; 1642 SocketSet writeSet; 1643 1644 /+ 1645 Generic event loop registration: 1646 1647 handle, operation (read/write), buffer (on posix it *might* be stack if a select loop), timeout (in real time), callback when op completed. 1648 1649 ....basically Windows style. Then it translates internally. 1650 1651 It should tell the thing if the buffer is reused or not 1652 +/ 1653 1654 1655 /++ 1656 This is made public for rudimentary event loop integration, but is still 1657 basically an internal detail. Try not to use it if you have another way. 1658 1659 This does a single iteration of the internal select()-based processing loop. 1660 1661 1662 Future directions: 1663 I want to merge the internal use of [WebSocket.eventLoop] with this; 1664 [advanceConnections] does just one run on the loop, whereas eventLoop 1665 runs it until all connections are closed. But they'd both process both 1666 pending http requests and active websockets. 1667 1668 After that, I want to be able to integrate in other event loops too. 1669 One might be to simply to reactor callbacks, then perhaps Windows overlapped 1670 i/o (that's just going to be tricky to retrofit into the existing select()-based 1671 code). It could then go fiber just by calling the resume function too. 1672 1673 The hard part is ensuring I keep this file stand-alone while offering these 1674 things. 1675 1676 This `advanceConnections` call will probably continue to work now that it is 1677 public, but it may not be wholly compatible with all the future features; you'd 1678 have to pick either the internal event loop or an external one you integrate, but not 1679 mix them. 1680 1681 History: 1682 This has been included in the library since almost day one, but 1683 it was private until April 13, 2021 (dub v9.5). 1684 1685 Params: 1686 maximumTimeout = the maximum time it will wait in select(). It may return much sooner than this if a connection timed out in the mean time. 1687 automaticallyRetryOnInterruption = internally loop on EINTR. 1688 1689 Returns: 1690 1691 0 = no error, work may remain so you should call `advanceConnections` again when you can 1692 1693 1 = passed `maximumTimeout` reached with no work done, yet requests are still in the queue. You may call `advanceConnections` again. 1694 1695 2 = no work to do, no point calling it again unless you've added new requests. Your program may exit if you have nothing to add since it means everything requested is now done. 1696 1697 3 = EINTR occurred on select(), you should check your interrupt flags if you set a signal handler, then call `advanceConnections` again if you aren't exiting. Only occurs if `automaticallyRetryOnInterruption` is set to `false` (the default when it is called externally). 1698 1699 any other value should be considered a non-recoverable error if you want to be forward compatible as I reserve the right to add more values later. 1700 +/ 1701 public int advanceConnections(Duration maximumTimeout = 10.seconds, bool automaticallyRetryOnInterruption = false) { 1702 debug(arsd_http2_verbose) writeln("advancing"); 1703 if(readSet is null) 1704 readSet = new SocketSet(); 1705 if(writeSet is null) 1706 writeSet = new SocketSet(); 1707 1708 ubyte[2048] buffer; 1709 1710 HttpRequest[16] removeFromPending; 1711 size_t removeFromPendingCount = 0; 1712 1713 bool hadAbortedRequest; 1714 1715 // are there pending requests? let's try to send them 1716 foreach(idx, pc; pending) { 1717 if(removeFromPendingCount == removeFromPending.length) 1718 break; 1719 1720 if(pc.state == HttpRequest.State.aborted) { 1721 removeFromPending[removeFromPendingCount++] = pc; 1722 hadAbortedRequest = true; 1723 continue; 1724 } 1725 1726 Socket socket; 1727 1728 try { 1729 socket = getOpenSocketOnHost(pc.proxy, pc.requestParameters.host, pc.requestParameters.port, pc.requestParameters.ssl, pc.requestParameters.unixSocketPath, pc.verifyPeer); 1730 } catch(ProxyException e) { 1731 // connection refused or timed out (I should disambiguate somehow)... 1732 pc.state = HttpRequest.State.aborted; 1733 1734 pc.responseData.code = 2; 1735 pc.responseData.codeText = e.msg ~ " from " ~ pc.proxy; 1736 1737 hadAbortedRequest = true; 1738 1739 removeFromPending[removeFromPendingCount++] = pc; 1740 continue; 1741 1742 } catch(SocketException e) { 1743 // connection refused or timed out (I should disambiguate somehow)... 1744 pc.state = HttpRequest.State.aborted; 1745 1746 pc.responseData.code = 2; 1747 pc.responseData.codeText = pc.proxy.length ? ("connection failed to proxy " ~ pc.proxy) : "connection failed"; 1748 1749 hadAbortedRequest = true; 1750 1751 removeFromPending[removeFromPendingCount++] = pc; 1752 continue; 1753 } catch(Exception e) { 1754 // connection failed due to other user error or SSL (i should disambiguate somehow)... 1755 pc.state = HttpRequest.State.aborted; 1756 1757 pc.responseData.code = 2; 1758 pc.responseData.codeText = e.msg; 1759 1760 hadAbortedRequest = true; 1761 1762 removeFromPending[removeFromPendingCount++] = pc; 1763 continue; 1764 1765 } 1766 1767 if(socket !is null) { 1768 activeRequestOnSocket[socket] = pc; 1769 assert(pc.sendBuffer.length); 1770 pc.state = State.connecting; 1771 1772 removeFromPending[removeFromPendingCount++] = pc; 1773 } 1774 } 1775 1776 import std.algorithm : remove; 1777 foreach(rp; removeFromPending[0 .. removeFromPendingCount]) 1778 pending = pending.remove!((a) => a is rp)(); 1779 1780 tryAgain: 1781 1782 Socket[16] inactive; 1783 int inactiveCount = 0; 1784 void killInactives() { 1785 foreach(s; inactive[0 .. inactiveCount]) { 1786 debug(arsd_http2) writeln("removing socket from active list ", cast(void*) s); 1787 activeRequestOnSocket.remove(s); 1788 cachedSockets++; 1789 } 1790 } 1791 1792 1793 readSet.reset(); 1794 writeSet.reset(); 1795 1796 bool hadOne = false; 1797 1798 auto minTimeout = maximumTimeout; 1799 auto now = MonoTime.currTime; 1800 1801 // active requests need to be read or written to 1802 foreach(sock, request; activeRequestOnSocket) { 1803 1804 if(request.state == State.aborted) { 1805 inactive[inactiveCount++] = sock; 1806 sock.close(); 1807 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1808 hadAbortedRequest = true; 1809 continue; 1810 } 1811 1812 // check the other sockets just for EOF, if they close, take them out of our list, 1813 // we'll reopen if needed upon request. 1814 readSet.add(sock); 1815 hadOne = true; 1816 1817 Duration timeo; 1818 if(request.timeoutFromInactivity <= now) 1819 timeo = 0.seconds; 1820 else 1821 timeo = request.timeoutFromInactivity - now; 1822 1823 if(timeo < minTimeout) 1824 minTimeout = timeo; 1825 1826 if(request.state == State.connecting || request.state == State.sslConnectPendingWrite || request.state == State.sendingHeaders || request.state == State.sendingBody) { 1827 writeSet.add(sock); 1828 hadOne = true; 1829 } 1830 } 1831 1832 if(!hadOne) { 1833 if(hadAbortedRequest) { 1834 killInactives(); 1835 return 0; // something got aborted, that's progress 1836 } 1837 return 2; // automatic timeout, nothing to do 1838 } 1839 1840 auto selectGot = Socket.select(readSet, writeSet, null, minTimeout); 1841 if(selectGot == 0) { /* timeout */ 1842 now = MonoTime.currTime; 1843 bool anyWorkDone = false; 1844 foreach(sock, request; activeRequestOnSocket) { 1845 1846 if(request.timeoutFromInactivity <= now) { 1847 request.state = HttpRequest.State.aborted; 1848 request.responseData.code = 5; 1849 if(request.state == State.connecting) 1850 request.responseData.codeText = "Connect timed out"; 1851 else 1852 request.responseData.codeText = "Request timed out"; 1853 1854 inactive[inactiveCount++] = sock; 1855 sock.close(); 1856 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1857 anyWorkDone = true; 1858 } 1859 } 1860 killInactives(); 1861 return anyWorkDone ? 0 : 1; 1862 // return 1; was an error to time out but now im making it on the individual request 1863 } else if(selectGot == -1) { /* interrupted */ 1864 /* 1865 version(Posix) { 1866 import core.stdc.errno; 1867 if(errno != EINTR) 1868 throw new Exception("select error: " ~ to!string(errno)); 1869 } 1870 */ 1871 if(automaticallyRetryOnInterruption) 1872 goto tryAgain; 1873 else 1874 return 3; 1875 } else { /* ready */ 1876 1877 void sslProceed(HttpRequest request, SslClientSocket s) { 1878 try { 1879 auto code = s.do_ssl_connect(); 1880 switch(code) { 1881 case 0: 1882 request.state = State.sendingHeaders; 1883 break; 1884 case SSL_ERROR_WANT_READ: 1885 request.state = State.sslConnectPendingRead; 1886 break; 1887 case SSL_ERROR_WANT_WRITE: 1888 request.state = State.sslConnectPendingWrite; 1889 break; 1890 default: 1891 assert(0); 1892 } 1893 } catch(Exception e) { 1894 request.state = State.aborted; 1895 1896 request.responseData.code = 2; 1897 request.responseData.codeText = e.msg; 1898 inactive[inactiveCount++] = s; 1899 s.close(); 1900 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, s); 1901 } 1902 } 1903 1904 1905 foreach(sock, request; activeRequestOnSocket) { 1906 // always need to try to send first in part because http works that way but 1907 // also because openssl will sometimes leave something ready to read even if we haven't 1908 // sent yet (probably leftover data from the crypto negotiation) and if that happens ssl 1909 // is liable to block forever hogging the connection and not letting it send... 1910 if(request.state == State.connecting) 1911 if(writeSet.isSet(sock) || readSet.isSet(sock)) { 1912 import core.stdc.stdint; 1913 int32_t error; 1914 int retopt = sock.getOption(SocketOptionLevel.SOCKET, SocketOption.ERROR, error); 1915 if(retopt < 0 || error != 0) { 1916 request.state = State.aborted; 1917 1918 request.responseData.code = 2; 1919 try { 1920 request.responseData.codeText = "connection failed - " ~ formatSocketError(error); 1921 } catch(Exception e) { 1922 request.responseData.codeText = "connection failed"; 1923 } 1924 inactive[inactiveCount++] = sock; 1925 sock.close(); 1926 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1927 continue; 1928 } else { 1929 if(auto s = cast(SslClientSocket) sock) { 1930 sslProceed(request, s); 1931 continue; 1932 } else { 1933 request.state = State.sendingHeaders; 1934 } 1935 } 1936 } 1937 1938 if(request.state == State.sslConnectPendingRead) 1939 if(readSet.isSet(sock)) { 1940 sslProceed(request, cast(SslClientSocket) sock); 1941 continue; 1942 } 1943 if(request.state == State.sslConnectPendingWrite) 1944 if(writeSet.isSet(sock)) { 1945 sslProceed(request, cast(SslClientSocket) sock); 1946 continue; 1947 } 1948 1949 if(request.state == State.sendingHeaders || request.state == State.sendingBody) 1950 if(writeSet.isSet(sock)) { 1951 request.timeoutFromInactivity = MonoTime.currTime + request.requestParameters.timeoutFromInactivity; 1952 assert(request.sendBuffer.length); 1953 auto sent = sock.send(request.sendBuffer); 1954 debug(arsd_http2_verbose) writeln(cast(void*) sock, "<send>", cast(string) request.sendBuffer, "</send>"); 1955 if(sent <= 0) { 1956 if(wouldHaveBlocked()) 1957 continue; 1958 1959 request.state = State.aborted; 1960 1961 request.responseData.code = 3; 1962 request.responseData.codeText = "send failed to server"; 1963 inactive[inactiveCount++] = sock; 1964 sock.close(); 1965 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1966 continue; 1967 1968 } 1969 request.sendBuffer = request.sendBuffer[sent .. $]; 1970 if(request.sendBuffer.length == 0) { 1971 request.state = State.waitingForResponse; 1972 1973 debug(arsd_http2_verbose) writeln("all sent"); 1974 } 1975 } 1976 1977 1978 if(readSet.isSet(sock)) { 1979 keep_going: 1980 request.timeoutFromInactivity = MonoTime.currTime + request.requestParameters.timeoutFromInactivity; 1981 auto got = sock.receive(buffer); 1982 debug(arsd_http2_verbose) { if(got < 0) writeln(lastSocketError); else writeln("====PACKET ",got,"=====",cast(string)buffer[0 .. got],"===/PACKET==="); } 1983 if(got < 0) { 1984 if(wouldHaveBlocked()) 1985 continue; 1986 debug(arsd_http2) writeln("receive error"); 1987 if(request.state != State.complete) { 1988 request.state = State.aborted; 1989 1990 request.responseData.code = 3; 1991 request.responseData.codeText = "receive error from server"; 1992 } 1993 inactive[inactiveCount++] = sock; 1994 sock.close(); 1995 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1996 } else if(got == 0) { 1997 // remote side disconnected 1998 debug(arsd_http2) writeln("remote disconnect"); 1999 if(request.state != State.complete) { 2000 request.state = State.aborted; 2001 2002 request.responseData.code = 3; 2003 request.responseData.codeText = "server disconnected"; 2004 } 2005 inactive[inactiveCount++] = sock; 2006 sock.close(); 2007 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 2008 } else { 2009 // data available 2010 bool stillAlive; 2011 2012 try { 2013 stillAlive = request.handleIncomingData(buffer[0 .. got]); 2014 /+ 2015 state needs to be set and public 2016 requestData.content/contentText needs to be around 2017 you need to be able to clear the content and keep processing for things like event sources. 2018 also need to be able to abort it. 2019 2020 and btw it should prolly just have evnet source as a pre-packaged thing. 2021 +/ 2022 } catch (Exception e) { 2023 debug(arsd_http2_verbose) { import std.stdio; writeln(e); } 2024 request.state = HttpRequest.State.aborted; 2025 request.responseData.code = 4; 2026 request.responseData.codeText = e.msg; 2027 2028 inactive[inactiveCount++] = sock; 2029 sock.close(); 2030 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 2031 continue; 2032 } 2033 2034 if(!stillAlive || request.state == HttpRequest.State.complete || request.state == HttpRequest.State.aborted) { 2035 //import std.stdio; writeln(cast(void*) sock, " ", stillAlive, " ", request.state); 2036 inactive[inactiveCount++] = sock; 2037 continue; 2038 // reuse the socket for another pending request, if we can 2039 } 2040 } 2041 2042 if(request.onDataReceived) 2043 request.onDataReceived(request); 2044 2045 version(with_openssl) 2046 if(auto s = cast(SslClientSocket) sock) { 2047 // select doesn't handle the case with stuff 2048 // left in the ssl buffer so i'm checking it separately 2049 if(s.dataPending()) { 2050 goto keep_going; 2051 } 2052 } 2053 } 2054 } 2055 } 2056 2057 killInactives(); 2058 2059 // we've completed a request, are there any more pending connection? if so, send them now 2060 2061 return 0; 2062 } 2063 } 2064 2065 public static void resetInternals() { 2066 socketsPerHost = null; 2067 activeRequestOnSocket = null; 2068 pending = null; 2069 2070 } 2071 2072 struct HeaderReadingState { 2073 bool justSawLf; 2074 bool justSawCr; 2075 bool atStartOfLine = true; 2076 bool readingLineContinuation; 2077 } 2078 HeaderReadingState headerReadingState; 2079 2080 struct BodyReadingState { 2081 bool isGzipped; 2082 bool isDeflated; 2083 2084 bool isChunked; 2085 int chunkedState; 2086 2087 // used for the chunk size if it is chunked 2088 int contentLengthRemaining; 2089 } 2090 BodyReadingState bodyReadingState; 2091 2092 bool closeSocketWhenComplete; 2093 2094 import std.zlib; 2095 UnCompress uncompress; 2096 2097 const(ubyte)[] leftoverDataFromLastTime; 2098 2099 bool handleIncomingData(scope const ubyte[] dataIn) { 2100 bool stillAlive = true; 2101 debug(arsd_http2) writeln("handleIncomingData, state: ", state); 2102 if(state == State.waitingForResponse) { 2103 state = State.readingHeaders; 2104 headerReadingState = HeaderReadingState.init; 2105 bodyReadingState = BodyReadingState.init; 2106 } 2107 2108 const(ubyte)[] data; 2109 if(leftoverDataFromLastTime.length) 2110 data = leftoverDataFromLastTime ~ dataIn[]; 2111 else 2112 data = dataIn[]; 2113 2114 if(state == State.readingHeaders) { 2115 void parseLastHeader() { 2116 assert(responseData.headers.length); 2117 if(responseData.headers.length == 1) { 2118 responseData.statusLine = responseData.headers[0]; 2119 import std.algorithm; 2120 auto parts = responseData.statusLine.splitter(" "); 2121 responseData.httpVersion = parts.front; 2122 parts.popFront(); 2123 if(parts.empty) 2124 throw new Exception("Corrupted response, bad status line"); 2125 responseData.code = to!int(parts.front()); 2126 parts.popFront(); 2127 responseData.codeText = ""; 2128 while(!parts.empty) { 2129 // FIXME: this sucks! 2130 responseData.codeText ~= parts.front(); 2131 parts.popFront(); 2132 if(!parts.empty) 2133 responseData.codeText ~= " "; 2134 } 2135 } else { 2136 // parse the new header 2137 auto header = responseData.headers[$-1]; 2138 2139 auto colon = header.indexOf(":"); 2140 if(colon < 0 || colon >= header.length) 2141 return; 2142 auto name = toLower(header[0 .. colon]); 2143 auto value = header[colon + 1 .. $].strip; // skip colon and strip whitespace 2144 2145 switch(name) { 2146 case "connection": 2147 if(value == "close") 2148 closeSocketWhenComplete = true; 2149 break; 2150 case "content-type": 2151 responseData.contentType = value; 2152 break; 2153 case "location": 2154 responseData.location = value; 2155 break; 2156 case "content-length": 2157 bodyReadingState.contentLengthRemaining = to!int(value); 2158 break; 2159 case "transfer-encoding": 2160 // note that if it is gzipped, it zips first, then chunks the compressed stream. 2161 // so we should always dechunk first, then feed into the decompressor 2162 if(value == "chunked") 2163 bodyReadingState.isChunked = true; 2164 else throw new Exception("Unknown Transfer-Encoding: " ~ value); 2165 break; 2166 case "content-encoding": 2167 if(value == "gzip") { 2168 bodyReadingState.isGzipped = true; 2169 uncompress = new UnCompress(); 2170 } else if(value == "deflate") { 2171 bodyReadingState.isDeflated = true; 2172 uncompress = new UnCompress(); 2173 } else throw new Exception("Unknown Content-Encoding: " ~ value); 2174 break; 2175 case "set-cookie": 2176 // handled elsewhere fyi 2177 break; 2178 default: 2179 // ignore 2180 } 2181 2182 responseData.headersHash[name] = value; 2183 } 2184 } 2185 2186 size_t position = 0; 2187 for(position = 0; position < data.length; position++) { 2188 if(headerReadingState.readingLineContinuation) { 2189 if(data[position] == ' ' || data[position] == '\t') 2190 continue; 2191 headerReadingState.readingLineContinuation = false; 2192 } 2193 2194 if(headerReadingState.atStartOfLine) { 2195 headerReadingState.atStartOfLine = false; 2196 // FIXME it being \r should never happen... and i don't think it does 2197 if(data[position] == '\r' || data[position] == '\n') { 2198 // done with headers 2199 2200 position++; // skip the \r 2201 2202 if(responseData.headers.length) 2203 parseLastHeader(); 2204 2205 if(responseData.code >= 100 && responseData.code < 200) { 2206 // "100 Continue" - we should continue uploading request data at this point 2207 // "101 Switching Protocols" - websocket, not expected here... 2208 // "102 Processing" - server still working, keep the connection alive 2209 // "103 Early Hints" - can have useful Link headers etc 2210 // 2211 // and other unrecognized ones can just safely be skipped 2212 2213 // FIXME: the headers shouldn't actually be reset; 103 Early Hints 2214 // can give useful headers we want to keep 2215 2216 responseData.headers = null; 2217 headerReadingState.atStartOfLine = true; 2218 2219 continue; // the \n will be skipped by the for loop advance 2220 } 2221 2222 if(this.requestParameters.method == HttpVerb.HEAD) 2223 state = State.complete; 2224 else 2225 state = State.readingBody; 2226 2227 // skip the \n before we break 2228 position++; 2229 2230 break; 2231 } else if(data[position] == ' ' || data[position] == '\t') { 2232 // line continuation, ignore all whitespace and collapse it into a space 2233 headerReadingState.readingLineContinuation = true; 2234 responseData.headers[$-1] ~= ' '; 2235 } else { 2236 // new header 2237 if(responseData.headers.length) 2238 parseLastHeader(); 2239 responseData.headers ~= ""; 2240 } 2241 } 2242 2243 if(data[position] == '\r') { 2244 headerReadingState.justSawCr = true; 2245 continue; 2246 } else 2247 headerReadingState.justSawCr = false; 2248 2249 if(data[position] == '\n') { 2250 headerReadingState.justSawLf = true; 2251 headerReadingState.atStartOfLine = true; 2252 continue; 2253 } else 2254 headerReadingState.justSawLf = false; 2255 2256 responseData.headers[$-1] ~= data[position]; 2257 } 2258 2259 data = data[position .. $]; 2260 } 2261 2262 if(state == State.readingBody) { 2263 if(bodyReadingState.isChunked) { 2264 // read the hex length, stopping at a \r\n, ignoring everything between the new line but after the first non-valid hex character 2265 // read binary data of that length. it is our content 2266 // repeat until a zero sized chunk 2267 // then read footers as headers. 2268 2269 start_over: 2270 for(int a = 0; a < data.length; a++) { 2271 final switch(bodyReadingState.chunkedState) { 2272 case 0: // reading hex 2273 char c = data[a]; 2274 if((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) { 2275 // just keep reading 2276 } else { 2277 int power = 1; 2278 bodyReadingState.contentLengthRemaining = 0; 2279 if(a == 0) 2280 break; // just wait for more data 2281 assert(a != 0, cast(string) data); 2282 for(int b = a-1; b >= 0; b--) { 2283 char cc = data[b]; 2284 if(cc >= 'a' && cc <= 'z') 2285 cc -= 0x20; 2286 int val = 0; 2287 if(cc >= '0' && cc <= '9') 2288 val = cc - '0'; 2289 else 2290 val = cc - 'A' + 10; 2291 2292 assert(val >= 0 && val <= 15, to!string(val)); 2293 bodyReadingState.contentLengthRemaining += power * val; 2294 power *= 16; 2295 } 2296 debug(arsd_http2_verbose) writeln("Chunk length: ", bodyReadingState.contentLengthRemaining); 2297 bodyReadingState.chunkedState = 1; 2298 data = data[a + 1 .. $]; 2299 goto start_over; 2300 } 2301 break; 2302 case 1: // reading until end of line 2303 char c = data[a]; 2304 if(c == '\n') { 2305 if(bodyReadingState.contentLengthRemaining == 0) 2306 bodyReadingState.chunkedState = 5; 2307 else 2308 bodyReadingState.chunkedState = 2; 2309 } 2310 data = data[a + 1 .. $]; 2311 goto start_over; 2312 case 2: // reading data 2313 auto can = a + bodyReadingState.contentLengthRemaining; 2314 if(can > data.length) 2315 can = cast(int) data.length; 2316 2317 auto newData = data[a .. can]; 2318 data = data[can .. $]; 2319 2320 //if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) 2321 // responseData.content ~= cast(ubyte[]) uncompress.uncompress(data[a .. can]); 2322 //else 2323 responseData.content ~= newData; 2324 2325 bodyReadingState.contentLengthRemaining -= newData.length; 2326 debug(arsd_http2_verbose) writeln("clr: ", bodyReadingState.contentLengthRemaining, " " , a, " ", can); 2327 assert(bodyReadingState.contentLengthRemaining >= 0); 2328 if(bodyReadingState.contentLengthRemaining == 0) { 2329 bodyReadingState.chunkedState = 3; 2330 } else { 2331 // will continue grabbing more 2332 } 2333 goto start_over; 2334 case 3: // reading 13/10 2335 assert(data[a] == 13); 2336 bodyReadingState.chunkedState++; 2337 data = data[a + 1 .. $]; 2338 goto start_over; 2339 case 4: // reading 10 at end of packet 2340 assert(data[a] == 10); 2341 data = data[a + 1 .. $]; 2342 bodyReadingState.chunkedState = 0; 2343 goto start_over; 2344 case 5: // reading footers 2345 //goto done; // FIXME 2346 2347 int footerReadingState = 0; 2348 int footerSize; 2349 2350 while(footerReadingState != 2 && a < data.length) { 2351 // import std.stdio; writeln(footerReadingState, " ", footerSize, " ", data); 2352 switch(footerReadingState) { 2353 case 0: 2354 if(data[a] == 13) 2355 footerReadingState++; 2356 else 2357 footerSize++; 2358 break; 2359 case 1: 2360 if(data[a] == 10) { 2361 if(footerSize == 0) { 2362 // all done, time to break 2363 footerReadingState++; 2364 2365 } else { 2366 // actually had a footer, try to read another 2367 footerReadingState = 0; 2368 footerSize = 0; 2369 } 2370 } else { 2371 throw new Exception("bad footer thing"); 2372 } 2373 break; 2374 default: 2375 assert(0); 2376 } 2377 2378 a++; 2379 } 2380 2381 if(footerReadingState != 2) 2382 break start_over; // haven't hit the end of the thing yet 2383 2384 bodyReadingState.chunkedState = 0; 2385 data = data[a .. $]; 2386 2387 if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) { 2388 auto n = uncompress.uncompress(responseData.content); 2389 n ~= uncompress.flush(); 2390 responseData.content = cast(ubyte[]) n; 2391 } 2392 2393 // responseData.content ~= cast(ubyte[]) uncompress.flush(); 2394 responseData.contentText = cast(string) responseData.content; 2395 2396 goto done; 2397 } 2398 } 2399 2400 } else { 2401 //if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) 2402 // responseData.content ~= cast(ubyte[]) uncompress.uncompress(data); 2403 //else 2404 responseData.content ~= data; 2405 //assert(data.length <= bodyReadingState.contentLengthRemaining, format("%d <= %d\n%s", data.length, bodyReadingState.contentLengthRemaining, cast(string)data)); 2406 { 2407 int use = cast(int) data.length; 2408 if(use > bodyReadingState.contentLengthRemaining) 2409 use = bodyReadingState.contentLengthRemaining; 2410 bodyReadingState.contentLengthRemaining -= use; 2411 data = data[use .. $]; 2412 } 2413 if(bodyReadingState.contentLengthRemaining == 0) { 2414 if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) { 2415 // import std.stdio; writeln(responseData.content.length, " ", responseData.content[0 .. 2], " .. ", responseData.content[$-2 .. $]); 2416 auto n = uncompress.uncompress(responseData.content); 2417 n ~= uncompress.flush(); 2418 responseData.content = cast(ubyte[]) n; 2419 responseData.contentText = cast(string) responseData.content; 2420 //responseData.content ~= cast(ubyte[]) uncompress.flush(); 2421 } else { 2422 responseData.contentText = cast(string) responseData.content; 2423 } 2424 2425 done: 2426 2427 if(retainCookies && client !is null) { 2428 client.retainCookies(responseData); 2429 } 2430 2431 if(followLocation && responseData.location.length) { 2432 if(maximumNumberOfRedirectsRemaining <= 0) { 2433 throw new Exception("Maximum number of redirects exceeded"); 2434 } else { 2435 maximumNumberOfRedirectsRemaining--; 2436 } 2437 2438 static bool first = true; 2439 //version(DigitalMars) if(!first) asm { int 3; } 2440 debug(arsd_http2) writeln("redirecting to ", responseData.location); 2441 populateFromInfo(Uri(responseData.location), HttpVerb.GET); 2442 //import std.stdio; writeln("redirected to ", responseData.location); 2443 first = false; 2444 responseData = HttpResponse.init; 2445 headerReadingState = HeaderReadingState.init; 2446 bodyReadingState = BodyReadingState.init; 2447 if(client !is null) { 2448 // FIXME: this won't clear cookies that were cleared in another request 2449 client.populateCookies(this); // they might have changed in the previous redirection cycle! 2450 } 2451 state = State.unsent; 2452 stillAlive = false; 2453 sendPrivate(false); 2454 } else { 2455 state = State.complete; 2456 // FIXME 2457 //if(closeSocketWhenComplete) 2458 //socket.close(); 2459 } 2460 } 2461 } 2462 } 2463 2464 if(data.length) 2465 leftoverDataFromLastTime = data.dup; 2466 else 2467 leftoverDataFromLastTime = null; 2468 2469 return stillAlive; 2470 } 2471 2472 } 2473 } 2474 2475 /++ 2476 Waits for the first of the given requests to be either aborted or completed. 2477 Returns the first one in that state, or `null` if the operation was interrupted 2478 or reached the given timeout before any completed. (If it returns null even before 2479 the timeout, it might be because the user pressed ctrl+c, so you should consider 2480 checking if you should cancel the operation. If not, you can simply call it again 2481 with the same arguments to start waiting again.) 2482 2483 You MUST check for null, even if you don't specify a timeout! 2484 2485 Note that if an individual request times out before any others request, it will 2486 return that timed out request, since that counts as completion. 2487 2488 If the return is not null, you should call `waitForCompletion` on the given request 2489 to get the response out. It will not have to wait since it is guaranteed to be 2490 finished when returned by this function; that will just give you the cached response. 2491 2492 (I thought about just having it return the response, but tying a response back to 2493 a request is harder than just getting the original request object back and taking 2494 the response out of it.) 2495 2496 Please note: if a request in the set has already completed or been aborted, it will 2497 always return the first one it sees upon calling the function. You may wish to remove 2498 them from the list before calling the function. 2499 2500 History: 2501 Added December 24, 2021 (dub v10.5) 2502 +/ 2503 HttpRequest waitForFirstToComplete(Duration timeout, HttpRequest[] requests...) { 2504 2505 foreach(request; requests) { 2506 if(request.state == HttpRequest.State.unsent) 2507 request.send(); 2508 else if(request.state == HttpRequest.State.complete) 2509 return request; 2510 else if(request.state == HttpRequest.State.aborted) 2511 return request; 2512 } 2513 2514 while(true) { 2515 if(auto err = HttpRequest.advanceConnections(timeout)) { 2516 switch(err) { 2517 case 1: return null; 2518 case 2: throw new Exception("HttpRequest.advanceConnections returned 2: nothing to do"); 2519 case 3: return null; 2520 default: throw new Exception("HttpRequest.advanceConnections got err " ~ to!string(err)); 2521 } 2522 } 2523 2524 foreach(request; requests) { 2525 if(request.state == HttpRequest.State.aborted || request.state == HttpRequest.State.complete) { 2526 request.waitForCompletion(); 2527 return request; 2528 } 2529 } 2530 2531 } 2532 } 2533 2534 /// ditto 2535 HttpRequest waitForFirstToComplete(HttpRequest[] requests...) { 2536 return waitForFirstToComplete(1.weeks, requests); 2537 } 2538 2539 /++ 2540 An input range that runs [waitForFirstToComplete] but only returning each request once. 2541 Before you loop over it, you can set some properties to customize behavior. 2542 2543 If it times out or is interrupted, it will prematurely run empty. You can set the delegate 2544 to process this. 2545 2546 Implementation note: each iteration through the loop does a O(n) check over each item remaining. 2547 This shouldn't matter, but if it does become an issue for you, let me know. 2548 2549 History: 2550 Added December 24, 2021 (dub v10.5) 2551 +/ 2552 struct HttpRequestsAsTheyComplete { 2553 /++ 2554 Seeds it with an overall timeout and the initial requests. 2555 It will send all the requests before returning, then will process 2556 the responses as they come. 2557 2558 Please note that it modifies the array of requests you pass in! It 2559 will keep a reference to it and reorder items on each call of popFront. 2560 You might want to pass a duplicate if you have another purpose for your 2561 array and don't want to see it shuffled. 2562 +/ 2563 this(Duration timeout, HttpRequest[] requests) { 2564 remainingRequests = requests; 2565 this.timeout = timeout; 2566 popFront(); 2567 } 2568 2569 /++ 2570 You can set this delegate to decide how to handle an interruption. Returning true 2571 from this will keep working. Returning false will terminate the loop. 2572 2573 If this is null, an interruption will always terminate the loop. 2574 2575 Note that interruptions can be caused by the garbage collector being triggered by 2576 another thread as well as by user action. If you don't set a SIGINT handler, it 2577 might be reasonable to always return true here. 2578 +/ 2579 bool delegate() onInterruption; 2580 2581 private HttpRequest[] remainingRequests; 2582 2583 /// The timeout you set in the constructor. You can change it if you want. 2584 Duration timeout; 2585 2586 /++ 2587 Adds another request to the work queue. It is safe to call this from inside the loop 2588 as you process other requests. 2589 +/ 2590 void appendRequest(HttpRequest request) { 2591 remainingRequests ~= request; 2592 } 2593 2594 /++ 2595 If the loop exited, it might be due to an interruption or a time out. If you like, you 2596 can call this to pick up the work again, 2597 2598 If it returns `false`, the work is indeed all finished and you should not re-enter the loop. 2599 2600 --- 2601 auto range = HttpRequestsAsTheyComplete(10.seconds, your_requests); 2602 process_loop: foreach(req; range) { 2603 // process req 2604 } 2605 // make sure we weren't interrupted because the user requested we cancel! 2606 // but then try to re-enter the range if possible 2607 if(!user_quit && range.reenter()) { 2608 // there's still something unprocessed in there 2609 // range.reenter returning true means it is no longer 2610 // empty, so we should try to loop over it again 2611 goto process_loop; // re-enter the loop 2612 } 2613 --- 2614 +/ 2615 bool reenter() { 2616 if(remainingRequests.length == 0) 2617 return false; 2618 empty = false; 2619 popFront(); 2620 return true; 2621 } 2622 2623 /// Standard range primitives. I reserve the right to change the variables to read-only properties in the future without notice. 2624 HttpRequest front; 2625 2626 /// ditto 2627 bool empty; 2628 2629 /// ditto 2630 void popFront() { 2631 resume: 2632 if(remainingRequests.length == 0) { 2633 empty = true; 2634 return; 2635 } 2636 2637 front = waitForFirstToComplete(timeout, remainingRequests); 2638 2639 if(front is null) { 2640 if(onInterruption) { 2641 if(onInterruption()) 2642 goto resume; 2643 } 2644 empty = true; 2645 return; 2646 } 2647 foreach(idx, req; remainingRequests) { 2648 if(req is front) { 2649 remainingRequests[idx] = remainingRequests[$ - 1]; 2650 remainingRequests = remainingRequests[0 .. $ - 1]; 2651 return; 2652 } 2653 } 2654 } 2655 } 2656 2657 // 2658 struct HttpRequestParameters { 2659 // FIXME: implement these 2660 //Duration timeoutTotal; // the whole request must finish in this time or else it fails,even if data is still trickling in 2661 Duration timeoutFromInactivity; // if there's no activity in this time it dies. basically the socket receive timeout 2662 2663 // debugging 2664 bool useHttp11 = true; /// 2665 bool acceptGzip = true; /// 2666 bool keepAlive = true; /// 2667 2668 // the request itself 2669 HttpVerb method; /// 2670 string host; /// 2671 ushort port; /// 2672 string uri; /// 2673 2674 bool ssl; /// 2675 2676 string userAgent; /// 2677 string authorization; /// 2678 2679 string[string] cookies; /// 2680 2681 string[] headers; /// do not duplicate host, content-length, content-type, or any others that have a specific property 2682 2683 string contentType; /// 2684 ubyte[] bodyData; /// 2685 2686 string unixSocketPath; /// 2687 } 2688 2689 interface IHttpClient { 2690 2691 } 2692 2693 /// 2694 enum HttpVerb { 2695 /// 2696 GET, 2697 /// 2698 HEAD, 2699 /// 2700 POST, 2701 /// 2702 PUT, 2703 /// 2704 DELETE, 2705 /// 2706 OPTIONS, 2707 /// 2708 TRACE, 2709 /// 2710 CONNECT, 2711 /// 2712 PATCH, 2713 /// 2714 MERGE 2715 } 2716 2717 /++ 2718 Supported file formats for [HttpClient.setClientCert]. These are loaded by OpenSSL 2719 in the current implementation. 2720 2721 History: 2722 Added February 3, 2022 (dub v10.6) 2723 +/ 2724 enum CertificateFileFormat { 2725 guess, /// try to guess the format from the file name and/or contents 2726 pem, /// the files are specifically in PEM format 2727 der /// the files are specifically in DER format 2728 } 2729 2730 /++ 2731 HttpClient keeps cookies, location, and some other state to reuse connections, when possible, like a web browser. 2732 You can use it as your entry point to make http requests. 2733 2734 See the example on [arsd.http2#examples]. 2735 +/ 2736 class HttpClient { 2737 /* Protocol restrictions, useful to disable when debugging servers */ 2738 bool useHttp11 = true; /// 2739 bool acceptGzip = true; /// 2740 bool keepAlive = true; /// 2741 2742 /++ 2743 Sets the client certificate used as a log in identifier on https connections. 2744 The certificate and key must be unencrypted at this time and both must be in 2745 the same file format. 2746 2747 Bugs: 2748 The current implementation sets the filenames into a static variable, 2749 meaning it is shared across all clients and connections. 2750 2751 Errors in the cert or key are only reported if the server reports an 2752 authentication failure. Make sure you are passing correct filenames 2753 and formats of you do see a failure. 2754 2755 History: 2756 Added February 2, 2022 (dub v10.6) 2757 +/ 2758 void setClientCertificate(string certFilename, string keyFilename, CertificateFileFormat certFormat = CertificateFileFormat.guess) { 2759 this.certFilename = certFilename; 2760 this.keyFilename = keyFilename; 2761 this.certFormat = certFormat; 2762 } 2763 2764 /++ 2765 Sets whether [HttpRequest]s created through this object (with [navigateTo], [request], etc.), will have the 2766 value of [HttpRequest.verifyPeer] of true or false upon construction. 2767 2768 History: 2769 Added April 5, 2022 (dub v10.8). Previously, there was an undocumented global value used. 2770 +/ 2771 bool defaultVerifyPeer = true; 2772 2773 // FIXME: try to not make these static 2774 private static string certFilename; 2775 private static string keyFilename; 2776 private static CertificateFileFormat certFormat; 2777 2778 /// 2779 @property Uri location() { 2780 return currentUrl; 2781 } 2782 2783 /++ 2784 Default timeout for requests created on this client. 2785 2786 History: 2787 Added March 31, 2021 2788 +/ 2789 Duration defaultTimeout = 10.seconds; 2790 2791 /++ 2792 High level function that works similarly to entering a url 2793 into a browser. 2794 2795 Follows locations, retain cookies, updates the current url, etc. 2796 +/ 2797 HttpRequest navigateTo(Uri where, HttpVerb method = HttpVerb.GET) { 2798 currentUrl = where.basedOn(currentUrl); 2799 currentDomain = where.host; 2800 2801 auto request = this.request(currentUrl, method); 2802 request.followLocation = true; 2803 request.retainCookies = true; 2804 2805 return request; 2806 } 2807 2808 /++ 2809 Creates a request without updating the current url state. If you want to save cookies, either call [retainCookies] with the response yourself 2810 or set [HttpRequest.retainCookies|request.retainCookies] to `true` on the returned object. But see important implementation shortcomings on [retainCookies]. 2811 +/ 2812 HttpRequest request(Uri uri, HttpVerb method = HttpVerb.GET, ubyte[] bodyData = null, string contentType = null) { 2813 string proxyToUse = getProxyFor(uri); 2814 2815 auto request = new HttpRequest(this, uri, method, cache, defaultTimeout, proxyToUse); 2816 2817 request.verifyPeer = this.defaultVerifyPeer; 2818 2819 request.requestParameters.userAgent = userAgent; 2820 request.requestParameters.authorization = authorization; 2821 2822 request.requestParameters.useHttp11 = this.useHttp11; 2823 request.requestParameters.acceptGzip = this.acceptGzip; 2824 request.requestParameters.keepAlive = this.keepAlive; 2825 2826 request.requestParameters.bodyData = bodyData; 2827 request.requestParameters.contentType = contentType; 2828 2829 populateCookies(request); 2830 2831 return request; 2832 2833 } 2834 2835 private void populateCookies(HttpRequest request) { 2836 // FIXME: what about expiration and the like? or domain/path checks? or Secure checks? 2837 // FIXME: is uri.host correct? i think it should include port number too. what fun. 2838 if(auto cookies = ""/*uri.host*/ in this.cookies) { 2839 foreach(cookie; *cookies) 2840 request.requestParameters.cookies[cookie.name] = cookie.value; 2841 } 2842 } 2843 2844 2845 /// ditto 2846 HttpRequest request(Uri uri, FormData fd, HttpVerb method = HttpVerb.POST) { 2847 return request(uri, method, fd.toBytes, fd.contentType); 2848 } 2849 2850 2851 private Uri currentUrl; 2852 private string currentDomain; 2853 private ICache cache; 2854 2855 /++ 2856 2857 +/ 2858 this(ICache cache = null) { 2859 this.defaultVerifyPeer = .defaultVerifyPeer_; 2860 this.cache = cache; 2861 loadDefaultProxy(); 2862 } 2863 2864 /++ 2865 Loads the system-default proxy. Note that the constructor does this automatically 2866 so you should rarely need to call this explicitly. 2867 2868 The environment variables are used, if present, on all operating systems. 2869 2870 History: 2871 no_proxy support added April 13, 2022 2872 2873 Added April 12, 2021 (included in dub v9.5) 2874 2875 Bugs: 2876 On Windows, it does NOT currently check the IE settings, but I do intend to 2877 implement that in the future. When I do, it will be classified as a bug fix, 2878 NOT a breaking change. 2879 +/ 2880 void loadDefaultProxy() { 2881 import std.process; 2882 httpProxy = environment.get("http_proxy", environment.get("HTTP_PROXY", null)); 2883 httpsProxy = environment.get("https_proxy", environment.get("HTTPS_PROXY", null)); 2884 auto noProxy = environment.get("no_proxy", environment.get("NO_PROXY", null)); 2885 if (noProxy.length) { 2886 proxyIgnore = noProxy.split(","); 2887 foreach (ref rule; proxyIgnore) 2888 rule = rule.strip; 2889 } 2890 2891 // FIXME: on Windows, I should use the Internet Explorer proxy settings 2892 } 2893 2894 /++ 2895 Checks if the given uri should be proxied according to the httpProxy, httpsProxy, proxyIgnore 2896 variables and returns either httpProxy, httpsProxy or null. 2897 2898 If neither `httpProxy` or `httpsProxy` are set this always returns `null`. Same if `proxyIgnore` 2899 contains `*`. 2900 2901 DNS is not resolved for proxyIgnore IPs, only IPs match IPs and hosts match hosts. 2902 +/ 2903 string getProxyFor(Uri uri) { 2904 string proxyToUse; 2905 switch(uri.scheme) { 2906 case "http": 2907 proxyToUse = httpProxy; 2908 break; 2909 case "https": 2910 proxyToUse = httpsProxy; 2911 break; 2912 default: 2913 proxyToUse = null; 2914 } 2915 2916 if (proxyToUse.length) { 2917 foreach (ignore; proxyIgnore) { 2918 if (matchProxyIgnore(ignore, uri)) { 2919 return null; 2920 } 2921 } 2922 } 2923 2924 return proxyToUse; 2925 } 2926 2927 /// Returns -1 on error, otherwise the IP as uint. Parsing is very strict. 2928 private static long tryParseIPv4(scope const(char)[] s) nothrow { 2929 import std.algorithm : findSplit, all; 2930 import std.ascii : isDigit; 2931 2932 static int parseNum(scope const(char)[] num) nothrow { 2933 if (num.length < 1 || num.length > 3 || !num.representation.all!isDigit) 2934 return -1; 2935 try { 2936 auto ret = num.to!int; 2937 return ret > 255 ? -1 : ret; 2938 } catch (Exception) { 2939 assert(false); 2940 } 2941 } 2942 2943 if (s.length < "0.0.0.0".length || s.length > "255.255.255.255".length) 2944 return -1; 2945 auto firstPair = s.findSplit("."); 2946 auto secondPair = firstPair[2].findSplit("."); 2947 auto thirdPair = secondPair[2].findSplit("."); 2948 auto a = parseNum(firstPair[0]); 2949 auto b = parseNum(secondPair[0]); 2950 auto c = parseNum(thirdPair[0]); 2951 auto d = parseNum(thirdPair[2]); 2952 if (a < 0 || b < 0 || c < 0 || d < 0) 2953 return -1; 2954 return (cast(uint)a << 24) | (b << 16) | (c << 8) | (d); 2955 } 2956 2957 unittest { 2958 assert(tryParseIPv4("0.0.0.0") == 0); 2959 assert(tryParseIPv4("127.0.0.1") == 0x7f000001); 2960 assert(tryParseIPv4("162.217.114.56") == 0xa2d97238); 2961 assert(tryParseIPv4("256.0.0.1") == -1); 2962 assert(tryParseIPv4("0.0.0.-2") == -1); 2963 assert(tryParseIPv4("0.0.0.a") == -1); 2964 assert(tryParseIPv4("0.0.0") == -1); 2965 assert(tryParseIPv4("0.0.0.0.0") == -1); 2966 } 2967 2968 /++ 2969 Returns true if the given no_proxy rule matches the uri. 2970 2971 Invalid IP ranges are silently ignored and return false. 2972 2973 See $(LREF proxyIgnore). 2974 +/ 2975 static bool matchProxyIgnore(scope const(char)[] rule, scope const Uri uri) nothrow { 2976 import std.algorithm; 2977 import std.ascii : isDigit; 2978 import std.uni : sicmp; 2979 2980 string uriHost = uri.host; 2981 if (uriHost.length && uriHost[$ - 1] == '.') 2982 uriHost = uriHost[0 .. $ - 1]; 2983 2984 if (rule == "*") 2985 return true; 2986 while (rule.length && rule[0] == '.') rule = rule[1 .. $]; 2987 2988 static int parsePort(scope const(char)[] portStr) nothrow { 2989 if (portStr.length < 1 || portStr.length > 5 || !portStr.representation.all!isDigit) 2990 return -1; 2991 try { 2992 return portStr.to!int; 2993 } catch (Exception) { 2994 assert(false, "to!int should succeed"); 2995 } 2996 } 2997 2998 if (sicmp(rule, uriHost) == 0 2999 || (uriHost.length > rule.length 3000 && sicmp(rule, uriHost[$ - rule.length .. $]) == 0 3001 && uriHost[$ - rule.length - 1] == '.')) 3002 return true; 3003 3004 if (rule.startsWith("[")) { // IPv6 3005 // below code is basically nothrow lastIndexOfAny("]:") 3006 ptrdiff_t lastColon = cast(ptrdiff_t) rule.length - 1; 3007 while (lastColon >= 0) { 3008 if (rule[lastColon] == ']' || rule[lastColon] == ':') 3009 break; 3010 lastColon--; 3011 } 3012 if (lastColon == -1) 3013 return false; // malformed 3014 3015 if (rule[lastColon] == ':') { // match with port 3016 auto port = parsePort(rule[lastColon + 1 .. $]); 3017 if (port != -1) { 3018 if (uri.effectivePort != port.to!int) 3019 return false; 3020 return uriHost == rule[0 .. lastColon]; 3021 } 3022 } 3023 // exact match of host already done above 3024 } else { 3025 auto slash = rule.lastIndexOfNothrow('/'); 3026 if (slash == -1) { // no IP range 3027 auto colon = rule.lastIndexOfNothrow(':'); 3028 auto host = colon == -1 ? rule : rule[0 .. colon]; 3029 auto port = colon != -1 ? parsePort(rule[colon + 1 .. $]) : -1; 3030 auto ip = tryParseIPv4(host); 3031 if (ip == -1) { // not an IPv4, test for host with port 3032 return port != -1 3033 && uri.effectivePort == port 3034 && uriHost == host; 3035 } else { 3036 // perform IPv4 equals 3037 auto other = tryParseIPv4(uriHost); 3038 if (other == -1) 3039 return false; // rule == IPv4, uri != IPv4 3040 if (port != -1) 3041 return uri.effectivePort == port 3042 && uriHost == host; 3043 else 3044 return uriHost == host; 3045 } 3046 } else { 3047 auto maskStr = rule[slash + 1 .. $]; 3048 auto ip = tryParseIPv4(rule[0 .. slash]); 3049 if (ip == -1) 3050 return false; 3051 if (maskStr.length && maskStr.length < 3 && maskStr.representation.all!isDigit) { 3052 // IPv4 range match 3053 int mask; 3054 try { 3055 mask = maskStr.to!int; 3056 } catch (Exception) { 3057 assert(false); 3058 } 3059 3060 auto other = tryParseIPv4(uriHost); 3061 if (other == -1) 3062 return false; // rule == IPv4, uri != IPv4 3063 3064 if (mask == 0) // matches all 3065 return true; 3066 if (mask > 32) // matches none 3067 return false; 3068 3069 auto shift = 32 - mask; 3070 return cast(uint)other >> shift 3071 == cast(uint)ip >> shift; 3072 } 3073 } 3074 } 3075 return false; 3076 } 3077 3078 unittest { 3079 assert(matchProxyIgnore("0.0.0.0/0", Uri("http://127.0.0.1:80/a"))); 3080 assert(matchProxyIgnore("0.0.0.0/0", Uri("http://127.0.0.1/a"))); 3081 assert(!matchProxyIgnore("0.0.0.0/0", Uri("https://dlang.org/a"))); 3082 assert(matchProxyIgnore("*", Uri("https://dlang.org/a"))); 3083 assert(matchProxyIgnore("127.0.0.0/8", Uri("http://127.0.0.1:80/a"))); 3084 assert(matchProxyIgnore("127.0.0.0/8", Uri("http://127.0.0.1/a"))); 3085 assert(matchProxyIgnore("127.0.0.1", Uri("http://127.0.0.1:1234/a"))); 3086 assert(!matchProxyIgnore("127.0.0.1:80", Uri("http://127.0.0.1:1234/a"))); 3087 assert(!matchProxyIgnore("127.0.0.1/8", Uri("http://localhost/a"))); // no DNS resolution / guessing 3088 assert(!matchProxyIgnore("0.0.0.0/1", Uri("http://localhost/a")) 3089 && !matchProxyIgnore("128.0.0.0/1", Uri("http://localhost/a"))); // no DNS resolution / guessing 2 3090 foreach (m; 1 .. 32) { 3091 assert(matchProxyIgnore(text("127.0.0.1/", m), Uri("http://127.0.0.1/a"))); 3092 assert(!matchProxyIgnore(text("127.0.0.1/", m), Uri("http://128.0.0.1/a"))); 3093 bool expectedMatch = m <= 24; 3094 assert(expectedMatch == matchProxyIgnore(text("127.0.1.0/", m), Uri("http://127.0.1.128/a")), m.to!string); 3095 } 3096 assert(matchProxyIgnore("localhost", Uri("http://localhost/a"))); 3097 assert(matchProxyIgnore("localhost", Uri("http://foo.localhost/a"))); 3098 assert(matchProxyIgnore("localhost", Uri("http://foo.localhost./a"))); 3099 assert(matchProxyIgnore(".localhost", Uri("http://localhost/a"))); 3100 assert(matchProxyIgnore(".localhost", Uri("http://foo.localhost/a"))); 3101 assert(matchProxyIgnore(".localhost", Uri("http://foo.localhost./a"))); 3102 assert(!matchProxyIgnore("foo.localhost", Uri("http://localhost/a"))); 3103 assert(matchProxyIgnore("foo.localhost", Uri("http://foo.localhost/a"))); 3104 assert(matchProxyIgnore("foo.localhost", Uri("http://foo.localhost./a"))); 3105 assert(!matchProxyIgnore("bar.localhost", Uri("http://localhost/a"))); 3106 assert(!matchProxyIgnore("bar.localhost", Uri("http://foo.localhost/a"))); 3107 assert(!matchProxyIgnore("bar.localhost", Uri("http://foo.localhost./a"))); 3108 assert(!matchProxyIgnore("bar.localhost", Uri("http://bbar.localhost./a"))); 3109 assert(matchProxyIgnore("[::1]", Uri("http://[::1]/a"))); 3110 assert(!matchProxyIgnore("[::1]", Uri("http://[::2]/a"))); 3111 assert(matchProxyIgnore("[::1]:80", Uri("http://[::1]/a"))); 3112 assert(!matchProxyIgnore("[::1]:443", Uri("http://[::1]/a"))); 3113 assert(!matchProxyIgnore("[::1]:80", Uri("https://[::1]/a"))); 3114 assert(matchProxyIgnore("[::1]:443", Uri("https://[::1]/a"))); 3115 assert(matchProxyIgnore("google.com", Uri("https://GOOGLE.COM/a"))); 3116 } 3117 3118 /++ 3119 Proxies to use for requests. The [HttpClient] constructor will set these to the system values, 3120 then you can reset it to `null` if you want to override and not use the proxy after all, or you 3121 can set it after construction to whatever. 3122 3123 The proxy from the client will be automatically set to the requests performed through it. You can 3124 also override on a per-request basis by creating the request and setting the `proxy` field there 3125 before sending it. 3126 3127 History: 3128 Added April 12, 2021 (included in dub v9.5) 3129 +/ 3130 string httpProxy; 3131 /// ditto 3132 string httpsProxy; 3133 /++ 3134 List of hosts or ips, optionally including a port, where not to proxy. 3135 3136 Each entry may be one of the following formats: 3137 - `127.0.0.1` (IPv4, any port) 3138 - `127.0.0.1:1234` (IPv4, specific port) 3139 - `127.0.0.1/8` (IPv4 range / CIDR block, any port) 3140 - `[::1]` (IPv6, any port) 3141 - `[::1]:1234` (IPv6, specific port) 3142 - `*` (all hosts and ports, basically don't proxy at all anymore) 3143 - `.domain.name`, `domain.name` (don't proxy the specified domain, 3144 leading dots are stripped and subdomains are also not proxied) 3145 - `.domain.name:1234`, `domain.name:1234` (same as above, with specific port) 3146 3147 No DNS resolution or regex is done in this list. 3148 3149 See https://about.gitlab.com/blog/2021/01/27/we-need-to-talk-no-proxy/ 3150 3151 History: 3152 Added April 13, 2022 3153 +/ 3154 string[] proxyIgnore; 3155 3156 /// See [retainCookies] for important caveats. 3157 void setCookie(string name, string value, string domain = null) { 3158 CookieHeader ch; 3159 3160 ch.name = name; 3161 ch.value = value; 3162 3163 setCookie(ch, domain); 3164 } 3165 3166 /// ditto 3167 void setCookie(CookieHeader ch, string domain = null) { 3168 if(domain is null) 3169 domain = currentDomain; 3170 3171 // FIXME: figure all this out or else cookies liable to get too long, in addition to the overwriting and oversharing issues in long scraping sessions 3172 cookies[""/*domain*/] ~= ch; 3173 } 3174 3175 /++ 3176 [HttpClient] does NOT automatically store cookies. You must explicitly retain them from a response by calling this method. 3177 3178 Examples: 3179 --- 3180 import arsd.http2; 3181 void main() { 3182 auto client = new HttpClient(); 3183 auto setRequest = client.request(Uri("http://arsdnet.net/cgi-bin/cookies/set")); 3184 auto setResponse = setRequest.waitForCompletion(); 3185 3186 auto request = client.request(Uri("http://arsdnet.net/cgi-bin/cookies/get")); 3187 auto response = request.waitForCompletion(); 3188 3189 // the cookie wasn't explicitly retained, so the server echos back nothing 3190 assert(response.responseText.length == 0); 3191 3192 // now keep the cookies from our original set 3193 client.retainCookies(setResponse); 3194 3195 request = client.request(Uri("http://arsdnet.net/cgi-bin/cookies/get")); 3196 response = request.waitForCompletion(); 3197 3198 // now it matches 3199 assert(response.responseText.length && response.responseText == setResponse.cookies["example-cookie"]); 3200 } 3201 --- 3202 3203 Bugs: 3204 It does NOT currently implement domain / path / secure separation nor cookie expiration. It assumes that if you call this function, you're ok with it. 3205 3206 You may want to use separate HttpClient instances if any sharing is unacceptable at this time. 3207 3208 History: 3209 Added July 5, 2021 (dub v10.2) 3210 +/ 3211 void retainCookies(HttpResponse fromResponse) { 3212 foreach(name, value; fromResponse.cookies) 3213 setCookie(name, value); 3214 } 3215 3216 /// 3217 void clearCookies(string domain = null) { 3218 if(domain is null) 3219 cookies = null; 3220 else 3221 cookies[domain] = null; 3222 } 3223 3224 // If you set these, they will be pre-filled on all requests made with this client 3225 string userAgent = "D arsd.html2"; /// 3226 string authorization; /// 3227 3228 /* inter-request state */ 3229 private CookieHeader[][string] cookies; 3230 } 3231 3232 private ptrdiff_t lastIndexOfNothrow(T)(scope T[] arr, T value) nothrow 3233 { 3234 ptrdiff_t ret = cast(ptrdiff_t)arr.length - 1; 3235 while (ret >= 0) { 3236 if (arr[ret] == value) 3237 return ret; 3238 ret--; 3239 } 3240 return ret; 3241 } 3242 3243 interface ICache { 3244 /++ 3245 The client is about to make the given `request`. It will ALWAYS pass it to the cache object first so you can decide if you want to and can provide a response. You should probably check the appropriate headers to see if you should even attempt to look up on the cache (HttpClient does NOT do this to give maximum flexibility to the cache implementor). 3246 3247 Return null if the cache does not provide. 3248 +/ 3249 const(HttpResponse)* getCachedResponse(HttpRequestParameters request); 3250 3251 /++ 3252 The given request has received the given response. The implementing class needs to decide if it wants to cache or not. Return true if it was added, false if you chose not to. 3253 3254 You may wish to examine headers, etc., in making the decision. The HttpClient will ALWAYS pass a request/response to this. 3255 +/ 3256 bool cacheResponse(HttpRequestParameters request, HttpResponse response); 3257 } 3258 3259 /+ 3260 // / Provides caching behavior similar to a real web browser 3261 class HttpCache : ICache { 3262 const(HttpResponse)* getCachedResponse(HttpRequestParameters request) { 3263 return null; 3264 } 3265 } 3266 3267 // / Gives simple maximum age caching, ignoring the actual http headers 3268 class SimpleCache : ICache { 3269 const(HttpResponse)* getCachedResponse(HttpRequestParameters request) { 3270 return null; 3271 } 3272 } 3273 +/ 3274 3275 /++ 3276 A pseudo-cache to provide a mock server. Construct one of these, 3277 populate it with test responses, and pass it to [HttpClient] to 3278 do a network-free test. 3279 3280 You should populate it with the [populate] method. Any request not 3281 pre-populated will return a "server refused connection" response. 3282 +/ 3283 class HttpMockProvider : ICache { 3284 /+ + 3285 3286 +/ 3287 version(none) 3288 this(Uri baseUrl, string defaultResponseContentType) { 3289 3290 } 3291 3292 this() {} 3293 3294 HttpResponse defaultResponse; 3295 3296 /// Implementation of the ICache interface. Hijacks all requests to return a pre-populated response or "server disconnected". 3297 const(HttpResponse)* getCachedResponse(HttpRequestParameters request) { 3298 import std.conv; 3299 auto defaultPort = request.ssl ? 443 : 80; 3300 string identifier = text( 3301 request.method, " ", 3302 request.ssl ? "https" : "http", "://", 3303 request.host, 3304 (request.port && request.port != defaultPort) ? (":" ~ to!string(request.port)) : "", 3305 request.uri 3306 ); 3307 3308 if(auto res = identifier in population) 3309 return res; 3310 return &defaultResponse; 3311 } 3312 3313 /// Implementation of the ICache interface. We never actually cache anything here since it is all about mock responses, not actually caching real data. 3314 bool cacheResponse(HttpRequestParameters request, HttpResponse response) { 3315 return false; 3316 } 3317 3318 /++ 3319 Convenience method to populate simple responses. For more complex 3320 work, use one of the other overloads where you build complete objects 3321 yourself. 3322 3323 Params: 3324 request = a verb and complete URL to mock as one string. 3325 For example "GET http://example.com/". If you provide only 3326 a partial URL, it will be based on the `baseUrl` you gave 3327 in the `HttpMockProvider` constructor. 3328 3329 responseCode = the HTTP response code, like 200 or 404. 3330 3331 response = the response body as a string. It is assumed 3332 to be of the `defaultResponseContentType` you passed in the 3333 `HttpMockProvider` constructor. 3334 +/ 3335 void populate(string request, int responseCode, string response) { 3336 3337 // FIXME: absolute-ize the URL in the request 3338 3339 HttpResponse r; 3340 r.code = responseCode; 3341 r.codeText = getHttpCodeText(r.code); 3342 3343 r.content = cast(ubyte[]) response; 3344 r.contentText = response; 3345 3346 population[request] = r; 3347 } 3348 3349 version(none) 3350 void populate(string method, string url, HttpResponse response) { 3351 // FIXME 3352 } 3353 3354 private HttpResponse[string] population; 3355 } 3356 3357 // modified from the one in cgi.d to just have the text 3358 private static string getHttpCodeText(int code) pure nothrow @nogc { 3359 switch(code) { 3360 // this module's proprietary extensions 3361 case 0: return null; 3362 case 1: return "request.abort called"; 3363 case 2: return "connection failed"; 3364 case 3: return "server disconnected"; 3365 case 4: return "exception thrown"; // actually should be some other thing 3366 case 5: return "Request timed out"; 3367 3368 // * * * standard ones * * * 3369 3370 // 1xx skipped since they shouldn't happen 3371 3372 // 3373 case 200: return "OK"; 3374 case 201: return "Created"; 3375 case 202: return "Accepted"; 3376 case 203: return "Non-Authoritative Information"; 3377 case 204: return "No Content"; 3378 case 205: return "Reset Content"; 3379 // 3380 case 300: return "Multiple Choices"; 3381 case 301: return "Moved Permanently"; 3382 case 302: return "Found"; 3383 case 303: return "See Other"; 3384 case 307: return "Temporary Redirect"; 3385 case 308: return "Permanent Redirect"; 3386 // 3387 case 400: return "Bad Request"; 3388 case 403: return "Forbidden"; 3389 case 404: return "Not Found"; 3390 case 405: return "Method Not Allowed"; 3391 case 406: return "Not Acceptable"; 3392 case 409: return "Conflict"; 3393 case 410: return "Gone"; 3394 // 3395 case 500: return "Internal Server Error"; 3396 case 501: return "Not Implemented"; 3397 case 502: return "Bad Gateway"; 3398 case 503: return "Service Unavailable"; 3399 // 3400 default: assert(0, "Unsupported http code"); 3401 } 3402 } 3403 3404 3405 /// 3406 struct HttpCookie { 3407 string name; /// 3408 string value; /// 3409 string domain; /// 3410 string path; /// 3411 //SysTime expirationDate; /// 3412 bool secure; /// 3413 bool httpOnly; /// 3414 } 3415 3416 // FIXME: websocket 3417 3418 version(testing) 3419 void main() { 3420 import std.stdio; 3421 auto client = new HttpClient(); 3422 auto request = client.navigateTo(Uri("http://localhost/chunked.php")); 3423 request.send(); 3424 auto request2 = client.navigateTo(Uri("http://dlang.org/")); 3425 request2.send(); 3426 3427 { 3428 auto response = request2.waitForCompletion(); 3429 //write(cast(string) response.content); 3430 } 3431 3432 auto response = request.waitForCompletion(); 3433 write(cast(string) response.content); 3434 3435 writeln(HttpRequest.socketsPerHost); 3436 } 3437 3438 3439 // From sslsocket.d, but this is the maintained version! 3440 version(use_openssl) { 3441 alias SslClientSocket = OpenSslSocket; 3442 3443 // CRL = Certificate Revocation List 3444 static immutable string[] sslErrorCodes = [ 3445 "OK (code 0)", 3446 "Unspecified SSL/TLS error (code 1)", 3447 "Unable to get TLS issuer certificate (code 2)", 3448 "Unable to get TLS CRL (code 3)", 3449 "Unable to decrypt TLS certificate signature (code 4)", 3450 "Unable to decrypt TLS CRL signature (code 5)", 3451 "Unable to decode TLS issuer public key (code 6)", 3452 "TLS certificate signature failure (code 7)", 3453 "TLS CRL signature failure (code 8)", 3454 "TLS certificate not yet valid (code 9)", 3455 "TLS certificate expired (code 10)", 3456 "TLS CRL not yet valid (code 11)", 3457 "TLS CRL expired (code 12)", 3458 "TLS error in certificate not before field (code 13)", 3459 "TLS error in certificate not after field (code 14)", 3460 "TLS error in CRL last update field (code 15)", 3461 "TLS error in CRL next update field (code 16)", 3462 "TLS system out of memory (code 17)", 3463 "TLS certificate is self-signed (code 18)", 3464 "Self-signed certificate in TLS chain (code 19)", 3465 "Unable to get TLS issuer certificate locally (code 20)", 3466 "Unable to verify TLS leaf signature (code 21)", 3467 "TLS certificate chain too long (code 22)", 3468 "TLS certificate was revoked (code 23)", 3469 "TLS CA is invalid (code 24)", 3470 "TLS error: path length exceeded (code 25)", 3471 "TLS error: invalid purpose (code 26)", 3472 "TLS error: certificate untrusted (code 27)", 3473 "TLS error: certificate rejected (code 28)", 3474 ]; 3475 3476 string getOpenSslErrorCode(long error) { 3477 if(error == 62) 3478 return "TLS certificate host name mismatch"; 3479 3480 if(error < 0 || error >= sslErrorCodes.length) 3481 return "SSL/TLS error code " ~ to!string(error); 3482 return sslErrorCodes[cast(size_t) error]; 3483 } 3484 3485 struct SSL; 3486 struct SSL_CTX; 3487 struct SSL_METHOD; 3488 struct X509_STORE_CTX; 3489 enum SSL_VERIFY_NONE = 0; 3490 enum SSL_VERIFY_PEER = 1; 3491 3492 // copy it into the buf[0 .. size] and return actual length you read. 3493 // rwflag == 0 when reading, 1 when writing. 3494 extern(C) alias pem_password_cb = int function(char* buffer, int bufferSize, int rwflag, void* userPointer); 3495 extern(C) alias print_errors_cb = int function(const char*, size_t, void*); 3496 extern(C) alias client_cert_cb = int function(SSL *ssl, X509 **x509, EVP_PKEY **pkey); 3497 extern(C) alias keylog_cb = void function(SSL*, char*); 3498 3499 struct X509; 3500 struct X509_STORE; 3501 struct EVP_PKEY; 3502 struct X509_VERIFY_PARAM; 3503 3504 import core.stdc.config; 3505 3506 enum SSL_ERROR_WANT_READ = 2; 3507 enum SSL_ERROR_WANT_WRITE = 3; 3508 3509 struct ossllib { 3510 __gshared static extern(C) { 3511 /* these are only on older openssl versions { */ 3512 int function() SSL_library_init; 3513 void function() SSL_load_error_strings; 3514 SSL_METHOD* function() SSLv23_client_method; 3515 /* } */ 3516 3517 void function(ulong, void*) OPENSSL_init_ssl; 3518 3519 SSL_CTX* function(const SSL_METHOD*) SSL_CTX_new; 3520 SSL* function(SSL_CTX*) SSL_new; 3521 int function(SSL*, int) SSL_set_fd; 3522 int function(SSL*) SSL_connect; 3523 int function(SSL*, const void*, int) SSL_write; 3524 int function(SSL*, void*, int) SSL_read; 3525 @trusted nothrow @nogc int function(SSL*) SSL_shutdown; 3526 void function(SSL*) SSL_free; 3527 void function(SSL_CTX*) SSL_CTX_free; 3528 3529 int function(const SSL*) SSL_pending; 3530 int function (const SSL *ssl, int ret) SSL_get_error; 3531 3532 void function(SSL*, int, void*) SSL_set_verify; 3533 3534 void function(SSL*, int, c_long, void*) SSL_ctrl; 3535 3536 SSL_METHOD* function() SSLv3_client_method; 3537 SSL_METHOD* function() TLS_client_method; 3538 3539 void function(SSL_CTX*, void function(SSL*, char* line)) SSL_CTX_set_keylog_callback; 3540 3541 int function(SSL_CTX*) SSL_CTX_set_default_verify_paths; 3542 3543 X509_STORE* function(SSL_CTX*) SSL_CTX_get_cert_store; 3544 c_long function(const SSL* ssl) SSL_get_verify_result; 3545 3546 X509_VERIFY_PARAM* function(const SSL*) SSL_get0_param; 3547 3548 /+ 3549 SSL_CTX_load_verify_locations 3550 SSL_CTX_set_client_CA_list 3551 +/ 3552 3553 // client cert things 3554 void function (SSL_CTX *ctx, int function(SSL *ssl, X509 **x509, EVP_PKEY **pkey)) SSL_CTX_set_client_cert_cb; 3555 } 3556 } 3557 3558 struct eallib { 3559 __gshared static extern(C) { 3560 /* these are only on older openssl versions { */ 3561 void function() OpenSSL_add_all_ciphers; 3562 void function() OpenSSL_add_all_digests; 3563 /* } */ 3564 3565 const(char)* function(int) OpenSSL_version; 3566 3567 void function(ulong, void*) OPENSSL_init_crypto; 3568 3569 void function(print_errors_cb, void*) ERR_print_errors_cb; 3570 3571 void function(X509*) X509_free; 3572 int function(X509_STORE*, X509*) X509_STORE_add_cert; 3573 3574 3575 X509* function(FILE *fp, X509 **x, pem_password_cb *cb, void *u) PEM_read_X509; 3576 EVP_PKEY* function(FILE *fp, EVP_PKEY **x, pem_password_cb *cb, void* userPointer) PEM_read_PrivateKey; 3577 3578 EVP_PKEY* function(FILE *fp, EVP_PKEY **a) d2i_PrivateKey_fp; 3579 X509* function(FILE *fp, X509 **x) d2i_X509_fp; 3580 3581 X509* function(X509** a, const(ubyte*)* pp, c_long length) d2i_X509; 3582 int function(X509* a, ubyte** o) i2d_X509; 3583 3584 int function(X509_VERIFY_PARAM* a, const char* b, size_t l) X509_VERIFY_PARAM_set1_host; 3585 3586 X509* function(X509_STORE_CTX *ctx) X509_STORE_CTX_get_current_cert; 3587 int function(X509_STORE_CTX *ctx) X509_STORE_CTX_get_error; 3588 } 3589 } 3590 3591 struct OpenSSL { 3592 static: 3593 3594 template opDispatch(string name) { 3595 auto opDispatch(T...)(T t) { 3596 static if(__traits(hasMember, ossllib, name)) { 3597 auto ptr = __traits(getMember, ossllib, name); 3598 } else static if(__traits(hasMember, eallib, name)) { 3599 auto ptr = __traits(getMember, eallib, name); 3600 } else static assert(0); 3601 3602 if(ptr is null) 3603 throw new Exception(name ~ " not loaded"); 3604 return ptr(t); 3605 } 3606 } 3607 3608 // macros in the original C 3609 SSL_METHOD* SSLv23_client_method() { 3610 if(ossllib.SSLv23_client_method) 3611 return ossllib.SSLv23_client_method(); 3612 else 3613 return ossllib.TLS_client_method(); 3614 } 3615 3616 void SSL_set_tlsext_host_name(SSL* a, const char* b) { 3617 if(ossllib.SSL_ctrl) 3618 return ossllib.SSL_ctrl(a, 55 /*SSL_CTRL_SET_TLSEXT_HOSTNAME*/, 0 /*TLSEXT_NAMETYPE_host_name*/, cast(void*) b); 3619 else throw new Exception("SSL_set_tlsext_host_name not loaded"); 3620 } 3621 3622 // special case 3623 @trusted nothrow @nogc int SSL_shutdown(SSL* a) { 3624 if(ossllib.SSL_shutdown) 3625 return ossllib.SSL_shutdown(a); 3626 assert(0); 3627 } 3628 3629 void SSL_CTX_keylog_cb_func(SSL_CTX* ctx, keylog_cb func) { 3630 // this isn't in openssl 1.0 and is non-essential, so it is allowed to fail. 3631 if(ossllib.SSL_CTX_set_keylog_callback) 3632 ossllib.SSL_CTX_set_keylog_callback(ctx, func); 3633 //else throw new Exception("SSL_CTX_keylog_cb_func not loaded"); 3634 } 3635 3636 } 3637 3638 extern(C) 3639 int collectSslErrors(const char* ptr, size_t len, void* user) @trusted { 3640 string* s = cast(string*) user; 3641 3642 (*s) ~= ptr[0 .. len]; 3643 3644 return 0; 3645 } 3646 3647 3648 private __gshared void* ossllib_handle; 3649 version(Windows) 3650 private __gshared void* oeaylib_handle; 3651 else 3652 alias oeaylib_handle = ossllib_handle; 3653 version(Posix) 3654 private import core.sys.posix.dlfcn; 3655 else version(Windows) 3656 private import core.sys.windows.windows; 3657 3658 import core.stdc.stdio; 3659 3660 private __gshared Object loadSslMutex = new Object; 3661 private __gshared bool sslLoaded = false; 3662 3663 void loadOpenSsl() { 3664 if(sslLoaded) 3665 return; 3666 synchronized(loadSslMutex) { 3667 3668 version(Posix) { 3669 version(OSX) { 3670 static immutable string[] ossllibs = [ 3671 "libssl.46.dylib", 3672 "libssl.44.dylib", 3673 "libssl.43.dylib", 3674 "libssl.35.dylib", 3675 "libssl.1.1.dylib", 3676 "libssl.dylib", 3677 "/usr/local/opt/openssl/lib/libssl.1.0.0.dylib", 3678 ]; 3679 } else { 3680 static immutable string[] ossllibs = [ 3681 "libssl.so.3", 3682 "libssl.so.1.1", 3683 "libssl.so.1.0.2", 3684 "libssl.so.1.0.1", 3685 "libssl.so.1.0.0", 3686 "libssl.so", 3687 ]; 3688 } 3689 3690 foreach(lib; ossllibs) { 3691 ossllib_handle = dlopen(lib.ptr, RTLD_NOW); 3692 if(ossllib_handle !is null) break; 3693 } 3694 } else version(Windows) { 3695 version(X86_64) { 3696 ossllib_handle = LoadLibraryW("libssl-1_1-x64.dll"w.ptr); 3697 oeaylib_handle = LoadLibraryW("libcrypto-1_1-x64.dll"w.ptr); 3698 } 3699 3700 static immutable wstring[] ossllibs = [ 3701 "libssl-3-x64.dll"w, 3702 "libssl-3.dll"w, 3703 "libssl-1_1.dll"w, 3704 "libssl32.dll"w, 3705 ]; 3706 3707 if(ossllib_handle is null) 3708 foreach(lib; ossllibs) { 3709 ossllib_handle = LoadLibraryW(lib.ptr); 3710 if(ossllib_handle !is null) break; 3711 } 3712 3713 static immutable wstring[] eaylibs = [ 3714 "libcrypto-3-x64.dll"w, 3715 "libcrypto-3.dll"w, 3716 "libcrypto-1_1.dll"w, 3717 "libeay32.dll", 3718 ]; 3719 3720 if(oeaylib_handle is null) 3721 foreach(lib; eaylibs) { 3722 oeaylib_handle = LoadLibraryW(lib.ptr); 3723 if (oeaylib_handle !is null) break; 3724 } 3725 3726 if(ossllib_handle is null) { 3727 ossllib_handle = LoadLibraryW("ssleay32.dll"w.ptr); 3728 oeaylib_handle = ossllib_handle; 3729 } 3730 } 3731 3732 if(ossllib_handle is null) 3733 throw new Exception("libssl library not found"); 3734 if(oeaylib_handle is null) 3735 throw new Exception("libeay32 library not found"); 3736 3737 foreach(memberName; __traits(allMembers, ossllib)) { 3738 alias t = typeof(__traits(getMember, ossllib, memberName)); 3739 version(Posix) 3740 __traits(getMember, ossllib, memberName) = cast(t) dlsym(ossllib_handle, memberName); 3741 else version(Windows) { 3742 __traits(getMember, ossllib, memberName) = cast(t) GetProcAddress(ossllib_handle, memberName); 3743 } 3744 } 3745 3746 foreach(memberName; __traits(allMembers, eallib)) { 3747 alias t = typeof(__traits(getMember, eallib, memberName)); 3748 version(Posix) 3749 __traits(getMember, eallib, memberName) = cast(t) dlsym(oeaylib_handle, memberName); 3750 else version(Windows) { 3751 __traits(getMember, eallib, memberName) = cast(t) GetProcAddress(oeaylib_handle, memberName); 3752 } 3753 } 3754 3755 3756 if(ossllib.SSL_library_init) 3757 ossllib.SSL_library_init(); 3758 else if(ossllib.OPENSSL_init_ssl) 3759 ossllib.OPENSSL_init_ssl(0, null); 3760 else throw new Exception("couldn't init openssl"); 3761 3762 if(eallib.OpenSSL_add_all_ciphers) { 3763 eallib.OpenSSL_add_all_ciphers(); 3764 if(eallib.OpenSSL_add_all_digests is null) 3765 throw new Exception("no add digests"); 3766 eallib.OpenSSL_add_all_digests(); 3767 } else if(eallib.OPENSSL_init_crypto) 3768 eallib.OPENSSL_init_crypto(0 /*OPENSSL_INIT_ADD_ALL_CIPHERS and ALL_DIGESTS together*/, null); 3769 else throw new Exception("couldn't init crypto openssl"); 3770 3771 if(ossllib.SSL_load_error_strings) 3772 ossllib.SSL_load_error_strings(); 3773 else if(ossllib.OPENSSL_init_ssl) 3774 ossllib.OPENSSL_init_ssl(0x00200000L, null); 3775 else throw new Exception("couldn't load openssl errors"); 3776 3777 sslLoaded = true; 3778 } 3779 } 3780 3781 /+ 3782 // I'm just gonna let the OS clean this up on process termination because otherwise SSL_free 3783 // might have trouble being run from the GC after this module is unloaded. 3784 shared static ~this() { 3785 if(ossllib_handle) { 3786 version(Windows) { 3787 FreeLibrary(oeaylib_handle); 3788 FreeLibrary(ossllib_handle); 3789 } else version(Posix) 3790 dlclose(ossllib_handle); 3791 ossllib_handle = null; 3792 } 3793 ossllib.tupleof = ossllib.tupleof.init; 3794 } 3795 +/ 3796 3797 //pragma(lib, "crypto"); 3798 //pragma(lib, "ssl"); 3799 extern(C) 3800 void write_to_file(SSL* ssl, char* line) 3801 { 3802 import std.stdio; 3803 import std.string; 3804 import std.process : environment; 3805 string logfile = environment.get("SSLKEYLOGFILE"); 3806 if (logfile !is null) 3807 { 3808 auto f = std.stdio.File(logfile, "a+"); 3809 f.writeln(fromStringz(line)); 3810 f.close(); 3811 } 3812 } 3813 3814 class OpenSslSocket : Socket { 3815 private SSL* ssl; 3816 private SSL_CTX* ctx; 3817 private void initSsl(bool verifyPeer, string hostname) { 3818 ctx = OpenSSL.SSL_CTX_new(OpenSSL.SSLv23_client_method()); 3819 assert(ctx !is null); 3820 3821 debug OpenSSL.SSL_CTX_keylog_cb_func(ctx, &write_to_file); 3822 ssl = OpenSSL.SSL_new(ctx); 3823 3824 if(hostname.length) { 3825 OpenSSL.SSL_set_tlsext_host_name(ssl, toStringz(hostname)); 3826 if(verifyPeer) 3827 OpenSSL.X509_VERIFY_PARAM_set1_host(OpenSSL.SSL_get0_param(ssl), hostname.ptr, hostname.length); 3828 } 3829 3830 if(verifyPeer) { 3831 OpenSSL.SSL_CTX_set_default_verify_paths(ctx); 3832 3833 version(Windows) { 3834 loadCertificatesFromRegistry(ctx); 3835 } 3836 3837 OpenSSL.SSL_set_verify(ssl, SSL_VERIFY_PEER, &verifyCertificateFromRegistryArsdHttp); 3838 } else 3839 OpenSSL.SSL_set_verify(ssl, SSL_VERIFY_NONE, null); 3840 3841 OpenSSL.SSL_set_fd(ssl, cast(int) this.handle); // on win64 it is necessary to truncate, but the value is never large anyway see http://openssl.6102.n7.nabble.com/Sockets-windows-64-bit-td36169.html 3842 3843 3844 OpenSSL.SSL_CTX_set_client_cert_cb(ctx, &cb); 3845 } 3846 3847 extern(C) 3848 static int cb(SSL* ssl, X509** x509, EVP_PKEY** pkey) { 3849 if(HttpClient.certFilename.length && HttpClient.keyFilename.length) { 3850 FILE* fpCert = fopen((HttpClient.certFilename ~ "\0").ptr, "rb"); 3851 if(fpCert is null) 3852 return 0; 3853 scope(exit) 3854 fclose(fpCert); 3855 FILE* fpKey = fopen((HttpClient.keyFilename ~ "\0").ptr, "rb"); 3856 if(fpKey is null) 3857 return 0; 3858 scope(exit) 3859 fclose(fpKey); 3860 3861 with(CertificateFileFormat) 3862 final switch(HttpClient.certFormat) { 3863 case guess: 3864 if(HttpClient.certFilename.endsWith(".pem") || HttpClient.keyFilename.endsWith(".pem")) 3865 goto case pem; 3866 else 3867 goto case der; 3868 case pem: 3869 *x509 = OpenSSL.PEM_read_X509(fpCert, null, null, null); 3870 *pkey = OpenSSL.PEM_read_PrivateKey(fpKey, null, null, null); 3871 break; 3872 case der: 3873 *x509 = OpenSSL.d2i_X509_fp(fpCert, null); 3874 *pkey = OpenSSL.d2i_PrivateKey_fp(fpKey, null); 3875 break; 3876 } 3877 3878 return 1; 3879 } 3880 3881 return 0; 3882 } 3883 3884 bool dataPending() { 3885 return OpenSSL.SSL_pending(ssl) > 0; 3886 } 3887 3888 @trusted 3889 override void connect(Address to) { 3890 super.connect(to); 3891 if(blocking) { 3892 do_ssl_connect(); 3893 } 3894 } 3895 3896 @trusted 3897 // returns true if it is finished, false if it would have blocked, throws if there's an error 3898 int do_ssl_connect() { 3899 if(OpenSSL.SSL_connect(ssl) == -1) { 3900 3901 auto errCode = OpenSSL.SSL_get_error(ssl, -1); 3902 if(errCode == SSL_ERROR_WANT_READ || errCode == SSL_ERROR_WANT_WRITE) { 3903 return errCode; 3904 } 3905 3906 string str; 3907 OpenSSL.ERR_print_errors_cb(&collectSslErrors, &str); 3908 int i; 3909 auto err = OpenSSL.SSL_get_verify_result(ssl); 3910 //printf("wtf\n"); 3911 //scanf("%d\n", i); 3912 throw new Exception("Secure connect failed: " ~ getOpenSslErrorCode(err)); 3913 } 3914 3915 return 0; 3916 } 3917 3918 @trusted 3919 override ptrdiff_t send(scope const(void)[] buf, SocketFlags flags) { 3920 //import std.stdio;writeln(cast(string) buf); 3921 debug(arsd_http2_verbose) writeln("ssl writing ", buf.length); 3922 auto retval = OpenSSL.SSL_write(ssl, buf.ptr, cast(uint) buf.length); 3923 3924 // don't need to throw anymore since it is checked elsewhere 3925 // code useful sometimes for debugging hence commenting instead of deleting 3926 version(none) 3927 if(retval == -1) { 3928 3929 string str; 3930 OpenSSL.ERR_print_errors_cb(&collectSslErrors, &str); 3931 int i; 3932 3933 //printf("wtf\n"); 3934 //scanf("%d\n", i); 3935 3936 throw new Exception("ssl send failed " ~ str); 3937 } 3938 return retval; 3939 3940 } 3941 override ptrdiff_t send(scope const(void)[] buf) { 3942 return send(buf, SocketFlags.NONE); 3943 } 3944 @trusted 3945 override ptrdiff_t receive(scope void[] buf, SocketFlags flags) { 3946 3947 debug(arsd_http2_verbose) writeln("ssl_read before"); 3948 auto retval = OpenSSL.SSL_read(ssl, buf.ptr, cast(int)buf.length); 3949 debug(arsd_http2_verbose) writeln("ssl_read after"); 3950 3951 // don't need to throw anymore since it is checked elsewhere 3952 // code useful sometimes for debugging hence commenting instead of deleting 3953 version(none) 3954 if(retval == -1) { 3955 3956 string str; 3957 OpenSSL.ERR_print_errors_cb(&collectSslErrors, &str); 3958 int i; 3959 3960 //printf("wtf\n"); 3961 //scanf("%d\n", i); 3962 3963 throw new Exception("ssl receive failed " ~ str); 3964 } 3965 return retval; 3966 } 3967 override ptrdiff_t receive(scope void[] buf) { 3968 return receive(buf, SocketFlags.NONE); 3969 } 3970 3971 this(AddressFamily af, SocketType type = SocketType.STREAM, string hostname = null, bool verifyPeer = true) { 3972 version(Windows) __traits(getMember, this, "_blocking") = true; // lol longstanding phobos bug setting this to false on init 3973 super(af, type); 3974 initSsl(verifyPeer, hostname); 3975 } 3976 3977 override void close() scope { 3978 if(ssl) OpenSSL.SSL_shutdown(ssl); 3979 super.close(); 3980 } 3981 3982 this(socket_t sock, AddressFamily af, string hostname, bool verifyPeer = true) { 3983 super(sock, af); 3984 initSsl(verifyPeer, hostname); 3985 } 3986 3987 void freeSsl() { 3988 if(ssl is null) 3989 return; 3990 OpenSSL.SSL_free(ssl); 3991 OpenSSL.SSL_CTX_free(ctx); 3992 ssl = null; 3993 } 3994 3995 ~this() { 3996 freeSsl(); 3997 } 3998 } 3999 } 4000 4001 4002 /++ 4003 An experimental component for working with REST apis. Note that it 4004 is a zero-argument template, so to create one, use `new HttpApiClient!()(args..)` 4005 or you will get "HttpApiClient is used as a type" compile errors. 4006 4007 This will probably not work for you yet, and I might change it significantly. 4008 4009 Requires [arsd.jsvar]. 4010 4011 4012 Here's a snippet to create a pull request on GitHub to Phobos: 4013 4014 --- 4015 auto github = new HttpApiClient!()("https://api.github.com/", "your personal api token here"); 4016 4017 // create the arguments object 4018 // see: https://developer.github.com/v3/pulls/#create-a-pull-request 4019 var args = var.emptyObject; 4020 args.title = "My Pull Request"; 4021 args.head = "yourusername:" ~ branchName; 4022 args.base = "master"; 4023 // note it is ["body"] instead of .body because `body` is a D keyword 4024 args["body"] = "My cool PR is opened by the API!"; 4025 args.maintainer_can_modify = true; 4026 4027 /+ 4028 Fun fact, you can also write that: 4029 4030 var args = [ 4031 "title": "My Pull Request".var, 4032 "head": "yourusername:" ~ branchName.var, 4033 "base" : "master".var, 4034 "body" : "My cool PR is opened by the API!".var, 4035 "maintainer_can_modify": true.var 4036 ]; 4037 4038 Note the .var constructor calls in there. If everything is the same type, you actually don't need that, but here since there's strings and bools, D won't allow the literal without explicit constructors to align them all. 4039 +/ 4040 4041 // this translates to `repos/dlang/phobos/pulls` and sends a POST request, 4042 // containing `args` as json, then immediately grabs the json result and extracts 4043 // the value `html_url` from it. `prUrl` is typed `var`, from arsd.jsvar. 4044 auto prUrl = github.rest.repos.dlang.phobos.pulls.POST(args).result.html_url; 4045 4046 writeln("Created: ", prUrl); 4047 --- 4048 4049 Why use this instead of just building the URL? Well, of course you can! This just makes 4050 it a bit more convenient than string concatenation and manages a few headers for you. 4051 4052 Subtypes could potentially add static type checks too. 4053 +/ 4054 class HttpApiClient() { 4055 import arsd.jsvar; 4056 4057 HttpClient httpClient; 4058 4059 alias HttpApiClientType = typeof(this); 4060 4061 string urlBase; 4062 string oauth2Token; 4063 string submittedContentType; 4064 4065 /++ 4066 Params: 4067 4068 urlBase = The base url for the api. Tends to be something like `https://api.example.com/v2/` or similar. 4069 oauth2Token = the authorization token for the service. You'll have to get it from somewhere else. 4070 submittedContentType = the content-type of POST, PUT, etc. bodies. 4071 httpClient = an injected http client, or null if you want to use a default-constructed one 4072 4073 History: 4074 The `httpClient` param was added on December 26, 2020. 4075 +/ 4076 this(string urlBase, string oauth2Token, string submittedContentType = "application/json", HttpClient httpClient = null) { 4077 if(httpClient is null) 4078 this.httpClient = new HttpClient(); 4079 else 4080 this.httpClient = httpClient; 4081 4082 assert(urlBase[0] == 'h'); 4083 assert(urlBase[$-1] == '/'); 4084 4085 this.urlBase = urlBase; 4086 this.oauth2Token = oauth2Token; 4087 this.submittedContentType = submittedContentType; 4088 } 4089 4090 /// 4091 static struct HttpRequestWrapper { 4092 HttpApiClientType apiClient; /// 4093 HttpRequest request; /// 4094 HttpResponse _response; 4095 4096 /// 4097 this(HttpApiClientType apiClient, HttpRequest request) { 4098 this.apiClient = apiClient; 4099 this.request = request; 4100 } 4101 4102 /// Returns the full [HttpResponse] object so you can inspect the headers 4103 @property HttpResponse response() { 4104 if(_response is HttpResponse.init) 4105 _response = request.waitForCompletion(); 4106 return _response; 4107 } 4108 4109 /++ 4110 Returns the parsed JSON from the body of the response. 4111 4112 Throws on non-2xx responses. 4113 +/ 4114 var result() { 4115 return apiClient.throwOnError(response); 4116 } 4117 4118 alias request this; 4119 } 4120 4121 /// 4122 HttpRequestWrapper request(string uri, HttpVerb requestMethod = HttpVerb.GET, ubyte[] bodyBytes = null) { 4123 if(uri[0] == '/') 4124 uri = uri[1 .. $]; 4125 4126 auto u = Uri(uri).basedOn(Uri(urlBase)); 4127 4128 auto req = httpClient.navigateTo(u, requestMethod); 4129 4130 if(oauth2Token.length) 4131 req.requestParameters.headers ~= "Authorization: Bearer " ~ oauth2Token; 4132 req.requestParameters.contentType = submittedContentType; 4133 req.requestParameters.bodyData = bodyBytes; 4134 4135 return HttpRequestWrapper(this, req); 4136 } 4137 4138 /// 4139 var throwOnError(HttpResponse res) { 4140 if(res.code < 200 || res.code >= 300) 4141 throw new Exception(res.codeText ~ " " ~ res.contentText); 4142 4143 var response = var.fromJson(res.contentText); 4144 if(response.errors) { 4145 throw new Exception(response.errors.toJson()); 4146 } 4147 4148 return response; 4149 } 4150 4151 /// 4152 @property RestBuilder rest() { 4153 return RestBuilder(this, null, null); 4154 } 4155 4156 // hipchat.rest.room["Tech Team"].history 4157 // gives: "/room/Tech%20Team/history" 4158 // 4159 // hipchat.rest.room["Tech Team"].history("page", "12) 4160 /// 4161 static struct RestBuilder { 4162 HttpApiClientType apiClient; 4163 string[] pathParts; 4164 string[2][] queryParts; 4165 this(HttpApiClientType apiClient, string[] pathParts, string[2][] queryParts) { 4166 this.apiClient = apiClient; 4167 this.pathParts = pathParts; 4168 this.queryParts = queryParts; 4169 } 4170 4171 RestBuilder _SELF() { 4172 return this; 4173 } 4174 4175 /// The args are so you can call opCall on the returned 4176 /// object, despite @property being broken af in D. 4177 RestBuilder opDispatch(string str, T)(string n, T v) { 4178 return RestBuilder(apiClient, pathParts ~ str, queryParts ~ [n, to!string(v)]); 4179 } 4180 4181 /// 4182 RestBuilder opDispatch(string str)() { 4183 return RestBuilder(apiClient, pathParts ~ str, queryParts); 4184 } 4185 4186 4187 /// 4188 RestBuilder opIndex(string str) { 4189 return RestBuilder(apiClient, pathParts ~ str, queryParts); 4190 } 4191 /// 4192 RestBuilder opIndex(var str) { 4193 return RestBuilder(apiClient, pathParts ~ str.get!string, queryParts); 4194 } 4195 /// 4196 RestBuilder opIndex(int i) { 4197 return RestBuilder(apiClient, pathParts ~ to!string(i), queryParts); 4198 } 4199 4200 /// 4201 RestBuilder opCall(T)(string name, T value) { 4202 return RestBuilder(apiClient, pathParts, queryParts ~ [name, to!string(value)]); 4203 } 4204 4205 /// 4206 string toUri() { 4207 import std.uri; 4208 string result; 4209 foreach(idx, part; pathParts) { 4210 if(idx) 4211 result ~= "/"; 4212 result ~= encodeComponent(part); 4213 } 4214 result ~= "?"; 4215 foreach(idx, part; queryParts) { 4216 if(idx) 4217 result ~= "&"; 4218 result ~= encodeComponent(part[0]); 4219 result ~= "="; 4220 result ~= encodeComponent(part[1]); 4221 } 4222 4223 return result; 4224 } 4225 4226 /// 4227 final HttpRequestWrapper GET() { return _EXECUTE(HttpVerb.GET, this.toUri(), ToBytesResult.init); } 4228 /// ditto 4229 final HttpRequestWrapper DELETE() { return _EXECUTE(HttpVerb.DELETE, this.toUri(), ToBytesResult.init); } 4230 4231 // need to be able to send: JSON, urlencoded, multipart/form-data, and raw stuff. 4232 /// ditto 4233 final HttpRequestWrapper POST(T...)(T t) { return _EXECUTE(HttpVerb.POST, this.toUri(), toBytes(t)); } 4234 /// ditto 4235 final HttpRequestWrapper PATCH(T...)(T t) { return _EXECUTE(HttpVerb.PATCH, this.toUri(), toBytes(t)); } 4236 /// ditto 4237 final HttpRequestWrapper PUT(T...)(T t) { return _EXECUTE(HttpVerb.PUT, this.toUri(), toBytes(t)); } 4238 4239 struct ToBytesResult { 4240 ubyte[] bytes; 4241 string contentType; 4242 } 4243 4244 private ToBytesResult toBytes(T...)(T t) { 4245 import std.conv : to; 4246 static if(T.length == 0) 4247 return ToBytesResult(null, null); 4248 else static if(T.length == 1 && is(T[0] == var)) 4249 return ToBytesResult(cast(ubyte[]) t[0].toJson(), "application/json"); // json data 4250 else static if(T.length == 1 && (is(T[0] == string) || is(T[0] == ubyte[]))) 4251 return ToBytesResult(cast(ubyte[]) t[0], null); // raw data 4252 else static if(T.length == 1 && is(T[0] : FormData)) 4253 return ToBytesResult(t[0].toBytes, t[0].contentType); 4254 else static if(T.length > 1 && T.length % 2 == 0 && is(T[0] == string)) { 4255 // string -> value pairs for a POST request 4256 string answer; 4257 foreach(idx, val; t) { 4258 static if(idx % 2 == 0) { 4259 if(answer.length) 4260 answer ~= "&"; 4261 answer ~= encodeComponent(val); // it had better be a string! lol 4262 answer ~= "="; 4263 } else { 4264 answer ~= encodeComponent(to!string(val)); 4265 } 4266 } 4267 4268 return ToBytesResult(cast(ubyte[]) answer, "application/x-www-form-urlencoded"); 4269 } 4270 else 4271 static assert(0); // FIXME 4272 4273 } 4274 4275 HttpRequestWrapper _EXECUTE(HttpVerb verb, string uri, ubyte[] bodyBytes) { 4276 return apiClient.request(uri, verb, bodyBytes); 4277 } 4278 4279 HttpRequestWrapper _EXECUTE(HttpVerb verb, string uri, ToBytesResult tbr) { 4280 auto r = apiClient.request(uri, verb, tbr.bytes); 4281 if(tbr.contentType !is null) 4282 r.requestParameters.contentType = tbr.contentType; 4283 return r; 4284 } 4285 } 4286 } 4287 4288 4289 // see also: arsd.cgi.encodeVariables 4290 /// Creates a multipart/form-data object that is suitable for file uploads and other kinds of POST 4291 class FormData { 4292 struct MimePart { 4293 string name; 4294 const(void)[] data; 4295 string contentType; 4296 string filename; 4297 } 4298 4299 MimePart[] parts; 4300 4301 /// 4302 void append(string key, in void[] value, string contentType = null, string filename = null) { 4303 parts ~= MimePart(key, value, contentType, filename); 4304 } 4305 4306 private string boundary = "0016e64be86203dd36047610926a"; // FIXME 4307 4308 string contentType() { 4309 return "multipart/form-data; boundary=" ~ boundary; 4310 } 4311 4312 /// 4313 ubyte[] toBytes() { 4314 string data; 4315 4316 foreach(part; parts) { 4317 data ~= "--" ~ boundary ~ "\r\n"; 4318 data ~= "Content-Disposition: form-data; name=\""~part.name~"\""; 4319 if(part.filename !is null) 4320 data ~= "; filename=\""~part.filename~"\""; 4321 data ~= "\r\n"; 4322 if(part.contentType !is null) 4323 data ~= "Content-Type: " ~ part.contentType ~ "\r\n"; 4324 data ~= "\r\n"; 4325 4326 data ~= cast(string) part.data; 4327 4328 data ~= "\r\n"; 4329 } 4330 4331 data ~= "--" ~ boundary ~ "--\r\n"; 4332 4333 return cast(ubyte[]) data; 4334 } 4335 } 4336 4337 private bool bicmp(in ubyte[] item, in char[] search) { 4338 if(item.length != search.length) return false; 4339 4340 foreach(i; 0 .. item.length) { 4341 ubyte a = item[i]; 4342 ubyte b = search[i]; 4343 if(a >= 'A' && a <= 'Z') 4344 a += 32; 4345 //if(b >= 'A' && b <= 'Z') 4346 //b += 32; 4347 if(a != b) 4348 return false; 4349 } 4350 4351 return true; 4352 } 4353 4354 /++ 4355 WebSocket client, based on the browser api, though also with other api options. 4356 4357 --- 4358 import arsd.http2; 4359 4360 void main() { 4361 auto ws = new WebSocket(Uri("ws://....")); 4362 4363 ws.onmessage = (in char[] msg) { 4364 ws.send("a reply"); 4365 }; 4366 4367 ws.connect(); 4368 4369 WebSocket.eventLoop(); 4370 } 4371 --- 4372 4373 Symbol_groups: 4374 foundational = 4375 Used with all API styles. 4376 4377 browser_api = 4378 API based on the standard in the browser. 4379 4380 event_loop_integration = 4381 Integrating with external event loops is done through static functions. You should 4382 call these BEFORE doing anything else with the WebSocket module or class. 4383 4384 $(PITFALL NOT IMPLEMENTED) 4385 --- 4386 WebSocket.setEventLoopProxy(arsd.simpledisplay.EventLoop.proxy.tupleof); 4387 // or something like that. it is not implemented yet. 4388 --- 4389 $(PITFALL NOT IMPLEMENTED) 4390 4391 blocking_api = 4392 The blocking API is best used when you only need basic functionality with a single connection. 4393 4394 --- 4395 WebSocketFrame msg; 4396 do { 4397 // FIXME good demo 4398 } while(msg); 4399 --- 4400 4401 Or to check for blocks before calling: 4402 4403 --- 4404 try_to_process_more: 4405 while(ws.isMessageBuffered()) { 4406 auto msg = ws.waitForNextMessage(); 4407 // process msg 4408 } 4409 if(ws.isDataPending()) { 4410 ws.lowLevelReceive(); 4411 goto try_to_process_more; 4412 } else { 4413 // nothing ready, you can do other things 4414 // or at least sleep a while before trying 4415 // to process more. 4416 if(ws.readyState == WebSocket.OPEN) { 4417 Thread.sleep(1.seconds); 4418 goto try_to_process_more; 4419 } 4420 } 4421 --- 4422 4423 +/ 4424 class WebSocket { 4425 private Uri uri; 4426 private string[string] cookies; 4427 4428 private string host; 4429 private ushort port; 4430 private bool ssl; 4431 4432 // used to decide if we mask outgoing msgs 4433 private bool isClient; 4434 4435 private MonoTime timeoutFromInactivity; 4436 private MonoTime nextPing; 4437 4438 /++ 4439 wss://echo.websocket.org 4440 +/ 4441 /// Group: foundational 4442 this(Uri uri, Config config = Config.init) 4443 //in (uri.scheme == "ws" || uri.scheme == "wss") 4444 in { assert(uri.scheme == "ws" || uri.scheme == "wss"); } do 4445 { 4446 this.uri = uri; 4447 this.config = config; 4448 4449 this.receiveBuffer = new ubyte[](config.initialReceiveBufferSize); 4450 4451 host = uri.host; 4452 ssl = uri.scheme == "wss"; 4453 port = cast(ushort) (uri.port ? uri.port : ssl ? 443 : 80); 4454 4455 if(ssl) { 4456 version(with_openssl) { 4457 loadOpenSsl(); 4458 socket = new SslClientSocket(family(uri.unixSocketPath), SocketType.STREAM, host, config.verifyPeer); 4459 } else 4460 throw new Exception("SSL not compiled in"); 4461 } else 4462 socket = new Socket(family(uri.unixSocketPath), SocketType.STREAM); 4463 4464 } 4465 4466 /++ 4467 4468 +/ 4469 /// Group: foundational 4470 void connect() { 4471 this.isClient = true; 4472 if(uri.unixSocketPath) 4473 socket.connect(new UnixAddress(uri.unixSocketPath)); 4474 else 4475 socket.connect(new InternetAddress(host, port)); // FIXME: ipv6 support... 4476 // FIXME: websocket handshake could and really should be async too. 4477 4478 auto uri = this.uri.path.length ? this.uri.path : "/"; 4479 if(this.uri.query.length) { 4480 uri ~= "?"; 4481 uri ~= this.uri.query; 4482 } 4483 4484 // the headers really shouldn't be bigger than this, at least 4485 // the chunks i need to process 4486 ubyte[4096] bufferBacking = void; 4487 ubyte[] buffer = bufferBacking[]; 4488 size_t pos; 4489 4490 void append(in char[][] items...) { 4491 foreach(what; items) { 4492 if((pos + what.length) > buffer.length) { 4493 buffer.length += 4096; 4494 } 4495 buffer[pos .. pos + what.length] = cast(ubyte[]) what[]; 4496 pos += what.length; 4497 } 4498 } 4499 4500 append("GET ", uri, " HTTP/1.1\r\n"); 4501 append("Host: ", this.uri.host, "\r\n"); 4502 4503 append("Upgrade: websocket\r\n"); 4504 append("Connection: Upgrade\r\n"); 4505 append("Sec-WebSocket-Version: 13\r\n"); 4506 4507 // FIXME: randomize this 4508 append("Sec-WebSocket-Key: x3JEHMbDL1EzLkh9GBhXDw==\r\n"); 4509 4510 if(config.protocol.length) 4511 append("Sec-WebSocket-Protocol: ", config.protocol, "\r\n"); 4512 if(config.origin.length) 4513 append("Origin: ", config.origin, "\r\n"); 4514 4515 foreach(h; config.additionalHeaders) { 4516 append(h); 4517 append("\r\n"); 4518 } 4519 4520 append("\r\n"); 4521 4522 auto remaining = buffer[0 .. pos]; 4523 //import std.stdio; writeln(host, " " , port, " ", cast(string) remaining); 4524 while(remaining.length) { 4525 auto r = socket.send(remaining); 4526 if(r < 0) 4527 throw new Exception(lastSocketError()); 4528 if(r == 0) 4529 throw new Exception("unexpected connection termination"); 4530 remaining = remaining[r .. $]; 4531 } 4532 4533 // the response shouldn't be especially large at this point, just 4534 // headers for the most part. gonna try to get it in the stack buffer. 4535 // then copy stuff after headers, if any, to the frame buffer. 4536 ubyte[] used; 4537 4538 void more() { 4539 auto r = socket.receive(buffer[used.length .. $]); 4540 4541 if(r < 0) 4542 throw new Exception(lastSocketError()); 4543 if(r == 0) 4544 throw new Exception("unexpected connection termination"); 4545 //import std.stdio;writef("%s", cast(string) buffer[used.length .. used.length + r]); 4546 4547 used = buffer[0 .. used.length + r]; 4548 } 4549 4550 more(); 4551 4552 import std.algorithm; 4553 if(!used.startsWith(cast(ubyte[]) "HTTP/1.1 101")) 4554 throw new Exception("didn't get a websocket answer"); 4555 // skip the status line 4556 while(used.length && used[0] != '\n') 4557 used = used[1 .. $]; 4558 4559 if(used.length == 0) 4560 throw new Exception("Remote server disconnected or didn't send enough information"); 4561 4562 if(used.length < 1) 4563 more(); 4564 4565 used = used[1 .. $]; // skip the \n 4566 4567 if(used.length == 0) 4568 more(); 4569 4570 // checks on the protocol from ehaders 4571 bool isWebsocket; 4572 bool isUpgrade; 4573 const(ubyte)[] protocol; 4574 const(ubyte)[] accept; 4575 4576 while(used.length) { 4577 if(used.length >= 2 && used[0] == '\r' && used[1] == '\n') { 4578 used = used[2 .. $]; 4579 break; // all done 4580 } 4581 int idxColon; 4582 while(idxColon < used.length && used[idxColon] != ':') 4583 idxColon++; 4584 if(idxColon == used.length) 4585 more(); 4586 auto idxStart = idxColon + 1; 4587 while(idxStart < used.length && used[idxStart] == ' ') 4588 idxStart++; 4589 if(idxStart == used.length) 4590 more(); 4591 auto idxEnd = idxStart; 4592 while(idxEnd < used.length && used[idxEnd] != '\r') 4593 idxEnd++; 4594 if(idxEnd == used.length) 4595 more(); 4596 4597 auto headerName = used[0 .. idxColon]; 4598 auto headerValue = used[idxStart .. idxEnd]; 4599 4600 // move past this header 4601 used = used[idxEnd .. $]; 4602 // and the \r\n 4603 if(2 <= used.length) 4604 used = used[2 .. $]; 4605 4606 if(headerName.bicmp("upgrade")) { 4607 if(headerValue.bicmp("websocket")) 4608 isWebsocket = true; 4609 } else if(headerName.bicmp("connection")) { 4610 if(headerValue.bicmp("upgrade")) 4611 isUpgrade = true; 4612 } else if(headerName.bicmp("sec-websocket-accept")) { 4613 accept = headerValue; 4614 } else if(headerName.bicmp("sec-websocket-protocol")) { 4615 protocol = headerValue; 4616 } 4617 4618 if(!used.length) { 4619 more(); 4620 } 4621 } 4622 4623 4624 if(!isWebsocket) 4625 throw new Exception("didn't answer as websocket"); 4626 if(!isUpgrade) 4627 throw new Exception("didn't answer as upgrade"); 4628 4629 4630 // FIXME: check protocol if config requested one 4631 // FIXME: check accept for the right hash 4632 4633 receiveBuffer[0 .. used.length] = used[]; 4634 receiveBufferUsedLength = used.length; 4635 4636 readyState_ = OPEN; 4637 4638 if(onopen) 4639 onopen(); 4640 4641 nextPing = MonoTime.currTime + config.pingFrequency.msecs; 4642 timeoutFromInactivity = MonoTime.currTime + config.timeoutFromInactivity; 4643 4644 registerActiveSocket(this); 4645 } 4646 4647 /++ 4648 Is data pending on the socket? Also check [isMessageBuffered] to see if there 4649 is already a message in memory too. 4650 4651 If this returns `true`, you can call [lowLevelReceive], then try [isMessageBuffered] 4652 again. 4653 +/ 4654 /// Group: blocking_api 4655 public bool isDataPending(Duration timeout = 0.seconds) { 4656 static SocketSet readSet; 4657 if(readSet is null) 4658 readSet = new SocketSet(); 4659 4660 version(with_openssl) 4661 if(auto s = cast(SslClientSocket) socket) { 4662 // select doesn't handle the case with stuff 4663 // left in the ssl buffer so i'm checking it separately 4664 if(s.dataPending()) { 4665 return true; 4666 } 4667 } 4668 4669 readSet.add(socket); 4670 4671 //tryAgain: 4672 auto selectGot = Socket.select(readSet, null, null, timeout); 4673 if(selectGot == 0) { /* timeout */ 4674 // timeout 4675 return false; 4676 } else if(selectGot == -1) { /* interrupted */ 4677 return false; 4678 } else { /* ready */ 4679 if(readSet.isSet(socket)) { 4680 return true; 4681 } 4682 } 4683 4684 return false; 4685 } 4686 4687 private void llsend(ubyte[] d) { 4688 if(readyState == CONNECTING) 4689 throw new Exception("WebSocket not connected when trying to send. Did you forget to call connect(); ?"); 4690 //connect(); 4691 //import std.stdio; writeln("LLSEND: ", d); 4692 while(d.length) { 4693 auto r = socket.send(d); 4694 if(r < 0 && wouldHaveBlocked()) { 4695 import core.thread; 4696 Thread.sleep(1.msecs); 4697 continue; 4698 } 4699 //import core.stdc.errno; import std.stdio; writeln(errno); 4700 if(r <= 0) { 4701 // import std.stdio; writeln(GetLastError()); 4702 throw new Exception("Socket send failed"); 4703 } 4704 d = d[r .. $]; 4705 } 4706 } 4707 4708 private void llclose() { 4709 // import std.stdio; writeln("LLCLOSE"); 4710 socket.shutdown(SocketShutdown.SEND); 4711 } 4712 4713 /++ 4714 Waits for more data off the low-level socket and adds it to the pending buffer. 4715 4716 Returns `true` if the connection is still active. 4717 +/ 4718 /// Group: blocking_api 4719 public bool lowLevelReceive() { 4720 if(readyState == CONNECTING) 4721 throw new Exception("WebSocket not connected when trying to receive. Did you forget to call connect(); ?"); 4722 if (receiveBufferUsedLength == receiveBuffer.length) 4723 { 4724 if (receiveBuffer.length == config.maximumReceiveBufferSize) 4725 throw new Exception("Maximum receive buffer size exhausted"); 4726 4727 import std.algorithm : min; 4728 receiveBuffer.length = min(receiveBuffer.length + config.initialReceiveBufferSize, 4729 config.maximumReceiveBufferSize); 4730 } 4731 auto r = socket.receive(receiveBuffer[receiveBufferUsedLength .. $]); 4732 if(r == 0) 4733 return false; 4734 if(r < 0 && wouldHaveBlocked()) 4735 return true; 4736 if(r <= 0) { 4737 //import std.stdio; writeln(WSAGetLastError()); 4738 throw new Exception("Socket receive failed"); 4739 } 4740 receiveBufferUsedLength += r; 4741 return true; 4742 } 4743 4744 private Socket socket; 4745 4746 /* copy/paste section { */ 4747 4748 private int readyState_; 4749 private ubyte[] receiveBuffer; 4750 private size_t receiveBufferUsedLength; 4751 4752 private Config config; 4753 4754 enum CONNECTING = 0; /// Socket has been created. The connection is not yet open. 4755 enum OPEN = 1; /// The connection is open and ready to communicate. 4756 enum CLOSING = 2; /// The connection is in the process of closing. 4757 enum CLOSED = 3; /// The connection is closed or couldn't be opened. 4758 4759 /++ 4760 4761 +/ 4762 /// Group: foundational 4763 static struct Config { 4764 /++ 4765 These control the size of the receive buffer. 4766 4767 It starts at the initial size, will temporarily 4768 balloon up to the maximum size, and will reuse 4769 a buffer up to the likely size. 4770 4771 Anything larger than the maximum size will cause 4772 the connection to be aborted and an exception thrown. 4773 This is to protect you against a peer trying to 4774 exhaust your memory, while keeping the user-level 4775 processing simple. 4776 +/ 4777 size_t initialReceiveBufferSize = 4096; 4778 size_t likelyReceiveBufferSize = 4096; /// ditto 4779 size_t maximumReceiveBufferSize = 10 * 1024 * 1024; /// ditto 4780 4781 /++ 4782 Maximum combined size of a message. 4783 +/ 4784 size_t maximumMessageSize = 10 * 1024 * 1024; 4785 4786 string[string] cookies; /// Cookies to send with the initial request. cookies[name] = value; 4787 string origin; /// Origin URL to send with the handshake, if desired. 4788 string protocol; /// the protocol header, if desired. 4789 4790 /++ 4791 Additional headers to put in the HTTP request. These should be formatted `Name: value`, like for example: 4792 4793 --- 4794 Config config; 4795 config.additionalHeaders ~= "Authorization: Bearer your_auth_token_here"; 4796 --- 4797 4798 History: 4799 Added February 19, 2021 (included in dub version 9.2) 4800 +/ 4801 string[] additionalHeaders; 4802 4803 int pingFrequency = 5000; /// Amount of time (in msecs) of idleness after which to send an automatic ping 4804 4805 /++ 4806 Amount of time to disconnect when there's no activity. Note that automatic pings will keep the connection alive; this timeout only occurs if there's absolutely nothing, including no responses to websocket ping frames. Since the default [pingFrequency] is only seconds, this one minute should never elapse unless the connection is actually dead. 4807 4808 The one thing to keep in mind is if your program is busy and doesn't check input, it might consider this a time out since there's no activity. The reason is that your program was busy rather than a connection failure, but it doesn't care. You should avoid long processing periods anyway though! 4809 4810 History: 4811 Added March 31, 2021 (included in dub version 9.4) 4812 +/ 4813 Duration timeoutFromInactivity = 1.minutes; 4814 4815 /++ 4816 For https connections, if this is `true`, it will fail to connect if the TLS certificate can not be 4817 verified. Setting this to `false` will skip this check and allow the connection to continue anyway. 4818 4819 History: 4820 Added April 5, 2022 (dub v10.8) 4821 4822 Prior to this, it always used the global (but undocumented) `defaultVerifyPeer` setting, and sometimes 4823 even if it was true, it would skip the verification. Now, it always respects this local setting. 4824 +/ 4825 bool verifyPeer = true; 4826 } 4827 4828 /++ 4829 Returns one of [CONNECTING], [OPEN], [CLOSING], or [CLOSED]. 4830 +/ 4831 int readyState() { 4832 return readyState_; 4833 } 4834 4835 /++ 4836 Closes the connection, sending a graceful teardown message to the other side. 4837 +/ 4838 /// Group: foundational 4839 void close(int code = 0, string reason = null) 4840 //in (reason.length < 123) 4841 in { assert(reason.length < 123); } do 4842 { 4843 if(readyState_ != OPEN) 4844 return; // it cool, we done 4845 WebSocketFrame wss; 4846 wss.fin = true; 4847 wss.masked = this.isClient; 4848 wss.opcode = WebSocketOpcode.close; 4849 wss.data = cast(ubyte[]) reason.dup; 4850 wss.send(&llsend); 4851 4852 readyState_ = CLOSING; 4853 4854 llclose(); 4855 } 4856 4857 /++ 4858 Sends a ping message to the server. This is done automatically by the library if you set a non-zero [Config.pingFrequency], but you can also send extra pings explicitly as well with this function. 4859 +/ 4860 /// Group: foundational 4861 void ping(in ubyte[] data = null) { 4862 WebSocketFrame wss; 4863 wss.fin = true; 4864 wss.masked = this.isClient; 4865 wss.opcode = WebSocketOpcode.ping; 4866 if(data !is null) wss.data = data.dup; 4867 wss.send(&llsend); 4868 } 4869 4870 /++ 4871 Sends a pong message to the server. This is normally done automatically in response to pings. 4872 +/ 4873 /// Group: foundational 4874 void pong(in ubyte[] data = null) { 4875 WebSocketFrame wss; 4876 wss.fin = true; 4877 wss.masked = this.isClient; 4878 wss.opcode = WebSocketOpcode.pong; 4879 wss.send(&llsend); 4880 if(data !is null) wss.data = data.dup; 4881 } 4882 4883 /++ 4884 Sends a text message through the websocket. 4885 +/ 4886 /// Group: foundational 4887 void send(in char[] textData) { 4888 WebSocketFrame wss; 4889 wss.fin = true; 4890 wss.masked = this.isClient; 4891 wss.opcode = WebSocketOpcode.text; 4892 wss.data = cast(ubyte[]) textData.dup; 4893 wss.send(&llsend); 4894 } 4895 4896 /++ 4897 Sends a binary message through the websocket. 4898 +/ 4899 /// Group: foundational 4900 void send(in ubyte[] binaryData) { 4901 WebSocketFrame wss; 4902 wss.masked = this.isClient; 4903 wss.fin = true; 4904 wss.opcode = WebSocketOpcode.binary; 4905 wss.data = cast(ubyte[]) binaryData.dup; 4906 wss.send(&llsend); 4907 } 4908 4909 /++ 4910 Waits for and returns the next complete message on the socket. 4911 4912 Note that the onmessage function is still called, right before 4913 this returns. 4914 +/ 4915 /// Group: blocking_api 4916 public WebSocketFrame waitForNextMessage() { 4917 do { 4918 auto m = processOnce(); 4919 if(m.populated) 4920 return m; 4921 } while(lowLevelReceive()); 4922 4923 return WebSocketFrame.init; // FIXME? maybe. 4924 } 4925 4926 /++ 4927 Tells if [waitForNextMessage] would block. 4928 +/ 4929 /// Group: blocking_api 4930 public bool waitForNextMessageWouldBlock() { 4931 checkAgain: 4932 if(isMessageBuffered()) 4933 return false; 4934 if(!isDataPending()) 4935 return true; 4936 while(isDataPending()) 4937 lowLevelReceive(); 4938 goto checkAgain; 4939 } 4940 4941 /++ 4942 Is there a message in the buffer already? 4943 If `true`, [waitForNextMessage] is guaranteed to return immediately. 4944 If `false`, check [isDataPending] as the next step. 4945 +/ 4946 /// Group: blocking_api 4947 public bool isMessageBuffered() { 4948 ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; 4949 auto s = d; 4950 if(d.length) { 4951 auto orig = d; 4952 auto m = WebSocketFrame.read(d); 4953 // that's how it indicates that it needs more data 4954 if(d !is orig) 4955 return true; 4956 } 4957 4958 return false; 4959 } 4960 4961 private ubyte continuingType; 4962 private ubyte[] continuingData; 4963 //private size_t continuingDataLength; 4964 4965 private WebSocketFrame processOnce() { 4966 ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; 4967 auto s = d; 4968 // FIXME: handle continuation frames more efficiently. it should really just reuse the receive buffer. 4969 WebSocketFrame m; 4970 if(d.length) { 4971 auto orig = d; 4972 m = WebSocketFrame.read(d); 4973 // that's how it indicates that it needs more data 4974 if(d is orig) 4975 return WebSocketFrame.init; 4976 m.unmaskInPlace(); 4977 switch(m.opcode) { 4978 case WebSocketOpcode.continuation: 4979 if(continuingData.length + m.data.length > config.maximumMessageSize) 4980 throw new Exception("message size exceeded"); 4981 4982 continuingData ~= m.data; 4983 if(m.fin) { 4984 if(ontextmessage) 4985 ontextmessage(cast(char[]) continuingData); 4986 if(onbinarymessage) 4987 onbinarymessage(continuingData); 4988 4989 continuingData = null; 4990 } 4991 break; 4992 case WebSocketOpcode.text: 4993 if(m.fin) { 4994 if(ontextmessage) 4995 ontextmessage(m.textData); 4996 } else { 4997 continuingType = m.opcode; 4998 //continuingDataLength = 0; 4999 continuingData = null; 5000 continuingData ~= m.data; 5001 } 5002 break; 5003 case WebSocketOpcode.binary: 5004 if(m.fin) { 5005 if(onbinarymessage) 5006 onbinarymessage(m.data); 5007 } else { 5008 continuingType = m.opcode; 5009 //continuingDataLength = 0; 5010 continuingData = null; 5011 continuingData ~= m.data; 5012 } 5013 break; 5014 case WebSocketOpcode.close: 5015 5016 //import std.stdio; writeln("closed ", cast(string) m.data); 5017 readyState_ = CLOSED; 5018 if(onclose) 5019 onclose(); 5020 5021 unregisterActiveSocket(this); 5022 break; 5023 case WebSocketOpcode.ping: 5024 // import std.stdio; writeln("ping received ", m.data); 5025 pong(m.data); 5026 break; 5027 case WebSocketOpcode.pong: 5028 // import std.stdio; writeln("pong received ", m.data); 5029 // just really references it is still alive, nbd. 5030 break; 5031 default: // ignore though i could and perhaps should throw too 5032 } 5033 } 5034 5035 if(d.length) { 5036 m.data = m.data.dup(); 5037 } 5038 5039 import core.stdc.string; 5040 memmove(receiveBuffer.ptr, d.ptr, d.length); 5041 receiveBufferUsedLength = d.length; 5042 5043 return m; 5044 } 5045 5046 private void autoprocess() { 5047 // FIXME 5048 do { 5049 processOnce(); 5050 } while(lowLevelReceive()); 5051 } 5052 5053 5054 void delegate() onclose; /// 5055 void delegate() onerror; /// 5056 void delegate(in char[]) ontextmessage; /// 5057 void delegate(in ubyte[]) onbinarymessage; /// 5058 void delegate() onopen; /// 5059 5060 /++ 5061 5062 +/ 5063 /// Group: browser_api 5064 void onmessage(void delegate(in char[]) dg) { 5065 ontextmessage = dg; 5066 } 5067 5068 /// ditto 5069 void onmessage(void delegate(in ubyte[]) dg) { 5070 onbinarymessage = dg; 5071 } 5072 5073 /* } end copy/paste */ 5074 5075 /* 5076 const int bufferedAmount // amount pending 5077 const string extensions 5078 5079 const string protocol 5080 const string url 5081 */ 5082 5083 static { 5084 /++ 5085 Runs an event loop with all known websockets on this thread until all websockets 5086 are closed or unregistered, or until you call [exitEventLoop], or set `*localLoopExited` 5087 to false (please note it may take a few seconds until it checks that flag again; it may 5088 not exit immediately). 5089 5090 History: 5091 The `localLoopExited` parameter was added August 22, 2022 (dub v10.9) 5092 5093 See_Also: 5094 [addToSimpledisplayEventLoop] 5095 +/ 5096 void eventLoop(shared(bool)* localLoopExited = null) { 5097 import core.atomic; 5098 atomicOp!"+="(numberOfEventLoops, 1); 5099 scope(exit) { 5100 if(atomicOp!"-="(numberOfEventLoops, 1) <= 0) 5101 loopExited = false; // reset it so we can reenter 5102 } 5103 5104 static SocketSet readSet; 5105 5106 if(readSet is null) 5107 readSet = new SocketSet(); 5108 5109 loopExited = false; 5110 5111 outermost: while(!loopExited && (localLoopExited is null || (*localLoopExited == false))) { 5112 readSet.reset(); 5113 5114 Duration timeout = 3.seconds; 5115 5116 auto now = MonoTime.currTime; 5117 bool hadAny; 5118 foreach(sock; activeSockets) { 5119 auto diff = sock.timeoutFromInactivity - now; 5120 if(diff <= 0.msecs) { 5121 // timeout 5122 if(sock.onerror) 5123 sock.onerror(); 5124 5125 sock.socket.close(); 5126 sock.readyState_ = CLOSED; 5127 unregisterActiveSocket(sock); 5128 continue outermost; 5129 } 5130 5131 if(diff < timeout) 5132 timeout = diff; 5133 5134 diff = sock.nextPing - now; 5135 5136 if(diff <= 0.msecs) { 5137 //sock.send(`{"action": "ping"}`); 5138 sock.ping(); 5139 sock.nextPing = now + sock.config.pingFrequency.msecs; 5140 } else { 5141 if(diff < timeout) 5142 timeout = diff; 5143 } 5144 5145 readSet.add(sock.socket); 5146 hadAny = true; 5147 } 5148 5149 if(!hadAny) { 5150 // import std.stdio; writeln("had none"); 5151 return; 5152 } 5153 5154 tryAgain: 5155 // import std.stdio; writeln(timeout); 5156 auto selectGot = Socket.select(readSet, null, null, timeout); 5157 if(selectGot == 0) { /* timeout */ 5158 // timeout 5159 continue; // it will be handled at the top of the loop 5160 } else if(selectGot == -1) { /* interrupted */ 5161 goto tryAgain; 5162 } else { 5163 foreach(sock; activeSockets) { 5164 if(readSet.isSet(sock.socket)) { 5165 sock.timeoutFromInactivity = MonoTime.currTime + sock.config.timeoutFromInactivity; 5166 if(!sock.lowLevelReceive()) { 5167 sock.readyState_ = CLOSED; 5168 unregisterActiveSocket(sock); 5169 continue outermost; 5170 } 5171 while(sock.processOnce().populated) {} 5172 selectGot--; 5173 if(selectGot <= 0) 5174 break; 5175 } 5176 } 5177 } 5178 } 5179 } 5180 5181 private static shared(int) numberOfEventLoops; 5182 5183 private __gshared bool loopExited; 5184 /++ 5185 Exits all running [WebSocket.eventLoop]s next time they loop around. You can call this from a signal handler or another thread. 5186 5187 Please note they may not loop around to check the flag for several seconds. Any new event loops will exit immediately until 5188 all current ones are closed. Once all event loops are exited, the flag is cleared and you can start the loop again. 5189 5190 This function is likely to be deprecated in the future due to its quirks and imprecise name. 5191 +/ 5192 void exitEventLoop() { 5193 loopExited = true; 5194 } 5195 5196 WebSocket[] activeSockets; 5197 5198 void registerActiveSocket(WebSocket s) { 5199 // ensure it isn't already there... 5200 assert(s !is null); 5201 foreach(i, a; activeSockets) 5202 if(a is s) 5203 return; 5204 activeSockets ~= s; 5205 } 5206 void unregisterActiveSocket(WebSocket s) { 5207 foreach(i, a; activeSockets) 5208 if(s is a) { 5209 activeSockets[i] = activeSockets[$-1]; 5210 activeSockets = activeSockets[0 .. $-1]; 5211 break; 5212 } 5213 } 5214 } 5215 } 5216 5217 private template imported(string mod) { 5218 mixin(`import imported = ` ~ mod ~ `;`); 5219 } 5220 5221 /++ 5222 Warning: you should call this AFTER websocket.connect or else it might throw on connect because the function sets nonblocking mode and the connect function doesn't handle that well (it throws on the "would block" condition in that function. easier to just do that first) 5223 +/ 5224 template addToSimpledisplayEventLoop() { 5225 import arsd.simpledisplay; 5226 void addToSimpledisplayEventLoop(WebSocket ws, imported!"arsd.simpledisplay".SimpleWindow window) { 5227 5228 void midprocess() { 5229 if(!ws.lowLevelReceive()) { 5230 ws.readyState_ = WebSocket.CLOSED; 5231 WebSocket.unregisterActiveSocket(ws); 5232 return; 5233 } 5234 while(ws.processOnce().populated) {} 5235 } 5236 5237 version(Posix) { 5238 auto reader = new PosixFdReader(&midprocess, ws.socket.handle); 5239 } else version(none) { 5240 if(WSAAsyncSelect(ws.socket.handle, window.hwnd, WM_USER + 150, FD_CLOSE | FD_READ)) 5241 throw new Exception("WSAAsyncSelect"); 5242 5243 window.handleNativeEvent = delegate int(HWND hwnd, UINT msg, WPARAM wParam, LPARAM lParam) { 5244 if(hwnd !is window.impl.hwnd) 5245 return 1; // we don't care... 5246 switch(msg) { 5247 case WM_USER + 150: // socket activity 5248 switch(LOWORD(lParam)) { 5249 case FD_READ: 5250 case FD_CLOSE: 5251 midprocess(); 5252 break; 5253 default: 5254 // nothing 5255 } 5256 break; 5257 default: return 1; // not handled, pass it on 5258 } 5259 return 0; 5260 }; 5261 5262 } else version(Windows) { 5263 ws.socket.blocking = false; // the WSAEventSelect does this anyway and doing it here lets phobos know about it. 5264 //CreateEvent(null, 0, 0, null); 5265 auto event = WSACreateEvent(); 5266 if(!event) { 5267 throw new Exception("WSACreateEvent"); 5268 } 5269 if(WSAEventSelect(ws.socket.handle, event, 1/*FD_READ*/ | (1<<5)/*FD_CLOSE*/)) { 5270 //import std.stdio; writeln(WSAGetLastError()); 5271 throw new Exception("WSAEventSelect"); 5272 } 5273 5274 auto handle = new WindowsHandleReader(&midprocess, event); 5275 5276 /+ 5277 static class Ready {} 5278 5279 Ready thisr = new Ready; 5280 5281 justCommunication.addEventListener((Ready r) { 5282 if(r is thisr) 5283 midprocess(); 5284 }); 5285 5286 import core.thread; 5287 auto thread = new Thread({ 5288 while(true) { 5289 WSAWaitForMultipleEvents(1, &event, true, -1/*WSA_INFINITE*/, false); 5290 justCommunication.postEvent(thisr); 5291 } 5292 }); 5293 thread.isDaemon = true; 5294 thread.start; 5295 +/ 5296 5297 } else static assert(0, "unsupported OS"); 5298 } 5299 } 5300 5301 version(Windows) { 5302 import core.sys.windows.windows; 5303 import core.sys.windows.winsock2; 5304 } 5305 5306 version(none) { 5307 extern(Windows) int WSAAsyncSelect(SOCKET, HWND, uint, int); 5308 enum int FD_CLOSE = 1 << 5; 5309 enum int FD_READ = 1 << 0; 5310 enum int WM_USER = 1024; 5311 } 5312 5313 version(Windows) { 5314 import core.stdc.config; 5315 extern(Windows) 5316 int WSAEventSelect(SOCKET, HANDLE /* to an Event */, c_long); 5317 5318 extern(Windows) 5319 HANDLE WSACreateEvent(); 5320 5321 extern(Windows) 5322 DWORD WSAWaitForMultipleEvents(DWORD, HANDLE*, BOOL, DWORD, BOOL); 5323 } 5324 5325 /* copy/paste from cgi.d */ 5326 public { 5327 enum WebSocketOpcode : ubyte { 5328 continuation = 0, 5329 text = 1, 5330 binary = 2, 5331 // 3, 4, 5, 6, 7 RESERVED 5332 close = 8, 5333 ping = 9, 5334 pong = 10, 5335 // 11,12,13,14,15 RESERVED 5336 } 5337 5338 public struct WebSocketFrame { 5339 private bool populated; 5340 bool fin; 5341 bool rsv1; 5342 bool rsv2; 5343 bool rsv3; 5344 WebSocketOpcode opcode; // 4 bits 5345 bool masked; 5346 ubyte lengthIndicator; // don't set this when building one to send 5347 ulong realLength; // don't use when sending 5348 ubyte[4] maskingKey; // don't set this when sending 5349 ubyte[] data; 5350 5351 static WebSocketFrame simpleMessage(WebSocketOpcode opcode, in void[] data) { 5352 WebSocketFrame msg; 5353 msg.fin = true; 5354 msg.opcode = opcode; 5355 msg.data = cast(ubyte[]) data.dup; // it is mutated below when masked, so need to be cautious and copy it, sigh 5356 5357 return msg; 5358 } 5359 5360 private void send(scope void delegate(ubyte[]) llsend) { 5361 ubyte[64] headerScratch; 5362 int headerScratchPos = 0; 5363 5364 realLength = data.length; 5365 5366 { 5367 ubyte b1; 5368 b1 |= cast(ubyte) opcode; 5369 b1 |= rsv3 ? (1 << 4) : 0; 5370 b1 |= rsv2 ? (1 << 5) : 0; 5371 b1 |= rsv1 ? (1 << 6) : 0; 5372 b1 |= fin ? (1 << 7) : 0; 5373 5374 headerScratch[0] = b1; 5375 headerScratchPos++; 5376 } 5377 5378 { 5379 headerScratchPos++; // we'll set header[1] at the end of this 5380 auto rlc = realLength; 5381 ubyte b2; 5382 b2 |= masked ? (1 << 7) : 0; 5383 5384 assert(headerScratchPos == 2); 5385 5386 if(realLength > 65535) { 5387 // use 64 bit length 5388 b2 |= 0x7f; 5389 5390 // FIXME: double check endinaness 5391 foreach(i; 0 .. 8) { 5392 headerScratch[2 + 7 - i] = rlc & 0x0ff; 5393 rlc >>>= 8; 5394 } 5395 5396 headerScratchPos += 8; 5397 } else if(realLength > 125) { 5398 // use 16 bit length 5399 b2 |= 0x7e; 5400 5401 // FIXME: double check endinaness 5402 foreach(i; 0 .. 2) { 5403 headerScratch[2 + 1 - i] = rlc & 0x0ff; 5404 rlc >>>= 8; 5405 } 5406 5407 headerScratchPos += 2; 5408 } else { 5409 // use 7 bit length 5410 b2 |= realLength & 0b_0111_1111; 5411 } 5412 5413 headerScratch[1] = b2; 5414 } 5415 5416 //assert(!masked, "masking key not properly implemented"); 5417 if(masked) { 5418 import std.random; 5419 foreach(ref item; maskingKey) 5420 item = uniform(ubyte.min, ubyte.max); 5421 headerScratch[headerScratchPos .. headerScratchPos + 4] = maskingKey[]; 5422 headerScratchPos += 4; 5423 5424 // we'll just mask it in place... 5425 int keyIdx = 0; 5426 foreach(i; 0 .. data.length) { 5427 data[i] = data[i] ^ maskingKey[keyIdx]; 5428 if(keyIdx == 3) 5429 keyIdx = 0; 5430 else 5431 keyIdx++; 5432 } 5433 } 5434 5435 //writeln("SENDING ", headerScratch[0 .. headerScratchPos], data); 5436 llsend(headerScratch[0 .. headerScratchPos]); 5437 if(data.length) 5438 llsend(data); 5439 } 5440 5441 static WebSocketFrame read(ref ubyte[] d) { 5442 WebSocketFrame msg; 5443 5444 auto orig = d; 5445 5446 WebSocketFrame needsMoreData() { 5447 d = orig; 5448 return WebSocketFrame.init; 5449 } 5450 5451 if(d.length < 2) 5452 return needsMoreData(); 5453 5454 ubyte b = d[0]; 5455 5456 msg.populated = true; 5457 5458 msg.opcode = cast(WebSocketOpcode) (b & 0x0f); 5459 b >>= 4; 5460 msg.rsv3 = b & 0x01; 5461 b >>= 1; 5462 msg.rsv2 = b & 0x01; 5463 b >>= 1; 5464 msg.rsv1 = b & 0x01; 5465 b >>= 1; 5466 msg.fin = b & 0x01; 5467 5468 b = d[1]; 5469 msg.masked = (b & 0b1000_0000) ? true : false; 5470 msg.lengthIndicator = b & 0b0111_1111; 5471 5472 d = d[2 .. $]; 5473 5474 if(msg.lengthIndicator == 0x7e) { 5475 // 16 bit length 5476 msg.realLength = 0; 5477 5478 if(d.length < 2) return needsMoreData(); 5479 5480 foreach(i; 0 .. 2) { 5481 msg.realLength |= d[0] << ((1-i) * 8); 5482 d = d[1 .. $]; 5483 } 5484 } else if(msg.lengthIndicator == 0x7f) { 5485 // 64 bit length 5486 msg.realLength = 0; 5487 5488 if(d.length < 8) return needsMoreData(); 5489 5490 foreach(i; 0 .. 8) { 5491 msg.realLength |= ulong(d[0]) << ((7-i) * 8); 5492 d = d[1 .. $]; 5493 } 5494 } else { 5495 // 7 bit length 5496 msg.realLength = msg.lengthIndicator; 5497 } 5498 5499 if(msg.masked) { 5500 5501 if(d.length < 4) return needsMoreData(); 5502 5503 msg.maskingKey = d[0 .. 4]; 5504 d = d[4 .. $]; 5505 } 5506 5507 if(msg.realLength > d.length) { 5508 return needsMoreData(); 5509 } 5510 5511 msg.data = d[0 .. cast(size_t) msg.realLength]; 5512 d = d[cast(size_t) msg.realLength .. $]; 5513 5514 return msg; 5515 } 5516 5517 void unmaskInPlace() { 5518 if(this.masked) { 5519 int keyIdx = 0; 5520 foreach(i; 0 .. this.data.length) { 5521 this.data[i] = this.data[i] ^ this.maskingKey[keyIdx]; 5522 if(keyIdx == 3) 5523 keyIdx = 0; 5524 else 5525 keyIdx++; 5526 } 5527 } 5528 } 5529 5530 char[] textData() { 5531 return cast(char[]) data; 5532 } 5533 } 5534 } 5535 5536 private extern(C) 5537 int verifyCertificateFromRegistryArsdHttp(int preverify_ok, X509_STORE_CTX* ctx) { 5538 version(Windows) { 5539 if(preverify_ok) 5540 return 1; 5541 5542 auto err_cert = OpenSSL.X509_STORE_CTX_get_current_cert(ctx); 5543 auto err = OpenSSL.X509_STORE_CTX_get_error(ctx); 5544 5545 if(err == 62) 5546 return 0; // hostname mismatch is an error we can trust; that means OpenSSL already found the certificate and rejected it 5547 5548 auto len = OpenSSL.i2d_X509(err_cert, null); 5549 if(len == -1) 5550 return 0; 5551 ubyte[] buffer = new ubyte[](len); 5552 auto ptr = buffer.ptr; 5553 len = OpenSSL.i2d_X509(err_cert, &ptr); 5554 if(len != buffer.length) 5555 return 0; 5556 5557 5558 CERT_CHAIN_PARA thing; 5559 thing.cbSize = thing.sizeof; 5560 auto context = CertCreateCertificateContext(X509_ASN_ENCODING, buffer.ptr, cast(int) buffer.length); 5561 if(context is null) 5562 return 0; 5563 scope(exit) CertFreeCertificateContext(context); 5564 5565 PCCERT_CHAIN_CONTEXT chain; 5566 if(CertGetCertificateChain(null, context, null, null, &thing, 0, null, &chain)) { 5567 scope(exit) 5568 CertFreeCertificateChain(chain); 5569 5570 DWORD errorStatus = chain.TrustStatus.dwErrorStatus; 5571 5572 if(errorStatus == 0) 5573 return 1; // Windows approved it, OK carry on 5574 // otherwise, sustain OpenSSL's original ruling 5575 } 5576 5577 return 0; 5578 } else { 5579 return preverify_ok; 5580 } 5581 } 5582 5583 5584 version(Windows) { 5585 pragma(lib, "crypt32"); 5586 import core.sys.windows.wincrypt; 5587 extern(Windows) { 5588 PCCERT_CONTEXT CertEnumCertificatesInStore(HCERTSTORE hCertStore, PCCERT_CONTEXT pPrevCertContext); 5589 // BOOL CertGetCertificateChain(HCERTCHAINENGINE hChainEngine, PCCERT_CONTEXT pCertContext, LPFILETIME pTime, HCERTSTORE hAdditionalStore, PCERT_CHAIN_PARA pChainPara, DWORD dwFlags, LPVOID pvReserved, PCCERT_CHAIN_CONTEXT *ppChainContext); 5590 PCCERT_CONTEXT CertCreateCertificateContext(DWORD dwCertEncodingType, const BYTE *pbCertEncoded, DWORD cbCertEncoded); 5591 } 5592 5593 void loadCertificatesFromRegistry(SSL_CTX* ctx) { 5594 auto store = CertOpenSystemStore(0, "ROOT"); 5595 if(store is null) { 5596 // import std.stdio; writeln("failed"); 5597 return; 5598 } 5599 scope(exit) 5600 CertCloseStore(store, 0); 5601 5602 X509_STORE* ssl_store = OpenSSL.SSL_CTX_get_cert_store(ctx); 5603 PCCERT_CONTEXT c; 5604 while((c = CertEnumCertificatesInStore(store, c)) !is null) { 5605 FILETIME na = c.pCertInfo.NotAfter; 5606 SYSTEMTIME st; 5607 FileTimeToSystemTime(&na, &st); 5608 5609 /+ 5610 _CRYPTOAPI_BLOB i = cast() c.pCertInfo.Issuer; 5611 5612 char[256] buffer; 5613 auto p = CertNameToStrA(X509_ASN_ENCODING, &i, CERT_SIMPLE_NAME_STR, buffer.ptr, cast(int) buffer.length); 5614 import std.stdio; writeln(buffer[0 .. p]); 5615 +/ 5616 5617 if(st.wYear <= 2021) { 5618 // see: https://www.openssl.org/blog/blog/2021/09/13/LetsEncryptRootCertExpire/ 5619 continue; // no point keeping an expired root cert and it can break Let's Encrypt anyway 5620 } 5621 5622 const(ubyte)* thing = c.pbCertEncoded; 5623 auto x509 = OpenSSL.d2i_X509(null, &thing, c.cbCertEncoded); 5624 if (x509) { 5625 auto success = OpenSSL.X509_STORE_add_cert(ssl_store, x509); 5626 //if(!success) 5627 //writeln("FAILED HERE"); 5628 OpenSSL.X509_free(x509); 5629 } else { 5630 //writeln("FAILED"); 5631 } 5632 } 5633 5634 CertFreeCertificateContext(c); 5635 5636 // import core.stdc.stdio; printf("%s\n", OpenSSL.OpenSSL_version(0)); 5637 } 5638 5639 5640 // because i use the FILE* in PEM_read_X509 and friends 5641 // gotta use this to bridge the MS C runtime functions 5642 // might be able to just change those to only use the BIO versions 5643 // instead 5644 5645 // only on MS C runtime 5646 version(CRuntime_Microsoft) {} else version=no_openssl_applink; 5647 5648 version(no_openssl_applink) {} else { 5649 private extern(C) { 5650 void _open(); 5651 void _read(); 5652 void _write(); 5653 void _lseek(); 5654 void _close(); 5655 int _fileno(FILE*); 5656 int _setmode(int, int); 5657 } 5658 export extern(C) void** OPENSSL_Applink() { 5659 import core.stdc.stdio; 5660 5661 static extern(C) void* app_stdin() { return cast(void*) stdin; } 5662 static extern(C) void* app_stdout() { return cast(void*) stdout; } 5663 static extern(C) void* app_stderr() { return cast(void*) stderr; } 5664 static extern(C) int app_feof(FILE* fp) { return feof(fp); } 5665 static extern(C) int app_ferror(FILE* fp) { return ferror(fp); } 5666 static extern(C) void app_clearerr(FILE* fp) { return clearerr(fp); } 5667 static extern(C) int app_fileno(FILE* fp) { return _fileno(fp); } 5668 static extern(C) int app_fsetmod(FILE* fp, char mod) { 5669 return _setmode(_fileno(fp), mod == 'b' ? _O_BINARY : _O_TEXT); 5670 } 5671 5672 static immutable void*[] table = [ 5673 cast(void*) 22, // applink max 5674 5675 &app_stdin, 5676 &app_stdout, 5677 &app_stderr, 5678 &fprintf, 5679 &fgets, 5680 &fread, 5681 &fwrite, 5682 &app_fsetmod, 5683 &app_feof, 5684 &fclose, 5685 5686 &fopen, 5687 &fseek, 5688 &ftell, 5689 &fflush, 5690 &app_ferror, 5691 &app_clearerr, 5692 &app_fileno, 5693 5694 &_open, 5695 &_read, 5696 &_write, 5697 &_lseek, 5698 &_close, 5699 ]; 5700 static assert(table.length == 23); 5701 5702 return cast(void**) table.ptr; 5703 } 5704 } 5705 } 5706 5707 unittest { 5708 auto client = new HttpClient(); 5709 auto response = client.navigateTo(Uri("data:,Hello%2C%20World%21")).waitForCompletion(); 5710 assert(response.contentTypeMimeType == "text/plain", response.contentType); 5711 assert(response.contentText == "Hello, World!", response.contentText); 5712 5713 response = client.navigateTo(Uri("data:text/plain;base64,SGVsbG8sIFdvcmxkIQ==")).waitForCompletion(); 5714 assert(response.contentTypeMimeType == "text/plain", response.contentType); 5715 assert(response.contentText == "Hello, World!", response.contentText); 5716 5717 response = client.navigateTo(Uri("data:text/html,%3Ch1%3EHello%2C%20World%21%3C%2Fh1%3E")).waitForCompletion(); 5718 assert(response.contentTypeMimeType == "text/html", response.contentType); 5719 assert(response.contentText == "<h1>Hello, World!</h1>", response.contentText); 5720 } 5721 5722 version(arsd_http2_unittests) 5723 unittest { 5724 import core.thread; 5725 5726 static void server() { 5727 import std.socket; 5728 auto socket = new TcpSocket(); 5729 socket.setOption(SocketOptionLevel.SOCKET, SocketOption.REUSEADDR, true); 5730 socket.bind(new InternetAddress(12346)); 5731 socket.listen(1); 5732 auto s = socket.accept(); 5733 socket.close(); 5734 5735 ubyte[1024] thing; 5736 auto g = s.receive(thing[]); 5737 5738 /+ 5739 string response = "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 9\r\n\r\nHello!!??"; 5740 auto packetSize = 2; 5741 +/ 5742 5743 auto packetSize = 1; 5744 string response = "HTTP/1.1 200 OK\r\nConnection: close\r\nTransfer-Encoding: chunked\r\n\r\n6\r\nHello!\r\n0\r\n\r\n"; 5745 5746 while(response.length) { 5747 s.send(response[0 .. packetSize]); 5748 response = response[packetSize .. $]; 5749 //import std.stdio; writeln(response); 5750 } 5751 5752 s.close(); 5753 } 5754 5755 auto thread = new Thread(&server); 5756 thread.start; 5757 5758 Thread.sleep(200.msecs); 5759 5760 auto response = get("http://localhost:12346/").waitForCompletion; 5761 assert(response.code == 200); 5762 //import std.stdio; writeln(response); 5763 5764 foreach(site; ["https://dlang.org/", "http://arsdnet.net", "https://phobos.dpldocs.info"]) { 5765 response = get(site).waitForCompletion; 5766 assert(response.code == 200); 5767 } 5768 5769 thread.join; 5770 } 5771 5772 /+ 5773 so the url params are arguments. it knows the request 5774 internally. other params are properties on the req 5775 5776 names may have different paths... those will just add ForSomething i think. 5777 5778 auto req = api.listMergeRequests 5779 req.page = 10; 5780 5781 or 5782 req.page(1) 5783 .bar("foo") 5784 5785 req.execute(); 5786 5787 5788 everything in the response is nullable access through the 5789 dynamic object, just with property getters there. need to make 5790 it static generated tho 5791 5792 other messages may be: isPresent and getDynamic 5793 5794 5795 AND/OR what about doing it like the rails objects 5796 5797 BroadcastMessage.get(4) 5798 // various properties 5799 5800 // it lists what you updated 5801 5802 BroadcastMessage.foo().bar().put(5) 5803 +/