1 // Copyright 2013-2022, Adam D. Ruppe. 2 3 // FIXME: websocket proxy support 4 // FIXME: ipv6 support 5 6 // FIXME: headers are supposed to be case insensitive. ugh. 7 8 /++ 9 This is version 2 of my http/1.1 client implementation. 10 11 12 It has no dependencies for basic operation, but does require OpenSSL 13 libraries (or compatible) to be support HTTPS. This dynamically loaded 14 on-demand (meaning it won't be loaded if you don't use it, but if you do 15 use it, the openssl dynamic libraries must be found in the system search path). 16 17 You can compile with `-version=without_openssl` to entirely disable ssl support. 18 19 http2.d, despite its name, does NOT implement HTTP/2.0, but this 20 shouldn't matter for 99.9% of usage, since all servers will continue 21 to support HTTP/1.1 for a very long time. 22 23 History: 24 Automatic `100 Continue` handling was added on September 28, 2021. It doesn't 25 set the Expect header, so it isn't supposed to happen, but plenty of web servers 26 don't follow the standard anyway. 27 +/ 28 module arsd.http2; 29 30 /// 31 unittest { 32 import arsd.http2; 33 34 void main() { 35 auto client = new HttpClient(); 36 37 auto request = client.request(Uri("http://dlang.org/")); 38 auto response = request.waitForCompletion(); 39 40 import std.stdio; 41 writeln(response.contentText); 42 writeln(response.code, " ", response.codeText); 43 writeln(response.contentType); 44 } 45 46 version(arsd_http2_integration_test) main(); // exclude from docs 47 } 48 49 // FIXME: I think I want to disable sigpipe here too. 50 51 import std.uri : encodeComponent; 52 53 debug(arsd_http2_verbose) debug=arsd_http2; 54 55 debug(arsd_http2) import std.stdio : writeln; 56 57 version=arsd_http_internal_implementation; 58 59 version(without_openssl) {} 60 else { 61 version=use_openssl; 62 version=with_openssl; 63 version(older_openssl) {} else 64 version=newer_openssl; 65 } 66 67 version(arsd_http_winhttp_implementation) { 68 pragma(lib, "winhttp") 69 import core.sys.windows.winhttp; 70 // FIXME: alter the dub package file too 71 72 // https://github.com/curl/curl/blob/master/lib/vtls/schannel.c 73 // https://docs.microsoft.com/en-us/windows/win32/secauthn/creating-an-schannel-security-context 74 75 76 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpreaddata 77 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpsendrequest 78 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpopenrequest 79 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpconnect 80 } 81 82 83 84 /++ 85 Demonstrates core functionality, using the [HttpClient], 86 [HttpRequest] (returned by [HttpClient.navigateTo|client.navigateTo]), 87 and [HttpResponse] (returned by [HttpRequest.waitForCompletion|request.waitForCompletion]). 88 89 +/ 90 unittest { 91 import arsd.http2; 92 93 void main() { 94 auto client = new HttpClient(); 95 auto request = client.navigateTo(Uri("http://dlang.org/")); 96 auto response = request.waitForCompletion(); 97 98 string returnedHtml = response.contentText; 99 } 100 } 101 102 // FIXME: multipart encoded file uploads needs implementation 103 // future: do web client api stuff 104 105 private __gshared bool defaultVerifyPeer_ = true; 106 107 void defaultVerifyPeer(bool v) { 108 defaultVerifyPeer_ = v; 109 } 110 111 debug import std.stdio; 112 113 import std.socket; 114 import core.time; 115 116 // FIXME: check Transfer-Encoding: gzip always 117 118 version(with_openssl) { 119 //pragma(lib, "crypto"); 120 //pragma(lib, "ssl"); 121 } 122 123 /+ 124 HttpRequest httpRequest(string method, string url, ubyte[] content, string[string] content) { 125 return null; 126 } 127 +/ 128 129 /** 130 auto request = get("http://arsdnet.net/"); 131 request.send(); 132 133 auto response = get("http://arsdnet.net/").waitForCompletion(); 134 */ 135 HttpRequest get(string url) { 136 auto client = new HttpClient(); 137 auto request = client.navigateTo(Uri(url)); 138 return request; 139 } 140 141 /** 142 Do not forget to call `waitForCompletion()` on the returned object! 143 */ 144 HttpRequest post(string url, string[string] req) { 145 auto client = new HttpClient(); 146 ubyte[] bdata; 147 foreach(k, v; req) { 148 if(bdata.length) 149 bdata ~= cast(ubyte[]) "&"; 150 bdata ~= cast(ubyte[]) encodeComponent(k); 151 bdata ~= cast(ubyte[]) "="; 152 bdata ~= cast(ubyte[]) encodeComponent(v); 153 } 154 auto request = client.request(Uri(url), HttpVerb.POST, bdata, "application/x-www-form-urlencoded"); 155 return request; 156 } 157 158 /// gets the text off a url. basic operation only. 159 string getText(string url) { 160 auto request = get(url); 161 auto response = request.waitForCompletion(); 162 return cast(string) response.content; 163 } 164 165 /+ 166 ubyte[] getBinary(string url, string[string] cookies = null) { 167 auto hr = httpRequest("GET", url, null, cookies); 168 if(hr.code != 200) 169 throw new Exception(format("HTTP answered %d instead of 200 on %s", hr.code, url)); 170 return hr.content; 171 } 172 173 /** 174 Gets a textual document, ignoring headers. Throws on non-text or error. 175 */ 176 string get(string url, string[string] cookies = null) { 177 auto hr = httpRequest("GET", url, null, cookies); 178 if(hr.code != 200) 179 throw new Exception(format("HTTP answered %d instead of 200 on %s", hr.code, url)); 180 if(hr.contentType.indexOf("text/") == -1) 181 throw new Exception(hr.contentType ~ " is bad content for conversion to string"); 182 return cast(string) hr.content; 183 184 } 185 186 static import std.uri; 187 188 string post(string url, string[string] args, string[string] cookies = null) { 189 string content; 190 191 foreach(name, arg; args) { 192 if(content.length) 193 content ~= "&"; 194 content ~= std.uri.encode(name) ~ "=" ~ std.uri.encode(arg); 195 } 196 197 auto hr = httpRequest("POST", url, cast(ubyte[]) content, cookies, ["Content-Type: application/x-www-form-urlencoded"]); 198 if(hr.code != 200) 199 throw new Exception(format("HTTP answered %d instead of 200", hr.code)); 200 if(hr.contentType.indexOf("text/") == -1) 201 throw new Exception(hr.contentType ~ " is bad content for conversion to string"); 202 203 return cast(string) hr.content; 204 } 205 206 +/ 207 208 /// 209 struct HttpResponse { 210 /++ 211 The HTTP response code, if the response was completed, or some value < 100 if it was aborted or failed. 212 213 Code 0 - initial value, nothing happened 214 Code 1 - you called request.abort 215 Code 2 - connection refused 216 Code 3 - connection succeeded, but server disconnected early 217 Code 4 - server sent corrupted response (or this code has a bug and processed it wrong) 218 Code 5 - request timed out 219 220 Code >= 100 - a HTTP response 221 +/ 222 int code; 223 string codeText; /// 224 225 string httpVersion; /// 226 227 string statusLine; /// 228 229 string contentType; /// The *full* content type header. See also [contentTypeMimeType] and [contentTypeCharset]. 230 string location; /// The location header 231 232 /++ 233 234 History: 235 Added December 5, 2020 (version 9.1) 236 +/ 237 bool wasSuccessful() { 238 return code >= 200 && code < 400; 239 } 240 241 /++ 242 Returns the mime type part of the [contentType] header. 243 244 History: 245 Added July 25, 2022 (version 10.9) 246 +/ 247 string contentTypeMimeType() { 248 auto idx = contentType.indexOf(";"); 249 if(idx == -1) 250 return contentType; 251 252 return contentType[0 .. idx].strip; 253 } 254 255 /// the charset out of content type, if present. `null` if not. 256 string contentTypeCharset() { 257 auto idx = contentType.indexOf("charset="); 258 if(idx == -1) 259 return null; 260 auto c = contentType[idx + "charset=".length .. $].strip; 261 if(c.length) 262 return c; 263 return null; 264 } 265 266 /++ 267 Names and values of cookies set in the response. 268 269 History: 270 Prior to July 5, 2021 (dub v10.2), this was a public field instead of a property. I did 271 not consider this a breaking change since the intended use is completely compatible with the 272 property, and it was not actually implemented properly before anyway. 273 +/ 274 @property string[string] cookies() const { 275 string[string] ret; 276 foreach(cookie; cookiesDetails) 277 ret[cookie.name] = cookie.value; 278 return ret; 279 } 280 /++ 281 The full parsed-out information of cookies set in the response. 282 283 History: 284 Added July 5, 2021 (dub v10.2). 285 +/ 286 @property CookieHeader[] cookiesDetails() inout { 287 CookieHeader[] ret; 288 foreach(header; headers) { 289 if(auto content = header.isHttpHeader("set-cookie")) { 290 // format: name=value, value might be double quoted. it MIGHT be url encoded, but im not going to attempt that since the RFC is silent. 291 // then there's optionally ; attr=value after that. attributes need not have a value 292 293 CookieHeader cookie; 294 295 auto remaining = content; 296 297 cookie_name: 298 foreach(idx, ch; remaining) { 299 if(ch == '=') { 300 cookie.name = remaining[0 .. idx].idup_if_needed; 301 remaining = remaining[idx + 1 .. $]; 302 break; 303 } 304 } 305 306 cookie_value: 307 308 { 309 auto idx = remaining.indexOf(";"); 310 if(idx == -1) { 311 cookie.value = remaining.idup_if_needed; 312 remaining = remaining[$..$]; 313 } else { 314 cookie.value = remaining[0 .. idx].idup_if_needed; 315 remaining = remaining[idx + 1 .. $].stripLeft; 316 } 317 318 if(cookie.value.length > 2 && cookie.value[0] == '"' && cookie.value[$-1] == '"') 319 cookie.value = cookie.value[1 .. $ - 1]; 320 } 321 322 cookie_attributes: 323 324 while(remaining.length) { 325 string name; 326 foreach(idx, ch; remaining) { 327 if(ch == '=') { 328 name = remaining[0 .. idx].idup_if_needed; 329 remaining = remaining[idx + 1 .. $]; 330 331 string value; 332 333 foreach(idx2, ch2; remaining) { 334 if(ch2 == ';') { 335 value = remaining[0 .. idx2].idup_if_needed; 336 remaining = remaining[idx2 + 1 .. $].stripLeft; 337 break; 338 } 339 } 340 341 if(value is null) { 342 value = remaining.idup_if_needed; 343 remaining = remaining[$ .. $]; 344 } 345 346 cookie.attributes[name] = value; 347 continue cookie_attributes; 348 } else if(ch == ';') { 349 name = remaining[0 .. idx].idup_if_needed; 350 remaining = remaining[idx + 1 .. $].stripLeft; 351 cookie.attributes[name] = ""; 352 continue cookie_attributes; 353 } 354 } 355 356 if(remaining.length) { 357 cookie.attributes[remaining.idup_if_needed] = ""; 358 remaining = remaining[$..$]; 359 360 } 361 } 362 363 ret ~= cookie; 364 } 365 } 366 return ret; 367 } 368 369 string[] headers; /// Array of all headers returned. 370 string[string] headersHash; /// 371 372 ubyte[] content; /// The raw content returned in the response body. 373 string contentText; /// [content], but casted to string (for convenience) 374 375 alias responseText = contentText; // just cuz I do this so often. 376 //alias body = content; 377 378 /++ 379 returns `new Document(this.contentText)`. Requires [arsd.dom]. 380 +/ 381 auto contentDom()() { 382 import arsd.dom; 383 return new Document(this.contentText); 384 385 } 386 387 /++ 388 returns `var.fromJson(this.contentText)`. Requires [arsd.jsvar]. 389 +/ 390 auto contentJson()() { 391 import arsd.jsvar; 392 return var.fromJson(this.contentText); 393 } 394 395 HttpRequestParameters requestParameters; /// 396 397 LinkHeader[] linksStored; 398 bool linksLazilyParsed; 399 400 HttpResponse deepCopy() const { 401 HttpResponse h = cast(HttpResponse) this; 402 h.headers = h.headers.dup; 403 h.headersHash = h.headersHash.dup; 404 h.content = h.content.dup; 405 h.linksStored = h.linksStored.dup; 406 return h; 407 } 408 409 /// Returns links header sorted by "rel" attribute. 410 /// It returns a new array on each call. 411 LinkHeader[string] linksHash() { 412 auto links = this.links(); 413 LinkHeader[string] ret; 414 foreach(link; links) 415 ret[link.rel] = link; 416 return ret; 417 } 418 419 /// Returns the Link header, parsed. 420 LinkHeader[] links() { 421 if(linksLazilyParsed) 422 return linksStored; 423 linksLazilyParsed = true; 424 LinkHeader[] ret; 425 426 auto hdrPtr = "link" in headersHash; 427 if(hdrPtr is null) 428 return ret; 429 430 auto header = *hdrPtr; 431 432 LinkHeader current; 433 434 while(header.length) { 435 char ch = header[0]; 436 437 if(ch == '<') { 438 // read url 439 header = header[1 .. $]; 440 size_t idx; 441 while(idx < header.length && header[idx] != '>') 442 idx++; 443 current.url = header[0 .. idx]; 444 header = header[idx .. $]; 445 } else if(ch == ';') { 446 // read attribute 447 header = header[1 .. $]; 448 header = header.stripLeft; 449 450 size_t idx; 451 while(idx < header.length && header[idx] != '=') 452 idx++; 453 454 string name = header[0 .. idx]; 455 if(idx + 1 < header.length) 456 header = header[idx + 1 .. $]; 457 else 458 header = header[$ .. $]; 459 460 string value; 461 462 if(header.length && header[0] == '"') { 463 // quoted value 464 header = header[1 .. $]; 465 idx = 0; 466 while(idx < header.length && header[idx] != '\"') 467 idx++; 468 value = header[0 .. idx]; 469 header = header[idx .. $]; 470 471 } else if(header.length) { 472 // unquoted value 473 idx = 0; 474 while(idx < header.length && header[idx] != ',' && header[idx] != ' ' && header[idx] != ';') 475 idx++; 476 477 value = header[0 .. idx]; 478 header = header[idx .. $].stripLeft; 479 } 480 481 name = name.toLower; 482 if(name == "rel") 483 current.rel = value; 484 else 485 current.attributes[name] = value; 486 487 } else if(ch == ',') { 488 // start another 489 ret ~= current; 490 current = LinkHeader.init; 491 } else if(ch == ' ' || ch == '\n' || ch == '\r' || ch == '\t') { 492 // ignore 493 } 494 495 if(header.length) 496 header = header[1 .. $]; 497 } 498 499 ret ~= current; 500 501 linksStored = ret; 502 503 return ret; 504 } 505 } 506 507 /+ 508 headerName MUST be all lower case and NOT have the colon on it 509 510 returns slice of the input thing after the header name 511 +/ 512 private inout(char)[] isHttpHeader(inout(char)[] thing, const(char)[] headerName) { 513 foreach(idx, ch; thing) { 514 if(idx < headerName.length) { 515 if(headerName[idx] == '-' && ch != '-') 516 return null; 517 if((ch | ' ') != headerName[idx]) 518 return null; 519 } else if(idx == headerName.length) { 520 if(ch != ':') 521 return null; 522 } else { 523 return thing[idx .. $].strip; 524 } 525 } 526 return null; 527 } 528 529 private string idup_if_needed(string s) { return s; } 530 private string idup_if_needed(const(char)[] s) { return s.idup; } 531 532 unittest { 533 assert("Cookie: foo=bar".isHttpHeader("cookie") == "foo=bar"); 534 assert("cookie: foo=bar".isHttpHeader("cookie") == "foo=bar"); 535 assert("cOOkie: foo=bar".isHttpHeader("cookie") == "foo=bar"); 536 assert("Set-Cookie: foo=bar".isHttpHeader("set-cookie") == "foo=bar"); 537 assert(!"".isHttpHeader("cookie")); 538 } 539 540 /// 541 struct LinkHeader { 542 string url; /// 543 string rel; /// 544 string[string] attributes; /// like title, rev, media, whatever attributes 545 } 546 547 /++ 548 History: 549 Added July 5, 2021 550 +/ 551 struct CookieHeader { 552 string name; 553 string value; 554 string[string] attributes; 555 } 556 557 import std.string; 558 static import std.algorithm; 559 import std.conv; 560 import std.range; 561 562 563 private AddressFamily family(string unixSocketPath) { 564 if(unixSocketPath.length) 565 return AddressFamily.UNIX; 566 else // FIXME: what about ipv6? 567 return AddressFamily.INET; 568 } 569 570 version(Windows) 571 private class UnixAddress : Address { 572 this(string) { 573 throw new Exception("No unix address support on this system in lib yet :("); 574 } 575 override sockaddr* name() { assert(0); } 576 override const(sockaddr)* name() const { assert(0); } 577 override int nameLen() const { assert(0); } 578 } 579 580 581 // Copy pasta from cgi.d, then stripped down. unix path thing added tho 582 /++ 583 Represents a URI. It offers named access to the components and relative uri resolution, though as a user of the library, you'd mostly just construct it like `Uri("http://example.com/index.html")`. 584 +/ 585 struct Uri { 586 alias toString this; // blargh idk a url really is a string, but should it be implicit? 587 588 // scheme://userinfo@host:port/path?query#fragment 589 590 string scheme; /// e.g. "http" in "http://example.com/" 591 string userinfo; /// the username (and possibly a password) in the uri 592 string host; /// the domain name 593 int port; /// port number, if given. Will be zero if a port was not explicitly given 594 string path; /// e.g. "/folder/file.html" in "http://example.com/folder/file.html" 595 string query; /// the stuff after the ? in a uri 596 string fragment; /// the stuff after the # in a uri. 597 598 /// Breaks down a uri string to its components 599 this(string uri) { 600 size_t lastGoodIndex; 601 foreach(char ch; uri) { 602 if(ch > 127) { 603 break; 604 } 605 lastGoodIndex++; 606 } 607 608 string replacement = uri[0 .. lastGoodIndex]; 609 foreach(char ch; uri[lastGoodIndex .. $]) { 610 if(ch > 127) { 611 // need to percent-encode any non-ascii in it 612 char[3] buffer; 613 buffer[0] = '%'; 614 615 auto first = ch / 16; 616 auto second = ch % 16; 617 first += (first >= 10) ? ('A'-10) : '0'; 618 second += (second >= 10) ? ('A'-10) : '0'; 619 620 buffer[1] = cast(char) first; 621 buffer[2] = cast(char) second; 622 623 replacement ~= buffer[]; 624 } else { 625 replacement ~= ch; 626 } 627 } 628 629 reparse(replacement); 630 } 631 632 /// Returns `port` if set, otherwise if scheme is https 443, otherwise always 80 633 int effectivePort() const @property nothrow pure @safe @nogc { 634 return port != 0 ? port 635 : scheme == "https" ? 443 : 80; 636 } 637 638 private string unixSocketPath = null; 639 /// Indicates it should be accessed through a unix socket instead of regular tcp. Returns new version without modifying this object. 640 Uri viaUnixSocket(string path) const { 641 Uri copy = this; 642 copy.unixSocketPath = path; 643 return copy; 644 } 645 646 /// Goes through a unix socket in the abstract namespace (linux only). Returns new version without modifying this object. 647 version(linux) 648 Uri viaAbstractSocket(string path) const { 649 Uri copy = this; 650 copy.unixSocketPath = "\0" ~ path; 651 return copy; 652 } 653 654 private void reparse(string uri) { 655 // from RFC 3986 656 // the ctRegex triples the compile time and makes ugly errors for no real benefit 657 // it was a nice experiment but just not worth it. 658 // enum ctr = ctRegex!r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?"; 659 /* 660 Captures: 661 0 = whole url 662 1 = scheme, with : 663 2 = scheme, no : 664 3 = authority, with // 665 4 = authority, no // 666 5 = path 667 6 = query string, with ? 668 7 = query string, no ? 669 8 = anchor, with # 670 9 = anchor, no # 671 */ 672 // Yikes, even regular, non-CT regex is also unacceptably slow to compile. 1.9s on my computer! 673 // instead, I will DIY and cut that down to 0.6s on the same computer. 674 /* 675 676 Note that authority is 677 user:password@domain:port 678 where the user:password@ part is optional, and the :port is optional. 679 680 Regex translation: 681 682 Scheme cannot have :, /, ?, or # in it, and must have one or more chars and end in a :. It is optional, but must be first. 683 Authority must start with //, but cannot have any other /, ?, or # in it. It is optional. 684 Path cannot have any ? or # in it. It is optional. 685 Query must start with ? and must not have # in it. It is optional. 686 Anchor must start with # and can have anything else in it to end of string. It is optional. 687 */ 688 689 this = Uri.init; // reset all state 690 691 // empty uri = nothing special 692 if(uri.length == 0) { 693 return; 694 } 695 696 size_t idx; 697 698 scheme_loop: foreach(char c; uri[idx .. $]) { 699 switch(c) { 700 case ':': 701 case '/': 702 case '?': 703 case '#': 704 break scheme_loop; 705 default: 706 } 707 idx++; 708 } 709 710 if(idx == 0 && uri[idx] == ':') { 711 // this is actually a path! we skip way ahead 712 goto path_loop; 713 } 714 715 if(idx == uri.length) { 716 // the whole thing is a path, apparently 717 path = uri; 718 return; 719 } 720 721 if(idx > 0 && uri[idx] == ':') { 722 scheme = uri[0 .. idx]; 723 idx++; 724 } else { 725 // we need to rewind; it found a / but no :, so the whole thing is prolly a path... 726 idx = 0; 727 } 728 729 if(idx + 2 < uri.length && uri[idx .. idx + 2] == "//") { 730 // we have an authority.... 731 idx += 2; 732 733 auto authority_start = idx; 734 authority_loop: foreach(char c; uri[idx .. $]) { 735 switch(c) { 736 case '/': 737 case '?': 738 case '#': 739 break authority_loop; 740 default: 741 } 742 idx++; 743 } 744 745 auto authority = uri[authority_start .. idx]; 746 747 auto idx2 = authority.indexOf("@"); 748 if(idx2 != -1) { 749 userinfo = authority[0 .. idx2]; 750 authority = authority[idx2 + 1 .. $]; 751 } 752 753 if(authority.length && authority[0] == '[') { 754 // ipv6 address special casing 755 idx2 = authority.indexOf(']'); 756 if(idx2 != -1) { 757 auto end = authority[idx2 + 1 .. $]; 758 if(end.length && end[0] == ':') 759 idx2 = idx2 + 1; 760 else 761 idx2 = -1; 762 } 763 } else { 764 idx2 = authority.indexOf(":"); 765 } 766 767 if(idx2 == -1) { 768 port = 0; // 0 means not specified; we should use the default for the scheme 769 host = authority; 770 } else { 771 host = authority[0 .. idx2]; 772 if(idx2 + 1 < authority.length) 773 port = to!int(authority[idx2 + 1 .. $]); 774 else 775 port = 0; 776 } 777 } 778 779 path_loop: 780 auto path_start = idx; 781 782 foreach(char c; uri[idx .. $]) { 783 if(c == '?' || c == '#') 784 break; 785 idx++; 786 } 787 788 path = uri[path_start .. idx]; 789 790 if(idx == uri.length) 791 return; // nothing more to examine... 792 793 if(uri[idx] == '?') { 794 idx++; 795 auto query_start = idx; 796 foreach(char c; uri[idx .. $]) { 797 if(c == '#') 798 break; 799 idx++; 800 } 801 query = uri[query_start .. idx]; 802 } 803 804 if(idx < uri.length && uri[idx] == '#') { 805 idx++; 806 fragment = uri[idx .. $]; 807 } 808 809 // uriInvalidated = false; 810 } 811 812 private string rebuildUri() const { 813 string ret; 814 if(scheme.length) 815 ret ~= scheme ~ ":"; 816 if(userinfo.length || host.length) 817 ret ~= "//"; 818 if(userinfo.length) 819 ret ~= userinfo ~ "@"; 820 if(host.length) 821 ret ~= host; 822 if(port) 823 ret ~= ":" ~ to!string(port); 824 825 ret ~= path; 826 827 if(query.length) 828 ret ~= "?" ~ query; 829 830 if(fragment.length) 831 ret ~= "#" ~ fragment; 832 833 // uri = ret; 834 // uriInvalidated = false; 835 return ret; 836 } 837 838 /// Converts the broken down parts back into a complete string 839 string toString() const { 840 // if(uriInvalidated) 841 return rebuildUri(); 842 } 843 844 /// Returns a new absolute Uri given a base. It treats this one as 845 /// relative where possible, but absolute if not. (If protocol, domain, or 846 /// other info is not set, the new one inherits it from the base.) 847 /// 848 /// Browsers use a function like this to figure out links in html. 849 Uri basedOn(in Uri baseUrl) const { 850 Uri n = this; // copies 851 if(n.scheme == "data") 852 return n; 853 // n.uriInvalidated = true; // make sure we regenerate... 854 855 // userinfo is not inherited... is this wrong? 856 857 // if anything is given in the existing url, we don't use the base anymore. 858 if(n.scheme.empty) { 859 n.scheme = baseUrl.scheme; 860 if(n.host.empty) { 861 n.host = baseUrl.host; 862 if(n.port == 0) { 863 n.port = baseUrl.port; 864 if(n.path.length > 0 && n.path[0] != '/') { 865 auto b = baseUrl.path[0 .. baseUrl.path.lastIndexOf("/") + 1]; 866 if(b.length == 0) 867 b = "/"; 868 n.path = b ~ n.path; 869 } else if(n.path.length == 0) { 870 n.path = baseUrl.path; 871 } 872 } 873 } 874 } 875 876 n.removeDots(); 877 878 // if still basically talking to the same thing, we should inherit the unix path 879 // too since basically the unix path is saying for this service, always use this override. 880 if(n.host == baseUrl.host && n.scheme == baseUrl.scheme && n.port == baseUrl.port) 881 n.unixSocketPath = baseUrl.unixSocketPath; 882 883 return n; 884 } 885 886 /++ 887 Resolves ../ and ./ parts of the path. Used in the implementation of [basedOn] and you could also use it to normalize things. 888 +/ 889 void removeDots() { 890 auto parts = this.path.split("/"); 891 string[] toKeep; 892 foreach(part; parts) { 893 if(part == ".") { 894 continue; 895 } else if(part == "..") { 896 //if(toKeep.length > 1) 897 toKeep = toKeep[0 .. $-1]; 898 //else 899 //toKeep = [""]; 900 continue; 901 } else { 902 //if(toKeep.length && toKeep[$-1].length == 0 && part.length == 0) 903 //continue; // skip a `//` situation 904 toKeep ~= part; 905 } 906 } 907 908 auto path = toKeep.join("/"); 909 if(path.length && path[0] != '/') 910 path = "/" ~ path; 911 912 this.path = path; 913 } 914 } 915 916 /* 917 void main(string args[]) { 918 write(post("http://arsdnet.net/bugs.php", ["test" : "hey", "again" : "what"])); 919 } 920 */ 921 922 /// 923 struct BasicAuth { 924 string username; /// 925 string password; /// 926 } 927 928 class ProxyException : Exception { 929 this(string msg) {super(msg); } 930 } 931 932 /** 933 Represents a HTTP request. You usually create these through a [HttpClient]. 934 935 936 --- 937 auto request = new HttpRequest(); // note that when there's no associated client, some features may not work 938 // normally you'd instead do `new HttpClient(); client.request(...)` 939 // set any properties here 940 941 // synchronous usage 942 auto reply = request.perform(); 943 944 // async usage, type 1: 945 request.send(); 946 request2.send(); 947 948 // wait until the first one is done, with the second one still in-flight 949 auto response = request.waitForCompletion(); 950 951 // async usage, type 2: 952 request.onDataReceived = (HttpRequest hr) { 953 if(hr.state == HttpRequest.State.complete) { 954 // use hr.responseData 955 } 956 }; 957 request.send(); // send, using the callback 958 959 // before terminating, be sure you wait for your requests to finish! 960 961 request.waitForCompletion(); 962 --- 963 */ 964 class HttpRequest { 965 966 /// Automatically follow a redirection? 967 bool followLocation = false; 968 969 /++ 970 Maximum number of redirections to follow (used only if [followLocation] is set to true). Will resolve with an error if a single request has more than this number of redirections. The default value is currently 10, but may change without notice. If you need a specific value, be sure to call this function. 971 972 If you want unlimited redirects, call it with `int.max`. If you set it to 0 but set [followLocation] to `true`, any attempt at redirection will abort the request. To disable automatically following redirection, set [followLocation] to `false` so you can process the 30x code yourself as a completed request. 973 974 History: 975 Added July 27, 2022 (dub v10.9) 976 +/ 977 void setMaximumNumberOfRedirects(int max = 10) { 978 maximumNumberOfRedirectsRemaining = max; 979 } 980 981 private int maximumNumberOfRedirectsRemaining; 982 983 /++ 984 Set to `true` to automatically retain cookies in the associated [HttpClient] from this request. 985 Note that you must have constructed the request from a `HttpClient` or at least passed one into the 986 constructor for this to have any effect. 987 988 Bugs: 989 See [HttpClient.retainCookies] for important caveats. 990 991 History: 992 Added July 5, 2021 (dub v10.2) 993 +/ 994 bool retainCookies = false; 995 996 private HttpClient client; 997 998 this() { 999 } 1000 1001 /// 1002 this(HttpClient client, Uri where, HttpVerb method, ICache cache = null, Duration timeout = 10.seconds, string proxy = null) { 1003 this.client = client; 1004 populateFromInfo(where, method); 1005 setTimeout(timeout); 1006 this.cache = cache; 1007 this.proxy = proxy; 1008 1009 setMaximumNumberOfRedirects(); 1010 } 1011 1012 1013 /// ditto 1014 this(Uri where, HttpVerb method, ICache cache = null, Duration timeout = 10.seconds, string proxy = null) { 1015 this(null, where, method, cache, timeout, proxy); 1016 } 1017 1018 /++ 1019 Sets the timeout from inactivity on the request. This is the amount of time that passes with no send or receive activity on the request before it fails with "request timed out" error. 1020 1021 History: 1022 Added March 31, 2021 1023 +/ 1024 void setTimeout(Duration timeout) { 1025 this.requestParameters.timeoutFromInactivity = timeout; 1026 this.timeoutFromInactivity = MonoTime.currTime + this.requestParameters.timeoutFromInactivity; 1027 } 1028 1029 private MonoTime timeoutFromInactivity; 1030 1031 private Uri where; 1032 1033 private ICache cache; 1034 1035 /++ 1036 Proxy to use for this request. It should be a URL or `null`. 1037 1038 This must be sent before you call [send]. 1039 1040 History: 1041 Added April 12, 2021 (dub v9.5) 1042 +/ 1043 string proxy; 1044 1045 /++ 1046 For https connections, if this is `true`, it will fail to connect if the TLS certificate can not be 1047 verified. Setting this to `false` will skip this check and allow the connection to continue anyway. 1048 1049 When the [HttpRequest] is constructed from a [HttpClient], it will inherit the value from the client 1050 instead of using the `= true` here. You can change this value any time before you call [send] (which 1051 is done implicitly if you call [waitForCompletion]). 1052 1053 History: 1054 Added April 5, 2022 (dub v10.8) 1055 1056 Prior to this, it always used the global (but undocumented) `defaultVerifyPeer` setting, and sometimes 1057 even if it was true, it would skip the verification. Now, it always respects this local setting. 1058 +/ 1059 bool verifyPeer = true; 1060 1061 1062 /// Final url after any redirections 1063 string finalUrl; 1064 1065 void populateFromInfo(Uri where, HttpVerb method) { 1066 auto parts = where.basedOn(this.where); 1067 this.where = parts; 1068 finalUrl = where.toString(); 1069 requestParameters.method = method; 1070 requestParameters.unixSocketPath = where.unixSocketPath; 1071 requestParameters.host = parts.host; 1072 requestParameters.port = cast(ushort) parts.effectivePort; 1073 requestParameters.ssl = parts.scheme == "https"; 1074 requestParameters.uri = parts.path.length ? parts.path : "/"; 1075 if(parts.query.length) { 1076 requestParameters.uri ~= "?"; 1077 requestParameters.uri ~= parts.query; 1078 } 1079 } 1080 1081 ~this() { 1082 } 1083 1084 ubyte[] sendBuffer; 1085 1086 HttpResponse responseData; 1087 private HttpClient parentClient; 1088 1089 size_t bodyBytesSent; 1090 size_t bodyBytesReceived; 1091 1092 State state_; 1093 State state() { return state_; } 1094 State state(State s) { 1095 assert(state_ != State.complete); 1096 return state_ = s; 1097 } 1098 /// Called when data is received. Check the state to see what data is available. 1099 void delegate(HttpRequest) onDataReceived; 1100 1101 enum State { 1102 /// The request has not yet been sent 1103 unsent, 1104 1105 /// The send() method has been called, but no data is 1106 /// sent on the socket yet because the connection is busy. 1107 pendingAvailableConnection, 1108 1109 /// connect has been called, but we're waiting on word of success 1110 connecting, 1111 1112 /// connecting a ssl, needing this 1113 sslConnectPendingRead, 1114 /// ditto 1115 sslConnectPendingWrite, 1116 1117 /// The headers are being sent now 1118 sendingHeaders, 1119 1120 // FIXME: allow Expect: 100-continue and separate the body send 1121 1122 /// The body is being sent now 1123 sendingBody, 1124 1125 /// The request has been sent but we haven't received any response yet 1126 waitingForResponse, 1127 1128 /// We have received some data and are currently receiving headers 1129 readingHeaders, 1130 1131 /// All headers are available but we're still waiting on the body 1132 readingBody, 1133 1134 /// The request is complete. 1135 complete, 1136 1137 /// The request is aborted, either by the abort() method, or as a result of the server disconnecting 1138 aborted 1139 } 1140 1141 /// Sends now and waits for the request to finish, returning the response. 1142 HttpResponse perform() { 1143 send(); 1144 return waitForCompletion(); 1145 } 1146 1147 /// Sends the request asynchronously. 1148 void send() { 1149 sendPrivate(true); 1150 } 1151 1152 private void sendPrivate(bool advance) { 1153 if(state != State.unsent && state != State.aborted) 1154 return; // already sent 1155 1156 if(cache !is null) { 1157 auto res = cache.getCachedResponse(this.requestParameters); 1158 if(res !is null) { 1159 state = State.complete; 1160 responseData = (*res).deepCopy(); 1161 return; 1162 } 1163 } 1164 1165 if(this.where.scheme == "data") { 1166 void error(string content) { 1167 responseData.code = 400; 1168 responseData.codeText = "Bad Request"; 1169 responseData.contentType = "text/plain"; 1170 responseData.content = cast(ubyte[]) content; 1171 responseData.contentText = content; 1172 state = State.complete; 1173 return; 1174 } 1175 1176 auto thing = this.where.path; 1177 // format is: type,data 1178 // type can have ;base64 1179 auto comma = thing.indexOf(","); 1180 if(comma == -1) 1181 return error("Invalid data uri, no comma found"); 1182 1183 auto type = thing[0 .. comma]; 1184 auto data = thing[comma + 1 .. $]; 1185 if(type.length == 0) 1186 type = "text/plain"; 1187 1188 import std.uri; 1189 auto bdata = cast(ubyte[]) decodeComponent(data); 1190 1191 if(type.indexOf(";base64") != -1) { 1192 import std.base64; 1193 try { 1194 bdata = Base64.decode(bdata); 1195 } catch(Exception e) { 1196 return error(e.msg); 1197 } 1198 } 1199 1200 responseData.code = 200; 1201 responseData.codeText = "OK"; 1202 responseData.contentType = type; 1203 responseData.content = bdata; 1204 responseData.contentText = cast(string) responseData.content; 1205 state = State.complete; 1206 return; 1207 } 1208 1209 string headers; 1210 1211 headers ~= to!string(requestParameters.method); 1212 headers ~= " "; 1213 if(proxy.length && !requestParameters.ssl) { 1214 // if we're doing a http proxy, we need to send a complete, absolute uri 1215 // so reconstruct it 1216 headers ~= "http://"; 1217 headers ~= requestParameters.host; 1218 if(requestParameters.port != 80) { 1219 headers ~= ":"; 1220 headers ~= to!string(requestParameters.port); 1221 } 1222 } 1223 1224 headers ~= requestParameters.uri; 1225 1226 if(requestParameters.useHttp11) 1227 headers ~= " HTTP/1.1\r\n"; 1228 else 1229 headers ~= " HTTP/1.0\r\n"; 1230 1231 // the whole authority section is supposed to be there, but curl doesn't send if default port 1232 // so I'll copy what they do 1233 headers ~= "Host: "; 1234 headers ~= requestParameters.host; 1235 if(requestParameters.port != 80 && requestParameters.port != 443) { 1236 headers ~= ":"; 1237 headers ~= to!string(requestParameters.port); 1238 } 1239 headers ~= "\r\n"; 1240 1241 bool specSaysRequestAlwaysHasBody = 1242 requestParameters.method == HttpVerb.POST || 1243 requestParameters.method == HttpVerb.PUT || 1244 requestParameters.method == HttpVerb.PATCH; 1245 1246 if(requestParameters.userAgent.length) 1247 headers ~= "User-Agent: "~requestParameters.userAgent~"\r\n"; 1248 if(requestParameters.contentType.length) 1249 headers ~= "Content-Type: "~requestParameters.contentType~"\r\n"; 1250 if(requestParameters.authorization.length) 1251 headers ~= "Authorization: "~requestParameters.authorization~"\r\n"; 1252 if(requestParameters.bodyData.length || specSaysRequestAlwaysHasBody) 1253 headers ~= "Content-Length: "~to!string(requestParameters.bodyData.length)~"\r\n"; 1254 if(requestParameters.acceptGzip) 1255 headers ~= "Accept-Encoding: gzip\r\n"; 1256 if(requestParameters.keepAlive) 1257 headers ~= "Connection: keep-alive\r\n"; 1258 1259 string cookieHeader; 1260 foreach(name, value; requestParameters.cookies) { 1261 if(cookieHeader is null) 1262 cookieHeader = "Cookie: "; 1263 else 1264 cookieHeader ~= "; "; 1265 cookieHeader ~= name; 1266 cookieHeader ~= "="; 1267 cookieHeader ~= value; 1268 } 1269 1270 if(cookieHeader !is null) { 1271 cookieHeader ~= "\r\n"; 1272 headers ~= cookieHeader; 1273 } 1274 1275 foreach(header; requestParameters.headers) 1276 headers ~= header ~ "\r\n"; 1277 1278 headers ~= "\r\n"; 1279 1280 // FIXME: separate this for 100 continue 1281 sendBuffer = cast(ubyte[]) headers ~ requestParameters.bodyData; 1282 1283 // import std.stdio; writeln("******* ", cast(string) sendBuffer); 1284 1285 responseData = HttpResponse.init; 1286 responseData.requestParameters = requestParameters; 1287 bodyBytesSent = 0; 1288 bodyBytesReceived = 0; 1289 state = State.pendingAvailableConnection; 1290 1291 bool alreadyPending = false; 1292 foreach(req; pending) 1293 if(req is this) { 1294 alreadyPending = true; 1295 break; 1296 } 1297 if(!alreadyPending) { 1298 pending ~= this; 1299 } 1300 1301 if(advance) 1302 HttpRequest.advanceConnections(requestParameters.timeoutFromInactivity); 1303 } 1304 1305 1306 /// Waits for the request to finish or timeout, whichever comes first. 1307 HttpResponse waitForCompletion() { 1308 while(state != State.aborted && state != State.complete) { 1309 if(state == State.unsent) { 1310 send(); 1311 continue; 1312 } 1313 if(auto err = HttpRequest.advanceConnections(requestParameters.timeoutFromInactivity)) { 1314 switch(err) { 1315 case 1: throw new Exception("HttpRequest.advanceConnections returned 1: all connections timed out"); 1316 case 2: throw new Exception("HttpRequest.advanceConnections returned 2: nothing to do"); 1317 case 3: continue; // EINTR 1318 default: throw new Exception("HttpRequest.advanceConnections got err " ~ to!string(err)); 1319 } 1320 } 1321 } 1322 1323 if(state == State.complete && responseData.code >= 200) 1324 if(cache !is null) 1325 cache.cacheResponse(this.requestParameters, this.responseData); 1326 1327 return responseData; 1328 } 1329 1330 /// Aborts this request. 1331 void abort() { 1332 this.state = State.aborted; 1333 this.responseData.code = 1; 1334 this.responseData.codeText = "request.abort called"; 1335 // the actual cancellation happens in the event loop 1336 } 1337 1338 HttpRequestParameters requestParameters; /// 1339 1340 version(arsd_http_winhttp_implementation) { 1341 public static void resetInternals() { 1342 1343 } 1344 1345 static assert(0, "implementation not finished"); 1346 } 1347 1348 1349 version(arsd_http_internal_implementation) { 1350 1351 /++ 1352 Changes the limit of number of open, inactive sockets. Reusing connections can provide a significant 1353 performance improvement, but the operating system can also impose a global limit on the number of open 1354 sockets and/or files that you don't want to run into. This lets you choose a balance right for you. 1355 1356 1357 When the total number of cached, inactive sockets approaches this maximum, it will check for ones closed by the 1358 server first. If there are none already closed by the server, it will select sockets at random from its connection 1359 cache and close them to make room for the new ones. 1360 1361 Please note: 1362 1363 $(LIST 1364 * there is always a limit of six open sockets per domain, per the common practice suggested by the http standard 1365 * the limit given here is thread-local. If you run multiple http clients/requests from multiple threads, don't set this too high or you might bump into the global limit from the OS. 1366 * setting this too low can waste connections because the server might close them, but they will never be garbage collected since my current implementation won't check for dead connections except when it thinks it is running close to the limit. 1367 ) 1368 1369 Setting it just right for your use case may provide an up to 10x performance boost. 1370 1371 This implementation is subject to change. If it does, I'll document it, but may not bump the version number. 1372 1373 History: 1374 Added August 10, 2022 (dub v10.9) 1375 +/ 1376 static void setConnectionCacheSize(int max = 32) { 1377 connectionCacheSize = max; 1378 } 1379 1380 private static { 1381 // we manage the actual connections. When a request is made on a particular 1382 // host, we try to reuse connections. We may open more than one connection per 1383 // host to do parallel requests. 1384 // 1385 // The key is the *domain name* and the port. Multiple domains on the same address will have separate connections. 1386 Socket[][string] socketsPerHost; 1387 1388 // only one request can be active on a given socket (at least HTTP < 2.0) so this is that 1389 HttpRequest[Socket] activeRequestOnSocket; 1390 HttpRequest[] pending; // and these are the requests that are waiting 1391 1392 int cachedSockets; 1393 int connectionCacheSize = 32; 1394 1395 /+ 1396 This is a somewhat expensive, but essential operation. If it isn't used in a heavy 1397 application, you'll risk running out of file descriptors. 1398 +/ 1399 void cleanOldSockets() { 1400 static struct CloseCandidate { 1401 string key; 1402 Socket socket; 1403 } 1404 1405 CloseCandidate[36] closeCandidates; 1406 int closeCandidatesPosition; 1407 1408 outer: foreach(key, sockets; socketsPerHost) { 1409 foreach(socket; sockets) { 1410 if(socket in activeRequestOnSocket) 1411 continue; // it is still in use; we can't close it 1412 1413 closeCandidates[closeCandidatesPosition++] = CloseCandidate(key, socket); 1414 if(closeCandidatesPosition == closeCandidates.length) 1415 break outer; 1416 } 1417 } 1418 1419 auto cc = closeCandidates[0 .. closeCandidatesPosition]; 1420 1421 if(cc.length == 0) 1422 return; // no candidates to even examine 1423 1424 // has the server closed any of these? if so, we also close and drop them 1425 static SocketSet readSet = null; 1426 if(readSet is null) 1427 readSet = new SocketSet(); 1428 readSet.reset(); 1429 1430 foreach(candidate; cc) { 1431 readSet.add(candidate.socket); 1432 } 1433 1434 int closeCount; 1435 1436 auto got = Socket.select(readSet, null, null, 0.msecs /* timeout, want it small since we just checking for eof */); 1437 if(got > 0) { 1438 foreach(ref candidate; cc) { 1439 if(readSet.isSet(candidate.socket)) { 1440 // if we can read when it isn't in use, that means eof; the 1441 // server closed it. 1442 candidate.socket.close(); 1443 loseSocketByKey(candidate.key, candidate.socket); 1444 closeCount++; 1445 } 1446 } 1447 debug(arsd_http2) writeln(closeCount, " from inactivity"); 1448 } else { 1449 // and if not, of the remaining ones, close a few just at random to bring us back beneath the arbitrary limit. 1450 1451 while(cc.length > 0 && (cachedSockets - closeCount) > connectionCacheSize) { 1452 import std.random; 1453 auto idx = uniform(0, cc.length); 1454 1455 cc[idx].socket.close(); 1456 loseSocketByKey(cc[idx].key, cc[idx].socket); 1457 1458 cc[idx] = cc[$ - 1]; 1459 cc = cc[0 .. $-1]; 1460 closeCount++; 1461 } 1462 debug(arsd_http2) writeln(closeCount, " from randomness"); 1463 } 1464 1465 cachedSockets -= closeCount; 1466 } 1467 1468 void loseSocketByKey(string key, Socket s) { 1469 if(auto list = key in socketsPerHost) { 1470 for(int a = 0; a < (*list).length; a++) { 1471 if((*list)[a] is s) { 1472 1473 for(int b = a; b < (*list).length - 1; b++) 1474 (*list)[b] = (*list)[b+1]; 1475 (*list) = (*list)[0 .. $-1]; 1476 break; 1477 } 1478 } 1479 } 1480 } 1481 1482 void loseSocket(string host, ushort port, bool ssl, Socket s) { 1483 import std.string; 1484 auto key = format("http%s://%s:%s", ssl ? "s" : "", host, port); 1485 1486 loseSocketByKey(key, s); 1487 } 1488 1489 Socket getOpenSocketOnHost(string proxy, string host, ushort port, bool ssl, string unixSocketPath, bool verifyPeer) { 1490 Socket openNewConnection() { 1491 Socket socket; 1492 if(ssl) { 1493 version(with_openssl) { 1494 loadOpenSsl(); 1495 socket = new SslClientSocket(family(unixSocketPath), SocketType.STREAM, host, verifyPeer); 1496 socket.blocking = false; 1497 } else 1498 throw new Exception("SSL not compiled in"); 1499 } else { 1500 socket = new Socket(family(unixSocketPath), SocketType.STREAM); 1501 socket.blocking = false; 1502 } 1503 1504 // FIXME: connect timeout? 1505 if(unixSocketPath) { 1506 import std.stdio; writeln(cast(ubyte[]) unixSocketPath); 1507 socket.connect(new UnixAddress(unixSocketPath)); 1508 } else { 1509 // FIXME: i should prolly do ipv6 if available too. 1510 if(host.length == 0) // this could arguably also be an in contract since it is user error, but the exception is good enough 1511 throw new Exception("No host given for request"); 1512 if(proxy.length) { 1513 if(proxy.indexOf("//") == -1) 1514 proxy = "http://" ~ proxy; 1515 auto proxyurl = Uri(proxy); 1516 1517 //auto proxyhttps = proxyurl.scheme == "https"; 1518 enum proxyhttps = false; // this isn't properly implemented and might never be necessary anyway so meh 1519 1520 // the precise types here are important to help with overload 1521 // resolution of the devirtualized call! 1522 Address pa = new InternetAddress(proxyurl.host, proxyurl.port ? cast(ushort) proxyurl.port : 80); 1523 1524 debug(arsd_http2) writeln("using proxy ", pa.toString()); 1525 1526 if(proxyhttps) { 1527 socket.connect(pa); 1528 } else { 1529 // the proxy never actually starts TLS, but if the request is tls then we need to CONNECT then upgrade the connection 1530 // using the parent class functions let us bypass the encryption 1531 socket.Socket.connect(pa); 1532 } 1533 1534 socket.blocking = true; // FIXME total hack to simplify the code here since it isn't really using the event loop yet 1535 1536 string message; 1537 if(ssl) { 1538 auto hostName = host ~ ":" ~ to!string(port); 1539 message = "CONNECT " ~ hostName ~ " HTTP/1.1\r\n"; 1540 message ~= "Host: " ~ hostName ~ "\r\n"; 1541 if(proxyurl.userinfo.length) { 1542 import std.base64; 1543 message ~= "Proxy-Authorization: Basic " ~ Base64.encode(cast(ubyte[]) proxyurl.userinfo) ~ "\r\n"; 1544 } 1545 message ~= "\r\n"; 1546 1547 // FIXME: what if proxy times out? should be reasonably fast too. 1548 if(proxyhttps) { 1549 socket.send(message, SocketFlags.NONE); 1550 } else { 1551 socket.Socket.send(message, SocketFlags.NONE); 1552 } 1553 1554 ubyte[1024] recvBuffer; 1555 // and last time 1556 ptrdiff_t rcvGot; 1557 if(proxyhttps) { 1558 rcvGot = socket.receive(recvBuffer[], SocketFlags.NONE); 1559 // bool verifyPeer = true; 1560 //(cast(OpenSslSocket)socket).freeSsl(); 1561 //(cast(OpenSslSocket)socket).initSsl(verifyPeer, host); 1562 } else { 1563 rcvGot = socket.Socket.receive(recvBuffer[], SocketFlags.NONE); 1564 } 1565 1566 if(rcvGot == -1) 1567 throw new ProxyException("proxy receive error"); 1568 auto got = cast(string) recvBuffer[0 .. rcvGot]; 1569 auto expect = "HTTP/1.1 200"; 1570 if(got.length < expect.length || (got[0 .. expect.length] != expect && got[0 .. expect.length] != "HTTP/1.0 200")) 1571 throw new ProxyException("Proxy rejected request: " ~ got[0 .. expect.length <= got.length ? expect.length : got.length]); 1572 1573 if(proxyhttps) { 1574 //(cast(OpenSslSocket)socket).do_ssl_connect(); 1575 } else { 1576 (cast(OpenSslSocket)socket).do_ssl_connect(); 1577 } 1578 } else { 1579 } 1580 } else { 1581 socket.connect(new InternetAddress(host, port)); 1582 } 1583 } 1584 1585 debug(arsd_http2) writeln("opening to ", host, ":", port, " ", cast(void*) socket, " ssl=", ssl); 1586 assert(socket.handle() !is socket_t.init); 1587 return socket; 1588 } 1589 1590 // import std.stdio; writeln(cachedSockets); 1591 if(cachedSockets > connectionCacheSize) 1592 cleanOldSockets(); 1593 1594 import std.string; 1595 auto key = format("http%s://%s:%s", ssl ? "s" : "", host, port); 1596 1597 if(auto hostListing = key in socketsPerHost) { 1598 // try to find an available socket that is already open 1599 foreach(socket; *hostListing) { 1600 if(socket !in activeRequestOnSocket) { 1601 // let's see if it has closed since we last tried 1602 // e.g. a server timeout or something. If so, we need 1603 // to lose this one and immediately open a new one. 1604 static SocketSet readSet = null; 1605 if(readSet is null) 1606 readSet = new SocketSet(); 1607 readSet.reset(); 1608 assert(socket !is null); 1609 assert(socket.handle() !is socket_t.init, socket is null ? "null" : socket.toString()); 1610 readSet.add(socket); 1611 auto got = Socket.select(readSet, null, null, 0.msecs /* timeout, want it small since we just checking for eof */); 1612 if(got > 0) { 1613 // we can read something off this... but there aren't 1614 // any active requests. Assume it is EOF and open a new one 1615 1616 socket.close(); 1617 loseSocket(host, port, ssl, socket); 1618 goto openNew; 1619 } 1620 cachedSockets--; 1621 return socket; 1622 } 1623 } 1624 1625 // if not too many already open, go ahead and do a new one 1626 if((*hostListing).length < 6) { 1627 auto socket = openNewConnection(); 1628 (*hostListing) ~= socket; 1629 return socket; 1630 } else 1631 return null; // too many, you'll have to wait 1632 } 1633 1634 openNew: 1635 1636 auto socket = openNewConnection(); 1637 socketsPerHost[key] ~= socket; 1638 return socket; 1639 } 1640 1641 SocketSet readSet; 1642 SocketSet writeSet; 1643 1644 /+ 1645 Generic event loop registration: 1646 1647 handle, operation (read/write), buffer (on posix it *might* be stack if a select loop), timeout (in real time), callback when op completed. 1648 1649 ....basically Windows style. Then it translates internally. 1650 1651 It should tell the thing if the buffer is reused or not 1652 +/ 1653 1654 1655 /++ 1656 This is made public for rudimentary event loop integration, but is still 1657 basically an internal detail. Try not to use it if you have another way. 1658 1659 This does a single iteration of the internal select()-based processing loop. 1660 1661 1662 Future directions: 1663 I want to merge the internal use of [WebSocket.eventLoop] with this; 1664 [advanceConnections] does just one run on the loop, whereas eventLoop 1665 runs it until all connections are closed. But they'd both process both 1666 pending http requests and active websockets. 1667 1668 After that, I want to be able to integrate in other event loops too. 1669 One might be to simply to reactor callbacks, then perhaps Windows overlapped 1670 i/o (that's just going to be tricky to retrofit into the existing select()-based 1671 code). It could then go fiber just by calling the resume function too. 1672 1673 The hard part is ensuring I keep this file stand-alone while offering these 1674 things. 1675 1676 This `advanceConnections` call will probably continue to work now that it is 1677 public, but it may not be wholly compatible with all the future features; you'd 1678 have to pick either the internal event loop or an external one you integrate, but not 1679 mix them. 1680 1681 History: 1682 This has been included in the library since almost day one, but 1683 it was private until April 13, 2021 (dub v9.5). 1684 1685 Params: 1686 maximumTimeout = the maximum time it will wait in select(). It may return much sooner than this if a connection timed out in the mean time. 1687 automaticallyRetryOnInterruption = internally loop on EINTR. 1688 1689 Returns: 1690 1691 0 = no error, work may remain so you should call `advanceConnections` again when you can 1692 1693 1 = passed `maximumTimeout` reached with no work done, yet requests are still in the queue. You may call `advanceConnections` again. 1694 1695 2 = no work to do, no point calling it again unless you've added new requests. Your program may exit if you have nothing to add since it means everything requested is now done. 1696 1697 3 = EINTR occurred on select(), you should check your interrupt flags if you set a signal handler, then call `advanceConnections` again if you aren't exiting. Only occurs if `automaticallyRetryOnInterruption` is set to `false` (the default when it is called externally). 1698 1699 any other value should be considered a non-recoverable error if you want to be forward compatible as I reserve the right to add more values later. 1700 +/ 1701 public int advanceConnections(Duration maximumTimeout = 10.seconds, bool automaticallyRetryOnInterruption = false) { 1702 debug(arsd_http2_verbose) writeln("advancing"); 1703 if(readSet is null) 1704 readSet = new SocketSet(); 1705 if(writeSet is null) 1706 writeSet = new SocketSet(); 1707 1708 ubyte[2048] buffer; 1709 1710 HttpRequest[16] removeFromPending; 1711 size_t removeFromPendingCount = 0; 1712 1713 bool hadAbortedRequest; 1714 1715 // are there pending requests? let's try to send them 1716 foreach(idx, pc; pending) { 1717 if(removeFromPendingCount == removeFromPending.length) 1718 break; 1719 1720 if(pc.state == HttpRequest.State.aborted) { 1721 removeFromPending[removeFromPendingCount++] = pc; 1722 hadAbortedRequest = true; 1723 continue; 1724 } 1725 1726 Socket socket; 1727 1728 try { 1729 socket = getOpenSocketOnHost(pc.proxy, pc.requestParameters.host, pc.requestParameters.port, pc.requestParameters.ssl, pc.requestParameters.unixSocketPath, pc.verifyPeer); 1730 } catch(ProxyException e) { 1731 // connection refused or timed out (I should disambiguate somehow)... 1732 pc.state = HttpRequest.State.aborted; 1733 1734 pc.responseData.code = 2; 1735 pc.responseData.codeText = e.msg ~ " from " ~ pc.proxy; 1736 1737 hadAbortedRequest = true; 1738 1739 removeFromPending[removeFromPendingCount++] = pc; 1740 continue; 1741 1742 } catch(SocketException e) { 1743 // connection refused or timed out (I should disambiguate somehow)... 1744 pc.state = HttpRequest.State.aborted; 1745 1746 pc.responseData.code = 2; 1747 pc.responseData.codeText = pc.proxy.length ? ("connection failed to proxy " ~ pc.proxy) : "connection failed"; 1748 1749 hadAbortedRequest = true; 1750 1751 removeFromPending[removeFromPendingCount++] = pc; 1752 continue; 1753 } catch(Exception e) { 1754 // connection failed due to other user error or SSL (i should disambiguate somehow)... 1755 pc.state = HttpRequest.State.aborted; 1756 1757 pc.responseData.code = 2; 1758 pc.responseData.codeText = e.msg; 1759 1760 hadAbortedRequest = true; 1761 1762 removeFromPending[removeFromPendingCount++] = pc; 1763 continue; 1764 1765 } 1766 1767 if(socket !is null) { 1768 activeRequestOnSocket[socket] = pc; 1769 assert(pc.sendBuffer.length); 1770 pc.state = State.connecting; 1771 1772 removeFromPending[removeFromPendingCount++] = pc; 1773 } 1774 } 1775 1776 import std.algorithm : remove; 1777 foreach(rp; removeFromPending[0 .. removeFromPendingCount]) 1778 pending = pending.remove!((a) => a is rp)(); 1779 1780 tryAgain: 1781 1782 Socket[16] inactive; 1783 int inactiveCount = 0; 1784 void killInactives() { 1785 foreach(s; inactive[0 .. inactiveCount]) { 1786 debug(arsd_http2) writeln("removing socket from active list ", cast(void*) s); 1787 activeRequestOnSocket.remove(s); 1788 cachedSockets++; 1789 } 1790 } 1791 1792 1793 readSet.reset(); 1794 writeSet.reset(); 1795 1796 bool hadOne = false; 1797 1798 auto minTimeout = maximumTimeout; 1799 auto now = MonoTime.currTime; 1800 1801 // active requests need to be read or written to 1802 foreach(sock, request; activeRequestOnSocket) { 1803 1804 if(request.state == State.aborted) { 1805 inactive[inactiveCount++] = sock; 1806 sock.close(); 1807 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1808 hadAbortedRequest = true; 1809 continue; 1810 } 1811 1812 // check the other sockets just for EOF, if they close, take them out of our list, 1813 // we'll reopen if needed upon request. 1814 readSet.add(sock); 1815 hadOne = true; 1816 1817 Duration timeo; 1818 if(request.timeoutFromInactivity <= now) 1819 timeo = 0.seconds; 1820 else 1821 timeo = request.timeoutFromInactivity - now; 1822 1823 if(timeo < minTimeout) 1824 minTimeout = timeo; 1825 1826 if(request.state == State.connecting || request.state == State.sslConnectPendingWrite || request.state == State.sendingHeaders || request.state == State.sendingBody) { 1827 writeSet.add(sock); 1828 hadOne = true; 1829 } 1830 } 1831 1832 if(!hadOne) { 1833 if(hadAbortedRequest) { 1834 killInactives(); 1835 return 0; // something got aborted, that's progress 1836 } 1837 return 2; // automatic timeout, nothing to do 1838 } 1839 1840 auto selectGot = Socket.select(readSet, writeSet, null, minTimeout); 1841 if(selectGot == 0) { /* timeout */ 1842 now = MonoTime.currTime; 1843 bool anyWorkDone = false; 1844 foreach(sock, request; activeRequestOnSocket) { 1845 1846 if(request.timeoutFromInactivity <= now) { 1847 request.state = HttpRequest.State.aborted; 1848 request.responseData.code = 5; 1849 if(request.state == State.connecting) 1850 request.responseData.codeText = "Connect timed out"; 1851 else 1852 request.responseData.codeText = "Request timed out"; 1853 1854 inactive[inactiveCount++] = sock; 1855 sock.close(); 1856 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1857 anyWorkDone = true; 1858 } 1859 } 1860 killInactives(); 1861 return anyWorkDone ? 0 : 1; 1862 // return 1; was an error to time out but now im making it on the individual request 1863 } else if(selectGot == -1) { /* interrupted */ 1864 /* 1865 version(Posix) { 1866 import core.stdc.errno; 1867 if(errno != EINTR) 1868 throw new Exception("select error: " ~ to!string(errno)); 1869 } 1870 */ 1871 if(automaticallyRetryOnInterruption) 1872 goto tryAgain; 1873 else 1874 return 3; 1875 } else { /* ready */ 1876 1877 void sslProceed(HttpRequest request, SslClientSocket s) { 1878 try { 1879 auto code = s.do_ssl_connect(); 1880 switch(code) { 1881 case 0: 1882 request.state = State.sendingHeaders; 1883 break; 1884 case SSL_ERROR_WANT_READ: 1885 request.state = State.sslConnectPendingRead; 1886 break; 1887 case SSL_ERROR_WANT_WRITE: 1888 request.state = State.sslConnectPendingWrite; 1889 break; 1890 default: 1891 assert(0); 1892 } 1893 } catch(Exception e) { 1894 request.state = State.aborted; 1895 1896 request.responseData.code = 2; 1897 request.responseData.codeText = e.msg; 1898 inactive[inactiveCount++] = s; 1899 s.close(); 1900 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, s); 1901 } 1902 } 1903 1904 1905 foreach(sock, request; activeRequestOnSocket) { 1906 // always need to try to send first in part because http works that way but 1907 // also because openssl will sometimes leave something ready to read even if we haven't 1908 // sent yet (probably leftover data from the crypto negotiation) and if that happens ssl 1909 // is liable to block forever hogging the connection and not letting it send... 1910 if(request.state == State.connecting) 1911 if(writeSet.isSet(sock) || readSet.isSet(sock)) { 1912 int error; 1913 int retopt = sock.getOption(SocketOptionLevel.SOCKET, SocketOption.ERROR, error); 1914 if(retopt < 0 || error != 0) { 1915 request.state = State.aborted; 1916 1917 request.responseData.code = 2; 1918 try { 1919 request.responseData.codeText = "connection failed - " ~ formatSocketError(error); 1920 } catch(Exception e) { 1921 request.responseData.codeText = "connection failed"; 1922 } 1923 inactive[inactiveCount++] = sock; 1924 sock.close(); 1925 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1926 continue; 1927 } else { 1928 if(auto s = cast(SslClientSocket) sock) { 1929 sslProceed(request, s); 1930 continue; 1931 } else { 1932 request.state = State.sendingHeaders; 1933 } 1934 } 1935 } 1936 1937 if(request.state == State.sslConnectPendingRead) 1938 if(readSet.isSet(sock)) { 1939 sslProceed(request, cast(SslClientSocket) sock); 1940 continue; 1941 } 1942 if(request.state == State.sslConnectPendingWrite) 1943 if(writeSet.isSet(sock)) { 1944 sslProceed(request, cast(SslClientSocket) sock); 1945 continue; 1946 } 1947 1948 if(request.state == State.sendingHeaders || request.state == State.sendingBody) 1949 if(writeSet.isSet(sock)) { 1950 request.timeoutFromInactivity = MonoTime.currTime + request.requestParameters.timeoutFromInactivity; 1951 assert(request.sendBuffer.length); 1952 auto sent = sock.send(request.sendBuffer); 1953 debug(arsd_http2_verbose) writeln(cast(void*) sock, "<send>", cast(string) request.sendBuffer, "</send>"); 1954 if(sent <= 0) { 1955 if(wouldHaveBlocked()) 1956 continue; 1957 1958 request.state = State.aborted; 1959 1960 request.responseData.code = 3; 1961 request.responseData.codeText = "send failed to server"; 1962 inactive[inactiveCount++] = sock; 1963 sock.close(); 1964 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1965 continue; 1966 1967 } 1968 request.sendBuffer = request.sendBuffer[sent .. $]; 1969 if(request.sendBuffer.length == 0) { 1970 request.state = State.waitingForResponse; 1971 1972 debug(arsd_http2_verbose) writeln("all sent"); 1973 } 1974 } 1975 1976 1977 if(readSet.isSet(sock)) { 1978 keep_going: 1979 request.timeoutFromInactivity = MonoTime.currTime + request.requestParameters.timeoutFromInactivity; 1980 auto got = sock.receive(buffer); 1981 debug(arsd_http2_verbose) { if(got < 0) writeln(lastSocketError); else writeln("====PACKET ",got,"=====",cast(string)buffer[0 .. got],"===/PACKET==="); } 1982 if(got < 0) { 1983 if(wouldHaveBlocked()) 1984 continue; 1985 debug(arsd_http2) writeln("receive error"); 1986 if(request.state != State.complete) { 1987 request.state = State.aborted; 1988 1989 request.responseData.code = 3; 1990 request.responseData.codeText = "receive error from server"; 1991 } 1992 inactive[inactiveCount++] = sock; 1993 sock.close(); 1994 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1995 } else if(got == 0) { 1996 // remote side disconnected 1997 debug(arsd_http2) writeln("remote disconnect"); 1998 if(request.state != State.complete) { 1999 request.state = State.aborted; 2000 2001 request.responseData.code = 3; 2002 request.responseData.codeText = "server disconnected"; 2003 } 2004 inactive[inactiveCount++] = sock; 2005 sock.close(); 2006 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 2007 } else { 2008 // data available 2009 bool stillAlive; 2010 2011 try { 2012 stillAlive = request.handleIncomingData(buffer[0 .. got]); 2013 /+ 2014 state needs to be set and public 2015 requestData.content/contentText needs to be around 2016 you need to be able to clear the content and keep processing for things like event sources. 2017 also need to be able to abort it. 2018 2019 and btw it should prolly just have evnet source as a pre-packaged thing. 2020 +/ 2021 } catch (Exception e) { 2022 debug(arsd_http2_verbose) { import std.stdio; writeln(e); } 2023 request.state = HttpRequest.State.aborted; 2024 request.responseData.code = 4; 2025 request.responseData.codeText = e.msg; 2026 2027 inactive[inactiveCount++] = sock; 2028 sock.close(); 2029 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 2030 continue; 2031 } 2032 2033 if(!stillAlive || request.state == HttpRequest.State.complete || request.state == HttpRequest.State.aborted) { 2034 //import std.stdio; writeln(cast(void*) sock, " ", stillAlive, " ", request.state); 2035 inactive[inactiveCount++] = sock; 2036 continue; 2037 // reuse the socket for another pending request, if we can 2038 } 2039 } 2040 2041 if(request.onDataReceived) 2042 request.onDataReceived(request); 2043 2044 version(with_openssl) 2045 if(auto s = cast(SslClientSocket) sock) { 2046 // select doesn't handle the case with stuff 2047 // left in the ssl buffer so i'm checking it separately 2048 if(s.dataPending()) { 2049 goto keep_going; 2050 } 2051 } 2052 } 2053 } 2054 } 2055 2056 killInactives(); 2057 2058 // we've completed a request, are there any more pending connection? if so, send them now 2059 2060 return 0; 2061 } 2062 } 2063 2064 public static void resetInternals() { 2065 socketsPerHost = null; 2066 activeRequestOnSocket = null; 2067 pending = null; 2068 2069 } 2070 2071 struct HeaderReadingState { 2072 bool justSawLf; 2073 bool justSawCr; 2074 bool atStartOfLine = true; 2075 bool readingLineContinuation; 2076 } 2077 HeaderReadingState headerReadingState; 2078 2079 struct BodyReadingState { 2080 bool isGzipped; 2081 bool isDeflated; 2082 2083 bool isChunked; 2084 int chunkedState; 2085 2086 // used for the chunk size if it is chunked 2087 int contentLengthRemaining; 2088 } 2089 BodyReadingState bodyReadingState; 2090 2091 bool closeSocketWhenComplete; 2092 2093 import std.zlib; 2094 UnCompress uncompress; 2095 2096 const(ubyte)[] leftoverDataFromLastTime; 2097 2098 bool handleIncomingData(scope const ubyte[] dataIn) { 2099 bool stillAlive = true; 2100 debug(arsd_http2) writeln("handleIncomingData, state: ", state); 2101 if(state == State.waitingForResponse) { 2102 state = State.readingHeaders; 2103 headerReadingState = HeaderReadingState.init; 2104 bodyReadingState = BodyReadingState.init; 2105 } 2106 2107 const(ubyte)[] data; 2108 if(leftoverDataFromLastTime.length) 2109 data = leftoverDataFromLastTime ~ dataIn[]; 2110 else 2111 data = dataIn[]; 2112 2113 if(state == State.readingHeaders) { 2114 void parseLastHeader() { 2115 assert(responseData.headers.length); 2116 if(responseData.headers.length == 1) { 2117 responseData.statusLine = responseData.headers[0]; 2118 import std.algorithm; 2119 auto parts = responseData.statusLine.splitter(" "); 2120 responseData.httpVersion = parts.front; 2121 parts.popFront(); 2122 if(parts.empty) 2123 throw new Exception("Corrupted response, bad status line"); 2124 responseData.code = to!int(parts.front()); 2125 parts.popFront(); 2126 responseData.codeText = ""; 2127 while(!parts.empty) { 2128 // FIXME: this sucks! 2129 responseData.codeText ~= parts.front(); 2130 parts.popFront(); 2131 if(!parts.empty) 2132 responseData.codeText ~= " "; 2133 } 2134 } else { 2135 // parse the new header 2136 auto header = responseData.headers[$-1]; 2137 2138 auto colon = header.indexOf(":"); 2139 if(colon < 0 || colon >= header.length) 2140 return; 2141 auto name = toLower(header[0 .. colon]); 2142 auto value = header[colon + 1 .. $].strip; // skip colon and strip whitespace 2143 2144 switch(name) { 2145 case "connection": 2146 if(value == "close") 2147 closeSocketWhenComplete = true; 2148 break; 2149 case "content-type": 2150 responseData.contentType = value; 2151 break; 2152 case "location": 2153 responseData.location = value; 2154 break; 2155 case "content-length": 2156 bodyReadingState.contentLengthRemaining = to!int(value); 2157 break; 2158 case "transfer-encoding": 2159 // note that if it is gzipped, it zips first, then chunks the compressed stream. 2160 // so we should always dechunk first, then feed into the decompressor 2161 if(value == "chunked") 2162 bodyReadingState.isChunked = true; 2163 else throw new Exception("Unknown Transfer-Encoding: " ~ value); 2164 break; 2165 case "content-encoding": 2166 if(value == "gzip") { 2167 bodyReadingState.isGzipped = true; 2168 uncompress = new UnCompress(); 2169 } else if(value == "deflate") { 2170 bodyReadingState.isDeflated = true; 2171 uncompress = new UnCompress(); 2172 } else throw new Exception("Unknown Content-Encoding: " ~ value); 2173 break; 2174 case "set-cookie": 2175 // handled elsewhere fyi 2176 break; 2177 default: 2178 // ignore 2179 } 2180 2181 responseData.headersHash[name] = value; 2182 } 2183 } 2184 2185 size_t position = 0; 2186 for(position = 0; position < data.length; position++) { 2187 if(headerReadingState.readingLineContinuation) { 2188 if(data[position] == ' ' || data[position] == '\t') 2189 continue; 2190 headerReadingState.readingLineContinuation = false; 2191 } 2192 2193 if(headerReadingState.atStartOfLine) { 2194 headerReadingState.atStartOfLine = false; 2195 // FIXME it being \r should never happen... and i don't think it does 2196 if(data[position] == '\r' || data[position] == '\n') { 2197 // done with headers 2198 2199 position++; // skip the \r 2200 2201 if(responseData.headers.length) 2202 parseLastHeader(); 2203 2204 if(responseData.code >= 100 && responseData.code < 200) { 2205 // "100 Continue" - we should continue uploading request data at this point 2206 // "101 Switching Protocols" - websocket, not expected here... 2207 // "102 Processing" - server still working, keep the connection alive 2208 // "103 Early Hints" - can have useful Link headers etc 2209 // 2210 // and other unrecognized ones can just safely be skipped 2211 2212 // FIXME: the headers shouldn't actually be reset; 103 Early Hints 2213 // can give useful headers we want to keep 2214 2215 responseData.headers = null; 2216 headerReadingState.atStartOfLine = true; 2217 2218 continue; // the \n will be skipped by the for loop advance 2219 } 2220 2221 if(this.requestParameters.method == HttpVerb.HEAD) 2222 state = State.complete; 2223 else 2224 state = State.readingBody; 2225 2226 // skip the \n before we break 2227 position++; 2228 2229 break; 2230 } else if(data[position] == ' ' || data[position] == '\t') { 2231 // line continuation, ignore all whitespace and collapse it into a space 2232 headerReadingState.readingLineContinuation = true; 2233 responseData.headers[$-1] ~= ' '; 2234 } else { 2235 // new header 2236 if(responseData.headers.length) 2237 parseLastHeader(); 2238 responseData.headers ~= ""; 2239 } 2240 } 2241 2242 if(data[position] == '\r') { 2243 headerReadingState.justSawCr = true; 2244 continue; 2245 } else 2246 headerReadingState.justSawCr = false; 2247 2248 if(data[position] == '\n') { 2249 headerReadingState.justSawLf = true; 2250 headerReadingState.atStartOfLine = true; 2251 continue; 2252 } else 2253 headerReadingState.justSawLf = false; 2254 2255 responseData.headers[$-1] ~= data[position]; 2256 } 2257 2258 data = data[position .. $]; 2259 } 2260 2261 if(state == State.readingBody) { 2262 if(bodyReadingState.isChunked) { 2263 // read the hex length, stopping at a \r\n, ignoring everything between the new line but after the first non-valid hex character 2264 // read binary data of that length. it is our content 2265 // repeat until a zero sized chunk 2266 // then read footers as headers. 2267 2268 start_over: 2269 for(int a = 0; a < data.length; a++) { 2270 final switch(bodyReadingState.chunkedState) { 2271 case 0: // reading hex 2272 char c = data[a]; 2273 if((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) { 2274 // just keep reading 2275 } else { 2276 int power = 1; 2277 bodyReadingState.contentLengthRemaining = 0; 2278 if(a == 0) 2279 break; // just wait for more data 2280 assert(a != 0, cast(string) data); 2281 for(int b = a-1; b >= 0; b--) { 2282 char cc = data[b]; 2283 if(cc >= 'a' && cc <= 'z') 2284 cc -= 0x20; 2285 int val = 0; 2286 if(cc >= '0' && cc <= '9') 2287 val = cc - '0'; 2288 else 2289 val = cc - 'A' + 10; 2290 2291 assert(val >= 0 && val <= 15, to!string(val)); 2292 bodyReadingState.contentLengthRemaining += power * val; 2293 power *= 16; 2294 } 2295 debug(arsd_http2_verbose) writeln("Chunk length: ", bodyReadingState.contentLengthRemaining); 2296 bodyReadingState.chunkedState = 1; 2297 data = data[a + 1 .. $]; 2298 goto start_over; 2299 } 2300 break; 2301 case 1: // reading until end of line 2302 char c = data[a]; 2303 if(c == '\n') { 2304 if(bodyReadingState.contentLengthRemaining == 0) 2305 bodyReadingState.chunkedState = 5; 2306 else 2307 bodyReadingState.chunkedState = 2; 2308 } 2309 data = data[a + 1 .. $]; 2310 goto start_over; 2311 case 2: // reading data 2312 auto can = a + bodyReadingState.contentLengthRemaining; 2313 if(can > data.length) 2314 can = cast(int) data.length; 2315 2316 auto newData = data[a .. can]; 2317 data = data[can .. $]; 2318 2319 //if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) 2320 // responseData.content ~= cast(ubyte[]) uncompress.uncompress(data[a .. can]); 2321 //else 2322 responseData.content ~= newData; 2323 2324 bodyReadingState.contentLengthRemaining -= newData.length; 2325 debug(arsd_http2_verbose) writeln("clr: ", bodyReadingState.contentLengthRemaining, " " , a, " ", can); 2326 assert(bodyReadingState.contentLengthRemaining >= 0); 2327 if(bodyReadingState.contentLengthRemaining == 0) { 2328 bodyReadingState.chunkedState = 3; 2329 } else { 2330 // will continue grabbing more 2331 } 2332 goto start_over; 2333 case 3: // reading 13/10 2334 assert(data[a] == 13); 2335 bodyReadingState.chunkedState++; 2336 data = data[a + 1 .. $]; 2337 goto start_over; 2338 case 4: // reading 10 at end of packet 2339 assert(data[a] == 10); 2340 data = data[a + 1 .. $]; 2341 bodyReadingState.chunkedState = 0; 2342 goto start_over; 2343 case 5: // reading footers 2344 //goto done; // FIXME 2345 2346 int footerReadingState = 0; 2347 int footerSize; 2348 2349 while(footerReadingState != 2 && a < data.length) { 2350 // import std.stdio; writeln(footerReadingState, " ", footerSize, " ", data); 2351 switch(footerReadingState) { 2352 case 0: 2353 if(data[a] == 13) 2354 footerReadingState++; 2355 else 2356 footerSize++; 2357 break; 2358 case 1: 2359 if(data[a] == 10) { 2360 if(footerSize == 0) { 2361 // all done, time to break 2362 footerReadingState++; 2363 2364 } else { 2365 // actually had a footer, try to read another 2366 footerReadingState = 0; 2367 footerSize = 0; 2368 } 2369 } else { 2370 throw new Exception("bad footer thing"); 2371 } 2372 break; 2373 default: 2374 assert(0); 2375 } 2376 2377 a++; 2378 } 2379 2380 if(footerReadingState != 2) 2381 break start_over; // haven't hit the end of the thing yet 2382 2383 bodyReadingState.chunkedState = 0; 2384 data = data[a .. $]; 2385 2386 if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) { 2387 auto n = uncompress.uncompress(responseData.content); 2388 n ~= uncompress.flush(); 2389 responseData.content = cast(ubyte[]) n; 2390 } 2391 2392 // responseData.content ~= cast(ubyte[]) uncompress.flush(); 2393 responseData.contentText = cast(string) responseData.content; 2394 2395 goto done; 2396 } 2397 } 2398 2399 } else { 2400 //if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) 2401 // responseData.content ~= cast(ubyte[]) uncompress.uncompress(data); 2402 //else 2403 responseData.content ~= data; 2404 //assert(data.length <= bodyReadingState.contentLengthRemaining, format("%d <= %d\n%s", data.length, bodyReadingState.contentLengthRemaining, cast(string)data)); 2405 { 2406 int use = cast(int) data.length; 2407 if(use > bodyReadingState.contentLengthRemaining) 2408 use = bodyReadingState.contentLengthRemaining; 2409 bodyReadingState.contentLengthRemaining -= use; 2410 data = data[use .. $]; 2411 } 2412 if(bodyReadingState.contentLengthRemaining == 0) { 2413 if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) { 2414 // import std.stdio; writeln(responseData.content.length, " ", responseData.content[0 .. 2], " .. ", responseData.content[$-2 .. $]); 2415 auto n = uncompress.uncompress(responseData.content); 2416 n ~= uncompress.flush(); 2417 responseData.content = cast(ubyte[]) n; 2418 responseData.contentText = cast(string) responseData.content; 2419 //responseData.content ~= cast(ubyte[]) uncompress.flush(); 2420 } else { 2421 responseData.contentText = cast(string) responseData.content; 2422 } 2423 2424 done: 2425 2426 if(retainCookies && client !is null) { 2427 client.retainCookies(responseData); 2428 } 2429 2430 if(followLocation && responseData.location.length) { 2431 if(maximumNumberOfRedirectsRemaining <= 0) { 2432 throw new Exception("Maximum number of redirects exceeded"); 2433 } else { 2434 maximumNumberOfRedirectsRemaining--; 2435 } 2436 2437 static bool first = true; 2438 //version(DigitalMars) if(!first) asm { int 3; } 2439 debug(arsd_http2) writeln("redirecting to ", responseData.location); 2440 populateFromInfo(Uri(responseData.location), HttpVerb.GET); 2441 //import std.stdio; writeln("redirected to ", responseData.location); 2442 first = false; 2443 responseData = HttpResponse.init; 2444 headerReadingState = HeaderReadingState.init; 2445 bodyReadingState = BodyReadingState.init; 2446 if(client !is null) { 2447 // FIXME: this won't clear cookies that were cleared in another request 2448 client.populateCookies(this); // they might have changed in the previous redirection cycle! 2449 } 2450 state = State.unsent; 2451 stillAlive = false; 2452 sendPrivate(false); 2453 } else { 2454 state = State.complete; 2455 // FIXME 2456 //if(closeSocketWhenComplete) 2457 //socket.close(); 2458 } 2459 } 2460 } 2461 } 2462 2463 if(data.length) 2464 leftoverDataFromLastTime = data.dup; 2465 else 2466 leftoverDataFromLastTime = null; 2467 2468 return stillAlive; 2469 } 2470 2471 } 2472 } 2473 2474 /++ 2475 Waits for the first of the given requests to be either aborted or completed. 2476 Returns the first one in that state, or `null` if the operation was interrupted 2477 or reached the given timeout before any completed. (If it returns null even before 2478 the timeout, it might be because the user pressed ctrl+c, so you should consider 2479 checking if you should cancel the operation. If not, you can simply call it again 2480 with the same arguments to start waiting again.) 2481 2482 You MUST check for null, even if you don't specify a timeout! 2483 2484 Note that if an individual request times out before any others request, it will 2485 return that timed out request, since that counts as completion. 2486 2487 If the return is not null, you should call `waitForCompletion` on the given request 2488 to get the response out. It will not have to wait since it is guaranteed to be 2489 finished when returned by this function; that will just give you the cached response. 2490 2491 (I thought about just having it return the response, but tying a response back to 2492 a request is harder than just getting the original request object back and taking 2493 the response out of it.) 2494 2495 Please note: if a request in the set has already completed or been aborted, it will 2496 always return the first one it sees upon calling the function. You may wish to remove 2497 them from the list before calling the function. 2498 2499 History: 2500 Added December 24, 2021 (dub v10.5) 2501 +/ 2502 HttpRequest waitForFirstToComplete(Duration timeout, HttpRequest[] requests...) { 2503 2504 foreach(request; requests) { 2505 if(request.state == HttpRequest.State.unsent) 2506 request.send(); 2507 else if(request.state == HttpRequest.State.complete) 2508 return request; 2509 else if(request.state == HttpRequest.State.aborted) 2510 return request; 2511 } 2512 2513 while(true) { 2514 if(auto err = HttpRequest.advanceConnections(timeout)) { 2515 switch(err) { 2516 case 1: return null; 2517 case 2: throw new Exception("HttpRequest.advanceConnections returned 2: nothing to do"); 2518 case 3: return null; 2519 default: throw new Exception("HttpRequest.advanceConnections got err " ~ to!string(err)); 2520 } 2521 } 2522 2523 foreach(request; requests) { 2524 if(request.state == HttpRequest.State.aborted || request.state == HttpRequest.State.complete) { 2525 request.waitForCompletion(); 2526 return request; 2527 } 2528 } 2529 2530 } 2531 } 2532 2533 /// ditto 2534 HttpRequest waitForFirstToComplete(HttpRequest[] requests...) { 2535 return waitForFirstToComplete(1.weeks, requests); 2536 } 2537 2538 /++ 2539 An input range that runs [waitForFirstToComplete] but only returning each request once. 2540 Before you loop over it, you can set some properties to customize behavior. 2541 2542 If it times out or is interrupted, it will prematurely run empty. You can set the delegate 2543 to process this. 2544 2545 Implementation note: each iteration through the loop does a O(n) check over each item remaining. 2546 This shouldn't matter, but if it does become an issue for you, let me know. 2547 2548 History: 2549 Added December 24, 2021 (dub v10.5) 2550 +/ 2551 struct HttpRequestsAsTheyComplete { 2552 /++ 2553 Seeds it with an overall timeout and the initial requests. 2554 It will send all the requests before returning, then will process 2555 the responses as they come. 2556 2557 Please note that it modifies the array of requests you pass in! It 2558 will keep a reference to it and reorder items on each call of popFront. 2559 You might want to pass a duplicate if you have another purpose for your 2560 array and don't want to see it shuffled. 2561 +/ 2562 this(Duration timeout, HttpRequest[] requests) { 2563 remainingRequests = requests; 2564 this.timeout = timeout; 2565 popFront(); 2566 } 2567 2568 /++ 2569 You can set this delegate to decide how to handle an interruption. Returning true 2570 from this will keep working. Returning false will terminate the loop. 2571 2572 If this is null, an interruption will always terminate the loop. 2573 2574 Note that interruptions can be caused by the garbage collector being triggered by 2575 another thread as well as by user action. If you don't set a SIGINT handler, it 2576 might be reasonable to always return true here. 2577 +/ 2578 bool delegate() onInterruption; 2579 2580 private HttpRequest[] remainingRequests; 2581 2582 /// The timeout you set in the constructor. You can change it if you want. 2583 Duration timeout; 2584 2585 /++ 2586 Adds another request to the work queue. It is safe to call this from inside the loop 2587 as you process other requests. 2588 +/ 2589 void appendRequest(HttpRequest request) { 2590 remainingRequests ~= request; 2591 } 2592 2593 /++ 2594 If the loop exited, it might be due to an interruption or a time out. If you like, you 2595 can call this to pick up the work again, 2596 2597 If it returns `false`, the work is indeed all finished and you should not re-enter the loop. 2598 2599 --- 2600 auto range = HttpRequestsAsTheyComplete(10.seconds, your_requests); 2601 process_loop: foreach(req; range) { 2602 // process req 2603 } 2604 // make sure we weren't interrupted because the user requested we cancel! 2605 // but then try to re-enter the range if possible 2606 if(!user_quit && range.reenter()) { 2607 // there's still something unprocessed in there 2608 // range.reenter returning true means it is no longer 2609 // empty, so we should try to loop over it again 2610 goto process_loop; // re-enter the loop 2611 } 2612 --- 2613 +/ 2614 bool reenter() { 2615 if(remainingRequests.length == 0) 2616 return false; 2617 empty = false; 2618 popFront(); 2619 return true; 2620 } 2621 2622 /// Standard range primitives. I reserve the right to change the variables to read-only properties in the future without notice. 2623 HttpRequest front; 2624 2625 /// ditto 2626 bool empty; 2627 2628 /// ditto 2629 void popFront() { 2630 resume: 2631 if(remainingRequests.length == 0) { 2632 empty = true; 2633 return; 2634 } 2635 2636 front = waitForFirstToComplete(timeout, remainingRequests); 2637 2638 if(front is null) { 2639 if(onInterruption) { 2640 if(onInterruption()) 2641 goto resume; 2642 } 2643 empty = true; 2644 return; 2645 } 2646 foreach(idx, req; remainingRequests) { 2647 if(req is front) { 2648 remainingRequests[idx] = remainingRequests[$ - 1]; 2649 remainingRequests = remainingRequests[0 .. $ - 1]; 2650 return; 2651 } 2652 } 2653 } 2654 } 2655 2656 // 2657 struct HttpRequestParameters { 2658 // FIXME: implement these 2659 //Duration timeoutTotal; // the whole request must finish in this time or else it fails,even if data is still trickling in 2660 Duration timeoutFromInactivity; // if there's no activity in this time it dies. basically the socket receive timeout 2661 2662 // debugging 2663 bool useHttp11 = true; /// 2664 bool acceptGzip = true; /// 2665 bool keepAlive = true; /// 2666 2667 // the request itself 2668 HttpVerb method; /// 2669 string host; /// 2670 ushort port; /// 2671 string uri; /// 2672 2673 bool ssl; /// 2674 2675 string userAgent; /// 2676 string authorization; /// 2677 2678 string[string] cookies; /// 2679 2680 string[] headers; /// do not duplicate host, content-length, content-type, or any others that have a specific property 2681 2682 string contentType; /// 2683 ubyte[] bodyData; /// 2684 2685 string unixSocketPath; /// 2686 } 2687 2688 interface IHttpClient { 2689 2690 } 2691 2692 /// 2693 enum HttpVerb { 2694 /// 2695 GET, 2696 /// 2697 HEAD, 2698 /// 2699 POST, 2700 /// 2701 PUT, 2702 /// 2703 DELETE, 2704 /// 2705 OPTIONS, 2706 /// 2707 TRACE, 2708 /// 2709 CONNECT, 2710 /// 2711 PATCH, 2712 /// 2713 MERGE 2714 } 2715 2716 /++ 2717 Supported file formats for [HttpClient.setClientCert]. These are loaded by OpenSSL 2718 in the current implementation. 2719 2720 History: 2721 Added February 3, 2022 (dub v10.6) 2722 +/ 2723 enum CertificateFileFormat { 2724 guess, /// try to guess the format from the file name and/or contents 2725 pem, /// the files are specifically in PEM format 2726 der /// the files are specifically in DER format 2727 } 2728 2729 /++ 2730 HttpClient keeps cookies, location, and some other state to reuse connections, when possible, like a web browser. 2731 You can use it as your entry point to make http requests. 2732 2733 See the example on [arsd.http2#examples]. 2734 +/ 2735 class HttpClient { 2736 /* Protocol restrictions, useful to disable when debugging servers */ 2737 bool useHttp11 = true; /// 2738 bool acceptGzip = true; /// 2739 bool keepAlive = true; /// 2740 2741 /++ 2742 Sets the client certificate used as a log in identifier on https connections. 2743 The certificate and key must be unencrypted at this time and both must be in 2744 the same file format. 2745 2746 Bugs: 2747 The current implementation sets the filenames into a static variable, 2748 meaning it is shared across all clients and connections. 2749 2750 Errors in the cert or key are only reported if the server reports an 2751 authentication failure. Make sure you are passing correct filenames 2752 and formats of you do see a failure. 2753 2754 History: 2755 Added February 2, 2022 (dub v10.6) 2756 +/ 2757 void setClientCertificate(string certFilename, string keyFilename, CertificateFileFormat certFormat = CertificateFileFormat.guess) { 2758 this.certFilename = certFilename; 2759 this.keyFilename = keyFilename; 2760 this.certFormat = certFormat; 2761 } 2762 2763 /++ 2764 Sets whether [HttpRequest]s created through this object (with [navigateTo], [request], etc.), will have the 2765 value of [HttpRequest.verifyPeer] of true or false upon construction. 2766 2767 History: 2768 Added April 5, 2022 (dub v10.8). Previously, there was an undocumented global value used. 2769 +/ 2770 bool defaultVerifyPeer = true; 2771 2772 // FIXME: try to not make these static 2773 private static string certFilename; 2774 private static string keyFilename; 2775 private static CertificateFileFormat certFormat; 2776 2777 /// 2778 @property Uri location() { 2779 return currentUrl; 2780 } 2781 2782 /++ 2783 Default timeout for requests created on this client. 2784 2785 History: 2786 Added March 31, 2021 2787 +/ 2788 Duration defaultTimeout = 10.seconds; 2789 2790 /++ 2791 High level function that works similarly to entering a url 2792 into a browser. 2793 2794 Follows locations, retain cookies, updates the current url, etc. 2795 +/ 2796 HttpRequest navigateTo(Uri where, HttpVerb method = HttpVerb.GET) { 2797 currentUrl = where.basedOn(currentUrl); 2798 currentDomain = where.host; 2799 2800 auto request = this.request(currentUrl, method); 2801 request.followLocation = true; 2802 request.retainCookies = true; 2803 2804 return request; 2805 } 2806 2807 /++ 2808 Creates a request without updating the current url state. If you want to save cookies, either call [retainCookies] with the response yourself 2809 or set [HttpRequest.retainCookies|request.retainCookies] to `true` on the returned object. But see important implementation shortcomings on [retainCookies]. 2810 +/ 2811 HttpRequest request(Uri uri, HttpVerb method = HttpVerb.GET, ubyte[] bodyData = null, string contentType = null) { 2812 string proxyToUse = getProxyFor(uri); 2813 2814 auto request = new HttpRequest(this, uri, method, cache, defaultTimeout, proxyToUse); 2815 2816 request.verifyPeer = this.defaultVerifyPeer; 2817 2818 request.requestParameters.userAgent = userAgent; 2819 request.requestParameters.authorization = authorization; 2820 2821 request.requestParameters.useHttp11 = this.useHttp11; 2822 request.requestParameters.acceptGzip = this.acceptGzip; 2823 request.requestParameters.keepAlive = this.keepAlive; 2824 2825 request.requestParameters.bodyData = bodyData; 2826 request.requestParameters.contentType = contentType; 2827 2828 populateCookies(request); 2829 2830 return request; 2831 2832 } 2833 2834 private void populateCookies(HttpRequest request) { 2835 // FIXME: what about expiration and the like? or domain/path checks? or Secure checks? 2836 // FIXME: is uri.host correct? i think it should include port number too. what fun. 2837 if(auto cookies = ""/*uri.host*/ in this.cookies) { 2838 foreach(cookie; *cookies) 2839 request.requestParameters.cookies[cookie.name] = cookie.value; 2840 } 2841 } 2842 2843 2844 /// ditto 2845 HttpRequest request(Uri uri, FormData fd, HttpVerb method = HttpVerb.POST) { 2846 return request(uri, method, fd.toBytes, fd.contentType); 2847 } 2848 2849 2850 private Uri currentUrl; 2851 private string currentDomain; 2852 private ICache cache; 2853 2854 /++ 2855 2856 +/ 2857 this(ICache cache = null) { 2858 this.defaultVerifyPeer = .defaultVerifyPeer_; 2859 this.cache = cache; 2860 loadDefaultProxy(); 2861 } 2862 2863 /++ 2864 Loads the system-default proxy. Note that the constructor does this automatically 2865 so you should rarely need to call this explicitly. 2866 2867 The environment variables are used, if present, on all operating systems. 2868 2869 History: 2870 no_proxy support added April 13, 2022 2871 2872 Added April 12, 2021 (included in dub v9.5) 2873 2874 Bugs: 2875 On Windows, it does NOT currently check the IE settings, but I do intend to 2876 implement that in the future. When I do, it will be classified as a bug fix, 2877 NOT a breaking change. 2878 +/ 2879 void loadDefaultProxy() { 2880 import std.process; 2881 httpProxy = environment.get("http_proxy", environment.get("HTTP_PROXY", null)); 2882 httpsProxy = environment.get("https_proxy", environment.get("HTTPS_PROXY", null)); 2883 auto noProxy = environment.get("no_proxy", environment.get("NO_PROXY", null)); 2884 if (noProxy.length) { 2885 proxyIgnore = noProxy.split(","); 2886 foreach (ref rule; proxyIgnore) 2887 rule = rule.strip; 2888 } 2889 2890 // FIXME: on Windows, I should use the Internet Explorer proxy settings 2891 } 2892 2893 /++ 2894 Checks if the given uri should be proxied according to the httpProxy, httpsProxy, proxyIgnore 2895 variables and returns either httpProxy, httpsProxy or null. 2896 2897 If neither `httpProxy` or `httpsProxy` are set this always returns `null`. Same if `proxyIgnore` 2898 contains `*`. 2899 2900 DNS is not resolved for proxyIgnore IPs, only IPs match IPs and hosts match hosts. 2901 +/ 2902 string getProxyFor(Uri uri) { 2903 string proxyToUse; 2904 switch(uri.scheme) { 2905 case "http": 2906 proxyToUse = httpProxy; 2907 break; 2908 case "https": 2909 proxyToUse = httpsProxy; 2910 break; 2911 default: 2912 proxyToUse = null; 2913 } 2914 2915 if (proxyToUse.length) { 2916 foreach (ignore; proxyIgnore) { 2917 if (matchProxyIgnore(ignore, uri)) { 2918 return null; 2919 } 2920 } 2921 } 2922 2923 return proxyToUse; 2924 } 2925 2926 /// Returns -1 on error, otherwise the IP as uint. Parsing is very strict. 2927 private static long tryParseIPv4(scope const(char)[] s) nothrow { 2928 import std.algorithm : findSplit, all; 2929 import std.ascii : isDigit; 2930 2931 static int parseNum(scope const(char)[] num) nothrow { 2932 if (num.length < 1 || num.length > 3 || !num.representation.all!isDigit) 2933 return -1; 2934 try { 2935 auto ret = num.to!int; 2936 return ret > 255 ? -1 : ret; 2937 } catch (Exception) { 2938 assert(false); 2939 } 2940 } 2941 2942 if (s.length < "0.0.0.0".length || s.length > "255.255.255.255".length) 2943 return -1; 2944 auto firstPair = s.findSplit("."); 2945 auto secondPair = firstPair[2].findSplit("."); 2946 auto thirdPair = secondPair[2].findSplit("."); 2947 auto a = parseNum(firstPair[0]); 2948 auto b = parseNum(secondPair[0]); 2949 auto c = parseNum(thirdPair[0]); 2950 auto d = parseNum(thirdPair[2]); 2951 if (a < 0 || b < 0 || c < 0 || d < 0) 2952 return -1; 2953 return (cast(uint)a << 24) | (b << 16) | (c << 8) | (d); 2954 } 2955 2956 unittest { 2957 assert(tryParseIPv4("0.0.0.0") == 0); 2958 assert(tryParseIPv4("127.0.0.1") == 0x7f000001); 2959 assert(tryParseIPv4("162.217.114.56") == 0xa2d97238); 2960 assert(tryParseIPv4("256.0.0.1") == -1); 2961 assert(tryParseIPv4("0.0.0.-2") == -1); 2962 assert(tryParseIPv4("0.0.0.a") == -1); 2963 assert(tryParseIPv4("0.0.0") == -1); 2964 assert(tryParseIPv4("0.0.0.0.0") == -1); 2965 } 2966 2967 /++ 2968 Returns true if the given no_proxy rule matches the uri. 2969 2970 Invalid IP ranges are silently ignored and return false. 2971 2972 See $(LREF proxyIgnore). 2973 +/ 2974 static bool matchProxyIgnore(scope const(char)[] rule, scope const Uri uri) nothrow { 2975 import std.algorithm; 2976 import std.ascii : isDigit; 2977 import std.uni : sicmp; 2978 2979 string uriHost = uri.host; 2980 if (uriHost.length && uriHost[$ - 1] == '.') 2981 uriHost = uriHost[0 .. $ - 1]; 2982 2983 if (rule == "*") 2984 return true; 2985 while (rule.length && rule[0] == '.') rule = rule[1 .. $]; 2986 2987 static int parsePort(scope const(char)[] portStr) nothrow { 2988 if (portStr.length < 1 || portStr.length > 5 || !portStr.representation.all!isDigit) 2989 return -1; 2990 try { 2991 return portStr.to!int; 2992 } catch (Exception) { 2993 assert(false, "to!int should succeed"); 2994 } 2995 } 2996 2997 if (sicmp(rule, uriHost) == 0 2998 || (uriHost.length > rule.length 2999 && sicmp(rule, uriHost[$ - rule.length .. $]) == 0 3000 && uriHost[$ - rule.length - 1] == '.')) 3001 return true; 3002 3003 if (rule.startsWith("[")) { // IPv6 3004 // below code is basically nothrow lastIndexOfAny("]:") 3005 ptrdiff_t lastColon = cast(ptrdiff_t) rule.length - 1; 3006 while (lastColon >= 0) { 3007 if (rule[lastColon] == ']' || rule[lastColon] == ':') 3008 break; 3009 lastColon--; 3010 } 3011 if (lastColon == -1) 3012 return false; // malformed 3013 3014 if (rule[lastColon] == ':') { // match with port 3015 auto port = parsePort(rule[lastColon + 1 .. $]); 3016 if (port != -1) { 3017 if (uri.effectivePort != port.to!int) 3018 return false; 3019 return uriHost == rule[0 .. lastColon]; 3020 } 3021 } 3022 // exact match of host already done above 3023 } else { 3024 auto slash = rule.lastIndexOfNothrow('/'); 3025 if (slash == -1) { // no IP range 3026 auto colon = rule.lastIndexOfNothrow(':'); 3027 auto host = colon == -1 ? rule : rule[0 .. colon]; 3028 auto port = colon != -1 ? parsePort(rule[colon + 1 .. $]) : -1; 3029 auto ip = tryParseIPv4(host); 3030 if (ip == -1) { // not an IPv4, test for host with port 3031 return port != -1 3032 && uri.effectivePort == port 3033 && uriHost == host; 3034 } else { 3035 // perform IPv4 equals 3036 auto other = tryParseIPv4(uriHost); 3037 if (other == -1) 3038 return false; // rule == IPv4, uri != IPv4 3039 if (port != -1) 3040 return uri.effectivePort == port 3041 && uriHost == host; 3042 else 3043 return uriHost == host; 3044 } 3045 } else { 3046 auto maskStr = rule[slash + 1 .. $]; 3047 auto ip = tryParseIPv4(rule[0 .. slash]); 3048 if (ip == -1) 3049 return false; 3050 if (maskStr.length && maskStr.length < 3 && maskStr.representation.all!isDigit) { 3051 // IPv4 range match 3052 int mask; 3053 try { 3054 mask = maskStr.to!int; 3055 } catch (Exception) { 3056 assert(false); 3057 } 3058 3059 auto other = tryParseIPv4(uriHost); 3060 if (other == -1) 3061 return false; // rule == IPv4, uri != IPv4 3062 3063 if (mask == 0) // matches all 3064 return true; 3065 if (mask > 32) // matches none 3066 return false; 3067 3068 auto shift = 32 - mask; 3069 return cast(uint)other >> shift 3070 == cast(uint)ip >> shift; 3071 } 3072 } 3073 } 3074 return false; 3075 } 3076 3077 unittest { 3078 assert(matchProxyIgnore("0.0.0.0/0", Uri("http://127.0.0.1:80/a"))); 3079 assert(matchProxyIgnore("0.0.0.0/0", Uri("http://127.0.0.1/a"))); 3080 assert(!matchProxyIgnore("0.0.0.0/0", Uri("https://dlang.org/a"))); 3081 assert(matchProxyIgnore("*", Uri("https://dlang.org/a"))); 3082 assert(matchProxyIgnore("127.0.0.0/8", Uri("http://127.0.0.1:80/a"))); 3083 assert(matchProxyIgnore("127.0.0.0/8", Uri("http://127.0.0.1/a"))); 3084 assert(matchProxyIgnore("127.0.0.1", Uri("http://127.0.0.1:1234/a"))); 3085 assert(!matchProxyIgnore("127.0.0.1:80", Uri("http://127.0.0.1:1234/a"))); 3086 assert(!matchProxyIgnore("127.0.0.1/8", Uri("http://localhost/a"))); // no DNS resolution / guessing 3087 assert(!matchProxyIgnore("0.0.0.0/1", Uri("http://localhost/a")) 3088 && !matchProxyIgnore("128.0.0.0/1", Uri("http://localhost/a"))); // no DNS resolution / guessing 2 3089 foreach (m; 1 .. 32) { 3090 assert(matchProxyIgnore(text("127.0.0.1/", m), Uri("http://127.0.0.1/a"))); 3091 assert(!matchProxyIgnore(text("127.0.0.1/", m), Uri("http://128.0.0.1/a"))); 3092 bool expectedMatch = m <= 24; 3093 assert(expectedMatch == matchProxyIgnore(text("127.0.1.0/", m), Uri("http://127.0.1.128/a")), m.to!string); 3094 } 3095 assert(matchProxyIgnore("localhost", Uri("http://localhost/a"))); 3096 assert(matchProxyIgnore("localhost", Uri("http://foo.localhost/a"))); 3097 assert(matchProxyIgnore("localhost", Uri("http://foo.localhost./a"))); 3098 assert(matchProxyIgnore(".localhost", Uri("http://localhost/a"))); 3099 assert(matchProxyIgnore(".localhost", Uri("http://foo.localhost/a"))); 3100 assert(matchProxyIgnore(".localhost", Uri("http://foo.localhost./a"))); 3101 assert(!matchProxyIgnore("foo.localhost", Uri("http://localhost/a"))); 3102 assert(matchProxyIgnore("foo.localhost", Uri("http://foo.localhost/a"))); 3103 assert(matchProxyIgnore("foo.localhost", Uri("http://foo.localhost./a"))); 3104 assert(!matchProxyIgnore("bar.localhost", Uri("http://localhost/a"))); 3105 assert(!matchProxyIgnore("bar.localhost", Uri("http://foo.localhost/a"))); 3106 assert(!matchProxyIgnore("bar.localhost", Uri("http://foo.localhost./a"))); 3107 assert(!matchProxyIgnore("bar.localhost", Uri("http://bbar.localhost./a"))); 3108 assert(matchProxyIgnore("[::1]", Uri("http://[::1]/a"))); 3109 assert(!matchProxyIgnore("[::1]", Uri("http://[::2]/a"))); 3110 assert(matchProxyIgnore("[::1]:80", Uri("http://[::1]/a"))); 3111 assert(!matchProxyIgnore("[::1]:443", Uri("http://[::1]/a"))); 3112 assert(!matchProxyIgnore("[::1]:80", Uri("https://[::1]/a"))); 3113 assert(matchProxyIgnore("[::1]:443", Uri("https://[::1]/a"))); 3114 assert(matchProxyIgnore("google.com", Uri("https://GOOGLE.COM/a"))); 3115 } 3116 3117 /++ 3118 Proxies to use for requests. The [HttpClient] constructor will set these to the system values, 3119 then you can reset it to `null` if you want to override and not use the proxy after all, or you 3120 can set it after construction to whatever. 3121 3122 The proxy from the client will be automatically set to the requests performed through it. You can 3123 also override on a per-request basis by creating the request and setting the `proxy` field there 3124 before sending it. 3125 3126 History: 3127 Added April 12, 2021 (included in dub v9.5) 3128 +/ 3129 string httpProxy; 3130 /// ditto 3131 string httpsProxy; 3132 /++ 3133 List of hosts or ips, optionally including a port, where not to proxy. 3134 3135 Each entry may be one of the following formats: 3136 - `127.0.0.1` (IPv4, any port) 3137 - `127.0.0.1:1234` (IPv4, specific port) 3138 - `127.0.0.1/8` (IPv4 range / CIDR block, any port) 3139 - `[::1]` (IPv6, any port) 3140 - `[::1]:1234` (IPv6, specific port) 3141 - `*` (all hosts and ports, basically don't proxy at all anymore) 3142 - `.domain.name`, `domain.name` (don't proxy the specified domain, 3143 leading dots are stripped and subdomains are also not proxied) 3144 - `.domain.name:1234`, `domain.name:1234` (same as above, with specific port) 3145 3146 No DNS resolution or regex is done in this list. 3147 3148 See https://about.gitlab.com/blog/2021/01/27/we-need-to-talk-no-proxy/ 3149 3150 History: 3151 Added April 13, 2022 3152 +/ 3153 string[] proxyIgnore; 3154 3155 /// See [retainCookies] for important caveats. 3156 void setCookie(string name, string value, string domain = null) { 3157 CookieHeader ch; 3158 3159 ch.name = name; 3160 ch.value = value; 3161 3162 setCookie(ch, domain); 3163 } 3164 3165 /// ditto 3166 void setCookie(CookieHeader ch, string domain = null) { 3167 if(domain is null) 3168 domain = currentDomain; 3169 3170 // FIXME: figure all this out or else cookies liable to get too long, in addition to the overwriting and oversharing issues in long scraping sessions 3171 cookies[""/*domain*/] ~= ch; 3172 } 3173 3174 /++ 3175 [HttpClient] does NOT automatically store cookies. You must explicitly retain them from a response by calling this method. 3176 3177 Examples: 3178 --- 3179 import arsd.http2; 3180 void main() { 3181 auto client = new HttpClient(); 3182 auto setRequest = client.request(Uri("http://arsdnet.net/cgi-bin/cookies/set")); 3183 auto setResponse = setRequest.waitForCompletion(); 3184 3185 auto request = client.request(Uri("http://arsdnet.net/cgi-bin/cookies/get")); 3186 auto response = request.waitForCompletion(); 3187 3188 // the cookie wasn't explicitly retained, so the server echos back nothing 3189 assert(response.responseText.length == 0); 3190 3191 // now keep the cookies from our original set 3192 client.retainCookies(setResponse); 3193 3194 request = client.request(Uri("http://arsdnet.net/cgi-bin/cookies/get")); 3195 response = request.waitForCompletion(); 3196 3197 // now it matches 3198 assert(response.responseText.length && response.responseText == setResponse.cookies["example-cookie"]); 3199 } 3200 --- 3201 3202 Bugs: 3203 It does NOT currently implement domain / path / secure separation nor cookie expiration. It assumes that if you call this function, you're ok with it. 3204 3205 You may want to use separate HttpClient instances if any sharing is unacceptable at this time. 3206 3207 History: 3208 Added July 5, 2021 (dub v10.2) 3209 +/ 3210 void retainCookies(HttpResponse fromResponse) { 3211 foreach(name, value; fromResponse.cookies) 3212 setCookie(name, value); 3213 } 3214 3215 /// 3216 void clearCookies(string domain = null) { 3217 if(domain is null) 3218 cookies = null; 3219 else 3220 cookies[domain] = null; 3221 } 3222 3223 // If you set these, they will be pre-filled on all requests made with this client 3224 string userAgent = "D arsd.html2"; /// 3225 string authorization; /// 3226 3227 /* inter-request state */ 3228 private CookieHeader[][string] cookies; 3229 } 3230 3231 private ptrdiff_t lastIndexOfNothrow(T)(scope T[] arr, T value) nothrow 3232 { 3233 ptrdiff_t ret = cast(ptrdiff_t)arr.length - 1; 3234 while (ret >= 0) { 3235 if (arr[ret] == value) 3236 return ret; 3237 ret--; 3238 } 3239 return ret; 3240 } 3241 3242 interface ICache { 3243 /++ 3244 The client is about to make the given `request`. It will ALWAYS pass it to the cache object first so you can decide if you want to and can provide a response. You should probably check the appropriate headers to see if you should even attempt to look up on the cache (HttpClient does NOT do this to give maximum flexibility to the cache implementor). 3245 3246 Return null if the cache does not provide. 3247 +/ 3248 const(HttpResponse)* getCachedResponse(HttpRequestParameters request); 3249 3250 /++ 3251 The given request has received the given response. The implementing class needs to decide if it wants to cache or not. Return true if it was added, false if you chose not to. 3252 3253 You may wish to examine headers, etc., in making the decision. The HttpClient will ALWAYS pass a request/response to this. 3254 +/ 3255 bool cacheResponse(HttpRequestParameters request, HttpResponse response); 3256 } 3257 3258 /+ 3259 // / Provides caching behavior similar to a real web browser 3260 class HttpCache : ICache { 3261 const(HttpResponse)* getCachedResponse(HttpRequestParameters request) { 3262 return null; 3263 } 3264 } 3265 3266 // / Gives simple maximum age caching, ignoring the actual http headers 3267 class SimpleCache : ICache { 3268 const(HttpResponse)* getCachedResponse(HttpRequestParameters request) { 3269 return null; 3270 } 3271 } 3272 +/ 3273 3274 /++ 3275 A pseudo-cache to provide a mock server. Construct one of these, 3276 populate it with test responses, and pass it to [HttpClient] to 3277 do a network-free test. 3278 3279 You should populate it with the [populate] method. Any request not 3280 pre-populated will return a "server refused connection" response. 3281 +/ 3282 class HttpMockProvider : ICache { 3283 /+ + 3284 3285 +/ 3286 version(none) 3287 this(Uri baseUrl, string defaultResponseContentType) { 3288 3289 } 3290 3291 this() {} 3292 3293 HttpResponse defaultResponse; 3294 3295 /// Implementation of the ICache interface. Hijacks all requests to return a pre-populated response or "server disconnected". 3296 const(HttpResponse)* getCachedResponse(HttpRequestParameters request) { 3297 import std.conv; 3298 auto defaultPort = request.ssl ? 443 : 80; 3299 string identifier = text( 3300 request.method, " ", 3301 request.ssl ? "https" : "http", "://", 3302 request.host, 3303 (request.port && request.port != defaultPort) ? (":" ~ to!string(request.port)) : "", 3304 request.uri 3305 ); 3306 3307 if(auto res = identifier in population) 3308 return res; 3309 return &defaultResponse; 3310 } 3311 3312 /// Implementation of the ICache interface. We never actually cache anything here since it is all about mock responses, not actually caching real data. 3313 bool cacheResponse(HttpRequestParameters request, HttpResponse response) { 3314 return false; 3315 } 3316 3317 /++ 3318 Convenience method to populate simple responses. For more complex 3319 work, use one of the other overloads where you build complete objects 3320 yourself. 3321 3322 Params: 3323 request = a verb and complete URL to mock as one string. 3324 For example "GET http://example.com/". If you provide only 3325 a partial URL, it will be based on the `baseUrl` you gave 3326 in the `HttpMockProvider` constructor. 3327 3328 responseCode = the HTTP response code, like 200 or 404. 3329 3330 response = the response body as a string. It is assumed 3331 to be of the `defaultResponseContentType` you passed in the 3332 `HttpMockProvider` constructor. 3333 +/ 3334 void populate(string request, int responseCode, string response) { 3335 3336 // FIXME: absolute-ize the URL in the request 3337 3338 HttpResponse r; 3339 r.code = responseCode; 3340 r.codeText = getHttpCodeText(r.code); 3341 3342 r.content = cast(ubyte[]) response; 3343 r.contentText = response; 3344 3345 population[request] = r; 3346 } 3347 3348 version(none) 3349 void populate(string method, string url, HttpResponse response) { 3350 // FIXME 3351 } 3352 3353 private HttpResponse[string] population; 3354 } 3355 3356 // modified from the one in cgi.d to just have the text 3357 private static string getHttpCodeText(int code) pure nothrow @nogc { 3358 switch(code) { 3359 // this module's proprietary extensions 3360 case 0: return null; 3361 case 1: return "request.abort called"; 3362 case 2: return "connection failed"; 3363 case 3: return "server disconnected"; 3364 case 4: return "exception thrown"; // actually should be some other thing 3365 case 5: return "Request timed out"; 3366 3367 // * * * standard ones * * * 3368 3369 // 1xx skipped since they shouldn't happen 3370 3371 // 3372 case 200: return "OK"; 3373 case 201: return "Created"; 3374 case 202: return "Accepted"; 3375 case 203: return "Non-Authoritative Information"; 3376 case 204: return "No Content"; 3377 case 205: return "Reset Content"; 3378 // 3379 case 300: return "Multiple Choices"; 3380 case 301: return "Moved Permanently"; 3381 case 302: return "Found"; 3382 case 303: return "See Other"; 3383 case 307: return "Temporary Redirect"; 3384 case 308: return "Permanent Redirect"; 3385 // 3386 case 400: return "Bad Request"; 3387 case 403: return "Forbidden"; 3388 case 404: return "Not Found"; 3389 case 405: return "Method Not Allowed"; 3390 case 406: return "Not Acceptable"; 3391 case 409: return "Conflict"; 3392 case 410: return "Gone"; 3393 // 3394 case 500: return "Internal Server Error"; 3395 case 501: return "Not Implemented"; 3396 case 502: return "Bad Gateway"; 3397 case 503: return "Service Unavailable"; 3398 // 3399 default: assert(0, "Unsupported http code"); 3400 } 3401 } 3402 3403 3404 /// 3405 struct HttpCookie { 3406 string name; /// 3407 string value; /// 3408 string domain; /// 3409 string path; /// 3410 //SysTime expirationDate; /// 3411 bool secure; /// 3412 bool httpOnly; /// 3413 } 3414 3415 // FIXME: websocket 3416 3417 version(testing) 3418 void main() { 3419 import std.stdio; 3420 auto client = new HttpClient(); 3421 auto request = client.navigateTo(Uri("http://localhost/chunked.php")); 3422 request.send(); 3423 auto request2 = client.navigateTo(Uri("http://dlang.org/")); 3424 request2.send(); 3425 3426 { 3427 auto response = request2.waitForCompletion(); 3428 //write(cast(string) response.content); 3429 } 3430 3431 auto response = request.waitForCompletion(); 3432 write(cast(string) response.content); 3433 3434 writeln(HttpRequest.socketsPerHost); 3435 } 3436 3437 3438 // From sslsocket.d, but this is the maintained version! 3439 version(use_openssl) { 3440 alias SslClientSocket = OpenSslSocket; 3441 3442 // CRL = Certificate Revocation List 3443 static immutable string[] sslErrorCodes = [ 3444 "OK (code 0)", 3445 "Unspecified SSL/TLS error (code 1)", 3446 "Unable to get TLS issuer certificate (code 2)", 3447 "Unable to get TLS CRL (code 3)", 3448 "Unable to decrypt TLS certificate signature (code 4)", 3449 "Unable to decrypt TLS CRL signature (code 5)", 3450 "Unable to decode TLS issuer public key (code 6)", 3451 "TLS certificate signature failure (code 7)", 3452 "TLS CRL signature failure (code 8)", 3453 "TLS certificate not yet valid (code 9)", 3454 "TLS certificate expired (code 10)", 3455 "TLS CRL not yet valid (code 11)", 3456 "TLS CRL expired (code 12)", 3457 "TLS error in certificate not before field (code 13)", 3458 "TLS error in certificate not after field (code 14)", 3459 "TLS error in CRL last update field (code 15)", 3460 "TLS error in CRL next update field (code 16)", 3461 "TLS system out of memory (code 17)", 3462 "TLS certificate is self-signed (code 18)", 3463 "Self-signed certificate in TLS chain (code 19)", 3464 "Unable to get TLS issuer certificate locally (code 20)", 3465 "Unable to verify TLS leaf signature (code 21)", 3466 "TLS certificate chain too long (code 22)", 3467 "TLS certificate was revoked (code 23)", 3468 "TLS CA is invalid (code 24)", 3469 "TLS error: path length exceeded (code 25)", 3470 "TLS error: invalid purpose (code 26)", 3471 "TLS error: certificate untrusted (code 27)", 3472 "TLS error: certificate rejected (code 28)", 3473 ]; 3474 3475 string getOpenSslErrorCode(long error) { 3476 if(error == 62) 3477 return "TLS certificate host name mismatch"; 3478 3479 if(error < 0 || error >= sslErrorCodes.length) 3480 return "SSL/TLS error code " ~ to!string(error); 3481 return sslErrorCodes[cast(size_t) error]; 3482 } 3483 3484 struct SSL; 3485 struct SSL_CTX; 3486 struct SSL_METHOD; 3487 struct X509_STORE_CTX; 3488 enum SSL_VERIFY_NONE = 0; 3489 enum SSL_VERIFY_PEER = 1; 3490 3491 // copy it into the buf[0 .. size] and return actual length you read. 3492 // rwflag == 0 when reading, 1 when writing. 3493 extern(C) alias pem_password_cb = int function(char* buffer, int bufferSize, int rwflag, void* userPointer); 3494 extern(C) alias print_errors_cb = int function(const char*, size_t, void*); 3495 extern(C) alias client_cert_cb = int function(SSL *ssl, X509 **x509, EVP_PKEY **pkey); 3496 extern(C) alias keylog_cb = void function(SSL*, char*); 3497 3498 struct X509; 3499 struct X509_STORE; 3500 struct EVP_PKEY; 3501 struct X509_VERIFY_PARAM; 3502 3503 import core.stdc.config; 3504 3505 enum SSL_ERROR_WANT_READ = 2; 3506 enum SSL_ERROR_WANT_WRITE = 3; 3507 3508 struct ossllib { 3509 __gshared static extern(C) { 3510 /* these are only on older openssl versions { */ 3511 int function() SSL_library_init; 3512 void function() SSL_load_error_strings; 3513 SSL_METHOD* function() SSLv23_client_method; 3514 /* } */ 3515 3516 void function(ulong, void*) OPENSSL_init_ssl; 3517 3518 SSL_CTX* function(const SSL_METHOD*) SSL_CTX_new; 3519 SSL* function(SSL_CTX*) SSL_new; 3520 int function(SSL*, int) SSL_set_fd; 3521 int function(SSL*) SSL_connect; 3522 int function(SSL*, const void*, int) SSL_write; 3523 int function(SSL*, void*, int) SSL_read; 3524 @trusted nothrow @nogc int function(SSL*) SSL_shutdown; 3525 void function(SSL*) SSL_free; 3526 void function(SSL_CTX*) SSL_CTX_free; 3527 3528 int function(const SSL*) SSL_pending; 3529 int function (const SSL *ssl, int ret) SSL_get_error; 3530 3531 void function(SSL*, int, void*) SSL_set_verify; 3532 3533 void function(SSL*, int, c_long, void*) SSL_ctrl; 3534 3535 SSL_METHOD* function() SSLv3_client_method; 3536 SSL_METHOD* function() TLS_client_method; 3537 3538 void function(SSL_CTX*, void function(SSL*, char* line)) SSL_CTX_set_keylog_callback; 3539 3540 int function(SSL_CTX*) SSL_CTX_set_default_verify_paths; 3541 3542 X509_STORE* function(SSL_CTX*) SSL_CTX_get_cert_store; 3543 c_long function(const SSL* ssl) SSL_get_verify_result; 3544 3545 X509_VERIFY_PARAM* function(const SSL*) SSL_get0_param; 3546 3547 /+ 3548 SSL_CTX_load_verify_locations 3549 SSL_CTX_set_client_CA_list 3550 +/ 3551 3552 // client cert things 3553 void function (SSL_CTX *ctx, int function(SSL *ssl, X509 **x509, EVP_PKEY **pkey)) SSL_CTX_set_client_cert_cb; 3554 } 3555 } 3556 3557 struct eallib { 3558 __gshared static extern(C) { 3559 /* these are only on older openssl versions { */ 3560 void function() OpenSSL_add_all_ciphers; 3561 void function() OpenSSL_add_all_digests; 3562 /* } */ 3563 3564 const(char)* function(int) OpenSSL_version; 3565 3566 void function(ulong, void*) OPENSSL_init_crypto; 3567 3568 void function(print_errors_cb, void*) ERR_print_errors_cb; 3569 3570 void function(X509*) X509_free; 3571 int function(X509_STORE*, X509*) X509_STORE_add_cert; 3572 3573 3574 X509* function(FILE *fp, X509 **x, pem_password_cb *cb, void *u) PEM_read_X509; 3575 EVP_PKEY* function(FILE *fp, EVP_PKEY **x, pem_password_cb *cb, void* userPointer) PEM_read_PrivateKey; 3576 3577 EVP_PKEY* function(FILE *fp, EVP_PKEY **a) d2i_PrivateKey_fp; 3578 X509* function(FILE *fp, X509 **x) d2i_X509_fp; 3579 3580 X509* function(X509** a, const(ubyte*)* pp, c_long length) d2i_X509; 3581 int function(X509* a, ubyte** o) i2d_X509; 3582 3583 int function(X509_VERIFY_PARAM* a, const char* b, size_t l) X509_VERIFY_PARAM_set1_host; 3584 3585 X509* function(X509_STORE_CTX *ctx) X509_STORE_CTX_get_current_cert; 3586 int function(X509_STORE_CTX *ctx) X509_STORE_CTX_get_error; 3587 } 3588 } 3589 3590 struct OpenSSL { 3591 static: 3592 3593 template opDispatch(string name) { 3594 auto opDispatch(T...)(T t) { 3595 static if(__traits(hasMember, ossllib, name)) { 3596 auto ptr = __traits(getMember, ossllib, name); 3597 } else static if(__traits(hasMember, eallib, name)) { 3598 auto ptr = __traits(getMember, eallib, name); 3599 } else static assert(0); 3600 3601 if(ptr is null) 3602 throw new Exception(name ~ " not loaded"); 3603 return ptr(t); 3604 } 3605 } 3606 3607 // macros in the original C 3608 SSL_METHOD* SSLv23_client_method() { 3609 if(ossllib.SSLv23_client_method) 3610 return ossllib.SSLv23_client_method(); 3611 else 3612 return ossllib.TLS_client_method(); 3613 } 3614 3615 void SSL_set_tlsext_host_name(SSL* a, const char* b) { 3616 if(ossllib.SSL_ctrl) 3617 return ossllib.SSL_ctrl(a, 55 /*SSL_CTRL_SET_TLSEXT_HOSTNAME*/, 0 /*TLSEXT_NAMETYPE_host_name*/, cast(void*) b); 3618 else throw new Exception("SSL_set_tlsext_host_name not loaded"); 3619 } 3620 3621 // special case 3622 @trusted nothrow @nogc int SSL_shutdown(SSL* a) { 3623 if(ossllib.SSL_shutdown) 3624 return ossllib.SSL_shutdown(a); 3625 assert(0); 3626 } 3627 3628 void SSL_CTX_keylog_cb_func(SSL_CTX* ctx, keylog_cb func) { 3629 // this isn't in openssl 1.0 and is non-essential, so it is allowed to fail. 3630 if(ossllib.SSL_CTX_set_keylog_callback) 3631 ossllib.SSL_CTX_set_keylog_callback(ctx, func); 3632 //else throw new Exception("SSL_CTX_keylog_cb_func not loaded"); 3633 } 3634 3635 } 3636 3637 extern(C) 3638 int collectSslErrors(const char* ptr, size_t len, void* user) @trusted { 3639 string* s = cast(string*) user; 3640 3641 (*s) ~= ptr[0 .. len]; 3642 3643 return 0; 3644 } 3645 3646 3647 private __gshared void* ossllib_handle; 3648 version(Windows) 3649 private __gshared void* oeaylib_handle; 3650 else 3651 alias oeaylib_handle = ossllib_handle; 3652 version(Posix) 3653 private import core.sys.posix.dlfcn; 3654 else version(Windows) 3655 private import core.sys.windows.windows; 3656 3657 import core.stdc.stdio; 3658 3659 private __gshared Object loadSslMutex = new Object; 3660 private __gshared bool sslLoaded = false; 3661 3662 void loadOpenSsl() { 3663 if(sslLoaded) 3664 return; 3665 synchronized(loadSslMutex) { 3666 3667 version(Posix) { 3668 version(OSX) { 3669 static immutable string[] ossllibs = [ 3670 "libssl.46.dylib", 3671 "libssl.44.dylib", 3672 "libssl.43.dylib", 3673 "libssl.35.dylib", 3674 "libssl.1.1.dylib", 3675 "libssl.dylib", 3676 "/usr/local/opt/openssl/lib/libssl.1.0.0.dylib", 3677 ]; 3678 } else { 3679 static immutable string[] ossllibs = [ 3680 "libssl.so.3", 3681 "libssl.so.1.1", 3682 "libssl.so.1.0.2", 3683 "libssl.so.1.0.1", 3684 "libssl.so.1.0.0", 3685 "libssl.so", 3686 ]; 3687 } 3688 3689 foreach(lib; ossllibs) { 3690 ossllib_handle = dlopen(lib.ptr, RTLD_NOW); 3691 if(ossllib_handle !is null) break; 3692 } 3693 } else version(Windows) { 3694 version(X86_64) { 3695 ossllib_handle = LoadLibraryW("libssl-1_1-x64.dll"w.ptr); 3696 oeaylib_handle = LoadLibraryW("libcrypto-1_1-x64.dll"w.ptr); 3697 } 3698 3699 static immutable wstring[] ossllibs = [ 3700 "libssl-3-x64.dll"w, 3701 "libssl-3.dll"w, 3702 "libssl-1_1.dll"w, 3703 "libssl32.dll"w, 3704 ]; 3705 3706 if(ossllib_handle is null) 3707 foreach(lib; ossllibs) { 3708 ossllib_handle = LoadLibraryW(lib.ptr); 3709 if(ossllib_handle !is null) break; 3710 } 3711 3712 static immutable wstring[] eaylibs = [ 3713 "libcrypto-3-x64.dll"w, 3714 "libcrypto-3.dll"w, 3715 "libcrypto-1_1.dll"w, 3716 "libeay32.dll", 3717 ]; 3718 3719 if(oeaylib_handle is null) 3720 foreach(lib; eaylibs) { 3721 oeaylib_handle = LoadLibraryW(lib.ptr); 3722 if (oeaylib_handle !is null) break; 3723 } 3724 3725 if(ossllib_handle is null) { 3726 ossllib_handle = LoadLibraryW("ssleay32.dll"w.ptr); 3727 oeaylib_handle = ossllib_handle; 3728 } 3729 } 3730 3731 if(ossllib_handle is null) 3732 throw new Exception("libssl library not found"); 3733 if(oeaylib_handle is null) 3734 throw new Exception("libeay32 library not found"); 3735 3736 foreach(memberName; __traits(allMembers, ossllib)) { 3737 alias t = typeof(__traits(getMember, ossllib, memberName)); 3738 version(Posix) 3739 __traits(getMember, ossllib, memberName) = cast(t) dlsym(ossllib_handle, memberName); 3740 else version(Windows) { 3741 __traits(getMember, ossllib, memberName) = cast(t) GetProcAddress(ossllib_handle, memberName); 3742 } 3743 } 3744 3745 foreach(memberName; __traits(allMembers, eallib)) { 3746 alias t = typeof(__traits(getMember, eallib, memberName)); 3747 version(Posix) 3748 __traits(getMember, eallib, memberName) = cast(t) dlsym(oeaylib_handle, memberName); 3749 else version(Windows) { 3750 __traits(getMember, eallib, memberName) = cast(t) GetProcAddress(oeaylib_handle, memberName); 3751 } 3752 } 3753 3754 3755 if(ossllib.SSL_library_init) 3756 ossllib.SSL_library_init(); 3757 else if(ossllib.OPENSSL_init_ssl) 3758 ossllib.OPENSSL_init_ssl(0, null); 3759 else throw new Exception("couldn't init openssl"); 3760 3761 if(eallib.OpenSSL_add_all_ciphers) { 3762 eallib.OpenSSL_add_all_ciphers(); 3763 if(eallib.OpenSSL_add_all_digests is null) 3764 throw new Exception("no add digests"); 3765 eallib.OpenSSL_add_all_digests(); 3766 } else if(eallib.OPENSSL_init_crypto) 3767 eallib.OPENSSL_init_crypto(0 /*OPENSSL_INIT_ADD_ALL_CIPHERS and ALL_DIGESTS together*/, null); 3768 else throw new Exception("couldn't init crypto openssl"); 3769 3770 if(ossllib.SSL_load_error_strings) 3771 ossllib.SSL_load_error_strings(); 3772 else if(ossllib.OPENSSL_init_ssl) 3773 ossllib.OPENSSL_init_ssl(0x00200000L, null); 3774 else throw new Exception("couldn't load openssl errors"); 3775 3776 sslLoaded = true; 3777 } 3778 } 3779 3780 /+ 3781 // I'm just gonna let the OS clean this up on process termination because otherwise SSL_free 3782 // might have trouble being run from the GC after this module is unloaded. 3783 shared static ~this() { 3784 if(ossllib_handle) { 3785 version(Windows) { 3786 FreeLibrary(oeaylib_handle); 3787 FreeLibrary(ossllib_handle); 3788 } else version(Posix) 3789 dlclose(ossllib_handle); 3790 ossllib_handle = null; 3791 } 3792 ossllib.tupleof = ossllib.tupleof.init; 3793 } 3794 +/ 3795 3796 //pragma(lib, "crypto"); 3797 //pragma(lib, "ssl"); 3798 extern(C) 3799 void write_to_file(SSL* ssl, char* line) 3800 { 3801 import std.stdio; 3802 import std.string; 3803 import std.process : environment; 3804 string logfile = environment.get("SSLKEYLOGFILE"); 3805 if (logfile !is null) 3806 { 3807 auto f = std.stdio.File(logfile, "a+"); 3808 f.writeln(fromStringz(line)); 3809 f.close(); 3810 } 3811 } 3812 3813 class OpenSslSocket : Socket { 3814 private SSL* ssl; 3815 private SSL_CTX* ctx; 3816 private void initSsl(bool verifyPeer, string hostname) { 3817 ctx = OpenSSL.SSL_CTX_new(OpenSSL.SSLv23_client_method()); 3818 assert(ctx !is null); 3819 3820 debug OpenSSL.SSL_CTX_keylog_cb_func(ctx, &write_to_file); 3821 ssl = OpenSSL.SSL_new(ctx); 3822 3823 if(hostname.length) { 3824 OpenSSL.SSL_set_tlsext_host_name(ssl, toStringz(hostname)); 3825 if(verifyPeer) 3826 OpenSSL.X509_VERIFY_PARAM_set1_host(OpenSSL.SSL_get0_param(ssl), hostname.ptr, hostname.length); 3827 } 3828 3829 if(verifyPeer) { 3830 OpenSSL.SSL_CTX_set_default_verify_paths(ctx); 3831 3832 version(Windows) { 3833 loadCertificatesFromRegistry(ctx); 3834 } 3835 3836 OpenSSL.SSL_set_verify(ssl, SSL_VERIFY_PEER, &verifyCertificateFromRegistryArsdHttp); 3837 } else 3838 OpenSSL.SSL_set_verify(ssl, SSL_VERIFY_NONE, null); 3839 3840 OpenSSL.SSL_set_fd(ssl, cast(int) this.handle); // on win64 it is necessary to truncate, but the value is never large anyway see http://openssl.6102.n7.nabble.com/Sockets-windows-64-bit-td36169.html 3841 3842 3843 OpenSSL.SSL_CTX_set_client_cert_cb(ctx, &cb); 3844 } 3845 3846 extern(C) 3847 static int cb(SSL* ssl, X509** x509, EVP_PKEY** pkey) { 3848 if(HttpClient.certFilename.length && HttpClient.keyFilename.length) { 3849 FILE* fpCert = fopen((HttpClient.certFilename ~ "\0").ptr, "rb"); 3850 if(fpCert is null) 3851 return 0; 3852 scope(exit) 3853 fclose(fpCert); 3854 FILE* fpKey = fopen((HttpClient.keyFilename ~ "\0").ptr, "rb"); 3855 if(fpKey is null) 3856 return 0; 3857 scope(exit) 3858 fclose(fpKey); 3859 3860 with(CertificateFileFormat) 3861 final switch(HttpClient.certFormat) { 3862 case guess: 3863 if(HttpClient.certFilename.endsWith(".pem") || HttpClient.keyFilename.endsWith(".pem")) 3864 goto case pem; 3865 else 3866 goto case der; 3867 case pem: 3868 *x509 = OpenSSL.PEM_read_X509(fpCert, null, null, null); 3869 *pkey = OpenSSL.PEM_read_PrivateKey(fpKey, null, null, null); 3870 break; 3871 case der: 3872 *x509 = OpenSSL.d2i_X509_fp(fpCert, null); 3873 *pkey = OpenSSL.d2i_PrivateKey_fp(fpKey, null); 3874 break; 3875 } 3876 3877 return 1; 3878 } 3879 3880 return 0; 3881 } 3882 3883 bool dataPending() { 3884 return OpenSSL.SSL_pending(ssl) > 0; 3885 } 3886 3887 @trusted 3888 override void connect(Address to) { 3889 super.connect(to); 3890 if(blocking) { 3891 do_ssl_connect(); 3892 } 3893 } 3894 3895 @trusted 3896 // returns true if it is finished, false if it would have blocked, throws if there's an error 3897 int do_ssl_connect() { 3898 if(OpenSSL.SSL_connect(ssl) == -1) { 3899 3900 auto errCode = OpenSSL.SSL_get_error(ssl, -1); 3901 if(errCode == SSL_ERROR_WANT_READ || errCode == SSL_ERROR_WANT_WRITE) { 3902 return errCode; 3903 } 3904 3905 string str; 3906 OpenSSL.ERR_print_errors_cb(&collectSslErrors, &str); 3907 int i; 3908 auto err = OpenSSL.SSL_get_verify_result(ssl); 3909 //printf("wtf\n"); 3910 //scanf("%d\n", i); 3911 throw new Exception("Secure connect failed: " ~ getOpenSslErrorCode(err)); 3912 } 3913 3914 return 0; 3915 } 3916 3917 @trusted 3918 override ptrdiff_t send(scope const(void)[] buf, SocketFlags flags) { 3919 //import std.stdio;writeln(cast(string) buf); 3920 debug(arsd_http2_verbose) writeln("ssl writing ", buf.length); 3921 auto retval = OpenSSL.SSL_write(ssl, buf.ptr, cast(uint) buf.length); 3922 3923 // don't need to throw anymore since it is checked elsewhere 3924 // code useful sometimes for debugging hence commenting instead of deleting 3925 version(none) 3926 if(retval == -1) { 3927 3928 string str; 3929 OpenSSL.ERR_print_errors_cb(&collectSslErrors, &str); 3930 int i; 3931 3932 //printf("wtf\n"); 3933 //scanf("%d\n", i); 3934 3935 throw new Exception("ssl send failed " ~ str); 3936 } 3937 return retval; 3938 3939 } 3940 override ptrdiff_t send(scope const(void)[] buf) { 3941 return send(buf, SocketFlags.NONE); 3942 } 3943 @trusted 3944 override ptrdiff_t receive(scope void[] buf, SocketFlags flags) { 3945 3946 debug(arsd_http2_verbose) writeln("ssl_read before"); 3947 auto retval = OpenSSL.SSL_read(ssl, buf.ptr, cast(int)buf.length); 3948 debug(arsd_http2_verbose) writeln("ssl_read after"); 3949 3950 // don't need to throw anymore since it is checked elsewhere 3951 // code useful sometimes for debugging hence commenting instead of deleting 3952 version(none) 3953 if(retval == -1) { 3954 3955 string str; 3956 OpenSSL.ERR_print_errors_cb(&collectSslErrors, &str); 3957 int i; 3958 3959 //printf("wtf\n"); 3960 //scanf("%d\n", i); 3961 3962 throw new Exception("ssl receive failed " ~ str); 3963 } 3964 return retval; 3965 } 3966 override ptrdiff_t receive(scope void[] buf) { 3967 return receive(buf, SocketFlags.NONE); 3968 } 3969 3970 this(AddressFamily af, SocketType type = SocketType.STREAM, string hostname = null, bool verifyPeer = true) { 3971 super(af, type); 3972 initSsl(verifyPeer, hostname); 3973 } 3974 3975 override void close() { 3976 if(ssl) OpenSSL.SSL_shutdown(ssl); 3977 super.close(); 3978 } 3979 3980 this(socket_t sock, AddressFamily af, string hostname, bool verifyPeer = true) { 3981 super(sock, af); 3982 initSsl(verifyPeer, hostname); 3983 } 3984 3985 void freeSsl() { 3986 if(ssl is null) 3987 return; 3988 OpenSSL.SSL_free(ssl); 3989 OpenSSL.SSL_CTX_free(ctx); 3990 ssl = null; 3991 } 3992 3993 ~this() { 3994 freeSsl(); 3995 } 3996 } 3997 } 3998 3999 4000 /++ 4001 An experimental component for working with REST apis. Note that it 4002 is a zero-argument template, so to create one, use `new HttpApiClient!()(args..)` 4003 or you will get "HttpApiClient is used as a type" compile errors. 4004 4005 This will probably not work for you yet, and I might change it significantly. 4006 4007 Requires [arsd.jsvar]. 4008 4009 4010 Here's a snippet to create a pull request on GitHub to Phobos: 4011 4012 --- 4013 auto github = new HttpApiClient!()("https://api.github.com/", "your personal api token here"); 4014 4015 // create the arguments object 4016 // see: https://developer.github.com/v3/pulls/#create-a-pull-request 4017 var args = var.emptyObject; 4018 args.title = "My Pull Request"; 4019 args.head = "yourusername:" ~ branchName; 4020 args.base = "master"; 4021 // note it is ["body"] instead of .body because `body` is a D keyword 4022 args["body"] = "My cool PR is opened by the API!"; 4023 args.maintainer_can_modify = true; 4024 4025 /+ 4026 Fun fact, you can also write that: 4027 4028 var args = [ 4029 "title": "My Pull Request".var, 4030 "head": "yourusername:" ~ branchName.var, 4031 "base" : "master".var, 4032 "body" : "My cool PR is opened by the API!".var, 4033 "maintainer_can_modify": true.var 4034 ]; 4035 4036 Note the .var constructor calls in there. If everything is the same type, you actually don't need that, but here since there's strings and bools, D won't allow the literal without explicit constructors to align them all. 4037 +/ 4038 4039 // this translates to `repos/dlang/phobos/pulls` and sends a POST request, 4040 // containing `args` as json, then immediately grabs the json result and extracts 4041 // the value `html_url` from it. `prUrl` is typed `var`, from arsd.jsvar. 4042 auto prUrl = github.rest.repos.dlang.phobos.pulls.POST(args).result.html_url; 4043 4044 writeln("Created: ", prUrl); 4045 --- 4046 4047 Why use this instead of just building the URL? Well, of course you can! This just makes 4048 it a bit more convenient than string concatenation and manages a few headers for you. 4049 4050 Subtypes could potentially add static type checks too. 4051 +/ 4052 class HttpApiClient() { 4053 import arsd.jsvar; 4054 4055 HttpClient httpClient; 4056 4057 alias HttpApiClientType = typeof(this); 4058 4059 string urlBase; 4060 string oauth2Token; 4061 string submittedContentType; 4062 4063 /++ 4064 Params: 4065 4066 urlBase = The base url for the api. Tends to be something like `https://api.example.com/v2/` or similar. 4067 oauth2Token = the authorization token for the service. You'll have to get it from somewhere else. 4068 submittedContentType = the content-type of POST, PUT, etc. bodies. 4069 httpClient = an injected http client, or null if you want to use a default-constructed one 4070 4071 History: 4072 The `httpClient` param was added on December 26, 2020. 4073 +/ 4074 this(string urlBase, string oauth2Token, string submittedContentType = "application/json", HttpClient httpClient = null) { 4075 if(httpClient is null) 4076 this.httpClient = new HttpClient(); 4077 else 4078 this.httpClient = httpClient; 4079 4080 assert(urlBase[0] == 'h'); 4081 assert(urlBase[$-1] == '/'); 4082 4083 this.urlBase = urlBase; 4084 this.oauth2Token = oauth2Token; 4085 this.submittedContentType = submittedContentType; 4086 } 4087 4088 /// 4089 static struct HttpRequestWrapper { 4090 HttpApiClientType apiClient; /// 4091 HttpRequest request; /// 4092 HttpResponse _response; 4093 4094 /// 4095 this(HttpApiClientType apiClient, HttpRequest request) { 4096 this.apiClient = apiClient; 4097 this.request = request; 4098 } 4099 4100 /// Returns the full [HttpResponse] object so you can inspect the headers 4101 @property HttpResponse response() { 4102 if(_response is HttpResponse.init) 4103 _response = request.waitForCompletion(); 4104 return _response; 4105 } 4106 4107 /++ 4108 Returns the parsed JSON from the body of the response. 4109 4110 Throws on non-2xx responses. 4111 +/ 4112 var result() { 4113 return apiClient.throwOnError(response); 4114 } 4115 4116 alias request this; 4117 } 4118 4119 /// 4120 HttpRequestWrapper request(string uri, HttpVerb requestMethod = HttpVerb.GET, ubyte[] bodyBytes = null) { 4121 if(uri[0] == '/') 4122 uri = uri[1 .. $]; 4123 4124 auto u = Uri(uri).basedOn(Uri(urlBase)); 4125 4126 auto req = httpClient.navigateTo(u, requestMethod); 4127 4128 if(oauth2Token.length) 4129 req.requestParameters.headers ~= "Authorization: Bearer " ~ oauth2Token; 4130 req.requestParameters.contentType = submittedContentType; 4131 req.requestParameters.bodyData = bodyBytes; 4132 4133 return HttpRequestWrapper(this, req); 4134 } 4135 4136 /// 4137 var throwOnError(HttpResponse res) { 4138 if(res.code < 200 || res.code >= 300) 4139 throw new Exception(res.codeText ~ " " ~ res.contentText); 4140 4141 var response = var.fromJson(res.contentText); 4142 if(response.errors) { 4143 throw new Exception(response.errors.toJson()); 4144 } 4145 4146 return response; 4147 } 4148 4149 /// 4150 @property RestBuilder rest() { 4151 return RestBuilder(this, null, null); 4152 } 4153 4154 // hipchat.rest.room["Tech Team"].history 4155 // gives: "/room/Tech%20Team/history" 4156 // 4157 // hipchat.rest.room["Tech Team"].history("page", "12) 4158 /// 4159 static struct RestBuilder { 4160 HttpApiClientType apiClient; 4161 string[] pathParts; 4162 string[2][] queryParts; 4163 this(HttpApiClientType apiClient, string[] pathParts, string[2][] queryParts) { 4164 this.apiClient = apiClient; 4165 this.pathParts = pathParts; 4166 this.queryParts = queryParts; 4167 } 4168 4169 RestBuilder _SELF() { 4170 return this; 4171 } 4172 4173 /// The args are so you can call opCall on the returned 4174 /// object, despite @property being broken af in D. 4175 RestBuilder opDispatch(string str, T)(string n, T v) { 4176 return RestBuilder(apiClient, pathParts ~ str, queryParts ~ [n, to!string(v)]); 4177 } 4178 4179 /// 4180 RestBuilder opDispatch(string str)() { 4181 return RestBuilder(apiClient, pathParts ~ str, queryParts); 4182 } 4183 4184 4185 /// 4186 RestBuilder opIndex(string str) { 4187 return RestBuilder(apiClient, pathParts ~ str, queryParts); 4188 } 4189 /// 4190 RestBuilder opIndex(var str) { 4191 return RestBuilder(apiClient, pathParts ~ str.get!string, queryParts); 4192 } 4193 /// 4194 RestBuilder opIndex(int i) { 4195 return RestBuilder(apiClient, pathParts ~ to!string(i), queryParts); 4196 } 4197 4198 /// 4199 RestBuilder opCall(T)(string name, T value) { 4200 return RestBuilder(apiClient, pathParts, queryParts ~ [name, to!string(value)]); 4201 } 4202 4203 /// 4204 string toUri() { 4205 import std.uri; 4206 string result; 4207 foreach(idx, part; pathParts) { 4208 if(idx) 4209 result ~= "/"; 4210 result ~= encodeComponent(part); 4211 } 4212 result ~= "?"; 4213 foreach(idx, part; queryParts) { 4214 if(idx) 4215 result ~= "&"; 4216 result ~= encodeComponent(part[0]); 4217 result ~= "="; 4218 result ~= encodeComponent(part[1]); 4219 } 4220 4221 return result; 4222 } 4223 4224 /// 4225 final HttpRequestWrapper GET() { return _EXECUTE(HttpVerb.GET, this.toUri(), ToBytesResult.init); } 4226 /// ditto 4227 final HttpRequestWrapper DELETE() { return _EXECUTE(HttpVerb.DELETE, this.toUri(), ToBytesResult.init); } 4228 4229 // need to be able to send: JSON, urlencoded, multipart/form-data, and raw stuff. 4230 /// ditto 4231 final HttpRequestWrapper POST(T...)(T t) { return _EXECUTE(HttpVerb.POST, this.toUri(), toBytes(t)); } 4232 /// ditto 4233 final HttpRequestWrapper PATCH(T...)(T t) { return _EXECUTE(HttpVerb.PATCH, this.toUri(), toBytes(t)); } 4234 /// ditto 4235 final HttpRequestWrapper PUT(T...)(T t) { return _EXECUTE(HttpVerb.PUT, this.toUri(), toBytes(t)); } 4236 4237 struct ToBytesResult { 4238 ubyte[] bytes; 4239 string contentType; 4240 } 4241 4242 private ToBytesResult toBytes(T...)(T t) { 4243 import std.conv : to; 4244 static if(T.length == 0) 4245 return ToBytesResult(null, null); 4246 else static if(T.length == 1 && is(T[0] == var)) 4247 return ToBytesResult(cast(ubyte[]) t[0].toJson(), "application/json"); // json data 4248 else static if(T.length == 1 && (is(T[0] == string) || is(T[0] == ubyte[]))) 4249 return ToBytesResult(cast(ubyte[]) t[0], null); // raw data 4250 else static if(T.length == 1 && is(T[0] : FormData)) 4251 return ToBytesResult(t[0].toBytes, t[0].contentType); 4252 else static if(T.length > 1 && T.length % 2 == 0 && is(T[0] == string)) { 4253 // string -> value pairs for a POST request 4254 string answer; 4255 foreach(idx, val; t) { 4256 static if(idx % 2 == 0) { 4257 if(answer.length) 4258 answer ~= "&"; 4259 answer ~= encodeComponent(val); // it had better be a string! lol 4260 answer ~= "="; 4261 } else { 4262 answer ~= encodeComponent(to!string(val)); 4263 } 4264 } 4265 4266 return ToBytesResult(cast(ubyte[]) answer, "application/x-www-form-urlencoded"); 4267 } 4268 else 4269 static assert(0); // FIXME 4270 4271 } 4272 4273 HttpRequestWrapper _EXECUTE(HttpVerb verb, string uri, ubyte[] bodyBytes) { 4274 return apiClient.request(uri, verb, bodyBytes); 4275 } 4276 4277 HttpRequestWrapper _EXECUTE(HttpVerb verb, string uri, ToBytesResult tbr) { 4278 auto r = apiClient.request(uri, verb, tbr.bytes); 4279 if(tbr.contentType !is null) 4280 r.requestParameters.contentType = tbr.contentType; 4281 return r; 4282 } 4283 } 4284 } 4285 4286 4287 // see also: arsd.cgi.encodeVariables 4288 /// Creates a multipart/form-data object that is suitable for file uploads and other kinds of POST 4289 class FormData { 4290 struct MimePart { 4291 string name; 4292 const(void)[] data; 4293 string contentType; 4294 string filename; 4295 } 4296 4297 MimePart[] parts; 4298 4299 /// 4300 void append(string key, in void[] value, string contentType = null, string filename = null) { 4301 parts ~= MimePart(key, value, contentType, filename); 4302 } 4303 4304 private string boundary = "0016e64be86203dd36047610926a"; // FIXME 4305 4306 string contentType() { 4307 return "multipart/form-data; boundary=" ~ boundary; 4308 } 4309 4310 /// 4311 ubyte[] toBytes() { 4312 string data; 4313 4314 foreach(part; parts) { 4315 data ~= "--" ~ boundary ~ "\r\n"; 4316 data ~= "Content-Disposition: form-data; name=\""~part.name~"\""; 4317 if(part.filename !is null) 4318 data ~= "; filename=\""~part.filename~"\""; 4319 data ~= "\r\n"; 4320 if(part.contentType !is null) 4321 data ~= "Content-Type: " ~ part.contentType ~ "\r\n"; 4322 data ~= "\r\n"; 4323 4324 data ~= cast(string) part.data; 4325 4326 data ~= "\r\n"; 4327 } 4328 4329 data ~= "--" ~ boundary ~ "--\r\n"; 4330 4331 return cast(ubyte[]) data; 4332 } 4333 } 4334 4335 private bool bicmp(in ubyte[] item, in char[] search) { 4336 if(item.length != search.length) return false; 4337 4338 foreach(i; 0 .. item.length) { 4339 ubyte a = item[i]; 4340 ubyte b = search[i]; 4341 if(a >= 'A' && a <= 'Z') 4342 a += 32; 4343 //if(b >= 'A' && b <= 'Z') 4344 //b += 32; 4345 if(a != b) 4346 return false; 4347 } 4348 4349 return true; 4350 } 4351 4352 /++ 4353 WebSocket client, based on the browser api, though also with other api options. 4354 4355 --- 4356 import arsd.http2; 4357 4358 void main() { 4359 auto ws = new WebSocket(Uri("ws://....")); 4360 4361 ws.onmessage = (in char[] msg) { 4362 ws.send("a reply"); 4363 }; 4364 4365 ws.connect(); 4366 4367 WebSocket.eventLoop(); 4368 } 4369 --- 4370 4371 Symbol_groups: 4372 foundational = 4373 Used with all API styles. 4374 4375 browser_api = 4376 API based on the standard in the browser. 4377 4378 event_loop_integration = 4379 Integrating with external event loops is done through static functions. You should 4380 call these BEFORE doing anything else with the WebSocket module or class. 4381 4382 $(PITFALL NOT IMPLEMENTED) 4383 --- 4384 WebSocket.setEventLoopProxy(arsd.simpledisplay.EventLoop.proxy.tupleof); 4385 // or something like that. it is not implemented yet. 4386 --- 4387 $(PITFALL NOT IMPLEMENTED) 4388 4389 blocking_api = 4390 The blocking API is best used when you only need basic functionality with a single connection. 4391 4392 --- 4393 WebSocketFrame msg; 4394 do { 4395 // FIXME good demo 4396 } while(msg); 4397 --- 4398 4399 Or to check for blocks before calling: 4400 4401 --- 4402 try_to_process_more: 4403 while(ws.isMessageBuffered()) { 4404 auto msg = ws.waitForNextMessage(); 4405 // process msg 4406 } 4407 if(ws.isDataPending()) { 4408 ws.lowLevelReceive(); 4409 goto try_to_process_more; 4410 } else { 4411 // nothing ready, you can do other things 4412 // or at least sleep a while before trying 4413 // to process more. 4414 if(ws.readyState == WebSocket.OPEN) { 4415 Thread.sleep(1.seconds); 4416 goto try_to_process_more; 4417 } 4418 } 4419 --- 4420 4421 +/ 4422 class WebSocket { 4423 private Uri uri; 4424 private string[string] cookies; 4425 4426 private string host; 4427 private ushort port; 4428 private bool ssl; 4429 4430 // used to decide if we mask outgoing msgs 4431 private bool isClient; 4432 4433 private MonoTime timeoutFromInactivity; 4434 private MonoTime nextPing; 4435 4436 /++ 4437 wss://echo.websocket.org 4438 +/ 4439 /// Group: foundational 4440 this(Uri uri, Config config = Config.init) 4441 //in (uri.scheme == "ws" || uri.scheme == "wss") 4442 in { assert(uri.scheme == "ws" || uri.scheme == "wss"); } do 4443 { 4444 this.uri = uri; 4445 this.config = config; 4446 4447 this.receiveBuffer = new ubyte[](config.initialReceiveBufferSize); 4448 4449 host = uri.host; 4450 ssl = uri.scheme == "wss"; 4451 port = cast(ushort) (uri.port ? uri.port : ssl ? 443 : 80); 4452 4453 if(ssl) { 4454 version(with_openssl) { 4455 loadOpenSsl(); 4456 socket = new SslClientSocket(family(uri.unixSocketPath), SocketType.STREAM, host, config.verifyPeer); 4457 } else 4458 throw new Exception("SSL not compiled in"); 4459 } else 4460 socket = new Socket(family(uri.unixSocketPath), SocketType.STREAM); 4461 4462 } 4463 4464 /++ 4465 4466 +/ 4467 /// Group: foundational 4468 void connect() { 4469 this.isClient = true; 4470 if(uri.unixSocketPath) 4471 socket.connect(new UnixAddress(uri.unixSocketPath)); 4472 else 4473 socket.connect(new InternetAddress(host, port)); // FIXME: ipv6 support... 4474 // FIXME: websocket handshake could and really should be async too. 4475 4476 auto uri = this.uri.path.length ? this.uri.path : "/"; 4477 if(this.uri.query.length) { 4478 uri ~= "?"; 4479 uri ~= this.uri.query; 4480 } 4481 4482 // the headers really shouldn't be bigger than this, at least 4483 // the chunks i need to process 4484 ubyte[4096] bufferBacking = void; 4485 ubyte[] buffer = bufferBacking[]; 4486 size_t pos; 4487 4488 void append(in char[][] items...) { 4489 foreach(what; items) { 4490 if((pos + what.length) > buffer.length) { 4491 buffer.length += 4096; 4492 } 4493 buffer[pos .. pos + what.length] = cast(ubyte[]) what[]; 4494 pos += what.length; 4495 } 4496 } 4497 4498 append("GET ", uri, " HTTP/1.1\r\n"); 4499 append("Host: ", this.uri.host, "\r\n"); 4500 4501 append("Upgrade: websocket\r\n"); 4502 append("Connection: Upgrade\r\n"); 4503 append("Sec-WebSocket-Version: 13\r\n"); 4504 4505 // FIXME: randomize this 4506 append("Sec-WebSocket-Key: x3JEHMbDL1EzLkh9GBhXDw==\r\n"); 4507 4508 if(config.protocol.length) 4509 append("Sec-WebSocket-Protocol: ", config.protocol, "\r\n"); 4510 if(config.origin.length) 4511 append("Origin: ", config.origin, "\r\n"); 4512 4513 foreach(h; config.additionalHeaders) { 4514 append(h); 4515 append("\r\n"); 4516 } 4517 4518 append("\r\n"); 4519 4520 auto remaining = buffer[0 .. pos]; 4521 //import std.stdio; writeln(host, " " , port, " ", cast(string) remaining); 4522 while(remaining.length) { 4523 auto r = socket.send(remaining); 4524 if(r < 0) 4525 throw new Exception(lastSocketError()); 4526 if(r == 0) 4527 throw new Exception("unexpected connection termination"); 4528 remaining = remaining[r .. $]; 4529 } 4530 4531 // the response shouldn't be especially large at this point, just 4532 // headers for the most part. gonna try to get it in the stack buffer. 4533 // then copy stuff after headers, if any, to the frame buffer. 4534 ubyte[] used; 4535 4536 void more() { 4537 auto r = socket.receive(buffer[used.length .. $]); 4538 4539 if(r < 0) 4540 throw new Exception(lastSocketError()); 4541 if(r == 0) 4542 throw new Exception("unexpected connection termination"); 4543 //import std.stdio;writef("%s", cast(string) buffer[used.length .. used.length + r]); 4544 4545 used = buffer[0 .. used.length + r]; 4546 } 4547 4548 more(); 4549 4550 import std.algorithm; 4551 if(!used.startsWith(cast(ubyte[]) "HTTP/1.1 101")) 4552 throw new Exception("didn't get a websocket answer"); 4553 // skip the status line 4554 while(used.length && used[0] != '\n') 4555 used = used[1 .. $]; 4556 4557 if(used.length == 0) 4558 throw new Exception("Remote server disconnected or didn't send enough information"); 4559 4560 if(used.length < 1) 4561 more(); 4562 4563 used = used[1 .. $]; // skip the \n 4564 4565 if(used.length == 0) 4566 more(); 4567 4568 // checks on the protocol from ehaders 4569 bool isWebsocket; 4570 bool isUpgrade; 4571 const(ubyte)[] protocol; 4572 const(ubyte)[] accept; 4573 4574 while(used.length) { 4575 if(used.length >= 2 && used[0] == '\r' && used[1] == '\n') { 4576 used = used[2 .. $]; 4577 break; // all done 4578 } 4579 int idxColon; 4580 while(idxColon < used.length && used[idxColon] != ':') 4581 idxColon++; 4582 if(idxColon == used.length) 4583 more(); 4584 auto idxStart = idxColon + 1; 4585 while(idxStart < used.length && used[idxStart] == ' ') 4586 idxStart++; 4587 if(idxStart == used.length) 4588 more(); 4589 auto idxEnd = idxStart; 4590 while(idxEnd < used.length && used[idxEnd] != '\r') 4591 idxEnd++; 4592 if(idxEnd == used.length) 4593 more(); 4594 4595 auto headerName = used[0 .. idxColon]; 4596 auto headerValue = used[idxStart .. idxEnd]; 4597 4598 // move past this header 4599 used = used[idxEnd .. $]; 4600 // and the \r\n 4601 if(2 <= used.length) 4602 used = used[2 .. $]; 4603 4604 if(headerName.bicmp("upgrade")) { 4605 if(headerValue.bicmp("websocket")) 4606 isWebsocket = true; 4607 } else if(headerName.bicmp("connection")) { 4608 if(headerValue.bicmp("upgrade")) 4609 isUpgrade = true; 4610 } else if(headerName.bicmp("sec-websocket-accept")) { 4611 accept = headerValue; 4612 } else if(headerName.bicmp("sec-websocket-protocol")) { 4613 protocol = headerValue; 4614 } 4615 4616 if(!used.length) { 4617 more(); 4618 } 4619 } 4620 4621 4622 if(!isWebsocket) 4623 throw new Exception("didn't answer as websocket"); 4624 if(!isUpgrade) 4625 throw new Exception("didn't answer as upgrade"); 4626 4627 4628 // FIXME: check protocol if config requested one 4629 // FIXME: check accept for the right hash 4630 4631 receiveBuffer[0 .. used.length] = used[]; 4632 receiveBufferUsedLength = used.length; 4633 4634 readyState_ = OPEN; 4635 4636 if(onopen) 4637 onopen(); 4638 4639 nextPing = MonoTime.currTime + config.pingFrequency.msecs; 4640 timeoutFromInactivity = MonoTime.currTime + config.timeoutFromInactivity; 4641 4642 registerActiveSocket(this); 4643 } 4644 4645 /++ 4646 Is data pending on the socket? Also check [isMessageBuffered] to see if there 4647 is already a message in memory too. 4648 4649 If this returns `true`, you can call [lowLevelReceive], then try [isMessageBuffered] 4650 again. 4651 +/ 4652 /// Group: blocking_api 4653 public bool isDataPending(Duration timeout = 0.seconds) { 4654 static SocketSet readSet; 4655 if(readSet is null) 4656 readSet = new SocketSet(); 4657 4658 version(with_openssl) 4659 if(auto s = cast(SslClientSocket) socket) { 4660 // select doesn't handle the case with stuff 4661 // left in the ssl buffer so i'm checking it separately 4662 if(s.dataPending()) { 4663 return true; 4664 } 4665 } 4666 4667 readSet.add(socket); 4668 4669 //tryAgain: 4670 auto selectGot = Socket.select(readSet, null, null, timeout); 4671 if(selectGot == 0) { /* timeout */ 4672 // timeout 4673 return false; 4674 } else if(selectGot == -1) { /* interrupted */ 4675 return false; 4676 } else { /* ready */ 4677 if(readSet.isSet(socket)) { 4678 return true; 4679 } 4680 } 4681 4682 return false; 4683 } 4684 4685 private void llsend(ubyte[] d) { 4686 if(readyState == CONNECTING) 4687 throw new Exception("WebSocket not connected when trying to send. Did you forget to call connect(); ?"); 4688 //connect(); 4689 //import std.stdio; writeln("LLSEND: ", d); 4690 while(d.length) { 4691 auto r = socket.send(d); 4692 if(r < 0 && wouldHaveBlocked()) { 4693 import core.thread; 4694 Thread.sleep(1.msecs); 4695 continue; 4696 } 4697 //import core.stdc.errno; import std.stdio; writeln(errno); 4698 if(r <= 0) { 4699 // import std.stdio; writeln(GetLastError()); 4700 throw new Exception("Socket send failed"); 4701 } 4702 d = d[r .. $]; 4703 } 4704 } 4705 4706 private void llclose() { 4707 // import std.stdio; writeln("LLCLOSE"); 4708 socket.shutdown(SocketShutdown.SEND); 4709 } 4710 4711 /++ 4712 Waits for more data off the low-level socket and adds it to the pending buffer. 4713 4714 Returns `true` if the connection is still active. 4715 +/ 4716 /// Group: blocking_api 4717 public bool lowLevelReceive() { 4718 if(readyState == CONNECTING) 4719 throw new Exception("WebSocket not connected when trying to receive. Did you forget to call connect(); ?"); 4720 if (receiveBufferUsedLength == receiveBuffer.length) 4721 { 4722 if (receiveBuffer.length == config.maximumReceiveBufferSize) 4723 throw new Exception("Maximum receive buffer size exhausted"); 4724 4725 import std.algorithm : min; 4726 receiveBuffer.length = min(receiveBuffer.length + config.initialReceiveBufferSize, 4727 config.maximumReceiveBufferSize); 4728 } 4729 auto r = socket.receive(receiveBuffer[receiveBufferUsedLength .. $]); 4730 if(r == 0) 4731 return false; 4732 if(r < 0 && wouldHaveBlocked()) 4733 return true; 4734 if(r <= 0) { 4735 //import std.stdio; writeln(WSAGetLastError()); 4736 throw new Exception("Socket receive failed"); 4737 } 4738 receiveBufferUsedLength += r; 4739 return true; 4740 } 4741 4742 private Socket socket; 4743 4744 /* copy/paste section { */ 4745 4746 private int readyState_; 4747 private ubyte[] receiveBuffer; 4748 private size_t receiveBufferUsedLength; 4749 4750 private Config config; 4751 4752 enum CONNECTING = 0; /// Socket has been created. The connection is not yet open. 4753 enum OPEN = 1; /// The connection is open and ready to communicate. 4754 enum CLOSING = 2; /// The connection is in the process of closing. 4755 enum CLOSED = 3; /// The connection is closed or couldn't be opened. 4756 4757 /++ 4758 4759 +/ 4760 /// Group: foundational 4761 static struct Config { 4762 /++ 4763 These control the size of the receive buffer. 4764 4765 It starts at the initial size, will temporarily 4766 balloon up to the maximum size, and will reuse 4767 a buffer up to the likely size. 4768 4769 Anything larger than the maximum size will cause 4770 the connection to be aborted and an exception thrown. 4771 This is to protect you against a peer trying to 4772 exhaust your memory, while keeping the user-level 4773 processing simple. 4774 +/ 4775 size_t initialReceiveBufferSize = 4096; 4776 size_t likelyReceiveBufferSize = 4096; /// ditto 4777 size_t maximumReceiveBufferSize = 10 * 1024 * 1024; /// ditto 4778 4779 /++ 4780 Maximum combined size of a message. 4781 +/ 4782 size_t maximumMessageSize = 10 * 1024 * 1024; 4783 4784 string[string] cookies; /// Cookies to send with the initial request. cookies[name] = value; 4785 string origin; /// Origin URL to send with the handshake, if desired. 4786 string protocol; /// the protocol header, if desired. 4787 4788 /++ 4789 Additional headers to put in the HTTP request. These should be formatted `Name: value`, like for example: 4790 4791 --- 4792 Config config; 4793 config.additionalHeaders ~= "Authorization: Bearer your_auth_token_here"; 4794 --- 4795 4796 History: 4797 Added February 19, 2021 (included in dub version 9.2) 4798 +/ 4799 string[] additionalHeaders; 4800 4801 int pingFrequency = 5000; /// Amount of time (in msecs) of idleness after which to send an automatic ping 4802 4803 /++ 4804 Amount of time to disconnect when there's no activity. Note that automatic pings will keep the connection alive; this timeout only occurs if there's absolutely nothing, including no responses to websocket ping frames. Since the default [pingFrequency] is only seconds, this one minute should never elapse unless the connection is actually dead. 4805 4806 The one thing to keep in mind is if your program is busy and doesn't check input, it might consider this a time out since there's no activity. The reason is that your program was busy rather than a connection failure, but it doesn't care. You should avoid long processing periods anyway though! 4807 4808 History: 4809 Added March 31, 2021 (included in dub version 9.4) 4810 +/ 4811 Duration timeoutFromInactivity = 1.minutes; 4812 4813 /++ 4814 For https connections, if this is `true`, it will fail to connect if the TLS certificate can not be 4815 verified. Setting this to `false` will skip this check and allow the connection to continue anyway. 4816 4817 History: 4818 Added April 5, 2022 (dub v10.8) 4819 4820 Prior to this, it always used the global (but undocumented) `defaultVerifyPeer` setting, and sometimes 4821 even if it was true, it would skip the verification. Now, it always respects this local setting. 4822 +/ 4823 bool verifyPeer = true; 4824 } 4825 4826 /++ 4827 Returns one of [CONNECTING], [OPEN], [CLOSING], or [CLOSED]. 4828 +/ 4829 int readyState() { 4830 return readyState_; 4831 } 4832 4833 /++ 4834 Closes the connection, sending a graceful teardown message to the other side. 4835 +/ 4836 /// Group: foundational 4837 void close(int code = 0, string reason = null) 4838 //in (reason.length < 123) 4839 in { assert(reason.length < 123); } do 4840 { 4841 if(readyState_ != OPEN) 4842 return; // it cool, we done 4843 WebSocketFrame wss; 4844 wss.fin = true; 4845 wss.masked = this.isClient; 4846 wss.opcode = WebSocketOpcode.close; 4847 wss.data = cast(ubyte[]) reason.dup; 4848 wss.send(&llsend); 4849 4850 readyState_ = CLOSING; 4851 4852 llclose(); 4853 } 4854 4855 /++ 4856 Sends a ping message to the server. This is done automatically by the library if you set a non-zero [Config.pingFrequency], but you can also send extra pings explicitly as well with this function. 4857 +/ 4858 /// Group: foundational 4859 void ping(in ubyte[] data = null) { 4860 WebSocketFrame wss; 4861 wss.fin = true; 4862 wss.masked = this.isClient; 4863 wss.opcode = WebSocketOpcode.ping; 4864 if(data !is null) wss.data = data.dup; 4865 wss.send(&llsend); 4866 } 4867 4868 /++ 4869 Sends a pong message to the server. This is normally done automatically in response to pings. 4870 +/ 4871 /// Group: foundational 4872 void pong(in ubyte[] data = null) { 4873 WebSocketFrame wss; 4874 wss.fin = true; 4875 wss.masked = this.isClient; 4876 wss.opcode = WebSocketOpcode.pong; 4877 wss.send(&llsend); 4878 if(data !is null) wss.data = data.dup; 4879 } 4880 4881 /++ 4882 Sends a text message through the websocket. 4883 +/ 4884 /// Group: foundational 4885 void send(in char[] textData) { 4886 WebSocketFrame wss; 4887 wss.fin = true; 4888 wss.masked = this.isClient; 4889 wss.opcode = WebSocketOpcode.text; 4890 wss.data = cast(ubyte[]) textData.dup; 4891 wss.send(&llsend); 4892 } 4893 4894 /++ 4895 Sends a binary message through the websocket. 4896 +/ 4897 /// Group: foundational 4898 void send(in ubyte[] binaryData) { 4899 WebSocketFrame wss; 4900 wss.masked = this.isClient; 4901 wss.fin = true; 4902 wss.opcode = WebSocketOpcode.binary; 4903 wss.data = cast(ubyte[]) binaryData.dup; 4904 wss.send(&llsend); 4905 } 4906 4907 /++ 4908 Waits for and returns the next complete message on the socket. 4909 4910 Note that the onmessage function is still called, right before 4911 this returns. 4912 +/ 4913 /// Group: blocking_api 4914 public WebSocketFrame waitForNextMessage() { 4915 do { 4916 auto m = processOnce(); 4917 if(m.populated) 4918 return m; 4919 } while(lowLevelReceive()); 4920 4921 return WebSocketFrame.init; // FIXME? maybe. 4922 } 4923 4924 /++ 4925 Tells if [waitForNextMessage] would block. 4926 +/ 4927 /// Group: blocking_api 4928 public bool waitForNextMessageWouldBlock() { 4929 checkAgain: 4930 if(isMessageBuffered()) 4931 return false; 4932 if(!isDataPending()) 4933 return true; 4934 while(isDataPending()) 4935 lowLevelReceive(); 4936 goto checkAgain; 4937 } 4938 4939 /++ 4940 Is there a message in the buffer already? 4941 If `true`, [waitForNextMessage] is guaranteed to return immediately. 4942 If `false`, check [isDataPending] as the next step. 4943 +/ 4944 /// Group: blocking_api 4945 public bool isMessageBuffered() { 4946 ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; 4947 auto s = d; 4948 if(d.length) { 4949 auto orig = d; 4950 auto m = WebSocketFrame.read(d); 4951 // that's how it indicates that it needs more data 4952 if(d !is orig) 4953 return true; 4954 } 4955 4956 return false; 4957 } 4958 4959 private ubyte continuingType; 4960 private ubyte[] continuingData; 4961 //private size_t continuingDataLength; 4962 4963 private WebSocketFrame processOnce() { 4964 ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; 4965 auto s = d; 4966 // FIXME: handle continuation frames more efficiently. it should really just reuse the receive buffer. 4967 WebSocketFrame m; 4968 if(d.length) { 4969 auto orig = d; 4970 m = WebSocketFrame.read(d); 4971 // that's how it indicates that it needs more data 4972 if(d is orig) 4973 return WebSocketFrame.init; 4974 m.unmaskInPlace(); 4975 switch(m.opcode) { 4976 case WebSocketOpcode.continuation: 4977 if(continuingData.length + m.data.length > config.maximumMessageSize) 4978 throw new Exception("message size exceeded"); 4979 4980 continuingData ~= m.data; 4981 if(m.fin) { 4982 if(ontextmessage) 4983 ontextmessage(cast(char[]) continuingData); 4984 if(onbinarymessage) 4985 onbinarymessage(continuingData); 4986 4987 continuingData = null; 4988 } 4989 break; 4990 case WebSocketOpcode.text: 4991 if(m.fin) { 4992 if(ontextmessage) 4993 ontextmessage(m.textData); 4994 } else { 4995 continuingType = m.opcode; 4996 //continuingDataLength = 0; 4997 continuingData = null; 4998 continuingData ~= m.data; 4999 } 5000 break; 5001 case WebSocketOpcode.binary: 5002 if(m.fin) { 5003 if(onbinarymessage) 5004 onbinarymessage(m.data); 5005 } else { 5006 continuingType = m.opcode; 5007 //continuingDataLength = 0; 5008 continuingData = null; 5009 continuingData ~= m.data; 5010 } 5011 break; 5012 case WebSocketOpcode.close: 5013 5014 //import std.stdio; writeln("closed ", cast(string) m.data); 5015 readyState_ = CLOSED; 5016 if(onclose) 5017 onclose(); 5018 5019 unregisterActiveSocket(this); 5020 break; 5021 case WebSocketOpcode.ping: 5022 // import std.stdio; writeln("ping received ", m.data); 5023 pong(m.data); 5024 break; 5025 case WebSocketOpcode.pong: 5026 // import std.stdio; writeln("pong received ", m.data); 5027 // just really references it is still alive, nbd. 5028 break; 5029 default: // ignore though i could and perhaps should throw too 5030 } 5031 } 5032 5033 if(d.length) { 5034 m.data = m.data.dup(); 5035 } 5036 5037 import core.stdc.string; 5038 memmove(receiveBuffer.ptr, d.ptr, d.length); 5039 receiveBufferUsedLength = d.length; 5040 5041 return m; 5042 } 5043 5044 private void autoprocess() { 5045 // FIXME 5046 do { 5047 processOnce(); 5048 } while(lowLevelReceive()); 5049 } 5050 5051 5052 void delegate() onclose; /// 5053 void delegate() onerror; /// 5054 void delegate(in char[]) ontextmessage; /// 5055 void delegate(in ubyte[]) onbinarymessage; /// 5056 void delegate() onopen; /// 5057 5058 /++ 5059 5060 +/ 5061 /// Group: browser_api 5062 void onmessage(void delegate(in char[]) dg) { 5063 ontextmessage = dg; 5064 } 5065 5066 /// ditto 5067 void onmessage(void delegate(in ubyte[]) dg) { 5068 onbinarymessage = dg; 5069 } 5070 5071 /* } end copy/paste */ 5072 5073 /* 5074 const int bufferedAmount // amount pending 5075 const string extensions 5076 5077 const string protocol 5078 const string url 5079 */ 5080 5081 static { 5082 /++ 5083 Runs an event loop with all known websockets on this thread until all websockets 5084 are closed or unregistered, or until you call [exitEventLoop], or set `*localLoopExited` 5085 to false (please note it may take a few seconds until it checks that flag again; it may 5086 not exit immediately). 5087 5088 History: 5089 The `localLoopExited` parameter was added August 22, 2022 (dub v10.9) 5090 5091 See_Also: 5092 [addToSimpledisplayEventLoop] 5093 +/ 5094 void eventLoop(shared(bool)* localLoopExited = null) { 5095 import core.atomic; 5096 atomicOp!"+="(numberOfEventLoops, 1); 5097 scope(exit) { 5098 if(atomicOp!"-="(numberOfEventLoops, 1) <= 0) 5099 loopExited = false; // reset it so we can reenter 5100 } 5101 5102 static SocketSet readSet; 5103 5104 if(readSet is null) 5105 readSet = new SocketSet(); 5106 5107 loopExited = false; 5108 5109 outermost: while(!loopExited && (localLoopExited is null || (*localLoopExited == false))) { 5110 readSet.reset(); 5111 5112 Duration timeout = 3.seconds; 5113 5114 auto now = MonoTime.currTime; 5115 bool hadAny; 5116 foreach(sock; activeSockets) { 5117 auto diff = sock.timeoutFromInactivity - now; 5118 if(diff <= 0.msecs) { 5119 // timeout 5120 if(sock.onerror) 5121 sock.onerror(); 5122 5123 sock.socket.close(); 5124 sock.readyState_ = CLOSED; 5125 unregisterActiveSocket(sock); 5126 continue outermost; 5127 } 5128 5129 if(diff < timeout) 5130 timeout = diff; 5131 5132 diff = sock.nextPing - now; 5133 5134 if(diff <= 0.msecs) { 5135 //sock.send(`{"action": "ping"}`); 5136 sock.ping(); 5137 sock.nextPing = now + sock.config.pingFrequency.msecs; 5138 } else { 5139 if(diff < timeout) 5140 timeout = diff; 5141 } 5142 5143 readSet.add(sock.socket); 5144 hadAny = true; 5145 } 5146 5147 if(!hadAny) { 5148 // import std.stdio; writeln("had none"); 5149 return; 5150 } 5151 5152 tryAgain: 5153 // import std.stdio; writeln(timeout); 5154 auto selectGot = Socket.select(readSet, null, null, timeout); 5155 if(selectGot == 0) { /* timeout */ 5156 // timeout 5157 continue; // it will be handled at the top of the loop 5158 } else if(selectGot == -1) { /* interrupted */ 5159 goto tryAgain; 5160 } else { 5161 foreach(sock; activeSockets) { 5162 if(readSet.isSet(sock.socket)) { 5163 sock.timeoutFromInactivity = MonoTime.currTime + sock.config.timeoutFromInactivity; 5164 if(!sock.lowLevelReceive()) { 5165 sock.readyState_ = CLOSED; 5166 unregisterActiveSocket(sock); 5167 continue outermost; 5168 } 5169 while(sock.processOnce().populated) {} 5170 selectGot--; 5171 if(selectGot <= 0) 5172 break; 5173 } 5174 } 5175 } 5176 } 5177 } 5178 5179 private static shared(int) numberOfEventLoops; 5180 5181 private __gshared bool loopExited; 5182 /++ 5183 Exits all running [WebSocket.eventLoop]s next time they loop around. You can call this from a signal handler or another thread. 5184 5185 Please note they may not loop around to check the flag for several seconds. Any new event loops will exit immediately until 5186 all current ones are closed. Once all event loops are exited, the flag is cleared and you can start the loop again. 5187 5188 This function is likely to be deprecated in the future due to its quirks and imprecise name. 5189 +/ 5190 void exitEventLoop() { 5191 loopExited = true; 5192 } 5193 5194 WebSocket[] activeSockets; 5195 5196 void registerActiveSocket(WebSocket s) { 5197 // ensure it isn't already there... 5198 assert(s !is null); 5199 foreach(i, a; activeSockets) 5200 if(a is s) 5201 return; 5202 activeSockets ~= s; 5203 } 5204 void unregisterActiveSocket(WebSocket s) { 5205 foreach(i, a; activeSockets) 5206 if(s is a) { 5207 activeSockets[i] = activeSockets[$-1]; 5208 activeSockets = activeSockets[0 .. $-1]; 5209 break; 5210 } 5211 } 5212 } 5213 } 5214 5215 private template imported(string mod) { 5216 mixin(`import imported = ` ~ mod ~ `;`); 5217 } 5218 5219 /++ 5220 Warning: you should call this AFTER websocket.connect or else it might throw on connect because the function sets nonblocking mode and the connect function doesn't handle that well (it throws on the "would block" condition in that function. easier to just do that first) 5221 +/ 5222 template addToSimpledisplayEventLoop() { 5223 import arsd.simpledisplay; 5224 void addToSimpledisplayEventLoop(WebSocket ws, imported!"arsd.simpledisplay".SimpleWindow window) { 5225 5226 void midprocess() { 5227 if(!ws.lowLevelReceive()) { 5228 ws.readyState_ = WebSocket.CLOSED; 5229 WebSocket.unregisterActiveSocket(ws); 5230 return; 5231 } 5232 while(ws.processOnce().populated) {} 5233 } 5234 5235 version(Posix) { 5236 auto reader = new PosixFdReader(&midprocess, ws.socket.handle); 5237 } else version(none) { 5238 if(WSAAsyncSelect(ws.socket.handle, window.hwnd, WM_USER + 150, FD_CLOSE | FD_READ)) 5239 throw new Exception("WSAAsyncSelect"); 5240 5241 window.handleNativeEvent = delegate int(HWND hwnd, UINT msg, WPARAM wParam, LPARAM lParam) { 5242 if(hwnd !is window.impl.hwnd) 5243 return 1; // we don't care... 5244 switch(msg) { 5245 case WM_USER + 150: // socket activity 5246 switch(LOWORD(lParam)) { 5247 case FD_READ: 5248 case FD_CLOSE: 5249 midprocess(); 5250 break; 5251 default: 5252 // nothing 5253 } 5254 break; 5255 default: return 1; // not handled, pass it on 5256 } 5257 return 0; 5258 }; 5259 5260 } else version(Windows) { 5261 ws.socket.blocking = false; // the WSAEventSelect does this anyway and doing it here lets phobos know about it. 5262 //CreateEvent(null, 0, 0, null); 5263 auto event = WSACreateEvent(); 5264 if(!event) { 5265 throw new Exception("WSACreateEvent"); 5266 } 5267 if(WSAEventSelect(ws.socket.handle, event, 1/*FD_READ*/ | (1<<5)/*FD_CLOSE*/)) { 5268 //import std.stdio; writeln(WSAGetLastError()); 5269 throw new Exception("WSAEventSelect"); 5270 } 5271 5272 auto handle = new WindowsHandleReader(&midprocess, event); 5273 5274 /+ 5275 static class Ready {} 5276 5277 Ready thisr = new Ready; 5278 5279 justCommunication.addEventListener((Ready r) { 5280 if(r is thisr) 5281 midprocess(); 5282 }); 5283 5284 import core.thread; 5285 auto thread = new Thread({ 5286 while(true) { 5287 WSAWaitForMultipleEvents(1, &event, true, -1/*WSA_INFINITE*/, false); 5288 justCommunication.postEvent(thisr); 5289 } 5290 }); 5291 thread.isDaemon = true; 5292 thread.start; 5293 +/ 5294 5295 } else static assert(0, "unsupported OS"); 5296 } 5297 } 5298 5299 version(Windows) { 5300 import core.sys.windows.windows; 5301 import core.sys.windows.winsock2; 5302 } 5303 5304 version(none) { 5305 extern(Windows) int WSAAsyncSelect(SOCKET, HWND, uint, int); 5306 enum int FD_CLOSE = 1 << 5; 5307 enum int FD_READ = 1 << 0; 5308 enum int WM_USER = 1024; 5309 } 5310 5311 version(Windows) { 5312 import core.stdc.config; 5313 extern(Windows) 5314 int WSAEventSelect(SOCKET, HANDLE /* to an Event */, c_long); 5315 5316 extern(Windows) 5317 HANDLE WSACreateEvent(); 5318 5319 extern(Windows) 5320 DWORD WSAWaitForMultipleEvents(DWORD, HANDLE*, BOOL, DWORD, BOOL); 5321 } 5322 5323 /* copy/paste from cgi.d */ 5324 public { 5325 enum WebSocketOpcode : ubyte { 5326 continuation = 0, 5327 text = 1, 5328 binary = 2, 5329 // 3, 4, 5, 6, 7 RESERVED 5330 close = 8, 5331 ping = 9, 5332 pong = 10, 5333 // 11,12,13,14,15 RESERVED 5334 } 5335 5336 public struct WebSocketFrame { 5337 private bool populated; 5338 bool fin; 5339 bool rsv1; 5340 bool rsv2; 5341 bool rsv3; 5342 WebSocketOpcode opcode; // 4 bits 5343 bool masked; 5344 ubyte lengthIndicator; // don't set this when building one to send 5345 ulong realLength; // don't use when sending 5346 ubyte[4] maskingKey; // don't set this when sending 5347 ubyte[] data; 5348 5349 static WebSocketFrame simpleMessage(WebSocketOpcode opcode, in void[] data) { 5350 WebSocketFrame msg; 5351 msg.fin = true; 5352 msg.opcode = opcode; 5353 msg.data = cast(ubyte[]) data.dup; // it is mutated below when masked, so need to be cautious and copy it, sigh 5354 5355 return msg; 5356 } 5357 5358 private void send(scope void delegate(ubyte[]) llsend) { 5359 ubyte[64] headerScratch; 5360 int headerScratchPos = 0; 5361 5362 realLength = data.length; 5363 5364 { 5365 ubyte b1; 5366 b1 |= cast(ubyte) opcode; 5367 b1 |= rsv3 ? (1 << 4) : 0; 5368 b1 |= rsv2 ? (1 << 5) : 0; 5369 b1 |= rsv1 ? (1 << 6) : 0; 5370 b1 |= fin ? (1 << 7) : 0; 5371 5372 headerScratch[0] = b1; 5373 headerScratchPos++; 5374 } 5375 5376 { 5377 headerScratchPos++; // we'll set header[1] at the end of this 5378 auto rlc = realLength; 5379 ubyte b2; 5380 b2 |= masked ? (1 << 7) : 0; 5381 5382 assert(headerScratchPos == 2); 5383 5384 if(realLength > 65535) { 5385 // use 64 bit length 5386 b2 |= 0x7f; 5387 5388 // FIXME: double check endinaness 5389 foreach(i; 0 .. 8) { 5390 headerScratch[2 + 7 - i] = rlc & 0x0ff; 5391 rlc >>>= 8; 5392 } 5393 5394 headerScratchPos += 8; 5395 } else if(realLength > 125) { 5396 // use 16 bit length 5397 b2 |= 0x7e; 5398 5399 // FIXME: double check endinaness 5400 foreach(i; 0 .. 2) { 5401 headerScratch[2 + 1 - i] = rlc & 0x0ff; 5402 rlc >>>= 8; 5403 } 5404 5405 headerScratchPos += 2; 5406 } else { 5407 // use 7 bit length 5408 b2 |= realLength & 0b_0111_1111; 5409 } 5410 5411 headerScratch[1] = b2; 5412 } 5413 5414 //assert(!masked, "masking key not properly implemented"); 5415 if(masked) { 5416 import std.random; 5417 foreach(ref item; maskingKey) 5418 item = uniform(ubyte.min, ubyte.max); 5419 headerScratch[headerScratchPos .. headerScratchPos + 4] = maskingKey[]; 5420 headerScratchPos += 4; 5421 5422 // we'll just mask it in place... 5423 int keyIdx = 0; 5424 foreach(i; 0 .. data.length) { 5425 data[i] = data[i] ^ maskingKey[keyIdx]; 5426 if(keyIdx == 3) 5427 keyIdx = 0; 5428 else 5429 keyIdx++; 5430 } 5431 } 5432 5433 //writeln("SENDING ", headerScratch[0 .. headerScratchPos], data); 5434 llsend(headerScratch[0 .. headerScratchPos]); 5435 if(data.length) 5436 llsend(data); 5437 } 5438 5439 static WebSocketFrame read(ref ubyte[] d) { 5440 WebSocketFrame msg; 5441 5442 auto orig = d; 5443 5444 WebSocketFrame needsMoreData() { 5445 d = orig; 5446 return WebSocketFrame.init; 5447 } 5448 5449 if(d.length < 2) 5450 return needsMoreData(); 5451 5452 ubyte b = d[0]; 5453 5454 msg.populated = true; 5455 5456 msg.opcode = cast(WebSocketOpcode) (b & 0x0f); 5457 b >>= 4; 5458 msg.rsv3 = b & 0x01; 5459 b >>= 1; 5460 msg.rsv2 = b & 0x01; 5461 b >>= 1; 5462 msg.rsv1 = b & 0x01; 5463 b >>= 1; 5464 msg.fin = b & 0x01; 5465 5466 b = d[1]; 5467 msg.masked = (b & 0b1000_0000) ? true : false; 5468 msg.lengthIndicator = b & 0b0111_1111; 5469 5470 d = d[2 .. $]; 5471 5472 if(msg.lengthIndicator == 0x7e) { 5473 // 16 bit length 5474 msg.realLength = 0; 5475 5476 if(d.length < 2) return needsMoreData(); 5477 5478 foreach(i; 0 .. 2) { 5479 msg.realLength |= d[0] << ((1-i) * 8); 5480 d = d[1 .. $]; 5481 } 5482 } else if(msg.lengthIndicator == 0x7f) { 5483 // 64 bit length 5484 msg.realLength = 0; 5485 5486 if(d.length < 8) return needsMoreData(); 5487 5488 foreach(i; 0 .. 8) { 5489 msg.realLength |= ulong(d[0]) << ((7-i) * 8); 5490 d = d[1 .. $]; 5491 } 5492 } else { 5493 // 7 bit length 5494 msg.realLength = msg.lengthIndicator; 5495 } 5496 5497 if(msg.masked) { 5498 5499 if(d.length < 4) return needsMoreData(); 5500 5501 msg.maskingKey = d[0 .. 4]; 5502 d = d[4 .. $]; 5503 } 5504 5505 if(msg.realLength > d.length) { 5506 return needsMoreData(); 5507 } 5508 5509 msg.data = d[0 .. cast(size_t) msg.realLength]; 5510 d = d[cast(size_t) msg.realLength .. $]; 5511 5512 return msg; 5513 } 5514 5515 void unmaskInPlace() { 5516 if(this.masked) { 5517 int keyIdx = 0; 5518 foreach(i; 0 .. this.data.length) { 5519 this.data[i] = this.data[i] ^ this.maskingKey[keyIdx]; 5520 if(keyIdx == 3) 5521 keyIdx = 0; 5522 else 5523 keyIdx++; 5524 } 5525 } 5526 } 5527 5528 char[] textData() { 5529 return cast(char[]) data; 5530 } 5531 } 5532 } 5533 5534 private extern(C) 5535 int verifyCertificateFromRegistryArsdHttp(int preverify_ok, X509_STORE_CTX* ctx) { 5536 version(Windows) { 5537 if(preverify_ok) 5538 return 1; 5539 5540 auto err_cert = OpenSSL.X509_STORE_CTX_get_current_cert(ctx); 5541 auto err = OpenSSL.X509_STORE_CTX_get_error(ctx); 5542 5543 if(err == 62) 5544 return 0; // hostname mismatch is an error we can trust; that means OpenSSL already found the certificate and rejected it 5545 5546 auto len = OpenSSL.i2d_X509(err_cert, null); 5547 if(len == -1) 5548 return 0; 5549 ubyte[] buffer = new ubyte[](len); 5550 auto ptr = buffer.ptr; 5551 len = OpenSSL.i2d_X509(err_cert, &ptr); 5552 if(len != buffer.length) 5553 return 0; 5554 5555 5556 CERT_CHAIN_PARA thing; 5557 thing.cbSize = thing.sizeof; 5558 auto context = CertCreateCertificateContext(X509_ASN_ENCODING, buffer.ptr, cast(int) buffer.length); 5559 if(context is null) 5560 return 0; 5561 scope(exit) CertFreeCertificateContext(context); 5562 5563 PCCERT_CHAIN_CONTEXT chain; 5564 if(CertGetCertificateChain(null, context, null, null, &thing, 0, null, &chain)) { 5565 scope(exit) 5566 CertFreeCertificateChain(chain); 5567 5568 DWORD errorStatus = chain.TrustStatus.dwErrorStatus; 5569 5570 if(errorStatus == 0) 5571 return 1; // Windows approved it, OK carry on 5572 // otherwise, sustain OpenSSL's original ruling 5573 } 5574 5575 return 0; 5576 } else { 5577 return preverify_ok; 5578 } 5579 } 5580 5581 5582 version(Windows) { 5583 pragma(lib, "crypt32"); 5584 import core.sys.windows.wincrypt; 5585 extern(Windows) { 5586 PCCERT_CONTEXT CertEnumCertificatesInStore(HCERTSTORE hCertStore, PCCERT_CONTEXT pPrevCertContext); 5587 // BOOL CertGetCertificateChain(HCERTCHAINENGINE hChainEngine, PCCERT_CONTEXT pCertContext, LPFILETIME pTime, HCERTSTORE hAdditionalStore, PCERT_CHAIN_PARA pChainPara, DWORD dwFlags, LPVOID pvReserved, PCCERT_CHAIN_CONTEXT *ppChainContext); 5588 PCCERT_CONTEXT CertCreateCertificateContext(DWORD dwCertEncodingType, const BYTE *pbCertEncoded, DWORD cbCertEncoded); 5589 } 5590 5591 void loadCertificatesFromRegistry(SSL_CTX* ctx) { 5592 auto store = CertOpenSystemStore(0, "ROOT"); 5593 if(store is null) { 5594 // import std.stdio; writeln("failed"); 5595 return; 5596 } 5597 scope(exit) 5598 CertCloseStore(store, 0); 5599 5600 X509_STORE* ssl_store = OpenSSL.SSL_CTX_get_cert_store(ctx); 5601 PCCERT_CONTEXT c; 5602 while((c = CertEnumCertificatesInStore(store, c)) !is null) { 5603 FILETIME na = c.pCertInfo.NotAfter; 5604 SYSTEMTIME st; 5605 FileTimeToSystemTime(&na, &st); 5606 5607 /+ 5608 _CRYPTOAPI_BLOB i = cast() c.pCertInfo.Issuer; 5609 5610 char[256] buffer; 5611 auto p = CertNameToStrA(X509_ASN_ENCODING, &i, CERT_SIMPLE_NAME_STR, buffer.ptr, cast(int) buffer.length); 5612 import std.stdio; writeln(buffer[0 .. p]); 5613 +/ 5614 5615 if(st.wYear <= 2021) { 5616 // see: https://www.openssl.org/blog/blog/2021/09/13/LetsEncryptRootCertExpire/ 5617 continue; // no point keeping an expired root cert and it can break Let's Encrypt anyway 5618 } 5619 5620 const(ubyte)* thing = c.pbCertEncoded; 5621 auto x509 = OpenSSL.d2i_X509(null, &thing, c.cbCertEncoded); 5622 if (x509) { 5623 auto success = OpenSSL.X509_STORE_add_cert(ssl_store, x509); 5624 //if(!success) 5625 //writeln("FAILED HERE"); 5626 OpenSSL.X509_free(x509); 5627 } else { 5628 //writeln("FAILED"); 5629 } 5630 } 5631 5632 CertFreeCertificateContext(c); 5633 5634 // import core.stdc.stdio; printf("%s\n", OpenSSL.OpenSSL_version(0)); 5635 } 5636 5637 5638 // because i use the FILE* in PEM_read_X509 and friends 5639 // gotta use this to bridge the MS C runtime functions 5640 // might be able to just change those to only use the BIO versions 5641 // instead 5642 5643 // only on MS C runtime 5644 version(CRuntime_Microsoft) {} else version=no_openssl_applink; 5645 5646 version(no_openssl_applink) {} else { 5647 private extern(C) { 5648 void _open(); 5649 void _read(); 5650 void _write(); 5651 void _lseek(); 5652 void _close(); 5653 int _fileno(FILE*); 5654 int _setmode(int, int); 5655 } 5656 export extern(C) void** OPENSSL_Applink() { 5657 import core.stdc.stdio; 5658 5659 static extern(C) void* app_stdin() { return cast(void*) stdin; } 5660 static extern(C) void* app_stdout() { return cast(void*) stdout; } 5661 static extern(C) void* app_stderr() { return cast(void*) stderr; } 5662 static extern(C) int app_feof(FILE* fp) { return feof(fp); } 5663 static extern(C) int app_ferror(FILE* fp) { return ferror(fp); } 5664 static extern(C) void app_clearerr(FILE* fp) { return clearerr(fp); } 5665 static extern(C) int app_fileno(FILE* fp) { return _fileno(fp); } 5666 static extern(C) int app_fsetmod(FILE* fp, char mod) { 5667 return _setmode(_fileno(fp), mod == 'b' ? _O_BINARY : _O_TEXT); 5668 } 5669 5670 static immutable void*[] table = [ 5671 cast(void*) 22, // applink max 5672 5673 &app_stdin, 5674 &app_stdout, 5675 &app_stderr, 5676 &fprintf, 5677 &fgets, 5678 &fread, 5679 &fwrite, 5680 &app_fsetmod, 5681 &app_feof, 5682 &fclose, 5683 5684 &fopen, 5685 &fseek, 5686 &ftell, 5687 &fflush, 5688 &app_ferror, 5689 &app_clearerr, 5690 &app_fileno, 5691 5692 &_open, 5693 &_read, 5694 &_write, 5695 &_lseek, 5696 &_close, 5697 ]; 5698 static assert(table.length == 23); 5699 5700 return cast(void**) table.ptr; 5701 } 5702 } 5703 } 5704 5705 unittest { 5706 auto client = new HttpClient(); 5707 auto response = client.navigateTo(Uri("data:,Hello%2C%20World%21")).waitForCompletion(); 5708 assert(response.contentTypeMimeType == "text/plain", response.contentType); 5709 assert(response.contentText == "Hello, World!", response.contentText); 5710 5711 response = client.navigateTo(Uri("data:text/plain;base64,SGVsbG8sIFdvcmxkIQ==")).waitForCompletion(); 5712 assert(response.contentTypeMimeType == "text/plain", response.contentType); 5713 assert(response.contentText == "Hello, World!", response.contentText); 5714 5715 response = client.navigateTo(Uri("data:text/html,%3Ch1%3EHello%2C%20World%21%3C%2Fh1%3E")).waitForCompletion(); 5716 assert(response.contentTypeMimeType == "text/html", response.contentType); 5717 assert(response.contentText == "<h1>Hello, World!</h1>", response.contentText); 5718 } 5719 5720 version(arsd_http2_unittests) 5721 unittest { 5722 import core.thread; 5723 5724 static void server() { 5725 import std.socket; 5726 auto socket = new TcpSocket(); 5727 socket.setOption(SocketOptionLevel.SOCKET, SocketOption.REUSEADDR, true); 5728 socket.bind(new InternetAddress(12346)); 5729 socket.listen(1); 5730 auto s = socket.accept(); 5731 socket.close(); 5732 5733 ubyte[1024] thing; 5734 auto g = s.receive(thing[]); 5735 5736 /+ 5737 string response = "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 9\r\n\r\nHello!!??"; 5738 auto packetSize = 2; 5739 +/ 5740 5741 auto packetSize = 1; 5742 string response = "HTTP/1.1 200 OK\r\nConnection: close\r\nTransfer-Encoding: chunked\r\n\r\n6\r\nHello!\r\n0\r\n\r\n"; 5743 5744 while(response.length) { 5745 s.send(response[0 .. packetSize]); 5746 response = response[packetSize .. $]; 5747 //import std.stdio; writeln(response); 5748 } 5749 5750 s.close(); 5751 } 5752 5753 auto thread = new Thread(&server); 5754 thread.start; 5755 5756 Thread.sleep(200.msecs); 5757 5758 auto response = get("http://localhost:12346/").waitForCompletion; 5759 assert(response.code == 200); 5760 //import std.stdio; writeln(response); 5761 5762 foreach(site; ["https://dlang.org/", "http://arsdnet.net", "https://phobos.dpldocs.info"]) { 5763 response = get(site).waitForCompletion; 5764 assert(response.code == 200); 5765 } 5766 5767 thread.join; 5768 } 5769 5770 /+ 5771 so the url params are arguments. it knows the request 5772 internally. other params are properties on the req 5773 5774 names may have different paths... those will just add ForSomething i think. 5775 5776 auto req = api.listMergeRequests 5777 req.page = 10; 5778 5779 or 5780 req.page(1) 5781 .bar("foo") 5782 5783 req.execute(); 5784 5785 5786 everything in the response is nullable access through the 5787 dynamic object, just with property getters there. need to make 5788 it static generated tho 5789 5790 other messages may be: isPresent and getDynamic 5791 5792 5793 AND/OR what about doing it like the rails objects 5794 5795 BroadcastMessage.get(4) 5796 // various properties 5797 5798 // it lists what you updated 5799 5800 BroadcastMessage.foo().bar().put(5) 5801 +/