1 // Copyright 2013-2022, Adam D. Ruppe. 2 3 // FIXME: websocket proxy support 4 // FIXME: ipv6 support 5 6 // FIXME: headers are supposed to be case insensitive. ugh. 7 8 /++ 9 This is version 2 of my http/1.1 client implementation. 10 11 12 It has no dependencies for basic operation, but does require OpenSSL 13 libraries (or compatible) to support HTTPS. This dynamically loaded 14 on-demand (meaning it won't be loaded if you don't use it, but if you do 15 use it, the openssl dynamic libraries must be found in the system search path). 16 17 On Windows, you can bundle the openssl dlls with your exe and they will be picked 18 up when distributed. 19 20 You can compile with `-version=without_openssl` to entirely disable ssl support. 21 22 http2.d, despite its name, does NOT implement HTTP/2.0, but this 23 shouldn't matter for 99.9% of usage, since all servers will continue 24 to support HTTP/1.1 for a very long time. 25 26 History: 27 Automatic `100 Continue` handling was added on September 28, 2021. It doesn't 28 set the Expect header, so it isn't supposed to happen, but plenty of web servers 29 don't follow the standard anyway. 30 31 A dependency on [arsd.core] was added on March 19, 2023 (dub v11.0). Previously, 32 module was stand-alone. You will have add the `core.d` file from the arsd repo 33 to your build now if you are managing the files and builds yourself. 34 35 The benefits of this dependency include some simplified implementation code which 36 makes it easier for me to add more api conveniences, better exceptions with more 37 information, and better event loop integration with other arsd modules beyond 38 just the simpledisplay adapters available previously. The new integration can 39 also make things like heartbeat timers easier for you to code. 40 +/ 41 module arsd.http2; 42 43 /// 44 unittest { 45 import arsd.http2; 46 47 void main() { 48 auto client = new HttpClient(); 49 50 auto request = client.request(Uri("http://dlang.org/")); 51 auto response = request.waitForCompletion(); 52 53 import std.stdio; 54 writeln(response.contentText); 55 writeln(response.code, " ", response.codeText); 56 writeln(response.contentType); 57 } 58 59 version(arsd_http2_integration_test) main(); // exclude from docs 60 } 61 62 static import arsd.core; 63 64 // FIXME: I think I want to disable sigpipe here too. 65 66 import std.uri : encodeComponent; 67 68 debug(arsd_http2_verbose) debug=arsd_http2; 69 70 debug(arsd_http2) import std.stdio : writeln; 71 72 version=arsd_http_internal_implementation; 73 74 version(without_openssl) {} 75 else { 76 version=use_openssl; 77 version=with_openssl; 78 version(older_openssl) {} else 79 version=newer_openssl; 80 } 81 82 version(arsd_http_winhttp_implementation) { 83 pragma(lib, "winhttp") 84 import core.sys.windows.winhttp; 85 // FIXME: alter the dub package file too 86 87 // https://github.com/curl/curl/blob/master/lib/vtls/schannel.c 88 // https://docs.microsoft.com/en-us/windows/win32/secauthn/creating-an-schannel-security-context 89 90 91 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpreaddata 92 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpsendrequest 93 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpopenrequest 94 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpconnect 95 } 96 97 98 99 /++ 100 Demonstrates core functionality, using the [HttpClient], 101 [HttpRequest] (returned by [HttpClient.navigateTo|client.navigateTo]), 102 and [HttpResponse] (returned by [HttpRequest.waitForCompletion|request.waitForCompletion]). 103 104 +/ 105 unittest { 106 import arsd.http2; 107 108 void main() { 109 auto client = new HttpClient(); 110 auto request = client.navigateTo(Uri("http://dlang.org/")); 111 auto response = request.waitForCompletion(); 112 113 string returnedHtml = response.contentText; 114 } 115 } 116 117 private __gshared bool defaultVerifyPeer_ = true; 118 119 void defaultVerifyPeer(bool v) { 120 defaultVerifyPeer_ = v; 121 } 122 123 debug import std.stdio; 124 125 import std.socket; 126 import core.time; 127 128 // FIXME: check Transfer-Encoding: gzip always 129 130 version(with_openssl) { 131 //pragma(lib, "crypto"); 132 //pragma(lib, "ssl"); 133 } 134 135 /+ 136 HttpRequest httpRequest(string method, string url, ubyte[] content, string[string] content) { 137 return null; 138 } 139 +/ 140 141 /** 142 auto request = get("http://arsdnet.net/"); 143 request.send(); 144 145 auto response = get("http://arsdnet.net/").waitForCompletion(); 146 */ 147 HttpRequest get(string url) { 148 auto client = new HttpClient(); 149 auto request = client.navigateTo(Uri(url)); 150 return request; 151 } 152 153 /** 154 Do not forget to call `waitForCompletion()` on the returned object! 155 */ 156 HttpRequest post(string url, string[string] req) { 157 auto client = new HttpClient(); 158 ubyte[] bdata; 159 foreach(k, v; req) { 160 if(bdata.length) 161 bdata ~= cast(ubyte[]) "&"; 162 bdata ~= cast(ubyte[]) encodeComponent(k); 163 bdata ~= cast(ubyte[]) "="; 164 bdata ~= cast(ubyte[]) encodeComponent(v); 165 } 166 auto request = client.request(Uri(url), HttpVerb.POST, bdata, "application/x-www-form-urlencoded"); 167 return request; 168 } 169 170 /// gets the text off a url. basic operation only. 171 string getText(string url) { 172 auto request = get(url); 173 auto response = request.waitForCompletion(); 174 return cast(string) response.content; 175 } 176 177 /+ 178 ubyte[] getBinary(string url, string[string] cookies = null) { 179 auto hr = httpRequest("GET", url, null, cookies); 180 if(hr.code != 200) 181 throw new Exception(format("HTTP answered %d instead of 200 on %s", hr.code, url)); 182 return hr.content; 183 } 184 185 /** 186 Gets a textual document, ignoring headers. Throws on non-text or error. 187 */ 188 string get(string url, string[string] cookies = null) { 189 auto hr = httpRequest("GET", url, null, cookies); 190 if(hr.code != 200) 191 throw new Exception(format("HTTP answered %d instead of 200 on %s", hr.code, url)); 192 if(hr.contentType.indexOf("text/") == -1) 193 throw new Exception(hr.contentType ~ " is bad content for conversion to string"); 194 return cast(string) hr.content; 195 196 } 197 198 static import std.uri; 199 200 string post(string url, string[string] args, string[string] cookies = null) { 201 string content; 202 203 foreach(name, arg; args) { 204 if(content.length) 205 content ~= "&"; 206 content ~= std.uri.encode(name) ~ "=" ~ std.uri.encode(arg); 207 } 208 209 auto hr = httpRequest("POST", url, cast(ubyte[]) content, cookies, ["Content-Type: application/x-www-form-urlencoded"]); 210 if(hr.code != 200) 211 throw new Exception(format("HTTP answered %d instead of 200", hr.code)); 212 if(hr.contentType.indexOf("text/") == -1) 213 throw new Exception(hr.contentType ~ " is bad content for conversion to string"); 214 215 return cast(string) hr.content; 216 } 217 218 +/ 219 220 /// 221 struct HttpResponse { 222 /++ 223 The HTTP response code, if the response was completed, or some value < 100 if it was aborted or failed. 224 225 Code 0 - initial value, nothing happened 226 Code 1 - you called request.abort 227 Code 2 - connection refused 228 Code 3 - connection succeeded, but server disconnected early 229 Code 4 - server sent corrupted response (or this code has a bug and processed it wrong) 230 Code 5 - request timed out 231 232 Code >= 100 - a HTTP response 233 +/ 234 int code; 235 string codeText; /// 236 237 string httpVersion; /// 238 239 string statusLine; /// 240 241 string contentType; /// The *full* content type header. See also [contentTypeMimeType] and [contentTypeCharset]. 242 string location; /// The location header 243 244 /++ 245 246 History: 247 Added December 5, 2020 (version 9.1) 248 +/ 249 bool wasSuccessful() { 250 return code >= 200 && code < 400; 251 } 252 253 /++ 254 Returns the mime type part of the [contentType] header. 255 256 History: 257 Added July 25, 2022 (version 10.9) 258 +/ 259 string contentTypeMimeType() { 260 auto idx = contentType.indexOf(";"); 261 if(idx == -1) 262 return contentType; 263 264 return contentType[0 .. idx].strip; 265 } 266 267 /// the charset out of content type, if present. `null` if not. 268 string contentTypeCharset() { 269 auto idx = contentType.indexOf("charset="); 270 if(idx == -1) 271 return null; 272 auto c = contentType[idx + "charset=".length .. $].strip; 273 if(c.length) 274 return c; 275 return null; 276 } 277 278 /++ 279 Names and values of cookies set in the response. 280 281 History: 282 Prior to July 5, 2021 (dub v10.2), this was a public field instead of a property. I did 283 not consider this a breaking change since the intended use is completely compatible with the 284 property, and it was not actually implemented properly before anyway. 285 +/ 286 @property string[string] cookies() const { 287 string[string] ret; 288 foreach(cookie; cookiesDetails) 289 ret[cookie.name] = cookie.value; 290 return ret; 291 } 292 /++ 293 The full parsed-out information of cookies set in the response. 294 295 History: 296 Added July 5, 2021 (dub v10.2). 297 +/ 298 @property CookieHeader[] cookiesDetails() inout { 299 CookieHeader[] ret; 300 foreach(header; headers) { 301 if(auto content = header.isHttpHeader("set-cookie")) { 302 // format: name=value, value might be double quoted. it MIGHT be url encoded, but im not going to attempt that since the RFC is silent. 303 // then there's optionally ; attr=value after that. attributes need not have a value 304 305 CookieHeader cookie; 306 307 auto remaining = content; 308 309 cookie_name: 310 foreach(idx, ch; remaining) { 311 if(ch == '=') { 312 cookie.name = remaining[0 .. idx].idup_if_needed; 313 remaining = remaining[idx + 1 .. $]; 314 break; 315 } 316 } 317 318 cookie_value: 319 320 { 321 auto idx = remaining.indexOf(";"); 322 if(idx == -1) { 323 cookie.value = remaining.idup_if_needed; 324 remaining = remaining[$..$]; 325 } else { 326 cookie.value = remaining[0 .. idx].idup_if_needed; 327 remaining = remaining[idx + 1 .. $].stripLeft; 328 } 329 330 if(cookie.value.length > 2 && cookie.value[0] == '"' && cookie.value[$-1] == '"') 331 cookie.value = cookie.value[1 .. $ - 1]; 332 } 333 334 cookie_attributes: 335 336 while(remaining.length) { 337 string name; 338 foreach(idx, ch; remaining) { 339 if(ch == '=') { 340 name = remaining[0 .. idx].idup_if_needed; 341 remaining = remaining[idx + 1 .. $]; 342 343 string value; 344 345 foreach(idx2, ch2; remaining) { 346 if(ch2 == ';') { 347 value = remaining[0 .. idx2].idup_if_needed; 348 remaining = remaining[idx2 + 1 .. $].stripLeft; 349 break; 350 } 351 } 352 353 if(value is null) { 354 value = remaining.idup_if_needed; 355 remaining = remaining[$ .. $]; 356 } 357 358 cookie.attributes[name] = value; 359 continue cookie_attributes; 360 } else if(ch == ';') { 361 name = remaining[0 .. idx].idup_if_needed; 362 remaining = remaining[idx + 1 .. $].stripLeft; 363 cookie.attributes[name] = ""; 364 continue cookie_attributes; 365 } 366 } 367 368 if(remaining.length) { 369 cookie.attributes[remaining.idup_if_needed] = ""; 370 remaining = remaining[$..$]; 371 372 } 373 } 374 375 ret ~= cookie; 376 } 377 } 378 return ret; 379 } 380 381 string[] headers; /// Array of all headers returned. 382 string[string] headersHash; /// 383 384 ubyte[] content; /// The raw content returned in the response body. 385 string contentText; /// [content], but casted to string (for convenience) 386 387 alias responseText = contentText; // just cuz I do this so often. 388 //alias body = content; 389 390 /++ 391 returns `new Document(this.contentText)`. Requires [arsd.dom]. 392 +/ 393 auto contentDom()() { 394 import arsd.dom; 395 return new Document(this.contentText); 396 397 } 398 399 /++ 400 returns `var.fromJson(this.contentText)`. Requires [arsd.jsvar]. 401 +/ 402 auto contentJson()() { 403 import arsd.jsvar; 404 return var.fromJson(this.contentText); 405 } 406 407 HttpRequestParameters requestParameters; /// 408 409 LinkHeader[] linksStored; 410 bool linksLazilyParsed; 411 412 HttpResponse deepCopy() const { 413 HttpResponse h = cast(HttpResponse) this; 414 h.headers = h.headers.dup; 415 h.headersHash = h.headersHash.dup; 416 h.content = h.content.dup; 417 h.linksStored = h.linksStored.dup; 418 return h; 419 } 420 421 /// Returns links header sorted by "rel" attribute. 422 /// It returns a new array on each call. 423 LinkHeader[string] linksHash() { 424 auto links = this.links(); 425 LinkHeader[string] ret; 426 foreach(link; links) 427 ret[link.rel] = link; 428 return ret; 429 } 430 431 /// Returns the Link header, parsed. 432 LinkHeader[] links() { 433 if(linksLazilyParsed) 434 return linksStored; 435 linksLazilyParsed = true; 436 LinkHeader[] ret; 437 438 auto hdrPtr = "link" in headersHash; 439 if(hdrPtr is null) 440 return ret; 441 442 auto header = *hdrPtr; 443 444 LinkHeader current; 445 446 while(header.length) { 447 char ch = header[0]; 448 449 if(ch == '<') { 450 // read url 451 header = header[1 .. $]; 452 size_t idx; 453 while(idx < header.length && header[idx] != '>') 454 idx++; 455 current.url = header[0 .. idx]; 456 header = header[idx .. $]; 457 } else if(ch == ';') { 458 // read attribute 459 header = header[1 .. $]; 460 header = header.stripLeft; 461 462 size_t idx; 463 while(idx < header.length && header[idx] != '=') 464 idx++; 465 466 string name = header[0 .. idx]; 467 if(idx + 1 < header.length) 468 header = header[idx + 1 .. $]; 469 else 470 header = header[$ .. $]; 471 472 string value; 473 474 if(header.length && header[0] == '"') { 475 // quoted value 476 header = header[1 .. $]; 477 idx = 0; 478 while(idx < header.length && header[idx] != '\"') 479 idx++; 480 value = header[0 .. idx]; 481 header = header[idx .. $]; 482 483 } else if(header.length) { 484 // unquoted value 485 idx = 0; 486 while(idx < header.length && header[idx] != ',' && header[idx] != ' ' && header[idx] != ';') 487 idx++; 488 489 value = header[0 .. idx]; 490 header = header[idx .. $].stripLeft; 491 } 492 493 name = name.toLower; 494 if(name == "rel") 495 current.rel = value; 496 else 497 current.attributes[name] = value; 498 499 } else if(ch == ',') { 500 // start another 501 ret ~= current; 502 current = LinkHeader.init; 503 } else if(ch == ' ' || ch == '\n' || ch == '\r' || ch == '\t') { 504 // ignore 505 } 506 507 if(header.length) 508 header = header[1 .. $]; 509 } 510 511 ret ~= current; 512 513 linksStored = ret; 514 515 return ret; 516 } 517 } 518 519 /+ 520 headerName MUST be all lower case and NOT have the colon on it 521 522 returns slice of the input thing after the header name 523 +/ 524 private inout(char)[] isHttpHeader(inout(char)[] thing, const(char)[] headerName) { 525 foreach(idx, ch; thing) { 526 if(idx < headerName.length) { 527 if(headerName[idx] == '-' && ch != '-') 528 return null; 529 if((ch | ' ') != headerName[idx]) 530 return null; 531 } else if(idx == headerName.length) { 532 if(ch != ':') 533 return null; 534 } else { 535 return thing[idx .. $].strip; 536 } 537 } 538 return null; 539 } 540 541 private string idup_if_needed(string s) { return s; } 542 private string idup_if_needed(const(char)[] s) { return s.idup; } 543 544 unittest { 545 assert("Cookie: foo=bar".isHttpHeader("cookie") == "foo=bar"); 546 assert("cookie: foo=bar".isHttpHeader("cookie") == "foo=bar"); 547 assert("cOOkie: foo=bar".isHttpHeader("cookie") == "foo=bar"); 548 assert("Set-Cookie: foo=bar".isHttpHeader("set-cookie") == "foo=bar"); 549 assert(!"".isHttpHeader("cookie")); 550 } 551 552 /// 553 struct LinkHeader { 554 string url; /// 555 string rel; /// 556 string[string] attributes; /// like title, rev, media, whatever attributes 557 } 558 559 /++ 560 History: 561 Added July 5, 2021 562 +/ 563 struct CookieHeader { 564 string name; 565 string value; 566 string[string] attributes; 567 } 568 569 import std.string; 570 static import std.algorithm; 571 import std.conv; 572 import std.range; 573 574 575 private AddressFamily family(string unixSocketPath) { 576 if(unixSocketPath.length) 577 return AddressFamily.UNIX; 578 else // FIXME: what about ipv6? 579 return AddressFamily.INET; 580 } 581 582 version(Windows) 583 private class UnixAddress : Address { 584 this(string) { 585 throw new Exception("No unix address support on this system in lib yet :("); 586 } 587 override sockaddr* name() { assert(0); } 588 override const(sockaddr)* name() const { assert(0); } 589 override int nameLen() const { assert(0); } 590 } 591 592 593 // Copy pasta from cgi.d, then stripped down. unix path thing added tho 594 /++ 595 Represents a URI. It offers named access to the components and relative uri resolution, though as a user of the library, you'd mostly just construct it like `Uri("http://example.com/index.html")`. 596 +/ 597 struct Uri { 598 alias toString this; // blargh idk a url really is a string, but should it be implicit? 599 600 // scheme://userinfo@host:port/path?query#fragment 601 602 string scheme; /// e.g. "http" in "http://example.com/" 603 string userinfo; /// the username (and possibly a password) in the uri 604 string host; /// the domain name 605 int port; /// port number, if given. Will be zero if a port was not explicitly given 606 string path; /// e.g. "/folder/file.html" in "http://example.com/folder/file.html" 607 string query; /// the stuff after the ? in a uri 608 string fragment; /// the stuff after the # in a uri. 609 610 /// Breaks down a uri string to its components 611 this(string uri) { 612 size_t lastGoodIndex; 613 foreach(char ch; uri) { 614 if(ch > 127) { 615 break; 616 } 617 lastGoodIndex++; 618 } 619 620 string replacement = uri[0 .. lastGoodIndex]; 621 foreach(char ch; uri[lastGoodIndex .. $]) { 622 if(ch > 127) { 623 // need to percent-encode any non-ascii in it 624 char[3] buffer; 625 buffer[0] = '%'; 626 627 auto first = ch / 16; 628 auto second = ch % 16; 629 first += (first >= 10) ? ('A'-10) : '0'; 630 second += (second >= 10) ? ('A'-10) : '0'; 631 632 buffer[1] = cast(char) first; 633 buffer[2] = cast(char) second; 634 635 replacement ~= buffer[]; 636 } else { 637 replacement ~= ch; 638 } 639 } 640 641 reparse(replacement); 642 } 643 644 /// Returns `port` if set, otherwise if scheme is https 443, otherwise always 80 645 int effectivePort() const @property nothrow pure @safe @nogc { 646 return port != 0 ? port 647 : scheme == "https" ? 443 : 80; 648 } 649 650 private string unixSocketPath = null; 651 /// Indicates it should be accessed through a unix socket instead of regular tcp. Returns new version without modifying this object. 652 Uri viaUnixSocket(string path) const { 653 Uri copy = this; 654 copy.unixSocketPath = path; 655 return copy; 656 } 657 658 /// Goes through a unix socket in the abstract namespace (linux only). Returns new version without modifying this object. 659 version(linux) 660 Uri viaAbstractSocket(string path) const { 661 Uri copy = this; 662 copy.unixSocketPath = "\0" ~ path; 663 return copy; 664 } 665 666 private void reparse(string uri) { 667 // from RFC 3986 668 // the ctRegex triples the compile time and makes ugly errors for no real benefit 669 // it was a nice experiment but just not worth it. 670 // enum ctr = ctRegex!r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?"; 671 /* 672 Captures: 673 0 = whole url 674 1 = scheme, with : 675 2 = scheme, no : 676 3 = authority, with // 677 4 = authority, no // 678 5 = path 679 6 = query string, with ? 680 7 = query string, no ? 681 8 = anchor, with # 682 9 = anchor, no # 683 */ 684 // Yikes, even regular, non-CT regex is also unacceptably slow to compile. 1.9s on my computer! 685 // instead, I will DIY and cut that down to 0.6s on the same computer. 686 /* 687 688 Note that authority is 689 user:password@domain:port 690 where the user:password@ part is optional, and the :port is optional. 691 692 Regex translation: 693 694 Scheme cannot have :, /, ?, or # in it, and must have one or more chars and end in a :. It is optional, but must be first. 695 Authority must start with //, but cannot have any other /, ?, or # in it. It is optional. 696 Path cannot have any ? or # in it. It is optional. 697 Query must start with ? and must not have # in it. It is optional. 698 Anchor must start with # and can have anything else in it to end of string. It is optional. 699 */ 700 701 this = Uri.init; // reset all state 702 703 // empty uri = nothing special 704 if(uri.length == 0) { 705 return; 706 } 707 708 size_t idx; 709 710 scheme_loop: foreach(char c; uri[idx .. $]) { 711 switch(c) { 712 case ':': 713 case '/': 714 case '?': 715 case '#': 716 break scheme_loop; 717 default: 718 } 719 idx++; 720 } 721 722 if(idx == 0 && uri[idx] == ':') { 723 // this is actually a path! we skip way ahead 724 goto path_loop; 725 } 726 727 if(idx == uri.length) { 728 // the whole thing is a path, apparently 729 path = uri; 730 return; 731 } 732 733 if(idx > 0 && uri[idx] == ':') { 734 scheme = uri[0 .. idx]; 735 idx++; 736 } else { 737 // we need to rewind; it found a / but no :, so the whole thing is prolly a path... 738 idx = 0; 739 } 740 741 if(idx + 2 < uri.length && uri[idx .. idx + 2] == "//") { 742 // we have an authority.... 743 idx += 2; 744 745 auto authority_start = idx; 746 authority_loop: foreach(char c; uri[idx .. $]) { 747 switch(c) { 748 case '/': 749 case '?': 750 case '#': 751 break authority_loop; 752 default: 753 } 754 idx++; 755 } 756 757 auto authority = uri[authority_start .. idx]; 758 759 auto idx2 = authority.indexOf("@"); 760 if(idx2 != -1) { 761 userinfo = authority[0 .. idx2]; 762 authority = authority[idx2 + 1 .. $]; 763 } 764 765 if(authority.length && authority[0] == '[') { 766 // ipv6 address special casing 767 idx2 = authority.indexOf(']'); 768 if(idx2 != -1) { 769 auto end = authority[idx2 + 1 .. $]; 770 if(end.length && end[0] == ':') 771 idx2 = idx2 + 1; 772 else 773 idx2 = -1; 774 } 775 } else { 776 idx2 = authority.indexOf(":"); 777 } 778 779 if(idx2 == -1) { 780 port = 0; // 0 means not specified; we should use the default for the scheme 781 host = authority; 782 } else { 783 host = authority[0 .. idx2]; 784 if(idx2 + 1 < authority.length) 785 port = to!int(authority[idx2 + 1 .. $]); 786 else 787 port = 0; 788 } 789 } 790 791 path_loop: 792 auto path_start = idx; 793 794 foreach(char c; uri[idx .. $]) { 795 if(c == '?' || c == '#') 796 break; 797 idx++; 798 } 799 800 path = uri[path_start .. idx]; 801 802 if(idx == uri.length) 803 return; // nothing more to examine... 804 805 if(uri[idx] == '?') { 806 idx++; 807 auto query_start = idx; 808 foreach(char c; uri[idx .. $]) { 809 if(c == '#') 810 break; 811 idx++; 812 } 813 query = uri[query_start .. idx]; 814 } 815 816 if(idx < uri.length && uri[idx] == '#') { 817 idx++; 818 fragment = uri[idx .. $]; 819 } 820 821 // uriInvalidated = false; 822 } 823 824 private string rebuildUri() const { 825 string ret; 826 if(scheme.length) 827 ret ~= scheme ~ ":"; 828 if(userinfo.length || host.length) 829 ret ~= "//"; 830 if(userinfo.length) 831 ret ~= userinfo ~ "@"; 832 if(host.length) 833 ret ~= host; 834 if(port) 835 ret ~= ":" ~ to!string(port); 836 837 ret ~= path; 838 839 if(query.length) 840 ret ~= "?" ~ query; 841 842 if(fragment.length) 843 ret ~= "#" ~ fragment; 844 845 // uri = ret; 846 // uriInvalidated = false; 847 return ret; 848 } 849 850 /// Converts the broken down parts back into a complete string 851 string toString() const { 852 // if(uriInvalidated) 853 return rebuildUri(); 854 } 855 856 /// Returns a new absolute Uri given a base. It treats this one as 857 /// relative where possible, but absolute if not. (If protocol, domain, or 858 /// other info is not set, the new one inherits it from the base.) 859 /// 860 /// Browsers use a function like this to figure out links in html. 861 Uri basedOn(in Uri baseUrl) const { 862 Uri n = this; // copies 863 if(n.scheme == "data") 864 return n; 865 // n.uriInvalidated = true; // make sure we regenerate... 866 867 // userinfo is not inherited... is this wrong? 868 869 // if anything is given in the existing url, we don't use the base anymore. 870 if(n.scheme.empty) { 871 n.scheme = baseUrl.scheme; 872 if(n.host.empty) { 873 n.host = baseUrl.host; 874 if(n.port == 0) { 875 n.port = baseUrl.port; 876 if(n.path.length > 0 && n.path[0] != '/') { 877 auto b = baseUrl.path[0 .. baseUrl.path.lastIndexOf("/") + 1]; 878 if(b.length == 0) 879 b = "/"; 880 n.path = b ~ n.path; 881 } else if(n.path.length == 0) { 882 n.path = baseUrl.path; 883 } 884 } 885 } 886 } 887 888 n.removeDots(); 889 890 // if still basically talking to the same thing, we should inherit the unix path 891 // too since basically the unix path is saying for this service, always use this override. 892 if(n.host == baseUrl.host && n.scheme == baseUrl.scheme && n.port == baseUrl.port) 893 n.unixSocketPath = baseUrl.unixSocketPath; 894 895 return n; 896 } 897 898 /++ 899 Resolves ../ and ./ parts of the path. Used in the implementation of [basedOn] and you could also use it to normalize things. 900 +/ 901 void removeDots() { 902 auto parts = this.path.split("/"); 903 string[] toKeep; 904 foreach(part; parts) { 905 if(part == ".") { 906 continue; 907 } else if(part == "..") { 908 //if(toKeep.length > 1) 909 toKeep = toKeep[0 .. $-1]; 910 //else 911 //toKeep = [""]; 912 continue; 913 } else { 914 //if(toKeep.length && toKeep[$-1].length == 0 && part.length == 0) 915 //continue; // skip a `//` situation 916 toKeep ~= part; 917 } 918 } 919 920 auto path = toKeep.join("/"); 921 if(path.length && path[0] != '/') 922 path = "/" ~ path; 923 924 this.path = path; 925 } 926 } 927 928 /* 929 void main(string args[]) { 930 write(post("http://arsdnet.net/bugs.php", ["test" : "hey", "again" : "what"])); 931 } 932 */ 933 934 /// 935 struct BasicAuth { 936 string username; /// 937 string password; /// 938 } 939 940 class ProxyException : Exception { 941 this(string msg) {super(msg); } 942 } 943 944 /** 945 Represents a HTTP request. You usually create these through a [HttpClient]. 946 947 948 --- 949 auto request = new HttpRequest(); // note that when there's no associated client, some features may not work 950 // normally you'd instead do `new HttpClient(); client.request(...)` 951 // set any properties here 952 953 // synchronous usage 954 auto reply = request.perform(); 955 956 // async usage, type 1: 957 request.send(); 958 request2.send(); 959 960 // wait until the first one is done, with the second one still in-flight 961 auto response = request.waitForCompletion(); 962 963 // async usage, type 2: 964 request.onDataReceived = (HttpRequest hr) { 965 if(hr.state == HttpRequest.State.complete) { 966 // use hr.responseData 967 } 968 }; 969 request.send(); // send, using the callback 970 971 // before terminating, be sure you wait for your requests to finish! 972 973 request.waitForCompletion(); 974 --- 975 */ 976 class HttpRequest { 977 978 /// Automatically follow a redirection? 979 bool followLocation = false; 980 981 /++ 982 Maximum number of redirections to follow (used only if [followLocation] is set to true). Will resolve with an error if a single request has more than this number of redirections. The default value is currently 10, but may change without notice. If you need a specific value, be sure to call this function. 983 984 If you want unlimited redirects, call it with `int.max`. If you set it to 0 but set [followLocation] to `true`, any attempt at redirection will abort the request. To disable automatically following redirection, set [followLocation] to `false` so you can process the 30x code yourself as a completed request. 985 986 History: 987 Added July 27, 2022 (dub v10.9) 988 +/ 989 void setMaximumNumberOfRedirects(int max = 10) { 990 maximumNumberOfRedirectsRemaining = max; 991 } 992 993 private int maximumNumberOfRedirectsRemaining; 994 995 /++ 996 Set to `true` to automatically retain cookies in the associated [HttpClient] from this request. 997 Note that you must have constructed the request from a `HttpClient` or at least passed one into the 998 constructor for this to have any effect. 999 1000 Bugs: 1001 See [HttpClient.retainCookies] for important caveats. 1002 1003 History: 1004 Added July 5, 2021 (dub v10.2) 1005 +/ 1006 bool retainCookies = false; 1007 1008 private HttpClient client; 1009 1010 this() { 1011 } 1012 1013 /// 1014 this(HttpClient client, Uri where, HttpVerb method, ICache cache = null, Duration timeout = 10.seconds, string proxy = null) { 1015 this.client = client; 1016 populateFromInfo(where, method); 1017 setTimeout(timeout); 1018 this.cache = cache; 1019 this.proxy = proxy; 1020 1021 setMaximumNumberOfRedirects(); 1022 } 1023 1024 1025 /// ditto 1026 this(Uri where, HttpVerb method, ICache cache = null, Duration timeout = 10.seconds, string proxy = null) { 1027 this(null, where, method, cache, timeout, proxy); 1028 } 1029 1030 /++ 1031 Sets the timeout from inactivity on the request. This is the amount of time that passes with no send or receive activity on the request before it fails with "request timed out" error. 1032 1033 History: 1034 Added March 31, 2021 1035 +/ 1036 void setTimeout(Duration timeout) { 1037 this.requestParameters.timeoutFromInactivity = timeout; 1038 this.timeoutFromInactivity = MonoTime.currTime + this.requestParameters.timeoutFromInactivity; 1039 } 1040 1041 private MonoTime timeoutFromInactivity; 1042 1043 private Uri where; 1044 1045 private ICache cache; 1046 1047 /++ 1048 Proxy to use for this request. It should be a URL or `null`. 1049 1050 This must be sent before you call [send]. 1051 1052 History: 1053 Added April 12, 2021 (dub v9.5) 1054 +/ 1055 string proxy; 1056 1057 /++ 1058 For https connections, if this is `true`, it will fail to connect if the TLS certificate can not be 1059 verified. Setting this to `false` will skip this check and allow the connection to continue anyway. 1060 1061 When the [HttpRequest] is constructed from a [HttpClient], it will inherit the value from the client 1062 instead of using the `= true` here. You can change this value any time before you call [send] (which 1063 is done implicitly if you call [waitForCompletion]). 1064 1065 History: 1066 Added April 5, 2022 (dub v10.8) 1067 1068 Prior to this, it always used the global (but undocumented) `defaultVerifyPeer` setting, and sometimes 1069 even if it was true, it would skip the verification. Now, it always respects this local setting. 1070 +/ 1071 bool verifyPeer = true; 1072 1073 1074 /// Final url after any redirections 1075 string finalUrl; 1076 1077 void populateFromInfo(Uri where, HttpVerb method) { 1078 auto parts = where.basedOn(this.where); 1079 this.where = parts; 1080 finalUrl = where.toString(); 1081 requestParameters.method = method; 1082 requestParameters.unixSocketPath = where.unixSocketPath; 1083 requestParameters.host = parts.host; 1084 requestParameters.port = cast(ushort) parts.effectivePort; 1085 requestParameters.ssl = parts.scheme == "https"; 1086 requestParameters.uri = parts.path.length ? parts.path : "/"; 1087 if(parts.query.length) { 1088 requestParameters.uri ~= "?"; 1089 requestParameters.uri ~= parts.query; 1090 } 1091 } 1092 1093 ~this() { 1094 } 1095 1096 ubyte[] sendBuffer; 1097 1098 HttpResponse responseData; 1099 private HttpClient parentClient; 1100 1101 size_t bodyBytesSent; 1102 size_t bodyBytesReceived; 1103 1104 State state_; 1105 State state() { return state_; } 1106 State state(State s) { 1107 assert(state_ != State.complete); 1108 return state_ = s; 1109 } 1110 /// Called when data is received. Check the state to see what data is available. 1111 void delegate(HttpRequest) onDataReceived; 1112 1113 enum State { 1114 /// The request has not yet been sent 1115 unsent, 1116 1117 /// The send() method has been called, but no data is 1118 /// sent on the socket yet because the connection is busy. 1119 pendingAvailableConnection, 1120 1121 /// connect has been called, but we're waiting on word of success 1122 connecting, 1123 1124 /// connecting a ssl, needing this 1125 sslConnectPendingRead, 1126 /// ditto 1127 sslConnectPendingWrite, 1128 1129 /// The headers are being sent now 1130 sendingHeaders, 1131 1132 // FIXME: allow Expect: 100-continue and separate the body send 1133 1134 /// The body is being sent now 1135 sendingBody, 1136 1137 /// The request has been sent but we haven't received any response yet 1138 waitingForResponse, 1139 1140 /// We have received some data and are currently receiving headers 1141 readingHeaders, 1142 1143 /// All headers are available but we're still waiting on the body 1144 readingBody, 1145 1146 /// The request is complete. 1147 complete, 1148 1149 /// The request is aborted, either by the abort() method, or as a result of the server disconnecting 1150 aborted 1151 } 1152 1153 /// Sends now and waits for the request to finish, returning the response. 1154 HttpResponse perform() { 1155 send(); 1156 return waitForCompletion(); 1157 } 1158 1159 /// Sends the request asynchronously. 1160 void send() { 1161 sendPrivate(true); 1162 } 1163 1164 private void sendPrivate(bool advance) { 1165 if(state != State.unsent && state != State.aborted) 1166 return; // already sent 1167 1168 if(cache !is null) { 1169 auto res = cache.getCachedResponse(this.requestParameters); 1170 if(res !is null) { 1171 state = State.complete; 1172 responseData = (*res).deepCopy(); 1173 return; 1174 } 1175 } 1176 1177 if(this.where.scheme == "data") { 1178 void error(string content) { 1179 responseData.code = 400; 1180 responseData.codeText = "Bad Request"; 1181 responseData.contentType = "text/plain"; 1182 responseData.content = cast(ubyte[]) content; 1183 responseData.contentText = content; 1184 state = State.complete; 1185 return; 1186 } 1187 1188 auto thing = this.where.path; 1189 // format is: type,data 1190 // type can have ;base64 1191 auto comma = thing.indexOf(","); 1192 if(comma == -1) 1193 return error("Invalid data uri, no comma found"); 1194 1195 auto type = thing[0 .. comma]; 1196 auto data = thing[comma + 1 .. $]; 1197 if(type.length == 0) 1198 type = "text/plain"; 1199 1200 import std.uri; 1201 auto bdata = cast(ubyte[]) decodeComponent(data); 1202 1203 if(type.indexOf(";base64") != -1) { 1204 import std.base64; 1205 try { 1206 bdata = Base64.decode(bdata); 1207 } catch(Exception e) { 1208 return error(e.msg); 1209 } 1210 } 1211 1212 responseData.code = 200; 1213 responseData.codeText = "OK"; 1214 responseData.contentType = type; 1215 responseData.content = bdata; 1216 responseData.contentText = cast(string) responseData.content; 1217 state = State.complete; 1218 return; 1219 } 1220 1221 string headers; 1222 1223 headers ~= to!string(requestParameters.method); 1224 headers ~= " "; 1225 if(proxy.length && !requestParameters.ssl) { 1226 // if we're doing a http proxy, we need to send a complete, absolute uri 1227 // so reconstruct it 1228 headers ~= "http://"; 1229 headers ~= requestParameters.host; 1230 if(requestParameters.port != 80) { 1231 headers ~= ":"; 1232 headers ~= to!string(requestParameters.port); 1233 } 1234 } 1235 1236 headers ~= requestParameters.uri; 1237 1238 if(requestParameters.useHttp11) 1239 headers ~= " HTTP/1.1\r\n"; 1240 else 1241 headers ~= " HTTP/1.0\r\n"; 1242 1243 // the whole authority section is supposed to be there, but curl doesn't send if default port 1244 // so I'll copy what they do 1245 headers ~= "Host: "; 1246 headers ~= requestParameters.host; 1247 if(requestParameters.port != 80 && requestParameters.port != 443) { 1248 headers ~= ":"; 1249 headers ~= to!string(requestParameters.port); 1250 } 1251 headers ~= "\r\n"; 1252 1253 bool specSaysRequestAlwaysHasBody = 1254 requestParameters.method == HttpVerb.POST || 1255 requestParameters.method == HttpVerb.PUT || 1256 requestParameters.method == HttpVerb.PATCH; 1257 1258 if(requestParameters.userAgent.length) 1259 headers ~= "User-Agent: "~requestParameters.userAgent~"\r\n"; 1260 if(requestParameters.contentType.length) 1261 headers ~= "Content-Type: "~requestParameters.contentType~"\r\n"; 1262 if(requestParameters.authorization.length) 1263 headers ~= "Authorization: "~requestParameters.authorization~"\r\n"; 1264 if(requestParameters.bodyData.length || specSaysRequestAlwaysHasBody) 1265 headers ~= "Content-Length: "~to!string(requestParameters.bodyData.length)~"\r\n"; 1266 if(requestParameters.acceptGzip) 1267 headers ~= "Accept-Encoding: gzip\r\n"; 1268 if(requestParameters.keepAlive) 1269 headers ~= "Connection: keep-alive\r\n"; 1270 1271 string cookieHeader; 1272 foreach(name, value; requestParameters.cookies) { 1273 if(cookieHeader is null) 1274 cookieHeader = "Cookie: "; 1275 else 1276 cookieHeader ~= "; "; 1277 cookieHeader ~= name; 1278 cookieHeader ~= "="; 1279 cookieHeader ~= value; 1280 } 1281 1282 if(cookieHeader !is null) { 1283 cookieHeader ~= "\r\n"; 1284 headers ~= cookieHeader; 1285 } 1286 1287 foreach(header; requestParameters.headers) 1288 headers ~= header ~ "\r\n"; 1289 1290 headers ~= "\r\n"; 1291 1292 // FIXME: separate this for 100 continue 1293 sendBuffer = cast(ubyte[]) headers ~ requestParameters.bodyData; 1294 1295 // import std.stdio; writeln("******* ", cast(string) sendBuffer); 1296 1297 responseData = HttpResponse.init; 1298 responseData.requestParameters = requestParameters; 1299 bodyBytesSent = 0; 1300 bodyBytesReceived = 0; 1301 state = State.pendingAvailableConnection; 1302 1303 bool alreadyPending = false; 1304 foreach(req; pending) 1305 if(req is this) { 1306 alreadyPending = true; 1307 break; 1308 } 1309 if(!alreadyPending) { 1310 pending ~= this; 1311 } 1312 1313 if(advance) 1314 HttpRequest.advanceConnections(requestParameters.timeoutFromInactivity); 1315 } 1316 1317 1318 /// Waits for the request to finish or timeout, whichever comes first. 1319 HttpResponse waitForCompletion() { 1320 while(state != State.aborted && state != State.complete) { 1321 if(state == State.unsent) { 1322 send(); 1323 continue; 1324 } 1325 if(auto err = HttpRequest.advanceConnections(requestParameters.timeoutFromInactivity)) { 1326 switch(err) { 1327 case 1: throw new Exception("HttpRequest.advanceConnections returned 1: all connections timed out"); 1328 case 2: throw new Exception("HttpRequest.advanceConnections returned 2: nothing to do"); 1329 case 3: continue; // EINTR 1330 default: throw new Exception("HttpRequest.advanceConnections got err " ~ to!string(err)); 1331 } 1332 } 1333 } 1334 1335 if(state == State.complete && responseData.code >= 200) 1336 if(cache !is null) 1337 cache.cacheResponse(this.requestParameters, this.responseData); 1338 1339 return responseData; 1340 } 1341 1342 /// Aborts this request. 1343 void abort() { 1344 this.state = State.aborted; 1345 this.responseData.code = 1; 1346 this.responseData.codeText = "request.abort called"; 1347 // the actual cancellation happens in the event loop 1348 } 1349 1350 HttpRequestParameters requestParameters; /// 1351 1352 version(arsd_http_winhttp_implementation) { 1353 public static void resetInternals() { 1354 1355 } 1356 1357 static assert(0, "implementation not finished"); 1358 } 1359 1360 1361 version(arsd_http_internal_implementation) { 1362 1363 /++ 1364 Changes the limit of number of open, inactive sockets. Reusing connections can provide a significant 1365 performance improvement, but the operating system can also impose a global limit on the number of open 1366 sockets and/or files that you don't want to run into. This lets you choose a balance right for you. 1367 1368 1369 When the total number of cached, inactive sockets approaches this maximum, it will check for ones closed by the 1370 server first. If there are none already closed by the server, it will select sockets at random from its connection 1371 cache and close them to make room for the new ones. 1372 1373 Please note: 1374 1375 $(LIST 1376 * there is always a limit of six open sockets per domain, per the common practice suggested by the http standard 1377 * the limit given here is thread-local. If you run multiple http clients/requests from multiple threads, don't set this too high or you might bump into the global limit from the OS. 1378 * setting this too low can waste connections because the server might close them, but they will never be garbage collected since my current implementation won't check for dead connections except when it thinks it is running close to the limit. 1379 ) 1380 1381 Setting it just right for your use case may provide an up to 10x performance boost. 1382 1383 This implementation is subject to change. If it does, I'll document it, but may not bump the version number. 1384 1385 History: 1386 Added August 10, 2022 (dub v10.9) 1387 +/ 1388 static void setConnectionCacheSize(int max = 32) { 1389 connectionCacheSize = max; 1390 } 1391 1392 private static { 1393 // we manage the actual connections. When a request is made on a particular 1394 // host, we try to reuse connections. We may open more than one connection per 1395 // host to do parallel requests. 1396 // 1397 // The key is the *domain name* and the port. Multiple domains on the same address will have separate connections. 1398 Socket[][string] socketsPerHost; 1399 1400 // only one request can be active on a given socket (at least HTTP < 2.0) so this is that 1401 HttpRequest[Socket] activeRequestOnSocket; 1402 HttpRequest[] pending; // and these are the requests that are waiting 1403 1404 int cachedSockets; 1405 int connectionCacheSize = 32; 1406 1407 /+ 1408 This is a somewhat expensive, but essential operation. If it isn't used in a heavy 1409 application, you'll risk running out of file descriptors. 1410 +/ 1411 void cleanOldSockets() { 1412 static struct CloseCandidate { 1413 string key; 1414 Socket socket; 1415 } 1416 1417 CloseCandidate[36] closeCandidates; 1418 int closeCandidatesPosition; 1419 1420 outer: foreach(key, sockets; socketsPerHost) { 1421 foreach(socket; sockets) { 1422 if(socket in activeRequestOnSocket) 1423 continue; // it is still in use; we can't close it 1424 1425 closeCandidates[closeCandidatesPosition++] = CloseCandidate(key, socket); 1426 if(closeCandidatesPosition == closeCandidates.length) 1427 break outer; 1428 } 1429 } 1430 1431 auto cc = closeCandidates[0 .. closeCandidatesPosition]; 1432 1433 if(cc.length == 0) 1434 return; // no candidates to even examine 1435 1436 // has the server closed any of these? if so, we also close and drop them 1437 static SocketSet readSet = null; 1438 if(readSet is null) 1439 readSet = new SocketSet(); 1440 readSet.reset(); 1441 1442 foreach(candidate; cc) { 1443 readSet.add(candidate.socket); 1444 } 1445 1446 int closeCount; 1447 1448 auto got = Socket.select(readSet, null, null, 0.msecs /* timeout, want it small since we just checking for eof */); 1449 if(got > 0) { 1450 foreach(ref candidate; cc) { 1451 if(readSet.isSet(candidate.socket)) { 1452 // if we can read when it isn't in use, that means eof; the 1453 // server closed it. 1454 candidate.socket.close(); 1455 loseSocketByKey(candidate.key, candidate.socket); 1456 closeCount++; 1457 } 1458 } 1459 debug(arsd_http2) writeln(closeCount, " from inactivity"); 1460 } else { 1461 // and if not, of the remaining ones, close a few just at random to bring us back beneath the arbitrary limit. 1462 1463 while(cc.length > 0 && (cachedSockets - closeCount) > connectionCacheSize) { 1464 import std.random; 1465 auto idx = uniform(0, cc.length); 1466 1467 cc[idx].socket.close(); 1468 loseSocketByKey(cc[idx].key, cc[idx].socket); 1469 1470 cc[idx] = cc[$ - 1]; 1471 cc = cc[0 .. $-1]; 1472 closeCount++; 1473 } 1474 debug(arsd_http2) writeln(closeCount, " from randomness"); 1475 } 1476 1477 cachedSockets -= closeCount; 1478 } 1479 1480 void loseSocketByKey(string key, Socket s) { 1481 if(auto list = key in socketsPerHost) { 1482 for(int a = 0; a < (*list).length; a++) { 1483 if((*list)[a] is s) { 1484 1485 for(int b = a; b < (*list).length - 1; b++) 1486 (*list)[b] = (*list)[b+1]; 1487 (*list) = (*list)[0 .. $-1]; 1488 break; 1489 } 1490 } 1491 } 1492 } 1493 1494 void loseSocket(string host, ushort port, bool ssl, Socket s) { 1495 import std.string; 1496 auto key = format("http%s://%s:%s", ssl ? "s" : "", host, port); 1497 1498 loseSocketByKey(key, s); 1499 } 1500 1501 Socket getOpenSocketOnHost(string proxy, string host, ushort port, bool ssl, string unixSocketPath, bool verifyPeer) { 1502 Socket openNewConnection() { 1503 Socket socket; 1504 if(ssl) { 1505 version(with_openssl) { 1506 loadOpenSsl(); 1507 socket = new SslClientSocket(family(unixSocketPath), SocketType.STREAM, host, verifyPeer); 1508 socket.blocking = false; 1509 } else 1510 throw new Exception("SSL not compiled in"); 1511 } else { 1512 socket = new Socket(family(unixSocketPath), SocketType.STREAM); 1513 socket.blocking = false; 1514 } 1515 1516 socket.setOption(SocketOptionLevel.TCP, SocketOption.TCP_NODELAY, 1); 1517 1518 // FIXME: connect timeout? 1519 if(unixSocketPath) { 1520 import std.stdio; writeln(cast(ubyte[]) unixSocketPath); 1521 socket.connect(new UnixAddress(unixSocketPath)); 1522 } else { 1523 // FIXME: i should prolly do ipv6 if available too. 1524 if(host.length == 0) // this could arguably also be an in contract since it is user error, but the exception is good enough 1525 throw new Exception("No host given for request"); 1526 if(proxy.length) { 1527 if(proxy.indexOf("//") == -1) 1528 proxy = "http://" ~ proxy; 1529 auto proxyurl = Uri(proxy); 1530 1531 //auto proxyhttps = proxyurl.scheme == "https"; 1532 enum proxyhttps = false; // this isn't properly implemented and might never be necessary anyway so meh 1533 1534 // the precise types here are important to help with overload 1535 // resolution of the devirtualized call! 1536 Address pa = new InternetAddress(proxyurl.host, proxyurl.port ? cast(ushort) proxyurl.port : 80); 1537 1538 debug(arsd_http2) writeln("using proxy ", pa.toString()); 1539 1540 if(proxyhttps) { 1541 socket.connect(pa); 1542 } else { 1543 // the proxy never actually starts TLS, but if the request is tls then we need to CONNECT then upgrade the connection 1544 // using the parent class functions let us bypass the encryption 1545 socket.Socket.connect(pa); 1546 } 1547 1548 socket.blocking = true; // FIXME total hack to simplify the code here since it isn't really using the event loop yet 1549 1550 string message; 1551 if(ssl) { 1552 auto hostName = host ~ ":" ~ to!string(port); 1553 message = "CONNECT " ~ hostName ~ " HTTP/1.1\r\n"; 1554 message ~= "Host: " ~ hostName ~ "\r\n"; 1555 if(proxyurl.userinfo.length) { 1556 import std.base64; 1557 message ~= "Proxy-Authorization: Basic " ~ Base64.encode(cast(ubyte[]) proxyurl.userinfo) ~ "\r\n"; 1558 } 1559 message ~= "\r\n"; 1560 1561 // FIXME: what if proxy times out? should be reasonably fast too. 1562 if(proxyhttps) { 1563 socket.send(message, SocketFlags.NONE); 1564 } else { 1565 socket.Socket.send(message, SocketFlags.NONE); 1566 } 1567 1568 ubyte[1024] recvBuffer; 1569 // and last time 1570 ptrdiff_t rcvGot; 1571 if(proxyhttps) { 1572 rcvGot = socket.receive(recvBuffer[], SocketFlags.NONE); 1573 // bool verifyPeer = true; 1574 //(cast(OpenSslSocket)socket).freeSsl(); 1575 //(cast(OpenSslSocket)socket).initSsl(verifyPeer, host); 1576 } else { 1577 rcvGot = socket.Socket.receive(recvBuffer[], SocketFlags.NONE); 1578 } 1579 1580 if(rcvGot == -1) 1581 throw new ProxyException("proxy receive error"); 1582 auto got = cast(string) recvBuffer[0 .. rcvGot]; 1583 auto expect = "HTTP/1.1 200"; 1584 if(got.length < expect.length || (got[0 .. expect.length] != expect && got[0 .. expect.length] != "HTTP/1.0 200")) 1585 throw new ProxyException("Proxy rejected request: " ~ got[0 .. expect.length <= got.length ? expect.length : got.length]); 1586 1587 if(proxyhttps) { 1588 //(cast(OpenSslSocket)socket).do_ssl_connect(); 1589 } else { 1590 (cast(OpenSslSocket)socket).do_ssl_connect(); 1591 } 1592 } else { 1593 } 1594 } else { 1595 socket.connect(new InternetAddress(host, port)); 1596 } 1597 } 1598 1599 debug(arsd_http2) writeln("opening to ", host, ":", port, " ", cast(void*) socket, " ssl=", ssl); 1600 assert(socket.handle() !is socket_t.init); 1601 return socket; 1602 } 1603 1604 // import std.stdio; writeln(cachedSockets); 1605 if(cachedSockets > connectionCacheSize) 1606 cleanOldSockets(); 1607 1608 import std.string; 1609 auto key = format("http%s://%s:%s", ssl ? "s" : "", host, port); 1610 1611 if(auto hostListing = key in socketsPerHost) { 1612 // try to find an available socket that is already open 1613 foreach(socket; *hostListing) { 1614 if(socket !in activeRequestOnSocket) { 1615 // let's see if it has closed since we last tried 1616 // e.g. a server timeout or something. If so, we need 1617 // to lose this one and immediately open a new one. 1618 static SocketSet readSet = null; 1619 if(readSet is null) 1620 readSet = new SocketSet(); 1621 readSet.reset(); 1622 assert(socket !is null); 1623 assert(socket.handle() !is socket_t.init, socket is null ? "null" : socket.toString()); 1624 readSet.add(socket); 1625 auto got = Socket.select(readSet, null, null, 0.msecs /* timeout, want it small since we just checking for eof */); 1626 if(got > 0) { 1627 // we can read something off this... but there aren't 1628 // any active requests. Assume it is EOF and open a new one 1629 1630 socket.close(); 1631 loseSocket(host, port, ssl, socket); 1632 goto openNew; 1633 } 1634 cachedSockets--; 1635 return socket; 1636 } 1637 } 1638 1639 // if not too many already open, go ahead and do a new one 1640 if((*hostListing).length < 6) { 1641 auto socket = openNewConnection(); 1642 (*hostListing) ~= socket; 1643 return socket; 1644 } else 1645 return null; // too many, you'll have to wait 1646 } 1647 1648 openNew: 1649 1650 auto socket = openNewConnection(); 1651 socketsPerHost[key] ~= socket; 1652 return socket; 1653 } 1654 1655 SocketSet readSet; 1656 SocketSet writeSet; 1657 1658 /+ 1659 Generic event loop registration: 1660 1661 handle, operation (read/write), buffer (on posix it *might* be stack if a select loop), timeout (in real time), callback when op completed. 1662 1663 ....basically Windows style. Then it translates internally. 1664 1665 It should tell the thing if the buffer is reused or not 1666 +/ 1667 1668 1669 /++ 1670 This is made public for rudimentary event loop integration, but is still 1671 basically an internal detail. Try not to use it if you have another way. 1672 1673 This does a single iteration of the internal select()-based processing loop. 1674 1675 1676 Future directions: 1677 I want to merge the internal use of [WebSocket.eventLoop] with this; 1678 [advanceConnections] does just one run on the loop, whereas eventLoop 1679 runs it until all connections are closed. But they'd both process both 1680 pending http requests and active websockets. 1681 1682 After that, I want to be able to integrate in other event loops too. 1683 One might be to simply to reactor callbacks, then perhaps Windows overlapped 1684 i/o (that's just going to be tricky to retrofit into the existing select()-based 1685 code). It could then go fiber just by calling the resume function too. 1686 1687 The hard part is ensuring I keep this file stand-alone while offering these 1688 things. 1689 1690 This `advanceConnections` call will probably continue to work now that it is 1691 public, but it may not be wholly compatible with all the future features; you'd 1692 have to pick either the internal event loop or an external one you integrate, but not 1693 mix them. 1694 1695 History: 1696 This has been included in the library since almost day one, but 1697 it was private until April 13, 2021 (dub v9.5). 1698 1699 Params: 1700 maximumTimeout = the maximum time it will wait in select(). It may return much sooner than this if a connection timed out in the mean time. 1701 automaticallyRetryOnInterruption = internally loop on EINTR. 1702 1703 Returns: 1704 1705 0 = no error, work may remain so you should call `advanceConnections` again when you can 1706 1707 1 = passed `maximumTimeout` reached with no work done, yet requests are still in the queue. You may call `advanceConnections` again. 1708 1709 2 = no work to do, no point calling it again unless you've added new requests. Your program may exit if you have nothing to add since it means everything requested is now done. 1710 1711 3 = EINTR occurred on select(), you should check your interrupt flags if you set a signal handler, then call `advanceConnections` again if you aren't exiting. Only occurs if `automaticallyRetryOnInterruption` is set to `false` (the default when it is called externally). 1712 1713 any other value should be considered a non-recoverable error if you want to be forward compatible as I reserve the right to add more values later. 1714 +/ 1715 public int advanceConnections(Duration maximumTimeout = 10.seconds, bool automaticallyRetryOnInterruption = false) { 1716 debug(arsd_http2_verbose) writeln("advancing"); 1717 if(readSet is null) 1718 readSet = new SocketSet(); 1719 if(writeSet is null) 1720 writeSet = new SocketSet(); 1721 1722 ubyte[2048] buffer; 1723 1724 HttpRequest[16] removeFromPending; 1725 size_t removeFromPendingCount = 0; 1726 1727 bool hadAbortedRequest; 1728 1729 // are there pending requests? let's try to send them 1730 foreach(idx, pc; pending) { 1731 if(removeFromPendingCount == removeFromPending.length) 1732 break; 1733 1734 if(pc.state == HttpRequest.State.aborted) { 1735 removeFromPending[removeFromPendingCount++] = pc; 1736 hadAbortedRequest = true; 1737 continue; 1738 } 1739 1740 Socket socket; 1741 1742 try { 1743 socket = getOpenSocketOnHost(pc.proxy, pc.requestParameters.host, pc.requestParameters.port, pc.requestParameters.ssl, pc.requestParameters.unixSocketPath, pc.verifyPeer); 1744 } catch(ProxyException e) { 1745 // connection refused or timed out (I should disambiguate somehow)... 1746 pc.state = HttpRequest.State.aborted; 1747 1748 pc.responseData.code = 2; 1749 pc.responseData.codeText = e.msg ~ " from " ~ pc.proxy; 1750 1751 hadAbortedRequest = true; 1752 1753 removeFromPending[removeFromPendingCount++] = pc; 1754 continue; 1755 1756 } catch(SocketException e) { 1757 // connection refused or timed out (I should disambiguate somehow)... 1758 pc.state = HttpRequest.State.aborted; 1759 1760 pc.responseData.code = 2; 1761 pc.responseData.codeText = pc.proxy.length ? ("connection failed to proxy " ~ pc.proxy) : "connection failed"; 1762 1763 hadAbortedRequest = true; 1764 1765 removeFromPending[removeFromPendingCount++] = pc; 1766 continue; 1767 } catch(Exception e) { 1768 // connection failed due to other user error or SSL (i should disambiguate somehow)... 1769 pc.state = HttpRequest.State.aborted; 1770 1771 pc.responseData.code = 2; 1772 pc.responseData.codeText = e.msg; 1773 1774 hadAbortedRequest = true; 1775 1776 removeFromPending[removeFromPendingCount++] = pc; 1777 continue; 1778 1779 } 1780 1781 if(socket !is null) { 1782 activeRequestOnSocket[socket] = pc; 1783 assert(pc.sendBuffer.length); 1784 pc.state = State.connecting; 1785 1786 removeFromPending[removeFromPendingCount++] = pc; 1787 } 1788 } 1789 1790 import std.algorithm : remove; 1791 foreach(rp; removeFromPending[0 .. removeFromPendingCount]) 1792 pending = pending.remove!((a) => a is rp)(); 1793 1794 tryAgain: 1795 1796 Socket[16] inactive; 1797 int inactiveCount = 0; 1798 void killInactives() { 1799 foreach(s; inactive[0 .. inactiveCount]) { 1800 debug(arsd_http2) writeln("removing socket from active list ", cast(void*) s); 1801 activeRequestOnSocket.remove(s); 1802 cachedSockets++; 1803 } 1804 } 1805 1806 1807 readSet.reset(); 1808 writeSet.reset(); 1809 1810 bool hadOne = false; 1811 1812 auto minTimeout = maximumTimeout; 1813 auto now = MonoTime.currTime; 1814 1815 // active requests need to be read or written to 1816 foreach(sock, request; activeRequestOnSocket) { 1817 1818 if(request.state == State.aborted) { 1819 inactive[inactiveCount++] = sock; 1820 sock.close(); 1821 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1822 hadAbortedRequest = true; 1823 continue; 1824 } 1825 1826 // check the other sockets just for EOF, if they close, take them out of our list, 1827 // we'll reopen if needed upon request. 1828 readSet.add(sock); 1829 hadOne = true; 1830 1831 Duration timeo; 1832 if(request.timeoutFromInactivity <= now) 1833 timeo = 0.seconds; 1834 else 1835 timeo = request.timeoutFromInactivity - now; 1836 1837 if(timeo < minTimeout) 1838 minTimeout = timeo; 1839 1840 if(request.state == State.connecting || request.state == State.sslConnectPendingWrite || request.state == State.sendingHeaders || request.state == State.sendingBody) { 1841 writeSet.add(sock); 1842 hadOne = true; 1843 } 1844 } 1845 1846 if(!hadOne) { 1847 if(hadAbortedRequest) { 1848 killInactives(); 1849 return 0; // something got aborted, that's progress 1850 } 1851 return 2; // automatic timeout, nothing to do 1852 } 1853 1854 auto selectGot = Socket.select(readSet, writeSet, null, minTimeout); 1855 if(selectGot == 0) { /* timeout */ 1856 now = MonoTime.currTime; 1857 bool anyWorkDone = false; 1858 foreach(sock, request; activeRequestOnSocket) { 1859 1860 if(request.timeoutFromInactivity <= now) { 1861 request.state = HttpRequest.State.aborted; 1862 request.responseData.code = 5; 1863 if(request.state == State.connecting) 1864 request.responseData.codeText = "Connect timed out"; 1865 else 1866 request.responseData.codeText = "Request timed out"; 1867 1868 inactive[inactiveCount++] = sock; 1869 sock.close(); 1870 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1871 anyWorkDone = true; 1872 } 1873 } 1874 killInactives(); 1875 return anyWorkDone ? 0 : 1; 1876 // return 1; was an error to time out but now im making it on the individual request 1877 } else if(selectGot == -1) { /* interrupted */ 1878 /* 1879 version(Posix) { 1880 import core.stdc.errno; 1881 if(errno != EINTR) 1882 throw new Exception("select error: " ~ to!string(errno)); 1883 } 1884 */ 1885 if(automaticallyRetryOnInterruption) 1886 goto tryAgain; 1887 else 1888 return 3; 1889 } else { /* ready */ 1890 1891 void sslProceed(HttpRequest request, SslClientSocket s) { 1892 try { 1893 auto code = s.do_ssl_connect(); 1894 switch(code) { 1895 case 0: 1896 request.state = State.sendingHeaders; 1897 break; 1898 case SSL_ERROR_WANT_READ: 1899 request.state = State.sslConnectPendingRead; 1900 break; 1901 case SSL_ERROR_WANT_WRITE: 1902 request.state = State.sslConnectPendingWrite; 1903 break; 1904 default: 1905 assert(0); 1906 } 1907 } catch(Exception e) { 1908 request.state = State.aborted; 1909 1910 request.responseData.code = 2; 1911 request.responseData.codeText = e.msg; 1912 inactive[inactiveCount++] = s; 1913 s.close(); 1914 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, s); 1915 } 1916 } 1917 1918 1919 foreach(sock, request; activeRequestOnSocket) { 1920 // always need to try to send first in part because http works that way but 1921 // also because openssl will sometimes leave something ready to read even if we haven't 1922 // sent yet (probably leftover data from the crypto negotiation) and if that happens ssl 1923 // is liable to block forever hogging the connection and not letting it send... 1924 if(request.state == State.connecting) 1925 if(writeSet.isSet(sock) || readSet.isSet(sock)) { 1926 import core.stdc.stdint; 1927 int32_t error; 1928 int retopt = sock.getOption(SocketOptionLevel.SOCKET, SocketOption.ERROR, error); 1929 if(retopt < 0 || error != 0) { 1930 request.state = State.aborted; 1931 1932 request.responseData.code = 2; 1933 try { 1934 request.responseData.codeText = "connection failed - " ~ formatSocketError(error); 1935 } catch(Exception e) { 1936 request.responseData.codeText = "connection failed"; 1937 } 1938 inactive[inactiveCount++] = sock; 1939 sock.close(); 1940 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1941 continue; 1942 } else { 1943 if(auto s = cast(SslClientSocket) sock) { 1944 sslProceed(request, s); 1945 continue; 1946 } else { 1947 request.state = State.sendingHeaders; 1948 } 1949 } 1950 } 1951 1952 if(request.state == State.sslConnectPendingRead) 1953 if(readSet.isSet(sock)) { 1954 sslProceed(request, cast(SslClientSocket) sock); 1955 continue; 1956 } 1957 if(request.state == State.sslConnectPendingWrite) 1958 if(writeSet.isSet(sock)) { 1959 sslProceed(request, cast(SslClientSocket) sock); 1960 continue; 1961 } 1962 1963 if(request.state == State.sendingHeaders || request.state == State.sendingBody) 1964 if(writeSet.isSet(sock)) { 1965 request.timeoutFromInactivity = MonoTime.currTime + request.requestParameters.timeoutFromInactivity; 1966 assert(request.sendBuffer.length); 1967 auto sent = sock.send(request.sendBuffer); 1968 debug(arsd_http2_verbose) writeln(cast(void*) sock, "<send>", cast(string) request.sendBuffer, "</send>"); 1969 if(sent <= 0) { 1970 if(wouldHaveBlocked()) 1971 continue; 1972 1973 request.state = State.aborted; 1974 1975 request.responseData.code = 3; 1976 request.responseData.codeText = "send failed to server"; 1977 inactive[inactiveCount++] = sock; 1978 sock.close(); 1979 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1980 continue; 1981 1982 } 1983 request.sendBuffer = request.sendBuffer[sent .. $]; 1984 if(request.sendBuffer.length == 0) { 1985 request.state = State.waitingForResponse; 1986 1987 debug(arsd_http2_verbose) writeln("all sent"); 1988 } 1989 } 1990 1991 1992 if(readSet.isSet(sock)) { 1993 keep_going: 1994 request.timeoutFromInactivity = MonoTime.currTime + request.requestParameters.timeoutFromInactivity; 1995 auto got = sock.receive(buffer); 1996 debug(arsd_http2_verbose) { if(got < 0) writeln(lastSocketError); else writeln("====PACKET ",got,"=====",cast(string)buffer[0 .. got],"===/PACKET==="); } 1997 if(got < 0) { 1998 if(wouldHaveBlocked()) 1999 continue; 2000 debug(arsd_http2) writeln("receive error"); 2001 if(request.state != State.complete) { 2002 request.state = State.aborted; 2003 2004 request.responseData.code = 3; 2005 request.responseData.codeText = "receive error from server"; 2006 } 2007 inactive[inactiveCount++] = sock; 2008 sock.close(); 2009 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 2010 } else if(got == 0) { 2011 // remote side disconnected 2012 debug(arsd_http2) writeln("remote disconnect"); 2013 if(request.state != State.complete) { 2014 request.state = State.aborted; 2015 2016 request.responseData.code = 3; 2017 request.responseData.codeText = "server disconnected"; 2018 } 2019 inactive[inactiveCount++] = sock; 2020 sock.close(); 2021 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 2022 } else { 2023 // data available 2024 bool stillAlive; 2025 2026 try { 2027 stillAlive = request.handleIncomingData(buffer[0 .. got]); 2028 /+ 2029 state needs to be set and public 2030 requestData.content/contentText needs to be around 2031 you need to be able to clear the content and keep processing for things like event sources. 2032 also need to be able to abort it. 2033 2034 and btw it should prolly just have evnet source as a pre-packaged thing. 2035 +/ 2036 } catch (Exception e) { 2037 debug(arsd_http2_verbose) { import std.stdio; writeln(e); } 2038 request.state = HttpRequest.State.aborted; 2039 request.responseData.code = 4; 2040 request.responseData.codeText = e.msg; 2041 2042 inactive[inactiveCount++] = sock; 2043 sock.close(); 2044 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 2045 } 2046 2047 if(!stillAlive || request.state == HttpRequest.State.complete || request.state == HttpRequest.State.aborted) { 2048 //import std.stdio; writeln(cast(void*) sock, " ", stillAlive, " ", request.state); 2049 inactive[inactiveCount++] = sock; 2050 // reuse the socket for another pending request, if we can 2051 } 2052 } 2053 2054 if(request.onDataReceived) 2055 request.onDataReceived(request); 2056 2057 version(with_openssl) 2058 if(auto s = cast(SslClientSocket) sock) { 2059 // select doesn't handle the case with stuff 2060 // left in the ssl buffer so i'm checking it separately 2061 if(s.dataPending()) { 2062 goto keep_going; 2063 } 2064 } 2065 } 2066 } 2067 } 2068 2069 killInactives(); 2070 2071 // we've completed a request, are there any more pending connection? if so, send them now 2072 2073 return 0; 2074 } 2075 } 2076 2077 public static void resetInternals() { 2078 socketsPerHost = null; 2079 activeRequestOnSocket = null; 2080 pending = null; 2081 2082 } 2083 2084 struct HeaderReadingState { 2085 bool justSawLf; 2086 bool justSawCr; 2087 bool atStartOfLine = true; 2088 bool readingLineContinuation; 2089 } 2090 HeaderReadingState headerReadingState; 2091 2092 struct BodyReadingState { 2093 bool isGzipped; 2094 bool isDeflated; 2095 2096 bool isChunked; 2097 int chunkedState; 2098 2099 // used for the chunk size if it is chunked 2100 int contentLengthRemaining; 2101 } 2102 BodyReadingState bodyReadingState; 2103 2104 bool closeSocketWhenComplete; 2105 2106 import std.zlib; 2107 UnCompress uncompress; 2108 2109 const(ubyte)[] leftoverDataFromLastTime; 2110 2111 bool handleIncomingData(scope const ubyte[] dataIn) { 2112 bool stillAlive = true; 2113 debug(arsd_http2) writeln("handleIncomingData, state: ", state); 2114 if(state == State.waitingForResponse) { 2115 state = State.readingHeaders; 2116 headerReadingState = HeaderReadingState.init; 2117 bodyReadingState = BodyReadingState.init; 2118 } 2119 2120 const(ubyte)[] data; 2121 if(leftoverDataFromLastTime.length) 2122 data = leftoverDataFromLastTime ~ dataIn[]; 2123 else 2124 data = dataIn[]; 2125 2126 if(state == State.readingHeaders) { 2127 void parseLastHeader() { 2128 assert(responseData.headers.length); 2129 if(responseData.headers.length == 1) { 2130 responseData.statusLine = responseData.headers[0]; 2131 import std.algorithm; 2132 auto parts = responseData.statusLine.splitter(" "); 2133 responseData.httpVersion = parts.front; 2134 parts.popFront(); 2135 if(parts.empty) 2136 throw new Exception("Corrupted response, bad status line"); 2137 responseData.code = to!int(parts.front()); 2138 parts.popFront(); 2139 responseData.codeText = ""; 2140 while(!parts.empty) { 2141 // FIXME: this sucks! 2142 responseData.codeText ~= parts.front(); 2143 parts.popFront(); 2144 if(!parts.empty) 2145 responseData.codeText ~= " "; 2146 } 2147 } else { 2148 // parse the new header 2149 auto header = responseData.headers[$-1]; 2150 2151 auto colon = header.indexOf(":"); 2152 if(colon < 0 || colon >= header.length) 2153 return; 2154 auto name = toLower(header[0 .. colon]); 2155 auto value = header[colon + 1 .. $].strip; // skip colon and strip whitespace 2156 2157 switch(name) { 2158 case "connection": 2159 if(value == "close") 2160 closeSocketWhenComplete = true; 2161 break; 2162 case "content-type": 2163 responseData.contentType = value; 2164 break; 2165 case "location": 2166 responseData.location = value; 2167 break; 2168 case "content-length": 2169 bodyReadingState.contentLengthRemaining = to!int(value); 2170 break; 2171 case "transfer-encoding": 2172 // note that if it is gzipped, it zips first, then chunks the compressed stream. 2173 // so we should always dechunk first, then feed into the decompressor 2174 if(value == "chunked") 2175 bodyReadingState.isChunked = true; 2176 else throw new Exception("Unknown Transfer-Encoding: " ~ value); 2177 break; 2178 case "content-encoding": 2179 if(value == "gzip") { 2180 bodyReadingState.isGzipped = true; 2181 uncompress = new UnCompress(); 2182 } else if(value == "deflate") { 2183 bodyReadingState.isDeflated = true; 2184 uncompress = new UnCompress(); 2185 } else throw new Exception("Unknown Content-Encoding: " ~ value); 2186 break; 2187 case "set-cookie": 2188 // handled elsewhere fyi 2189 break; 2190 default: 2191 // ignore 2192 } 2193 2194 responseData.headersHash[name] = value; 2195 } 2196 } 2197 2198 size_t position = 0; 2199 for(position = 0; position < data.length; position++) { 2200 if(headerReadingState.readingLineContinuation) { 2201 if(data[position] == ' ' || data[position] == '\t') 2202 continue; 2203 headerReadingState.readingLineContinuation = false; 2204 } 2205 2206 if(headerReadingState.atStartOfLine) { 2207 headerReadingState.atStartOfLine = false; 2208 // FIXME it being \r should never happen... and i don't think it does 2209 if(data[position] == '\r' || data[position] == '\n') { 2210 // done with headers 2211 2212 position++; // skip the \r 2213 2214 if(responseData.headers.length) 2215 parseLastHeader(); 2216 2217 if(responseData.code >= 100 && responseData.code < 200) { 2218 // "100 Continue" - we should continue uploading request data at this point 2219 // "101 Switching Protocols" - websocket, not expected here... 2220 // "102 Processing" - server still working, keep the connection alive 2221 // "103 Early Hints" - can have useful Link headers etc 2222 // 2223 // and other unrecognized ones can just safely be skipped 2224 2225 // FIXME: the headers shouldn't actually be reset; 103 Early Hints 2226 // can give useful headers we want to keep 2227 2228 responseData.headers = null; 2229 headerReadingState.atStartOfLine = true; 2230 2231 continue; // the \n will be skipped by the for loop advance 2232 } 2233 2234 if(this.requestParameters.method == HttpVerb.HEAD) 2235 state = State.complete; 2236 else 2237 state = State.readingBody; 2238 2239 // skip the \n before we break 2240 position++; 2241 2242 break; 2243 } else if(data[position] == ' ' || data[position] == '\t') { 2244 // line continuation, ignore all whitespace and collapse it into a space 2245 headerReadingState.readingLineContinuation = true; 2246 responseData.headers[$-1] ~= ' '; 2247 } else { 2248 // new header 2249 if(responseData.headers.length) 2250 parseLastHeader(); 2251 responseData.headers ~= ""; 2252 } 2253 } 2254 2255 if(data[position] == '\r') { 2256 headerReadingState.justSawCr = true; 2257 continue; 2258 } else 2259 headerReadingState.justSawCr = false; 2260 2261 if(data[position] == '\n') { 2262 headerReadingState.justSawLf = true; 2263 headerReadingState.atStartOfLine = true; 2264 continue; 2265 } else 2266 headerReadingState.justSawLf = false; 2267 2268 responseData.headers[$-1] ~= data[position]; 2269 } 2270 2271 data = data[position .. $]; 2272 } 2273 2274 if(state == State.readingBody) { 2275 if(bodyReadingState.isChunked) { 2276 // read the hex length, stopping at a \r\n, ignoring everything between the new line but after the first non-valid hex character 2277 // read binary data of that length. it is our content 2278 // repeat until a zero sized chunk 2279 // then read footers as headers. 2280 2281 start_over: 2282 for(int a = 0; a < data.length; a++) { 2283 final switch(bodyReadingState.chunkedState) { 2284 case 0: // reading hex 2285 char c = data[a]; 2286 if((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) { 2287 // just keep reading 2288 } else { 2289 int power = 1; 2290 bodyReadingState.contentLengthRemaining = 0; 2291 if(a == 0) 2292 break; // just wait for more data 2293 assert(a != 0, cast(string) data); 2294 for(int b = a-1; b >= 0; b--) { 2295 char cc = data[b]; 2296 if(cc >= 'a' && cc <= 'z') 2297 cc -= 0x20; 2298 int val = 0; 2299 if(cc >= '0' && cc <= '9') 2300 val = cc - '0'; 2301 else 2302 val = cc - 'A' + 10; 2303 2304 assert(val >= 0 && val <= 15, to!string(val)); 2305 bodyReadingState.contentLengthRemaining += power * val; 2306 power *= 16; 2307 } 2308 debug(arsd_http2_verbose) writeln("Chunk length: ", bodyReadingState.contentLengthRemaining); 2309 bodyReadingState.chunkedState = 1; 2310 data = data[a + 1 .. $]; 2311 goto start_over; 2312 } 2313 break; 2314 case 1: // reading until end of line 2315 char c = data[a]; 2316 if(c == '\n') { 2317 if(bodyReadingState.contentLengthRemaining == 0) 2318 bodyReadingState.chunkedState = 5; 2319 else 2320 bodyReadingState.chunkedState = 2; 2321 } 2322 data = data[a + 1 .. $]; 2323 goto start_over; 2324 case 2: // reading data 2325 auto can = a + bodyReadingState.contentLengthRemaining; 2326 if(can > data.length) 2327 can = cast(int) data.length; 2328 2329 auto newData = data[a .. can]; 2330 data = data[can .. $]; 2331 2332 //if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) 2333 // responseData.content ~= cast(ubyte[]) uncompress.uncompress(data[a .. can]); 2334 //else 2335 responseData.content ~= newData; 2336 2337 bodyReadingState.contentLengthRemaining -= newData.length; 2338 debug(arsd_http2_verbose) writeln("clr: ", bodyReadingState.contentLengthRemaining, " " , a, " ", can); 2339 assert(bodyReadingState.contentLengthRemaining >= 0); 2340 if(bodyReadingState.contentLengthRemaining == 0) { 2341 bodyReadingState.chunkedState = 3; 2342 } else { 2343 // will continue grabbing more 2344 } 2345 goto start_over; 2346 case 3: // reading 13/10 2347 assert(data[a] == 13); 2348 bodyReadingState.chunkedState++; 2349 data = data[a + 1 .. $]; 2350 goto start_over; 2351 case 4: // reading 10 at end of packet 2352 assert(data[a] == 10); 2353 data = data[a + 1 .. $]; 2354 bodyReadingState.chunkedState = 0; 2355 goto start_over; 2356 case 5: // reading footers 2357 //goto done; // FIXME 2358 2359 int footerReadingState = 0; 2360 int footerSize; 2361 2362 while(footerReadingState != 2 && a < data.length) { 2363 // import std.stdio; writeln(footerReadingState, " ", footerSize, " ", data); 2364 switch(footerReadingState) { 2365 case 0: 2366 if(data[a] == 13) 2367 footerReadingState++; 2368 else 2369 footerSize++; 2370 break; 2371 case 1: 2372 if(data[a] == 10) { 2373 if(footerSize == 0) { 2374 // all done, time to break 2375 footerReadingState++; 2376 2377 } else { 2378 // actually had a footer, try to read another 2379 footerReadingState = 0; 2380 footerSize = 0; 2381 } 2382 } else { 2383 throw new Exception("bad footer thing"); 2384 } 2385 break; 2386 default: 2387 assert(0); 2388 } 2389 2390 a++; 2391 } 2392 2393 if(footerReadingState != 2) 2394 break start_over; // haven't hit the end of the thing yet 2395 2396 bodyReadingState.chunkedState = 0; 2397 data = data[a .. $]; 2398 2399 if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) { 2400 auto n = uncompress.uncompress(responseData.content); 2401 n ~= uncompress.flush(); 2402 responseData.content = cast(ubyte[]) n; 2403 } 2404 2405 // responseData.content ~= cast(ubyte[]) uncompress.flush(); 2406 responseData.contentText = cast(string) responseData.content; 2407 2408 goto done; 2409 } 2410 } 2411 2412 } else { 2413 //if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) 2414 // responseData.content ~= cast(ubyte[]) uncompress.uncompress(data); 2415 //else 2416 responseData.content ~= data; 2417 //assert(data.length <= bodyReadingState.contentLengthRemaining, format("%d <= %d\n%s", data.length, bodyReadingState.contentLengthRemaining, cast(string)data)); 2418 { 2419 int use = cast(int) data.length; 2420 if(use > bodyReadingState.contentLengthRemaining) 2421 use = bodyReadingState.contentLengthRemaining; 2422 bodyReadingState.contentLengthRemaining -= use; 2423 data = data[use .. $]; 2424 } 2425 if(bodyReadingState.contentLengthRemaining == 0) { 2426 if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) { 2427 // import std.stdio; writeln(responseData.content.length, " ", responseData.content[0 .. 2], " .. ", responseData.content[$-2 .. $]); 2428 auto n = uncompress.uncompress(responseData.content); 2429 n ~= uncompress.flush(); 2430 responseData.content = cast(ubyte[]) n; 2431 responseData.contentText = cast(string) responseData.content; 2432 //responseData.content ~= cast(ubyte[]) uncompress.flush(); 2433 } else { 2434 responseData.contentText = cast(string) responseData.content; 2435 } 2436 2437 done: 2438 2439 if(retainCookies && client !is null) { 2440 client.retainCookies(responseData); 2441 } 2442 2443 if(followLocation && responseData.location.length) { 2444 if(maximumNumberOfRedirectsRemaining <= 0) { 2445 throw new Exception("Maximum number of redirects exceeded"); 2446 } else { 2447 maximumNumberOfRedirectsRemaining--; 2448 } 2449 2450 static bool first = true; 2451 //version(DigitalMars) if(!first) asm { int 3; } 2452 debug(arsd_http2) writeln("redirecting to ", responseData.location); 2453 populateFromInfo(Uri(responseData.location), HttpVerb.GET); 2454 //import std.stdio; writeln("redirected to ", responseData.location); 2455 first = false; 2456 responseData = HttpResponse.init; 2457 headerReadingState = HeaderReadingState.init; 2458 bodyReadingState = BodyReadingState.init; 2459 if(client !is null) { 2460 // FIXME: this won't clear cookies that were cleared in another request 2461 client.populateCookies(this); // they might have changed in the previous redirection cycle! 2462 } 2463 state = State.unsent; 2464 stillAlive = false; 2465 sendPrivate(false); 2466 } else { 2467 state = State.complete; 2468 // FIXME 2469 //if(closeSocketWhenComplete) 2470 //socket.close(); 2471 } 2472 } 2473 } 2474 } 2475 2476 if(data.length) 2477 leftoverDataFromLastTime = data.dup; 2478 else 2479 leftoverDataFromLastTime = null; 2480 2481 return stillAlive; 2482 } 2483 2484 } 2485 } 2486 2487 /++ 2488 Waits for the first of the given requests to be either aborted or completed. 2489 Returns the first one in that state, or `null` if the operation was interrupted 2490 or reached the given timeout before any completed. (If it returns null even before 2491 the timeout, it might be because the user pressed ctrl+c, so you should consider 2492 checking if you should cancel the operation. If not, you can simply call it again 2493 with the same arguments to start waiting again.) 2494 2495 You MUST check for null, even if you don't specify a timeout! 2496 2497 Note that if an individual request times out before any others request, it will 2498 return that timed out request, since that counts as completion. 2499 2500 If the return is not null, you should call `waitForCompletion` on the given request 2501 to get the response out. It will not have to wait since it is guaranteed to be 2502 finished when returned by this function; that will just give you the cached response. 2503 2504 (I thought about just having it return the response, but tying a response back to 2505 a request is harder than just getting the original request object back and taking 2506 the response out of it.) 2507 2508 Please note: if a request in the set has already completed or been aborted, it will 2509 always return the first one it sees upon calling the function. You may wish to remove 2510 them from the list before calling the function. 2511 2512 History: 2513 Added December 24, 2021 (dub v10.5) 2514 +/ 2515 HttpRequest waitForFirstToComplete(Duration timeout, HttpRequest[] requests...) { 2516 2517 foreach(request; requests) { 2518 if(request.state == HttpRequest.State.unsent) 2519 request.send(); 2520 else if(request.state == HttpRequest.State.complete) 2521 return request; 2522 else if(request.state == HttpRequest.State.aborted) 2523 return request; 2524 } 2525 2526 while(true) { 2527 if(auto err = HttpRequest.advanceConnections(timeout)) { 2528 switch(err) { 2529 case 1: return null; 2530 case 2: throw new Exception("HttpRequest.advanceConnections returned 2: nothing to do"); 2531 case 3: return null; 2532 default: throw new Exception("HttpRequest.advanceConnections got err " ~ to!string(err)); 2533 } 2534 } 2535 2536 foreach(request; requests) { 2537 if(request.state == HttpRequest.State.aborted || request.state == HttpRequest.State.complete) { 2538 request.waitForCompletion(); 2539 return request; 2540 } 2541 } 2542 2543 } 2544 } 2545 2546 /// ditto 2547 HttpRequest waitForFirstToComplete(HttpRequest[] requests...) { 2548 return waitForFirstToComplete(1.weeks, requests); 2549 } 2550 2551 /++ 2552 An input range that runs [waitForFirstToComplete] but only returning each request once. 2553 Before you loop over it, you can set some properties to customize behavior. 2554 2555 If it times out or is interrupted, it will prematurely run empty. You can set the delegate 2556 to process this. 2557 2558 Implementation note: each iteration through the loop does a O(n) check over each item remaining. 2559 This shouldn't matter, but if it does become an issue for you, let me know. 2560 2561 History: 2562 Added December 24, 2021 (dub v10.5) 2563 +/ 2564 struct HttpRequestsAsTheyComplete { 2565 /++ 2566 Seeds it with an overall timeout and the initial requests. 2567 It will send all the requests before returning, then will process 2568 the responses as they come. 2569 2570 Please note that it modifies the array of requests you pass in! It 2571 will keep a reference to it and reorder items on each call of popFront. 2572 You might want to pass a duplicate if you have another purpose for your 2573 array and don't want to see it shuffled. 2574 +/ 2575 this(Duration timeout, HttpRequest[] requests) { 2576 remainingRequests = requests; 2577 this.timeout = timeout; 2578 popFront(); 2579 } 2580 2581 /++ 2582 You can set this delegate to decide how to handle an interruption. Returning true 2583 from this will keep working. Returning false will terminate the loop. 2584 2585 If this is null, an interruption will always terminate the loop. 2586 2587 Note that interruptions can be caused by the garbage collector being triggered by 2588 another thread as well as by user action. If you don't set a SIGINT handler, it 2589 might be reasonable to always return true here. 2590 +/ 2591 bool delegate() onInterruption; 2592 2593 private HttpRequest[] remainingRequests; 2594 2595 /// The timeout you set in the constructor. You can change it if you want. 2596 Duration timeout; 2597 2598 /++ 2599 Adds another request to the work queue. It is safe to call this from inside the loop 2600 as you process other requests. 2601 +/ 2602 void appendRequest(HttpRequest request) { 2603 remainingRequests ~= request; 2604 } 2605 2606 /++ 2607 If the loop exited, it might be due to an interruption or a time out. If you like, you 2608 can call this to pick up the work again, 2609 2610 If it returns `false`, the work is indeed all finished and you should not re-enter the loop. 2611 2612 --- 2613 auto range = HttpRequestsAsTheyComplete(10.seconds, your_requests); 2614 process_loop: foreach(req; range) { 2615 // process req 2616 } 2617 // make sure we weren't interrupted because the user requested we cancel! 2618 // but then try to re-enter the range if possible 2619 if(!user_quit && range.reenter()) { 2620 // there's still something unprocessed in there 2621 // range.reenter returning true means it is no longer 2622 // empty, so we should try to loop over it again 2623 goto process_loop; // re-enter the loop 2624 } 2625 --- 2626 +/ 2627 bool reenter() { 2628 if(remainingRequests.length == 0) 2629 return false; 2630 empty = false; 2631 popFront(); 2632 return true; 2633 } 2634 2635 /// Standard range primitives. I reserve the right to change the variables to read-only properties in the future without notice. 2636 HttpRequest front; 2637 2638 /// ditto 2639 bool empty; 2640 2641 /// ditto 2642 void popFront() { 2643 resume: 2644 if(remainingRequests.length == 0) { 2645 empty = true; 2646 return; 2647 } 2648 2649 front = waitForFirstToComplete(timeout, remainingRequests); 2650 2651 if(front is null) { 2652 if(onInterruption) { 2653 if(onInterruption()) 2654 goto resume; 2655 } 2656 empty = true; 2657 return; 2658 } 2659 foreach(idx, req; remainingRequests) { 2660 if(req is front) { 2661 remainingRequests[idx] = remainingRequests[$ - 1]; 2662 remainingRequests = remainingRequests[0 .. $ - 1]; 2663 return; 2664 } 2665 } 2666 } 2667 } 2668 2669 // 2670 struct HttpRequestParameters { 2671 // FIXME: implement these 2672 //Duration timeoutTotal; // the whole request must finish in this time or else it fails,even if data is still trickling in 2673 Duration timeoutFromInactivity; // if there's no activity in this time it dies. basically the socket receive timeout 2674 2675 // debugging 2676 bool useHttp11 = true; /// 2677 bool acceptGzip = true; /// 2678 bool keepAlive = true; /// 2679 2680 // the request itself 2681 HttpVerb method; /// 2682 string host; /// 2683 ushort port; /// 2684 string uri; /// 2685 2686 bool ssl; /// 2687 2688 string userAgent; /// 2689 string authorization; /// 2690 2691 string[string] cookies; /// 2692 2693 string[] headers; /// do not duplicate host, content-length, content-type, or any others that have a specific property 2694 2695 string contentType; /// 2696 ubyte[] bodyData; /// 2697 2698 string unixSocketPath; /// 2699 } 2700 2701 interface IHttpClient { 2702 2703 } 2704 2705 /// 2706 enum HttpVerb { 2707 /// 2708 GET, 2709 /// 2710 HEAD, 2711 /// 2712 POST, 2713 /// 2714 PUT, 2715 /// 2716 DELETE, 2717 /// 2718 OPTIONS, 2719 /// 2720 TRACE, 2721 /// 2722 CONNECT, 2723 /// 2724 PATCH, 2725 /// 2726 MERGE 2727 } 2728 2729 /++ 2730 Supported file formats for [HttpClient.setClientCert]. These are loaded by OpenSSL 2731 in the current implementation. 2732 2733 History: 2734 Added February 3, 2022 (dub v10.6) 2735 +/ 2736 enum CertificateFileFormat { 2737 guess, /// try to guess the format from the file name and/or contents 2738 pem, /// the files are specifically in PEM format 2739 der /// the files are specifically in DER format 2740 } 2741 2742 /++ 2743 HttpClient keeps cookies, location, and some other state to reuse connections, when possible, like a web browser. 2744 You can use it as your entry point to make http requests. 2745 2746 See the example on [arsd.http2#examples]. 2747 +/ 2748 class HttpClient { 2749 /* Protocol restrictions, useful to disable when debugging servers */ 2750 bool useHttp11 = true; /// 2751 bool acceptGzip = true; /// 2752 bool keepAlive = true; /// 2753 2754 /++ 2755 Sets the client certificate used as a log in identifier on https connections. 2756 The certificate and key must be unencrypted at this time and both must be in 2757 the same file format. 2758 2759 Bugs: 2760 The current implementation sets the filenames into a static variable, 2761 meaning it is shared across all clients and connections. 2762 2763 Errors in the cert or key are only reported if the server reports an 2764 authentication failure. Make sure you are passing correct filenames 2765 and formats of you do see a failure. 2766 2767 History: 2768 Added February 2, 2022 (dub v10.6) 2769 +/ 2770 void setClientCertificate(string certFilename, string keyFilename, CertificateFileFormat certFormat = CertificateFileFormat.guess) { 2771 this.certFilename = certFilename; 2772 this.keyFilename = keyFilename; 2773 this.certFormat = certFormat; 2774 } 2775 2776 /++ 2777 Sets whether [HttpRequest]s created through this object (with [navigateTo], [request], etc.), will have the 2778 value of [HttpRequest.verifyPeer] of true or false upon construction. 2779 2780 History: 2781 Added April 5, 2022 (dub v10.8). Previously, there was an undocumented global value used. 2782 +/ 2783 bool defaultVerifyPeer = true; 2784 2785 /++ 2786 Adds a header to be automatically appended to each request created through this client. 2787 2788 If you add duplicate headers, it will add multiple copies. 2789 2790 You should NOT use this to add headers that can be set through other properties like [userAgent], [authorization], or [setCookie]. 2791 2792 History: 2793 Added July 12, 2023 2794 +/ 2795 void addDefaultHeader(string key, string value) { 2796 defaultHeaders ~= key ~ ": " ~ value; 2797 } 2798 2799 private string[] defaultHeaders; 2800 2801 // FIXME: getCookies api 2802 // FIXME: an easy way to download files 2803 2804 // FIXME: try to not make these static 2805 private static string certFilename; 2806 private static string keyFilename; 2807 private static CertificateFileFormat certFormat; 2808 2809 /// 2810 @property Uri location() { 2811 return currentUrl; 2812 } 2813 2814 /++ 2815 Default timeout for requests created on this client. 2816 2817 History: 2818 Added March 31, 2021 2819 +/ 2820 Duration defaultTimeout = 10.seconds; 2821 2822 /++ 2823 High level function that works similarly to entering a url 2824 into a browser. 2825 2826 Follows locations, retain cookies, updates the current url, etc. 2827 +/ 2828 HttpRequest navigateTo(Uri where, HttpVerb method = HttpVerb.GET) { 2829 currentUrl = where.basedOn(currentUrl); 2830 currentDomain = where.host; 2831 2832 auto request = this.request(currentUrl, method); 2833 request.followLocation = true; 2834 request.retainCookies = true; 2835 2836 return request; 2837 } 2838 2839 /++ 2840 Creates a request without updating the current url state. If you want to save cookies, either call [retainCookies] with the response yourself 2841 or set [HttpRequest.retainCookies|request.retainCookies] to `true` on the returned object. But see important implementation shortcomings on [retainCookies]. 2842 2843 To upload files, you can use the [FormData] overload. 2844 +/ 2845 HttpRequest request(Uri uri, HttpVerb method = HttpVerb.GET, ubyte[] bodyData = null, string contentType = null) { 2846 string proxyToUse = getProxyFor(uri); 2847 2848 auto request = new HttpRequest(this, uri, method, cache, defaultTimeout, proxyToUse); 2849 2850 request.verifyPeer = this.defaultVerifyPeer; 2851 2852 request.requestParameters.userAgent = userAgent; 2853 request.requestParameters.authorization = authorization; 2854 2855 request.requestParameters.useHttp11 = this.useHttp11; 2856 request.requestParameters.acceptGzip = this.acceptGzip; 2857 request.requestParameters.keepAlive = this.keepAlive; 2858 2859 request.requestParameters.bodyData = bodyData; 2860 request.requestParameters.contentType = contentType; 2861 2862 request.requestParameters.headers = this.defaultHeaders; 2863 2864 populateCookies(request); 2865 2866 return request; 2867 } 2868 2869 /// ditto 2870 HttpRequest request(Uri uri, FormData fd, HttpVerb method = HttpVerb.POST) { 2871 return request(uri, method, fd.toBytes, fd.contentType); 2872 } 2873 2874 2875 private void populateCookies(HttpRequest request) { 2876 // FIXME: what about expiration and the like? or domain/path checks? or Secure checks? 2877 // FIXME: is uri.host correct? i think it should include port number too. what fun. 2878 if(auto cookies = ""/*uri.host*/ in this.cookies) { 2879 foreach(cookie; *cookies) 2880 request.requestParameters.cookies[cookie.name] = cookie.value; 2881 } 2882 } 2883 2884 private Uri currentUrl; 2885 private string currentDomain; 2886 private ICache cache; 2887 2888 /++ 2889 2890 +/ 2891 this(ICache cache = null) { 2892 this.defaultVerifyPeer = .defaultVerifyPeer_; 2893 this.cache = cache; 2894 loadDefaultProxy(); 2895 } 2896 2897 /++ 2898 Loads the system-default proxy. Note that the constructor does this automatically 2899 so you should rarely need to call this explicitly. 2900 2901 The environment variables are used, if present, on all operating systems. 2902 2903 History: 2904 no_proxy support added April 13, 2022 2905 2906 Added April 12, 2021 (included in dub v9.5) 2907 2908 Bugs: 2909 On Windows, it does NOT currently check the IE settings, but I do intend to 2910 implement that in the future. When I do, it will be classified as a bug fix, 2911 NOT a breaking change. 2912 +/ 2913 void loadDefaultProxy() { 2914 import std.process; 2915 httpProxy = environment.get("http_proxy", environment.get("HTTP_PROXY", null)); 2916 httpsProxy = environment.get("https_proxy", environment.get("HTTPS_PROXY", null)); 2917 auto noProxy = environment.get("no_proxy", environment.get("NO_PROXY", null)); 2918 if (noProxy.length) { 2919 proxyIgnore = noProxy.split(","); 2920 foreach (ref rule; proxyIgnore) 2921 rule = rule.strip; 2922 } 2923 2924 // FIXME: on Windows, I should use the Internet Explorer proxy settings 2925 } 2926 2927 /++ 2928 Checks if the given uri should be proxied according to the httpProxy, httpsProxy, proxyIgnore 2929 variables and returns either httpProxy, httpsProxy or null. 2930 2931 If neither `httpProxy` or `httpsProxy` are set this always returns `null`. Same if `proxyIgnore` 2932 contains `*`. 2933 2934 DNS is not resolved for proxyIgnore IPs, only IPs match IPs and hosts match hosts. 2935 +/ 2936 string getProxyFor(Uri uri) { 2937 string proxyToUse; 2938 switch(uri.scheme) { 2939 case "http": 2940 proxyToUse = httpProxy; 2941 break; 2942 case "https": 2943 proxyToUse = httpsProxy; 2944 break; 2945 default: 2946 proxyToUse = null; 2947 } 2948 2949 if (proxyToUse.length) { 2950 foreach (ignore; proxyIgnore) { 2951 if (matchProxyIgnore(ignore, uri)) { 2952 return null; 2953 } 2954 } 2955 } 2956 2957 return proxyToUse; 2958 } 2959 2960 /// Returns -1 on error, otherwise the IP as uint. Parsing is very strict. 2961 private static long tryParseIPv4(scope const(char)[] s) nothrow { 2962 import std.algorithm : findSplit, all; 2963 import std.ascii : isDigit; 2964 2965 static int parseNum(scope const(char)[] num) nothrow { 2966 if (num.length < 1 || num.length > 3 || !num.representation.all!isDigit) 2967 return -1; 2968 try { 2969 auto ret = num.to!int; 2970 return ret > 255 ? -1 : ret; 2971 } catch (Exception) { 2972 assert(false); 2973 } 2974 } 2975 2976 if (s.length < "0.0.0.0".length || s.length > "255.255.255.255".length) 2977 return -1; 2978 auto firstPair = s.findSplit("."); 2979 auto secondPair = firstPair[2].findSplit("."); 2980 auto thirdPair = secondPair[2].findSplit("."); 2981 auto a = parseNum(firstPair[0]); 2982 auto b = parseNum(secondPair[0]); 2983 auto c = parseNum(thirdPair[0]); 2984 auto d = parseNum(thirdPair[2]); 2985 if (a < 0 || b < 0 || c < 0 || d < 0) 2986 return -1; 2987 return (cast(uint)a << 24) | (b << 16) | (c << 8) | (d); 2988 } 2989 2990 unittest { 2991 assert(tryParseIPv4("0.0.0.0") == 0); 2992 assert(tryParseIPv4("127.0.0.1") == 0x7f000001); 2993 assert(tryParseIPv4("162.217.114.56") == 0xa2d97238); 2994 assert(tryParseIPv4("256.0.0.1") == -1); 2995 assert(tryParseIPv4("0.0.0.-2") == -1); 2996 assert(tryParseIPv4("0.0.0.a") == -1); 2997 assert(tryParseIPv4("0.0.0") == -1); 2998 assert(tryParseIPv4("0.0.0.0.0") == -1); 2999 } 3000 3001 /++ 3002 Returns true if the given no_proxy rule matches the uri. 3003 3004 Invalid IP ranges are silently ignored and return false. 3005 3006 See $(LREF proxyIgnore). 3007 +/ 3008 static bool matchProxyIgnore(scope const(char)[] rule, scope const Uri uri) nothrow { 3009 import std.algorithm; 3010 import std.ascii : isDigit; 3011 import std.uni : sicmp; 3012 3013 string uriHost = uri.host; 3014 if (uriHost.length && uriHost[$ - 1] == '.') 3015 uriHost = uriHost[0 .. $ - 1]; 3016 3017 if (rule == "*") 3018 return true; 3019 while (rule.length && rule[0] == '.') rule = rule[1 .. $]; 3020 3021 static int parsePort(scope const(char)[] portStr) nothrow { 3022 if (portStr.length < 1 || portStr.length > 5 || !portStr.representation.all!isDigit) 3023 return -1; 3024 try { 3025 return portStr.to!int; 3026 } catch (Exception) { 3027 assert(false, "to!int should succeed"); 3028 } 3029 } 3030 3031 if (sicmp(rule, uriHost) == 0 3032 || (uriHost.length > rule.length 3033 && sicmp(rule, uriHost[$ - rule.length .. $]) == 0 3034 && uriHost[$ - rule.length - 1] == '.')) 3035 return true; 3036 3037 if (rule.startsWith("[")) { // IPv6 3038 // below code is basically nothrow lastIndexOfAny("]:") 3039 ptrdiff_t lastColon = cast(ptrdiff_t) rule.length - 1; 3040 while (lastColon >= 0) { 3041 if (rule[lastColon] == ']' || rule[lastColon] == ':') 3042 break; 3043 lastColon--; 3044 } 3045 if (lastColon == -1) 3046 return false; // malformed 3047 3048 if (rule[lastColon] == ':') { // match with port 3049 auto port = parsePort(rule[lastColon + 1 .. $]); 3050 if (port != -1) { 3051 if (uri.effectivePort != port.to!int) 3052 return false; 3053 return uriHost == rule[0 .. lastColon]; 3054 } 3055 } 3056 // exact match of host already done above 3057 } else { 3058 auto slash = rule.lastIndexOfNothrow('/'); 3059 if (slash == -1) { // no IP range 3060 auto colon = rule.lastIndexOfNothrow(':'); 3061 auto host = colon == -1 ? rule : rule[0 .. colon]; 3062 auto port = colon != -1 ? parsePort(rule[colon + 1 .. $]) : -1; 3063 auto ip = tryParseIPv4(host); 3064 if (ip == -1) { // not an IPv4, test for host with port 3065 return port != -1 3066 && uri.effectivePort == port 3067 && uriHost == host; 3068 } else { 3069 // perform IPv4 equals 3070 auto other = tryParseIPv4(uriHost); 3071 if (other == -1) 3072 return false; // rule == IPv4, uri != IPv4 3073 if (port != -1) 3074 return uri.effectivePort == port 3075 && uriHost == host; 3076 else 3077 return uriHost == host; 3078 } 3079 } else { 3080 auto maskStr = rule[slash + 1 .. $]; 3081 auto ip = tryParseIPv4(rule[0 .. slash]); 3082 if (ip == -1) 3083 return false; 3084 if (maskStr.length && maskStr.length < 3 && maskStr.representation.all!isDigit) { 3085 // IPv4 range match 3086 int mask; 3087 try { 3088 mask = maskStr.to!int; 3089 } catch (Exception) { 3090 assert(false); 3091 } 3092 3093 auto other = tryParseIPv4(uriHost); 3094 if (other == -1) 3095 return false; // rule == IPv4, uri != IPv4 3096 3097 if (mask == 0) // matches all 3098 return true; 3099 if (mask > 32) // matches none 3100 return false; 3101 3102 auto shift = 32 - mask; 3103 return cast(uint)other >> shift 3104 == cast(uint)ip >> shift; 3105 } 3106 } 3107 } 3108 return false; 3109 } 3110 3111 unittest { 3112 assert(matchProxyIgnore("0.0.0.0/0", Uri("http://127.0.0.1:80/a"))); 3113 assert(matchProxyIgnore("0.0.0.0/0", Uri("http://127.0.0.1/a"))); 3114 assert(!matchProxyIgnore("0.0.0.0/0", Uri("https://dlang.org/a"))); 3115 assert(matchProxyIgnore("*", Uri("https://dlang.org/a"))); 3116 assert(matchProxyIgnore("127.0.0.0/8", Uri("http://127.0.0.1:80/a"))); 3117 assert(matchProxyIgnore("127.0.0.0/8", Uri("http://127.0.0.1/a"))); 3118 assert(matchProxyIgnore("127.0.0.1", Uri("http://127.0.0.1:1234/a"))); 3119 assert(!matchProxyIgnore("127.0.0.1:80", Uri("http://127.0.0.1:1234/a"))); 3120 assert(!matchProxyIgnore("127.0.0.1/8", Uri("http://localhost/a"))); // no DNS resolution / guessing 3121 assert(!matchProxyIgnore("0.0.0.0/1", Uri("http://localhost/a")) 3122 && !matchProxyIgnore("128.0.0.0/1", Uri("http://localhost/a"))); // no DNS resolution / guessing 2 3123 foreach (m; 1 .. 32) { 3124 assert(matchProxyIgnore(text("127.0.0.1/", m), Uri("http://127.0.0.1/a"))); 3125 assert(!matchProxyIgnore(text("127.0.0.1/", m), Uri("http://128.0.0.1/a"))); 3126 bool expectedMatch = m <= 24; 3127 assert(expectedMatch == matchProxyIgnore(text("127.0.1.0/", m), Uri("http://127.0.1.128/a")), m.to!string); 3128 } 3129 assert(matchProxyIgnore("localhost", Uri("http://localhost/a"))); 3130 assert(matchProxyIgnore("localhost", Uri("http://foo.localhost/a"))); 3131 assert(matchProxyIgnore("localhost", Uri("http://foo.localhost./a"))); 3132 assert(matchProxyIgnore(".localhost", Uri("http://localhost/a"))); 3133 assert(matchProxyIgnore(".localhost", Uri("http://foo.localhost/a"))); 3134 assert(matchProxyIgnore(".localhost", Uri("http://foo.localhost./a"))); 3135 assert(!matchProxyIgnore("foo.localhost", Uri("http://localhost/a"))); 3136 assert(matchProxyIgnore("foo.localhost", Uri("http://foo.localhost/a"))); 3137 assert(matchProxyIgnore("foo.localhost", Uri("http://foo.localhost./a"))); 3138 assert(!matchProxyIgnore("bar.localhost", Uri("http://localhost/a"))); 3139 assert(!matchProxyIgnore("bar.localhost", Uri("http://foo.localhost/a"))); 3140 assert(!matchProxyIgnore("bar.localhost", Uri("http://foo.localhost./a"))); 3141 assert(!matchProxyIgnore("bar.localhost", Uri("http://bbar.localhost./a"))); 3142 assert(matchProxyIgnore("[::1]", Uri("http://[::1]/a"))); 3143 assert(!matchProxyIgnore("[::1]", Uri("http://[::2]/a"))); 3144 assert(matchProxyIgnore("[::1]:80", Uri("http://[::1]/a"))); 3145 assert(!matchProxyIgnore("[::1]:443", Uri("http://[::1]/a"))); 3146 assert(!matchProxyIgnore("[::1]:80", Uri("https://[::1]/a"))); 3147 assert(matchProxyIgnore("[::1]:443", Uri("https://[::1]/a"))); 3148 assert(matchProxyIgnore("google.com", Uri("https://GOOGLE.COM/a"))); 3149 } 3150 3151 /++ 3152 Proxies to use for requests. The [HttpClient] constructor will set these to the system values, 3153 then you can reset it to `null` if you want to override and not use the proxy after all, or you 3154 can set it after construction to whatever. 3155 3156 The proxy from the client will be automatically set to the requests performed through it. You can 3157 also override on a per-request basis by creating the request and setting the `proxy` field there 3158 before sending it. 3159 3160 History: 3161 Added April 12, 2021 (included in dub v9.5) 3162 +/ 3163 string httpProxy; 3164 /// ditto 3165 string httpsProxy; 3166 /++ 3167 List of hosts or ips, optionally including a port, where not to proxy. 3168 3169 Each entry may be one of the following formats: 3170 - `127.0.0.1` (IPv4, any port) 3171 - `127.0.0.1:1234` (IPv4, specific port) 3172 - `127.0.0.1/8` (IPv4 range / CIDR block, any port) 3173 - `[::1]` (IPv6, any port) 3174 - `[::1]:1234` (IPv6, specific port) 3175 - `*` (all hosts and ports, basically don't proxy at all anymore) 3176 - `.domain.name`, `domain.name` (don't proxy the specified domain, 3177 leading dots are stripped and subdomains are also not proxied) 3178 - `.domain.name:1234`, `domain.name:1234` (same as above, with specific port) 3179 3180 No DNS resolution or regex is done in this list. 3181 3182 See https://about.gitlab.com/blog/2021/01/27/we-need-to-talk-no-proxy/ 3183 3184 History: 3185 Added April 13, 2022 3186 +/ 3187 string[] proxyIgnore; 3188 3189 /// See [retainCookies] for important caveats. 3190 void setCookie(string name, string value, string domain = null) { 3191 CookieHeader ch; 3192 3193 ch.name = name; 3194 ch.value = value; 3195 3196 setCookie(ch, domain); 3197 } 3198 3199 /// ditto 3200 void setCookie(CookieHeader ch, string domain = null) { 3201 if(domain is null) 3202 domain = currentDomain; 3203 3204 // FIXME: figure all this out or else cookies liable to get too long, in addition to the overwriting and oversharing issues in long scraping sessions 3205 cookies[""/*domain*/] ~= ch; 3206 } 3207 3208 /++ 3209 [HttpClient] does NOT automatically store cookies. You must explicitly retain them from a response by calling this method. 3210 3211 Examples: 3212 --- 3213 import arsd.http2; 3214 void main() { 3215 auto client = new HttpClient(); 3216 auto setRequest = client.request(Uri("http://arsdnet.net/cgi-bin/cookies/set")); 3217 auto setResponse = setRequest.waitForCompletion(); 3218 3219 auto request = client.request(Uri("http://arsdnet.net/cgi-bin/cookies/get")); 3220 auto response = request.waitForCompletion(); 3221 3222 // the cookie wasn't explicitly retained, so the server echos back nothing 3223 assert(response.responseText.length == 0); 3224 3225 // now keep the cookies from our original set 3226 client.retainCookies(setResponse); 3227 3228 request = client.request(Uri("http://arsdnet.net/cgi-bin/cookies/get")); 3229 response = request.waitForCompletion(); 3230 3231 // now it matches 3232 assert(response.responseText.length && response.responseText == setResponse.cookies["example-cookie"]); 3233 } 3234 --- 3235 3236 Bugs: 3237 It does NOT currently implement domain / path / secure separation nor cookie expiration. It assumes that if you call this function, you're ok with it. 3238 3239 You may want to use separate HttpClient instances if any sharing is unacceptable at this time. 3240 3241 History: 3242 Added July 5, 2021 (dub v10.2) 3243 +/ 3244 void retainCookies(HttpResponse fromResponse) { 3245 foreach(name, value; fromResponse.cookies) 3246 setCookie(name, value); 3247 } 3248 3249 /// 3250 void clearCookies(string domain = null) { 3251 if(domain is null) 3252 cookies = null; 3253 else 3254 cookies[domain] = null; 3255 } 3256 3257 // If you set these, they will be pre-filled on all requests made with this client 3258 string userAgent = "D arsd.html2"; /// 3259 string authorization; /// 3260 3261 /* inter-request state */ 3262 private CookieHeader[][string] cookies; 3263 } 3264 3265 private ptrdiff_t lastIndexOfNothrow(T)(scope T[] arr, T value) nothrow 3266 { 3267 ptrdiff_t ret = cast(ptrdiff_t)arr.length - 1; 3268 while (ret >= 0) { 3269 if (arr[ret] == value) 3270 return ret; 3271 ret--; 3272 } 3273 return ret; 3274 } 3275 3276 interface ICache { 3277 /++ 3278 The client is about to make the given `request`. It will ALWAYS pass it to the cache object first so you can decide if you want to and can provide a response. You should probably check the appropriate headers to see if you should even attempt to look up on the cache (HttpClient does NOT do this to give maximum flexibility to the cache implementor). 3279 3280 Return null if the cache does not provide. 3281 +/ 3282 const(HttpResponse)* getCachedResponse(HttpRequestParameters request); 3283 3284 /++ 3285 The given request has received the given response. The implementing class needs to decide if it wants to cache or not. Return true if it was added, false if you chose not to. 3286 3287 You may wish to examine headers, etc., in making the decision. The HttpClient will ALWAYS pass a request/response to this. 3288 +/ 3289 bool cacheResponse(HttpRequestParameters request, HttpResponse response); 3290 } 3291 3292 /+ 3293 // / Provides caching behavior similar to a real web browser 3294 class HttpCache : ICache { 3295 const(HttpResponse)* getCachedResponse(HttpRequestParameters request) { 3296 return null; 3297 } 3298 } 3299 3300 // / Gives simple maximum age caching, ignoring the actual http headers 3301 class SimpleCache : ICache { 3302 const(HttpResponse)* getCachedResponse(HttpRequestParameters request) { 3303 return null; 3304 } 3305 } 3306 +/ 3307 3308 /++ 3309 A pseudo-cache to provide a mock server. Construct one of these, 3310 populate it with test responses, and pass it to [HttpClient] to 3311 do a network-free test. 3312 3313 You should populate it with the [populate] method. Any request not 3314 pre-populated will return a "server refused connection" response. 3315 +/ 3316 class HttpMockProvider : ICache { 3317 /+ + 3318 3319 +/ 3320 version(none) 3321 this(Uri baseUrl, string defaultResponseContentType) { 3322 3323 } 3324 3325 this() {} 3326 3327 HttpResponse defaultResponse; 3328 3329 /// Implementation of the ICache interface. Hijacks all requests to return a pre-populated response or "server disconnected". 3330 const(HttpResponse)* getCachedResponse(HttpRequestParameters request) { 3331 import std.conv; 3332 auto defaultPort = request.ssl ? 443 : 80; 3333 string identifier = text( 3334 request.method, " ", 3335 request.ssl ? "https" : "http", "://", 3336 request.host, 3337 (request.port && request.port != defaultPort) ? (":" ~ to!string(request.port)) : "", 3338 request.uri 3339 ); 3340 3341 if(auto res = identifier in population) 3342 return res; 3343 return &defaultResponse; 3344 } 3345 3346 /// Implementation of the ICache interface. We never actually cache anything here since it is all about mock responses, not actually caching real data. 3347 bool cacheResponse(HttpRequestParameters request, HttpResponse response) { 3348 return false; 3349 } 3350 3351 /++ 3352 Convenience method to populate simple responses. For more complex 3353 work, use one of the other overloads where you build complete objects 3354 yourself. 3355 3356 Params: 3357 request = a verb and complete URL to mock as one string. 3358 For example "GET http://example.com/". If you provide only 3359 a partial URL, it will be based on the `baseUrl` you gave 3360 in the `HttpMockProvider` constructor. 3361 3362 responseCode = the HTTP response code, like 200 or 404. 3363 3364 response = the response body as a string. It is assumed 3365 to be of the `defaultResponseContentType` you passed in the 3366 `HttpMockProvider` constructor. 3367 +/ 3368 void populate(string request, int responseCode, string response) { 3369 3370 // FIXME: absolute-ize the URL in the request 3371 3372 HttpResponse r; 3373 r.code = responseCode; 3374 r.codeText = getHttpCodeText(r.code); 3375 3376 r.content = cast(ubyte[]) response; 3377 r.contentText = response; 3378 3379 population[request] = r; 3380 } 3381 3382 version(none) 3383 void populate(string method, string url, HttpResponse response) { 3384 // FIXME 3385 } 3386 3387 private HttpResponse[string] population; 3388 } 3389 3390 // modified from the one in cgi.d to just have the text 3391 private static string getHttpCodeText(int code) pure nothrow @nogc { 3392 switch(code) { 3393 // this module's proprietary extensions 3394 case 0: return null; 3395 case 1: return "request.abort called"; 3396 case 2: return "connection failed"; 3397 case 3: return "server disconnected"; 3398 case 4: return "exception thrown"; // actually should be some other thing 3399 case 5: return "Request timed out"; 3400 3401 // * * * standard ones * * * 3402 3403 // 1xx skipped since they shouldn't happen 3404 3405 // 3406 case 200: return "OK"; 3407 case 201: return "Created"; 3408 case 202: return "Accepted"; 3409 case 203: return "Non-Authoritative Information"; 3410 case 204: return "No Content"; 3411 case 205: return "Reset Content"; 3412 // 3413 case 300: return "Multiple Choices"; 3414 case 301: return "Moved Permanently"; 3415 case 302: return "Found"; 3416 case 303: return "See Other"; 3417 case 307: return "Temporary Redirect"; 3418 case 308: return "Permanent Redirect"; 3419 // 3420 case 400: return "Bad Request"; 3421 case 403: return "Forbidden"; 3422 case 404: return "Not Found"; 3423 case 405: return "Method Not Allowed"; 3424 case 406: return "Not Acceptable"; 3425 case 409: return "Conflict"; 3426 case 410: return "Gone"; 3427 // 3428 case 500: return "Internal Server Error"; 3429 case 501: return "Not Implemented"; 3430 case 502: return "Bad Gateway"; 3431 case 503: return "Service Unavailable"; 3432 // 3433 default: assert(0, "Unsupported http code"); 3434 } 3435 } 3436 3437 3438 /// 3439 struct HttpCookie { 3440 string name; /// 3441 string value; /// 3442 string domain; /// 3443 string path; /// 3444 //SysTime expirationDate; /// 3445 bool secure; /// 3446 bool httpOnly; /// 3447 } 3448 3449 // FIXME: websocket 3450 3451 version(testing) 3452 void main() { 3453 import std.stdio; 3454 auto client = new HttpClient(); 3455 auto request = client.navigateTo(Uri("http://localhost/chunked.php")); 3456 request.send(); 3457 auto request2 = client.navigateTo(Uri("http://dlang.org/")); 3458 request2.send(); 3459 3460 { 3461 auto response = request2.waitForCompletion(); 3462 //write(cast(string) response.content); 3463 } 3464 3465 auto response = request.waitForCompletion(); 3466 write(cast(string) response.content); 3467 3468 writeln(HttpRequest.socketsPerHost); 3469 } 3470 3471 3472 // From sslsocket.d, but this is the maintained version! 3473 version(use_openssl) { 3474 alias SslClientSocket = OpenSslSocket; 3475 3476 // CRL = Certificate Revocation List 3477 static immutable string[] sslErrorCodes = [ 3478 "OK (code 0)", 3479 "Unspecified SSL/TLS error (code 1)", 3480 "Unable to get TLS issuer certificate (code 2)", 3481 "Unable to get TLS CRL (code 3)", 3482 "Unable to decrypt TLS certificate signature (code 4)", 3483 "Unable to decrypt TLS CRL signature (code 5)", 3484 "Unable to decode TLS issuer public key (code 6)", 3485 "TLS certificate signature failure (code 7)", 3486 "TLS CRL signature failure (code 8)", 3487 "TLS certificate not yet valid (code 9)", 3488 "TLS certificate expired (code 10)", 3489 "TLS CRL not yet valid (code 11)", 3490 "TLS CRL expired (code 12)", 3491 "TLS error in certificate not before field (code 13)", 3492 "TLS error in certificate not after field (code 14)", 3493 "TLS error in CRL last update field (code 15)", 3494 "TLS error in CRL next update field (code 16)", 3495 "TLS system out of memory (code 17)", 3496 "TLS certificate is self-signed (code 18)", 3497 "Self-signed certificate in TLS chain (code 19)", 3498 "Unable to get TLS issuer certificate locally (code 20)", 3499 "Unable to verify TLS leaf signature (code 21)", 3500 "TLS certificate chain too long (code 22)", 3501 "TLS certificate was revoked (code 23)", 3502 "TLS CA is invalid (code 24)", 3503 "TLS error: path length exceeded (code 25)", 3504 "TLS error: invalid purpose (code 26)", 3505 "TLS error: certificate untrusted (code 27)", 3506 "TLS error: certificate rejected (code 28)", 3507 ]; 3508 3509 string getOpenSslErrorCode(long error) { 3510 if(error == 62) 3511 return "TLS certificate host name mismatch"; 3512 3513 if(error < 0 || error >= sslErrorCodes.length) 3514 return "SSL/TLS error code " ~ to!string(error); 3515 return sslErrorCodes[cast(size_t) error]; 3516 } 3517 3518 struct SSL; 3519 struct SSL_CTX; 3520 struct SSL_METHOD; 3521 struct X509_STORE_CTX; 3522 enum SSL_VERIFY_NONE = 0; 3523 enum SSL_VERIFY_PEER = 1; 3524 3525 // copy it into the buf[0 .. size] and return actual length you read. 3526 // rwflag == 0 when reading, 1 when writing. 3527 extern(C) alias pem_password_cb = int function(char* buffer, int bufferSize, int rwflag, void* userPointer); 3528 extern(C) alias print_errors_cb = int function(const char*, size_t, void*); 3529 extern(C) alias client_cert_cb = int function(SSL *ssl, X509 **x509, EVP_PKEY **pkey); 3530 extern(C) alias keylog_cb = void function(SSL*, char*); 3531 3532 struct X509; 3533 struct X509_STORE; 3534 struct EVP_PKEY; 3535 struct X509_VERIFY_PARAM; 3536 3537 import core.stdc.config; 3538 3539 enum SSL_ERROR_WANT_READ = 2; 3540 enum SSL_ERROR_WANT_WRITE = 3; 3541 3542 struct ossllib { 3543 __gshared static extern(C) { 3544 /* these are only on older openssl versions { */ 3545 int function() SSL_library_init; 3546 void function() SSL_load_error_strings; 3547 SSL_METHOD* function() SSLv23_client_method; 3548 /* } */ 3549 3550 void function(ulong, void*) OPENSSL_init_ssl; 3551 3552 SSL_CTX* function(const SSL_METHOD*) SSL_CTX_new; 3553 SSL* function(SSL_CTX*) SSL_new; 3554 int function(SSL*, int) SSL_set_fd; 3555 int function(SSL*) SSL_connect; 3556 int function(SSL*, const void*, int) SSL_write; 3557 int function(SSL*, void*, int) SSL_read; 3558 @trusted nothrow @nogc int function(SSL*) SSL_shutdown; 3559 void function(SSL*) SSL_free; 3560 void function(SSL_CTX*) SSL_CTX_free; 3561 3562 int function(const SSL*) SSL_pending; 3563 int function (const SSL *ssl, int ret) SSL_get_error; 3564 3565 void function(SSL*, int, void*) SSL_set_verify; 3566 3567 void function(SSL*, int, c_long, void*) SSL_ctrl; 3568 3569 SSL_METHOD* function() SSLv3_client_method; 3570 SSL_METHOD* function() TLS_client_method; 3571 3572 void function(SSL_CTX*, void function(SSL*, char* line)) SSL_CTX_set_keylog_callback; 3573 3574 int function(SSL_CTX*) SSL_CTX_set_default_verify_paths; 3575 3576 X509_STORE* function(SSL_CTX*) SSL_CTX_get_cert_store; 3577 c_long function(const SSL* ssl) SSL_get_verify_result; 3578 3579 X509_VERIFY_PARAM* function(const SSL*) SSL_get0_param; 3580 3581 /+ 3582 SSL_CTX_load_verify_locations 3583 SSL_CTX_set_client_CA_list 3584 +/ 3585 3586 // client cert things 3587 void function (SSL_CTX *ctx, int function(SSL *ssl, X509 **x509, EVP_PKEY **pkey)) SSL_CTX_set_client_cert_cb; 3588 } 3589 } 3590 3591 struct eallib { 3592 __gshared static extern(C) { 3593 /* these are only on older openssl versions { */ 3594 void function() OpenSSL_add_all_ciphers; 3595 void function() OpenSSL_add_all_digests; 3596 /* } */ 3597 3598 const(char)* function(int) OpenSSL_version; 3599 3600 void function(ulong, void*) OPENSSL_init_crypto; 3601 3602 void function(print_errors_cb, void*) ERR_print_errors_cb; 3603 3604 void function(X509*) X509_free; 3605 int function(X509_STORE*, X509*) X509_STORE_add_cert; 3606 3607 3608 X509* function(FILE *fp, X509 **x, pem_password_cb *cb, void *u) PEM_read_X509; 3609 EVP_PKEY* function(FILE *fp, EVP_PKEY **x, pem_password_cb *cb, void* userPointer) PEM_read_PrivateKey; 3610 3611 EVP_PKEY* function(FILE *fp, EVP_PKEY **a) d2i_PrivateKey_fp; 3612 X509* function(FILE *fp, X509 **x) d2i_X509_fp; 3613 3614 X509* function(X509** a, const(ubyte*)* pp, c_long length) d2i_X509; 3615 int function(X509* a, ubyte** o) i2d_X509; 3616 3617 int function(X509_VERIFY_PARAM* a, const char* b, size_t l) X509_VERIFY_PARAM_set1_host; 3618 3619 X509* function(X509_STORE_CTX *ctx) X509_STORE_CTX_get_current_cert; 3620 int function(X509_STORE_CTX *ctx) X509_STORE_CTX_get_error; 3621 } 3622 } 3623 3624 struct OpenSSL { 3625 static: 3626 3627 template opDispatch(string name) { 3628 auto opDispatch(T...)(T t) { 3629 static if(__traits(hasMember, ossllib, name)) { 3630 auto ptr = __traits(getMember, ossllib, name); 3631 } else static if(__traits(hasMember, eallib, name)) { 3632 auto ptr = __traits(getMember, eallib, name); 3633 } else static assert(0); 3634 3635 if(ptr is null) 3636 throw new Exception(name ~ " not loaded"); 3637 return ptr(t); 3638 } 3639 } 3640 3641 // macros in the original C 3642 SSL_METHOD* SSLv23_client_method() { 3643 if(ossllib.SSLv23_client_method) 3644 return ossllib.SSLv23_client_method(); 3645 else 3646 return ossllib.TLS_client_method(); 3647 } 3648 3649 void SSL_set_tlsext_host_name(SSL* a, const char* b) { 3650 if(ossllib.SSL_ctrl) 3651 return ossllib.SSL_ctrl(a, 55 /*SSL_CTRL_SET_TLSEXT_HOSTNAME*/, 0 /*TLSEXT_NAMETYPE_host_name*/, cast(void*) b); 3652 else throw new Exception("SSL_set_tlsext_host_name not loaded"); 3653 } 3654 3655 // special case 3656 @trusted nothrow @nogc int SSL_shutdown(SSL* a) { 3657 if(ossllib.SSL_shutdown) 3658 return ossllib.SSL_shutdown(a); 3659 assert(0); 3660 } 3661 3662 void SSL_CTX_keylog_cb_func(SSL_CTX* ctx, keylog_cb func) { 3663 // this isn't in openssl 1.0 and is non-essential, so it is allowed to fail. 3664 if(ossllib.SSL_CTX_set_keylog_callback) 3665 ossllib.SSL_CTX_set_keylog_callback(ctx, func); 3666 //else throw new Exception("SSL_CTX_keylog_cb_func not loaded"); 3667 } 3668 3669 } 3670 3671 extern(C) 3672 int collectSslErrors(const char* ptr, size_t len, void* user) @trusted { 3673 string* s = cast(string*) user; 3674 3675 (*s) ~= ptr[0 .. len]; 3676 3677 return 0; 3678 } 3679 3680 3681 private __gshared void* ossllib_handle; 3682 version(Windows) 3683 private __gshared void* oeaylib_handle; 3684 else 3685 alias oeaylib_handle = ossllib_handle; 3686 version(Posix) 3687 private import core.sys.posix.dlfcn; 3688 else version(Windows) 3689 private import core.sys.windows.windows; 3690 3691 import core.stdc.stdio; 3692 3693 private __gshared Object loadSslMutex = new Object; 3694 private __gshared bool sslLoaded = false; 3695 3696 void loadOpenSsl() { 3697 if(sslLoaded) 3698 return; 3699 synchronized(loadSslMutex) { 3700 3701 version(Posix) { 3702 version(OSX) { 3703 static immutable string[] ossllibs = [ 3704 "libssl.46.dylib", 3705 "libssl.44.dylib", 3706 "libssl.43.dylib", 3707 "libssl.35.dylib", 3708 "libssl.1.1.dylib", 3709 "libssl.dylib", 3710 "/usr/local/opt/openssl/lib/libssl.1.0.0.dylib", 3711 ]; 3712 } else { 3713 static immutable string[] ossllibs = [ 3714 "libssl.so.3", 3715 "libssl.so.1.1", 3716 "libssl.so.1.0.2", 3717 "libssl.so.1.0.1", 3718 "libssl.so.1.0.0", 3719 "libssl.so", 3720 ]; 3721 } 3722 3723 foreach(lib; ossllibs) { 3724 ossllib_handle = dlopen(lib.ptr, RTLD_NOW); 3725 if(ossllib_handle !is null) break; 3726 } 3727 } else version(Windows) { 3728 version(X86_64) { 3729 ossllib_handle = LoadLibraryW("libssl-1_1-x64.dll"w.ptr); 3730 oeaylib_handle = LoadLibraryW("libcrypto-1_1-x64.dll"w.ptr); 3731 } 3732 3733 static immutable wstring[] ossllibs = [ 3734 "libssl-3-x64.dll"w, 3735 "libssl-3.dll"w, 3736 "libssl-1_1.dll"w, 3737 "libssl32.dll"w, 3738 ]; 3739 3740 if(ossllib_handle is null) 3741 foreach(lib; ossllibs) { 3742 ossllib_handle = LoadLibraryW(lib.ptr); 3743 if(ossllib_handle !is null) break; 3744 } 3745 3746 static immutable wstring[] eaylibs = [ 3747 "libcrypto-3-x64.dll"w, 3748 "libcrypto-3.dll"w, 3749 "libcrypto-1_1.dll"w, 3750 "libeay32.dll", 3751 ]; 3752 3753 if(oeaylib_handle is null) 3754 foreach(lib; eaylibs) { 3755 oeaylib_handle = LoadLibraryW(lib.ptr); 3756 if (oeaylib_handle !is null) break; 3757 } 3758 3759 if(ossllib_handle is null) { 3760 ossllib_handle = LoadLibraryW("ssleay32.dll"w.ptr); 3761 oeaylib_handle = ossllib_handle; 3762 } 3763 } 3764 3765 if(ossllib_handle is null) 3766 throw new Exception("libssl library not found"); 3767 if(oeaylib_handle is null) 3768 throw new Exception("libeay32 library not found"); 3769 3770 foreach(memberName; __traits(allMembers, ossllib)) { 3771 alias t = typeof(__traits(getMember, ossllib, memberName)); 3772 version(Posix) 3773 __traits(getMember, ossllib, memberName) = cast(t) dlsym(ossllib_handle, memberName); 3774 else version(Windows) { 3775 __traits(getMember, ossllib, memberName) = cast(t) GetProcAddress(ossllib_handle, memberName); 3776 } 3777 } 3778 3779 foreach(memberName; __traits(allMembers, eallib)) { 3780 alias t = typeof(__traits(getMember, eallib, memberName)); 3781 version(Posix) 3782 __traits(getMember, eallib, memberName) = cast(t) dlsym(oeaylib_handle, memberName); 3783 else version(Windows) { 3784 __traits(getMember, eallib, memberName) = cast(t) GetProcAddress(oeaylib_handle, memberName); 3785 } 3786 } 3787 3788 3789 if(ossllib.SSL_library_init) 3790 ossllib.SSL_library_init(); 3791 else if(ossllib.OPENSSL_init_ssl) 3792 ossllib.OPENSSL_init_ssl(0, null); 3793 else throw new Exception("couldn't init openssl"); 3794 3795 if(eallib.OpenSSL_add_all_ciphers) { 3796 eallib.OpenSSL_add_all_ciphers(); 3797 if(eallib.OpenSSL_add_all_digests is null) 3798 throw new Exception("no add digests"); 3799 eallib.OpenSSL_add_all_digests(); 3800 } else if(eallib.OPENSSL_init_crypto) 3801 eallib.OPENSSL_init_crypto(0 /*OPENSSL_INIT_ADD_ALL_CIPHERS and ALL_DIGESTS together*/, null); 3802 else throw new Exception("couldn't init crypto openssl"); 3803 3804 if(ossllib.SSL_load_error_strings) 3805 ossllib.SSL_load_error_strings(); 3806 else if(ossllib.OPENSSL_init_ssl) 3807 ossllib.OPENSSL_init_ssl(0x00200000L, null); 3808 else throw new Exception("couldn't load openssl errors"); 3809 3810 sslLoaded = true; 3811 } 3812 } 3813 3814 /+ 3815 // I'm just gonna let the OS clean this up on process termination because otherwise SSL_free 3816 // might have trouble being run from the GC after this module is unloaded. 3817 shared static ~this() { 3818 if(ossllib_handle) { 3819 version(Windows) { 3820 FreeLibrary(oeaylib_handle); 3821 FreeLibrary(ossllib_handle); 3822 } else version(Posix) 3823 dlclose(ossllib_handle); 3824 ossllib_handle = null; 3825 } 3826 ossllib.tupleof = ossllib.tupleof.init; 3827 } 3828 +/ 3829 3830 //pragma(lib, "crypto"); 3831 //pragma(lib, "ssl"); 3832 extern(C) 3833 void write_to_file(SSL* ssl, char* line) 3834 { 3835 import std.stdio; 3836 import std.string; 3837 import std.process : environment; 3838 string logfile = environment.get("SSLKEYLOGFILE"); 3839 if (logfile !is null) 3840 { 3841 auto f = std.stdio.File(logfile, "a+"); 3842 f.writeln(fromStringz(line)); 3843 f.close(); 3844 } 3845 } 3846 3847 class OpenSslSocket : Socket { 3848 private SSL* ssl; 3849 private SSL_CTX* ctx; 3850 private void initSsl(bool verifyPeer, string hostname) { 3851 ctx = OpenSSL.SSL_CTX_new(OpenSSL.SSLv23_client_method()); 3852 assert(ctx !is null); 3853 3854 debug OpenSSL.SSL_CTX_keylog_cb_func(ctx, &write_to_file); 3855 ssl = OpenSSL.SSL_new(ctx); 3856 3857 if(hostname.length) { 3858 OpenSSL.SSL_set_tlsext_host_name(ssl, toStringz(hostname)); 3859 if(verifyPeer) 3860 OpenSSL.X509_VERIFY_PARAM_set1_host(OpenSSL.SSL_get0_param(ssl), hostname.ptr, hostname.length); 3861 } 3862 3863 if(verifyPeer) { 3864 OpenSSL.SSL_CTX_set_default_verify_paths(ctx); 3865 3866 version(Windows) { 3867 loadCertificatesFromRegistry(ctx); 3868 } 3869 3870 OpenSSL.SSL_set_verify(ssl, SSL_VERIFY_PEER, &verifyCertificateFromRegistryArsdHttp); 3871 } else 3872 OpenSSL.SSL_set_verify(ssl, SSL_VERIFY_NONE, null); 3873 3874 OpenSSL.SSL_set_fd(ssl, cast(int) this.handle); // on win64 it is necessary to truncate, but the value is never large anyway see http://openssl.6102.n7.nabble.com/Sockets-windows-64-bit-td36169.html 3875 3876 3877 OpenSSL.SSL_CTX_set_client_cert_cb(ctx, &cb); 3878 } 3879 3880 extern(C) 3881 static int cb(SSL* ssl, X509** x509, EVP_PKEY** pkey) { 3882 if(HttpClient.certFilename.length && HttpClient.keyFilename.length) { 3883 FILE* fpCert = fopen((HttpClient.certFilename ~ "\0").ptr, "rb"); 3884 if(fpCert is null) 3885 return 0; 3886 scope(exit) 3887 fclose(fpCert); 3888 FILE* fpKey = fopen((HttpClient.keyFilename ~ "\0").ptr, "rb"); 3889 if(fpKey is null) 3890 return 0; 3891 scope(exit) 3892 fclose(fpKey); 3893 3894 with(CertificateFileFormat) 3895 final switch(HttpClient.certFormat) { 3896 case guess: 3897 if(HttpClient.certFilename.endsWith(".pem") || HttpClient.keyFilename.endsWith(".pem")) 3898 goto case pem; 3899 else 3900 goto case der; 3901 case pem: 3902 *x509 = OpenSSL.PEM_read_X509(fpCert, null, null, null); 3903 *pkey = OpenSSL.PEM_read_PrivateKey(fpKey, null, null, null); 3904 break; 3905 case der: 3906 *x509 = OpenSSL.d2i_X509_fp(fpCert, null); 3907 *pkey = OpenSSL.d2i_PrivateKey_fp(fpKey, null); 3908 break; 3909 } 3910 3911 return 1; 3912 } 3913 3914 return 0; 3915 } 3916 3917 bool dataPending() { 3918 return OpenSSL.SSL_pending(ssl) > 0; 3919 } 3920 3921 @trusted 3922 override void connect(Address to) { 3923 super.connect(to); 3924 if(blocking) { 3925 do_ssl_connect(); 3926 } 3927 } 3928 3929 @trusted 3930 // returns true if it is finished, false if it would have blocked, throws if there's an error 3931 int do_ssl_connect() { 3932 if(OpenSSL.SSL_connect(ssl) == -1) { 3933 3934 auto errCode = OpenSSL.SSL_get_error(ssl, -1); 3935 if(errCode == SSL_ERROR_WANT_READ || errCode == SSL_ERROR_WANT_WRITE) { 3936 return errCode; 3937 } 3938 3939 string str; 3940 OpenSSL.ERR_print_errors_cb(&collectSslErrors, &str); 3941 int i; 3942 auto err = OpenSSL.SSL_get_verify_result(ssl); 3943 //printf("wtf\n"); 3944 //scanf("%d\n", i); 3945 throw new Exception("Secure connect failed: " ~ getOpenSslErrorCode(err)); 3946 } 3947 3948 return 0; 3949 } 3950 3951 @trusted 3952 override ptrdiff_t send(scope const(void)[] buf, SocketFlags flags) { 3953 //import std.stdio;writeln(cast(string) buf); 3954 debug(arsd_http2_verbose) writeln("ssl writing ", buf.length); 3955 auto retval = OpenSSL.SSL_write(ssl, buf.ptr, cast(uint) buf.length); 3956 3957 // don't need to throw anymore since it is checked elsewhere 3958 // code useful sometimes for debugging hence commenting instead of deleting 3959 version(none) 3960 if(retval == -1) { 3961 3962 string str; 3963 OpenSSL.ERR_print_errors_cb(&collectSslErrors, &str); 3964 int i; 3965 3966 //printf("wtf\n"); 3967 //scanf("%d\n", i); 3968 3969 throw new Exception("ssl send failed " ~ str); 3970 } 3971 return retval; 3972 3973 } 3974 override ptrdiff_t send(scope const(void)[] buf) { 3975 return send(buf, SocketFlags.NONE); 3976 } 3977 @trusted 3978 override ptrdiff_t receive(scope void[] buf, SocketFlags flags) { 3979 3980 debug(arsd_http2_verbose) writeln("ssl_read before"); 3981 auto retval = OpenSSL.SSL_read(ssl, buf.ptr, cast(int)buf.length); 3982 debug(arsd_http2_verbose) writeln("ssl_read after"); 3983 3984 // don't need to throw anymore since it is checked elsewhere 3985 // code useful sometimes for debugging hence commenting instead of deleting 3986 version(none) 3987 if(retval == -1) { 3988 3989 string str; 3990 OpenSSL.ERR_print_errors_cb(&collectSslErrors, &str); 3991 int i; 3992 3993 //printf("wtf\n"); 3994 //scanf("%d\n", i); 3995 3996 throw new Exception("ssl receive failed " ~ str); 3997 } 3998 return retval; 3999 } 4000 override ptrdiff_t receive(scope void[] buf) { 4001 return receive(buf, SocketFlags.NONE); 4002 } 4003 4004 this(AddressFamily af, SocketType type = SocketType.STREAM, string hostname = null, bool verifyPeer = true) { 4005 version(Windows) __traits(getMember, this, "_blocking") = true; // lol longstanding phobos bug setting this to false on init 4006 super(af, type); 4007 initSsl(verifyPeer, hostname); 4008 } 4009 4010 override void close() scope { 4011 if(ssl) OpenSSL.SSL_shutdown(ssl); 4012 super.close(); 4013 } 4014 4015 this(socket_t sock, AddressFamily af, string hostname, bool verifyPeer = true) { 4016 super(sock, af); 4017 initSsl(verifyPeer, hostname); 4018 } 4019 4020 void freeSsl() { 4021 if(ssl is null) 4022 return; 4023 OpenSSL.SSL_free(ssl); 4024 OpenSSL.SSL_CTX_free(ctx); 4025 ssl = null; 4026 } 4027 4028 ~this() { 4029 freeSsl(); 4030 } 4031 } 4032 } 4033 4034 4035 /++ 4036 An experimental component for working with REST apis. Note that it 4037 is a zero-argument template, so to create one, use `new HttpApiClient!()(args..)` 4038 or you will get "HttpApiClient is used as a type" compile errors. 4039 4040 This will probably not work for you yet, and I might change it significantly. 4041 4042 Requires [arsd.jsvar]. 4043 4044 4045 Here's a snippet to create a pull request on GitHub to Phobos: 4046 4047 --- 4048 auto github = new HttpApiClient!()("https://api.github.com/", "your personal api token here"); 4049 4050 // create the arguments object 4051 // see: https://developer.github.com/v3/pulls/#create-a-pull-request 4052 var args = var.emptyObject; 4053 args.title = "My Pull Request"; 4054 args.head = "yourusername:" ~ branchName; 4055 args.base = "master"; 4056 // note it is ["body"] instead of .body because `body` is a D keyword 4057 args["body"] = "My cool PR is opened by the API!"; 4058 args.maintainer_can_modify = true; 4059 4060 /+ 4061 Fun fact, you can also write that: 4062 4063 var args = [ 4064 "title": "My Pull Request".var, 4065 "head": "yourusername:" ~ branchName.var, 4066 "base" : "master".var, 4067 "body" : "My cool PR is opened by the API!".var, 4068 "maintainer_can_modify": true.var 4069 ]; 4070 4071 Note the .var constructor calls in there. If everything is the same type, you actually don't need that, but here since there's strings and bools, D won't allow the literal without explicit constructors to align them all. 4072 +/ 4073 4074 // this translates to `repos/dlang/phobos/pulls` and sends a POST request, 4075 // containing `args` as json, then immediately grabs the json result and extracts 4076 // the value `html_url` from it. `prUrl` is typed `var`, from arsd.jsvar. 4077 auto prUrl = github.rest.repos.dlang.phobos.pulls.POST(args).result.html_url; 4078 4079 writeln("Created: ", prUrl); 4080 --- 4081 4082 Why use this instead of just building the URL? Well, of course you can! This just makes 4083 it a bit more convenient than string concatenation and manages a few headers for you. 4084 4085 Subtypes could potentially add static type checks too. 4086 +/ 4087 class HttpApiClient() { 4088 import arsd.jsvar; 4089 4090 HttpClient httpClient; 4091 4092 alias HttpApiClientType = typeof(this); 4093 4094 string urlBase; 4095 string oauth2Token; 4096 string submittedContentType; 4097 4098 /++ 4099 Params: 4100 4101 urlBase = The base url for the api. Tends to be something like `https://api.example.com/v2/` or similar. 4102 oauth2Token = the authorization token for the service. You'll have to get it from somewhere else. 4103 submittedContentType = the content-type of POST, PUT, etc. bodies. 4104 httpClient = an injected http client, or null if you want to use a default-constructed one 4105 4106 History: 4107 The `httpClient` param was added on December 26, 2020. 4108 +/ 4109 this(string urlBase, string oauth2Token, string submittedContentType = "application/json", HttpClient httpClient = null) { 4110 if(httpClient is null) 4111 this.httpClient = new HttpClient(); 4112 else 4113 this.httpClient = httpClient; 4114 4115 assert(urlBase[0] == 'h'); 4116 assert(urlBase[$-1] == '/'); 4117 4118 this.urlBase = urlBase; 4119 this.oauth2Token = oauth2Token; 4120 this.submittedContentType = submittedContentType; 4121 } 4122 4123 /// 4124 static struct HttpRequestWrapper { 4125 HttpApiClientType apiClient; /// 4126 HttpRequest request; /// 4127 HttpResponse _response; 4128 4129 /// 4130 this(HttpApiClientType apiClient, HttpRequest request) { 4131 this.apiClient = apiClient; 4132 this.request = request; 4133 } 4134 4135 /// Returns the full [HttpResponse] object so you can inspect the headers 4136 @property HttpResponse response() { 4137 if(_response is HttpResponse.init) 4138 _response = request.waitForCompletion(); 4139 return _response; 4140 } 4141 4142 /++ 4143 Returns the parsed JSON from the body of the response. 4144 4145 Throws on non-2xx responses. 4146 +/ 4147 var result() { 4148 return apiClient.throwOnError(response); 4149 } 4150 4151 alias request this; 4152 } 4153 4154 /// 4155 HttpRequestWrapper request(string uri, HttpVerb requestMethod = HttpVerb.GET, ubyte[] bodyBytes = null) { 4156 if(uri[0] == '/') 4157 uri = uri[1 .. $]; 4158 4159 auto u = Uri(uri).basedOn(Uri(urlBase)); 4160 4161 auto req = httpClient.navigateTo(u, requestMethod); 4162 4163 if(oauth2Token.length) 4164 req.requestParameters.headers ~= "Authorization: Bearer " ~ oauth2Token; 4165 req.requestParameters.contentType = submittedContentType; 4166 req.requestParameters.bodyData = bodyBytes; 4167 4168 return HttpRequestWrapper(this, req); 4169 } 4170 4171 /// 4172 var throwOnError(HttpResponse res) { 4173 if(res.code < 200 || res.code >= 300) 4174 throw new Exception(res.codeText ~ " " ~ res.contentText); 4175 4176 var response = var.fromJson(res.contentText); 4177 if(response.errors) { 4178 throw new Exception(response.errors.toJson()); 4179 } 4180 4181 return response; 4182 } 4183 4184 /// 4185 @property RestBuilder rest() { 4186 return RestBuilder(this, null, null); 4187 } 4188 4189 // hipchat.rest.room["Tech Team"].history 4190 // gives: "/room/Tech%20Team/history" 4191 // 4192 // hipchat.rest.room["Tech Team"].history("page", "12) 4193 /// 4194 static struct RestBuilder { 4195 HttpApiClientType apiClient; 4196 string[] pathParts; 4197 string[2][] queryParts; 4198 this(HttpApiClientType apiClient, string[] pathParts, string[2][] queryParts) { 4199 this.apiClient = apiClient; 4200 this.pathParts = pathParts; 4201 this.queryParts = queryParts; 4202 } 4203 4204 RestBuilder _SELF() { 4205 return this; 4206 } 4207 4208 /// The args are so you can call opCall on the returned 4209 /// object, despite @property being broken af in D. 4210 RestBuilder opDispatch(string str, T)(string n, T v) { 4211 return RestBuilder(apiClient, pathParts ~ str, queryParts ~ [n, to!string(v)]); 4212 } 4213 4214 /// 4215 RestBuilder opDispatch(string str)() { 4216 return RestBuilder(apiClient, pathParts ~ str, queryParts); 4217 } 4218 4219 4220 /// 4221 RestBuilder opIndex(string str) { 4222 return RestBuilder(apiClient, pathParts ~ str, queryParts); 4223 } 4224 /// 4225 RestBuilder opIndex(var str) { 4226 return RestBuilder(apiClient, pathParts ~ str.get!string, queryParts); 4227 } 4228 /// 4229 RestBuilder opIndex(int i) { 4230 return RestBuilder(apiClient, pathParts ~ to!string(i), queryParts); 4231 } 4232 4233 /// 4234 RestBuilder opCall(T)(string name, T value) { 4235 return RestBuilder(apiClient, pathParts, queryParts ~ [name, to!string(value)]); 4236 } 4237 4238 /// 4239 string toUri() { 4240 import std.uri; 4241 string result; 4242 foreach(idx, part; pathParts) { 4243 if(idx) 4244 result ~= "/"; 4245 result ~= encodeComponent(part); 4246 } 4247 result ~= "?"; 4248 foreach(idx, part; queryParts) { 4249 if(idx) 4250 result ~= "&"; 4251 result ~= encodeComponent(part[0]); 4252 result ~= "="; 4253 result ~= encodeComponent(part[1]); 4254 } 4255 4256 return result; 4257 } 4258 4259 /// 4260 final HttpRequestWrapper GET() { return _EXECUTE(HttpVerb.GET, this.toUri(), ToBytesResult.init); } 4261 /// ditto 4262 final HttpRequestWrapper DELETE() { return _EXECUTE(HttpVerb.DELETE, this.toUri(), ToBytesResult.init); } 4263 4264 // need to be able to send: JSON, urlencoded, multipart/form-data, and raw stuff. 4265 /// ditto 4266 final HttpRequestWrapper POST(T...)(T t) { return _EXECUTE(HttpVerb.POST, this.toUri(), toBytes(t)); } 4267 /// ditto 4268 final HttpRequestWrapper PATCH(T...)(T t) { return _EXECUTE(HttpVerb.PATCH, this.toUri(), toBytes(t)); } 4269 /// ditto 4270 final HttpRequestWrapper PUT(T...)(T t) { return _EXECUTE(HttpVerb.PUT, this.toUri(), toBytes(t)); } 4271 4272 struct ToBytesResult { 4273 ubyte[] bytes; 4274 string contentType; 4275 } 4276 4277 private ToBytesResult toBytes(T...)(T t) { 4278 import std.conv : to; 4279 static if(T.length == 0) 4280 return ToBytesResult(null, null); 4281 else static if(T.length == 1 && is(T[0] == var)) 4282 return ToBytesResult(cast(ubyte[]) t[0].toJson(), "application/json"); // json data 4283 else static if(T.length == 1 && (is(T[0] == string) || is(T[0] == ubyte[]))) 4284 return ToBytesResult(cast(ubyte[]) t[0], null); // raw data 4285 else static if(T.length == 1 && is(T[0] : FormData)) 4286 return ToBytesResult(t[0].toBytes, t[0].contentType); 4287 else static if(T.length > 1 && T.length % 2 == 0 && is(T[0] == string)) { 4288 // string -> value pairs for a POST request 4289 string answer; 4290 foreach(idx, val; t) { 4291 static if(idx % 2 == 0) { 4292 if(answer.length) 4293 answer ~= "&"; 4294 answer ~= encodeComponent(val); // it had better be a string! lol 4295 answer ~= "="; 4296 } else { 4297 answer ~= encodeComponent(to!string(val)); 4298 } 4299 } 4300 4301 return ToBytesResult(cast(ubyte[]) answer, "application/x-www-form-urlencoded"); 4302 } 4303 else 4304 static assert(0); // FIXME 4305 4306 } 4307 4308 HttpRequestWrapper _EXECUTE(HttpVerb verb, string uri, ubyte[] bodyBytes) { 4309 return apiClient.request(uri, verb, bodyBytes); 4310 } 4311 4312 HttpRequestWrapper _EXECUTE(HttpVerb verb, string uri, ToBytesResult tbr) { 4313 auto r = apiClient.request(uri, verb, tbr.bytes); 4314 if(tbr.contentType !is null) 4315 r.requestParameters.contentType = tbr.contentType; 4316 return r; 4317 } 4318 } 4319 } 4320 4321 4322 // see also: arsd.cgi.encodeVariables 4323 /++ 4324 Creates a multipart/form-data object that is suitable for file uploads and other kinds of POST. 4325 4326 It has a set of names and values of mime components. Names can be repeated. They will be presented in the same order in which you add them. You will mostly want to use the [append] method. 4327 4328 You can pass this directly to [HttpClient.request]. 4329 4330 Based on: https://developer.mozilla.org/en-US/docs/Web/API/FormData 4331 4332 --- 4333 auto fd = new FormData(); 4334 // add some data, plain string first 4335 fd.append("name", "Adam"); 4336 // then a file 4337 fd.append("photo", std.file.read("adam.jpg"), "image/jpeg", "adam.jpg"); 4338 4339 // post it! 4340 auto client = new HttpClient(); 4341 client.request(Uri("http://example.com/people"), fd).waitForCompletion(); 4342 --- 4343 4344 History: 4345 Added June 8, 2018 4346 +/ 4347 class FormData { 4348 static struct MimePart { 4349 string name; 4350 const(void)[] data; 4351 string contentType; 4352 string filename; 4353 } 4354 4355 private MimePart[] parts; 4356 private string boundary = "0016e64be86203dd36047610926a"; // FIXME 4357 4358 /++ 4359 Appends the given entry to the request. This can be a simple key/value pair of strings or file uploads. 4360 4361 For a simple key/value pair, leave `contentType` and `filename` as `null`. 4362 4363 For file uploads, please note that many servers require filename be given for a file upload and it may not allow you to put in a path. I suggest using [std.path.baseName] to strip off path information from a file you are loading. 4364 4365 The `contentType` is generally verified by servers for file uploads. 4366 +/ 4367 void append(string key, const(void)[] value, string contentType = null, string filename = null) { 4368 parts ~= MimePart(key, value, contentType, filename); 4369 } 4370 4371 /++ 4372 Deletes any entries from the set with the given key. 4373 4374 History: 4375 Added June 7, 2023 (dub v11.0) 4376 +/ 4377 void deleteKey(string key) { 4378 MimePart[] newParts; 4379 foreach(part; parts) 4380 if(part.name != key) 4381 newParts ~= part; 4382 parts = newParts; 4383 } 4384 4385 /++ 4386 Returns the first entry with the given key, or `MimePart.init` if there is nothing. 4387 4388 History: 4389 Added June 7, 2023 (dub v11.0) 4390 +/ 4391 MimePart get(string key) { 4392 foreach(part; parts) 4393 if(part.name == key) 4394 return part; 4395 return MimePart.init; 4396 } 4397 4398 /++ 4399 Returns the all entries with the given key. 4400 4401 History: 4402 Added June 7, 2023 (dub v11.0) 4403 +/ 4404 MimePart[] getAll(string key) { 4405 MimePart[] answer; 4406 foreach(part; parts) 4407 if(part.name == key) 4408 answer ~= part; 4409 return answer; 4410 } 4411 4412 /++ 4413 Returns true if the given key exists in the set. 4414 4415 History: 4416 Added June 7, 2023 (dub v11.0) 4417 +/ 4418 bool has(string key) { 4419 return get(key).name == key; 4420 } 4421 4422 /++ 4423 Sets the given key to the given value if it exists, or appends it if it doesn't. 4424 4425 You probably want [append] instead. 4426 4427 See_Also: 4428 [append] 4429 4430 History: 4431 Added June 7, 2023 (dub v11.0) 4432 +/ 4433 void set(string key, const(void)[] value, string contentType, string filename) { 4434 foreach(ref part; parts) 4435 if(part.name == key) { 4436 part.data = value; 4437 part.contentType = contentType; 4438 part.filename = filename; 4439 return; 4440 } 4441 4442 append(key, value, contentType, filename); 4443 } 4444 4445 /++ 4446 Returns all the current entries in the object. 4447 4448 History: 4449 Added June 7, 2023 (dub v11.0) 4450 +/ 4451 MimePart[] entries() { 4452 return parts; 4453 } 4454 4455 // FIXME: 4456 // keys iterator 4457 // values iterator 4458 4459 /++ 4460 Gets the content type header that should be set in the request. This includes the type and boundary that is applicable to the [toBytes] method. 4461 +/ 4462 string contentType() { 4463 return "multipart/form-data; boundary=" ~ boundary; 4464 } 4465 4466 /++ 4467 Returns bytes applicable for the body of this request. Use the [contentType] method to get the appropriate content type header with the right boundary. 4468 +/ 4469 ubyte[] toBytes() { 4470 string data; 4471 4472 foreach(part; parts) { 4473 data ~= "--" ~ boundary ~ "\r\n"; 4474 data ~= "Content-Disposition: form-data; name=\""~part.name~"\""; 4475 if(part.filename !is null) 4476 data ~= "; filename=\""~part.filename~"\""; 4477 data ~= "\r\n"; 4478 if(part.contentType !is null) 4479 data ~= "Content-Type: " ~ part.contentType ~ "\r\n"; 4480 data ~= "\r\n"; 4481 4482 data ~= cast(string) part.data; 4483 4484 data ~= "\r\n"; 4485 } 4486 4487 data ~= "--" ~ boundary ~ "--\r\n"; 4488 4489 return cast(ubyte[]) data; 4490 } 4491 } 4492 4493 private bool bicmp(in ubyte[] item, in char[] search) { 4494 if(item.length != search.length) return false; 4495 4496 foreach(i; 0 .. item.length) { 4497 ubyte a = item[i]; 4498 ubyte b = search[i]; 4499 if(a >= 'A' && a <= 'Z') 4500 a += 32; 4501 //if(b >= 'A' && b <= 'Z') 4502 //b += 32; 4503 if(a != b) 4504 return false; 4505 } 4506 4507 return true; 4508 } 4509 4510 /++ 4511 WebSocket client, based on the browser api, though also with other api options. 4512 4513 --- 4514 import arsd.http2; 4515 4516 void main() { 4517 auto ws = new WebSocket(Uri("ws://....")); 4518 4519 ws.onmessage = (in char[] msg) { 4520 ws.send("a reply"); 4521 }; 4522 4523 ws.connect(); 4524 4525 WebSocket.eventLoop(); 4526 } 4527 --- 4528 4529 Symbol_groups: 4530 foundational = 4531 Used with all API styles. 4532 4533 browser_api = 4534 API based on the standard in the browser. 4535 4536 event_loop_integration = 4537 Integrating with external event loops is done through static functions. You should 4538 call these BEFORE doing anything else with the WebSocket module or class. 4539 4540 $(PITFALL NOT IMPLEMENTED) 4541 --- 4542 WebSocket.setEventLoopProxy(arsd.simpledisplay.EventLoop.proxy.tupleof); 4543 // or something like that. it is not implemented yet. 4544 --- 4545 $(PITFALL NOT IMPLEMENTED) 4546 4547 blocking_api = 4548 The blocking API is best used when you only need basic functionality with a single connection. 4549 4550 --- 4551 WebSocketFrame msg; 4552 do { 4553 // FIXME good demo 4554 } while(msg); 4555 --- 4556 4557 Or to check for blocks before calling: 4558 4559 --- 4560 try_to_process_more: 4561 while(ws.isMessageBuffered()) { 4562 auto msg = ws.waitForNextMessage(); 4563 // process msg 4564 } 4565 if(ws.isDataPending()) { 4566 ws.lowLevelReceive(); 4567 goto try_to_process_more; 4568 } else { 4569 // nothing ready, you can do other things 4570 // or at least sleep a while before trying 4571 // to process more. 4572 if(ws.readyState == WebSocket.OPEN) { 4573 Thread.sleep(1.seconds); 4574 goto try_to_process_more; 4575 } 4576 } 4577 --- 4578 4579 +/ 4580 class WebSocket { 4581 private Uri uri; 4582 private string[string] cookies; 4583 4584 private string host; 4585 private ushort port; 4586 private bool ssl; 4587 4588 // used to decide if we mask outgoing msgs 4589 private bool isClient; 4590 4591 private MonoTime timeoutFromInactivity; 4592 private MonoTime nextPing; 4593 4594 /++ 4595 wss://echo.websocket.org 4596 +/ 4597 /// Group: foundational 4598 this(Uri uri, Config config = Config.init) 4599 //in (uri.scheme == "ws" || uri.scheme == "wss") 4600 in { assert(uri.scheme == "ws" || uri.scheme == "wss"); } do 4601 { 4602 this.uri = uri; 4603 this.config = config; 4604 4605 this.receiveBuffer = new ubyte[](config.initialReceiveBufferSize); 4606 4607 host = uri.host; 4608 ssl = uri.scheme == "wss"; 4609 port = cast(ushort) (uri.port ? uri.port : ssl ? 443 : 80); 4610 4611 if(ssl) { 4612 version(with_openssl) { 4613 loadOpenSsl(); 4614 socket = new SslClientSocket(family(uri.unixSocketPath), SocketType.STREAM, host, config.verifyPeer); 4615 } else 4616 throw new Exception("SSL not compiled in"); 4617 } else 4618 socket = new Socket(family(uri.unixSocketPath), SocketType.STREAM); 4619 4620 socket.setOption(SocketOptionLevel.TCP, SocketOption.TCP_NODELAY, 1); 4621 cookies = config.cookies; 4622 } 4623 4624 /++ 4625 4626 +/ 4627 /// Group: foundational 4628 void connect() { 4629 this.isClient = true; 4630 4631 socket.blocking = false; 4632 4633 if(uri.unixSocketPath) 4634 socket.connect(new UnixAddress(uri.unixSocketPath)); 4635 else 4636 socket.connect(new InternetAddress(host, port)); // FIXME: ipv6 support... 4637 4638 4639 auto readSet = new SocketSet(); 4640 auto writeSet = new SocketSet(); 4641 4642 readSet.reset(); 4643 writeSet.reset(); 4644 4645 readSet.add(socket); 4646 writeSet.add(socket); 4647 4648 auto selectGot = Socket.select(readSet, writeSet, null, config.timeoutFromInactivity); 4649 if(selectGot == -1) { 4650 // interrupted 4651 4652 throw new Exception("Websocket connection interrupted - retry might succeed"); 4653 } else if(selectGot == 0) { 4654 // time out 4655 socket.close(); 4656 throw new Exception("Websocket connection timed out"); 4657 } else { 4658 if(writeSet.isSet(socket) || readSet.isSet(socket)) { 4659 import core.stdc.stdint; 4660 int32_t error; 4661 int retopt = socket.getOption(SocketOptionLevel.SOCKET, SocketOption.ERROR, error); 4662 if(retopt < 0 || error != 0) { 4663 socket.close(); 4664 throw new Exception("Websocket connection failed - " ~ formatSocketError(error)); 4665 } else { 4666 // FIXME: websocket handshake could and really should be async too. 4667 socket.blocking = true; // just convenience 4668 if(auto s = cast(SslClientSocket) socket) { 4669 s.do_ssl_connect(); 4670 } else { 4671 // we're ready 4672 } 4673 } 4674 } 4675 } 4676 4677 auto uri = this.uri.path.length ? this.uri.path : "/"; 4678 if(this.uri.query.length) { 4679 uri ~= "?"; 4680 uri ~= this.uri.query; 4681 } 4682 4683 // the headers really shouldn't be bigger than this, at least 4684 // the chunks i need to process 4685 ubyte[4096] bufferBacking = void; 4686 ubyte[] buffer = bufferBacking[]; 4687 size_t pos; 4688 4689 void append(in char[][] items...) { 4690 foreach(what; items) { 4691 if((pos + what.length) > buffer.length) { 4692 buffer.length += 4096; 4693 } 4694 buffer[pos .. pos + what.length] = cast(ubyte[]) what[]; 4695 pos += what.length; 4696 } 4697 } 4698 4699 append("GET ", uri, " HTTP/1.1\r\n"); 4700 append("Host: ", this.uri.host, "\r\n"); 4701 4702 append("Upgrade: websocket\r\n"); 4703 append("Connection: Upgrade\r\n"); 4704 append("Sec-WebSocket-Version: 13\r\n"); 4705 4706 // FIXME: randomize this 4707 append("Sec-WebSocket-Key: x3JEHMbDL1EzLkh9GBhXDw==\r\n"); 4708 if(cookies.length > 0) { 4709 append("Cookie: "); 4710 bool first=true; 4711 foreach(k,v;cookies) { 4712 if(first) first = false; 4713 else append("; "); 4714 append(k); 4715 append("="); 4716 append(v); 4717 } 4718 append("\r\n"); 4719 } 4720 /* 4721 //This is equivalent but has dependencies 4722 import std.format; 4723 import std.algorithm : map; 4724 append(format("cookie: %-(%s %)\r\n",cookies.byKeyValue.map!(t=>format("%s=%s",t.key,t.value)))); 4725 */ 4726 4727 if(config.protocol.length) 4728 append("Sec-WebSocket-Protocol: ", config.protocol, "\r\n"); 4729 if(config.origin.length) 4730 append("Origin: ", config.origin, "\r\n"); 4731 4732 foreach(h; config.additionalHeaders) { 4733 append(h); 4734 append("\r\n"); 4735 } 4736 4737 append("\r\n"); 4738 4739 auto remaining = buffer[0 .. pos]; 4740 //import std.stdio; writeln(host, " " , port, " ", cast(string) remaining); 4741 while(remaining.length) { 4742 auto r = socket.send(remaining); 4743 if(r < 0) 4744 throw new Exception(lastSocketError()); 4745 if(r == 0) 4746 throw new Exception("unexpected connection termination"); 4747 remaining = remaining[r .. $]; 4748 } 4749 4750 // the response shouldn't be especially large at this point, just 4751 // headers for the most part. gonna try to get it in the stack buffer. 4752 // then copy stuff after headers, if any, to the frame buffer. 4753 ubyte[] used; 4754 4755 void more() { 4756 auto r = socket.receive(buffer[used.length .. $]); 4757 4758 if(r < 0) 4759 throw new Exception(lastSocketError()); 4760 if(r == 0) 4761 throw new Exception("unexpected connection termination"); 4762 //import std.stdio;writef("%s", cast(string) buffer[used.length .. used.length + r]); 4763 4764 used = buffer[0 .. used.length + r]; 4765 } 4766 4767 more(); 4768 4769 import std.algorithm; 4770 if(!used.startsWith(cast(ubyte[]) "HTTP/1.1 101")) 4771 throw new Exception("didn't get a websocket answer"); 4772 // skip the status line 4773 while(used.length && used[0] != '\n') 4774 used = used[1 .. $]; 4775 4776 if(used.length == 0) 4777 throw new Exception("Remote server disconnected or didn't send enough information"); 4778 4779 if(used.length < 1) 4780 more(); 4781 4782 used = used[1 .. $]; // skip the \n 4783 4784 if(used.length == 0) 4785 more(); 4786 4787 // checks on the protocol from ehaders 4788 bool isWebsocket; 4789 bool isUpgrade; 4790 const(ubyte)[] protocol; 4791 const(ubyte)[] accept; 4792 4793 while(used.length) { 4794 if(used.length >= 2 && used[0] == '\r' && used[1] == '\n') { 4795 used = used[2 .. $]; 4796 break; // all done 4797 } 4798 int idxColon; 4799 while(idxColon < used.length && used[idxColon] != ':') 4800 idxColon++; 4801 if(idxColon == used.length) 4802 more(); 4803 auto idxStart = idxColon + 1; 4804 while(idxStart < used.length && used[idxStart] == ' ') 4805 idxStart++; 4806 if(idxStart == used.length) 4807 more(); 4808 auto idxEnd = idxStart; 4809 while(idxEnd < used.length && used[idxEnd] != '\r') 4810 idxEnd++; 4811 if(idxEnd == used.length) 4812 more(); 4813 4814 auto headerName = used[0 .. idxColon]; 4815 auto headerValue = used[idxStart .. idxEnd]; 4816 4817 // move past this header 4818 used = used[idxEnd .. $]; 4819 // and the \r\n 4820 if(2 <= used.length) 4821 used = used[2 .. $]; 4822 4823 if(headerName.bicmp("upgrade")) { 4824 if(headerValue.bicmp("websocket")) 4825 isWebsocket = true; 4826 } else if(headerName.bicmp("connection")) { 4827 if(headerValue.bicmp("upgrade")) 4828 isUpgrade = true; 4829 } else if(headerName.bicmp("sec-websocket-accept")) { 4830 accept = headerValue; 4831 } else if(headerName.bicmp("sec-websocket-protocol")) { 4832 protocol = headerValue; 4833 } 4834 4835 if(!used.length) { 4836 more(); 4837 } 4838 } 4839 4840 4841 if(!isWebsocket) 4842 throw new Exception("didn't answer as websocket"); 4843 if(!isUpgrade) 4844 throw new Exception("didn't answer as upgrade"); 4845 4846 4847 // FIXME: check protocol if config requested one 4848 // FIXME: check accept for the right hash 4849 4850 receiveBuffer[0 .. used.length] = used[]; 4851 receiveBufferUsedLength = used.length; 4852 4853 readyState_ = OPEN; 4854 4855 if(onopen) 4856 onopen(); 4857 4858 nextPing = MonoTime.currTime + config.pingFrequency.msecs; 4859 timeoutFromInactivity = MonoTime.currTime + config.timeoutFromInactivity; 4860 4861 registerActiveSocket(this); 4862 } 4863 4864 /++ 4865 Is data pending on the socket? Also check [isMessageBuffered] to see if there 4866 is already a message in memory too. 4867 4868 If this returns `true`, you can call [lowLevelReceive], then try [isMessageBuffered] 4869 again. 4870 +/ 4871 /// Group: blocking_api 4872 public bool isDataPending(Duration timeout = 0.seconds) { 4873 static SocketSet readSet; 4874 if(readSet is null) 4875 readSet = new SocketSet(); 4876 4877 version(with_openssl) 4878 if(auto s = cast(SslClientSocket) socket) { 4879 // select doesn't handle the case with stuff 4880 // left in the ssl buffer so i'm checking it separately 4881 if(s.dataPending()) { 4882 return true; 4883 } 4884 } 4885 4886 readSet.reset(); 4887 4888 readSet.add(socket); 4889 4890 //tryAgain: 4891 auto selectGot = Socket.select(readSet, null, null, timeout); 4892 if(selectGot == 0) { /* timeout */ 4893 // timeout 4894 return false; 4895 } else if(selectGot == -1) { /* interrupted */ 4896 return false; 4897 } else { /* ready */ 4898 if(readSet.isSet(socket)) { 4899 return true; 4900 } 4901 } 4902 4903 return false; 4904 } 4905 4906 private void llsend(ubyte[] d) { 4907 if(readyState == CONNECTING) 4908 throw new Exception("WebSocket not connected when trying to send. Did you forget to call connect(); ?"); 4909 //connect(); 4910 //import std.stdio; writeln("LLSEND: ", d); 4911 while(d.length) { 4912 auto r = socket.send(d); 4913 if(r < 0 && wouldHaveBlocked()) { 4914 import core.thread; 4915 Thread.sleep(1.msecs); 4916 continue; 4917 } 4918 //import core.stdc.errno; import std.stdio; writeln(errno); 4919 if(r <= 0) { 4920 // import std.stdio; writeln(GetLastError()); 4921 throw new Exception("Socket send failed"); 4922 } 4923 d = d[r .. $]; 4924 } 4925 } 4926 4927 private void llclose() { 4928 // import std.stdio; writeln("LLCLOSE"); 4929 socket.shutdown(SocketShutdown.SEND); 4930 } 4931 4932 /++ 4933 Waits for more data off the low-level socket and adds it to the pending buffer. 4934 4935 Returns `true` if the connection is still active. 4936 +/ 4937 /// Group: blocking_api 4938 public bool lowLevelReceive() { 4939 if(readyState == CONNECTING) 4940 throw new Exception("WebSocket not connected when trying to receive. Did you forget to call connect(); ?"); 4941 if (receiveBufferUsedLength == receiveBuffer.length) 4942 { 4943 if (receiveBuffer.length == config.maximumReceiveBufferSize) 4944 throw new Exception("Maximum receive buffer size exhausted"); 4945 4946 import std.algorithm : min; 4947 receiveBuffer.length = min(receiveBuffer.length + config.initialReceiveBufferSize, 4948 config.maximumReceiveBufferSize); 4949 } 4950 auto r = socket.receive(receiveBuffer[receiveBufferUsedLength .. $]); 4951 if(r == 0) 4952 return false; 4953 if(r < 0 && wouldHaveBlocked()) 4954 return true; 4955 if(r <= 0) { 4956 //import std.stdio; writeln(WSAGetLastError()); 4957 throw new Exception("Socket receive failed"); 4958 } 4959 receiveBufferUsedLength += r; 4960 return true; 4961 } 4962 4963 private Socket socket; 4964 4965 /* copy/paste section { */ 4966 4967 private int readyState_; 4968 private ubyte[] receiveBuffer; 4969 private size_t receiveBufferUsedLength; 4970 4971 private Config config; 4972 4973 enum CONNECTING = 0; /// Socket has been created. The connection is not yet open. 4974 enum OPEN = 1; /// The connection is open and ready to communicate. 4975 enum CLOSING = 2; /// The connection is in the process of closing. 4976 enum CLOSED = 3; /// The connection is closed or couldn't be opened. 4977 4978 /++ 4979 4980 +/ 4981 /// Group: foundational 4982 static struct Config { 4983 /++ 4984 These control the size of the receive buffer. 4985 4986 It starts at the initial size, will temporarily 4987 balloon up to the maximum size, and will reuse 4988 a buffer up to the likely size. 4989 4990 Anything larger than the maximum size will cause 4991 the connection to be aborted and an exception thrown. 4992 This is to protect you against a peer trying to 4993 exhaust your memory, while keeping the user-level 4994 processing simple. 4995 +/ 4996 size_t initialReceiveBufferSize = 4096; 4997 size_t likelyReceiveBufferSize = 4096; /// ditto 4998 size_t maximumReceiveBufferSize = 10 * 1024 * 1024; /// ditto 4999 5000 /++ 5001 Maximum combined size of a message. 5002 +/ 5003 size_t maximumMessageSize = 10 * 1024 * 1024; 5004 5005 string[string] cookies; /// Cookies to send with the initial request. cookies[name] = value; 5006 string origin; /// Origin URL to send with the handshake, if desired. 5007 string protocol; /// the protocol header, if desired. 5008 5009 /++ 5010 Additional headers to put in the HTTP request. These should be formatted `Name: value`, like for example: 5011 5012 --- 5013 Config config; 5014 config.additionalHeaders ~= "Authorization: Bearer your_auth_token_here"; 5015 --- 5016 5017 History: 5018 Added February 19, 2021 (included in dub version 9.2) 5019 +/ 5020 string[] additionalHeaders; 5021 5022 /++ 5023 Amount of time (in msecs) of idleness after which to send an automatic ping 5024 5025 Please note how this interacts with [timeoutFromInactivity] - a ping counts as activity that 5026 keeps the socket alive. 5027 +/ 5028 int pingFrequency = 5000; 5029 5030 /++ 5031 Amount of time to disconnect when there's no activity. Note that automatic pings will keep the connection alive; this timeout only occurs if there's absolutely nothing, including no responses to websocket ping frames. Since the default [pingFrequency] is only seconds, this one minute should never elapse unless the connection is actually dead. 5032 5033 The one thing to keep in mind is if your program is busy and doesn't check input, it might consider this a time out since there's no activity. The reason is that your program was busy rather than a connection failure, but it doesn't care. You should avoid long processing periods anyway though! 5034 5035 History: 5036 Added March 31, 2021 (included in dub version 9.4) 5037 +/ 5038 Duration timeoutFromInactivity = 1.minutes; 5039 5040 /++ 5041 For https connections, if this is `true`, it will fail to connect if the TLS certificate can not be 5042 verified. Setting this to `false` will skip this check and allow the connection to continue anyway. 5043 5044 History: 5045 Added April 5, 2022 (dub v10.8) 5046 5047 Prior to this, it always used the global (but undocumented) `defaultVerifyPeer` setting, and sometimes 5048 even if it was true, it would skip the verification. Now, it always respects this local setting. 5049 +/ 5050 bool verifyPeer = true; 5051 } 5052 5053 /++ 5054 Returns one of [CONNECTING], [OPEN], [CLOSING], or [CLOSED]. 5055 +/ 5056 int readyState() { 5057 return readyState_; 5058 } 5059 5060 /++ 5061 Closes the connection, sending a graceful teardown message to the other side. 5062 5063 Code 1000 is the normal closure code. 5064 5065 History: 5066 The default `code` was changed to 1000 on January 9, 2023. Previously it was 0, 5067 but also ignored anyway. 5068 +/ 5069 /// Group: foundational 5070 void close(int code = 1000, string reason = null) 5071 //in (reason.length < 123) 5072 in { assert(reason.length < 123); } do 5073 { 5074 if(readyState_ != OPEN) 5075 return; // it cool, we done 5076 WebSocketFrame wss; 5077 wss.fin = true; 5078 wss.masked = this.isClient; 5079 wss.opcode = WebSocketOpcode.close; 5080 wss.data = [ubyte((code >> 8) & 0xff), ubyte(code & 0xff)] ~ cast(ubyte[]) reason.dup; 5081 wss.send(&llsend); 5082 5083 readyState_ = CLOSING; 5084 5085 closeCalled = true; 5086 5087 llclose(); 5088 } 5089 5090 private bool closeCalled; 5091 5092 /++ 5093 Sends a ping message to the server. This is done automatically by the library if you set a non-zero [Config.pingFrequency], but you can also send extra pings explicitly as well with this function. 5094 +/ 5095 /// Group: foundational 5096 void ping(in ubyte[] data = null) { 5097 WebSocketFrame wss; 5098 wss.fin = true; 5099 wss.masked = this.isClient; 5100 wss.opcode = WebSocketOpcode.ping; 5101 if(data !is null) wss.data = data.dup; 5102 wss.send(&llsend); 5103 } 5104 5105 /++ 5106 Sends a pong message to the server. This is normally done automatically in response to pings. 5107 +/ 5108 /// Group: foundational 5109 void pong(in ubyte[] data = null) { 5110 WebSocketFrame wss; 5111 wss.fin = true; 5112 wss.masked = this.isClient; 5113 wss.opcode = WebSocketOpcode.pong; 5114 if(data !is null) wss.data = data.dup; 5115 wss.send(&llsend); 5116 } 5117 5118 /++ 5119 Sends a text message through the websocket. 5120 +/ 5121 /// Group: foundational 5122 void send(in char[] textData) { 5123 WebSocketFrame wss; 5124 wss.fin = true; 5125 wss.masked = this.isClient; 5126 wss.opcode = WebSocketOpcode.text; 5127 wss.data = cast(ubyte[]) textData.dup; 5128 wss.send(&llsend); 5129 } 5130 5131 /++ 5132 Sends a binary message through the websocket. 5133 +/ 5134 /// Group: foundational 5135 void send(in ubyte[] binaryData) { 5136 WebSocketFrame wss; 5137 wss.masked = this.isClient; 5138 wss.fin = true; 5139 wss.opcode = WebSocketOpcode.binary; 5140 wss.data = cast(ubyte[]) binaryData.dup; 5141 wss.send(&llsend); 5142 } 5143 5144 /++ 5145 Waits for and returns the next complete message on the socket. 5146 5147 Note that the onmessage function is still called, right before 5148 this returns. 5149 +/ 5150 /// Group: blocking_api 5151 public WebSocketFrame waitForNextMessage() { 5152 do { 5153 auto m = processOnce(); 5154 if(m.populated) 5155 return m; 5156 } while(lowLevelReceive()); 5157 5158 return WebSocketFrame.init; // FIXME? maybe. 5159 } 5160 5161 /++ 5162 Tells if [waitForNextMessage] would block. 5163 +/ 5164 /// Group: blocking_api 5165 public bool waitForNextMessageWouldBlock() { 5166 checkAgain: 5167 if(isMessageBuffered()) 5168 return false; 5169 if(!isDataPending()) 5170 return true; 5171 while(isDataPending()) 5172 lowLevelReceive(); 5173 goto checkAgain; 5174 } 5175 5176 /++ 5177 Is there a message in the buffer already? 5178 If `true`, [waitForNextMessage] is guaranteed to return immediately. 5179 If `false`, check [isDataPending] as the next step. 5180 +/ 5181 /// Group: blocking_api 5182 public bool isMessageBuffered() { 5183 ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; 5184 auto s = d; 5185 if(d.length) { 5186 auto orig = d; 5187 auto m = WebSocketFrame.read(d); 5188 // that's how it indicates that it needs more data 5189 if(d !is orig) 5190 return true; 5191 } 5192 5193 return false; 5194 } 5195 5196 private ubyte continuingType; 5197 private ubyte[] continuingData; 5198 //private size_t continuingDataLength; 5199 5200 private WebSocketFrame processOnce() { 5201 ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; 5202 auto s = d; 5203 // FIXME: handle continuation frames more efficiently. it should really just reuse the receive buffer. 5204 WebSocketFrame m; 5205 if(d.length) { 5206 auto orig = d; 5207 m = WebSocketFrame.read(d); 5208 // that's how it indicates that it needs more data 5209 if(d is orig) 5210 return WebSocketFrame.init; 5211 m.unmaskInPlace(); 5212 switch(m.opcode) { 5213 case WebSocketOpcode.continuation: 5214 if(continuingData.length + m.data.length > config.maximumMessageSize) 5215 throw new Exception("message size exceeded"); 5216 5217 continuingData ~= m.data; 5218 if(m.fin) { 5219 if(ontextmessage) 5220 ontextmessage(cast(char[]) continuingData); 5221 if(onbinarymessage) 5222 onbinarymessage(continuingData); 5223 5224 continuingData = null; 5225 } 5226 break; 5227 case WebSocketOpcode.text: 5228 if(m.fin) { 5229 if(ontextmessage) 5230 ontextmessage(m.textData); 5231 } else { 5232 continuingType = m.opcode; 5233 //continuingDataLength = 0; 5234 continuingData = null; 5235 continuingData ~= m.data; 5236 } 5237 break; 5238 case WebSocketOpcode.binary: 5239 if(m.fin) { 5240 if(onbinarymessage) 5241 onbinarymessage(m.data); 5242 } else { 5243 continuingType = m.opcode; 5244 //continuingDataLength = 0; 5245 continuingData = null; 5246 continuingData ~= m.data; 5247 } 5248 break; 5249 case WebSocketOpcode.close: 5250 5251 //import std.stdio; writeln("closed ", cast(string) m.data); 5252 5253 ushort code = CloseEvent.StandardCloseCodes.noStatusCodePresent; 5254 const(char)[] reason; 5255 5256 if(m.data.length >= 2) { 5257 code = (m.data[0] << 8) | m.data[1]; 5258 reason = (cast(char[]) m.data[2 .. $]); 5259 } 5260 5261 if(onclose) 5262 onclose(CloseEvent(code, reason, true)); 5263 5264 // if we receive one and haven't sent one back we're supposed to echo it back and close. 5265 if(!closeCalled) 5266 close(code, reason.idup); 5267 5268 readyState_ = CLOSED; 5269 5270 unregisterActiveSocket(this); 5271 break; 5272 case WebSocketOpcode.ping: 5273 // import std.stdio; writeln("ping received ", m.data); 5274 pong(m.data); 5275 break; 5276 case WebSocketOpcode.pong: 5277 // import std.stdio; writeln("pong received ", m.data); 5278 // just really references it is still alive, nbd. 5279 break; 5280 default: // ignore though i could and perhaps should throw too 5281 } 5282 } 5283 5284 if(d.length) { 5285 m.data = m.data.dup(); 5286 } 5287 5288 import core.stdc.string; 5289 memmove(receiveBuffer.ptr, d.ptr, d.length); 5290 receiveBufferUsedLength = d.length; 5291 5292 return m; 5293 } 5294 5295 private void autoprocess() { 5296 // FIXME 5297 do { 5298 processOnce(); 5299 } while(lowLevelReceive()); 5300 } 5301 5302 /++ 5303 Arguments for the close event. The `code` and `reason` are provided from the close message on the websocket, if they are present. The spec says code 1000 indicates a normal, default reason close, but reserves the code range from 3000-5000 for future definition; the 3000s can be registered with IANA and the 4000's are application private use. The `reason` should be user readable, but not displayed to the end user. `wasClean` is true if the server actually sent a close event, false if it just disconnected. 5304 5305 $(PITFALL 5306 The `reason` argument references a temporary buffer and there's no guarantee it will remain valid once your callback returns. It may be freed and will very likely be overwritten. If you want to keep the reason beyond the callback, make sure you `.idup` it. 5307 ) 5308 5309 History: 5310 Added March 19, 2023 (dub v11.0). 5311 +/ 5312 static struct CloseEvent { 5313 ushort code; 5314 const(char)[] reason; 5315 bool wasClean; 5316 5317 string extendedErrorInformationUnstable; 5318 5319 /++ 5320 See https://www.rfc-editor.org/rfc/rfc6455#section-7.4.1 for details. 5321 +/ 5322 enum StandardCloseCodes { 5323 purposeFulfilled = 1000, 5324 goingAway = 1001, 5325 protocolError = 1002, 5326 unacceptableData = 1003, // e.g. got text message when you can only handle binary 5327 Reserved = 1004, 5328 noStatusCodePresent = 1005, // not set by endpoint. 5329 abnormalClosure = 1006, // not set by endpoint. closed without a Close control. FIXME: maybe keep a copy of errno around for these 5330 inconsistentData = 1007, // e.g. utf8 validation failed 5331 genericPolicyViolation = 1008, 5332 messageTooBig = 1009, 5333 clientRequiredExtensionMissing = 1010, // only the client should send this 5334 unnexpectedCondition = 1011, 5335 unverifiedCertificate = 1015, // not set by client 5336 } 5337 } 5338 5339 /++ 5340 The `CloseEvent` you get references a temporary buffer that may be overwritten after your handler returns. If you want to keep it or the `event.reason` member, remember to `.idup` it. 5341 5342 History: 5343 The `CloseEvent` was changed to a [arsd.core.FlexibleDelegate] on March 19, 2023 (dub v11.0). Before that, `onclose` was a public member of type `void delegate()`. This change means setters still work with or without the [CloseEvent] argument. 5344 5345 Your onclose method is now also called on abnormal terminations. Check the `wasClean` member of the `CloseEvent` to know if it came from a close frame or other cause. 5346 +/ 5347 arsd.core.FlexibleDelegate!(void delegate(CloseEvent event)) onclose; 5348 void delegate() onerror; /// 5349 void delegate(in char[]) ontextmessage; /// 5350 void delegate(in ubyte[]) onbinarymessage; /// 5351 void delegate() onopen; /// 5352 5353 /++ 5354 5355 +/ 5356 /// Group: browser_api 5357 void onmessage(void delegate(in char[]) dg) { 5358 ontextmessage = dg; 5359 } 5360 5361 /// ditto 5362 void onmessage(void delegate(in ubyte[]) dg) { 5363 onbinarymessage = dg; 5364 } 5365 5366 /* } end copy/paste */ 5367 5368 /* 5369 const int bufferedAmount // amount pending 5370 const string extensions 5371 5372 const string protocol 5373 const string url 5374 */ 5375 5376 static { 5377 /++ 5378 Runs an event loop with all known websockets on this thread until all websockets 5379 are closed or unregistered, or until you call [exitEventLoop], or set `*localLoopExited` 5380 to false (please note it may take a few seconds until it checks that flag again; it may 5381 not exit immediately). 5382 5383 History: 5384 The `localLoopExited` parameter was added August 22, 2022 (dub v10.9) 5385 5386 See_Also: 5387 [addToSimpledisplayEventLoop] 5388 +/ 5389 void eventLoop(shared(bool)* localLoopExited = null) { 5390 import core.atomic; 5391 atomicOp!"+="(numberOfEventLoops, 1); 5392 scope(exit) { 5393 if(atomicOp!"-="(numberOfEventLoops, 1) <= 0) 5394 loopExited = false; // reset it so we can reenter 5395 } 5396 5397 static SocketSet readSet; 5398 5399 if(readSet is null) 5400 readSet = new SocketSet(); 5401 5402 loopExited = false; 5403 5404 outermost: while(!loopExited && (localLoopExited is null || (*localLoopExited == false))) { 5405 readSet.reset(); 5406 5407 Duration timeout = 3.seconds; 5408 5409 auto now = MonoTime.currTime; 5410 bool hadAny; 5411 foreach(sock; activeSockets) { 5412 auto diff = sock.timeoutFromInactivity - now; 5413 if(diff <= 0.msecs) { 5414 // timeout 5415 if(sock.onerror) 5416 sock.onerror(); 5417 5418 if(sock.onclose) 5419 sock.onclose(CloseEvent(CloseEvent.StandardCloseCodes.abnormalClosure, "Connection timed out", false, null)); 5420 5421 sock.socket.close(); 5422 sock.readyState_ = CLOSED; 5423 unregisterActiveSocket(sock); 5424 continue outermost; 5425 } 5426 5427 if(diff < timeout) 5428 timeout = diff; 5429 5430 diff = sock.nextPing - now; 5431 5432 if(diff <= 0.msecs) { 5433 //sock.send(`{"action": "ping"}`); 5434 sock.ping(); 5435 sock.nextPing = now + sock.config.pingFrequency.msecs; 5436 } else { 5437 if(diff < timeout) 5438 timeout = diff; 5439 } 5440 5441 readSet.add(sock.socket); 5442 hadAny = true; 5443 } 5444 5445 if(!hadAny) { 5446 // import std.stdio; writeln("had none"); 5447 return; 5448 } 5449 5450 tryAgain: 5451 // import std.stdio; writeln(timeout); 5452 auto selectGot = Socket.select(readSet, null, null, timeout); 5453 if(selectGot == 0) { /* timeout */ 5454 // timeout 5455 continue; // it will be handled at the top of the loop 5456 } else if(selectGot == -1) { /* interrupted */ 5457 goto tryAgain; 5458 } else { 5459 foreach(sock; activeSockets) { 5460 if(readSet.isSet(sock.socket)) { 5461 sock.timeoutFromInactivity = MonoTime.currTime + sock.config.timeoutFromInactivity; 5462 if(!sock.lowLevelReceive()) { 5463 sock.readyState_ = CLOSED; 5464 5465 if(sock.onerror) 5466 sock.onerror(); 5467 5468 if(sock.onclose) 5469 sock.onclose(CloseEvent(CloseEvent.StandardCloseCodes.abnormalClosure, "Connection lost", false, lastSocketError())); 5470 5471 unregisterActiveSocket(sock); 5472 continue outermost; 5473 } 5474 while(sock.processOnce().populated) {} 5475 selectGot--; 5476 if(selectGot <= 0) 5477 break; 5478 } 5479 } 5480 } 5481 } 5482 } 5483 5484 private static shared(int) numberOfEventLoops; 5485 5486 private __gshared bool loopExited; 5487 /++ 5488 Exits all running [WebSocket.eventLoop]s next time they loop around. You can call this from a signal handler or another thread. 5489 5490 Please note they may not loop around to check the flag for several seconds. Any new event loops will exit immediately until 5491 all current ones are closed. Once all event loops are exited, the flag is cleared and you can start the loop again. 5492 5493 This function is likely to be deprecated in the future due to its quirks and imprecise name. 5494 +/ 5495 void exitEventLoop() { 5496 loopExited = true; 5497 } 5498 5499 WebSocket[] activeSockets; 5500 5501 void registerActiveSocket(WebSocket s) { 5502 // ensure it isn't already there... 5503 assert(s !is null); 5504 foreach(i, a; activeSockets) 5505 if(a is s) 5506 return; 5507 activeSockets ~= s; 5508 } 5509 void unregisterActiveSocket(WebSocket s) { 5510 foreach(i, a; activeSockets) 5511 if(s is a) { 5512 activeSockets[i] = activeSockets[$-1]; 5513 activeSockets = activeSockets[0 .. $-1]; 5514 break; 5515 } 5516 } 5517 } 5518 } 5519 5520 private template imported(string mod) { 5521 mixin(`import imported = ` ~ mod ~ `;`); 5522 } 5523 5524 /++ 5525 Warning: you should call this AFTER websocket.connect or else it might throw on connect because the function sets nonblocking mode and the connect function doesn't handle that well (it throws on the "would block" condition in that function. easier to just do that first) 5526 +/ 5527 template addToSimpledisplayEventLoop() { 5528 import arsd.simpledisplay; 5529 void addToSimpledisplayEventLoop(WebSocket ws, imported!"arsd.simpledisplay".SimpleWindow window) { 5530 5531 version(Windows) 5532 auto event = WSACreateEvent(); 5533 // FIXME: supposed to close event too 5534 5535 void midprocess() { 5536 version(Windows) 5537 ResetEvent(event); 5538 if(!ws.lowLevelReceive()) { 5539 ws.readyState_ = WebSocket.CLOSED; 5540 WebSocket.unregisterActiveSocket(ws); 5541 return; 5542 } 5543 while(ws.processOnce().populated) {} 5544 } 5545 5546 version(Posix) { 5547 auto reader = new PosixFdReader(&midprocess, ws.socket.handle); 5548 } else version(none) { 5549 if(WSAAsyncSelect(ws.socket.handle, window.hwnd, WM_USER + 150, FD_CLOSE | FD_READ)) 5550 throw new Exception("WSAAsyncSelect"); 5551 5552 window.handleNativeEvent = delegate int(HWND hwnd, UINT msg, WPARAM wParam, LPARAM lParam) { 5553 if(hwnd !is window.impl.hwnd) 5554 return 1; // we don't care... 5555 switch(msg) { 5556 case WM_USER + 150: // socket activity 5557 switch(LOWORD(lParam)) { 5558 case FD_READ: 5559 case FD_CLOSE: 5560 midprocess(); 5561 break; 5562 default: 5563 // nothing 5564 } 5565 break; 5566 default: return 1; // not handled, pass it on 5567 } 5568 return 0; 5569 }; 5570 5571 } else version(Windows) { 5572 ws.socket.blocking = false; // the WSAEventSelect does this anyway and doing it here lets phobos know about it. 5573 //CreateEvent(null, 0, 0, null); 5574 if(!event) { 5575 throw new Exception("WSACreateEvent"); 5576 } 5577 if(WSAEventSelect(ws.socket.handle, event, 1/*FD_READ*/ | (1<<5)/*FD_CLOSE*/)) { 5578 //import std.stdio; writeln(WSAGetLastError()); 5579 throw new Exception("WSAEventSelect"); 5580 } 5581 5582 auto handle = new WindowsHandleReader(&midprocess, event); 5583 5584 /+ 5585 static class Ready {} 5586 5587 Ready thisr = new Ready; 5588 5589 justCommunication.addEventListener((Ready r) { 5590 if(r is thisr) 5591 midprocess(); 5592 }); 5593 5594 import core.thread; 5595 auto thread = new Thread({ 5596 while(true) { 5597 WSAWaitForMultipleEvents(1, &event, true, -1/*WSA_INFINITE*/, false); 5598 justCommunication.postEvent(thisr); 5599 } 5600 }); 5601 thread.isDaemon = true; 5602 thread.start; 5603 +/ 5604 5605 } else static assert(0, "unsupported OS"); 5606 } 5607 } 5608 5609 version(Windows) { 5610 import core.sys.windows.windows; 5611 import core.sys.windows.winsock2; 5612 } 5613 5614 version(none) { 5615 extern(Windows) int WSAAsyncSelect(SOCKET, HWND, uint, int); 5616 enum int FD_CLOSE = 1 << 5; 5617 enum int FD_READ = 1 << 0; 5618 enum int WM_USER = 1024; 5619 } 5620 5621 version(Windows) { 5622 import core.stdc.config; 5623 extern(Windows) 5624 int WSAEventSelect(SOCKET, HANDLE /* to an Event */, c_long); 5625 5626 extern(Windows) 5627 HANDLE WSACreateEvent(); 5628 5629 extern(Windows) 5630 DWORD WSAWaitForMultipleEvents(DWORD, HANDLE*, BOOL, DWORD, BOOL); 5631 } 5632 5633 /* copy/paste from cgi.d */ 5634 public { 5635 enum WebSocketOpcode : ubyte { 5636 continuation = 0, 5637 text = 1, 5638 binary = 2, 5639 // 3, 4, 5, 6, 7 RESERVED 5640 close = 8, 5641 ping = 9, 5642 pong = 10, 5643 // 11,12,13,14,15 RESERVED 5644 } 5645 5646 public struct WebSocketFrame { 5647 private bool populated; 5648 bool fin; 5649 bool rsv1; 5650 bool rsv2; 5651 bool rsv3; 5652 WebSocketOpcode opcode; // 4 bits 5653 bool masked; 5654 ubyte lengthIndicator; // don't set this when building one to send 5655 ulong realLength; // don't use when sending 5656 ubyte[4] maskingKey; // don't set this when sending 5657 ubyte[] data; 5658 5659 static WebSocketFrame simpleMessage(WebSocketOpcode opcode, in void[] data) { 5660 WebSocketFrame msg; 5661 msg.fin = true; 5662 msg.opcode = opcode; 5663 msg.data = cast(ubyte[]) data.dup; // it is mutated below when masked, so need to be cautious and copy it, sigh 5664 5665 return msg; 5666 } 5667 5668 private void send(scope void delegate(ubyte[]) llsend) { 5669 ubyte[64] headerScratch; 5670 int headerScratchPos = 0; 5671 5672 realLength = data.length; 5673 5674 { 5675 ubyte b1; 5676 b1 |= cast(ubyte) opcode; 5677 b1 |= rsv3 ? (1 << 4) : 0; 5678 b1 |= rsv2 ? (1 << 5) : 0; 5679 b1 |= rsv1 ? (1 << 6) : 0; 5680 b1 |= fin ? (1 << 7) : 0; 5681 5682 headerScratch[0] = b1; 5683 headerScratchPos++; 5684 } 5685 5686 { 5687 headerScratchPos++; // we'll set header[1] at the end of this 5688 auto rlc = realLength; 5689 ubyte b2; 5690 b2 |= masked ? (1 << 7) : 0; 5691 5692 assert(headerScratchPos == 2); 5693 5694 if(realLength > 65535) { 5695 // use 64 bit length 5696 b2 |= 0x7f; 5697 5698 // FIXME: double check endinaness 5699 foreach(i; 0 .. 8) { 5700 headerScratch[2 + 7 - i] = rlc & 0x0ff; 5701 rlc >>>= 8; 5702 } 5703 5704 headerScratchPos += 8; 5705 } else if(realLength > 125) { 5706 // use 16 bit length 5707 b2 |= 0x7e; 5708 5709 // FIXME: double check endinaness 5710 foreach(i; 0 .. 2) { 5711 headerScratch[2 + 1 - i] = rlc & 0x0ff; 5712 rlc >>>= 8; 5713 } 5714 5715 headerScratchPos += 2; 5716 } else { 5717 // use 7 bit length 5718 b2 |= realLength & 0b_0111_1111; 5719 } 5720 5721 headerScratch[1] = b2; 5722 } 5723 5724 //assert(!masked, "masking key not properly implemented"); 5725 if(masked) { 5726 import std.random; 5727 foreach(ref item; maskingKey) 5728 item = uniform(ubyte.min, ubyte.max); 5729 headerScratch[headerScratchPos .. headerScratchPos + 4] = maskingKey[]; 5730 headerScratchPos += 4; 5731 5732 // we'll just mask it in place... 5733 int keyIdx = 0; 5734 foreach(i; 0 .. data.length) { 5735 data[i] = data[i] ^ maskingKey[keyIdx]; 5736 if(keyIdx == 3) 5737 keyIdx = 0; 5738 else 5739 keyIdx++; 5740 } 5741 } 5742 5743 //writeln("SENDING ", headerScratch[0 .. headerScratchPos], data); 5744 llsend(headerScratch[0 .. headerScratchPos]); 5745 if(data.length) 5746 llsend(data); 5747 } 5748 5749 static WebSocketFrame read(ref ubyte[] d) { 5750 WebSocketFrame msg; 5751 5752 auto orig = d; 5753 5754 WebSocketFrame needsMoreData() { 5755 d = orig; 5756 return WebSocketFrame.init; 5757 } 5758 5759 if(d.length < 2) 5760 return needsMoreData(); 5761 5762 ubyte b = d[0]; 5763 5764 msg.populated = true; 5765 5766 msg.opcode = cast(WebSocketOpcode) (b & 0x0f); 5767 b >>= 4; 5768 msg.rsv3 = b & 0x01; 5769 b >>= 1; 5770 msg.rsv2 = b & 0x01; 5771 b >>= 1; 5772 msg.rsv1 = b & 0x01; 5773 b >>= 1; 5774 msg.fin = b & 0x01; 5775 5776 b = d[1]; 5777 msg.masked = (b & 0b1000_0000) ? true : false; 5778 msg.lengthIndicator = b & 0b0111_1111; 5779 5780 d = d[2 .. $]; 5781 5782 if(msg.lengthIndicator == 0x7e) { 5783 // 16 bit length 5784 msg.realLength = 0; 5785 5786 if(d.length < 2) return needsMoreData(); 5787 5788 foreach(i; 0 .. 2) { 5789 msg.realLength |= d[0] << ((1-i) * 8); 5790 d = d[1 .. $]; 5791 } 5792 } else if(msg.lengthIndicator == 0x7f) { 5793 // 64 bit length 5794 msg.realLength = 0; 5795 5796 if(d.length < 8) return needsMoreData(); 5797 5798 foreach(i; 0 .. 8) { 5799 msg.realLength |= ulong(d[0]) << ((7-i) * 8); 5800 d = d[1 .. $]; 5801 } 5802 } else { 5803 // 7 bit length 5804 msg.realLength = msg.lengthIndicator; 5805 } 5806 5807 if(msg.masked) { 5808 5809 if(d.length < 4) return needsMoreData(); 5810 5811 msg.maskingKey = d[0 .. 4]; 5812 d = d[4 .. $]; 5813 } 5814 5815 if(msg.realLength > d.length) { 5816 return needsMoreData(); 5817 } 5818 5819 msg.data = d[0 .. cast(size_t) msg.realLength]; 5820 d = d[cast(size_t) msg.realLength .. $]; 5821 5822 return msg; 5823 } 5824 5825 void unmaskInPlace() { 5826 if(this.masked) { 5827 int keyIdx = 0; 5828 foreach(i; 0 .. this.data.length) { 5829 this.data[i] = this.data[i] ^ this.maskingKey[keyIdx]; 5830 if(keyIdx == 3) 5831 keyIdx = 0; 5832 else 5833 keyIdx++; 5834 } 5835 } 5836 } 5837 5838 char[] textData() { 5839 return cast(char[]) data; 5840 } 5841 } 5842 } 5843 5844 private extern(C) 5845 int verifyCertificateFromRegistryArsdHttp(int preverify_ok, X509_STORE_CTX* ctx) { 5846 version(Windows) { 5847 if(preverify_ok) 5848 return 1; 5849 5850 auto err_cert = OpenSSL.X509_STORE_CTX_get_current_cert(ctx); 5851 auto err = OpenSSL.X509_STORE_CTX_get_error(ctx); 5852 5853 if(err == 62) 5854 return 0; // hostname mismatch is an error we can trust; that means OpenSSL already found the certificate and rejected it 5855 5856 auto len = OpenSSL.i2d_X509(err_cert, null); 5857 if(len == -1) 5858 return 0; 5859 ubyte[] buffer = new ubyte[](len); 5860 auto ptr = buffer.ptr; 5861 len = OpenSSL.i2d_X509(err_cert, &ptr); 5862 if(len != buffer.length) 5863 return 0; 5864 5865 5866 CERT_CHAIN_PARA thing; 5867 thing.cbSize = thing.sizeof; 5868 auto context = CertCreateCertificateContext(X509_ASN_ENCODING, buffer.ptr, cast(int) buffer.length); 5869 if(context is null) 5870 return 0; 5871 scope(exit) CertFreeCertificateContext(context); 5872 5873 PCCERT_CHAIN_CONTEXT chain; 5874 if(CertGetCertificateChain(null, context, null, null, &thing, 0, null, &chain)) { 5875 scope(exit) 5876 CertFreeCertificateChain(chain); 5877 5878 DWORD errorStatus = chain.TrustStatus.dwErrorStatus; 5879 5880 if(errorStatus == 0) 5881 return 1; // Windows approved it, OK carry on 5882 // otherwise, sustain OpenSSL's original ruling 5883 } 5884 5885 return 0; 5886 } else { 5887 return preverify_ok; 5888 } 5889 } 5890 5891 5892 version(Windows) { 5893 pragma(lib, "crypt32"); 5894 import core.sys.windows.wincrypt; 5895 extern(Windows) { 5896 PCCERT_CONTEXT CertEnumCertificatesInStore(HCERTSTORE hCertStore, PCCERT_CONTEXT pPrevCertContext); 5897 // BOOL CertGetCertificateChain(HCERTCHAINENGINE hChainEngine, PCCERT_CONTEXT pCertContext, LPFILETIME pTime, HCERTSTORE hAdditionalStore, PCERT_CHAIN_PARA pChainPara, DWORD dwFlags, LPVOID pvReserved, PCCERT_CHAIN_CONTEXT *ppChainContext); 5898 PCCERT_CONTEXT CertCreateCertificateContext(DWORD dwCertEncodingType, const BYTE *pbCertEncoded, DWORD cbCertEncoded); 5899 } 5900 5901 void loadCertificatesFromRegistry(SSL_CTX* ctx) { 5902 auto store = CertOpenSystemStore(0, "ROOT"); 5903 if(store is null) { 5904 // import std.stdio; writeln("failed"); 5905 return; 5906 } 5907 scope(exit) 5908 CertCloseStore(store, 0); 5909 5910 X509_STORE* ssl_store = OpenSSL.SSL_CTX_get_cert_store(ctx); 5911 PCCERT_CONTEXT c; 5912 while((c = CertEnumCertificatesInStore(store, c)) !is null) { 5913 FILETIME na = c.pCertInfo.NotAfter; 5914 SYSTEMTIME st; 5915 FileTimeToSystemTime(&na, &st); 5916 5917 /+ 5918 _CRYPTOAPI_BLOB i = cast() c.pCertInfo.Issuer; 5919 5920 char[256] buffer; 5921 auto p = CertNameToStrA(X509_ASN_ENCODING, &i, CERT_SIMPLE_NAME_STR, buffer.ptr, cast(int) buffer.length); 5922 import std.stdio; writeln(buffer[0 .. p]); 5923 +/ 5924 5925 if(st.wYear <= 2021) { 5926 // see: https://www.openssl.org/blog/blog/2021/09/13/LetsEncryptRootCertExpire/ 5927 continue; // no point keeping an expired root cert and it can break Let's Encrypt anyway 5928 } 5929 5930 const(ubyte)* thing = c.pbCertEncoded; 5931 auto x509 = OpenSSL.d2i_X509(null, &thing, c.cbCertEncoded); 5932 if (x509) { 5933 auto success = OpenSSL.X509_STORE_add_cert(ssl_store, x509); 5934 //if(!success) 5935 //writeln("FAILED HERE"); 5936 OpenSSL.X509_free(x509); 5937 } else { 5938 //writeln("FAILED"); 5939 } 5940 } 5941 5942 CertFreeCertificateContext(c); 5943 5944 // import core.stdc.stdio; printf("%s\n", OpenSSL.OpenSSL_version(0)); 5945 } 5946 5947 5948 // because i use the FILE* in PEM_read_X509 and friends 5949 // gotta use this to bridge the MS C runtime functions 5950 // might be able to just change those to only use the BIO versions 5951 // instead 5952 5953 // only on MS C runtime 5954 version(CRuntime_Microsoft) {} else version=no_openssl_applink; 5955 5956 version(no_openssl_applink) {} else { 5957 private extern(C) { 5958 void _open(); 5959 void _read(); 5960 void _write(); 5961 void _lseek(); 5962 void _close(); 5963 int _fileno(FILE*); 5964 int _setmode(int, int); 5965 } 5966 export extern(C) void** OPENSSL_Applink() { 5967 import core.stdc.stdio; 5968 5969 static extern(C) void* app_stdin() { return cast(void*) stdin; } 5970 static extern(C) void* app_stdout() { return cast(void*) stdout; } 5971 static extern(C) void* app_stderr() { return cast(void*) stderr; } 5972 static extern(C) int app_feof(FILE* fp) { return feof(fp); } 5973 static extern(C) int app_ferror(FILE* fp) { return ferror(fp); } 5974 static extern(C) void app_clearerr(FILE* fp) { return clearerr(fp); } 5975 static extern(C) int app_fileno(FILE* fp) { return _fileno(fp); } 5976 static extern(C) int app_fsetmod(FILE* fp, char mod) { 5977 return _setmode(_fileno(fp), mod == 'b' ? _O_BINARY : _O_TEXT); 5978 } 5979 5980 static immutable void*[] table = [ 5981 cast(void*) 22, // applink max 5982 5983 &app_stdin, 5984 &app_stdout, 5985 &app_stderr, 5986 &fprintf, 5987 &fgets, 5988 &fread, 5989 &fwrite, 5990 &app_fsetmod, 5991 &app_feof, 5992 &fclose, 5993 5994 &fopen, 5995 &fseek, 5996 &ftell, 5997 &fflush, 5998 &app_ferror, 5999 &app_clearerr, 6000 &app_fileno, 6001 6002 &_open, 6003 &_read, 6004 &_write, 6005 &_lseek, 6006 &_close, 6007 ]; 6008 static assert(table.length == 23); 6009 6010 return cast(void**) table.ptr; 6011 } 6012 } 6013 } 6014 6015 unittest { 6016 auto client = new HttpClient(); 6017 auto response = client.navigateTo(Uri("data:,Hello%2C%20World%21")).waitForCompletion(); 6018 assert(response.contentTypeMimeType == "text/plain", response.contentType); 6019 assert(response.contentText == "Hello, World!", response.contentText); 6020 6021 response = client.navigateTo(Uri("data:text/plain;base64,SGVsbG8sIFdvcmxkIQ==")).waitForCompletion(); 6022 assert(response.contentTypeMimeType == "text/plain", response.contentType); 6023 assert(response.contentText == "Hello, World!", response.contentText); 6024 6025 response = client.navigateTo(Uri("data:text/html,%3Ch1%3EHello%2C%20World%21%3C%2Fh1%3E")).waitForCompletion(); 6026 assert(response.contentTypeMimeType == "text/html", response.contentType); 6027 assert(response.contentText == "<h1>Hello, World!</h1>", response.contentText); 6028 } 6029 6030 version(arsd_http2_unittests) 6031 unittest { 6032 import core.thread; 6033 6034 static void server() { 6035 import std.socket; 6036 auto socket = new TcpSocket(); 6037 socket.setOption(SocketOptionLevel.SOCKET, SocketOption.REUSEADDR, true); 6038 socket.bind(new InternetAddress(12346)); 6039 socket.listen(1); 6040 auto s = socket.accept(); 6041 socket.close(); 6042 6043 ubyte[1024] thing; 6044 auto g = s.receive(thing[]); 6045 6046 /+ 6047 string response = "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 9\r\n\r\nHello!!??"; 6048 auto packetSize = 2; 6049 +/ 6050 6051 auto packetSize = 1; 6052 string response = "HTTP/1.1 200 OK\r\nConnection: close\r\nTransfer-Encoding: chunked\r\n\r\n6\r\nHello!\r\n0\r\n\r\n"; 6053 6054 while(response.length) { 6055 s.send(response[0 .. packetSize]); 6056 response = response[packetSize .. $]; 6057 //import std.stdio; writeln(response); 6058 } 6059 6060 s.close(); 6061 } 6062 6063 auto thread = new Thread(&server); 6064 thread.start; 6065 6066 Thread.sleep(200.msecs); 6067 6068 auto response = get("http://localhost:12346/").waitForCompletion; 6069 assert(response.code == 200); 6070 //import std.stdio; writeln(response); 6071 6072 foreach(site; ["https://dlang.org/", "http://arsdnet.net", "https://phobos.dpldocs.info"]) { 6073 response = get(site).waitForCompletion; 6074 assert(response.code == 200); 6075 } 6076 6077 thread.join; 6078 } 6079 6080 /+ 6081 so the url params are arguments. it knows the request 6082 internally. other params are properties on the req 6083 6084 names may have different paths... those will just add ForSomething i think. 6085 6086 auto req = api.listMergeRequests 6087 req.page = 10; 6088 6089 or 6090 req.page(1) 6091 .bar("foo") 6092 6093 req.execute(); 6094 6095 6096 everything in the response is nullable access through the 6097 dynamic object, just with property getters there. need to make 6098 it static generated tho 6099 6100 other messages may be: isPresent and getDynamic 6101 6102 6103 AND/OR what about doing it like the rails objects 6104 6105 BroadcastMessage.get(4) 6106 // various properties 6107 6108 // it lists what you updated 6109 6110 BroadcastMessage.foo().bar().put(5) 6111 +/