1 // Copyright 2013-2022, Adam D. Ruppe. 2 3 // FIXME: websocket proxy support 4 // FIXME: ipv6 support 5 6 // FIXME: headers are supposed to be case insensitive. ugh. 7 8 /++ 9 This is version 2 of my http/1.1 client implementation. 10 11 12 It has no dependencies for basic operation, but does require OpenSSL 13 libraries (or compatible) to be support HTTPS. This dynamically loaded 14 on-demand (meaning it won't be loaded if you don't use it, but if you do 15 use it, the openssl dynamic libraries must be found in the system search path). 16 17 You can compile with `-version=without_openssl` to entirely disable ssl support. 18 19 http2.d, despite its name, does NOT implement HTTP/2.0, but this 20 shouldn't matter for 99.9% of usage, since all servers will continue 21 to support HTTP/1.1 for a very long time. 22 23 History: 24 Automatic `100 Continue` handling was added on September 28, 2021. It doesn't 25 set the Expect header, so it isn't supposed to happen, but plenty of web servers 26 don't follow the standard anyway. 27 28 A dependency on [arsd.core] was added on March 19, 2023 (dub v11.0). Previously, 29 module was stand-alone. You will have add the `core.d` file from the arsd repo 30 to your build now if you are managing the files and builds yourself. 31 32 The benefits of this dependency include some simplified implementation code which 33 makes it easier for me to add more api conveniences, better exceptions with more 34 information, and better event loop integration with other arsd modules beyond 35 just the simpledisplay adapters available previously. The new integration can 36 also make things like heartbeat timers easier for you to code. 37 +/ 38 module arsd.http2; 39 40 import arsd.core; 41 42 /// 43 unittest { 44 import arsd.http2; 45 46 void main() { 47 auto client = new HttpClient(); 48 49 auto request = client.request(Uri("http://dlang.org/")); 50 auto response = request.waitForCompletion(); 51 52 import std.stdio; 53 writeln(response.contentText); 54 writeln(response.code, " ", response.codeText); 55 writeln(response.contentType); 56 } 57 58 version(arsd_http2_integration_test) main(); // exclude from docs 59 } 60 61 // FIXME: I think I want to disable sigpipe here too. 62 63 import std.uri : encodeComponent; 64 65 debug(arsd_http2_verbose) debug=arsd_http2; 66 67 debug(arsd_http2) import std.stdio : writeln; 68 69 version=arsd_http_internal_implementation; 70 71 version(without_openssl) {} 72 else { 73 version=use_openssl; 74 version=with_openssl; 75 version(older_openssl) {} else 76 version=newer_openssl; 77 } 78 79 version(arsd_http_winhttp_implementation) { 80 pragma(lib, "winhttp") 81 import core.sys.windows.winhttp; 82 // FIXME: alter the dub package file too 83 84 // https://github.com/curl/curl/blob/master/lib/vtls/schannel.c 85 // https://docs.microsoft.com/en-us/windows/win32/secauthn/creating-an-schannel-security-context 86 87 88 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpreaddata 89 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpsendrequest 90 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpopenrequest 91 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpconnect 92 } 93 94 95 96 /++ 97 Demonstrates core functionality, using the [HttpClient], 98 [HttpRequest] (returned by [HttpClient.navigateTo|client.navigateTo]), 99 and [HttpResponse] (returned by [HttpRequest.waitForCompletion|request.waitForCompletion]). 100 101 +/ 102 unittest { 103 import arsd.http2; 104 105 void main() { 106 auto client = new HttpClient(); 107 auto request = client.navigateTo(Uri("http://dlang.org/")); 108 auto response = request.waitForCompletion(); 109 110 string returnedHtml = response.contentText; 111 } 112 } 113 114 // FIXME: multipart encoded file uploads needs implementation 115 // future: do web client api stuff 116 117 private __gshared bool defaultVerifyPeer_ = true; 118 119 void defaultVerifyPeer(bool v) { 120 defaultVerifyPeer_ = v; 121 } 122 123 debug import std.stdio; 124 125 import std.socket; 126 import core.time; 127 128 // FIXME: check Transfer-Encoding: gzip always 129 130 version(with_openssl) { 131 //pragma(lib, "crypto"); 132 //pragma(lib, "ssl"); 133 } 134 135 /+ 136 HttpRequest httpRequest(string method, string url, ubyte[] content, string[string] content) { 137 return null; 138 } 139 +/ 140 141 /** 142 auto request = get("http://arsdnet.net/"); 143 request.send(); 144 145 auto response = get("http://arsdnet.net/").waitForCompletion(); 146 */ 147 HttpRequest get(string url) { 148 auto client = new HttpClient(); 149 auto request = client.navigateTo(Uri(url)); 150 return request; 151 } 152 153 /** 154 Do not forget to call `waitForCompletion()` on the returned object! 155 */ 156 HttpRequest post(string url, string[string] req) { 157 auto client = new HttpClient(); 158 ubyte[] bdata; 159 foreach(k, v; req) { 160 if(bdata.length) 161 bdata ~= cast(ubyte[]) "&"; 162 bdata ~= cast(ubyte[]) encodeComponent(k); 163 bdata ~= cast(ubyte[]) "="; 164 bdata ~= cast(ubyte[]) encodeComponent(v); 165 } 166 auto request = client.request(Uri(url), HttpVerb.POST, bdata, "application/x-www-form-urlencoded"); 167 return request; 168 } 169 170 /// gets the text off a url. basic operation only. 171 string getText(string url) { 172 auto request = get(url); 173 auto response = request.waitForCompletion(); 174 return cast(string) response.content; 175 } 176 177 /+ 178 ubyte[] getBinary(string url, string[string] cookies = null) { 179 auto hr = httpRequest("GET", url, null, cookies); 180 if(hr.code != 200) 181 throw new Exception(format("HTTP answered %d instead of 200 on %s", hr.code, url)); 182 return hr.content; 183 } 184 185 /** 186 Gets a textual document, ignoring headers. Throws on non-text or error. 187 */ 188 string get(string url, string[string] cookies = null) { 189 auto hr = httpRequest("GET", url, null, cookies); 190 if(hr.code != 200) 191 throw new Exception(format("HTTP answered %d instead of 200 on %s", hr.code, url)); 192 if(hr.contentType.indexOf("text/") == -1) 193 throw new Exception(hr.contentType ~ " is bad content for conversion to string"); 194 return cast(string) hr.content; 195 196 } 197 198 static import std.uri; 199 200 string post(string url, string[string] args, string[string] cookies = null) { 201 string content; 202 203 foreach(name, arg; args) { 204 if(content.length) 205 content ~= "&"; 206 content ~= std.uri.encode(name) ~ "=" ~ std.uri.encode(arg); 207 } 208 209 auto hr = httpRequest("POST", url, cast(ubyte[]) content, cookies, ["Content-Type: application/x-www-form-urlencoded"]); 210 if(hr.code != 200) 211 throw new Exception(format("HTTP answered %d instead of 200", hr.code)); 212 if(hr.contentType.indexOf("text/") == -1) 213 throw new Exception(hr.contentType ~ " is bad content for conversion to string"); 214 215 return cast(string) hr.content; 216 } 217 218 +/ 219 220 /// 221 struct HttpResponse { 222 /++ 223 The HTTP response code, if the response was completed, or some value < 100 if it was aborted or failed. 224 225 Code 0 - initial value, nothing happened 226 Code 1 - you called request.abort 227 Code 2 - connection refused 228 Code 3 - connection succeeded, but server disconnected early 229 Code 4 - server sent corrupted response (or this code has a bug and processed it wrong) 230 Code 5 - request timed out 231 232 Code >= 100 - a HTTP response 233 +/ 234 int code; 235 string codeText; /// 236 237 string httpVersion; /// 238 239 string statusLine; /// 240 241 string contentType; /// The *full* content type header. See also [contentTypeMimeType] and [contentTypeCharset]. 242 string location; /// The location header 243 244 /++ 245 246 History: 247 Added December 5, 2020 (version 9.1) 248 +/ 249 bool wasSuccessful() { 250 return code >= 200 && code < 400; 251 } 252 253 /++ 254 Returns the mime type part of the [contentType] header. 255 256 History: 257 Added July 25, 2022 (version 10.9) 258 +/ 259 string contentTypeMimeType() { 260 auto idx = contentType.indexOf(";"); 261 if(idx == -1) 262 return contentType; 263 264 return contentType[0 .. idx].strip; 265 } 266 267 /// the charset out of content type, if present. `null` if not. 268 string contentTypeCharset() { 269 auto idx = contentType.indexOf("charset="); 270 if(idx == -1) 271 return null; 272 auto c = contentType[idx + "charset=".length .. $].strip; 273 if(c.length) 274 return c; 275 return null; 276 } 277 278 /++ 279 Names and values of cookies set in the response. 280 281 History: 282 Prior to July 5, 2021 (dub v10.2), this was a public field instead of a property. I did 283 not consider this a breaking change since the intended use is completely compatible with the 284 property, and it was not actually implemented properly before anyway. 285 +/ 286 @property string[string] cookies() const { 287 string[string] ret; 288 foreach(cookie; cookiesDetails) 289 ret[cookie.name] = cookie.value; 290 return ret; 291 } 292 /++ 293 The full parsed-out information of cookies set in the response. 294 295 History: 296 Added July 5, 2021 (dub v10.2). 297 +/ 298 @property CookieHeader[] cookiesDetails() inout { 299 CookieHeader[] ret; 300 foreach(header; headers) { 301 if(auto content = header.isHttpHeader("set-cookie")) { 302 // format: name=value, value might be double quoted. it MIGHT be url encoded, but im not going to attempt that since the RFC is silent. 303 // then there's optionally ; attr=value after that. attributes need not have a value 304 305 CookieHeader cookie; 306 307 auto remaining = content; 308 309 cookie_name: 310 foreach(idx, ch; remaining) { 311 if(ch == '=') { 312 cookie.name = remaining[0 .. idx].idup_if_needed; 313 remaining = remaining[idx + 1 .. $]; 314 break; 315 } 316 } 317 318 cookie_value: 319 320 { 321 auto idx = remaining.indexOf(";"); 322 if(idx == -1) { 323 cookie.value = remaining.idup_if_needed; 324 remaining = remaining[$..$]; 325 } else { 326 cookie.value = remaining[0 .. idx].idup_if_needed; 327 remaining = remaining[idx + 1 .. $].stripLeft; 328 } 329 330 if(cookie.value.length > 2 && cookie.value[0] == '"' && cookie.value[$-1] == '"') 331 cookie.value = cookie.value[1 .. $ - 1]; 332 } 333 334 cookie_attributes: 335 336 while(remaining.length) { 337 string name; 338 foreach(idx, ch; remaining) { 339 if(ch == '=') { 340 name = remaining[0 .. idx].idup_if_needed; 341 remaining = remaining[idx + 1 .. $]; 342 343 string value; 344 345 foreach(idx2, ch2; remaining) { 346 if(ch2 == ';') { 347 value = remaining[0 .. idx2].idup_if_needed; 348 remaining = remaining[idx2 + 1 .. $].stripLeft; 349 break; 350 } 351 } 352 353 if(value is null) { 354 value = remaining.idup_if_needed; 355 remaining = remaining[$ .. $]; 356 } 357 358 cookie.attributes[name] = value; 359 continue cookie_attributes; 360 } else if(ch == ';') { 361 name = remaining[0 .. idx].idup_if_needed; 362 remaining = remaining[idx + 1 .. $].stripLeft; 363 cookie.attributes[name] = ""; 364 continue cookie_attributes; 365 } 366 } 367 368 if(remaining.length) { 369 cookie.attributes[remaining.idup_if_needed] = ""; 370 remaining = remaining[$..$]; 371 372 } 373 } 374 375 ret ~= cookie; 376 } 377 } 378 return ret; 379 } 380 381 string[] headers; /// Array of all headers returned. 382 string[string] headersHash; /// 383 384 ubyte[] content; /// The raw content returned in the response body. 385 string contentText; /// [content], but casted to string (for convenience) 386 387 alias responseText = contentText; // just cuz I do this so often. 388 //alias body = content; 389 390 /++ 391 returns `new Document(this.contentText)`. Requires [arsd.dom]. 392 +/ 393 auto contentDom()() { 394 import arsd.dom; 395 return new Document(this.contentText); 396 397 } 398 399 /++ 400 returns `var.fromJson(this.contentText)`. Requires [arsd.jsvar]. 401 +/ 402 auto contentJson()() { 403 import arsd.jsvar; 404 return var.fromJson(this.contentText); 405 } 406 407 HttpRequestParameters requestParameters; /// 408 409 LinkHeader[] linksStored; 410 bool linksLazilyParsed; 411 412 HttpResponse deepCopy() const { 413 HttpResponse h = cast(HttpResponse) this; 414 h.headers = h.headers.dup; 415 h.headersHash = h.headersHash.dup; 416 h.content = h.content.dup; 417 h.linksStored = h.linksStored.dup; 418 return h; 419 } 420 421 /// Returns links header sorted by "rel" attribute. 422 /// It returns a new array on each call. 423 LinkHeader[string] linksHash() { 424 auto links = this.links(); 425 LinkHeader[string] ret; 426 foreach(link; links) 427 ret[link.rel] = link; 428 return ret; 429 } 430 431 /// Returns the Link header, parsed. 432 LinkHeader[] links() { 433 if(linksLazilyParsed) 434 return linksStored; 435 linksLazilyParsed = true; 436 LinkHeader[] ret; 437 438 auto hdrPtr = "link" in headersHash; 439 if(hdrPtr is null) 440 return ret; 441 442 auto header = *hdrPtr; 443 444 LinkHeader current; 445 446 while(header.length) { 447 char ch = header[0]; 448 449 if(ch == '<') { 450 // read url 451 header = header[1 .. $]; 452 size_t idx; 453 while(idx < header.length && header[idx] != '>') 454 idx++; 455 current.url = header[0 .. idx]; 456 header = header[idx .. $]; 457 } else if(ch == ';') { 458 // read attribute 459 header = header[1 .. $]; 460 header = header.stripLeft; 461 462 size_t idx; 463 while(idx < header.length && header[idx] != '=') 464 idx++; 465 466 string name = header[0 .. idx]; 467 if(idx + 1 < header.length) 468 header = header[idx + 1 .. $]; 469 else 470 header = header[$ .. $]; 471 472 string value; 473 474 if(header.length && header[0] == '"') { 475 // quoted value 476 header = header[1 .. $]; 477 idx = 0; 478 while(idx < header.length && header[idx] != '\"') 479 idx++; 480 value = header[0 .. idx]; 481 header = header[idx .. $]; 482 483 } else if(header.length) { 484 // unquoted value 485 idx = 0; 486 while(idx < header.length && header[idx] != ',' && header[idx] != ' ' && header[idx] != ';') 487 idx++; 488 489 value = header[0 .. idx]; 490 header = header[idx .. $].stripLeft; 491 } 492 493 name = name.toLower; 494 if(name == "rel") 495 current.rel = value; 496 else 497 current.attributes[name] = value; 498 499 } else if(ch == ',') { 500 // start another 501 ret ~= current; 502 current = LinkHeader.init; 503 } else if(ch == ' ' || ch == '\n' || ch == '\r' || ch == '\t') { 504 // ignore 505 } 506 507 if(header.length) 508 header = header[1 .. $]; 509 } 510 511 ret ~= current; 512 513 linksStored = ret; 514 515 return ret; 516 } 517 } 518 519 /+ 520 headerName MUST be all lower case and NOT have the colon on it 521 522 returns slice of the input thing after the header name 523 +/ 524 private inout(char)[] isHttpHeader(inout(char)[] thing, const(char)[] headerName) { 525 foreach(idx, ch; thing) { 526 if(idx < headerName.length) { 527 if(headerName[idx] == '-' && ch != '-') 528 return null; 529 if((ch | ' ') != headerName[idx]) 530 return null; 531 } else if(idx == headerName.length) { 532 if(ch != ':') 533 return null; 534 } else { 535 return thing[idx .. $].strip; 536 } 537 } 538 return null; 539 } 540 541 private string idup_if_needed(string s) { return s; } 542 private string idup_if_needed(const(char)[] s) { return s.idup; } 543 544 unittest { 545 assert("Cookie: foo=bar".isHttpHeader("cookie") == "foo=bar"); 546 assert("cookie: foo=bar".isHttpHeader("cookie") == "foo=bar"); 547 assert("cOOkie: foo=bar".isHttpHeader("cookie") == "foo=bar"); 548 assert("Set-Cookie: foo=bar".isHttpHeader("set-cookie") == "foo=bar"); 549 assert(!"".isHttpHeader("cookie")); 550 } 551 552 /// 553 struct LinkHeader { 554 string url; /// 555 string rel; /// 556 string[string] attributes; /// like title, rev, media, whatever attributes 557 } 558 559 /++ 560 History: 561 Added July 5, 2021 562 +/ 563 struct CookieHeader { 564 string name; 565 string value; 566 string[string] attributes; 567 } 568 569 import std.string; 570 static import std.algorithm; 571 import std.conv; 572 import std.range; 573 574 575 private AddressFamily family(string unixSocketPath) { 576 if(unixSocketPath.length) 577 return AddressFamily.UNIX; 578 else // FIXME: what about ipv6? 579 return AddressFamily.INET; 580 } 581 582 version(Windows) 583 private class UnixAddress : Address { 584 this(string) { 585 throw new Exception("No unix address support on this system in lib yet :("); 586 } 587 override sockaddr* name() { assert(0); } 588 override const(sockaddr)* name() const { assert(0); } 589 override int nameLen() const { assert(0); } 590 } 591 592 593 // Copy pasta from cgi.d, then stripped down. unix path thing added tho 594 /++ 595 Represents a URI. It offers named access to the components and relative uri resolution, though as a user of the library, you'd mostly just construct it like `Uri("http://example.com/index.html")`. 596 +/ 597 struct Uri { 598 alias toString this; // blargh idk a url really is a string, but should it be implicit? 599 600 // scheme://userinfo@host:port/path?query#fragment 601 602 string scheme; /// e.g. "http" in "http://example.com/" 603 string userinfo; /// the username (and possibly a password) in the uri 604 string host; /// the domain name 605 int port; /// port number, if given. Will be zero if a port was not explicitly given 606 string path; /// e.g. "/folder/file.html" in "http://example.com/folder/file.html" 607 string query; /// the stuff after the ? in a uri 608 string fragment; /// the stuff after the # in a uri. 609 610 /// Breaks down a uri string to its components 611 this(string uri) { 612 size_t lastGoodIndex; 613 foreach(char ch; uri) { 614 if(ch > 127) { 615 break; 616 } 617 lastGoodIndex++; 618 } 619 620 string replacement = uri[0 .. lastGoodIndex]; 621 foreach(char ch; uri[lastGoodIndex .. $]) { 622 if(ch > 127) { 623 // need to percent-encode any non-ascii in it 624 char[3] buffer; 625 buffer[0] = '%'; 626 627 auto first = ch / 16; 628 auto second = ch % 16; 629 first += (first >= 10) ? ('A'-10) : '0'; 630 second += (second >= 10) ? ('A'-10) : '0'; 631 632 buffer[1] = cast(char) first; 633 buffer[2] = cast(char) second; 634 635 replacement ~= buffer[]; 636 } else { 637 replacement ~= ch; 638 } 639 } 640 641 reparse(replacement); 642 } 643 644 /// Returns `port` if set, otherwise if scheme is https 443, otherwise always 80 645 int effectivePort() const @property nothrow pure @safe @nogc { 646 return port != 0 ? port 647 : scheme == "https" ? 443 : 80; 648 } 649 650 private string unixSocketPath = null; 651 /// Indicates it should be accessed through a unix socket instead of regular tcp. Returns new version without modifying this object. 652 Uri viaUnixSocket(string path) const { 653 Uri copy = this; 654 copy.unixSocketPath = path; 655 return copy; 656 } 657 658 /// Goes through a unix socket in the abstract namespace (linux only). Returns new version without modifying this object. 659 version(linux) 660 Uri viaAbstractSocket(string path) const { 661 Uri copy = this; 662 copy.unixSocketPath = "\0" ~ path; 663 return copy; 664 } 665 666 private void reparse(string uri) { 667 // from RFC 3986 668 // the ctRegex triples the compile time and makes ugly errors for no real benefit 669 // it was a nice experiment but just not worth it. 670 // enum ctr = ctRegex!r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?"; 671 /* 672 Captures: 673 0 = whole url 674 1 = scheme, with : 675 2 = scheme, no : 676 3 = authority, with // 677 4 = authority, no // 678 5 = path 679 6 = query string, with ? 680 7 = query string, no ? 681 8 = anchor, with # 682 9 = anchor, no # 683 */ 684 // Yikes, even regular, non-CT regex is also unacceptably slow to compile. 1.9s on my computer! 685 // instead, I will DIY and cut that down to 0.6s on the same computer. 686 /* 687 688 Note that authority is 689 user:password@domain:port 690 where the user:password@ part is optional, and the :port is optional. 691 692 Regex translation: 693 694 Scheme cannot have :, /, ?, or # in it, and must have one or more chars and end in a :. It is optional, but must be first. 695 Authority must start with //, but cannot have any other /, ?, or # in it. It is optional. 696 Path cannot have any ? or # in it. It is optional. 697 Query must start with ? and must not have # in it. It is optional. 698 Anchor must start with # and can have anything else in it to end of string. It is optional. 699 */ 700 701 this = Uri.init; // reset all state 702 703 // empty uri = nothing special 704 if(uri.length == 0) { 705 return; 706 } 707 708 size_t idx; 709 710 scheme_loop: foreach(char c; uri[idx .. $]) { 711 switch(c) { 712 case ':': 713 case '/': 714 case '?': 715 case '#': 716 break scheme_loop; 717 default: 718 } 719 idx++; 720 } 721 722 if(idx == 0 && uri[idx] == ':') { 723 // this is actually a path! we skip way ahead 724 goto path_loop; 725 } 726 727 if(idx == uri.length) { 728 // the whole thing is a path, apparently 729 path = uri; 730 return; 731 } 732 733 if(idx > 0 && uri[idx] == ':') { 734 scheme = uri[0 .. idx]; 735 idx++; 736 } else { 737 // we need to rewind; it found a / but no :, so the whole thing is prolly a path... 738 idx = 0; 739 } 740 741 if(idx + 2 < uri.length && uri[idx .. idx + 2] == "//") { 742 // we have an authority.... 743 idx += 2; 744 745 auto authority_start = idx; 746 authority_loop: foreach(char c; uri[idx .. $]) { 747 switch(c) { 748 case '/': 749 case '?': 750 case '#': 751 break authority_loop; 752 default: 753 } 754 idx++; 755 } 756 757 auto authority = uri[authority_start .. idx]; 758 759 auto idx2 = authority.indexOf("@"); 760 if(idx2 != -1) { 761 userinfo = authority[0 .. idx2]; 762 authority = authority[idx2 + 1 .. $]; 763 } 764 765 if(authority.length && authority[0] == '[') { 766 // ipv6 address special casing 767 idx2 = authority.indexOf(']'); 768 if(idx2 != -1) { 769 auto end = authority[idx2 + 1 .. $]; 770 if(end.length && end[0] == ':') 771 idx2 = idx2 + 1; 772 else 773 idx2 = -1; 774 } 775 } else { 776 idx2 = authority.indexOf(":"); 777 } 778 779 if(idx2 == -1) { 780 port = 0; // 0 means not specified; we should use the default for the scheme 781 host = authority; 782 } else { 783 host = authority[0 .. idx2]; 784 if(idx2 + 1 < authority.length) 785 port = to!int(authority[idx2 + 1 .. $]); 786 else 787 port = 0; 788 } 789 } 790 791 path_loop: 792 auto path_start = idx; 793 794 foreach(char c; uri[idx .. $]) { 795 if(c == '?' || c == '#') 796 break; 797 idx++; 798 } 799 800 path = uri[path_start .. idx]; 801 802 if(idx == uri.length) 803 return; // nothing more to examine... 804 805 if(uri[idx] == '?') { 806 idx++; 807 auto query_start = idx; 808 foreach(char c; uri[idx .. $]) { 809 if(c == '#') 810 break; 811 idx++; 812 } 813 query = uri[query_start .. idx]; 814 } 815 816 if(idx < uri.length && uri[idx] == '#') { 817 idx++; 818 fragment = uri[idx .. $]; 819 } 820 821 // uriInvalidated = false; 822 } 823 824 private string rebuildUri() const { 825 string ret; 826 if(scheme.length) 827 ret ~= scheme ~ ":"; 828 if(userinfo.length || host.length) 829 ret ~= "//"; 830 if(userinfo.length) 831 ret ~= userinfo ~ "@"; 832 if(host.length) 833 ret ~= host; 834 if(port) 835 ret ~= ":" ~ to!string(port); 836 837 ret ~= path; 838 839 if(query.length) 840 ret ~= "?" ~ query; 841 842 if(fragment.length) 843 ret ~= "#" ~ fragment; 844 845 // uri = ret; 846 // uriInvalidated = false; 847 return ret; 848 } 849 850 /// Converts the broken down parts back into a complete string 851 string toString() const { 852 // if(uriInvalidated) 853 return rebuildUri(); 854 } 855 856 /// Returns a new absolute Uri given a base. It treats this one as 857 /// relative where possible, but absolute if not. (If protocol, domain, or 858 /// other info is not set, the new one inherits it from the base.) 859 /// 860 /// Browsers use a function like this to figure out links in html. 861 Uri basedOn(in Uri baseUrl) const { 862 Uri n = this; // copies 863 if(n.scheme == "data") 864 return n; 865 // n.uriInvalidated = true; // make sure we regenerate... 866 867 // userinfo is not inherited... is this wrong? 868 869 // if anything is given in the existing url, we don't use the base anymore. 870 if(n.scheme.empty) { 871 n.scheme = baseUrl.scheme; 872 if(n.host.empty) { 873 n.host = baseUrl.host; 874 if(n.port == 0) { 875 n.port = baseUrl.port; 876 if(n.path.length > 0 && n.path[0] != '/') { 877 auto b = baseUrl.path[0 .. baseUrl.path.lastIndexOf("/") + 1]; 878 if(b.length == 0) 879 b = "/"; 880 n.path = b ~ n.path; 881 } else if(n.path.length == 0) { 882 n.path = baseUrl.path; 883 } 884 } 885 } 886 } 887 888 n.removeDots(); 889 890 // if still basically talking to the same thing, we should inherit the unix path 891 // too since basically the unix path is saying for this service, always use this override. 892 if(n.host == baseUrl.host && n.scheme == baseUrl.scheme && n.port == baseUrl.port) 893 n.unixSocketPath = baseUrl.unixSocketPath; 894 895 return n; 896 } 897 898 /++ 899 Resolves ../ and ./ parts of the path. Used in the implementation of [basedOn] and you could also use it to normalize things. 900 +/ 901 void removeDots() { 902 auto parts = this.path.split("/"); 903 string[] toKeep; 904 foreach(part; parts) { 905 if(part == ".") { 906 continue; 907 } else if(part == "..") { 908 //if(toKeep.length > 1) 909 toKeep = toKeep[0 .. $-1]; 910 //else 911 //toKeep = [""]; 912 continue; 913 } else { 914 //if(toKeep.length && toKeep[$-1].length == 0 && part.length == 0) 915 //continue; // skip a `//` situation 916 toKeep ~= part; 917 } 918 } 919 920 auto path = toKeep.join("/"); 921 if(path.length && path[0] != '/') 922 path = "/" ~ path; 923 924 this.path = path; 925 } 926 } 927 928 /* 929 void main(string args[]) { 930 write(post("http://arsdnet.net/bugs.php", ["test" : "hey", "again" : "what"])); 931 } 932 */ 933 934 /// 935 struct BasicAuth { 936 string username; /// 937 string password; /// 938 } 939 940 class ProxyException : Exception { 941 this(string msg) {super(msg); } 942 } 943 944 /** 945 Represents a HTTP request. You usually create these through a [HttpClient]. 946 947 948 --- 949 auto request = new HttpRequest(); // note that when there's no associated client, some features may not work 950 // normally you'd instead do `new HttpClient(); client.request(...)` 951 // set any properties here 952 953 // synchronous usage 954 auto reply = request.perform(); 955 956 // async usage, type 1: 957 request.send(); 958 request2.send(); 959 960 // wait until the first one is done, with the second one still in-flight 961 auto response = request.waitForCompletion(); 962 963 // async usage, type 2: 964 request.onDataReceived = (HttpRequest hr) { 965 if(hr.state == HttpRequest.State.complete) { 966 // use hr.responseData 967 } 968 }; 969 request.send(); // send, using the callback 970 971 // before terminating, be sure you wait for your requests to finish! 972 973 request.waitForCompletion(); 974 --- 975 */ 976 class HttpRequest { 977 978 /// Automatically follow a redirection? 979 bool followLocation = false; 980 981 /++ 982 Maximum number of redirections to follow (used only if [followLocation] is set to true). Will resolve with an error if a single request has more than this number of redirections. The default value is currently 10, but may change without notice. If you need a specific value, be sure to call this function. 983 984 If you want unlimited redirects, call it with `int.max`. If you set it to 0 but set [followLocation] to `true`, any attempt at redirection will abort the request. To disable automatically following redirection, set [followLocation] to `false` so you can process the 30x code yourself as a completed request. 985 986 History: 987 Added July 27, 2022 (dub v10.9) 988 +/ 989 void setMaximumNumberOfRedirects(int max = 10) { 990 maximumNumberOfRedirectsRemaining = max; 991 } 992 993 private int maximumNumberOfRedirectsRemaining; 994 995 /++ 996 Set to `true` to automatically retain cookies in the associated [HttpClient] from this request. 997 Note that you must have constructed the request from a `HttpClient` or at least passed one into the 998 constructor for this to have any effect. 999 1000 Bugs: 1001 See [HttpClient.retainCookies] for important caveats. 1002 1003 History: 1004 Added July 5, 2021 (dub v10.2) 1005 +/ 1006 bool retainCookies = false; 1007 1008 private HttpClient client; 1009 1010 this() { 1011 } 1012 1013 /// 1014 this(HttpClient client, Uri where, HttpVerb method, ICache cache = null, Duration timeout = 10.seconds, string proxy = null) { 1015 this.client = client; 1016 populateFromInfo(where, method); 1017 setTimeout(timeout); 1018 this.cache = cache; 1019 this.proxy = proxy; 1020 1021 setMaximumNumberOfRedirects(); 1022 } 1023 1024 1025 /// ditto 1026 this(Uri where, HttpVerb method, ICache cache = null, Duration timeout = 10.seconds, string proxy = null) { 1027 this(null, where, method, cache, timeout, proxy); 1028 } 1029 1030 /++ 1031 Sets the timeout from inactivity on the request. This is the amount of time that passes with no send or receive activity on the request before it fails with "request timed out" error. 1032 1033 History: 1034 Added March 31, 2021 1035 +/ 1036 void setTimeout(Duration timeout) { 1037 this.requestParameters.timeoutFromInactivity = timeout; 1038 this.timeoutFromInactivity = MonoTime.currTime + this.requestParameters.timeoutFromInactivity; 1039 } 1040 1041 private MonoTime timeoutFromInactivity; 1042 1043 private Uri where; 1044 1045 private ICache cache; 1046 1047 /++ 1048 Proxy to use for this request. It should be a URL or `null`. 1049 1050 This must be sent before you call [send]. 1051 1052 History: 1053 Added April 12, 2021 (dub v9.5) 1054 +/ 1055 string proxy; 1056 1057 /++ 1058 For https connections, if this is `true`, it will fail to connect if the TLS certificate can not be 1059 verified. Setting this to `false` will skip this check and allow the connection to continue anyway. 1060 1061 When the [HttpRequest] is constructed from a [HttpClient], it will inherit the value from the client 1062 instead of using the `= true` here. You can change this value any time before you call [send] (which 1063 is done implicitly if you call [waitForCompletion]). 1064 1065 History: 1066 Added April 5, 2022 (dub v10.8) 1067 1068 Prior to this, it always used the global (but undocumented) `defaultVerifyPeer` setting, and sometimes 1069 even if it was true, it would skip the verification. Now, it always respects this local setting. 1070 +/ 1071 bool verifyPeer = true; 1072 1073 1074 /// Final url after any redirections 1075 string finalUrl; 1076 1077 void populateFromInfo(Uri where, HttpVerb method) { 1078 auto parts = where.basedOn(this.where); 1079 this.where = parts; 1080 finalUrl = where.toString(); 1081 requestParameters.method = method; 1082 requestParameters.unixSocketPath = where.unixSocketPath; 1083 requestParameters.host = parts.host; 1084 requestParameters.port = cast(ushort) parts.effectivePort; 1085 requestParameters.ssl = parts.scheme == "https"; 1086 requestParameters.uri = parts.path.length ? parts.path : "/"; 1087 if(parts.query.length) { 1088 requestParameters.uri ~= "?"; 1089 requestParameters.uri ~= parts.query; 1090 } 1091 } 1092 1093 ~this() { 1094 } 1095 1096 ubyte[] sendBuffer; 1097 1098 HttpResponse responseData; 1099 private HttpClient parentClient; 1100 1101 size_t bodyBytesSent; 1102 size_t bodyBytesReceived; 1103 1104 State state_; 1105 State state() { return state_; } 1106 State state(State s) { 1107 assert(state_ != State.complete); 1108 return state_ = s; 1109 } 1110 /// Called when data is received. Check the state to see what data is available. 1111 void delegate(HttpRequest) onDataReceived; 1112 1113 enum State { 1114 /// The request has not yet been sent 1115 unsent, 1116 1117 /// The send() method has been called, but no data is 1118 /// sent on the socket yet because the connection is busy. 1119 pendingAvailableConnection, 1120 1121 /// connect has been called, but we're waiting on word of success 1122 connecting, 1123 1124 /// connecting a ssl, needing this 1125 sslConnectPendingRead, 1126 /// ditto 1127 sslConnectPendingWrite, 1128 1129 /// The headers are being sent now 1130 sendingHeaders, 1131 1132 // FIXME: allow Expect: 100-continue and separate the body send 1133 1134 /// The body is being sent now 1135 sendingBody, 1136 1137 /// The request has been sent but we haven't received any response yet 1138 waitingForResponse, 1139 1140 /// We have received some data and are currently receiving headers 1141 readingHeaders, 1142 1143 /// All headers are available but we're still waiting on the body 1144 readingBody, 1145 1146 /// The request is complete. 1147 complete, 1148 1149 /// The request is aborted, either by the abort() method, or as a result of the server disconnecting 1150 aborted 1151 } 1152 1153 /// Sends now and waits for the request to finish, returning the response. 1154 HttpResponse perform() { 1155 send(); 1156 return waitForCompletion(); 1157 } 1158 1159 /// Sends the request asynchronously. 1160 void send() { 1161 sendPrivate(true); 1162 } 1163 1164 private void sendPrivate(bool advance) { 1165 if(state != State.unsent && state != State.aborted) 1166 return; // already sent 1167 1168 if(cache !is null) { 1169 auto res = cache.getCachedResponse(this.requestParameters); 1170 if(res !is null) { 1171 state = State.complete; 1172 responseData = (*res).deepCopy(); 1173 return; 1174 } 1175 } 1176 1177 if(this.where.scheme == "data") { 1178 void error(string content) { 1179 responseData.code = 400; 1180 responseData.codeText = "Bad Request"; 1181 responseData.contentType = "text/plain"; 1182 responseData.content = cast(ubyte[]) content; 1183 responseData.contentText = content; 1184 state = State.complete; 1185 return; 1186 } 1187 1188 auto thing = this.where.path; 1189 // format is: type,data 1190 // type can have ;base64 1191 auto comma = thing.indexOf(","); 1192 if(comma == -1) 1193 return error("Invalid data uri, no comma found"); 1194 1195 auto type = thing[0 .. comma]; 1196 auto data = thing[comma + 1 .. $]; 1197 if(type.length == 0) 1198 type = "text/plain"; 1199 1200 import std.uri; 1201 auto bdata = cast(ubyte[]) decodeComponent(data); 1202 1203 if(type.indexOf(";base64") != -1) { 1204 import std.base64; 1205 try { 1206 bdata = Base64.decode(bdata); 1207 } catch(Exception e) { 1208 return error(e.msg); 1209 } 1210 } 1211 1212 responseData.code = 200; 1213 responseData.codeText = "OK"; 1214 responseData.contentType = type; 1215 responseData.content = bdata; 1216 responseData.contentText = cast(string) responseData.content; 1217 state = State.complete; 1218 return; 1219 } 1220 1221 string headers; 1222 1223 headers ~= to!string(requestParameters.method); 1224 headers ~= " "; 1225 if(proxy.length && !requestParameters.ssl) { 1226 // if we're doing a http proxy, we need to send a complete, absolute uri 1227 // so reconstruct it 1228 headers ~= "http://"; 1229 headers ~= requestParameters.host; 1230 if(requestParameters.port != 80) { 1231 headers ~= ":"; 1232 headers ~= to!string(requestParameters.port); 1233 } 1234 } 1235 1236 headers ~= requestParameters.uri; 1237 1238 if(requestParameters.useHttp11) 1239 headers ~= " HTTP/1.1\r\n"; 1240 else 1241 headers ~= " HTTP/1.0\r\n"; 1242 1243 // the whole authority section is supposed to be there, but curl doesn't send if default port 1244 // so I'll copy what they do 1245 headers ~= "Host: "; 1246 headers ~= requestParameters.host; 1247 if(requestParameters.port != 80 && requestParameters.port != 443) { 1248 headers ~= ":"; 1249 headers ~= to!string(requestParameters.port); 1250 } 1251 headers ~= "\r\n"; 1252 1253 bool specSaysRequestAlwaysHasBody = 1254 requestParameters.method == HttpVerb.POST || 1255 requestParameters.method == HttpVerb.PUT || 1256 requestParameters.method == HttpVerb.PATCH; 1257 1258 if(requestParameters.userAgent.length) 1259 headers ~= "User-Agent: "~requestParameters.userAgent~"\r\n"; 1260 if(requestParameters.contentType.length) 1261 headers ~= "Content-Type: "~requestParameters.contentType~"\r\n"; 1262 if(requestParameters.authorization.length) 1263 headers ~= "Authorization: "~requestParameters.authorization~"\r\n"; 1264 if(requestParameters.bodyData.length || specSaysRequestAlwaysHasBody) 1265 headers ~= "Content-Length: "~to!string(requestParameters.bodyData.length)~"\r\n"; 1266 if(requestParameters.acceptGzip) 1267 headers ~= "Accept-Encoding: gzip\r\n"; 1268 if(requestParameters.keepAlive) 1269 headers ~= "Connection: keep-alive\r\n"; 1270 1271 string cookieHeader; 1272 foreach(name, value; requestParameters.cookies) { 1273 if(cookieHeader is null) 1274 cookieHeader = "Cookie: "; 1275 else 1276 cookieHeader ~= "; "; 1277 cookieHeader ~= name; 1278 cookieHeader ~= "="; 1279 cookieHeader ~= value; 1280 } 1281 1282 if(cookieHeader !is null) { 1283 cookieHeader ~= "\r\n"; 1284 headers ~= cookieHeader; 1285 } 1286 1287 foreach(header; requestParameters.headers) 1288 headers ~= header ~ "\r\n"; 1289 1290 headers ~= "\r\n"; 1291 1292 // FIXME: separate this for 100 continue 1293 sendBuffer = cast(ubyte[]) headers ~ requestParameters.bodyData; 1294 1295 // import std.stdio; writeln("******* ", cast(string) sendBuffer); 1296 1297 responseData = HttpResponse.init; 1298 responseData.requestParameters = requestParameters; 1299 bodyBytesSent = 0; 1300 bodyBytesReceived = 0; 1301 state = State.pendingAvailableConnection; 1302 1303 bool alreadyPending = false; 1304 foreach(req; pending) 1305 if(req is this) { 1306 alreadyPending = true; 1307 break; 1308 } 1309 if(!alreadyPending) { 1310 pending ~= this; 1311 } 1312 1313 if(advance) 1314 HttpRequest.advanceConnections(requestParameters.timeoutFromInactivity); 1315 } 1316 1317 1318 /// Waits for the request to finish or timeout, whichever comes first. 1319 HttpResponse waitForCompletion() { 1320 while(state != State.aborted && state != State.complete) { 1321 if(state == State.unsent) { 1322 send(); 1323 continue; 1324 } 1325 if(auto err = HttpRequest.advanceConnections(requestParameters.timeoutFromInactivity)) { 1326 switch(err) { 1327 case 1: throw new Exception("HttpRequest.advanceConnections returned 1: all connections timed out"); 1328 case 2: throw new Exception("HttpRequest.advanceConnections returned 2: nothing to do"); 1329 case 3: continue; // EINTR 1330 default: throw new Exception("HttpRequest.advanceConnections got err " ~ to!string(err)); 1331 } 1332 } 1333 } 1334 1335 if(state == State.complete && responseData.code >= 200) 1336 if(cache !is null) 1337 cache.cacheResponse(this.requestParameters, this.responseData); 1338 1339 return responseData; 1340 } 1341 1342 /// Aborts this request. 1343 void abort() { 1344 this.state = State.aborted; 1345 this.responseData.code = 1; 1346 this.responseData.codeText = "request.abort called"; 1347 // the actual cancellation happens in the event loop 1348 } 1349 1350 HttpRequestParameters requestParameters; /// 1351 1352 version(arsd_http_winhttp_implementation) { 1353 public static void resetInternals() { 1354 1355 } 1356 1357 static assert(0, "implementation not finished"); 1358 } 1359 1360 1361 version(arsd_http_internal_implementation) { 1362 1363 /++ 1364 Changes the limit of number of open, inactive sockets. Reusing connections can provide a significant 1365 performance improvement, but the operating system can also impose a global limit on the number of open 1366 sockets and/or files that you don't want to run into. This lets you choose a balance right for you. 1367 1368 1369 When the total number of cached, inactive sockets approaches this maximum, it will check for ones closed by the 1370 server first. If there are none already closed by the server, it will select sockets at random from its connection 1371 cache and close them to make room for the new ones. 1372 1373 Please note: 1374 1375 $(LIST 1376 * there is always a limit of six open sockets per domain, per the common practice suggested by the http standard 1377 * the limit given here is thread-local. If you run multiple http clients/requests from multiple threads, don't set this too high or you might bump into the global limit from the OS. 1378 * setting this too low can waste connections because the server might close them, but they will never be garbage collected since my current implementation won't check for dead connections except when it thinks it is running close to the limit. 1379 ) 1380 1381 Setting it just right for your use case may provide an up to 10x performance boost. 1382 1383 This implementation is subject to change. If it does, I'll document it, but may not bump the version number. 1384 1385 History: 1386 Added August 10, 2022 (dub v10.9) 1387 +/ 1388 static void setConnectionCacheSize(int max = 32) { 1389 connectionCacheSize = max; 1390 } 1391 1392 private static { 1393 // we manage the actual connections. When a request is made on a particular 1394 // host, we try to reuse connections. We may open more than one connection per 1395 // host to do parallel requests. 1396 // 1397 // The key is the *domain name* and the port. Multiple domains on the same address will have separate connections. 1398 Socket[][string] socketsPerHost; 1399 1400 // only one request can be active on a given socket (at least HTTP < 2.0) so this is that 1401 HttpRequest[Socket] activeRequestOnSocket; 1402 HttpRequest[] pending; // and these are the requests that are waiting 1403 1404 int cachedSockets; 1405 int connectionCacheSize = 32; 1406 1407 /+ 1408 This is a somewhat expensive, but essential operation. If it isn't used in a heavy 1409 application, you'll risk running out of file descriptors. 1410 +/ 1411 void cleanOldSockets() { 1412 static struct CloseCandidate { 1413 string key; 1414 Socket socket; 1415 } 1416 1417 CloseCandidate[36] closeCandidates; 1418 int closeCandidatesPosition; 1419 1420 outer: foreach(key, sockets; socketsPerHost) { 1421 foreach(socket; sockets) { 1422 if(socket in activeRequestOnSocket) 1423 continue; // it is still in use; we can't close it 1424 1425 closeCandidates[closeCandidatesPosition++] = CloseCandidate(key, socket); 1426 if(closeCandidatesPosition == closeCandidates.length) 1427 break outer; 1428 } 1429 } 1430 1431 auto cc = closeCandidates[0 .. closeCandidatesPosition]; 1432 1433 if(cc.length == 0) 1434 return; // no candidates to even examine 1435 1436 // has the server closed any of these? if so, we also close and drop them 1437 static SocketSet readSet = null; 1438 if(readSet is null) 1439 readSet = new SocketSet(); 1440 readSet.reset(); 1441 1442 foreach(candidate; cc) { 1443 readSet.add(candidate.socket); 1444 } 1445 1446 int closeCount; 1447 1448 auto got = Socket.select(readSet, null, null, 0.msecs /* timeout, want it small since we just checking for eof */); 1449 if(got > 0) { 1450 foreach(ref candidate; cc) { 1451 if(readSet.isSet(candidate.socket)) { 1452 // if we can read when it isn't in use, that means eof; the 1453 // server closed it. 1454 candidate.socket.close(); 1455 loseSocketByKey(candidate.key, candidate.socket); 1456 closeCount++; 1457 } 1458 } 1459 debug(arsd_http2) writeln(closeCount, " from inactivity"); 1460 } else { 1461 // and if not, of the remaining ones, close a few just at random to bring us back beneath the arbitrary limit. 1462 1463 while(cc.length > 0 && (cachedSockets - closeCount) > connectionCacheSize) { 1464 import std.random; 1465 auto idx = uniform(0, cc.length); 1466 1467 cc[idx].socket.close(); 1468 loseSocketByKey(cc[idx].key, cc[idx].socket); 1469 1470 cc[idx] = cc[$ - 1]; 1471 cc = cc[0 .. $-1]; 1472 closeCount++; 1473 } 1474 debug(arsd_http2) writeln(closeCount, " from randomness"); 1475 } 1476 1477 cachedSockets -= closeCount; 1478 } 1479 1480 void loseSocketByKey(string key, Socket s) { 1481 if(auto list = key in socketsPerHost) { 1482 for(int a = 0; a < (*list).length; a++) { 1483 if((*list)[a] is s) { 1484 1485 for(int b = a; b < (*list).length - 1; b++) 1486 (*list)[b] = (*list)[b+1]; 1487 (*list) = (*list)[0 .. $-1]; 1488 break; 1489 } 1490 } 1491 } 1492 } 1493 1494 void loseSocket(string host, ushort port, bool ssl, Socket s) { 1495 import std.string; 1496 auto key = format("http%s://%s:%s", ssl ? "s" : "", host, port); 1497 1498 loseSocketByKey(key, s); 1499 } 1500 1501 Socket getOpenSocketOnHost(string proxy, string host, ushort port, bool ssl, string unixSocketPath, bool verifyPeer) { 1502 Socket openNewConnection() { 1503 Socket socket; 1504 if(ssl) { 1505 version(with_openssl) { 1506 loadOpenSsl(); 1507 socket = new SslClientSocket(family(unixSocketPath), SocketType.STREAM, host, verifyPeer); 1508 socket.blocking = false; 1509 } else 1510 throw new Exception("SSL not compiled in"); 1511 } else { 1512 socket = new Socket(family(unixSocketPath), SocketType.STREAM); 1513 socket.blocking = false; 1514 } 1515 1516 socket.setOption(SocketOptionLevel.TCP, SocketOption.TCP_NODELAY, 1); 1517 1518 // FIXME: connect timeout? 1519 if(unixSocketPath) { 1520 import std.stdio; writeln(cast(ubyte[]) unixSocketPath); 1521 socket.connect(new UnixAddress(unixSocketPath)); 1522 } else { 1523 // FIXME: i should prolly do ipv6 if available too. 1524 if(host.length == 0) // this could arguably also be an in contract since it is user error, but the exception is good enough 1525 throw new Exception("No host given for request"); 1526 if(proxy.length) { 1527 if(proxy.indexOf("//") == -1) 1528 proxy = "http://" ~ proxy; 1529 auto proxyurl = Uri(proxy); 1530 1531 //auto proxyhttps = proxyurl.scheme == "https"; 1532 enum proxyhttps = false; // this isn't properly implemented and might never be necessary anyway so meh 1533 1534 // the precise types here are important to help with overload 1535 // resolution of the devirtualized call! 1536 Address pa = new InternetAddress(proxyurl.host, proxyurl.port ? cast(ushort) proxyurl.port : 80); 1537 1538 debug(arsd_http2) writeln("using proxy ", pa.toString()); 1539 1540 if(proxyhttps) { 1541 socket.connect(pa); 1542 } else { 1543 // the proxy never actually starts TLS, but if the request is tls then we need to CONNECT then upgrade the connection 1544 // using the parent class functions let us bypass the encryption 1545 socket.Socket.connect(pa); 1546 } 1547 1548 socket.blocking = true; // FIXME total hack to simplify the code here since it isn't really using the event loop yet 1549 1550 string message; 1551 if(ssl) { 1552 auto hostName = host ~ ":" ~ to!string(port); 1553 message = "CONNECT " ~ hostName ~ " HTTP/1.1\r\n"; 1554 message ~= "Host: " ~ hostName ~ "\r\n"; 1555 if(proxyurl.userinfo.length) { 1556 import std.base64; 1557 message ~= "Proxy-Authorization: Basic " ~ Base64.encode(cast(ubyte[]) proxyurl.userinfo) ~ "\r\n"; 1558 } 1559 message ~= "\r\n"; 1560 1561 // FIXME: what if proxy times out? should be reasonably fast too. 1562 if(proxyhttps) { 1563 socket.send(message, SocketFlags.NONE); 1564 } else { 1565 socket.Socket.send(message, SocketFlags.NONE); 1566 } 1567 1568 ubyte[1024] recvBuffer; 1569 // and last time 1570 ptrdiff_t rcvGot; 1571 if(proxyhttps) { 1572 rcvGot = socket.receive(recvBuffer[], SocketFlags.NONE); 1573 // bool verifyPeer = true; 1574 //(cast(OpenSslSocket)socket).freeSsl(); 1575 //(cast(OpenSslSocket)socket).initSsl(verifyPeer, host); 1576 } else { 1577 rcvGot = socket.Socket.receive(recvBuffer[], SocketFlags.NONE); 1578 } 1579 1580 if(rcvGot == -1) 1581 throw new ProxyException("proxy receive error"); 1582 auto got = cast(string) recvBuffer[0 .. rcvGot]; 1583 auto expect = "HTTP/1.1 200"; 1584 if(got.length < expect.length || (got[0 .. expect.length] != expect && got[0 .. expect.length] != "HTTP/1.0 200")) 1585 throw new ProxyException("Proxy rejected request: " ~ got[0 .. expect.length <= got.length ? expect.length : got.length]); 1586 1587 if(proxyhttps) { 1588 //(cast(OpenSslSocket)socket).do_ssl_connect(); 1589 } else { 1590 (cast(OpenSslSocket)socket).do_ssl_connect(); 1591 } 1592 } else { 1593 } 1594 } else { 1595 socket.connect(new InternetAddress(host, port)); 1596 } 1597 } 1598 1599 debug(arsd_http2) writeln("opening to ", host, ":", port, " ", cast(void*) socket, " ssl=", ssl); 1600 assert(socket.handle() !is socket_t.init); 1601 return socket; 1602 } 1603 1604 // import std.stdio; writeln(cachedSockets); 1605 if(cachedSockets > connectionCacheSize) 1606 cleanOldSockets(); 1607 1608 import std.string; 1609 auto key = format("http%s://%s:%s", ssl ? "s" : "", host, port); 1610 1611 if(auto hostListing = key in socketsPerHost) { 1612 // try to find an available socket that is already open 1613 foreach(socket; *hostListing) { 1614 if(socket !in activeRequestOnSocket) { 1615 // let's see if it has closed since we last tried 1616 // e.g. a server timeout or something. If so, we need 1617 // to lose this one and immediately open a new one. 1618 static SocketSet readSet = null; 1619 if(readSet is null) 1620 readSet = new SocketSet(); 1621 readSet.reset(); 1622 assert(socket !is null); 1623 assert(socket.handle() !is socket_t.init, socket is null ? "null" : socket.toString()); 1624 readSet.add(socket); 1625 auto got = Socket.select(readSet, null, null, 0.msecs /* timeout, want it small since we just checking for eof */); 1626 if(got > 0) { 1627 // we can read something off this... but there aren't 1628 // any active requests. Assume it is EOF and open a new one 1629 1630 socket.close(); 1631 loseSocket(host, port, ssl, socket); 1632 goto openNew; 1633 } 1634 cachedSockets--; 1635 return socket; 1636 } 1637 } 1638 1639 // if not too many already open, go ahead and do a new one 1640 if((*hostListing).length < 6) { 1641 auto socket = openNewConnection(); 1642 (*hostListing) ~= socket; 1643 return socket; 1644 } else 1645 return null; // too many, you'll have to wait 1646 } 1647 1648 openNew: 1649 1650 auto socket = openNewConnection(); 1651 socketsPerHost[key] ~= socket; 1652 return socket; 1653 } 1654 1655 SocketSet readSet; 1656 SocketSet writeSet; 1657 1658 /+ 1659 Generic event loop registration: 1660 1661 handle, operation (read/write), buffer (on posix it *might* be stack if a select loop), timeout (in real time), callback when op completed. 1662 1663 ....basically Windows style. Then it translates internally. 1664 1665 It should tell the thing if the buffer is reused or not 1666 +/ 1667 1668 1669 /++ 1670 This is made public for rudimentary event loop integration, but is still 1671 basically an internal detail. Try not to use it if you have another way. 1672 1673 This does a single iteration of the internal select()-based processing loop. 1674 1675 1676 Future directions: 1677 I want to merge the internal use of [WebSocket.eventLoop] with this; 1678 [advanceConnections] does just one run on the loop, whereas eventLoop 1679 runs it until all connections are closed. But they'd both process both 1680 pending http requests and active websockets. 1681 1682 After that, I want to be able to integrate in other event loops too. 1683 One might be to simply to reactor callbacks, then perhaps Windows overlapped 1684 i/o (that's just going to be tricky to retrofit into the existing select()-based 1685 code). It could then go fiber just by calling the resume function too. 1686 1687 The hard part is ensuring I keep this file stand-alone while offering these 1688 things. 1689 1690 This `advanceConnections` call will probably continue to work now that it is 1691 public, but it may not be wholly compatible with all the future features; you'd 1692 have to pick either the internal event loop or an external one you integrate, but not 1693 mix them. 1694 1695 History: 1696 This has been included in the library since almost day one, but 1697 it was private until April 13, 2021 (dub v9.5). 1698 1699 Params: 1700 maximumTimeout = the maximum time it will wait in select(). It may return much sooner than this if a connection timed out in the mean time. 1701 automaticallyRetryOnInterruption = internally loop on EINTR. 1702 1703 Returns: 1704 1705 0 = no error, work may remain so you should call `advanceConnections` again when you can 1706 1707 1 = passed `maximumTimeout` reached with no work done, yet requests are still in the queue. You may call `advanceConnections` again. 1708 1709 2 = no work to do, no point calling it again unless you've added new requests. Your program may exit if you have nothing to add since it means everything requested is now done. 1710 1711 3 = EINTR occurred on select(), you should check your interrupt flags if you set a signal handler, then call `advanceConnections` again if you aren't exiting. Only occurs if `automaticallyRetryOnInterruption` is set to `false` (the default when it is called externally). 1712 1713 any other value should be considered a non-recoverable error if you want to be forward compatible as I reserve the right to add more values later. 1714 +/ 1715 public int advanceConnections(Duration maximumTimeout = 10.seconds, bool automaticallyRetryOnInterruption = false) { 1716 debug(arsd_http2_verbose) writeln("advancing"); 1717 if(readSet is null) 1718 readSet = new SocketSet(); 1719 if(writeSet is null) 1720 writeSet = new SocketSet(); 1721 1722 ubyte[2048] buffer; 1723 1724 HttpRequest[16] removeFromPending; 1725 size_t removeFromPendingCount = 0; 1726 1727 bool hadAbortedRequest; 1728 1729 // are there pending requests? let's try to send them 1730 foreach(idx, pc; pending) { 1731 if(removeFromPendingCount == removeFromPending.length) 1732 break; 1733 1734 if(pc.state == HttpRequest.State.aborted) { 1735 removeFromPending[removeFromPendingCount++] = pc; 1736 hadAbortedRequest = true; 1737 continue; 1738 } 1739 1740 Socket socket; 1741 1742 try { 1743 socket = getOpenSocketOnHost(pc.proxy, pc.requestParameters.host, pc.requestParameters.port, pc.requestParameters.ssl, pc.requestParameters.unixSocketPath, pc.verifyPeer); 1744 } catch(ProxyException e) { 1745 // connection refused or timed out (I should disambiguate somehow)... 1746 pc.state = HttpRequest.State.aborted; 1747 1748 pc.responseData.code = 2; 1749 pc.responseData.codeText = e.msg ~ " from " ~ pc.proxy; 1750 1751 hadAbortedRequest = true; 1752 1753 removeFromPending[removeFromPendingCount++] = pc; 1754 continue; 1755 1756 } catch(SocketException e) { 1757 // connection refused or timed out (I should disambiguate somehow)... 1758 pc.state = HttpRequest.State.aborted; 1759 1760 pc.responseData.code = 2; 1761 pc.responseData.codeText = pc.proxy.length ? ("connection failed to proxy " ~ pc.proxy) : "connection failed"; 1762 1763 hadAbortedRequest = true; 1764 1765 removeFromPending[removeFromPendingCount++] = pc; 1766 continue; 1767 } catch(Exception e) { 1768 // connection failed due to other user error or SSL (i should disambiguate somehow)... 1769 pc.state = HttpRequest.State.aborted; 1770 1771 pc.responseData.code = 2; 1772 pc.responseData.codeText = e.msg; 1773 1774 hadAbortedRequest = true; 1775 1776 removeFromPending[removeFromPendingCount++] = pc; 1777 continue; 1778 1779 } 1780 1781 if(socket !is null) { 1782 activeRequestOnSocket[socket] = pc; 1783 assert(pc.sendBuffer.length); 1784 pc.state = State.connecting; 1785 1786 removeFromPending[removeFromPendingCount++] = pc; 1787 } 1788 } 1789 1790 import std.algorithm : remove; 1791 foreach(rp; removeFromPending[0 .. removeFromPendingCount]) 1792 pending = pending.remove!((a) => a is rp)(); 1793 1794 tryAgain: 1795 1796 Socket[16] inactive; 1797 int inactiveCount = 0; 1798 void killInactives() { 1799 foreach(s; inactive[0 .. inactiveCount]) { 1800 debug(arsd_http2) writeln("removing socket from active list ", cast(void*) s); 1801 activeRequestOnSocket.remove(s); 1802 cachedSockets++; 1803 } 1804 } 1805 1806 1807 readSet.reset(); 1808 writeSet.reset(); 1809 1810 bool hadOne = false; 1811 1812 auto minTimeout = maximumTimeout; 1813 auto now = MonoTime.currTime; 1814 1815 // active requests need to be read or written to 1816 foreach(sock, request; activeRequestOnSocket) { 1817 1818 if(request.state == State.aborted) { 1819 inactive[inactiveCount++] = sock; 1820 sock.close(); 1821 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1822 hadAbortedRequest = true; 1823 continue; 1824 } 1825 1826 // check the other sockets just for EOF, if they close, take them out of our list, 1827 // we'll reopen if needed upon request. 1828 readSet.add(sock); 1829 hadOne = true; 1830 1831 Duration timeo; 1832 if(request.timeoutFromInactivity <= now) 1833 timeo = 0.seconds; 1834 else 1835 timeo = request.timeoutFromInactivity - now; 1836 1837 if(timeo < minTimeout) 1838 minTimeout = timeo; 1839 1840 if(request.state == State.connecting || request.state == State.sslConnectPendingWrite || request.state == State.sendingHeaders || request.state == State.sendingBody) { 1841 writeSet.add(sock); 1842 hadOne = true; 1843 } 1844 } 1845 1846 if(!hadOne) { 1847 if(hadAbortedRequest) { 1848 killInactives(); 1849 return 0; // something got aborted, that's progress 1850 } 1851 return 2; // automatic timeout, nothing to do 1852 } 1853 1854 auto selectGot = Socket.select(readSet, writeSet, null, minTimeout); 1855 if(selectGot == 0) { /* timeout */ 1856 now = MonoTime.currTime; 1857 bool anyWorkDone = false; 1858 foreach(sock, request; activeRequestOnSocket) { 1859 1860 if(request.timeoutFromInactivity <= now) { 1861 request.state = HttpRequest.State.aborted; 1862 request.responseData.code = 5; 1863 if(request.state == State.connecting) 1864 request.responseData.codeText = "Connect timed out"; 1865 else 1866 request.responseData.codeText = "Request timed out"; 1867 1868 inactive[inactiveCount++] = sock; 1869 sock.close(); 1870 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1871 anyWorkDone = true; 1872 } 1873 } 1874 killInactives(); 1875 return anyWorkDone ? 0 : 1; 1876 // return 1; was an error to time out but now im making it on the individual request 1877 } else if(selectGot == -1) { /* interrupted */ 1878 /* 1879 version(Posix) { 1880 import core.stdc.errno; 1881 if(errno != EINTR) 1882 throw new Exception("select error: " ~ to!string(errno)); 1883 } 1884 */ 1885 if(automaticallyRetryOnInterruption) 1886 goto tryAgain; 1887 else 1888 return 3; 1889 } else { /* ready */ 1890 1891 void sslProceed(HttpRequest request, SslClientSocket s) { 1892 try { 1893 auto code = s.do_ssl_connect(); 1894 switch(code) { 1895 case 0: 1896 request.state = State.sendingHeaders; 1897 break; 1898 case SSL_ERROR_WANT_READ: 1899 request.state = State.sslConnectPendingRead; 1900 break; 1901 case SSL_ERROR_WANT_WRITE: 1902 request.state = State.sslConnectPendingWrite; 1903 break; 1904 default: 1905 assert(0); 1906 } 1907 } catch(Exception e) { 1908 request.state = State.aborted; 1909 1910 request.responseData.code = 2; 1911 request.responseData.codeText = e.msg; 1912 inactive[inactiveCount++] = s; 1913 s.close(); 1914 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, s); 1915 } 1916 } 1917 1918 1919 foreach(sock, request; activeRequestOnSocket) { 1920 // always need to try to send first in part because http works that way but 1921 // also because openssl will sometimes leave something ready to read even if we haven't 1922 // sent yet (probably leftover data from the crypto negotiation) and if that happens ssl 1923 // is liable to block forever hogging the connection and not letting it send... 1924 if(request.state == State.connecting) 1925 if(writeSet.isSet(sock) || readSet.isSet(sock)) { 1926 import core.stdc.stdint; 1927 int32_t error; 1928 int retopt = sock.getOption(SocketOptionLevel.SOCKET, SocketOption.ERROR, error); 1929 if(retopt < 0 || error != 0) { 1930 request.state = State.aborted; 1931 1932 request.responseData.code = 2; 1933 try { 1934 request.responseData.codeText = "connection failed - " ~ formatSocketError(error); 1935 } catch(Exception e) { 1936 request.responseData.codeText = "connection failed"; 1937 } 1938 inactive[inactiveCount++] = sock; 1939 sock.close(); 1940 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1941 continue; 1942 } else { 1943 if(auto s = cast(SslClientSocket) sock) { 1944 sslProceed(request, s); 1945 continue; 1946 } else { 1947 request.state = State.sendingHeaders; 1948 } 1949 } 1950 } 1951 1952 if(request.state == State.sslConnectPendingRead) 1953 if(readSet.isSet(sock)) { 1954 sslProceed(request, cast(SslClientSocket) sock); 1955 continue; 1956 } 1957 if(request.state == State.sslConnectPendingWrite) 1958 if(writeSet.isSet(sock)) { 1959 sslProceed(request, cast(SslClientSocket) sock); 1960 continue; 1961 } 1962 1963 if(request.state == State.sendingHeaders || request.state == State.sendingBody) 1964 if(writeSet.isSet(sock)) { 1965 request.timeoutFromInactivity = MonoTime.currTime + request.requestParameters.timeoutFromInactivity; 1966 assert(request.sendBuffer.length); 1967 auto sent = sock.send(request.sendBuffer); 1968 debug(arsd_http2_verbose) writeln(cast(void*) sock, "<send>", cast(string) request.sendBuffer, "</send>"); 1969 if(sent <= 0) { 1970 if(wouldHaveBlocked()) 1971 continue; 1972 1973 request.state = State.aborted; 1974 1975 request.responseData.code = 3; 1976 request.responseData.codeText = "send failed to server"; 1977 inactive[inactiveCount++] = sock; 1978 sock.close(); 1979 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1980 continue; 1981 1982 } 1983 request.sendBuffer = request.sendBuffer[sent .. $]; 1984 if(request.sendBuffer.length == 0) { 1985 request.state = State.waitingForResponse; 1986 1987 debug(arsd_http2_verbose) writeln("all sent"); 1988 } 1989 } 1990 1991 1992 if(readSet.isSet(sock)) { 1993 keep_going: 1994 request.timeoutFromInactivity = MonoTime.currTime + request.requestParameters.timeoutFromInactivity; 1995 auto got = sock.receive(buffer); 1996 debug(arsd_http2_verbose) { if(got < 0) writeln(lastSocketError); else writeln("====PACKET ",got,"=====",cast(string)buffer[0 .. got],"===/PACKET==="); } 1997 if(got < 0) { 1998 if(wouldHaveBlocked()) 1999 continue; 2000 debug(arsd_http2) writeln("receive error"); 2001 if(request.state != State.complete) { 2002 request.state = State.aborted; 2003 2004 request.responseData.code = 3; 2005 request.responseData.codeText = "receive error from server"; 2006 } 2007 inactive[inactiveCount++] = sock; 2008 sock.close(); 2009 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 2010 } else if(got == 0) { 2011 // remote side disconnected 2012 debug(arsd_http2) writeln("remote disconnect"); 2013 if(request.state != State.complete) { 2014 request.state = State.aborted; 2015 2016 request.responseData.code = 3; 2017 request.responseData.codeText = "server disconnected"; 2018 } 2019 inactive[inactiveCount++] = sock; 2020 sock.close(); 2021 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 2022 } else { 2023 // data available 2024 bool stillAlive; 2025 2026 try { 2027 stillAlive = request.handleIncomingData(buffer[0 .. got]); 2028 /+ 2029 state needs to be set and public 2030 requestData.content/contentText needs to be around 2031 you need to be able to clear the content and keep processing for things like event sources. 2032 also need to be able to abort it. 2033 2034 and btw it should prolly just have evnet source as a pre-packaged thing. 2035 +/ 2036 } catch (Exception e) { 2037 debug(arsd_http2_verbose) { import std.stdio; writeln(e); } 2038 request.state = HttpRequest.State.aborted; 2039 request.responseData.code = 4; 2040 request.responseData.codeText = e.msg; 2041 2042 inactive[inactiveCount++] = sock; 2043 sock.close(); 2044 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 2045 continue; 2046 } 2047 2048 if(!stillAlive || request.state == HttpRequest.State.complete || request.state == HttpRequest.State.aborted) { 2049 //import std.stdio; writeln(cast(void*) sock, " ", stillAlive, " ", request.state); 2050 inactive[inactiveCount++] = sock; 2051 continue; 2052 // reuse the socket for another pending request, if we can 2053 } 2054 } 2055 2056 if(request.onDataReceived) 2057 request.onDataReceived(request); 2058 2059 version(with_openssl) 2060 if(auto s = cast(SslClientSocket) sock) { 2061 // select doesn't handle the case with stuff 2062 // left in the ssl buffer so i'm checking it separately 2063 if(s.dataPending()) { 2064 goto keep_going; 2065 } 2066 } 2067 } 2068 } 2069 } 2070 2071 killInactives(); 2072 2073 // we've completed a request, are there any more pending connection? if so, send them now 2074 2075 return 0; 2076 } 2077 } 2078 2079 public static void resetInternals() { 2080 socketsPerHost = null; 2081 activeRequestOnSocket = null; 2082 pending = null; 2083 2084 } 2085 2086 struct HeaderReadingState { 2087 bool justSawLf; 2088 bool justSawCr; 2089 bool atStartOfLine = true; 2090 bool readingLineContinuation; 2091 } 2092 HeaderReadingState headerReadingState; 2093 2094 struct BodyReadingState { 2095 bool isGzipped; 2096 bool isDeflated; 2097 2098 bool isChunked; 2099 int chunkedState; 2100 2101 // used for the chunk size if it is chunked 2102 int contentLengthRemaining; 2103 } 2104 BodyReadingState bodyReadingState; 2105 2106 bool closeSocketWhenComplete; 2107 2108 import std.zlib; 2109 UnCompress uncompress; 2110 2111 const(ubyte)[] leftoverDataFromLastTime; 2112 2113 bool handleIncomingData(scope const ubyte[] dataIn) { 2114 bool stillAlive = true; 2115 debug(arsd_http2) writeln("handleIncomingData, state: ", state); 2116 if(state == State.waitingForResponse) { 2117 state = State.readingHeaders; 2118 headerReadingState = HeaderReadingState.init; 2119 bodyReadingState = BodyReadingState.init; 2120 } 2121 2122 const(ubyte)[] data; 2123 if(leftoverDataFromLastTime.length) 2124 data = leftoverDataFromLastTime ~ dataIn[]; 2125 else 2126 data = dataIn[]; 2127 2128 if(state == State.readingHeaders) { 2129 void parseLastHeader() { 2130 assert(responseData.headers.length); 2131 if(responseData.headers.length == 1) { 2132 responseData.statusLine = responseData.headers[0]; 2133 import std.algorithm; 2134 auto parts = responseData.statusLine.splitter(" "); 2135 responseData.httpVersion = parts.front; 2136 parts.popFront(); 2137 if(parts.empty) 2138 throw new Exception("Corrupted response, bad status line"); 2139 responseData.code = to!int(parts.front()); 2140 parts.popFront(); 2141 responseData.codeText = ""; 2142 while(!parts.empty) { 2143 // FIXME: this sucks! 2144 responseData.codeText ~= parts.front(); 2145 parts.popFront(); 2146 if(!parts.empty) 2147 responseData.codeText ~= " "; 2148 } 2149 } else { 2150 // parse the new header 2151 auto header = responseData.headers[$-1]; 2152 2153 auto colon = header.indexOf(":"); 2154 if(colon < 0 || colon >= header.length) 2155 return; 2156 auto name = toLower(header[0 .. colon]); 2157 auto value = header[colon + 1 .. $].strip; // skip colon and strip whitespace 2158 2159 switch(name) { 2160 case "connection": 2161 if(value == "close") 2162 closeSocketWhenComplete = true; 2163 break; 2164 case "content-type": 2165 responseData.contentType = value; 2166 break; 2167 case "location": 2168 responseData.location = value; 2169 break; 2170 case "content-length": 2171 bodyReadingState.contentLengthRemaining = to!int(value); 2172 break; 2173 case "transfer-encoding": 2174 // note that if it is gzipped, it zips first, then chunks the compressed stream. 2175 // so we should always dechunk first, then feed into the decompressor 2176 if(value == "chunked") 2177 bodyReadingState.isChunked = true; 2178 else throw new Exception("Unknown Transfer-Encoding: " ~ value); 2179 break; 2180 case "content-encoding": 2181 if(value == "gzip") { 2182 bodyReadingState.isGzipped = true; 2183 uncompress = new UnCompress(); 2184 } else if(value == "deflate") { 2185 bodyReadingState.isDeflated = true; 2186 uncompress = new UnCompress(); 2187 } else throw new Exception("Unknown Content-Encoding: " ~ value); 2188 break; 2189 case "set-cookie": 2190 // handled elsewhere fyi 2191 break; 2192 default: 2193 // ignore 2194 } 2195 2196 responseData.headersHash[name] = value; 2197 } 2198 } 2199 2200 size_t position = 0; 2201 for(position = 0; position < data.length; position++) { 2202 if(headerReadingState.readingLineContinuation) { 2203 if(data[position] == ' ' || data[position] == '\t') 2204 continue; 2205 headerReadingState.readingLineContinuation = false; 2206 } 2207 2208 if(headerReadingState.atStartOfLine) { 2209 headerReadingState.atStartOfLine = false; 2210 // FIXME it being \r should never happen... and i don't think it does 2211 if(data[position] == '\r' || data[position] == '\n') { 2212 // done with headers 2213 2214 position++; // skip the \r 2215 2216 if(responseData.headers.length) 2217 parseLastHeader(); 2218 2219 if(responseData.code >= 100 && responseData.code < 200) { 2220 // "100 Continue" - we should continue uploading request data at this point 2221 // "101 Switching Protocols" - websocket, not expected here... 2222 // "102 Processing" - server still working, keep the connection alive 2223 // "103 Early Hints" - can have useful Link headers etc 2224 // 2225 // and other unrecognized ones can just safely be skipped 2226 2227 // FIXME: the headers shouldn't actually be reset; 103 Early Hints 2228 // can give useful headers we want to keep 2229 2230 responseData.headers = null; 2231 headerReadingState.atStartOfLine = true; 2232 2233 continue; // the \n will be skipped by the for loop advance 2234 } 2235 2236 if(this.requestParameters.method == HttpVerb.HEAD) 2237 state = State.complete; 2238 else 2239 state = State.readingBody; 2240 2241 // skip the \n before we break 2242 position++; 2243 2244 break; 2245 } else if(data[position] == ' ' || data[position] == '\t') { 2246 // line continuation, ignore all whitespace and collapse it into a space 2247 headerReadingState.readingLineContinuation = true; 2248 responseData.headers[$-1] ~= ' '; 2249 } else { 2250 // new header 2251 if(responseData.headers.length) 2252 parseLastHeader(); 2253 responseData.headers ~= ""; 2254 } 2255 } 2256 2257 if(data[position] == '\r') { 2258 headerReadingState.justSawCr = true; 2259 continue; 2260 } else 2261 headerReadingState.justSawCr = false; 2262 2263 if(data[position] == '\n') { 2264 headerReadingState.justSawLf = true; 2265 headerReadingState.atStartOfLine = true; 2266 continue; 2267 } else 2268 headerReadingState.justSawLf = false; 2269 2270 responseData.headers[$-1] ~= data[position]; 2271 } 2272 2273 data = data[position .. $]; 2274 } 2275 2276 if(state == State.readingBody) { 2277 if(bodyReadingState.isChunked) { 2278 // read the hex length, stopping at a \r\n, ignoring everything between the new line but after the first non-valid hex character 2279 // read binary data of that length. it is our content 2280 // repeat until a zero sized chunk 2281 // then read footers as headers. 2282 2283 start_over: 2284 for(int a = 0; a < data.length; a++) { 2285 final switch(bodyReadingState.chunkedState) { 2286 case 0: // reading hex 2287 char c = data[a]; 2288 if((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) { 2289 // just keep reading 2290 } else { 2291 int power = 1; 2292 bodyReadingState.contentLengthRemaining = 0; 2293 if(a == 0) 2294 break; // just wait for more data 2295 assert(a != 0, cast(string) data); 2296 for(int b = a-1; b >= 0; b--) { 2297 char cc = data[b]; 2298 if(cc >= 'a' && cc <= 'z') 2299 cc -= 0x20; 2300 int val = 0; 2301 if(cc >= '0' && cc <= '9') 2302 val = cc - '0'; 2303 else 2304 val = cc - 'A' + 10; 2305 2306 assert(val >= 0 && val <= 15, to!string(val)); 2307 bodyReadingState.contentLengthRemaining += power * val; 2308 power *= 16; 2309 } 2310 debug(arsd_http2_verbose) writeln("Chunk length: ", bodyReadingState.contentLengthRemaining); 2311 bodyReadingState.chunkedState = 1; 2312 data = data[a + 1 .. $]; 2313 goto start_over; 2314 } 2315 break; 2316 case 1: // reading until end of line 2317 char c = data[a]; 2318 if(c == '\n') { 2319 if(bodyReadingState.contentLengthRemaining == 0) 2320 bodyReadingState.chunkedState = 5; 2321 else 2322 bodyReadingState.chunkedState = 2; 2323 } 2324 data = data[a + 1 .. $]; 2325 goto start_over; 2326 case 2: // reading data 2327 auto can = a + bodyReadingState.contentLengthRemaining; 2328 if(can > data.length) 2329 can = cast(int) data.length; 2330 2331 auto newData = data[a .. can]; 2332 data = data[can .. $]; 2333 2334 //if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) 2335 // responseData.content ~= cast(ubyte[]) uncompress.uncompress(data[a .. can]); 2336 //else 2337 responseData.content ~= newData; 2338 2339 bodyReadingState.contentLengthRemaining -= newData.length; 2340 debug(arsd_http2_verbose) writeln("clr: ", bodyReadingState.contentLengthRemaining, " " , a, " ", can); 2341 assert(bodyReadingState.contentLengthRemaining >= 0); 2342 if(bodyReadingState.contentLengthRemaining == 0) { 2343 bodyReadingState.chunkedState = 3; 2344 } else { 2345 // will continue grabbing more 2346 } 2347 goto start_over; 2348 case 3: // reading 13/10 2349 assert(data[a] == 13); 2350 bodyReadingState.chunkedState++; 2351 data = data[a + 1 .. $]; 2352 goto start_over; 2353 case 4: // reading 10 at end of packet 2354 assert(data[a] == 10); 2355 data = data[a + 1 .. $]; 2356 bodyReadingState.chunkedState = 0; 2357 goto start_over; 2358 case 5: // reading footers 2359 //goto done; // FIXME 2360 2361 int footerReadingState = 0; 2362 int footerSize; 2363 2364 while(footerReadingState != 2 && a < data.length) { 2365 // import std.stdio; writeln(footerReadingState, " ", footerSize, " ", data); 2366 switch(footerReadingState) { 2367 case 0: 2368 if(data[a] == 13) 2369 footerReadingState++; 2370 else 2371 footerSize++; 2372 break; 2373 case 1: 2374 if(data[a] == 10) { 2375 if(footerSize == 0) { 2376 // all done, time to break 2377 footerReadingState++; 2378 2379 } else { 2380 // actually had a footer, try to read another 2381 footerReadingState = 0; 2382 footerSize = 0; 2383 } 2384 } else { 2385 throw new Exception("bad footer thing"); 2386 } 2387 break; 2388 default: 2389 assert(0); 2390 } 2391 2392 a++; 2393 } 2394 2395 if(footerReadingState != 2) 2396 break start_over; // haven't hit the end of the thing yet 2397 2398 bodyReadingState.chunkedState = 0; 2399 data = data[a .. $]; 2400 2401 if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) { 2402 auto n = uncompress.uncompress(responseData.content); 2403 n ~= uncompress.flush(); 2404 responseData.content = cast(ubyte[]) n; 2405 } 2406 2407 // responseData.content ~= cast(ubyte[]) uncompress.flush(); 2408 responseData.contentText = cast(string) responseData.content; 2409 2410 goto done; 2411 } 2412 } 2413 2414 } else { 2415 //if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) 2416 // responseData.content ~= cast(ubyte[]) uncompress.uncompress(data); 2417 //else 2418 responseData.content ~= data; 2419 //assert(data.length <= bodyReadingState.contentLengthRemaining, format("%d <= %d\n%s", data.length, bodyReadingState.contentLengthRemaining, cast(string)data)); 2420 { 2421 int use = cast(int) data.length; 2422 if(use > bodyReadingState.contentLengthRemaining) 2423 use = bodyReadingState.contentLengthRemaining; 2424 bodyReadingState.contentLengthRemaining -= use; 2425 data = data[use .. $]; 2426 } 2427 if(bodyReadingState.contentLengthRemaining == 0) { 2428 if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) { 2429 // import std.stdio; writeln(responseData.content.length, " ", responseData.content[0 .. 2], " .. ", responseData.content[$-2 .. $]); 2430 auto n = uncompress.uncompress(responseData.content); 2431 n ~= uncompress.flush(); 2432 responseData.content = cast(ubyte[]) n; 2433 responseData.contentText = cast(string) responseData.content; 2434 //responseData.content ~= cast(ubyte[]) uncompress.flush(); 2435 } else { 2436 responseData.contentText = cast(string) responseData.content; 2437 } 2438 2439 done: 2440 2441 if(retainCookies && client !is null) { 2442 client.retainCookies(responseData); 2443 } 2444 2445 if(followLocation && responseData.location.length) { 2446 if(maximumNumberOfRedirectsRemaining <= 0) { 2447 throw new Exception("Maximum number of redirects exceeded"); 2448 } else { 2449 maximumNumberOfRedirectsRemaining--; 2450 } 2451 2452 static bool first = true; 2453 //version(DigitalMars) if(!first) asm { int 3; } 2454 debug(arsd_http2) writeln("redirecting to ", responseData.location); 2455 populateFromInfo(Uri(responseData.location), HttpVerb.GET); 2456 //import std.stdio; writeln("redirected to ", responseData.location); 2457 first = false; 2458 responseData = HttpResponse.init; 2459 headerReadingState = HeaderReadingState.init; 2460 bodyReadingState = BodyReadingState.init; 2461 if(client !is null) { 2462 // FIXME: this won't clear cookies that were cleared in another request 2463 client.populateCookies(this); // they might have changed in the previous redirection cycle! 2464 } 2465 state = State.unsent; 2466 stillAlive = false; 2467 sendPrivate(false); 2468 } else { 2469 state = State.complete; 2470 // FIXME 2471 //if(closeSocketWhenComplete) 2472 //socket.close(); 2473 } 2474 } 2475 } 2476 } 2477 2478 if(data.length) 2479 leftoverDataFromLastTime = data.dup; 2480 else 2481 leftoverDataFromLastTime = null; 2482 2483 return stillAlive; 2484 } 2485 2486 } 2487 } 2488 2489 /++ 2490 Waits for the first of the given requests to be either aborted or completed. 2491 Returns the first one in that state, or `null` if the operation was interrupted 2492 or reached the given timeout before any completed. (If it returns null even before 2493 the timeout, it might be because the user pressed ctrl+c, so you should consider 2494 checking if you should cancel the operation. If not, you can simply call it again 2495 with the same arguments to start waiting again.) 2496 2497 You MUST check for null, even if you don't specify a timeout! 2498 2499 Note that if an individual request times out before any others request, it will 2500 return that timed out request, since that counts as completion. 2501 2502 If the return is not null, you should call `waitForCompletion` on the given request 2503 to get the response out. It will not have to wait since it is guaranteed to be 2504 finished when returned by this function; that will just give you the cached response. 2505 2506 (I thought about just having it return the response, but tying a response back to 2507 a request is harder than just getting the original request object back and taking 2508 the response out of it.) 2509 2510 Please note: if a request in the set has already completed or been aborted, it will 2511 always return the first one it sees upon calling the function. You may wish to remove 2512 them from the list before calling the function. 2513 2514 History: 2515 Added December 24, 2021 (dub v10.5) 2516 +/ 2517 HttpRequest waitForFirstToComplete(Duration timeout, HttpRequest[] requests...) { 2518 2519 foreach(request; requests) { 2520 if(request.state == HttpRequest.State.unsent) 2521 request.send(); 2522 else if(request.state == HttpRequest.State.complete) 2523 return request; 2524 else if(request.state == HttpRequest.State.aborted) 2525 return request; 2526 } 2527 2528 while(true) { 2529 if(auto err = HttpRequest.advanceConnections(timeout)) { 2530 switch(err) { 2531 case 1: return null; 2532 case 2: throw new Exception("HttpRequest.advanceConnections returned 2: nothing to do"); 2533 case 3: return null; 2534 default: throw new Exception("HttpRequest.advanceConnections got err " ~ to!string(err)); 2535 } 2536 } 2537 2538 foreach(request; requests) { 2539 if(request.state == HttpRequest.State.aborted || request.state == HttpRequest.State.complete) { 2540 request.waitForCompletion(); 2541 return request; 2542 } 2543 } 2544 2545 } 2546 } 2547 2548 /// ditto 2549 HttpRequest waitForFirstToComplete(HttpRequest[] requests...) { 2550 return waitForFirstToComplete(1.weeks, requests); 2551 } 2552 2553 /++ 2554 An input range that runs [waitForFirstToComplete] but only returning each request once. 2555 Before you loop over it, you can set some properties to customize behavior. 2556 2557 If it times out or is interrupted, it will prematurely run empty. You can set the delegate 2558 to process this. 2559 2560 Implementation note: each iteration through the loop does a O(n) check over each item remaining. 2561 This shouldn't matter, but if it does become an issue for you, let me know. 2562 2563 History: 2564 Added December 24, 2021 (dub v10.5) 2565 +/ 2566 struct HttpRequestsAsTheyComplete { 2567 /++ 2568 Seeds it with an overall timeout and the initial requests. 2569 It will send all the requests before returning, then will process 2570 the responses as they come. 2571 2572 Please note that it modifies the array of requests you pass in! It 2573 will keep a reference to it and reorder items on each call of popFront. 2574 You might want to pass a duplicate if you have another purpose for your 2575 array and don't want to see it shuffled. 2576 +/ 2577 this(Duration timeout, HttpRequest[] requests) { 2578 remainingRequests = requests; 2579 this.timeout = timeout; 2580 popFront(); 2581 } 2582 2583 /++ 2584 You can set this delegate to decide how to handle an interruption. Returning true 2585 from this will keep working. Returning false will terminate the loop. 2586 2587 If this is null, an interruption will always terminate the loop. 2588 2589 Note that interruptions can be caused by the garbage collector being triggered by 2590 another thread as well as by user action. If you don't set a SIGINT handler, it 2591 might be reasonable to always return true here. 2592 +/ 2593 bool delegate() onInterruption; 2594 2595 private HttpRequest[] remainingRequests; 2596 2597 /// The timeout you set in the constructor. You can change it if you want. 2598 Duration timeout; 2599 2600 /++ 2601 Adds another request to the work queue. It is safe to call this from inside the loop 2602 as you process other requests. 2603 +/ 2604 void appendRequest(HttpRequest request) { 2605 remainingRequests ~= request; 2606 } 2607 2608 /++ 2609 If the loop exited, it might be due to an interruption or a time out. If you like, you 2610 can call this to pick up the work again, 2611 2612 If it returns `false`, the work is indeed all finished and you should not re-enter the loop. 2613 2614 --- 2615 auto range = HttpRequestsAsTheyComplete(10.seconds, your_requests); 2616 process_loop: foreach(req; range) { 2617 // process req 2618 } 2619 // make sure we weren't interrupted because the user requested we cancel! 2620 // but then try to re-enter the range if possible 2621 if(!user_quit && range.reenter()) { 2622 // there's still something unprocessed in there 2623 // range.reenter returning true means it is no longer 2624 // empty, so we should try to loop over it again 2625 goto process_loop; // re-enter the loop 2626 } 2627 --- 2628 +/ 2629 bool reenter() { 2630 if(remainingRequests.length == 0) 2631 return false; 2632 empty = false; 2633 popFront(); 2634 return true; 2635 } 2636 2637 /// Standard range primitives. I reserve the right to change the variables to read-only properties in the future without notice. 2638 HttpRequest front; 2639 2640 /// ditto 2641 bool empty; 2642 2643 /// ditto 2644 void popFront() { 2645 resume: 2646 if(remainingRequests.length == 0) { 2647 empty = true; 2648 return; 2649 } 2650 2651 front = waitForFirstToComplete(timeout, remainingRequests); 2652 2653 if(front is null) { 2654 if(onInterruption) { 2655 if(onInterruption()) 2656 goto resume; 2657 } 2658 empty = true; 2659 return; 2660 } 2661 foreach(idx, req; remainingRequests) { 2662 if(req is front) { 2663 remainingRequests[idx] = remainingRequests[$ - 1]; 2664 remainingRequests = remainingRequests[0 .. $ - 1]; 2665 return; 2666 } 2667 } 2668 } 2669 } 2670 2671 // 2672 struct HttpRequestParameters { 2673 // FIXME: implement these 2674 //Duration timeoutTotal; // the whole request must finish in this time or else it fails,even if data is still trickling in 2675 Duration timeoutFromInactivity; // if there's no activity in this time it dies. basically the socket receive timeout 2676 2677 // debugging 2678 bool useHttp11 = true; /// 2679 bool acceptGzip = true; /// 2680 bool keepAlive = true; /// 2681 2682 // the request itself 2683 HttpVerb method; /// 2684 string host; /// 2685 ushort port; /// 2686 string uri; /// 2687 2688 bool ssl; /// 2689 2690 string userAgent; /// 2691 string authorization; /// 2692 2693 string[string] cookies; /// 2694 2695 string[] headers; /// do not duplicate host, content-length, content-type, or any others that have a specific property 2696 2697 string contentType; /// 2698 ubyte[] bodyData; /// 2699 2700 string unixSocketPath; /// 2701 } 2702 2703 interface IHttpClient { 2704 2705 } 2706 2707 /// 2708 enum HttpVerb { 2709 /// 2710 GET, 2711 /// 2712 HEAD, 2713 /// 2714 POST, 2715 /// 2716 PUT, 2717 /// 2718 DELETE, 2719 /// 2720 OPTIONS, 2721 /// 2722 TRACE, 2723 /// 2724 CONNECT, 2725 /// 2726 PATCH, 2727 /// 2728 MERGE 2729 } 2730 2731 /++ 2732 Supported file formats for [HttpClient.setClientCert]. These are loaded by OpenSSL 2733 in the current implementation. 2734 2735 History: 2736 Added February 3, 2022 (dub v10.6) 2737 +/ 2738 enum CertificateFileFormat { 2739 guess, /// try to guess the format from the file name and/or contents 2740 pem, /// the files are specifically in PEM format 2741 der /// the files are specifically in DER format 2742 } 2743 2744 /++ 2745 HttpClient keeps cookies, location, and some other state to reuse connections, when possible, like a web browser. 2746 You can use it as your entry point to make http requests. 2747 2748 See the example on [arsd.http2#examples]. 2749 +/ 2750 class HttpClient { 2751 /* Protocol restrictions, useful to disable when debugging servers */ 2752 bool useHttp11 = true; /// 2753 bool acceptGzip = true; /// 2754 bool keepAlive = true; /// 2755 2756 /++ 2757 Sets the client certificate used as a log in identifier on https connections. 2758 The certificate and key must be unencrypted at this time and both must be in 2759 the same file format. 2760 2761 Bugs: 2762 The current implementation sets the filenames into a static variable, 2763 meaning it is shared across all clients and connections. 2764 2765 Errors in the cert or key are only reported if the server reports an 2766 authentication failure. Make sure you are passing correct filenames 2767 and formats of you do see a failure. 2768 2769 History: 2770 Added February 2, 2022 (dub v10.6) 2771 +/ 2772 void setClientCertificate(string certFilename, string keyFilename, CertificateFileFormat certFormat = CertificateFileFormat.guess) { 2773 this.certFilename = certFilename; 2774 this.keyFilename = keyFilename; 2775 this.certFormat = certFormat; 2776 } 2777 2778 /++ 2779 Sets whether [HttpRequest]s created through this object (with [navigateTo], [request], etc.), will have the 2780 value of [HttpRequest.verifyPeer] of true or false upon construction. 2781 2782 History: 2783 Added April 5, 2022 (dub v10.8). Previously, there was an undocumented global value used. 2784 +/ 2785 bool defaultVerifyPeer = true; 2786 2787 // FIXME: try to not make these static 2788 private static string certFilename; 2789 private static string keyFilename; 2790 private static CertificateFileFormat certFormat; 2791 2792 /// 2793 @property Uri location() { 2794 return currentUrl; 2795 } 2796 2797 /++ 2798 Default timeout for requests created on this client. 2799 2800 History: 2801 Added March 31, 2021 2802 +/ 2803 Duration defaultTimeout = 10.seconds; 2804 2805 /++ 2806 High level function that works similarly to entering a url 2807 into a browser. 2808 2809 Follows locations, retain cookies, updates the current url, etc. 2810 +/ 2811 HttpRequest navigateTo(Uri where, HttpVerb method = HttpVerb.GET) { 2812 currentUrl = where.basedOn(currentUrl); 2813 currentDomain = where.host; 2814 2815 auto request = this.request(currentUrl, method); 2816 request.followLocation = true; 2817 request.retainCookies = true; 2818 2819 return request; 2820 } 2821 2822 /++ 2823 Creates a request without updating the current url state. If you want to save cookies, either call [retainCookies] with the response yourself 2824 or set [HttpRequest.retainCookies|request.retainCookies] to `true` on the returned object. But see important implementation shortcomings on [retainCookies]. 2825 +/ 2826 HttpRequest request(Uri uri, HttpVerb method = HttpVerb.GET, ubyte[] bodyData = null, string contentType = null) { 2827 string proxyToUse = getProxyFor(uri); 2828 2829 auto request = new HttpRequest(this, uri, method, cache, defaultTimeout, proxyToUse); 2830 2831 request.verifyPeer = this.defaultVerifyPeer; 2832 2833 request.requestParameters.userAgent = userAgent; 2834 request.requestParameters.authorization = authorization; 2835 2836 request.requestParameters.useHttp11 = this.useHttp11; 2837 request.requestParameters.acceptGzip = this.acceptGzip; 2838 request.requestParameters.keepAlive = this.keepAlive; 2839 2840 request.requestParameters.bodyData = bodyData; 2841 request.requestParameters.contentType = contentType; 2842 2843 populateCookies(request); 2844 2845 return request; 2846 2847 } 2848 2849 private void populateCookies(HttpRequest request) { 2850 // FIXME: what about expiration and the like? or domain/path checks? or Secure checks? 2851 // FIXME: is uri.host correct? i think it should include port number too. what fun. 2852 if(auto cookies = ""/*uri.host*/ in this.cookies) { 2853 foreach(cookie; *cookies) 2854 request.requestParameters.cookies[cookie.name] = cookie.value; 2855 } 2856 } 2857 2858 2859 /// ditto 2860 HttpRequest request(Uri uri, FormData fd, HttpVerb method = HttpVerb.POST) { 2861 return request(uri, method, fd.toBytes, fd.contentType); 2862 } 2863 2864 2865 private Uri currentUrl; 2866 private string currentDomain; 2867 private ICache cache; 2868 2869 /++ 2870 2871 +/ 2872 this(ICache cache = null) { 2873 this.defaultVerifyPeer = .defaultVerifyPeer_; 2874 this.cache = cache; 2875 loadDefaultProxy(); 2876 } 2877 2878 /++ 2879 Loads the system-default proxy. Note that the constructor does this automatically 2880 so you should rarely need to call this explicitly. 2881 2882 The environment variables are used, if present, on all operating systems. 2883 2884 History: 2885 no_proxy support added April 13, 2022 2886 2887 Added April 12, 2021 (included in dub v9.5) 2888 2889 Bugs: 2890 On Windows, it does NOT currently check the IE settings, but I do intend to 2891 implement that in the future. When I do, it will be classified as a bug fix, 2892 NOT a breaking change. 2893 +/ 2894 void loadDefaultProxy() { 2895 import std.process; 2896 httpProxy = environment.get("http_proxy", environment.get("HTTP_PROXY", null)); 2897 httpsProxy = environment.get("https_proxy", environment.get("HTTPS_PROXY", null)); 2898 auto noProxy = environment.get("no_proxy", environment.get("NO_PROXY", null)); 2899 if (noProxy.length) { 2900 proxyIgnore = noProxy.split(","); 2901 foreach (ref rule; proxyIgnore) 2902 rule = rule.strip; 2903 } 2904 2905 // FIXME: on Windows, I should use the Internet Explorer proxy settings 2906 } 2907 2908 /++ 2909 Checks if the given uri should be proxied according to the httpProxy, httpsProxy, proxyIgnore 2910 variables and returns either httpProxy, httpsProxy or null. 2911 2912 If neither `httpProxy` or `httpsProxy` are set this always returns `null`. Same if `proxyIgnore` 2913 contains `*`. 2914 2915 DNS is not resolved for proxyIgnore IPs, only IPs match IPs and hosts match hosts. 2916 +/ 2917 string getProxyFor(Uri uri) { 2918 string proxyToUse; 2919 switch(uri.scheme) { 2920 case "http": 2921 proxyToUse = httpProxy; 2922 break; 2923 case "https": 2924 proxyToUse = httpsProxy; 2925 break; 2926 default: 2927 proxyToUse = null; 2928 } 2929 2930 if (proxyToUse.length) { 2931 foreach (ignore; proxyIgnore) { 2932 if (matchProxyIgnore(ignore, uri)) { 2933 return null; 2934 } 2935 } 2936 } 2937 2938 return proxyToUse; 2939 } 2940 2941 /// Returns -1 on error, otherwise the IP as uint. Parsing is very strict. 2942 private static long tryParseIPv4(scope const(char)[] s) nothrow { 2943 import std.algorithm : findSplit, all; 2944 import std.ascii : isDigit; 2945 2946 static int parseNum(scope const(char)[] num) nothrow { 2947 if (num.length < 1 || num.length > 3 || !num.representation.all!isDigit) 2948 return -1; 2949 try { 2950 auto ret = num.to!int; 2951 return ret > 255 ? -1 : ret; 2952 } catch (Exception) { 2953 assert(false); 2954 } 2955 } 2956 2957 if (s.length < "0.0.0.0".length || s.length > "255.255.255.255".length) 2958 return -1; 2959 auto firstPair = s.findSplit("."); 2960 auto secondPair = firstPair[2].findSplit("."); 2961 auto thirdPair = secondPair[2].findSplit("."); 2962 auto a = parseNum(firstPair[0]); 2963 auto b = parseNum(secondPair[0]); 2964 auto c = parseNum(thirdPair[0]); 2965 auto d = parseNum(thirdPair[2]); 2966 if (a < 0 || b < 0 || c < 0 || d < 0) 2967 return -1; 2968 return (cast(uint)a << 24) | (b << 16) | (c << 8) | (d); 2969 } 2970 2971 unittest { 2972 assert(tryParseIPv4("0.0.0.0") == 0); 2973 assert(tryParseIPv4("127.0.0.1") == 0x7f000001); 2974 assert(tryParseIPv4("162.217.114.56") == 0xa2d97238); 2975 assert(tryParseIPv4("256.0.0.1") == -1); 2976 assert(tryParseIPv4("0.0.0.-2") == -1); 2977 assert(tryParseIPv4("0.0.0.a") == -1); 2978 assert(tryParseIPv4("0.0.0") == -1); 2979 assert(tryParseIPv4("0.0.0.0.0") == -1); 2980 } 2981 2982 /++ 2983 Returns true if the given no_proxy rule matches the uri. 2984 2985 Invalid IP ranges are silently ignored and return false. 2986 2987 See $(LREF proxyIgnore). 2988 +/ 2989 static bool matchProxyIgnore(scope const(char)[] rule, scope const Uri uri) nothrow { 2990 import std.algorithm; 2991 import std.ascii : isDigit; 2992 import std.uni : sicmp; 2993 2994 string uriHost = uri.host; 2995 if (uriHost.length && uriHost[$ - 1] == '.') 2996 uriHost = uriHost[0 .. $ - 1]; 2997 2998 if (rule == "*") 2999 return true; 3000 while (rule.length && rule[0] == '.') rule = rule[1 .. $]; 3001 3002 static int parsePort(scope const(char)[] portStr) nothrow { 3003 if (portStr.length < 1 || portStr.length > 5 || !portStr.representation.all!isDigit) 3004 return -1; 3005 try { 3006 return portStr.to!int; 3007 } catch (Exception) { 3008 assert(false, "to!int should succeed"); 3009 } 3010 } 3011 3012 if (sicmp(rule, uriHost) == 0 3013 || (uriHost.length > rule.length 3014 && sicmp(rule, uriHost[$ - rule.length .. $]) == 0 3015 && uriHost[$ - rule.length - 1] == '.')) 3016 return true; 3017 3018 if (rule.startsWith("[")) { // IPv6 3019 // below code is basically nothrow lastIndexOfAny("]:") 3020 ptrdiff_t lastColon = cast(ptrdiff_t) rule.length - 1; 3021 while (lastColon >= 0) { 3022 if (rule[lastColon] == ']' || rule[lastColon] == ':') 3023 break; 3024 lastColon--; 3025 } 3026 if (lastColon == -1) 3027 return false; // malformed 3028 3029 if (rule[lastColon] == ':') { // match with port 3030 auto port = parsePort(rule[lastColon + 1 .. $]); 3031 if (port != -1) { 3032 if (uri.effectivePort != port.to!int) 3033 return false; 3034 return uriHost == rule[0 .. lastColon]; 3035 } 3036 } 3037 // exact match of host already done above 3038 } else { 3039 auto slash = rule.lastIndexOfNothrow('/'); 3040 if (slash == -1) { // no IP range 3041 auto colon = rule.lastIndexOfNothrow(':'); 3042 auto host = colon == -1 ? rule : rule[0 .. colon]; 3043 auto port = colon != -1 ? parsePort(rule[colon + 1 .. $]) : -1; 3044 auto ip = tryParseIPv4(host); 3045 if (ip == -1) { // not an IPv4, test for host with port 3046 return port != -1 3047 && uri.effectivePort == port 3048 && uriHost == host; 3049 } else { 3050 // perform IPv4 equals 3051 auto other = tryParseIPv4(uriHost); 3052 if (other == -1) 3053 return false; // rule == IPv4, uri != IPv4 3054 if (port != -1) 3055 return uri.effectivePort == port 3056 && uriHost == host; 3057 else 3058 return uriHost == host; 3059 } 3060 } else { 3061 auto maskStr = rule[slash + 1 .. $]; 3062 auto ip = tryParseIPv4(rule[0 .. slash]); 3063 if (ip == -1) 3064 return false; 3065 if (maskStr.length && maskStr.length < 3 && maskStr.representation.all!isDigit) { 3066 // IPv4 range match 3067 int mask; 3068 try { 3069 mask = maskStr.to!int; 3070 } catch (Exception) { 3071 assert(false); 3072 } 3073 3074 auto other = tryParseIPv4(uriHost); 3075 if (other == -1) 3076 return false; // rule == IPv4, uri != IPv4 3077 3078 if (mask == 0) // matches all 3079 return true; 3080 if (mask > 32) // matches none 3081 return false; 3082 3083 auto shift = 32 - mask; 3084 return cast(uint)other >> shift 3085 == cast(uint)ip >> shift; 3086 } 3087 } 3088 } 3089 return false; 3090 } 3091 3092 unittest { 3093 assert(matchProxyIgnore("0.0.0.0/0", Uri("http://127.0.0.1:80/a"))); 3094 assert(matchProxyIgnore("0.0.0.0/0", Uri("http://127.0.0.1/a"))); 3095 assert(!matchProxyIgnore("0.0.0.0/0", Uri("https://dlang.org/a"))); 3096 assert(matchProxyIgnore("*", Uri("https://dlang.org/a"))); 3097 assert(matchProxyIgnore("127.0.0.0/8", Uri("http://127.0.0.1:80/a"))); 3098 assert(matchProxyIgnore("127.0.0.0/8", Uri("http://127.0.0.1/a"))); 3099 assert(matchProxyIgnore("127.0.0.1", Uri("http://127.0.0.1:1234/a"))); 3100 assert(!matchProxyIgnore("127.0.0.1:80", Uri("http://127.0.0.1:1234/a"))); 3101 assert(!matchProxyIgnore("127.0.0.1/8", Uri("http://localhost/a"))); // no DNS resolution / guessing 3102 assert(!matchProxyIgnore("0.0.0.0/1", Uri("http://localhost/a")) 3103 && !matchProxyIgnore("128.0.0.0/1", Uri("http://localhost/a"))); // no DNS resolution / guessing 2 3104 foreach (m; 1 .. 32) { 3105 assert(matchProxyIgnore(text("127.0.0.1/", m), Uri("http://127.0.0.1/a"))); 3106 assert(!matchProxyIgnore(text("127.0.0.1/", m), Uri("http://128.0.0.1/a"))); 3107 bool expectedMatch = m <= 24; 3108 assert(expectedMatch == matchProxyIgnore(text("127.0.1.0/", m), Uri("http://127.0.1.128/a")), m.to!string); 3109 } 3110 assert(matchProxyIgnore("localhost", Uri("http://localhost/a"))); 3111 assert(matchProxyIgnore("localhost", Uri("http://foo.localhost/a"))); 3112 assert(matchProxyIgnore("localhost", Uri("http://foo.localhost./a"))); 3113 assert(matchProxyIgnore(".localhost", Uri("http://localhost/a"))); 3114 assert(matchProxyIgnore(".localhost", Uri("http://foo.localhost/a"))); 3115 assert(matchProxyIgnore(".localhost", Uri("http://foo.localhost./a"))); 3116 assert(!matchProxyIgnore("foo.localhost", Uri("http://localhost/a"))); 3117 assert(matchProxyIgnore("foo.localhost", Uri("http://foo.localhost/a"))); 3118 assert(matchProxyIgnore("foo.localhost", Uri("http://foo.localhost./a"))); 3119 assert(!matchProxyIgnore("bar.localhost", Uri("http://localhost/a"))); 3120 assert(!matchProxyIgnore("bar.localhost", Uri("http://foo.localhost/a"))); 3121 assert(!matchProxyIgnore("bar.localhost", Uri("http://foo.localhost./a"))); 3122 assert(!matchProxyIgnore("bar.localhost", Uri("http://bbar.localhost./a"))); 3123 assert(matchProxyIgnore("[::1]", Uri("http://[::1]/a"))); 3124 assert(!matchProxyIgnore("[::1]", Uri("http://[::2]/a"))); 3125 assert(matchProxyIgnore("[::1]:80", Uri("http://[::1]/a"))); 3126 assert(!matchProxyIgnore("[::1]:443", Uri("http://[::1]/a"))); 3127 assert(!matchProxyIgnore("[::1]:80", Uri("https://[::1]/a"))); 3128 assert(matchProxyIgnore("[::1]:443", Uri("https://[::1]/a"))); 3129 assert(matchProxyIgnore("google.com", Uri("https://GOOGLE.COM/a"))); 3130 } 3131 3132 /++ 3133 Proxies to use for requests. The [HttpClient] constructor will set these to the system values, 3134 then you can reset it to `null` if you want to override and not use the proxy after all, or you 3135 can set it after construction to whatever. 3136 3137 The proxy from the client will be automatically set to the requests performed through it. You can 3138 also override on a per-request basis by creating the request and setting the `proxy` field there 3139 before sending it. 3140 3141 History: 3142 Added April 12, 2021 (included in dub v9.5) 3143 +/ 3144 string httpProxy; 3145 /// ditto 3146 string httpsProxy; 3147 /++ 3148 List of hosts or ips, optionally including a port, where not to proxy. 3149 3150 Each entry may be one of the following formats: 3151 - `127.0.0.1` (IPv4, any port) 3152 - `127.0.0.1:1234` (IPv4, specific port) 3153 - `127.0.0.1/8` (IPv4 range / CIDR block, any port) 3154 - `[::1]` (IPv6, any port) 3155 - `[::1]:1234` (IPv6, specific port) 3156 - `*` (all hosts and ports, basically don't proxy at all anymore) 3157 - `.domain.name`, `domain.name` (don't proxy the specified domain, 3158 leading dots are stripped and subdomains are also not proxied) 3159 - `.domain.name:1234`, `domain.name:1234` (same as above, with specific port) 3160 3161 No DNS resolution or regex is done in this list. 3162 3163 See https://about.gitlab.com/blog/2021/01/27/we-need-to-talk-no-proxy/ 3164 3165 History: 3166 Added April 13, 2022 3167 +/ 3168 string[] proxyIgnore; 3169 3170 /// See [retainCookies] for important caveats. 3171 void setCookie(string name, string value, string domain = null) { 3172 CookieHeader ch; 3173 3174 ch.name = name; 3175 ch.value = value; 3176 3177 setCookie(ch, domain); 3178 } 3179 3180 /// ditto 3181 void setCookie(CookieHeader ch, string domain = null) { 3182 if(domain is null) 3183 domain = currentDomain; 3184 3185 // FIXME: figure all this out or else cookies liable to get too long, in addition to the overwriting and oversharing issues in long scraping sessions 3186 cookies[""/*domain*/] ~= ch; 3187 } 3188 3189 /++ 3190 [HttpClient] does NOT automatically store cookies. You must explicitly retain them from a response by calling this method. 3191 3192 Examples: 3193 --- 3194 import arsd.http2; 3195 void main() { 3196 auto client = new HttpClient(); 3197 auto setRequest = client.request(Uri("http://arsdnet.net/cgi-bin/cookies/set")); 3198 auto setResponse = setRequest.waitForCompletion(); 3199 3200 auto request = client.request(Uri("http://arsdnet.net/cgi-bin/cookies/get")); 3201 auto response = request.waitForCompletion(); 3202 3203 // the cookie wasn't explicitly retained, so the server echos back nothing 3204 assert(response.responseText.length == 0); 3205 3206 // now keep the cookies from our original set 3207 client.retainCookies(setResponse); 3208 3209 request = client.request(Uri("http://arsdnet.net/cgi-bin/cookies/get")); 3210 response = request.waitForCompletion(); 3211 3212 // now it matches 3213 assert(response.responseText.length && response.responseText == setResponse.cookies["example-cookie"]); 3214 } 3215 --- 3216 3217 Bugs: 3218 It does NOT currently implement domain / path / secure separation nor cookie expiration. It assumes that if you call this function, you're ok with it. 3219 3220 You may want to use separate HttpClient instances if any sharing is unacceptable at this time. 3221 3222 History: 3223 Added July 5, 2021 (dub v10.2) 3224 +/ 3225 void retainCookies(HttpResponse fromResponse) { 3226 foreach(name, value; fromResponse.cookies) 3227 setCookie(name, value); 3228 } 3229 3230 /// 3231 void clearCookies(string domain = null) { 3232 if(domain is null) 3233 cookies = null; 3234 else 3235 cookies[domain] = null; 3236 } 3237 3238 // If you set these, they will be pre-filled on all requests made with this client 3239 string userAgent = "D arsd.html2"; /// 3240 string authorization; /// 3241 3242 /* inter-request state */ 3243 private CookieHeader[][string] cookies; 3244 } 3245 3246 private ptrdiff_t lastIndexOfNothrow(T)(scope T[] arr, T value) nothrow 3247 { 3248 ptrdiff_t ret = cast(ptrdiff_t)arr.length - 1; 3249 while (ret >= 0) { 3250 if (arr[ret] == value) 3251 return ret; 3252 ret--; 3253 } 3254 return ret; 3255 } 3256 3257 interface ICache { 3258 /++ 3259 The client is about to make the given `request`. It will ALWAYS pass it to the cache object first so you can decide if you want to and can provide a response. You should probably check the appropriate headers to see if you should even attempt to look up on the cache (HttpClient does NOT do this to give maximum flexibility to the cache implementor). 3260 3261 Return null if the cache does not provide. 3262 +/ 3263 const(HttpResponse)* getCachedResponse(HttpRequestParameters request); 3264 3265 /++ 3266 The given request has received the given response. The implementing class needs to decide if it wants to cache or not. Return true if it was added, false if you chose not to. 3267 3268 You may wish to examine headers, etc., in making the decision. The HttpClient will ALWAYS pass a request/response to this. 3269 +/ 3270 bool cacheResponse(HttpRequestParameters request, HttpResponse response); 3271 } 3272 3273 /+ 3274 // / Provides caching behavior similar to a real web browser 3275 class HttpCache : ICache { 3276 const(HttpResponse)* getCachedResponse(HttpRequestParameters request) { 3277 return null; 3278 } 3279 } 3280 3281 // / Gives simple maximum age caching, ignoring the actual http headers 3282 class SimpleCache : ICache { 3283 const(HttpResponse)* getCachedResponse(HttpRequestParameters request) { 3284 return null; 3285 } 3286 } 3287 +/ 3288 3289 /++ 3290 A pseudo-cache to provide a mock server. Construct one of these, 3291 populate it with test responses, and pass it to [HttpClient] to 3292 do a network-free test. 3293 3294 You should populate it with the [populate] method. Any request not 3295 pre-populated will return a "server refused connection" response. 3296 +/ 3297 class HttpMockProvider : ICache { 3298 /+ + 3299 3300 +/ 3301 version(none) 3302 this(Uri baseUrl, string defaultResponseContentType) { 3303 3304 } 3305 3306 this() {} 3307 3308 HttpResponse defaultResponse; 3309 3310 /// Implementation of the ICache interface. Hijacks all requests to return a pre-populated response or "server disconnected". 3311 const(HttpResponse)* getCachedResponse(HttpRequestParameters request) { 3312 import std.conv; 3313 auto defaultPort = request.ssl ? 443 : 80; 3314 string identifier = text( 3315 request.method, " ", 3316 request.ssl ? "https" : "http", "://", 3317 request.host, 3318 (request.port && request.port != defaultPort) ? (":" ~ to!string(request.port)) : "", 3319 request.uri 3320 ); 3321 3322 if(auto res = identifier in population) 3323 return res; 3324 return &defaultResponse; 3325 } 3326 3327 /// Implementation of the ICache interface. We never actually cache anything here since it is all about mock responses, not actually caching real data. 3328 bool cacheResponse(HttpRequestParameters request, HttpResponse response) { 3329 return false; 3330 } 3331 3332 /++ 3333 Convenience method to populate simple responses. For more complex 3334 work, use one of the other overloads where you build complete objects 3335 yourself. 3336 3337 Params: 3338 request = a verb and complete URL to mock as one string. 3339 For example "GET http://example.com/". If you provide only 3340 a partial URL, it will be based on the `baseUrl` you gave 3341 in the `HttpMockProvider` constructor. 3342 3343 responseCode = the HTTP response code, like 200 or 404. 3344 3345 response = the response body as a string. It is assumed 3346 to be of the `defaultResponseContentType` you passed in the 3347 `HttpMockProvider` constructor. 3348 +/ 3349 void populate(string request, int responseCode, string response) { 3350 3351 // FIXME: absolute-ize the URL in the request 3352 3353 HttpResponse r; 3354 r.code = responseCode; 3355 r.codeText = getHttpCodeText(r.code); 3356 3357 r.content = cast(ubyte[]) response; 3358 r.contentText = response; 3359 3360 population[request] = r; 3361 } 3362 3363 version(none) 3364 void populate(string method, string url, HttpResponse response) { 3365 // FIXME 3366 } 3367 3368 private HttpResponse[string] population; 3369 } 3370 3371 // modified from the one in cgi.d to just have the text 3372 private static string getHttpCodeText(int code) pure nothrow @nogc { 3373 switch(code) { 3374 // this module's proprietary extensions 3375 case 0: return null; 3376 case 1: return "request.abort called"; 3377 case 2: return "connection failed"; 3378 case 3: return "server disconnected"; 3379 case 4: return "exception thrown"; // actually should be some other thing 3380 case 5: return "Request timed out"; 3381 3382 // * * * standard ones * * * 3383 3384 // 1xx skipped since they shouldn't happen 3385 3386 // 3387 case 200: return "OK"; 3388 case 201: return "Created"; 3389 case 202: return "Accepted"; 3390 case 203: return "Non-Authoritative Information"; 3391 case 204: return "No Content"; 3392 case 205: return "Reset Content"; 3393 // 3394 case 300: return "Multiple Choices"; 3395 case 301: return "Moved Permanently"; 3396 case 302: return "Found"; 3397 case 303: return "See Other"; 3398 case 307: return "Temporary Redirect"; 3399 case 308: return "Permanent Redirect"; 3400 // 3401 case 400: return "Bad Request"; 3402 case 403: return "Forbidden"; 3403 case 404: return "Not Found"; 3404 case 405: return "Method Not Allowed"; 3405 case 406: return "Not Acceptable"; 3406 case 409: return "Conflict"; 3407 case 410: return "Gone"; 3408 // 3409 case 500: return "Internal Server Error"; 3410 case 501: return "Not Implemented"; 3411 case 502: return "Bad Gateway"; 3412 case 503: return "Service Unavailable"; 3413 // 3414 default: assert(0, "Unsupported http code"); 3415 } 3416 } 3417 3418 3419 /// 3420 struct HttpCookie { 3421 string name; /// 3422 string value; /// 3423 string domain; /// 3424 string path; /// 3425 //SysTime expirationDate; /// 3426 bool secure; /// 3427 bool httpOnly; /// 3428 } 3429 3430 // FIXME: websocket 3431 3432 version(testing) 3433 void main() { 3434 import std.stdio; 3435 auto client = new HttpClient(); 3436 auto request = client.navigateTo(Uri("http://localhost/chunked.php")); 3437 request.send(); 3438 auto request2 = client.navigateTo(Uri("http://dlang.org/")); 3439 request2.send(); 3440 3441 { 3442 auto response = request2.waitForCompletion(); 3443 //write(cast(string) response.content); 3444 } 3445 3446 auto response = request.waitForCompletion(); 3447 write(cast(string) response.content); 3448 3449 writeln(HttpRequest.socketsPerHost); 3450 } 3451 3452 3453 // From sslsocket.d, but this is the maintained version! 3454 version(use_openssl) { 3455 alias SslClientSocket = OpenSslSocket; 3456 3457 // CRL = Certificate Revocation List 3458 static immutable string[] sslErrorCodes = [ 3459 "OK (code 0)", 3460 "Unspecified SSL/TLS error (code 1)", 3461 "Unable to get TLS issuer certificate (code 2)", 3462 "Unable to get TLS CRL (code 3)", 3463 "Unable to decrypt TLS certificate signature (code 4)", 3464 "Unable to decrypt TLS CRL signature (code 5)", 3465 "Unable to decode TLS issuer public key (code 6)", 3466 "TLS certificate signature failure (code 7)", 3467 "TLS CRL signature failure (code 8)", 3468 "TLS certificate not yet valid (code 9)", 3469 "TLS certificate expired (code 10)", 3470 "TLS CRL not yet valid (code 11)", 3471 "TLS CRL expired (code 12)", 3472 "TLS error in certificate not before field (code 13)", 3473 "TLS error in certificate not after field (code 14)", 3474 "TLS error in CRL last update field (code 15)", 3475 "TLS error in CRL next update field (code 16)", 3476 "TLS system out of memory (code 17)", 3477 "TLS certificate is self-signed (code 18)", 3478 "Self-signed certificate in TLS chain (code 19)", 3479 "Unable to get TLS issuer certificate locally (code 20)", 3480 "Unable to verify TLS leaf signature (code 21)", 3481 "TLS certificate chain too long (code 22)", 3482 "TLS certificate was revoked (code 23)", 3483 "TLS CA is invalid (code 24)", 3484 "TLS error: path length exceeded (code 25)", 3485 "TLS error: invalid purpose (code 26)", 3486 "TLS error: certificate untrusted (code 27)", 3487 "TLS error: certificate rejected (code 28)", 3488 ]; 3489 3490 string getOpenSslErrorCode(long error) { 3491 if(error == 62) 3492 return "TLS certificate host name mismatch"; 3493 3494 if(error < 0 || error >= sslErrorCodes.length) 3495 return "SSL/TLS error code " ~ to!string(error); 3496 return sslErrorCodes[cast(size_t) error]; 3497 } 3498 3499 struct SSL; 3500 struct SSL_CTX; 3501 struct SSL_METHOD; 3502 struct X509_STORE_CTX; 3503 enum SSL_VERIFY_NONE = 0; 3504 enum SSL_VERIFY_PEER = 1; 3505 3506 // copy it into the buf[0 .. size] and return actual length you read. 3507 // rwflag == 0 when reading, 1 when writing. 3508 extern(C) alias pem_password_cb = int function(char* buffer, int bufferSize, int rwflag, void* userPointer); 3509 extern(C) alias print_errors_cb = int function(const char*, size_t, void*); 3510 extern(C) alias client_cert_cb = int function(SSL *ssl, X509 **x509, EVP_PKEY **pkey); 3511 extern(C) alias keylog_cb = void function(SSL*, char*); 3512 3513 struct X509; 3514 struct X509_STORE; 3515 struct EVP_PKEY; 3516 struct X509_VERIFY_PARAM; 3517 3518 import core.stdc.config; 3519 3520 enum SSL_ERROR_WANT_READ = 2; 3521 enum SSL_ERROR_WANT_WRITE = 3; 3522 3523 struct ossllib { 3524 __gshared static extern(C) { 3525 /* these are only on older openssl versions { */ 3526 int function() SSL_library_init; 3527 void function() SSL_load_error_strings; 3528 SSL_METHOD* function() SSLv23_client_method; 3529 /* } */ 3530 3531 void function(ulong, void*) OPENSSL_init_ssl; 3532 3533 SSL_CTX* function(const SSL_METHOD*) SSL_CTX_new; 3534 SSL* function(SSL_CTX*) SSL_new; 3535 int function(SSL*, int) SSL_set_fd; 3536 int function(SSL*) SSL_connect; 3537 int function(SSL*, const void*, int) SSL_write; 3538 int function(SSL*, void*, int) SSL_read; 3539 @trusted nothrow @nogc int function(SSL*) SSL_shutdown; 3540 void function(SSL*) SSL_free; 3541 void function(SSL_CTX*) SSL_CTX_free; 3542 3543 int function(const SSL*) SSL_pending; 3544 int function (const SSL *ssl, int ret) SSL_get_error; 3545 3546 void function(SSL*, int, void*) SSL_set_verify; 3547 3548 void function(SSL*, int, c_long, void*) SSL_ctrl; 3549 3550 SSL_METHOD* function() SSLv3_client_method; 3551 SSL_METHOD* function() TLS_client_method; 3552 3553 void function(SSL_CTX*, void function(SSL*, char* line)) SSL_CTX_set_keylog_callback; 3554 3555 int function(SSL_CTX*) SSL_CTX_set_default_verify_paths; 3556 3557 X509_STORE* function(SSL_CTX*) SSL_CTX_get_cert_store; 3558 c_long function(const SSL* ssl) SSL_get_verify_result; 3559 3560 X509_VERIFY_PARAM* function(const SSL*) SSL_get0_param; 3561 3562 /+ 3563 SSL_CTX_load_verify_locations 3564 SSL_CTX_set_client_CA_list 3565 +/ 3566 3567 // client cert things 3568 void function (SSL_CTX *ctx, int function(SSL *ssl, X509 **x509, EVP_PKEY **pkey)) SSL_CTX_set_client_cert_cb; 3569 } 3570 } 3571 3572 struct eallib { 3573 __gshared static extern(C) { 3574 /* these are only on older openssl versions { */ 3575 void function() OpenSSL_add_all_ciphers; 3576 void function() OpenSSL_add_all_digests; 3577 /* } */ 3578 3579 const(char)* function(int) OpenSSL_version; 3580 3581 void function(ulong, void*) OPENSSL_init_crypto; 3582 3583 void function(print_errors_cb, void*) ERR_print_errors_cb; 3584 3585 void function(X509*) X509_free; 3586 int function(X509_STORE*, X509*) X509_STORE_add_cert; 3587 3588 3589 X509* function(FILE *fp, X509 **x, pem_password_cb *cb, void *u) PEM_read_X509; 3590 EVP_PKEY* function(FILE *fp, EVP_PKEY **x, pem_password_cb *cb, void* userPointer) PEM_read_PrivateKey; 3591 3592 EVP_PKEY* function(FILE *fp, EVP_PKEY **a) d2i_PrivateKey_fp; 3593 X509* function(FILE *fp, X509 **x) d2i_X509_fp; 3594 3595 X509* function(X509** a, const(ubyte*)* pp, c_long length) d2i_X509; 3596 int function(X509* a, ubyte** o) i2d_X509; 3597 3598 int function(X509_VERIFY_PARAM* a, const char* b, size_t l) X509_VERIFY_PARAM_set1_host; 3599 3600 X509* function(X509_STORE_CTX *ctx) X509_STORE_CTX_get_current_cert; 3601 int function(X509_STORE_CTX *ctx) X509_STORE_CTX_get_error; 3602 } 3603 } 3604 3605 struct OpenSSL { 3606 static: 3607 3608 template opDispatch(string name) { 3609 auto opDispatch(T...)(T t) { 3610 static if(__traits(hasMember, ossllib, name)) { 3611 auto ptr = __traits(getMember, ossllib, name); 3612 } else static if(__traits(hasMember, eallib, name)) { 3613 auto ptr = __traits(getMember, eallib, name); 3614 } else static assert(0); 3615 3616 if(ptr is null) 3617 throw new Exception(name ~ " not loaded"); 3618 return ptr(t); 3619 } 3620 } 3621 3622 // macros in the original C 3623 SSL_METHOD* SSLv23_client_method() { 3624 if(ossllib.SSLv23_client_method) 3625 return ossllib.SSLv23_client_method(); 3626 else 3627 return ossllib.TLS_client_method(); 3628 } 3629 3630 void SSL_set_tlsext_host_name(SSL* a, const char* b) { 3631 if(ossllib.SSL_ctrl) 3632 return ossllib.SSL_ctrl(a, 55 /*SSL_CTRL_SET_TLSEXT_HOSTNAME*/, 0 /*TLSEXT_NAMETYPE_host_name*/, cast(void*) b); 3633 else throw new Exception("SSL_set_tlsext_host_name not loaded"); 3634 } 3635 3636 // special case 3637 @trusted nothrow @nogc int SSL_shutdown(SSL* a) { 3638 if(ossllib.SSL_shutdown) 3639 return ossllib.SSL_shutdown(a); 3640 assert(0); 3641 } 3642 3643 void SSL_CTX_keylog_cb_func(SSL_CTX* ctx, keylog_cb func) { 3644 // this isn't in openssl 1.0 and is non-essential, so it is allowed to fail. 3645 if(ossllib.SSL_CTX_set_keylog_callback) 3646 ossllib.SSL_CTX_set_keylog_callback(ctx, func); 3647 //else throw new Exception("SSL_CTX_keylog_cb_func not loaded"); 3648 } 3649 3650 } 3651 3652 extern(C) 3653 int collectSslErrors(const char* ptr, size_t len, void* user) @trusted { 3654 string* s = cast(string*) user; 3655 3656 (*s) ~= ptr[0 .. len]; 3657 3658 return 0; 3659 } 3660 3661 3662 private __gshared void* ossllib_handle; 3663 version(Windows) 3664 private __gshared void* oeaylib_handle; 3665 else 3666 alias oeaylib_handle = ossllib_handle; 3667 version(Posix) 3668 private import core.sys.posix.dlfcn; 3669 else version(Windows) 3670 private import core.sys.windows.windows; 3671 3672 import core.stdc.stdio; 3673 3674 private __gshared Object loadSslMutex = new Object; 3675 private __gshared bool sslLoaded = false; 3676 3677 void loadOpenSsl() { 3678 if(sslLoaded) 3679 return; 3680 synchronized(loadSslMutex) { 3681 3682 version(Posix) { 3683 version(OSX) { 3684 static immutable string[] ossllibs = [ 3685 "libssl.46.dylib", 3686 "libssl.44.dylib", 3687 "libssl.43.dylib", 3688 "libssl.35.dylib", 3689 "libssl.1.1.dylib", 3690 "libssl.dylib", 3691 "/usr/local/opt/openssl/lib/libssl.1.0.0.dylib", 3692 ]; 3693 } else { 3694 static immutable string[] ossllibs = [ 3695 "libssl.so.3", 3696 "libssl.so.1.1", 3697 "libssl.so.1.0.2", 3698 "libssl.so.1.0.1", 3699 "libssl.so.1.0.0", 3700 "libssl.so", 3701 ]; 3702 } 3703 3704 foreach(lib; ossllibs) { 3705 ossllib_handle = dlopen(lib.ptr, RTLD_NOW); 3706 if(ossllib_handle !is null) break; 3707 } 3708 } else version(Windows) { 3709 version(X86_64) { 3710 ossllib_handle = LoadLibraryW("libssl-1_1-x64.dll"w.ptr); 3711 oeaylib_handle = LoadLibraryW("libcrypto-1_1-x64.dll"w.ptr); 3712 } 3713 3714 static immutable wstring[] ossllibs = [ 3715 "libssl-3-x64.dll"w, 3716 "libssl-3.dll"w, 3717 "libssl-1_1.dll"w, 3718 "libssl32.dll"w, 3719 ]; 3720 3721 if(ossllib_handle is null) 3722 foreach(lib; ossllibs) { 3723 ossllib_handle = LoadLibraryW(lib.ptr); 3724 if(ossllib_handle !is null) break; 3725 } 3726 3727 static immutable wstring[] eaylibs = [ 3728 "libcrypto-3-x64.dll"w, 3729 "libcrypto-3.dll"w, 3730 "libcrypto-1_1.dll"w, 3731 "libeay32.dll", 3732 ]; 3733 3734 if(oeaylib_handle is null) 3735 foreach(lib; eaylibs) { 3736 oeaylib_handle = LoadLibraryW(lib.ptr); 3737 if (oeaylib_handle !is null) break; 3738 } 3739 3740 if(ossllib_handle is null) { 3741 ossllib_handle = LoadLibraryW("ssleay32.dll"w.ptr); 3742 oeaylib_handle = ossllib_handle; 3743 } 3744 } 3745 3746 if(ossllib_handle is null) 3747 throw new Exception("libssl library not found"); 3748 if(oeaylib_handle is null) 3749 throw new Exception("libeay32 library not found"); 3750 3751 foreach(memberName; __traits(allMembers, ossllib)) { 3752 alias t = typeof(__traits(getMember, ossllib, memberName)); 3753 version(Posix) 3754 __traits(getMember, ossllib, memberName) = cast(t) dlsym(ossllib_handle, memberName); 3755 else version(Windows) { 3756 __traits(getMember, ossllib, memberName) = cast(t) GetProcAddress(ossllib_handle, memberName); 3757 } 3758 } 3759 3760 foreach(memberName; __traits(allMembers, eallib)) { 3761 alias t = typeof(__traits(getMember, eallib, memberName)); 3762 version(Posix) 3763 __traits(getMember, eallib, memberName) = cast(t) dlsym(oeaylib_handle, memberName); 3764 else version(Windows) { 3765 __traits(getMember, eallib, memberName) = cast(t) GetProcAddress(oeaylib_handle, memberName); 3766 } 3767 } 3768 3769 3770 if(ossllib.SSL_library_init) 3771 ossllib.SSL_library_init(); 3772 else if(ossllib.OPENSSL_init_ssl) 3773 ossllib.OPENSSL_init_ssl(0, null); 3774 else throw new Exception("couldn't init openssl"); 3775 3776 if(eallib.OpenSSL_add_all_ciphers) { 3777 eallib.OpenSSL_add_all_ciphers(); 3778 if(eallib.OpenSSL_add_all_digests is null) 3779 throw new Exception("no add digests"); 3780 eallib.OpenSSL_add_all_digests(); 3781 } else if(eallib.OPENSSL_init_crypto) 3782 eallib.OPENSSL_init_crypto(0 /*OPENSSL_INIT_ADD_ALL_CIPHERS and ALL_DIGESTS together*/, null); 3783 else throw new Exception("couldn't init crypto openssl"); 3784 3785 if(ossllib.SSL_load_error_strings) 3786 ossllib.SSL_load_error_strings(); 3787 else if(ossllib.OPENSSL_init_ssl) 3788 ossllib.OPENSSL_init_ssl(0x00200000L, null); 3789 else throw new Exception("couldn't load openssl errors"); 3790 3791 sslLoaded = true; 3792 } 3793 } 3794 3795 /+ 3796 // I'm just gonna let the OS clean this up on process termination because otherwise SSL_free 3797 // might have trouble being run from the GC after this module is unloaded. 3798 shared static ~this() { 3799 if(ossllib_handle) { 3800 version(Windows) { 3801 FreeLibrary(oeaylib_handle); 3802 FreeLibrary(ossllib_handle); 3803 } else version(Posix) 3804 dlclose(ossllib_handle); 3805 ossllib_handle = null; 3806 } 3807 ossllib.tupleof = ossllib.tupleof.init; 3808 } 3809 +/ 3810 3811 //pragma(lib, "crypto"); 3812 //pragma(lib, "ssl"); 3813 extern(C) 3814 void write_to_file(SSL* ssl, char* line) 3815 { 3816 import std.stdio; 3817 import std.string; 3818 import std.process : environment; 3819 string logfile = environment.get("SSLKEYLOGFILE"); 3820 if (logfile !is null) 3821 { 3822 auto f = std.stdio.File(logfile, "a+"); 3823 f.writeln(fromStringz(line)); 3824 f.close(); 3825 } 3826 } 3827 3828 class OpenSslSocket : Socket { 3829 private SSL* ssl; 3830 private SSL_CTX* ctx; 3831 private void initSsl(bool verifyPeer, string hostname) { 3832 ctx = OpenSSL.SSL_CTX_new(OpenSSL.SSLv23_client_method()); 3833 assert(ctx !is null); 3834 3835 debug OpenSSL.SSL_CTX_keylog_cb_func(ctx, &write_to_file); 3836 ssl = OpenSSL.SSL_new(ctx); 3837 3838 if(hostname.length) { 3839 OpenSSL.SSL_set_tlsext_host_name(ssl, toStringz(hostname)); 3840 if(verifyPeer) 3841 OpenSSL.X509_VERIFY_PARAM_set1_host(OpenSSL.SSL_get0_param(ssl), hostname.ptr, hostname.length); 3842 } 3843 3844 if(verifyPeer) { 3845 OpenSSL.SSL_CTX_set_default_verify_paths(ctx); 3846 3847 version(Windows) { 3848 loadCertificatesFromRegistry(ctx); 3849 } 3850 3851 OpenSSL.SSL_set_verify(ssl, SSL_VERIFY_PEER, &verifyCertificateFromRegistryArsdHttp); 3852 } else 3853 OpenSSL.SSL_set_verify(ssl, SSL_VERIFY_NONE, null); 3854 3855 OpenSSL.SSL_set_fd(ssl, cast(int) this.handle); // on win64 it is necessary to truncate, but the value is never large anyway see http://openssl.6102.n7.nabble.com/Sockets-windows-64-bit-td36169.html 3856 3857 3858 OpenSSL.SSL_CTX_set_client_cert_cb(ctx, &cb); 3859 } 3860 3861 extern(C) 3862 static int cb(SSL* ssl, X509** x509, EVP_PKEY** pkey) { 3863 if(HttpClient.certFilename.length && HttpClient.keyFilename.length) { 3864 FILE* fpCert = fopen((HttpClient.certFilename ~ "\0").ptr, "rb"); 3865 if(fpCert is null) 3866 return 0; 3867 scope(exit) 3868 fclose(fpCert); 3869 FILE* fpKey = fopen((HttpClient.keyFilename ~ "\0").ptr, "rb"); 3870 if(fpKey is null) 3871 return 0; 3872 scope(exit) 3873 fclose(fpKey); 3874 3875 with(CertificateFileFormat) 3876 final switch(HttpClient.certFormat) { 3877 case guess: 3878 if(HttpClient.certFilename.endsWith(".pem") || HttpClient.keyFilename.endsWith(".pem")) 3879 goto case pem; 3880 else 3881 goto case der; 3882 case pem: 3883 *x509 = OpenSSL.PEM_read_X509(fpCert, null, null, null); 3884 *pkey = OpenSSL.PEM_read_PrivateKey(fpKey, null, null, null); 3885 break; 3886 case der: 3887 *x509 = OpenSSL.d2i_X509_fp(fpCert, null); 3888 *pkey = OpenSSL.d2i_PrivateKey_fp(fpKey, null); 3889 break; 3890 } 3891 3892 return 1; 3893 } 3894 3895 return 0; 3896 } 3897 3898 bool dataPending() { 3899 return OpenSSL.SSL_pending(ssl) > 0; 3900 } 3901 3902 @trusted 3903 override void connect(Address to) { 3904 super.connect(to); 3905 if(blocking) { 3906 do_ssl_connect(); 3907 } 3908 } 3909 3910 @trusted 3911 // returns true if it is finished, false if it would have blocked, throws if there's an error 3912 int do_ssl_connect() { 3913 if(OpenSSL.SSL_connect(ssl) == -1) { 3914 3915 auto errCode = OpenSSL.SSL_get_error(ssl, -1); 3916 if(errCode == SSL_ERROR_WANT_READ || errCode == SSL_ERROR_WANT_WRITE) { 3917 return errCode; 3918 } 3919 3920 string str; 3921 OpenSSL.ERR_print_errors_cb(&collectSslErrors, &str); 3922 int i; 3923 auto err = OpenSSL.SSL_get_verify_result(ssl); 3924 //printf("wtf\n"); 3925 //scanf("%d\n", i); 3926 throw new Exception("Secure connect failed: " ~ getOpenSslErrorCode(err)); 3927 } 3928 3929 return 0; 3930 } 3931 3932 @trusted 3933 override ptrdiff_t send(scope const(void)[] buf, SocketFlags flags) { 3934 //import std.stdio;writeln(cast(string) buf); 3935 debug(arsd_http2_verbose) writeln("ssl writing ", buf.length); 3936 auto retval = OpenSSL.SSL_write(ssl, buf.ptr, cast(uint) buf.length); 3937 3938 // don't need to throw anymore since it is checked elsewhere 3939 // code useful sometimes for debugging hence commenting instead of deleting 3940 version(none) 3941 if(retval == -1) { 3942 3943 string str; 3944 OpenSSL.ERR_print_errors_cb(&collectSslErrors, &str); 3945 int i; 3946 3947 //printf("wtf\n"); 3948 //scanf("%d\n", i); 3949 3950 throw new Exception("ssl send failed " ~ str); 3951 } 3952 return retval; 3953 3954 } 3955 override ptrdiff_t send(scope const(void)[] buf) { 3956 return send(buf, SocketFlags.NONE); 3957 } 3958 @trusted 3959 override ptrdiff_t receive(scope void[] buf, SocketFlags flags) { 3960 3961 debug(arsd_http2_verbose) writeln("ssl_read before"); 3962 auto retval = OpenSSL.SSL_read(ssl, buf.ptr, cast(int)buf.length); 3963 debug(arsd_http2_verbose) writeln("ssl_read after"); 3964 3965 // don't need to throw anymore since it is checked elsewhere 3966 // code useful sometimes for debugging hence commenting instead of deleting 3967 version(none) 3968 if(retval == -1) { 3969 3970 string str; 3971 OpenSSL.ERR_print_errors_cb(&collectSslErrors, &str); 3972 int i; 3973 3974 //printf("wtf\n"); 3975 //scanf("%d\n", i); 3976 3977 throw new Exception("ssl receive failed " ~ str); 3978 } 3979 return retval; 3980 } 3981 override ptrdiff_t receive(scope void[] buf) { 3982 return receive(buf, SocketFlags.NONE); 3983 } 3984 3985 this(AddressFamily af, SocketType type = SocketType.STREAM, string hostname = null, bool verifyPeer = true) { 3986 version(Windows) __traits(getMember, this, "_blocking") = true; // lol longstanding phobos bug setting this to false on init 3987 super(af, type); 3988 initSsl(verifyPeer, hostname); 3989 } 3990 3991 override void close() scope { 3992 if(ssl) OpenSSL.SSL_shutdown(ssl); 3993 super.close(); 3994 } 3995 3996 this(socket_t sock, AddressFamily af, string hostname, bool verifyPeer = true) { 3997 super(sock, af); 3998 initSsl(verifyPeer, hostname); 3999 } 4000 4001 void freeSsl() { 4002 if(ssl is null) 4003 return; 4004 OpenSSL.SSL_free(ssl); 4005 OpenSSL.SSL_CTX_free(ctx); 4006 ssl = null; 4007 } 4008 4009 ~this() { 4010 freeSsl(); 4011 } 4012 } 4013 } 4014 4015 4016 /++ 4017 An experimental component for working with REST apis. Note that it 4018 is a zero-argument template, so to create one, use `new HttpApiClient!()(args..)` 4019 or you will get "HttpApiClient is used as a type" compile errors. 4020 4021 This will probably not work for you yet, and I might change it significantly. 4022 4023 Requires [arsd.jsvar]. 4024 4025 4026 Here's a snippet to create a pull request on GitHub to Phobos: 4027 4028 --- 4029 auto github = new HttpApiClient!()("https://api.github.com/", "your personal api token here"); 4030 4031 // create the arguments object 4032 // see: https://developer.github.com/v3/pulls/#create-a-pull-request 4033 var args = var.emptyObject; 4034 args.title = "My Pull Request"; 4035 args.head = "yourusername:" ~ branchName; 4036 args.base = "master"; 4037 // note it is ["body"] instead of .body because `body` is a D keyword 4038 args["body"] = "My cool PR is opened by the API!"; 4039 args.maintainer_can_modify = true; 4040 4041 /+ 4042 Fun fact, you can also write that: 4043 4044 var args = [ 4045 "title": "My Pull Request".var, 4046 "head": "yourusername:" ~ branchName.var, 4047 "base" : "master".var, 4048 "body" : "My cool PR is opened by the API!".var, 4049 "maintainer_can_modify": true.var 4050 ]; 4051 4052 Note the .var constructor calls in there. If everything is the same type, you actually don't need that, but here since there's strings and bools, D won't allow the literal without explicit constructors to align them all. 4053 +/ 4054 4055 // this translates to `repos/dlang/phobos/pulls` and sends a POST request, 4056 // containing `args` as json, then immediately grabs the json result and extracts 4057 // the value `html_url` from it. `prUrl` is typed `var`, from arsd.jsvar. 4058 auto prUrl = github.rest.repos.dlang.phobos.pulls.POST(args).result.html_url; 4059 4060 writeln("Created: ", prUrl); 4061 --- 4062 4063 Why use this instead of just building the URL? Well, of course you can! This just makes 4064 it a bit more convenient than string concatenation and manages a few headers for you. 4065 4066 Subtypes could potentially add static type checks too. 4067 +/ 4068 class HttpApiClient() { 4069 import arsd.jsvar; 4070 4071 HttpClient httpClient; 4072 4073 alias HttpApiClientType = typeof(this); 4074 4075 string urlBase; 4076 string oauth2Token; 4077 string submittedContentType; 4078 4079 /++ 4080 Params: 4081 4082 urlBase = The base url for the api. Tends to be something like `https://api.example.com/v2/` or similar. 4083 oauth2Token = the authorization token for the service. You'll have to get it from somewhere else. 4084 submittedContentType = the content-type of POST, PUT, etc. bodies. 4085 httpClient = an injected http client, or null if you want to use a default-constructed one 4086 4087 History: 4088 The `httpClient` param was added on December 26, 2020. 4089 +/ 4090 this(string urlBase, string oauth2Token, string submittedContentType = "application/json", HttpClient httpClient = null) { 4091 if(httpClient is null) 4092 this.httpClient = new HttpClient(); 4093 else 4094 this.httpClient = httpClient; 4095 4096 assert(urlBase[0] == 'h'); 4097 assert(urlBase[$-1] == '/'); 4098 4099 this.urlBase = urlBase; 4100 this.oauth2Token = oauth2Token; 4101 this.submittedContentType = submittedContentType; 4102 } 4103 4104 /// 4105 static struct HttpRequestWrapper { 4106 HttpApiClientType apiClient; /// 4107 HttpRequest request; /// 4108 HttpResponse _response; 4109 4110 /// 4111 this(HttpApiClientType apiClient, HttpRequest request) { 4112 this.apiClient = apiClient; 4113 this.request = request; 4114 } 4115 4116 /// Returns the full [HttpResponse] object so you can inspect the headers 4117 @property HttpResponse response() { 4118 if(_response is HttpResponse.init) 4119 _response = request.waitForCompletion(); 4120 return _response; 4121 } 4122 4123 /++ 4124 Returns the parsed JSON from the body of the response. 4125 4126 Throws on non-2xx responses. 4127 +/ 4128 var result() { 4129 return apiClient.throwOnError(response); 4130 } 4131 4132 alias request this; 4133 } 4134 4135 /// 4136 HttpRequestWrapper request(string uri, HttpVerb requestMethod = HttpVerb.GET, ubyte[] bodyBytes = null) { 4137 if(uri[0] == '/') 4138 uri = uri[1 .. $]; 4139 4140 auto u = Uri(uri).basedOn(Uri(urlBase)); 4141 4142 auto req = httpClient.navigateTo(u, requestMethod); 4143 4144 if(oauth2Token.length) 4145 req.requestParameters.headers ~= "Authorization: Bearer " ~ oauth2Token; 4146 req.requestParameters.contentType = submittedContentType; 4147 req.requestParameters.bodyData = bodyBytes; 4148 4149 return HttpRequestWrapper(this, req); 4150 } 4151 4152 /// 4153 var throwOnError(HttpResponse res) { 4154 if(res.code < 200 || res.code >= 300) 4155 throw new Exception(res.codeText ~ " " ~ res.contentText); 4156 4157 var response = var.fromJson(res.contentText); 4158 if(response.errors) { 4159 throw new Exception(response.errors.toJson()); 4160 } 4161 4162 return response; 4163 } 4164 4165 /// 4166 @property RestBuilder rest() { 4167 return RestBuilder(this, null, null); 4168 } 4169 4170 // hipchat.rest.room["Tech Team"].history 4171 // gives: "/room/Tech%20Team/history" 4172 // 4173 // hipchat.rest.room["Tech Team"].history("page", "12) 4174 /// 4175 static struct RestBuilder { 4176 HttpApiClientType apiClient; 4177 string[] pathParts; 4178 string[2][] queryParts; 4179 this(HttpApiClientType apiClient, string[] pathParts, string[2][] queryParts) { 4180 this.apiClient = apiClient; 4181 this.pathParts = pathParts; 4182 this.queryParts = queryParts; 4183 } 4184 4185 RestBuilder _SELF() { 4186 return this; 4187 } 4188 4189 /// The args are so you can call opCall on the returned 4190 /// object, despite @property being broken af in D. 4191 RestBuilder opDispatch(string str, T)(string n, T v) { 4192 return RestBuilder(apiClient, pathParts ~ str, queryParts ~ [n, to!string(v)]); 4193 } 4194 4195 /// 4196 RestBuilder opDispatch(string str)() { 4197 return RestBuilder(apiClient, pathParts ~ str, queryParts); 4198 } 4199 4200 4201 /// 4202 RestBuilder opIndex(string str) { 4203 return RestBuilder(apiClient, pathParts ~ str, queryParts); 4204 } 4205 /// 4206 RestBuilder opIndex(var str) { 4207 return RestBuilder(apiClient, pathParts ~ str.get!string, queryParts); 4208 } 4209 /// 4210 RestBuilder opIndex(int i) { 4211 return RestBuilder(apiClient, pathParts ~ to!string(i), queryParts); 4212 } 4213 4214 /// 4215 RestBuilder opCall(T)(string name, T value) { 4216 return RestBuilder(apiClient, pathParts, queryParts ~ [name, to!string(value)]); 4217 } 4218 4219 /// 4220 string toUri() { 4221 import std.uri; 4222 string result; 4223 foreach(idx, part; pathParts) { 4224 if(idx) 4225 result ~= "/"; 4226 result ~= encodeComponent(part); 4227 } 4228 result ~= "?"; 4229 foreach(idx, part; queryParts) { 4230 if(idx) 4231 result ~= "&"; 4232 result ~= encodeComponent(part[0]); 4233 result ~= "="; 4234 result ~= encodeComponent(part[1]); 4235 } 4236 4237 return result; 4238 } 4239 4240 /// 4241 final HttpRequestWrapper GET() { return _EXECUTE(HttpVerb.GET, this.toUri(), ToBytesResult.init); } 4242 /// ditto 4243 final HttpRequestWrapper DELETE() { return _EXECUTE(HttpVerb.DELETE, this.toUri(), ToBytesResult.init); } 4244 4245 // need to be able to send: JSON, urlencoded, multipart/form-data, and raw stuff. 4246 /// ditto 4247 final HttpRequestWrapper POST(T...)(T t) { return _EXECUTE(HttpVerb.POST, this.toUri(), toBytes(t)); } 4248 /// ditto 4249 final HttpRequestWrapper PATCH(T...)(T t) { return _EXECUTE(HttpVerb.PATCH, this.toUri(), toBytes(t)); } 4250 /// ditto 4251 final HttpRequestWrapper PUT(T...)(T t) { return _EXECUTE(HttpVerb.PUT, this.toUri(), toBytes(t)); } 4252 4253 struct ToBytesResult { 4254 ubyte[] bytes; 4255 string contentType; 4256 } 4257 4258 private ToBytesResult toBytes(T...)(T t) { 4259 import std.conv : to; 4260 static if(T.length == 0) 4261 return ToBytesResult(null, null); 4262 else static if(T.length == 1 && is(T[0] == var)) 4263 return ToBytesResult(cast(ubyte[]) t[0].toJson(), "application/json"); // json data 4264 else static if(T.length == 1 && (is(T[0] == string) || is(T[0] == ubyte[]))) 4265 return ToBytesResult(cast(ubyte[]) t[0], null); // raw data 4266 else static if(T.length == 1 && is(T[0] : FormData)) 4267 return ToBytesResult(t[0].toBytes, t[0].contentType); 4268 else static if(T.length > 1 && T.length % 2 == 0 && is(T[0] == string)) { 4269 // string -> value pairs for a POST request 4270 string answer; 4271 foreach(idx, val; t) { 4272 static if(idx % 2 == 0) { 4273 if(answer.length) 4274 answer ~= "&"; 4275 answer ~= encodeComponent(val); // it had better be a string! lol 4276 answer ~= "="; 4277 } else { 4278 answer ~= encodeComponent(to!string(val)); 4279 } 4280 } 4281 4282 return ToBytesResult(cast(ubyte[]) answer, "application/x-www-form-urlencoded"); 4283 } 4284 else 4285 static assert(0); // FIXME 4286 4287 } 4288 4289 HttpRequestWrapper _EXECUTE(HttpVerb verb, string uri, ubyte[] bodyBytes) { 4290 return apiClient.request(uri, verb, bodyBytes); 4291 } 4292 4293 HttpRequestWrapper _EXECUTE(HttpVerb verb, string uri, ToBytesResult tbr) { 4294 auto r = apiClient.request(uri, verb, tbr.bytes); 4295 if(tbr.contentType !is null) 4296 r.requestParameters.contentType = tbr.contentType; 4297 return r; 4298 } 4299 } 4300 } 4301 4302 4303 // see also: arsd.cgi.encodeVariables 4304 /// Creates a multipart/form-data object that is suitable for file uploads and other kinds of POST 4305 class FormData { 4306 struct MimePart { 4307 string name; 4308 const(void)[] data; 4309 string contentType; 4310 string filename; 4311 } 4312 4313 MimePart[] parts; 4314 4315 /// 4316 void append(string key, in void[] value, string contentType = null, string filename = null) { 4317 parts ~= MimePart(key, value, contentType, filename); 4318 } 4319 4320 private string boundary = "0016e64be86203dd36047610926a"; // FIXME 4321 4322 string contentType() { 4323 return "multipart/form-data; boundary=" ~ boundary; 4324 } 4325 4326 /// 4327 ubyte[] toBytes() { 4328 string data; 4329 4330 foreach(part; parts) { 4331 data ~= "--" ~ boundary ~ "\r\n"; 4332 data ~= "Content-Disposition: form-data; name=\""~part.name~"\""; 4333 if(part.filename !is null) 4334 data ~= "; filename=\""~part.filename~"\""; 4335 data ~= "\r\n"; 4336 if(part.contentType !is null) 4337 data ~= "Content-Type: " ~ part.contentType ~ "\r\n"; 4338 data ~= "\r\n"; 4339 4340 data ~= cast(string) part.data; 4341 4342 data ~= "\r\n"; 4343 } 4344 4345 data ~= "--" ~ boundary ~ "--\r\n"; 4346 4347 return cast(ubyte[]) data; 4348 } 4349 } 4350 4351 private bool bicmp(in ubyte[] item, in char[] search) { 4352 if(item.length != search.length) return false; 4353 4354 foreach(i; 0 .. item.length) { 4355 ubyte a = item[i]; 4356 ubyte b = search[i]; 4357 if(a >= 'A' && a <= 'Z') 4358 a += 32; 4359 //if(b >= 'A' && b <= 'Z') 4360 //b += 32; 4361 if(a != b) 4362 return false; 4363 } 4364 4365 return true; 4366 } 4367 4368 /++ 4369 WebSocket client, based on the browser api, though also with other api options. 4370 4371 --- 4372 import arsd.http2; 4373 4374 void main() { 4375 auto ws = new WebSocket(Uri("ws://....")); 4376 4377 ws.onmessage = (in char[] msg) { 4378 ws.send("a reply"); 4379 }; 4380 4381 ws.connect(); 4382 4383 WebSocket.eventLoop(); 4384 } 4385 --- 4386 4387 Symbol_groups: 4388 foundational = 4389 Used with all API styles. 4390 4391 browser_api = 4392 API based on the standard in the browser. 4393 4394 event_loop_integration = 4395 Integrating with external event loops is done through static functions. You should 4396 call these BEFORE doing anything else with the WebSocket module or class. 4397 4398 $(PITFALL NOT IMPLEMENTED) 4399 --- 4400 WebSocket.setEventLoopProxy(arsd.simpledisplay.EventLoop.proxy.tupleof); 4401 // or something like that. it is not implemented yet. 4402 --- 4403 $(PITFALL NOT IMPLEMENTED) 4404 4405 blocking_api = 4406 The blocking API is best used when you only need basic functionality with a single connection. 4407 4408 --- 4409 WebSocketFrame msg; 4410 do { 4411 // FIXME good demo 4412 } while(msg); 4413 --- 4414 4415 Or to check for blocks before calling: 4416 4417 --- 4418 try_to_process_more: 4419 while(ws.isMessageBuffered()) { 4420 auto msg = ws.waitForNextMessage(); 4421 // process msg 4422 } 4423 if(ws.isDataPending()) { 4424 ws.lowLevelReceive(); 4425 goto try_to_process_more; 4426 } else { 4427 // nothing ready, you can do other things 4428 // or at least sleep a while before trying 4429 // to process more. 4430 if(ws.readyState == WebSocket.OPEN) { 4431 Thread.sleep(1.seconds); 4432 goto try_to_process_more; 4433 } 4434 } 4435 --- 4436 4437 +/ 4438 class WebSocket { 4439 private Uri uri; 4440 private string[string] cookies; 4441 4442 private string host; 4443 private ushort port; 4444 private bool ssl; 4445 4446 // used to decide if we mask outgoing msgs 4447 private bool isClient; 4448 4449 private MonoTime timeoutFromInactivity; 4450 private MonoTime nextPing; 4451 4452 /++ 4453 wss://echo.websocket.org 4454 +/ 4455 /// Group: foundational 4456 this(Uri uri, Config config = Config.init) 4457 //in (uri.scheme == "ws" || uri.scheme == "wss") 4458 in { assert(uri.scheme == "ws" || uri.scheme == "wss"); } do 4459 { 4460 this.uri = uri; 4461 this.config = config; 4462 4463 this.receiveBuffer = new ubyte[](config.initialReceiveBufferSize); 4464 4465 host = uri.host; 4466 ssl = uri.scheme == "wss"; 4467 port = cast(ushort) (uri.port ? uri.port : ssl ? 443 : 80); 4468 4469 if(ssl) { 4470 version(with_openssl) { 4471 loadOpenSsl(); 4472 socket = new SslClientSocket(family(uri.unixSocketPath), SocketType.STREAM, host, config.verifyPeer); 4473 } else 4474 throw new Exception("SSL not compiled in"); 4475 } else 4476 socket = new Socket(family(uri.unixSocketPath), SocketType.STREAM); 4477 4478 socket.setOption(SocketOptionLevel.TCP, SocketOption.TCP_NODELAY, 1); 4479 } 4480 4481 /++ 4482 4483 +/ 4484 /// Group: foundational 4485 void connect() { 4486 this.isClient = true; 4487 4488 socket.blocking = false; 4489 4490 if(uri.unixSocketPath) 4491 socket.connect(new UnixAddress(uri.unixSocketPath)); 4492 else 4493 socket.connect(new InternetAddress(host, port)); // FIXME: ipv6 support... 4494 4495 4496 auto readSet = new SocketSet(); 4497 auto writeSet = new SocketSet(); 4498 4499 readSet.reset(); 4500 writeSet.reset(); 4501 4502 readSet.add(socket); 4503 writeSet.add(socket); 4504 4505 auto selectGot = Socket.select(readSet, writeSet, null, config.timeoutFromInactivity); 4506 if(selectGot == -1) { 4507 // interrupted 4508 4509 throw new Exception("Websocket connection interrupted - retry might succeed"); 4510 } else if(selectGot == 0) { 4511 // time out 4512 socket.close(); 4513 throw new Exception("Websocket connection timed out"); 4514 } else { 4515 if(writeSet.isSet(socket) || readSet.isSet(socket)) { 4516 import core.stdc.stdint; 4517 int32_t error; 4518 int retopt = socket.getOption(SocketOptionLevel.SOCKET, SocketOption.ERROR, error); 4519 if(retopt < 0 || error != 0) { 4520 socket.close(); 4521 throw new Exception("Websocket connection failed - " ~ formatSocketError(error)); 4522 } else { 4523 // FIXME: websocket handshake could and really should be async too. 4524 socket.blocking = true; // just convenience 4525 if(auto s = cast(SslClientSocket) socket) { 4526 s.do_ssl_connect(); 4527 } else { 4528 // we're ready 4529 } 4530 } 4531 } 4532 } 4533 4534 auto uri = this.uri.path.length ? this.uri.path : "/"; 4535 if(this.uri.query.length) { 4536 uri ~= "?"; 4537 uri ~= this.uri.query; 4538 } 4539 4540 // the headers really shouldn't be bigger than this, at least 4541 // the chunks i need to process 4542 ubyte[4096] bufferBacking = void; 4543 ubyte[] buffer = bufferBacking[]; 4544 size_t pos; 4545 4546 void append(in char[][] items...) { 4547 foreach(what; items) { 4548 if((pos + what.length) > buffer.length) { 4549 buffer.length += 4096; 4550 } 4551 buffer[pos .. pos + what.length] = cast(ubyte[]) what[]; 4552 pos += what.length; 4553 } 4554 } 4555 4556 append("GET ", uri, " HTTP/1.1\r\n"); 4557 append("Host: ", this.uri.host, "\r\n"); 4558 4559 append("Upgrade: websocket\r\n"); 4560 append("Connection: Upgrade\r\n"); 4561 append("Sec-WebSocket-Version: 13\r\n"); 4562 4563 // FIXME: randomize this 4564 append("Sec-WebSocket-Key: x3JEHMbDL1EzLkh9GBhXDw==\r\n"); 4565 4566 if(config.protocol.length) 4567 append("Sec-WebSocket-Protocol: ", config.protocol, "\r\n"); 4568 if(config.origin.length) 4569 append("Origin: ", config.origin, "\r\n"); 4570 4571 foreach(h; config.additionalHeaders) { 4572 append(h); 4573 append("\r\n"); 4574 } 4575 4576 append("\r\n"); 4577 4578 auto remaining = buffer[0 .. pos]; 4579 //import std.stdio; writeln(host, " " , port, " ", cast(string) remaining); 4580 while(remaining.length) { 4581 auto r = socket.send(remaining); 4582 if(r < 0) 4583 throw new Exception(lastSocketError()); 4584 if(r == 0) 4585 throw new Exception("unexpected connection termination"); 4586 remaining = remaining[r .. $]; 4587 } 4588 4589 // the response shouldn't be especially large at this point, just 4590 // headers for the most part. gonna try to get it in the stack buffer. 4591 // then copy stuff after headers, if any, to the frame buffer. 4592 ubyte[] used; 4593 4594 void more() { 4595 auto r = socket.receive(buffer[used.length .. $]); 4596 4597 if(r < 0) 4598 throw new Exception(lastSocketError()); 4599 if(r == 0) 4600 throw new Exception("unexpected connection termination"); 4601 //import std.stdio;writef("%s", cast(string) buffer[used.length .. used.length + r]); 4602 4603 used = buffer[0 .. used.length + r]; 4604 } 4605 4606 more(); 4607 4608 import std.algorithm; 4609 if(!used.startsWith(cast(ubyte[]) "HTTP/1.1 101")) 4610 throw new Exception("didn't get a websocket answer"); 4611 // skip the status line 4612 while(used.length && used[0] != '\n') 4613 used = used[1 .. $]; 4614 4615 if(used.length == 0) 4616 throw new Exception("Remote server disconnected or didn't send enough information"); 4617 4618 if(used.length < 1) 4619 more(); 4620 4621 used = used[1 .. $]; // skip the \n 4622 4623 if(used.length == 0) 4624 more(); 4625 4626 // checks on the protocol from ehaders 4627 bool isWebsocket; 4628 bool isUpgrade; 4629 const(ubyte)[] protocol; 4630 const(ubyte)[] accept; 4631 4632 while(used.length) { 4633 if(used.length >= 2 && used[0] == '\r' && used[1] == '\n') { 4634 used = used[2 .. $]; 4635 break; // all done 4636 } 4637 int idxColon; 4638 while(idxColon < used.length && used[idxColon] != ':') 4639 idxColon++; 4640 if(idxColon == used.length) 4641 more(); 4642 auto idxStart = idxColon + 1; 4643 while(idxStart < used.length && used[idxStart] == ' ') 4644 idxStart++; 4645 if(idxStart == used.length) 4646 more(); 4647 auto idxEnd = idxStart; 4648 while(idxEnd < used.length && used[idxEnd] != '\r') 4649 idxEnd++; 4650 if(idxEnd == used.length) 4651 more(); 4652 4653 auto headerName = used[0 .. idxColon]; 4654 auto headerValue = used[idxStart .. idxEnd]; 4655 4656 // move past this header 4657 used = used[idxEnd .. $]; 4658 // and the \r\n 4659 if(2 <= used.length) 4660 used = used[2 .. $]; 4661 4662 if(headerName.bicmp("upgrade")) { 4663 if(headerValue.bicmp("websocket")) 4664 isWebsocket = true; 4665 } else if(headerName.bicmp("connection")) { 4666 if(headerValue.bicmp("upgrade")) 4667 isUpgrade = true; 4668 } else if(headerName.bicmp("sec-websocket-accept")) { 4669 accept = headerValue; 4670 } else if(headerName.bicmp("sec-websocket-protocol")) { 4671 protocol = headerValue; 4672 } 4673 4674 if(!used.length) { 4675 more(); 4676 } 4677 } 4678 4679 4680 if(!isWebsocket) 4681 throw new Exception("didn't answer as websocket"); 4682 if(!isUpgrade) 4683 throw new Exception("didn't answer as upgrade"); 4684 4685 4686 // FIXME: check protocol if config requested one 4687 // FIXME: check accept for the right hash 4688 4689 receiveBuffer[0 .. used.length] = used[]; 4690 receiveBufferUsedLength = used.length; 4691 4692 readyState_ = OPEN; 4693 4694 if(onopen) 4695 onopen(); 4696 4697 nextPing = MonoTime.currTime + config.pingFrequency.msecs; 4698 timeoutFromInactivity = MonoTime.currTime + config.timeoutFromInactivity; 4699 4700 registerActiveSocket(this); 4701 } 4702 4703 /++ 4704 Is data pending on the socket? Also check [isMessageBuffered] to see if there 4705 is already a message in memory too. 4706 4707 If this returns `true`, you can call [lowLevelReceive], then try [isMessageBuffered] 4708 again. 4709 +/ 4710 /// Group: blocking_api 4711 public bool isDataPending(Duration timeout = 0.seconds) { 4712 static SocketSet readSet; 4713 if(readSet is null) 4714 readSet = new SocketSet(); 4715 4716 version(with_openssl) 4717 if(auto s = cast(SslClientSocket) socket) { 4718 // select doesn't handle the case with stuff 4719 // left in the ssl buffer so i'm checking it separately 4720 if(s.dataPending()) { 4721 return true; 4722 } 4723 } 4724 4725 readSet.reset(); 4726 4727 readSet.add(socket); 4728 4729 //tryAgain: 4730 auto selectGot = Socket.select(readSet, null, null, timeout); 4731 if(selectGot == 0) { /* timeout */ 4732 // timeout 4733 return false; 4734 } else if(selectGot == -1) { /* interrupted */ 4735 return false; 4736 } else { /* ready */ 4737 if(readSet.isSet(socket)) { 4738 return true; 4739 } 4740 } 4741 4742 return false; 4743 } 4744 4745 private void llsend(ubyte[] d) { 4746 if(readyState == CONNECTING) 4747 throw new Exception("WebSocket not connected when trying to send. Did you forget to call connect(); ?"); 4748 //connect(); 4749 //import std.stdio; writeln("LLSEND: ", d); 4750 while(d.length) { 4751 auto r = socket.send(d); 4752 if(r < 0 && wouldHaveBlocked()) { 4753 import core.thread; 4754 Thread.sleep(1.msecs); 4755 continue; 4756 } 4757 //import core.stdc.errno; import std.stdio; writeln(errno); 4758 if(r <= 0) { 4759 // import std.stdio; writeln(GetLastError()); 4760 throw new Exception("Socket send failed"); 4761 } 4762 d = d[r .. $]; 4763 } 4764 } 4765 4766 private void llclose() { 4767 // import std.stdio; writeln("LLCLOSE"); 4768 socket.shutdown(SocketShutdown.SEND); 4769 } 4770 4771 /++ 4772 Waits for more data off the low-level socket and adds it to the pending buffer. 4773 4774 Returns `true` if the connection is still active. 4775 +/ 4776 /// Group: blocking_api 4777 public bool lowLevelReceive() { 4778 if(readyState == CONNECTING) 4779 throw new Exception("WebSocket not connected when trying to receive. Did you forget to call connect(); ?"); 4780 if (receiveBufferUsedLength == receiveBuffer.length) 4781 { 4782 if (receiveBuffer.length == config.maximumReceiveBufferSize) 4783 throw new Exception("Maximum receive buffer size exhausted"); 4784 4785 import std.algorithm : min; 4786 receiveBuffer.length = min(receiveBuffer.length + config.initialReceiveBufferSize, 4787 config.maximumReceiveBufferSize); 4788 } 4789 auto r = socket.receive(receiveBuffer[receiveBufferUsedLength .. $]); 4790 if(r == 0) 4791 return false; 4792 if(r < 0 && wouldHaveBlocked()) 4793 return true; 4794 if(r <= 0) { 4795 //import std.stdio; writeln(WSAGetLastError()); 4796 throw new Exception("Socket receive failed"); 4797 } 4798 receiveBufferUsedLength += r; 4799 return true; 4800 } 4801 4802 private Socket socket; 4803 4804 /* copy/paste section { */ 4805 4806 private int readyState_; 4807 private ubyte[] receiveBuffer; 4808 private size_t receiveBufferUsedLength; 4809 4810 private Config config; 4811 4812 enum CONNECTING = 0; /// Socket has been created. The connection is not yet open. 4813 enum OPEN = 1; /// The connection is open and ready to communicate. 4814 enum CLOSING = 2; /// The connection is in the process of closing. 4815 enum CLOSED = 3; /// The connection is closed or couldn't be opened. 4816 4817 /++ 4818 4819 +/ 4820 /// Group: foundational 4821 static struct Config { 4822 /++ 4823 These control the size of the receive buffer. 4824 4825 It starts at the initial size, will temporarily 4826 balloon up to the maximum size, and will reuse 4827 a buffer up to the likely size. 4828 4829 Anything larger than the maximum size will cause 4830 the connection to be aborted and an exception thrown. 4831 This is to protect you against a peer trying to 4832 exhaust your memory, while keeping the user-level 4833 processing simple. 4834 +/ 4835 size_t initialReceiveBufferSize = 4096; 4836 size_t likelyReceiveBufferSize = 4096; /// ditto 4837 size_t maximumReceiveBufferSize = 10 * 1024 * 1024; /// ditto 4838 4839 /++ 4840 Maximum combined size of a message. 4841 +/ 4842 size_t maximumMessageSize = 10 * 1024 * 1024; 4843 4844 string[string] cookies; /// Cookies to send with the initial request. cookies[name] = value; 4845 string origin; /// Origin URL to send with the handshake, if desired. 4846 string protocol; /// the protocol header, if desired. 4847 4848 /++ 4849 Additional headers to put in the HTTP request. These should be formatted `Name: value`, like for example: 4850 4851 --- 4852 Config config; 4853 config.additionalHeaders ~= "Authorization: Bearer your_auth_token_here"; 4854 --- 4855 4856 History: 4857 Added February 19, 2021 (included in dub version 9.2) 4858 +/ 4859 string[] additionalHeaders; 4860 4861 /++ 4862 Amount of time (in msecs) of idleness after which to send an automatic ping 4863 4864 Please note how this interacts with [timeoutFromInactivity] - a ping counts as activity that 4865 keeps the socket alive. 4866 +/ 4867 int pingFrequency = 5000; 4868 4869 /++ 4870 Amount of time to disconnect when there's no activity. Note that automatic pings will keep the connection alive; this timeout only occurs if there's absolutely nothing, including no responses to websocket ping frames. Since the default [pingFrequency] is only seconds, this one minute should never elapse unless the connection is actually dead. 4871 4872 The one thing to keep in mind is if your program is busy and doesn't check input, it might consider this a time out since there's no activity. The reason is that your program was busy rather than a connection failure, but it doesn't care. You should avoid long processing periods anyway though! 4873 4874 History: 4875 Added March 31, 2021 (included in dub version 9.4) 4876 +/ 4877 Duration timeoutFromInactivity = 1.minutes; 4878 4879 /++ 4880 For https connections, if this is `true`, it will fail to connect if the TLS certificate can not be 4881 verified. Setting this to `false` will skip this check and allow the connection to continue anyway. 4882 4883 History: 4884 Added April 5, 2022 (dub v10.8) 4885 4886 Prior to this, it always used the global (but undocumented) `defaultVerifyPeer` setting, and sometimes 4887 even if it was true, it would skip the verification. Now, it always respects this local setting. 4888 +/ 4889 bool verifyPeer = true; 4890 } 4891 4892 /++ 4893 Returns one of [CONNECTING], [OPEN], [CLOSING], or [CLOSED]. 4894 +/ 4895 int readyState() { 4896 return readyState_; 4897 } 4898 4899 /++ 4900 Closes the connection, sending a graceful teardown message to the other side. 4901 4902 Code 1000 is the normal closure code. 4903 4904 History: 4905 The default `code` was changed to 1000 on January 9, 2023. Previously it was 0, 4906 but also ignored anyway. 4907 +/ 4908 /// Group: foundational 4909 void close(int code = 1000, string reason = null) 4910 //in (reason.length < 123) 4911 in { assert(reason.length < 123); } do 4912 { 4913 if(readyState_ != OPEN) 4914 return; // it cool, we done 4915 WebSocketFrame wss; 4916 wss.fin = true; 4917 wss.masked = this.isClient; 4918 wss.opcode = WebSocketOpcode.close; 4919 wss.data = [ubyte((code >> 8) & 0xff), ubyte(code & 0xff)] ~ cast(ubyte[]) reason.dup; 4920 wss.send(&llsend); 4921 4922 readyState_ = CLOSING; 4923 4924 closeCalled = true; 4925 4926 llclose(); 4927 } 4928 4929 private bool closeCalled; 4930 4931 /++ 4932 Sends a ping message to the server. This is done automatically by the library if you set a non-zero [Config.pingFrequency], but you can also send extra pings explicitly as well with this function. 4933 +/ 4934 /// Group: foundational 4935 void ping(in ubyte[] data = null) { 4936 WebSocketFrame wss; 4937 wss.fin = true; 4938 wss.masked = this.isClient; 4939 wss.opcode = WebSocketOpcode.ping; 4940 if(data !is null) wss.data = data.dup; 4941 wss.send(&llsend); 4942 } 4943 4944 /++ 4945 Sends a pong message to the server. This is normally done automatically in response to pings. 4946 +/ 4947 /// Group: foundational 4948 void pong(in ubyte[] data = null) { 4949 WebSocketFrame wss; 4950 wss.fin = true; 4951 wss.masked = this.isClient; 4952 wss.opcode = WebSocketOpcode.pong; 4953 if(data !is null) wss.data = data.dup; 4954 wss.send(&llsend); 4955 } 4956 4957 /++ 4958 Sends a text message through the websocket. 4959 +/ 4960 /// Group: foundational 4961 void send(in char[] textData) { 4962 WebSocketFrame wss; 4963 wss.fin = true; 4964 wss.masked = this.isClient; 4965 wss.opcode = WebSocketOpcode.text; 4966 wss.data = cast(ubyte[]) textData.dup; 4967 wss.send(&llsend); 4968 } 4969 4970 /++ 4971 Sends a binary message through the websocket. 4972 +/ 4973 /// Group: foundational 4974 void send(in ubyte[] binaryData) { 4975 WebSocketFrame wss; 4976 wss.masked = this.isClient; 4977 wss.fin = true; 4978 wss.opcode = WebSocketOpcode.binary; 4979 wss.data = cast(ubyte[]) binaryData.dup; 4980 wss.send(&llsend); 4981 } 4982 4983 /++ 4984 Waits for and returns the next complete message on the socket. 4985 4986 Note that the onmessage function is still called, right before 4987 this returns. 4988 +/ 4989 /// Group: blocking_api 4990 public WebSocketFrame waitForNextMessage() { 4991 do { 4992 auto m = processOnce(); 4993 if(m.populated) 4994 return m; 4995 } while(lowLevelReceive()); 4996 4997 return WebSocketFrame.init; // FIXME? maybe. 4998 } 4999 5000 /++ 5001 Tells if [waitForNextMessage] would block. 5002 +/ 5003 /// Group: blocking_api 5004 public bool waitForNextMessageWouldBlock() { 5005 checkAgain: 5006 if(isMessageBuffered()) 5007 return false; 5008 if(!isDataPending()) 5009 return true; 5010 while(isDataPending()) 5011 lowLevelReceive(); 5012 goto checkAgain; 5013 } 5014 5015 /++ 5016 Is there a message in the buffer already? 5017 If `true`, [waitForNextMessage] is guaranteed to return immediately. 5018 If `false`, check [isDataPending] as the next step. 5019 +/ 5020 /// Group: blocking_api 5021 public bool isMessageBuffered() { 5022 ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; 5023 auto s = d; 5024 if(d.length) { 5025 auto orig = d; 5026 auto m = WebSocketFrame.read(d); 5027 // that's how it indicates that it needs more data 5028 if(d !is orig) 5029 return true; 5030 } 5031 5032 return false; 5033 } 5034 5035 private ubyte continuingType; 5036 private ubyte[] continuingData; 5037 //private size_t continuingDataLength; 5038 5039 private WebSocketFrame processOnce() { 5040 ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; 5041 auto s = d; 5042 // FIXME: handle continuation frames more efficiently. it should really just reuse the receive buffer. 5043 WebSocketFrame m; 5044 if(d.length) { 5045 auto orig = d; 5046 m = WebSocketFrame.read(d); 5047 // that's how it indicates that it needs more data 5048 if(d is orig) 5049 return WebSocketFrame.init; 5050 m.unmaskInPlace(); 5051 switch(m.opcode) { 5052 case WebSocketOpcode.continuation: 5053 if(continuingData.length + m.data.length > config.maximumMessageSize) 5054 throw new Exception("message size exceeded"); 5055 5056 continuingData ~= m.data; 5057 if(m.fin) { 5058 if(ontextmessage) 5059 ontextmessage(cast(char[]) continuingData); 5060 if(onbinarymessage) 5061 onbinarymessage(continuingData); 5062 5063 continuingData = null; 5064 } 5065 break; 5066 case WebSocketOpcode.text: 5067 if(m.fin) { 5068 if(ontextmessage) 5069 ontextmessage(m.textData); 5070 } else { 5071 continuingType = m.opcode; 5072 //continuingDataLength = 0; 5073 continuingData = null; 5074 continuingData ~= m.data; 5075 } 5076 break; 5077 case WebSocketOpcode.binary: 5078 if(m.fin) { 5079 if(onbinarymessage) 5080 onbinarymessage(m.data); 5081 } else { 5082 continuingType = m.opcode; 5083 //continuingDataLength = 0; 5084 continuingData = null; 5085 continuingData ~= m.data; 5086 } 5087 break; 5088 case WebSocketOpcode.close: 5089 5090 //import std.stdio; writeln("closed ", cast(string) m.data); 5091 5092 ushort code = CloseEvent.StandardCloseCodes.noStatusCodePresent; 5093 const(char)[] reason; 5094 5095 if(m.data.length >= 2) { 5096 code = (m.data[0] << 8) | m.data[1]; 5097 reason = (cast(char[]) m.data[2 .. $]); 5098 } 5099 5100 if(onclose) 5101 onclose(CloseEvent(code, reason, true)); 5102 5103 // if we receive one and haven't sent one back we're supposed to echo it back and close. 5104 if(!closeCalled) 5105 close(code, reason.idup); 5106 5107 readyState_ = CLOSED; 5108 5109 unregisterActiveSocket(this); 5110 break; 5111 case WebSocketOpcode.ping: 5112 // import std.stdio; writeln("ping received ", m.data); 5113 pong(m.data); 5114 break; 5115 case WebSocketOpcode.pong: 5116 // import std.stdio; writeln("pong received ", m.data); 5117 // just really references it is still alive, nbd. 5118 break; 5119 default: // ignore though i could and perhaps should throw too 5120 } 5121 } 5122 5123 if(d.length) { 5124 m.data = m.data.dup(); 5125 } 5126 5127 import core.stdc.string; 5128 memmove(receiveBuffer.ptr, d.ptr, d.length); 5129 receiveBufferUsedLength = d.length; 5130 5131 return m; 5132 } 5133 5134 private void autoprocess() { 5135 // FIXME 5136 do { 5137 processOnce(); 5138 } while(lowLevelReceive()); 5139 } 5140 5141 /++ 5142 Arguments for the close event. The `code` and `reason` are provided from the close message on the websocket, if they are present. The spec says code 1000 indicates a normal, default reason close, but reserves the code range from 3000-5000 for future definition; the 3000s can be registered with IANA and the 4000's are application private use. The `reason` should be user readable, but not displayed to the end user. `wasClean` is true if the server actually sent a close event, false if it just disconnected. 5143 5144 $(PITFALL 5145 The `reason` argument references a temporary buffer and there's no guarantee it will remain valid once your callback returns. It may be freed and will very likely be overwritten. If you want to keep the reason beyond the callback, make sure you `.idup` it. 5146 ) 5147 5148 History: 5149 Added March 19, 2023 (dub v11.0). 5150 +/ 5151 static struct CloseEvent { 5152 ushort code; 5153 const(char)[] reason; 5154 bool wasClean; 5155 5156 string extendedErrorInformationUnstable; 5157 5158 /++ 5159 See https://www.rfc-editor.org/rfc/rfc6455#section-7.4.1 for details. 5160 +/ 5161 enum StandardCloseCodes { 5162 purposeFulfilled = 1000, 5163 goingAway = 1001, 5164 protocolError = 1002, 5165 unacceptableData = 1003, // e.g. got text message when you can only handle binary 5166 Reserved = 1004, 5167 noStatusCodePresent = 1005, // not set by endpoint. 5168 abnormalClosure = 1006, // not set by endpoint. closed without a Close control. FIXME: maybe keep a copy of errno around for these 5169 inconsistentData = 1007, // e.g. utf8 validation failed 5170 genericPolicyViolation = 1008, 5171 messageTooBig = 1009, 5172 clientRequiredExtensionMissing = 1010, // only the client should send this 5173 unnexpectedCondition = 1011, 5174 unverifiedCertificate = 1015, // not set by client 5175 } 5176 } 5177 5178 /++ 5179 The `CloseEvent` you get references a temporary buffer that may be overwritten after your handler returns. If you want to keep it or the `event.reason` member, remember to `.idup` it. 5180 5181 History: 5182 The `CloseEvent` was changed to a [arsd.core.FlexibleDelegate] on March 19, 2023 (dub v11.0). Before that, `onclose` was a public member of type `void delegate()`. This change means setters still work with or without the [CloseEvent] argument. 5183 5184 Your onclose method is now also called on abnormal terminations. Check the `wasClean` member of the `CloseEvent` to know if it came from a close frame or other cause. 5185 +/ 5186 FlexibleDelegate!(void delegate(CloseEvent event)) onclose; 5187 void delegate() onerror; /// 5188 void delegate(in char[]) ontextmessage; /// 5189 void delegate(in ubyte[]) onbinarymessage; /// 5190 void delegate() onopen; /// 5191 5192 /++ 5193 5194 +/ 5195 /// Group: browser_api 5196 void onmessage(void delegate(in char[]) dg) { 5197 ontextmessage = dg; 5198 } 5199 5200 /// ditto 5201 void onmessage(void delegate(in ubyte[]) dg) { 5202 onbinarymessage = dg; 5203 } 5204 5205 /* } end copy/paste */ 5206 5207 /* 5208 const int bufferedAmount // amount pending 5209 const string extensions 5210 5211 const string protocol 5212 const string url 5213 */ 5214 5215 static { 5216 /++ 5217 Runs an event loop with all known websockets on this thread until all websockets 5218 are closed or unregistered, or until you call [exitEventLoop], or set `*localLoopExited` 5219 to false (please note it may take a few seconds until it checks that flag again; it may 5220 not exit immediately). 5221 5222 History: 5223 The `localLoopExited` parameter was added August 22, 2022 (dub v10.9) 5224 5225 See_Also: 5226 [addToSimpledisplayEventLoop] 5227 +/ 5228 void eventLoop(shared(bool)* localLoopExited = null) { 5229 import core.atomic; 5230 atomicOp!"+="(numberOfEventLoops, 1); 5231 scope(exit) { 5232 if(atomicOp!"-="(numberOfEventLoops, 1) <= 0) 5233 loopExited = false; // reset it so we can reenter 5234 } 5235 5236 static SocketSet readSet; 5237 5238 if(readSet is null) 5239 readSet = new SocketSet(); 5240 5241 loopExited = false; 5242 5243 outermost: while(!loopExited && (localLoopExited is null || (*localLoopExited == false))) { 5244 readSet.reset(); 5245 5246 Duration timeout = 3.seconds; 5247 5248 auto now = MonoTime.currTime; 5249 bool hadAny; 5250 foreach(sock; activeSockets) { 5251 auto diff = sock.timeoutFromInactivity - now; 5252 if(diff <= 0.msecs) { 5253 // timeout 5254 if(sock.onerror) 5255 sock.onerror(); 5256 5257 if(sock.onclose) 5258 sock.onclose(CloseEvent(CloseEvent.StandardCloseCodes.abnormalClosure, "Connection timed out", false, null)); 5259 5260 sock.socket.close(); 5261 sock.readyState_ = CLOSED; 5262 unregisterActiveSocket(sock); 5263 continue outermost; 5264 } 5265 5266 if(diff < timeout) 5267 timeout = diff; 5268 5269 diff = sock.nextPing - now; 5270 5271 if(diff <= 0.msecs) { 5272 //sock.send(`{"action": "ping"}`); 5273 sock.ping(); 5274 sock.nextPing = now + sock.config.pingFrequency.msecs; 5275 } else { 5276 if(diff < timeout) 5277 timeout = diff; 5278 } 5279 5280 readSet.add(sock.socket); 5281 hadAny = true; 5282 } 5283 5284 if(!hadAny) { 5285 // import std.stdio; writeln("had none"); 5286 return; 5287 } 5288 5289 tryAgain: 5290 // import std.stdio; writeln(timeout); 5291 auto selectGot = Socket.select(readSet, null, null, timeout); 5292 if(selectGot == 0) { /* timeout */ 5293 // timeout 5294 continue; // it will be handled at the top of the loop 5295 } else if(selectGot == -1) { /* interrupted */ 5296 goto tryAgain; 5297 } else { 5298 foreach(sock; activeSockets) { 5299 if(readSet.isSet(sock.socket)) { 5300 sock.timeoutFromInactivity = MonoTime.currTime + sock.config.timeoutFromInactivity; 5301 if(!sock.lowLevelReceive()) { 5302 sock.readyState_ = CLOSED; 5303 5304 if(sock.onerror) 5305 sock.onerror(); 5306 5307 if(sock.onclose) 5308 sock.onclose(CloseEvent(CloseEvent.StandardCloseCodes.abnormalClosure, "Connection lost", false, lastSocketError())); 5309 5310 unregisterActiveSocket(sock); 5311 continue outermost; 5312 } 5313 while(sock.processOnce().populated) {} 5314 selectGot--; 5315 if(selectGot <= 0) 5316 break; 5317 } 5318 } 5319 } 5320 } 5321 } 5322 5323 private static shared(int) numberOfEventLoops; 5324 5325 private __gshared bool loopExited; 5326 /++ 5327 Exits all running [WebSocket.eventLoop]s next time they loop around. You can call this from a signal handler or another thread. 5328 5329 Please note they may not loop around to check the flag for several seconds. Any new event loops will exit immediately until 5330 all current ones are closed. Once all event loops are exited, the flag is cleared and you can start the loop again. 5331 5332 This function is likely to be deprecated in the future due to its quirks and imprecise name. 5333 +/ 5334 void exitEventLoop() { 5335 loopExited = true; 5336 } 5337 5338 WebSocket[] activeSockets; 5339 5340 void registerActiveSocket(WebSocket s) { 5341 // ensure it isn't already there... 5342 assert(s !is null); 5343 foreach(i, a; activeSockets) 5344 if(a is s) 5345 return; 5346 activeSockets ~= s; 5347 } 5348 void unregisterActiveSocket(WebSocket s) { 5349 foreach(i, a; activeSockets) 5350 if(s is a) { 5351 activeSockets[i] = activeSockets[$-1]; 5352 activeSockets = activeSockets[0 .. $-1]; 5353 break; 5354 } 5355 } 5356 } 5357 } 5358 5359 private template imported(string mod) { 5360 mixin(`import imported = ` ~ mod ~ `;`); 5361 } 5362 5363 /++ 5364 Warning: you should call this AFTER websocket.connect or else it might throw on connect because the function sets nonblocking mode and the connect function doesn't handle that well (it throws on the "would block" condition in that function. easier to just do that first) 5365 +/ 5366 template addToSimpledisplayEventLoop() { 5367 import arsd.simpledisplay; 5368 void addToSimpledisplayEventLoop(WebSocket ws, imported!"arsd.simpledisplay".SimpleWindow window) { 5369 5370 void midprocess() { 5371 if(!ws.lowLevelReceive()) { 5372 ws.readyState_ = WebSocket.CLOSED; 5373 WebSocket.unregisterActiveSocket(ws); 5374 return; 5375 } 5376 while(ws.processOnce().populated) {} 5377 } 5378 5379 version(Posix) { 5380 auto reader = new PosixFdReader(&midprocess, ws.socket.handle); 5381 } else version(none) { 5382 if(WSAAsyncSelect(ws.socket.handle, window.hwnd, WM_USER + 150, FD_CLOSE | FD_READ)) 5383 throw new Exception("WSAAsyncSelect"); 5384 5385 window.handleNativeEvent = delegate int(HWND hwnd, UINT msg, WPARAM wParam, LPARAM lParam) { 5386 if(hwnd !is window.impl.hwnd) 5387 return 1; // we don't care... 5388 switch(msg) { 5389 case WM_USER + 150: // socket activity 5390 switch(LOWORD(lParam)) { 5391 case FD_READ: 5392 case FD_CLOSE: 5393 midprocess(); 5394 break; 5395 default: 5396 // nothing 5397 } 5398 break; 5399 default: return 1; // not handled, pass it on 5400 } 5401 return 0; 5402 }; 5403 5404 } else version(Windows) { 5405 ws.socket.blocking = false; // the WSAEventSelect does this anyway and doing it here lets phobos know about it. 5406 //CreateEvent(null, 0, 0, null); 5407 auto event = WSACreateEvent(); 5408 if(!event) { 5409 throw new Exception("WSACreateEvent"); 5410 } 5411 if(WSAEventSelect(ws.socket.handle, event, 1/*FD_READ*/ | (1<<5)/*FD_CLOSE*/)) { 5412 //import std.stdio; writeln(WSAGetLastError()); 5413 throw new Exception("WSAEventSelect"); 5414 } 5415 5416 auto handle = new WindowsHandleReader(&midprocess, event); 5417 5418 /+ 5419 static class Ready {} 5420 5421 Ready thisr = new Ready; 5422 5423 justCommunication.addEventListener((Ready r) { 5424 if(r is thisr) 5425 midprocess(); 5426 }); 5427 5428 import core.thread; 5429 auto thread = new Thread({ 5430 while(true) { 5431 WSAWaitForMultipleEvents(1, &event, true, -1/*WSA_INFINITE*/, false); 5432 justCommunication.postEvent(thisr); 5433 } 5434 }); 5435 thread.isDaemon = true; 5436 thread.start; 5437 +/ 5438 5439 } else static assert(0, "unsupported OS"); 5440 } 5441 } 5442 5443 version(Windows) { 5444 import core.sys.windows.windows; 5445 import core.sys.windows.winsock2; 5446 } 5447 5448 version(none) { 5449 extern(Windows) int WSAAsyncSelect(SOCKET, HWND, uint, int); 5450 enum int FD_CLOSE = 1 << 5; 5451 enum int FD_READ = 1 << 0; 5452 enum int WM_USER = 1024; 5453 } 5454 5455 version(Windows) { 5456 import core.stdc.config; 5457 extern(Windows) 5458 int WSAEventSelect(SOCKET, HANDLE /* to an Event */, c_long); 5459 5460 extern(Windows) 5461 HANDLE WSACreateEvent(); 5462 5463 extern(Windows) 5464 DWORD WSAWaitForMultipleEvents(DWORD, HANDLE*, BOOL, DWORD, BOOL); 5465 } 5466 5467 /* copy/paste from cgi.d */ 5468 public { 5469 enum WebSocketOpcode : ubyte { 5470 continuation = 0, 5471 text = 1, 5472 binary = 2, 5473 // 3, 4, 5, 6, 7 RESERVED 5474 close = 8, 5475 ping = 9, 5476 pong = 10, 5477 // 11,12,13,14,15 RESERVED 5478 } 5479 5480 public struct WebSocketFrame { 5481 private bool populated; 5482 bool fin; 5483 bool rsv1; 5484 bool rsv2; 5485 bool rsv3; 5486 WebSocketOpcode opcode; // 4 bits 5487 bool masked; 5488 ubyte lengthIndicator; // don't set this when building one to send 5489 ulong realLength; // don't use when sending 5490 ubyte[4] maskingKey; // don't set this when sending 5491 ubyte[] data; 5492 5493 static WebSocketFrame simpleMessage(WebSocketOpcode opcode, in void[] data) { 5494 WebSocketFrame msg; 5495 msg.fin = true; 5496 msg.opcode = opcode; 5497 msg.data = cast(ubyte[]) data.dup; // it is mutated below when masked, so need to be cautious and copy it, sigh 5498 5499 return msg; 5500 } 5501 5502 private void send(scope void delegate(ubyte[]) llsend) { 5503 ubyte[64] headerScratch; 5504 int headerScratchPos = 0; 5505 5506 realLength = data.length; 5507 5508 { 5509 ubyte b1; 5510 b1 |= cast(ubyte) opcode; 5511 b1 |= rsv3 ? (1 << 4) : 0; 5512 b1 |= rsv2 ? (1 << 5) : 0; 5513 b1 |= rsv1 ? (1 << 6) : 0; 5514 b1 |= fin ? (1 << 7) : 0; 5515 5516 headerScratch[0] = b1; 5517 headerScratchPos++; 5518 } 5519 5520 { 5521 headerScratchPos++; // we'll set header[1] at the end of this 5522 auto rlc = realLength; 5523 ubyte b2; 5524 b2 |= masked ? (1 << 7) : 0; 5525 5526 assert(headerScratchPos == 2); 5527 5528 if(realLength > 65535) { 5529 // use 64 bit length 5530 b2 |= 0x7f; 5531 5532 // FIXME: double check endinaness 5533 foreach(i; 0 .. 8) { 5534 headerScratch[2 + 7 - i] = rlc & 0x0ff; 5535 rlc >>>= 8; 5536 } 5537 5538 headerScratchPos += 8; 5539 } else if(realLength > 125) { 5540 // use 16 bit length 5541 b2 |= 0x7e; 5542 5543 // FIXME: double check endinaness 5544 foreach(i; 0 .. 2) { 5545 headerScratch[2 + 1 - i] = rlc & 0x0ff; 5546 rlc >>>= 8; 5547 } 5548 5549 headerScratchPos += 2; 5550 } else { 5551 // use 7 bit length 5552 b2 |= realLength & 0b_0111_1111; 5553 } 5554 5555 headerScratch[1] = b2; 5556 } 5557 5558 //assert(!masked, "masking key not properly implemented"); 5559 if(masked) { 5560 import std.random; 5561 foreach(ref item; maskingKey) 5562 item = uniform(ubyte.min, ubyte.max); 5563 headerScratch[headerScratchPos .. headerScratchPos + 4] = maskingKey[]; 5564 headerScratchPos += 4; 5565 5566 // we'll just mask it in place... 5567 int keyIdx = 0; 5568 foreach(i; 0 .. data.length) { 5569 data[i] = data[i] ^ maskingKey[keyIdx]; 5570 if(keyIdx == 3) 5571 keyIdx = 0; 5572 else 5573 keyIdx++; 5574 } 5575 } 5576 5577 //writeln("SENDING ", headerScratch[0 .. headerScratchPos], data); 5578 llsend(headerScratch[0 .. headerScratchPos]); 5579 if(data.length) 5580 llsend(data); 5581 } 5582 5583 static WebSocketFrame read(ref ubyte[] d) { 5584 WebSocketFrame msg; 5585 5586 auto orig = d; 5587 5588 WebSocketFrame needsMoreData() { 5589 d = orig; 5590 return WebSocketFrame.init; 5591 } 5592 5593 if(d.length < 2) 5594 return needsMoreData(); 5595 5596 ubyte b = d[0]; 5597 5598 msg.populated = true; 5599 5600 msg.opcode = cast(WebSocketOpcode) (b & 0x0f); 5601 b >>= 4; 5602 msg.rsv3 = b & 0x01; 5603 b >>= 1; 5604 msg.rsv2 = b & 0x01; 5605 b >>= 1; 5606 msg.rsv1 = b & 0x01; 5607 b >>= 1; 5608 msg.fin = b & 0x01; 5609 5610 b = d[1]; 5611 msg.masked = (b & 0b1000_0000) ? true : false; 5612 msg.lengthIndicator = b & 0b0111_1111; 5613 5614 d = d[2 .. $]; 5615 5616 if(msg.lengthIndicator == 0x7e) { 5617 // 16 bit length 5618 msg.realLength = 0; 5619 5620 if(d.length < 2) return needsMoreData(); 5621 5622 foreach(i; 0 .. 2) { 5623 msg.realLength |= d[0] << ((1-i) * 8); 5624 d = d[1 .. $]; 5625 } 5626 } else if(msg.lengthIndicator == 0x7f) { 5627 // 64 bit length 5628 msg.realLength = 0; 5629 5630 if(d.length < 8) return needsMoreData(); 5631 5632 foreach(i; 0 .. 8) { 5633 msg.realLength |= ulong(d[0]) << ((7-i) * 8); 5634 d = d[1 .. $]; 5635 } 5636 } else { 5637 // 7 bit length 5638 msg.realLength = msg.lengthIndicator; 5639 } 5640 5641 if(msg.masked) { 5642 5643 if(d.length < 4) return needsMoreData(); 5644 5645 msg.maskingKey = d[0 .. 4]; 5646 d = d[4 .. $]; 5647 } 5648 5649 if(msg.realLength > d.length) { 5650 return needsMoreData(); 5651 } 5652 5653 msg.data = d[0 .. cast(size_t) msg.realLength]; 5654 d = d[cast(size_t) msg.realLength .. $]; 5655 5656 return msg; 5657 } 5658 5659 void unmaskInPlace() { 5660 if(this.masked) { 5661 int keyIdx = 0; 5662 foreach(i; 0 .. this.data.length) { 5663 this.data[i] = this.data[i] ^ this.maskingKey[keyIdx]; 5664 if(keyIdx == 3) 5665 keyIdx = 0; 5666 else 5667 keyIdx++; 5668 } 5669 } 5670 } 5671 5672 char[] textData() { 5673 return cast(char[]) data; 5674 } 5675 } 5676 } 5677 5678 private extern(C) 5679 int verifyCertificateFromRegistryArsdHttp(int preverify_ok, X509_STORE_CTX* ctx) { 5680 version(Windows) { 5681 if(preverify_ok) 5682 return 1; 5683 5684 auto err_cert = OpenSSL.X509_STORE_CTX_get_current_cert(ctx); 5685 auto err = OpenSSL.X509_STORE_CTX_get_error(ctx); 5686 5687 if(err == 62) 5688 return 0; // hostname mismatch is an error we can trust; that means OpenSSL already found the certificate and rejected it 5689 5690 auto len = OpenSSL.i2d_X509(err_cert, null); 5691 if(len == -1) 5692 return 0; 5693 ubyte[] buffer = new ubyte[](len); 5694 auto ptr = buffer.ptr; 5695 len = OpenSSL.i2d_X509(err_cert, &ptr); 5696 if(len != buffer.length) 5697 return 0; 5698 5699 5700 CERT_CHAIN_PARA thing; 5701 thing.cbSize = thing.sizeof; 5702 auto context = CertCreateCertificateContext(X509_ASN_ENCODING, buffer.ptr, cast(int) buffer.length); 5703 if(context is null) 5704 return 0; 5705 scope(exit) CertFreeCertificateContext(context); 5706 5707 PCCERT_CHAIN_CONTEXT chain; 5708 if(CertGetCertificateChain(null, context, null, null, &thing, 0, null, &chain)) { 5709 scope(exit) 5710 CertFreeCertificateChain(chain); 5711 5712 DWORD errorStatus = chain.TrustStatus.dwErrorStatus; 5713 5714 if(errorStatus == 0) 5715 return 1; // Windows approved it, OK carry on 5716 // otherwise, sustain OpenSSL's original ruling 5717 } 5718 5719 return 0; 5720 } else { 5721 return preverify_ok; 5722 } 5723 } 5724 5725 5726 version(Windows) { 5727 pragma(lib, "crypt32"); 5728 import core.sys.windows.wincrypt; 5729 extern(Windows) { 5730 PCCERT_CONTEXT CertEnumCertificatesInStore(HCERTSTORE hCertStore, PCCERT_CONTEXT pPrevCertContext); 5731 // BOOL CertGetCertificateChain(HCERTCHAINENGINE hChainEngine, PCCERT_CONTEXT pCertContext, LPFILETIME pTime, HCERTSTORE hAdditionalStore, PCERT_CHAIN_PARA pChainPara, DWORD dwFlags, LPVOID pvReserved, PCCERT_CHAIN_CONTEXT *ppChainContext); 5732 PCCERT_CONTEXT CertCreateCertificateContext(DWORD dwCertEncodingType, const BYTE *pbCertEncoded, DWORD cbCertEncoded); 5733 } 5734 5735 void loadCertificatesFromRegistry(SSL_CTX* ctx) { 5736 auto store = CertOpenSystemStore(0, "ROOT"); 5737 if(store is null) { 5738 // import std.stdio; writeln("failed"); 5739 return; 5740 } 5741 scope(exit) 5742 CertCloseStore(store, 0); 5743 5744 X509_STORE* ssl_store = OpenSSL.SSL_CTX_get_cert_store(ctx); 5745 PCCERT_CONTEXT c; 5746 while((c = CertEnumCertificatesInStore(store, c)) !is null) { 5747 FILETIME na = c.pCertInfo.NotAfter; 5748 SYSTEMTIME st; 5749 FileTimeToSystemTime(&na, &st); 5750 5751 /+ 5752 _CRYPTOAPI_BLOB i = cast() c.pCertInfo.Issuer; 5753 5754 char[256] buffer; 5755 auto p = CertNameToStrA(X509_ASN_ENCODING, &i, CERT_SIMPLE_NAME_STR, buffer.ptr, cast(int) buffer.length); 5756 import std.stdio; writeln(buffer[0 .. p]); 5757 +/ 5758 5759 if(st.wYear <= 2021) { 5760 // see: https://www.openssl.org/blog/blog/2021/09/13/LetsEncryptRootCertExpire/ 5761 continue; // no point keeping an expired root cert and it can break Let's Encrypt anyway 5762 } 5763 5764 const(ubyte)* thing = c.pbCertEncoded; 5765 auto x509 = OpenSSL.d2i_X509(null, &thing, c.cbCertEncoded); 5766 if (x509) { 5767 auto success = OpenSSL.X509_STORE_add_cert(ssl_store, x509); 5768 //if(!success) 5769 //writeln("FAILED HERE"); 5770 OpenSSL.X509_free(x509); 5771 } else { 5772 //writeln("FAILED"); 5773 } 5774 } 5775 5776 CertFreeCertificateContext(c); 5777 5778 // import core.stdc.stdio; printf("%s\n", OpenSSL.OpenSSL_version(0)); 5779 } 5780 5781 5782 // because i use the FILE* in PEM_read_X509 and friends 5783 // gotta use this to bridge the MS C runtime functions 5784 // might be able to just change those to only use the BIO versions 5785 // instead 5786 5787 // only on MS C runtime 5788 version(CRuntime_Microsoft) {} else version=no_openssl_applink; 5789 5790 version(no_openssl_applink) {} else { 5791 private extern(C) { 5792 void _open(); 5793 void _read(); 5794 void _write(); 5795 void _lseek(); 5796 void _close(); 5797 int _fileno(FILE*); 5798 int _setmode(int, int); 5799 } 5800 export extern(C) void** OPENSSL_Applink() { 5801 import core.stdc.stdio; 5802 5803 static extern(C) void* app_stdin() { return cast(void*) stdin; } 5804 static extern(C) void* app_stdout() { return cast(void*) stdout; } 5805 static extern(C) void* app_stderr() { return cast(void*) stderr; } 5806 static extern(C) int app_feof(FILE* fp) { return feof(fp); } 5807 static extern(C) int app_ferror(FILE* fp) { return ferror(fp); } 5808 static extern(C) void app_clearerr(FILE* fp) { return clearerr(fp); } 5809 static extern(C) int app_fileno(FILE* fp) { return _fileno(fp); } 5810 static extern(C) int app_fsetmod(FILE* fp, char mod) { 5811 return _setmode(_fileno(fp), mod == 'b' ? _O_BINARY : _O_TEXT); 5812 } 5813 5814 static immutable void*[] table = [ 5815 cast(void*) 22, // applink max 5816 5817 &app_stdin, 5818 &app_stdout, 5819 &app_stderr, 5820 &fprintf, 5821 &fgets, 5822 &fread, 5823 &fwrite, 5824 &app_fsetmod, 5825 &app_feof, 5826 &fclose, 5827 5828 &fopen, 5829 &fseek, 5830 &ftell, 5831 &fflush, 5832 &app_ferror, 5833 &app_clearerr, 5834 &app_fileno, 5835 5836 &_open, 5837 &_read, 5838 &_write, 5839 &_lseek, 5840 &_close, 5841 ]; 5842 static assert(table.length == 23); 5843 5844 return cast(void**) table.ptr; 5845 } 5846 } 5847 } 5848 5849 unittest { 5850 auto client = new HttpClient(); 5851 auto response = client.navigateTo(Uri("data:,Hello%2C%20World%21")).waitForCompletion(); 5852 assert(response.contentTypeMimeType == "text/plain", response.contentType); 5853 assert(response.contentText == "Hello, World!", response.contentText); 5854 5855 response = client.navigateTo(Uri("data:text/plain;base64,SGVsbG8sIFdvcmxkIQ==")).waitForCompletion(); 5856 assert(response.contentTypeMimeType == "text/plain", response.contentType); 5857 assert(response.contentText == "Hello, World!", response.contentText); 5858 5859 response = client.navigateTo(Uri("data:text/html,%3Ch1%3EHello%2C%20World%21%3C%2Fh1%3E")).waitForCompletion(); 5860 assert(response.contentTypeMimeType == "text/html", response.contentType); 5861 assert(response.contentText == "<h1>Hello, World!</h1>", response.contentText); 5862 } 5863 5864 version(arsd_http2_unittests) 5865 unittest { 5866 import core.thread; 5867 5868 static void server() { 5869 import std.socket; 5870 auto socket = new TcpSocket(); 5871 socket.setOption(SocketOptionLevel.SOCKET, SocketOption.REUSEADDR, true); 5872 socket.bind(new InternetAddress(12346)); 5873 socket.listen(1); 5874 auto s = socket.accept(); 5875 socket.close(); 5876 5877 ubyte[1024] thing; 5878 auto g = s.receive(thing[]); 5879 5880 /+ 5881 string response = "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 9\r\n\r\nHello!!??"; 5882 auto packetSize = 2; 5883 +/ 5884 5885 auto packetSize = 1; 5886 string response = "HTTP/1.1 200 OK\r\nConnection: close\r\nTransfer-Encoding: chunked\r\n\r\n6\r\nHello!\r\n0\r\n\r\n"; 5887 5888 while(response.length) { 5889 s.send(response[0 .. packetSize]); 5890 response = response[packetSize .. $]; 5891 //import std.stdio; writeln(response); 5892 } 5893 5894 s.close(); 5895 } 5896 5897 auto thread = new Thread(&server); 5898 thread.start; 5899 5900 Thread.sleep(200.msecs); 5901 5902 auto response = get("http://localhost:12346/").waitForCompletion; 5903 assert(response.code == 200); 5904 //import std.stdio; writeln(response); 5905 5906 foreach(site; ["https://dlang.org/", "http://arsdnet.net", "https://phobos.dpldocs.info"]) { 5907 response = get(site).waitForCompletion; 5908 assert(response.code == 200); 5909 } 5910 5911 thread.join; 5912 } 5913 5914 /+ 5915 so the url params are arguments. it knows the request 5916 internally. other params are properties on the req 5917 5918 names may have different paths... those will just add ForSomething i think. 5919 5920 auto req = api.listMergeRequests 5921 req.page = 10; 5922 5923 or 5924 req.page(1) 5925 .bar("foo") 5926 5927 req.execute(); 5928 5929 5930 everything in the response is nullable access through the 5931 dynamic object, just with property getters there. need to make 5932 it static generated tho 5933 5934 other messages may be: isPresent and getDynamic 5935 5936 5937 AND/OR what about doing it like the rails objects 5938 5939 BroadcastMessage.get(4) 5940 // various properties 5941 5942 // it lists what you updated 5943 5944 BroadcastMessage.foo().bar().put(5) 5945 +/