1 // Copyright 2013-2021, Adam D. Ruppe. 2 3 // FIXME: websocket proxy support 4 // FIXME: ipv6 support 5 6 // FIXME: headers are supposed to be case insensitive. ugh. 7 8 /++ 9 This is version 2 of my http/1.1 client implementation. 10 11 12 It has no dependencies for basic operation, but does require OpenSSL 13 libraries (or compatible) to be support HTTPS. This dynamically loaded 14 on-demand (meaning it won't be loaded if you don't use it, but if you do 15 use it, the openssl dynamic libraries must be found in the system search path). 16 17 You can compile with `-version=without_openssl` to entirely disable ssl support. 18 19 http2.d, despite its name, does NOT implement HTTP/2.0, but this 20 shouldn't matter for 99.9% of usage, since all servers will continue 21 to support HTTP/1.1 for a very long time. 22 23 History: 24 Automatic `100 Continue` handling was added on September 28, 2021. It doesn't 25 set the Expect header, so it isn't supposed to happen, but plenty of web servers 26 don't follow the standard anyway. 27 +/ 28 module arsd.http2; 29 30 /// 31 unittest { 32 import arsd.http2; 33 34 void main() { 35 auto client = new HttpClient(); 36 37 auto request = client.request(Uri("http://dlang.org/")); 38 auto response = request.waitForCompletion(); 39 40 import std.stdio; 41 writeln(response.contentText); 42 writeln(response.code, " ", response.codeText); 43 writeln(response.contentType); 44 } 45 46 version(arsd_http2_integration_test) main(); // exclude from docs 47 } 48 49 // FIXME: I think I want to disable sigpipe here too. 50 51 import std.uri : encodeComponent; 52 53 debug(arsd_http2_verbose) debug=arsd_http2; 54 55 debug(arsd_http2) import std.stdio : writeln; 56 57 version=arsd_http_internal_implementation; 58 59 version(without_openssl) {} 60 else { 61 version=use_openssl; 62 version=with_openssl; 63 version(older_openssl) {} else 64 version=newer_openssl; 65 } 66 67 version(arsd_http_winhttp_implementation) { 68 pragma(lib, "winhttp") 69 import core.sys.windows.winhttp; 70 // FIXME: alter the dub package file too 71 72 // https://github.com/curl/curl/blob/master/lib/vtls/schannel.c 73 // https://docs.microsoft.com/en-us/windows/win32/secauthn/creating-an-schannel-security-context 74 75 76 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpreaddata 77 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpsendrequest 78 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpopenrequest 79 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpconnect 80 } 81 82 83 84 /++ 85 Demonstrates core functionality, using the [HttpClient], 86 [HttpRequest] (returned by [HttpClient.navigateTo|client.navigateTo]), 87 and [HttpResponse] (returned by [HttpRequest.waitForCompletion|request.waitForCompletion]). 88 89 +/ 90 unittest { 91 import arsd.http2; 92 93 void main() { 94 auto client = new HttpClient(); 95 auto request = client.navigateTo(Uri("http://dlang.org/")); 96 auto response = request.waitForCompletion(); 97 98 string returnedHtml = response.contentText; 99 } 100 } 101 102 // FIXME: multipart encoded file uploads needs implementation 103 // future: do web client api stuff 104 105 __gshared bool defaultVerifyPeer = true; // public but intentionally undocumented 106 107 debug import std.stdio; 108 109 import std.socket; 110 import core.time; 111 112 // FIXME: check Transfer-Encoding: gzip always 113 114 version(with_openssl) { 115 //pragma(lib, "crypto"); 116 //pragma(lib, "ssl"); 117 } 118 119 /+ 120 HttpRequest httpRequest(string method, string url, ubyte[] content, string[string] content) { 121 return null; 122 } 123 +/ 124 125 /** 126 auto request = get("http://arsdnet.net/"); 127 request.send(); 128 129 auto response = get("http://arsdnet.net/").waitForCompletion(); 130 */ 131 HttpRequest get(string url) { 132 auto client = new HttpClient(); 133 auto request = client.navigateTo(Uri(url)); 134 return request; 135 } 136 137 /** 138 Do not forget to call `waitForCompletion()` on the returned object! 139 */ 140 HttpRequest post(string url, string[string] req) { 141 auto client = new HttpClient(); 142 ubyte[] bdata; 143 foreach(k, v; req) { 144 if(bdata.length) 145 bdata ~= cast(ubyte[]) "&"; 146 bdata ~= cast(ubyte[]) encodeComponent(k); 147 bdata ~= cast(ubyte[]) "="; 148 bdata ~= cast(ubyte[]) encodeComponent(v); 149 } 150 auto request = client.request(Uri(url), HttpVerb.POST, bdata, "application/x-www-form-urlencoded"); 151 return request; 152 } 153 154 /// gets the text off a url. basic operation only. 155 string getText(string url) { 156 auto request = get(url); 157 auto response = request.waitForCompletion(); 158 return cast(string) response.content; 159 } 160 161 /+ 162 ubyte[] getBinary(string url, string[string] cookies = null) { 163 auto hr = httpRequest("GET", url, null, cookies); 164 if(hr.code != 200) 165 throw new Exception(format("HTTP answered %d instead of 200 on %s", hr.code, url)); 166 return hr.content; 167 } 168 169 /** 170 Gets a textual document, ignoring headers. Throws on non-text or error. 171 */ 172 string get(string url, string[string] cookies = null) { 173 auto hr = httpRequest("GET", url, null, cookies); 174 if(hr.code != 200) 175 throw new Exception(format("HTTP answered %d instead of 200 on %s", hr.code, url)); 176 if(hr.contentType.indexOf("text/") == -1) 177 throw new Exception(hr.contentType ~ " is bad content for conversion to string"); 178 return cast(string) hr.content; 179 180 } 181 182 static import std.uri; 183 184 string post(string url, string[string] args, string[string] cookies = null) { 185 string content; 186 187 foreach(name, arg; args) { 188 if(content.length) 189 content ~= "&"; 190 content ~= std.uri.encode(name) ~ "=" ~ std.uri.encode(arg); 191 } 192 193 auto hr = httpRequest("POST", url, cast(ubyte[]) content, cookies, ["Content-Type: application/x-www-form-urlencoded"]); 194 if(hr.code != 200) 195 throw new Exception(format("HTTP answered %d instead of 200", hr.code)); 196 if(hr.contentType.indexOf("text/") == -1) 197 throw new Exception(hr.contentType ~ " is bad content for conversion to string"); 198 199 return cast(string) hr.content; 200 } 201 202 +/ 203 204 /// 205 struct HttpResponse { 206 /++ 207 The HTTP response code, if the response was completed, or some value < 100 if it was aborted or failed. 208 209 Code 0 - initial value, nothing happened 210 Code 1 - you called request.abort 211 Code 2 - connection refused 212 Code 3 - connection succeeded, but server disconnected early 213 Code 4 - server sent corrupted response (or this code has a bug and processed it wrong) 214 Code 5 - request timed out 215 216 Code >= 100 - a HTTP response 217 +/ 218 int code; 219 string codeText; /// 220 221 string httpVersion; /// 222 223 string statusLine; /// 224 225 string contentType; /// The content type header 226 string location; /// The location header 227 228 /++ 229 230 History: 231 Added December 5, 2020 (version 9.1) 232 +/ 233 bool wasSuccessful() { 234 return code >= 200 && code < 400; 235 } 236 237 /// the charset out of content type, if present. `null` if not. 238 string contentTypeCharset() { 239 auto idx = contentType.indexOf("charset="); 240 if(idx == -1) 241 return null; 242 auto c = contentType[idx + "charset=".length .. $].strip; 243 if(c.length) 244 return c; 245 return null; 246 } 247 248 /++ 249 Names and values of cookies set in the response. 250 251 History: 252 Prior to July 5, 2021 (dub v10.2), this was a public field instead of a property. I did 253 not consider this a breaking change since the intended use is completely compatible with the 254 property, and it was not actually implemented properly before anyway. 255 +/ 256 @property string[string] cookies() const { 257 string[string] ret; 258 foreach(cookie; cookiesDetails) 259 ret[cookie.name] = cookie.value; 260 return ret; 261 } 262 /++ 263 The full parsed-out information of cookies set in the response. 264 265 History: 266 Added July 5, 2021 (dub v10.2). 267 +/ 268 @property CookieHeader[] cookiesDetails() inout { 269 CookieHeader[] ret; 270 foreach(header; headers) { 271 if(auto content = header.isHttpHeader("set-cookie")) { 272 // format: name=value, value might be double quoted. it MIGHT be url encoded, but im not going to attempt that since the RFC is silent. 273 // then there's optionally ; attr=value after that. attributes need not have a value 274 275 CookieHeader cookie; 276 277 auto remaining = content; 278 279 cookie_name: 280 foreach(idx, ch; remaining) { 281 if(ch == '=') { 282 cookie.name = remaining[0 .. idx].idup_if_needed; 283 remaining = remaining[idx + 1 .. $]; 284 break; 285 } 286 } 287 288 cookie_value: 289 290 { 291 auto idx = remaining.indexOf(";"); 292 if(idx == -1) { 293 cookie.value = remaining; 294 remaining = remaining[$..$]; 295 } else { 296 cookie.value = remaining[0 .. idx]; 297 remaining = remaining[idx + 1 .. $].stripLeft; 298 } 299 300 if(cookie.value.length > 2 && cookie.value[0] == '"' && cookie.value[$-1] == '"') 301 cookie.value = cookie.value[1 .. $ - 1]; 302 } 303 304 cookie_attributes: 305 306 while(remaining.length) { 307 string name; 308 foreach(idx, ch; remaining) { 309 if(ch == '=') { 310 name = remaining[0 .. idx].idup_if_needed; 311 remaining = remaining[idx + 1 .. $]; 312 313 string value; 314 315 foreach(idx2, ch2; remaining) { 316 if(ch2 == ';') { 317 value = remaining[0 .. idx2].idup_if_needed; 318 remaining = remaining[idx2 + 1 .. $].stripLeft; 319 break; 320 } 321 } 322 323 if(value is null) { 324 value = remaining.idup_if_needed; 325 remaining = remaining[$ .. $]; 326 } 327 328 cookie.attributes[name] = value; 329 continue cookie_attributes; 330 } else if(ch == ';') { 331 name = remaining[0 .. idx].idup_if_needed; 332 remaining = remaining[idx + 1 .. $].stripLeft; 333 cookie.attributes[name] = ""; 334 continue cookie_attributes; 335 } 336 } 337 338 if(remaining.length) { 339 cookie.attributes[remaining.idup_if_needed] = ""; 340 remaining = remaining[$..$]; 341 342 } 343 } 344 345 ret ~= cookie; 346 } 347 } 348 return ret; 349 } 350 351 string[] headers; /// Array of all headers returned. 352 string[string] headersHash; /// 353 354 ubyte[] content; /// The raw content returned in the response body. 355 string contentText; /// [content], but casted to string (for convenience) 356 357 alias responseText = contentText; // just cuz I do this so often. 358 //alias body = content; 359 360 /++ 361 returns `new Document(this.contentText)`. Requires [arsd.dom]. 362 +/ 363 auto contentDom()() { 364 import arsd.dom; 365 return new Document(this.contentText); 366 367 } 368 369 /++ 370 returns `var.fromJson(this.contentText)`. Requires [arsd.jsvar]. 371 +/ 372 auto contentJson()() { 373 import arsd.jsvar; 374 return var.fromJson(this.contentText); 375 } 376 377 HttpRequestParameters requestParameters; /// 378 379 LinkHeader[] linksStored; 380 bool linksLazilyParsed; 381 382 HttpResponse deepCopy() const { 383 HttpResponse h = cast(HttpResponse) this; 384 h.headers = h.headers.dup; 385 h.headersHash = h.headersHash.dup; 386 h.content = h.content.dup; 387 h.linksStored = h.linksStored.dup; 388 return h; 389 } 390 391 /// Returns links header sorted by "rel" attribute. 392 /// It returns a new array on each call. 393 LinkHeader[string] linksHash() { 394 auto links = this.links(); 395 LinkHeader[string] ret; 396 foreach(link; links) 397 ret[link.rel] = link; 398 return ret; 399 } 400 401 /// Returns the Link header, parsed. 402 LinkHeader[] links() { 403 if(linksLazilyParsed) 404 return linksStored; 405 linksLazilyParsed = true; 406 LinkHeader[] ret; 407 408 auto hdrPtr = "Link" in headersHash; 409 if(hdrPtr is null) 410 return ret; 411 412 auto header = *hdrPtr; 413 414 LinkHeader current; 415 416 while(header.length) { 417 char ch = header[0]; 418 419 if(ch == '<') { 420 // read url 421 header = header[1 .. $]; 422 size_t idx; 423 while(idx < header.length && header[idx] != '>') 424 idx++; 425 current.url = header[0 .. idx]; 426 header = header[idx .. $]; 427 } else if(ch == ';') { 428 // read attribute 429 header = header[1 .. $]; 430 header = header.stripLeft; 431 432 size_t idx; 433 while(idx < header.length && header[idx] != '=') 434 idx++; 435 436 string name = header[0 .. idx]; 437 header = header[idx + 1 .. $]; 438 439 string value; 440 441 if(header.length && header[0] == '"') { 442 // quoted value 443 header = header[1 .. $]; 444 idx = 0; 445 while(idx < header.length && header[idx] != '\"') 446 idx++; 447 value = header[0 .. idx]; 448 header = header[idx .. $]; 449 450 } else if(header.length) { 451 // unquoted value 452 idx = 0; 453 while(idx < header.length && header[idx] != ',' && header[idx] != ' ' && header[idx] != ';') 454 idx++; 455 456 value = header[0 .. idx]; 457 header = header[idx .. $].stripLeft; 458 } 459 460 name = name.toLower; 461 if(name == "rel") 462 current.rel = value; 463 else 464 current.attributes[name] = value; 465 466 } else if(ch == ',') { 467 // start another 468 ret ~= current; 469 current = LinkHeader.init; 470 } else if(ch == ' ' || ch == '\n' || ch == '\r' || ch == '\t') { 471 // ignore 472 } 473 474 if(header.length) 475 header = header[1 .. $]; 476 } 477 478 ret ~= current; 479 480 linksStored = ret; 481 482 return ret; 483 } 484 } 485 486 /+ 487 headerName MUST be all lower case and NOT have the colon on it 488 489 returns slice of the input thing after the header name 490 +/ 491 private inout(char)[] isHttpHeader(inout(char)[] thing, const(char)[] headerName) { 492 foreach(idx, ch; thing) { 493 if(idx < headerName.length) { 494 if(headerName[idx] == '-' && ch != '-') 495 return null; 496 if((ch | ' ') != headerName[idx]) 497 return null; 498 } else if(idx == headerName.length) { 499 if(ch != ':') 500 return null; 501 } else { 502 return thing[idx .. $].strip; 503 } 504 } 505 return null; 506 } 507 508 private string idup_if_needed(string s) { return s; } 509 private string idup_if_needed(const(char)[] s) { return s.idup; } 510 511 unittest { 512 assert("Cookie: foo=bar".isHttpHeader("cookie") == "foo=bar"); 513 assert("cookie: foo=bar".isHttpHeader("cookie") == "foo=bar"); 514 assert("cOOkie: foo=bar".isHttpHeader("cookie") == "foo=bar"); 515 assert("Set-Cookie: foo=bar".isHttpHeader("set-cookie") == "foo=bar"); 516 assert(!"".isHttpHeader("cookie")); 517 } 518 519 /// 520 struct LinkHeader { 521 string url; /// 522 string rel; /// 523 string[string] attributes; /// like title, rev, media, whatever attributes 524 } 525 526 /++ 527 History: 528 Added July 5, 2021 529 +/ 530 struct CookieHeader { 531 string name; 532 string value; 533 string[string] attributes; 534 } 535 536 import std.string; 537 static import std.algorithm; 538 import std.conv; 539 import std.range; 540 541 542 private AddressFamily family(string unixSocketPath) { 543 if(unixSocketPath.length) 544 return AddressFamily.UNIX; 545 else // FIXME: what about ipv6? 546 return AddressFamily.INET; 547 } 548 549 version(Windows) 550 private class UnixAddress : Address { 551 this(string) { 552 throw new Exception("No unix address support on this system in lib yet :("); 553 } 554 override sockaddr* name() { assert(0); } 555 override const(sockaddr)* name() const { assert(0); } 556 override int nameLen() const { assert(0); } 557 } 558 559 560 // Copy pasta from cgi.d, then stripped down. unix path thing added tho 561 /// 562 struct Uri { 563 alias toString this; // blargh idk a url really is a string, but should it be implicit? 564 565 // scheme//userinfo@host:port/path?query#fragment 566 567 string scheme; /// e.g. "http" in "http://example.com/" 568 string userinfo; /// the username (and possibly a password) in the uri 569 string host; /// the domain name 570 int port; /// port number, if given. Will be zero if a port was not explicitly given 571 string path; /// e.g. "/folder/file.html" in "http://example.com/folder/file.html" 572 string query; /// the stuff after the ? in a uri 573 string fragment; /// the stuff after the # in a uri. 574 575 /// Breaks down a uri string to its components 576 this(string uri) { 577 reparse(uri); 578 } 579 580 private string unixSocketPath = null; 581 /// Indicates it should be accessed through a unix socket instead of regular tcp. Returns new version without modifying this object. 582 Uri viaUnixSocket(string path) const { 583 Uri copy = this; 584 copy.unixSocketPath = path; 585 return copy; 586 } 587 588 /// Goes through a unix socket in the abstract namespace (linux only). Returns new version without modifying this object. 589 version(linux) 590 Uri viaAbstractSocket(string path) const { 591 Uri copy = this; 592 copy.unixSocketPath = "\0" ~ path; 593 return copy; 594 } 595 596 private void reparse(string uri) { 597 // from RFC 3986 598 // the ctRegex triples the compile time and makes ugly errors for no real benefit 599 // it was a nice experiment but just not worth it. 600 // enum ctr = ctRegex!r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?"; 601 /* 602 Captures: 603 0 = whole url 604 1 = scheme, with : 605 2 = scheme, no : 606 3 = authority, with // 607 4 = authority, no // 608 5 = path 609 6 = query string, with ? 610 7 = query string, no ? 611 8 = anchor, with # 612 9 = anchor, no # 613 */ 614 // Yikes, even regular, non-CT regex is also unacceptably slow to compile. 1.9s on my computer! 615 // instead, I will DIY and cut that down to 0.6s on the same computer. 616 /* 617 618 Note that authority is 619 user:password@domain:port 620 where the user:password@ part is optional, and the :port is optional. 621 622 Regex translation: 623 624 Scheme cannot have :, /, ?, or # in it, and must have one or more chars and end in a :. It is optional, but must be first. 625 Authority must start with //, but cannot have any other /, ?, or # in it. It is optional. 626 Path cannot have any ? or # in it. It is optional. 627 Query must start with ? and must not have # in it. It is optional. 628 Anchor must start with # and can have anything else in it to end of string. It is optional. 629 */ 630 631 this = Uri.init; // reset all state 632 633 // empty uri = nothing special 634 if(uri.length == 0) { 635 return; 636 } 637 638 size_t idx; 639 640 scheme_loop: foreach(char c; uri[idx .. $]) { 641 switch(c) { 642 case ':': 643 case '/': 644 case '?': 645 case '#': 646 break scheme_loop; 647 default: 648 } 649 idx++; 650 } 651 652 if(idx == 0 && uri[idx] == ':') { 653 // this is actually a path! we skip way ahead 654 goto path_loop; 655 } 656 657 if(idx == uri.length) { 658 // the whole thing is a path, apparently 659 path = uri; 660 return; 661 } 662 663 if(idx > 0 && uri[idx] == ':') { 664 scheme = uri[0 .. idx]; 665 idx++; 666 } else { 667 // we need to rewind; it found a / but no :, so the whole thing is prolly a path... 668 idx = 0; 669 } 670 671 if(idx + 2 < uri.length && uri[idx .. idx + 2] == "//") { 672 // we have an authority.... 673 idx += 2; 674 675 auto authority_start = idx; 676 authority_loop: foreach(char c; uri[idx .. $]) { 677 switch(c) { 678 case '/': 679 case '?': 680 case '#': 681 break authority_loop; 682 default: 683 } 684 idx++; 685 } 686 687 auto authority = uri[authority_start .. idx]; 688 689 auto idx2 = authority.indexOf("@"); 690 if(idx2 != -1) { 691 userinfo = authority[0 .. idx2]; 692 authority = authority[idx2 + 1 .. $]; 693 } 694 695 if(authority.length && authority[0] == '[') { 696 // ipv6 address special casing 697 idx2 = authority.indexOf(']'); 698 if(idx2 != -1) { 699 auto end = authority[idx2 + 1 .. $]; 700 if(end.length && end[0] == ':') 701 idx2 = idx2 + 1; 702 else 703 idx2 = -1; 704 } 705 } else { 706 idx2 = authority.indexOf(":"); 707 } 708 709 if(idx2 == -1) { 710 port = 0; // 0 means not specified; we should use the default for the scheme 711 host = authority; 712 } else { 713 host = authority[0 .. idx2]; 714 port = to!int(authority[idx2 + 1 .. $]); 715 } 716 } 717 718 path_loop: 719 auto path_start = idx; 720 721 foreach(char c; uri[idx .. $]) { 722 if(c == '?' || c == '#') 723 break; 724 idx++; 725 } 726 727 path = uri[path_start .. idx]; 728 729 if(idx == uri.length) 730 return; // nothing more to examine... 731 732 if(uri[idx] == '?') { 733 idx++; 734 auto query_start = idx; 735 foreach(char c; uri[idx .. $]) { 736 if(c == '#') 737 break; 738 idx++; 739 } 740 query = uri[query_start .. idx]; 741 } 742 743 if(idx < uri.length && uri[idx] == '#') { 744 idx++; 745 fragment = uri[idx .. $]; 746 } 747 748 // uriInvalidated = false; 749 } 750 751 private string rebuildUri() const { 752 string ret; 753 if(scheme.length) 754 ret ~= scheme ~ ":"; 755 if(userinfo.length || host.length) 756 ret ~= "//"; 757 if(userinfo.length) 758 ret ~= userinfo ~ "@"; 759 if(host.length) 760 ret ~= host; 761 if(port) 762 ret ~= ":" ~ to!string(port); 763 764 ret ~= path; 765 766 if(query.length) 767 ret ~= "?" ~ query; 768 769 if(fragment.length) 770 ret ~= "#" ~ fragment; 771 772 // uri = ret; 773 // uriInvalidated = false; 774 return ret; 775 } 776 777 /// Converts the broken down parts back into a complete string 778 string toString() const { 779 // if(uriInvalidated) 780 return rebuildUri(); 781 } 782 783 /// Returns a new absolute Uri given a base. It treats this one as 784 /// relative where possible, but absolute if not. (If protocol, domain, or 785 /// other info is not set, the new one inherits it from the base.) 786 /// 787 /// Browsers use a function like this to figure out links in html. 788 Uri basedOn(in Uri baseUrl) const { 789 Uri n = this; // copies 790 // n.uriInvalidated = true; // make sure we regenerate... 791 792 // userinfo is not inherited... is this wrong? 793 794 // if anything is given in the existing url, we don't use the base anymore. 795 if(n.scheme.empty) { 796 n.scheme = baseUrl.scheme; 797 if(n.host.empty) { 798 n.host = baseUrl.host; 799 if(n.port == 0) { 800 n.port = baseUrl.port; 801 if(n.path.length > 0 && n.path[0] != '/') { 802 auto b = baseUrl.path[0 .. baseUrl.path.lastIndexOf("/") + 1]; 803 if(b.length == 0) 804 b = "/"; 805 n.path = b ~ n.path; 806 } else if(n.path.length == 0) { 807 n.path = baseUrl.path; 808 } 809 } 810 } 811 } 812 813 n.removeDots(); 814 815 // if still basically talking to the same thing, we should inherit the unix path 816 // too since basically the unix path is saying for this service, always use this override. 817 if(n.host == baseUrl.host && n.scheme == baseUrl.scheme && n.port == baseUrl.port) 818 n.unixSocketPath = baseUrl.unixSocketPath; 819 820 return n; 821 } 822 823 void removeDots() { 824 auto parts = this.path.split("/"); 825 string[] toKeep; 826 foreach(part; parts) { 827 if(part == ".") { 828 continue; 829 } else if(part == "..") { 830 //if(toKeep.length > 1) 831 toKeep = toKeep[0 .. $-1]; 832 //else 833 //toKeep = [""]; 834 continue; 835 } else { 836 //if(toKeep.length && toKeep[$-1].length == 0 && part.length == 0) 837 //continue; // skip a `//` situation 838 toKeep ~= part; 839 } 840 } 841 842 auto path = toKeep.join("/"); 843 if(path.length && path[0] != '/') 844 path = "/" ~ path; 845 846 this.path = path; 847 } 848 } 849 850 /* 851 void main(string args[]) { 852 write(post("http://arsdnet.net/bugs.php", ["test" : "hey", "again" : "what"])); 853 } 854 */ 855 856 /// 857 struct BasicAuth { 858 string username; /// 859 string password; /// 860 } 861 862 class ProxyException : Exception { 863 this(string msg) {super(msg); } 864 } 865 866 /** 867 Represents a HTTP request. You usually create these through a [HttpClient]. 868 869 870 --- 871 auto request = new HttpRequest(); // note that when there's no associated client, some features may not work 872 // normally you'd instead do `new HttpClient(); client.request(...)` 873 // set any properties here 874 875 // synchronous usage 876 auto reply = request.perform(); 877 878 // async usage, type 1: 879 request.send(); 880 request2.send(); 881 882 // wait until the first one is done, with the second one still in-flight 883 auto response = request.waitForCompletion(); 884 885 // async usage, type 2: 886 request.onDataReceived = (HttpRequest hr) { 887 if(hr.state == HttpRequest.State.complete) { 888 // use hr.responseData 889 } 890 }; 891 request.send(); // send, using the callback 892 893 // before terminating, be sure you wait for your requests to finish! 894 895 request.waitForCompletion(); 896 --- 897 */ 898 class HttpRequest { 899 900 /// Automatically follow a redirection? 901 bool followLocation = false; 902 903 /++ 904 Set to `true` to automatically retain cookies in the associated [HttpClient] from this request. 905 Note that you must have constructed the request from a `HttpClient` or at least passed one into the 906 constructor for this to have any effect. 907 908 Bugs: 909 See [HttpClient.retainCookies] for important caveats. 910 911 History: 912 Added July 5, 2021 (dub v10.2) 913 +/ 914 bool retainCookies = false; 915 916 private HttpClient client; 917 918 this() { 919 } 920 921 /// 922 this(HttpClient client, Uri where, HttpVerb method, ICache cache = null, Duration timeout = 10.seconds, string proxy = null) { 923 this.client = client; 924 populateFromInfo(where, method); 925 setTimeout(timeout); 926 this.cache = cache; 927 this.proxy = proxy; 928 } 929 930 931 /// ditto 932 this(Uri where, HttpVerb method, ICache cache = null, Duration timeout = 10.seconds, string proxy = null) { 933 this(null, where, method, cache, timeout, proxy); 934 } 935 936 /++ 937 Sets the timeout from inactivity on the request. This is the amount of time that passes with no send or receive activity on the request before it fails with "request timed out" error. 938 939 History: 940 Added March 31, 2021 941 +/ 942 void setTimeout(Duration timeout) { 943 this.requestParameters.timeoutFromInactivity = timeout; 944 this.timeoutFromInactivity = MonoTime.currTime + this.requestParameters.timeoutFromInactivity; 945 } 946 947 private MonoTime timeoutFromInactivity; 948 949 private Uri where; 950 951 private ICache cache; 952 953 /++ 954 Proxy to use for this request. It should be a URL or `null`. 955 956 This must be sent before you call `send`. 957 958 History: 959 Added April 12, 2021 (dub v9.5) 960 +/ 961 string proxy; 962 963 /// Final url after any redirections 964 string finalUrl; 965 966 void populateFromInfo(Uri where, HttpVerb method) { 967 auto parts = where.basedOn(this.where); 968 this.where = parts; 969 finalUrl = where.toString(); 970 requestParameters.method = method; 971 requestParameters.unixSocketPath = where.unixSocketPath; 972 requestParameters.host = parts.host; 973 requestParameters.port = cast(ushort) parts.port; 974 requestParameters.ssl = parts.scheme == "https"; 975 if(parts.port == 0) 976 requestParameters.port = requestParameters.ssl ? 443 : 80; 977 requestParameters.uri = parts.path.length ? parts.path : "/"; 978 if(parts.query.length) { 979 requestParameters.uri ~= "?"; 980 requestParameters.uri ~= parts.query; 981 } 982 } 983 984 ~this() { 985 } 986 987 ubyte[] sendBuffer; 988 989 HttpResponse responseData; 990 private HttpClient parentClient; 991 992 size_t bodyBytesSent; 993 size_t bodyBytesReceived; 994 995 State state_; 996 State state() { return state_; } 997 State state(State s) { 998 assert(state_ != State.complete); 999 return state_ = s; 1000 } 1001 /// Called when data is received. Check the state to see what data is available. 1002 void delegate(HttpRequest) onDataReceived; 1003 1004 enum State { 1005 /// The request has not yet been sent 1006 unsent, 1007 1008 /// The send() method has been called, but no data is 1009 /// sent on the socket yet because the connection is busy. 1010 pendingAvailableConnection, 1011 1012 /// The headers are being sent now 1013 sendingHeaders, 1014 1015 /// The body is being sent now 1016 sendingBody, 1017 1018 /// The request has been sent but we haven't received any response yet 1019 waitingForResponse, 1020 1021 /// We have received some data and are currently receiving headers 1022 readingHeaders, 1023 1024 /// All headers are available but we're still waiting on the body 1025 readingBody, 1026 1027 /// The request is complete. 1028 complete, 1029 1030 /// The request is aborted, either by the abort() method, or as a result of the server disconnecting 1031 aborted 1032 } 1033 1034 /// Sends now and waits for the request to finish, returning the response. 1035 HttpResponse perform() { 1036 send(); 1037 return waitForCompletion(); 1038 } 1039 1040 /// Sends the request asynchronously. 1041 void send() { 1042 sendPrivate(true); 1043 } 1044 1045 private void sendPrivate(bool advance) { 1046 if(state != State.unsent && state != State.aborted) 1047 return; // already sent 1048 1049 if(cache !is null) { 1050 auto res = cache.getCachedResponse(this.requestParameters); 1051 if(res !is null) { 1052 state = State.complete; 1053 responseData = (*res).deepCopy(); 1054 return; 1055 } 1056 } 1057 1058 string headers; 1059 1060 headers ~= to!string(requestParameters.method); 1061 headers ~= " "; 1062 if(proxy.length && !requestParameters.ssl) { 1063 // if we're doing a http proxy, we need to send a complete, absolute uri 1064 // so reconstruct it 1065 headers ~= "http://"; 1066 headers ~= requestParameters.host; 1067 if(requestParameters.port != 80) { 1068 headers ~= ":"; 1069 headers ~= to!string(requestParameters.port); 1070 } 1071 } 1072 1073 headers ~= requestParameters.uri; 1074 1075 if(requestParameters.useHttp11) 1076 headers ~= " HTTP/1.1\r\n"; 1077 else 1078 headers ~= " HTTP/1.0\r\n"; 1079 1080 // the whole authority section is supposed to be there, but curl doesn't send if default port 1081 // so I'll copy what they do 1082 headers ~= "Host: "; 1083 headers ~= requestParameters.host; 1084 if(requestParameters.port != 80 && requestParameters.port != 443) { 1085 headers ~= ":"; 1086 headers ~= to!string(requestParameters.port); 1087 } 1088 headers ~= "\r\n"; 1089 1090 if(requestParameters.userAgent.length) 1091 headers ~= "User-Agent: "~requestParameters.userAgent~"\r\n"; 1092 if(requestParameters.contentType.length) 1093 headers ~= "Content-Type: "~requestParameters.contentType~"\r\n"; 1094 if(requestParameters.authorization.length) 1095 headers ~= "Authorization: "~requestParameters.authorization~"\r\n"; 1096 if(requestParameters.bodyData.length) 1097 headers ~= "Content-Length: "~to!string(requestParameters.bodyData.length)~"\r\n"; 1098 if(requestParameters.acceptGzip) 1099 headers ~= "Accept-Encoding: gzip\r\n"; 1100 if(requestParameters.keepAlive) 1101 headers ~= "Connection: keep-alive\r\n"; 1102 1103 string cookieHeader; 1104 foreach(name, value; requestParameters.cookies) { 1105 if(cookieHeader is null) 1106 cookieHeader = "Cookie: "; 1107 else 1108 cookieHeader ~= "; "; 1109 cookieHeader ~= name; 1110 cookieHeader ~= "="; 1111 cookieHeader ~= value; 1112 } 1113 1114 if(cookieHeader !is null) { 1115 cookieHeader ~= "\r\n"; 1116 headers ~= cookieHeader; 1117 } 1118 1119 foreach(header; requestParameters.headers) 1120 headers ~= header ~ "\r\n"; 1121 1122 headers ~= "\r\n"; 1123 1124 sendBuffer = cast(ubyte[]) headers ~ requestParameters.bodyData; 1125 1126 // import std.stdio; writeln("******* ", sendBuffer); 1127 1128 responseData = HttpResponse.init; 1129 responseData.requestParameters = requestParameters; 1130 bodyBytesSent = 0; 1131 bodyBytesReceived = 0; 1132 state = State.pendingAvailableConnection; 1133 1134 bool alreadyPending = false; 1135 foreach(req; pending) 1136 if(req is this) { 1137 alreadyPending = true; 1138 break; 1139 } 1140 if(!alreadyPending) { 1141 pending ~= this; 1142 } 1143 1144 if(advance) 1145 HttpRequest.advanceConnections(requestParameters.timeoutFromInactivity); 1146 } 1147 1148 1149 /// Waits for the request to finish or timeout, whichever comes first. 1150 HttpResponse waitForCompletion() { 1151 while(state != State.aborted && state != State.complete) { 1152 if(state == State.unsent) { 1153 send(); 1154 continue; 1155 } 1156 if(auto err = HttpRequest.advanceConnections(requestParameters.timeoutFromInactivity)) { 1157 switch(err) { 1158 case 1: throw new Exception("HttpRequest.advanceConnections returned 1: all connections timed out"); 1159 case 2: throw new Exception("HttpRequest.advanceConnections returned 2: nothing to do"); 1160 case 3: continue; // EINTR 1161 default: throw new Exception("HttpRequest.advanceConnections got err " ~ to!string(err)); 1162 } 1163 } 1164 } 1165 1166 if(state == State.complete && responseData.code >= 200) 1167 if(cache !is null) 1168 cache.cacheResponse(this.requestParameters, this.responseData); 1169 1170 return responseData; 1171 } 1172 1173 /// Aborts this request. 1174 void abort() { 1175 this.state = State.aborted; 1176 this.responseData.code = 1; 1177 this.responseData.codeText = "request.abort called"; 1178 // the actual cancellation happens in the event loop 1179 } 1180 1181 HttpRequestParameters requestParameters; /// 1182 1183 version(arsd_http_winhttp_implementation) { 1184 public static void resetInternals() { 1185 1186 } 1187 1188 static assert(0, "implementation not finished"); 1189 } 1190 1191 1192 version(arsd_http_internal_implementation) { 1193 private static { 1194 // we manage the actual connections. When a request is made on a particular 1195 // host, we try to reuse connections. We may open more than one connection per 1196 // host to do parallel requests. 1197 // 1198 // The key is the *domain name* and the port. Multiple domains on the same address will have separate connections. 1199 Socket[][string] socketsPerHost; 1200 1201 void loseSocket(string host, ushort port, bool ssl, Socket s) { 1202 import std.string; 1203 auto key = format("http%s://%s:%s", ssl ? "s" : "", host, port); 1204 1205 if(auto list = key in socketsPerHost) { 1206 for(int a = 0; a < (*list).length; a++) { 1207 if((*list)[a] is s) { 1208 1209 for(int b = a; b < (*list).length - 1; b++) 1210 (*list)[b] = (*list)[b+1]; 1211 (*list) = (*list)[0 .. $-1]; 1212 break; 1213 } 1214 } 1215 } 1216 } 1217 1218 Socket getOpenSocketOnHost(string proxy, string host, ushort port, bool ssl, string unixSocketPath) { 1219 Socket openNewConnection() { 1220 Socket socket; 1221 if(ssl) { 1222 version(with_openssl) { 1223 loadOpenSsl(); 1224 socket = new SslClientSocket(family(unixSocketPath), SocketType.STREAM, host, defaultVerifyPeer); 1225 } else 1226 throw new Exception("SSL not compiled in"); 1227 } else 1228 socket = new Socket(family(unixSocketPath), SocketType.STREAM); 1229 1230 // FIXME: connect timeout? 1231 if(unixSocketPath) { 1232 import std.stdio; writeln(cast(ubyte[]) unixSocketPath); 1233 socket.connect(new UnixAddress(unixSocketPath)); 1234 } else { 1235 // FIXME: i should prolly do ipv6 if available too. 1236 if(host.length == 0) // this could arguably also be an in contract since it is user error, but the exception is good enough 1237 throw new Exception("No host given for request"); 1238 if(proxy.length) { 1239 if(proxy.indexOf("//") == -1) 1240 proxy = "http://" ~ proxy; 1241 auto proxyurl = Uri(proxy); 1242 1243 //auto proxyhttps = proxyurl.scheme == "https"; 1244 enum proxyhttps = false; // this isn't properly implemented and might never be necessary anyway so meh 1245 1246 // the precise types here are important to help with overload 1247 // resolution of the devirtualized call! 1248 Address pa = new InternetAddress(proxyurl.host, proxyurl.port ? cast(ushort) proxyurl.port : 80); 1249 1250 debug(arsd_http2) writeln("using proxy ", pa.toString()); 1251 1252 if(proxyhttps) { 1253 socket.connect(pa); 1254 } else { 1255 // the proxy never actually starts TLS, but if the request is tls then we need to CONNECT then upgrade the connection 1256 // using the parent class functions let us bypass the encryption 1257 socket.Socket.connect(pa); 1258 } 1259 1260 string message; 1261 if(ssl) { 1262 auto hostName = host ~ ":" ~ to!string(port); 1263 message = "CONNECT " ~ hostName ~ " HTTP/1.1\r\n"; 1264 message ~= "Host: " ~ hostName ~ "\r\n"; 1265 if(proxyurl.userinfo.length) { 1266 import std.base64; 1267 message ~= "Proxy-Authorization: Basic " ~ Base64.encode(cast(ubyte[]) proxyurl.userinfo) ~ "\r\n"; 1268 } 1269 message ~= "\r\n"; 1270 1271 // FIXME: what if proxy times out? should be reasonably fast too. 1272 if(proxyhttps) { 1273 socket.send(message, SocketFlags.NONE); 1274 } else { 1275 socket.Socket.send(message, SocketFlags.NONE); 1276 } 1277 1278 ubyte[1024] recvBuffer; 1279 // and last time 1280 ptrdiff_t rcvGot; 1281 if(proxyhttps) { 1282 rcvGot = socket.receive(recvBuffer[], SocketFlags.NONE); 1283 // bool verifyPeer = true; 1284 //(cast(OpenSslSocket)socket).freeSsl(); 1285 //(cast(OpenSslSocket)socket).initSsl(verifyPeer, host); 1286 } else { 1287 rcvGot = socket.Socket.receive(recvBuffer[], SocketFlags.NONE); 1288 } 1289 1290 if(rcvGot == -1) 1291 throw new ProxyException("proxy receive error"); 1292 auto got = cast(string) recvBuffer[0 .. rcvGot]; 1293 auto expect = "HTTP/1.1 200"; 1294 if(got.length < expect.length || (got[0 .. expect.length] != expect && got[0 .. expect.length] != "HTTP/1.0 200")) 1295 throw new ProxyException("Proxy rejected request: " ~ got[0 .. expect.length <= got.length ? expect.length : got.length]); 1296 1297 if(proxyhttps) { 1298 //(cast(OpenSslSocket)socket).do_ssl_connect(); 1299 } else { 1300 (cast(OpenSslSocket)socket).do_ssl_connect(); 1301 } 1302 } else { 1303 } 1304 } else { 1305 socket.connect(new InternetAddress(host, port)); 1306 } 1307 } 1308 1309 debug(arsd_http2) writeln("opening to ", host, ":", port, " ", cast(void*) socket); 1310 assert(socket.handle() !is socket_t.init); 1311 return socket; 1312 } 1313 1314 import std.string; 1315 auto key = format("http%s://%s:%s", ssl ? "s" : "", host, port); 1316 1317 if(auto hostListing = key in socketsPerHost) { 1318 // try to find an available socket that is already open 1319 foreach(socket; *hostListing) { 1320 if(socket !in activeRequestOnSocket) { 1321 // let's see if it has closed since we last tried 1322 // e.g. a server timeout or something. If so, we need 1323 // to lose this one and immediately open a new one. 1324 static SocketSet readSet = null; 1325 if(readSet is null) 1326 readSet = new SocketSet(); 1327 readSet.reset(); 1328 assert(socket !is null); 1329 assert(socket.handle() !is socket_t.init, socket is null ? "null" : socket.toString()); 1330 readSet.add(socket); 1331 auto got = Socket.select(readSet, null, null, 5.msecs /* timeout */); 1332 if(got > 0) { 1333 // we can read something off this... but there aren't 1334 // any active requests. Assume it is EOF and open a new one 1335 1336 socket.close(); 1337 loseSocket(host, port, ssl, socket); 1338 goto openNew; 1339 } 1340 return socket; 1341 } 1342 } 1343 1344 // if not too many already open, go ahead and do a new one 1345 if((*hostListing).length < 6) { 1346 auto socket = openNewConnection(); 1347 (*hostListing) ~= socket; 1348 return socket; 1349 } else 1350 return null; // too many, you'll have to wait 1351 } 1352 1353 openNew: 1354 1355 auto socket = openNewConnection(); 1356 socketsPerHost[key] ~= socket; 1357 return socket; 1358 } 1359 1360 // only one request can be active on a given socket (at least HTTP < 2.0) so this is that 1361 HttpRequest[Socket] activeRequestOnSocket; 1362 HttpRequest[] pending; // and these are the requests that are waiting 1363 1364 SocketSet readSet; 1365 SocketSet writeSet; 1366 1367 /+ 1368 Generic event loop registration: 1369 1370 handle, operation (read/write), buffer (on posix it *might* be stack if a select loop), timeout (in real time), callback when op completed. 1371 1372 ....basically Windows style. Then it translates internally. 1373 1374 It should tell the thing if the buffer is reused or not 1375 +/ 1376 1377 1378 /++ 1379 This is made public for rudimentary event loop integration, but is still 1380 basically an internal detail. Try not to use it if you have another way. 1381 1382 This does a single iteration of the internal select()-based processing loop. 1383 1384 1385 Future directions: 1386 I want to merge the internal use of [WebSocket.eventLoop] with this; 1387 [advanceConnections] does just one run on the loop, whereas eventLoop 1388 runs it until all connections are closed. But they'd both process both 1389 pending http requests and active websockets. 1390 1391 After that, I want to be able to integrate in other event loops too. 1392 One might be to simply to reactor callbacks, then perhaps Windows overlapped 1393 i/o (that's just going to be tricky to retrofit into the existing select()-based 1394 code). It could then go fiber just by calling the resume function too. 1395 1396 The hard part is ensuring I keep this file stand-alone while offering these 1397 things. 1398 1399 This `advanceConnections` call will probably continue to work now that it is 1400 public, but it may not be wholly compatible with all the future features; you'd 1401 have to pick either the internal event loop or an external one you integrate, but not 1402 mix them. 1403 1404 History: 1405 This has been included in the library since almost day one, but 1406 it was private until April 13, 2021 (dub v9.5). 1407 1408 Params: 1409 maximumTimeout = the maximum time it will wait in select(). It may return much sooner than this if a connection timed out in the mean time. 1410 automaticallyRetryOnInterruption = internally loop on EINTR. 1411 1412 Returns: 1413 1414 0 = no error, work may remain so you should call `advanceConnections` again when you can 1415 1416 1 = passed `maximumTimeout` reached with no work done, yet requests are still in the queue. You may call `advanceConnections` again. 1417 1418 2 = no work to do, no point calling it again unless you've added new requests. Your program may exit if you have nothing to add since it means everything requested is now done. 1419 1420 3 = EINTR occurred on select(), you should check your interrupt flags if you set a signal handler, then call `advanceConnections` again if you aren't exiting. Only occurs if `automaticallyRetryOnInterruption` is set to `false` (the default when it is called externally). 1421 1422 any other value should be considered a non-recoverable error if you want to be forward compatible as I reserve the right to add more values later. 1423 +/ 1424 public int advanceConnections(Duration maximumTimeout = 10.seconds, bool automaticallyRetryOnInterruption = false) { 1425 if(readSet is null) 1426 readSet = new SocketSet(); 1427 if(writeSet is null) 1428 writeSet = new SocketSet(); 1429 1430 ubyte[2048] buffer; 1431 1432 HttpRequest[16] removeFromPending; 1433 size_t removeFromPendingCount = 0; 1434 1435 bool hadAbortedRequest; 1436 1437 // are there pending requests? let's try to send them 1438 foreach(idx, pc; pending) { 1439 if(removeFromPendingCount == removeFromPending.length) 1440 break; 1441 1442 if(pc.state == HttpRequest.State.aborted) { 1443 removeFromPending[removeFromPendingCount++] = pc; 1444 hadAbortedRequest = true; 1445 continue; 1446 } 1447 1448 Socket socket; 1449 1450 try { 1451 socket = getOpenSocketOnHost(pc.proxy, pc.requestParameters.host, pc.requestParameters.port, pc.requestParameters.ssl, pc.requestParameters.unixSocketPath); 1452 } catch(ProxyException e) { 1453 // connection refused or timed out (I should disambiguate somehow)... 1454 pc.state = HttpRequest.State.aborted; 1455 1456 pc.responseData.code = 2; 1457 pc.responseData.codeText = e.msg ~ " from " ~ pc.proxy; 1458 1459 hadAbortedRequest = true; 1460 1461 removeFromPending[removeFromPendingCount++] = pc; 1462 continue; 1463 1464 } catch(SocketException e) { 1465 // connection refused or timed out (I should disambiguate somehow)... 1466 pc.state = HttpRequest.State.aborted; 1467 1468 pc.responseData.code = 2; 1469 pc.responseData.codeText = pc.proxy.length ? ("connection failed to proxy " ~ pc.proxy) : "connection failed"; 1470 1471 hadAbortedRequest = true; 1472 1473 removeFromPending[removeFromPendingCount++] = pc; 1474 continue; 1475 } catch(Exception e) { 1476 // connection failed due to other user error 1477 pc.state = HttpRequest.State.aborted; 1478 1479 pc.responseData.code = 2; 1480 pc.responseData.codeText = e.msg; 1481 1482 hadAbortedRequest = true; 1483 1484 removeFromPending[removeFromPendingCount++] = pc; 1485 continue; 1486 1487 } 1488 1489 if(socket !is null) { 1490 activeRequestOnSocket[socket] = pc; 1491 assert(pc.sendBuffer.length); 1492 pc.state = State.sendingHeaders; 1493 1494 removeFromPending[removeFromPendingCount++] = pc; 1495 } 1496 } 1497 1498 import std.algorithm : remove; 1499 foreach(rp; removeFromPending[0 .. removeFromPendingCount]) 1500 pending = pending.remove!((a) => a is rp)(); 1501 1502 tryAgain: 1503 1504 Socket[16] inactive; 1505 int inactiveCount = 0; 1506 void killInactives() { 1507 foreach(s; inactive[0 .. inactiveCount]) { 1508 debug(arsd_http2) writeln("removing socket from active list ", cast(void*) s); 1509 activeRequestOnSocket.remove(s); 1510 } 1511 } 1512 1513 1514 readSet.reset(); 1515 writeSet.reset(); 1516 1517 bool hadOne = false; 1518 1519 auto minTimeout = maximumTimeout; 1520 auto now = MonoTime.currTime; 1521 1522 // active requests need to be read or written to 1523 foreach(sock, request; activeRequestOnSocket) { 1524 1525 if(request.state == State.aborted) { 1526 inactive[inactiveCount++] = sock; 1527 sock.close(); 1528 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1529 hadAbortedRequest = true; 1530 continue; 1531 } 1532 1533 // check the other sockets just for EOF, if they close, take them out of our list, 1534 // we'll reopen if needed upon request. 1535 readSet.add(sock); 1536 hadOne = true; 1537 1538 Duration timeo; 1539 if(request.timeoutFromInactivity <= now) 1540 timeo = 0.seconds; 1541 else 1542 timeo = request.timeoutFromInactivity - now; 1543 1544 if(timeo < minTimeout) 1545 minTimeout = timeo; 1546 1547 if(request.state == State.sendingHeaders || request.state == State.sendingBody) { 1548 writeSet.add(sock); 1549 hadOne = true; 1550 } 1551 } 1552 1553 if(!hadOne) { 1554 if(hadAbortedRequest) { 1555 killInactives(); 1556 return 0; // something got aborted, that's progress 1557 } 1558 return 2; // automatic timeout, nothing to do 1559 } 1560 1561 auto selectGot = Socket.select(readSet, writeSet, null, minTimeout); 1562 if(selectGot == 0) { /* timeout */ 1563 now = MonoTime.currTime; 1564 bool anyWorkDone = false; 1565 foreach(sock, request; activeRequestOnSocket) { 1566 1567 if(request.timeoutFromInactivity <= now) { 1568 request.state = HttpRequest.State.aborted; 1569 request.responseData.code = 5; 1570 request.responseData.codeText = "Request timed out"; 1571 1572 inactive[inactiveCount++] = sock; 1573 sock.close(); 1574 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1575 anyWorkDone = true; 1576 } 1577 } 1578 killInactives(); 1579 return anyWorkDone ? 0 : 1; 1580 // return 1; was an error to time out but now im making it on the individual request 1581 } else if(selectGot == -1) { /* interrupted */ 1582 /* 1583 version(Posix) { 1584 import core.stdc.errno; 1585 if(errno != EINTR) 1586 throw new Exception("select error: " ~ to!string(errno)); 1587 } 1588 */ 1589 if(automaticallyRetryOnInterruption) 1590 goto tryAgain; 1591 else 1592 return 3; 1593 } else { /* ready */ 1594 foreach(sock, request; activeRequestOnSocket) { 1595 if(readSet.isSet(sock)) { 1596 keep_going: 1597 request.timeoutFromInactivity = MonoTime.currTime + request.requestParameters.timeoutFromInactivity; 1598 auto got = sock.receive(buffer); 1599 debug(arsd_http2_verbose) writeln("====PACKET ",got,"=====",cast(string)buffer[0 .. got],"===/PACKET==="); 1600 if(got < 0) { 1601 throw new Exception("receive error"); 1602 } else if(got == 0) { 1603 // remote side disconnected 1604 debug(arsd_http2) writeln("remote disconnect"); 1605 if(request.state != State.complete) { 1606 request.state = State.aborted; 1607 1608 request.responseData.code = 3; 1609 request.responseData.codeText = "server disconnected"; 1610 } 1611 inactive[inactiveCount++] = sock; 1612 sock.close(); 1613 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1614 } else { 1615 // data available 1616 bool stillAlive; 1617 1618 try { 1619 stillAlive = request.handleIncomingData(buffer[0 .. got]); 1620 } catch (Exception e) { 1621 request.state = HttpRequest.State.aborted; 1622 request.responseData.code = 4; 1623 request.responseData.codeText = e.msg; 1624 1625 inactive[inactiveCount++] = sock; 1626 sock.close(); 1627 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1628 continue; 1629 } 1630 1631 if(!stillAlive || request.state == HttpRequest.State.complete || request.state == HttpRequest.State.aborted) { 1632 //import std.stdio; writeln(cast(void*) sock, " ", stillAlive, " ", request.state); 1633 inactive[inactiveCount++] = sock; 1634 continue; 1635 // reuse the socket for another pending request, if we can 1636 } 1637 } 1638 1639 if(request.onDataReceived) 1640 request.onDataReceived(request); 1641 1642 version(with_openssl) 1643 if(auto s = cast(SslClientSocket) sock) { 1644 // select doesn't handle the case with stuff 1645 // left in the ssl buffer so i'm checking it separately 1646 if(s.dataPending()) { 1647 goto keep_going; 1648 } 1649 } 1650 } 1651 1652 if(request.state == State.sendingHeaders || request.state == State.sendingBody) 1653 if(writeSet.isSet(sock)) { 1654 request.timeoutFromInactivity = MonoTime.currTime + request.requestParameters.timeoutFromInactivity; 1655 assert(request.sendBuffer.length); 1656 auto sent = sock.send(request.sendBuffer); 1657 debug(arsd_http2_verbose) writeln(cast(void*) sock, "<send>", cast(string) request.sendBuffer, "</send>"); 1658 if(sent <= 0) 1659 throw new Exception("send error " ~ lastSocketError); 1660 request.sendBuffer = request.sendBuffer[sent .. $]; 1661 if(request.sendBuffer.length == 0) { 1662 request.state = State.waitingForResponse; 1663 } 1664 } 1665 } 1666 } 1667 1668 killInactives(); 1669 1670 // we've completed a request, are there any more pending connection? if so, send them now 1671 1672 return 0; 1673 } 1674 } 1675 1676 public static void resetInternals() { 1677 socketsPerHost = null; 1678 activeRequestOnSocket = null; 1679 pending = null; 1680 1681 } 1682 1683 struct HeaderReadingState { 1684 bool justSawLf; 1685 bool justSawCr; 1686 bool atStartOfLine = true; 1687 bool readingLineContinuation; 1688 } 1689 HeaderReadingState headerReadingState; 1690 1691 struct BodyReadingState { 1692 bool isGzipped; 1693 bool isDeflated; 1694 1695 bool isChunked; 1696 int chunkedState; 1697 1698 // used for the chunk size if it is chunked 1699 int contentLengthRemaining; 1700 } 1701 BodyReadingState bodyReadingState; 1702 1703 bool closeSocketWhenComplete; 1704 1705 import std.zlib; 1706 UnCompress uncompress; 1707 1708 const(ubyte)[] leftoverDataFromLastTime; 1709 1710 bool handleIncomingData(scope const ubyte[] dataIn) { 1711 bool stillAlive = true; 1712 debug(arsd_http2) writeln("handleIncomingData, state: ", state); 1713 if(state == State.waitingForResponse) { 1714 state = State.readingHeaders; 1715 headerReadingState = HeaderReadingState.init; 1716 bodyReadingState = BodyReadingState.init; 1717 } 1718 1719 const(ubyte)[] data; 1720 if(leftoverDataFromLastTime.length) 1721 data = leftoverDataFromLastTime ~ dataIn[]; 1722 else 1723 data = dataIn[]; 1724 1725 if(state == State.readingHeaders) { 1726 void parseLastHeader() { 1727 assert(responseData.headers.length); 1728 if(responseData.headers.length == 1) { 1729 responseData.statusLine = responseData.headers[0]; 1730 import std.algorithm; 1731 auto parts = responseData.statusLine.splitter(" "); 1732 responseData.httpVersion = parts.front; 1733 parts.popFront(); 1734 if(parts.empty) 1735 throw new Exception("Corrupted response, bad status line"); 1736 responseData.code = to!int(parts.front()); 1737 parts.popFront(); 1738 responseData.codeText = ""; 1739 while(!parts.empty) { 1740 // FIXME: this sucks! 1741 responseData.codeText ~= parts.front(); 1742 parts.popFront(); 1743 if(!parts.empty) 1744 responseData.codeText ~= " "; 1745 } 1746 } else { 1747 // parse the new header 1748 auto header = responseData.headers[$-1]; 1749 1750 auto colon = header.indexOf(":"); 1751 if(colon == -1) 1752 return; 1753 auto name = header[0 .. colon]; 1754 if(colon + 1 == header.length || colon + 2 == header.length) // assuming a space there 1755 return; // empty header, idk 1756 assert(colon + 2 < header.length, header); 1757 auto value = header[colon + 2 .. $]; // skipping the colon itself and the following space 1758 1759 switch(name) { 1760 case "Connection": 1761 case "connection": 1762 if(value == "close") 1763 closeSocketWhenComplete = true; 1764 break; 1765 case "Content-Type": 1766 case "content-type": 1767 responseData.contentType = value; 1768 break; 1769 case "Location": 1770 case "location": 1771 responseData.location = value; 1772 break; 1773 case "Content-Length": 1774 case "content-length": 1775 bodyReadingState.contentLengthRemaining = to!int(value); 1776 break; 1777 case "Transfer-Encoding": 1778 case "transfer-encoding": 1779 // note that if it is gzipped, it zips first, then chunks the compressed stream. 1780 // so we should always dechunk first, then feed into the decompressor 1781 if(value.strip == "chunked") 1782 bodyReadingState.isChunked = true; 1783 else throw new Exception("Unknown Transfer-Encoding: " ~ value); 1784 break; 1785 case "Content-Encoding": 1786 case "content-encoding": 1787 if(value == "gzip") { 1788 bodyReadingState.isGzipped = true; 1789 uncompress = new UnCompress(); 1790 } else if(value == "deflate") { 1791 bodyReadingState.isDeflated = true; 1792 uncompress = new UnCompress(); 1793 } else throw new Exception("Unknown Content-Encoding: " ~ value); 1794 break; 1795 case "Set-Cookie": 1796 case "set-cookie": 1797 // handled elsewhere fyi 1798 break; 1799 default: 1800 // ignore 1801 } 1802 1803 responseData.headersHash[name] = value; 1804 } 1805 } 1806 1807 size_t position = 0; 1808 for(position = 0; position < data.length; position++) { 1809 if(headerReadingState.readingLineContinuation) { 1810 if(data[position] == ' ' || data[position] == '\t') 1811 continue; 1812 headerReadingState.readingLineContinuation = false; 1813 } 1814 1815 if(headerReadingState.atStartOfLine) { 1816 headerReadingState.atStartOfLine = false; 1817 if(data[position] == '\r' || data[position] == '\n') { 1818 // done with headers 1819 if(data[position] == '\r' && (position + 1) < data.length && data[position + 1] == '\n') 1820 position++; 1821 if(responseData.headers.length && responseData.headers[0].indexOf(" 100 ") != -1) { 1822 // HTTP/1.1 100 Continue 1823 // here we just discard the continue message and carry on; it is just informational anyway 1824 // it arguably should be smarter though 1825 responseData.headers = null; 1826 headerReadingState.atStartOfLine = true; 1827 1828 continue; 1829 } else { 1830 if(this.requestParameters.method == HttpVerb.HEAD) 1831 state = State.complete; 1832 else 1833 state = State.readingBody; 1834 } 1835 position++; // skip the newline 1836 break; 1837 } else if(data[position] == ' ' || data[position] == '\t') { 1838 // line continuation, ignore all whitespace and collapse it into a space 1839 headerReadingState.readingLineContinuation = true; 1840 responseData.headers[$-1] ~= ' '; 1841 } else { 1842 // new header 1843 if(responseData.headers.length) 1844 parseLastHeader(); 1845 responseData.headers ~= ""; 1846 } 1847 } 1848 1849 if(data[position] == '\r') { 1850 headerReadingState.justSawCr = true; 1851 continue; 1852 } else 1853 headerReadingState.justSawCr = false; 1854 1855 if(data[position] == '\n') { 1856 headerReadingState.justSawLf = true; 1857 headerReadingState.atStartOfLine = true; 1858 continue; 1859 } else 1860 headerReadingState.justSawLf = false; 1861 1862 responseData.headers[$-1] ~= data[position]; 1863 } 1864 1865 if(responseData.headers.length) 1866 parseLastHeader(); 1867 data = data[position .. $]; 1868 } 1869 1870 if(state == State.readingBody) { 1871 if(bodyReadingState.isChunked) { 1872 // read the hex length, stopping at a \r\n, ignoring everything between the new line but after the first non-valid hex character 1873 // read binary data of that length. it is our content 1874 // repeat until a zero sized chunk 1875 // then read footers as headers. 1876 1877 start_over: 1878 for(int a = 0; a < data.length; a++) { 1879 final switch(bodyReadingState.chunkedState) { 1880 case 0: // reading hex 1881 char c = data[a]; 1882 if((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) { 1883 // just keep reading 1884 } else { 1885 int power = 1; 1886 bodyReadingState.contentLengthRemaining = 0; 1887 assert(a != 0, cast(string) data); 1888 for(int b = a-1; b >= 0; b--) { 1889 char cc = data[b]; 1890 if(cc >= 'a' && cc <= 'z') 1891 cc -= 0x20; 1892 int val = 0; 1893 if(cc >= '0' && cc <= '9') 1894 val = cc - '0'; 1895 else 1896 val = cc - 'A' + 10; 1897 1898 assert(val >= 0 && val <= 15, to!string(val)); 1899 bodyReadingState.contentLengthRemaining += power * val; 1900 power *= 16; 1901 } 1902 debug(arsd_http2_verbose) writeln("Chunk length: ", bodyReadingState.contentLengthRemaining); 1903 bodyReadingState.chunkedState = 1; 1904 data = data[a + 1 .. $]; 1905 goto start_over; 1906 } 1907 break; 1908 case 1: // reading until end of line 1909 char c = data[a]; 1910 if(c == '\n') { 1911 if(bodyReadingState.contentLengthRemaining == 0) 1912 bodyReadingState.chunkedState = 5; 1913 else 1914 bodyReadingState.chunkedState = 2; 1915 } 1916 data = data[a + 1 .. $]; 1917 goto start_over; 1918 case 2: // reading data 1919 auto can = a + bodyReadingState.contentLengthRemaining; 1920 if(can > data.length) 1921 can = cast(int) data.length; 1922 1923 auto newData = data[a .. can]; 1924 data = data[can .. $]; 1925 1926 //if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) 1927 // responseData.content ~= cast(ubyte[]) uncompress.uncompress(data[a .. can]); 1928 //else 1929 responseData.content ~= newData; 1930 1931 bodyReadingState.contentLengthRemaining -= newData.length; 1932 debug(arsd_http2_verbose) writeln("clr: ", bodyReadingState.contentLengthRemaining, " " , a, " ", can); 1933 assert(bodyReadingState.contentLengthRemaining >= 0); 1934 if(bodyReadingState.contentLengthRemaining == 0) { 1935 bodyReadingState.chunkedState = 3; 1936 } else { 1937 // will continue grabbing more 1938 } 1939 goto start_over; 1940 case 3: // reading 13/10 1941 assert(data[a] == 13); 1942 bodyReadingState.chunkedState++; 1943 data = data[a + 1 .. $]; 1944 goto start_over; 1945 case 4: // reading 10 at end of packet 1946 assert(data[a] == 10); 1947 data = data[a + 1 .. $]; 1948 bodyReadingState.chunkedState = 0; 1949 goto start_over; 1950 case 5: // reading footers 1951 //goto done; // FIXME 1952 1953 while(data[a] != 10) { 1954 a++; 1955 if(a == data.length) 1956 return stillAlive; // in the footer state we're just discarding everything until we're done so this should be ok 1957 } 1958 1959 bodyReadingState.chunkedState = 0; 1960 data = data[a + 1 .. $]; 1961 1962 if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) { 1963 auto n = uncompress.uncompress(responseData.content); 1964 n ~= uncompress.flush(); 1965 responseData.content = cast(ubyte[]) n; 1966 } 1967 1968 // responseData.content ~= cast(ubyte[]) uncompress.flush(); 1969 responseData.contentText = cast(string) responseData.content; 1970 1971 goto done; 1972 } 1973 } 1974 1975 } else { 1976 //if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) 1977 // responseData.content ~= cast(ubyte[]) uncompress.uncompress(data); 1978 //else 1979 responseData.content ~= data; 1980 //assert(data.length <= bodyReadingState.contentLengthRemaining, format("%d <= %d\n%s", data.length, bodyReadingState.contentLengthRemaining, cast(string)data)); 1981 { 1982 int use = cast(int) data.length; 1983 if(use > bodyReadingState.contentLengthRemaining) 1984 use = bodyReadingState.contentLengthRemaining; 1985 bodyReadingState.contentLengthRemaining -= use; 1986 data = data[use .. $]; 1987 } 1988 if(bodyReadingState.contentLengthRemaining == 0) { 1989 if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) { 1990 auto n = uncompress.uncompress(responseData.content); 1991 n ~= uncompress.flush(); 1992 responseData.content = cast(ubyte[]) n; 1993 responseData.contentText = cast(string) responseData.content; 1994 //responseData.content ~= cast(ubyte[]) uncompress.flush(); 1995 } else { 1996 responseData.contentText = cast(string) responseData.content; 1997 } 1998 1999 done: 2000 2001 if(retainCookies && client !is null) { 2002 client.retainCookies(responseData); 2003 } 2004 2005 if(followLocation && responseData.location.length) { 2006 static bool first = true; 2007 //version(DigitalMars) if(!first) asm { int 3; } 2008 populateFromInfo(Uri(responseData.location), HttpVerb.GET); 2009 //import std.stdio; writeln("redirected to ", responseData.location); 2010 first = false; 2011 responseData = HttpResponse.init; 2012 headerReadingState = HeaderReadingState.init; 2013 bodyReadingState = BodyReadingState.init; 2014 state = State.unsent; 2015 stillAlive = false; 2016 sendPrivate(false); 2017 } else { 2018 state = State.complete; 2019 // FIXME 2020 //if(closeSocketWhenComplete) 2021 //socket.close(); 2022 } 2023 } 2024 } 2025 } 2026 2027 if(data.length) 2028 leftoverDataFromLastTime = data.dup; 2029 else 2030 leftoverDataFromLastTime = null; 2031 2032 return stillAlive; 2033 } 2034 2035 } 2036 } 2037 2038 /// 2039 struct HttpRequestParameters { 2040 // FIXME: implement these 2041 //Duration timeoutTotal; // the whole request must finish in this time or else it fails,even if data is still trickling in 2042 Duration timeoutFromInactivity; // if there's no activity in this time it dies. basically the socket receive timeout 2043 2044 // debugging 2045 bool useHttp11 = true; /// 2046 bool acceptGzip = true; /// 2047 bool keepAlive = true; /// 2048 2049 // the request itself 2050 HttpVerb method; /// 2051 string host; /// 2052 ushort port; /// 2053 string uri; /// 2054 2055 bool ssl; /// 2056 2057 string userAgent; /// 2058 string authorization; /// 2059 2060 string[string] cookies; /// 2061 2062 string[] headers; /// do not duplicate host, content-length, content-type, or any others that have a specific property 2063 2064 string contentType; /// 2065 ubyte[] bodyData; /// 2066 2067 string unixSocketPath; 2068 } 2069 2070 interface IHttpClient { 2071 2072 } 2073 2074 /// 2075 enum HttpVerb { 2076 /// 2077 GET, 2078 /// 2079 HEAD, 2080 /// 2081 POST, 2082 /// 2083 PUT, 2084 /// 2085 DELETE, 2086 /// 2087 OPTIONS, 2088 /// 2089 TRACE, 2090 /// 2091 CONNECT, 2092 /// 2093 PATCH, 2094 /// 2095 MERGE 2096 } 2097 2098 /++ 2099 HttpClient keeps cookies, location, and some other state to reuse connections, when possible, like a web browser. 2100 You can use it as your entry point to make http requests. 2101 2102 See the example on [arsd.http2#examples]. 2103 +/ 2104 class HttpClient { 2105 /* Protocol restrictions, useful to disable when debugging servers */ 2106 bool useHttp11 = true; /// 2107 bool acceptGzip = true; /// 2108 bool keepAlive = true; /// 2109 2110 /// 2111 @property Uri location() { 2112 return currentUrl; 2113 } 2114 2115 /++ 2116 Default timeout for requests created on this client. 2117 2118 History: 2119 Added March 31, 2021 2120 +/ 2121 Duration defaultTimeout = 10.seconds; 2122 2123 /++ 2124 High level function that works similarly to entering a url 2125 into a browser. 2126 2127 Follows locations, retain cookies, updates the current url, etc. 2128 +/ 2129 HttpRequest navigateTo(Uri where, HttpVerb method = HttpVerb.GET) { 2130 currentUrl = where.basedOn(currentUrl); 2131 currentDomain = where.host; 2132 2133 auto request = this.request(currentUrl, method); 2134 request.followLocation = true; 2135 request.retainCookies = true; 2136 2137 return request; 2138 } 2139 2140 /++ 2141 Creates a request without updating the current url state. If you want to save cookies, either call [retainCookies] with the response yourself 2142 or set [HttpRequest.retainCookies|request.retainCookies] to `true` on the returned object. But see important implementation shortcomings on [retainCookies]. 2143 +/ 2144 HttpRequest request(Uri uri, HttpVerb method = HttpVerb.GET, ubyte[] bodyData = null, string contentType = null) { 2145 string proxyToUse; 2146 switch(uri.scheme) { 2147 case "http": 2148 proxyToUse = httpProxy; 2149 break; 2150 case "https": 2151 proxyToUse = httpsProxy; 2152 break; 2153 default: 2154 proxyToUse = null; 2155 } 2156 2157 auto request = new HttpRequest(this, uri, method, cache, defaultTimeout, proxyToUse); 2158 2159 request.requestParameters.userAgent = userAgent; 2160 request.requestParameters.authorization = authorization; 2161 2162 request.requestParameters.useHttp11 = this.useHttp11; 2163 request.requestParameters.acceptGzip = this.acceptGzip; 2164 request.requestParameters.keepAlive = this.keepAlive; 2165 2166 request.requestParameters.bodyData = bodyData; 2167 request.requestParameters.contentType = contentType; 2168 2169 // FIXME: what about expiration and the like? or domain/path checks? or Secure checks? 2170 // FIXME: is uri.host correct? i think it should include port number too. what fun. 2171 if(auto cookies = ""/*uri.host*/ in this.cookies) { 2172 foreach(cookie; *cookies) 2173 request.requestParameters.cookies[cookie.name] = cookie.value; 2174 } 2175 2176 return request; 2177 2178 } 2179 2180 /// ditto 2181 HttpRequest request(Uri uri, FormData fd, HttpVerb method = HttpVerb.POST) { 2182 return request(uri, method, fd.toBytes, fd.contentType); 2183 } 2184 2185 2186 private Uri currentUrl; 2187 private string currentDomain; 2188 private ICache cache; 2189 2190 this(ICache cache = null) { 2191 this.cache = cache; 2192 loadDefaultProxy(); 2193 } 2194 2195 /++ 2196 Loads the system-default proxy. Note that the constructor does this automatically 2197 so you should rarely need to call this explicitly. 2198 2199 The environment variables are used, if present, on all operating systems. 2200 2201 History: 2202 Added April 12, 2021 (included in dub v9.5) 2203 2204 Bugs: 2205 On Windows, it does NOT currently check the IE settings, but I do intend to 2206 implement that in the future. When I do, it will be classified as a bug fix, 2207 NOT a breaking change. 2208 +/ 2209 void loadDefaultProxy() { 2210 import std.process; 2211 httpProxy = environment.get("http_proxy", environment.get("HTTP_PROXY", null)); 2212 httpsProxy = environment.get("https_proxy", environment.get("HTTPS_PROXY", null)); 2213 2214 // FIXME: on Windows, I should use the Internet Explorer proxy settings 2215 } 2216 2217 /++ 2218 Proxies to use for requests. The [HttpClient] constructor will set these to the system values, 2219 then you can reset it to `null` if you want to override and not use the proxy after all, or you 2220 can set it after construction to whatever. 2221 2222 The proxy from the client will be automatically set to the requests performed through it. You can 2223 also override on a per-request basis by creating the request and setting the `proxy` field there 2224 before sending it. 2225 2226 History: 2227 Added April 12, 2021 (included in dub v9.5) 2228 +/ 2229 string httpProxy; 2230 /// ditto 2231 string httpsProxy; 2232 2233 /// See [retainCookies] for important caveats. 2234 void setCookie(string name, string value, string domain = null) { 2235 CookieHeader ch; 2236 2237 ch.name = name; 2238 ch.value = value; 2239 2240 setCookie(ch, domain); 2241 } 2242 2243 /// ditto 2244 void setCookie(CookieHeader ch, string domain = null) { 2245 if(domain is null) 2246 domain = currentDomain; 2247 2248 cookies[""/*domain*/] ~= ch; 2249 } 2250 2251 /++ 2252 [HttpClient] does NOT automatically store cookies. You must explicitly retain them from a response by calling this method. 2253 2254 Examples: 2255 --- 2256 import arsd.http2; 2257 void main() { 2258 auto client = new HttpClient(); 2259 auto setRequest = client.request(Uri("http://arsdnet.net/cgi-bin/cookies/set")); 2260 auto setResponse = setRequest.waitForCompletion(); 2261 2262 auto request = client.request(Uri("http://arsdnet.net/cgi-bin/cookies/get")); 2263 auto response = request.waitForCompletion(); 2264 2265 // the cookie wasn't explicitly retained, so the server echos back nothing 2266 assert(response.responseText.length == 0); 2267 2268 // now keep the cookies from our original set 2269 client.retainCookies(setResponse); 2270 2271 request = client.request(Uri("http://arsdnet.net/cgi-bin/cookies/get")); 2272 response = request.waitForCompletion(); 2273 2274 // now it matches 2275 assert(response.responseText.length && response.responseText == setResponse.cookies["example-cookie"]); 2276 } 2277 --- 2278 2279 Bugs: 2280 It does NOT currently implement domain / path / secure separation nor cookie expiration. It assumes that if you call this function, you're ok with it. 2281 2282 You may want to use separate HttpClient instances if any sharing is unacceptable at this time. 2283 2284 History: 2285 Added July 5, 2021 (dub v10.2) 2286 +/ 2287 void retainCookies(HttpResponse fromResponse) { 2288 foreach(name, value; fromResponse.cookies) 2289 setCookie(name, value); 2290 } 2291 2292 /// 2293 void clearCookies(string domain = null) { 2294 if(domain is null) 2295 cookies = null; 2296 else 2297 cookies[domain] = null; 2298 } 2299 2300 // If you set these, they will be pre-filled on all requests made with this client 2301 string userAgent = "D arsd.html2"; /// 2302 string authorization; /// 2303 2304 /* inter-request state */ 2305 private CookieHeader[][string] cookies; 2306 } 2307 2308 interface ICache { 2309 /++ 2310 The client is about to make the given `request`. It will ALWAYS pass it to the cache object first so you can decide if you want to and can provide a response. You should probably check the appropriate headers to see if you should even attempt to look up on the cache (HttpClient does NOT do this to give maximum flexibility to the cache implementor). 2311 2312 Return null if the cache does not provide. 2313 +/ 2314 const(HttpResponse)* getCachedResponse(HttpRequestParameters request); 2315 2316 /++ 2317 The given request has received the given response. The implementing class needs to decide if it wants to cache or not. Return true if it was added, false if you chose not to. 2318 2319 You may wish to examine headers, etc., in making the decision. The HttpClient will ALWAYS pass a request/response to this. 2320 +/ 2321 bool cacheResponse(HttpRequestParameters request, HttpResponse response); 2322 } 2323 2324 /+ 2325 // / Provides caching behavior similar to a real web browser 2326 class HttpCache : ICache { 2327 const(HttpResponse)* getCachedResponse(HttpRequestParameters request) { 2328 return null; 2329 } 2330 } 2331 2332 // / Gives simple maximum age caching, ignoring the actual http headers 2333 class SimpleCache : ICache { 2334 const(HttpResponse)* getCachedResponse(HttpRequestParameters request) { 2335 return null; 2336 } 2337 } 2338 +/ 2339 2340 /++ 2341 A pseudo-cache to provide a mock server. Construct one of these, 2342 populate it with test responses, and pass it to [HttpClient] to 2343 do a network-free test. 2344 2345 You should populate it with the [populate] method. Any request not 2346 pre-populated will return a "server refused connection" response. 2347 +/ 2348 class HttpMockProvider : ICache { 2349 /+ + 2350 2351 +/ 2352 version(none) 2353 this(Uri baseUrl, string defaultResponseContentType) { 2354 2355 } 2356 2357 this() {} 2358 2359 HttpResponse defaultResponse; 2360 2361 /// Implementation of the ICache interface. Hijacks all requests to return a pre-populated response or "server disconnected". 2362 const(HttpResponse)* getCachedResponse(HttpRequestParameters request) { 2363 import std.conv; 2364 auto defaultPort = request.ssl ? 443 : 80; 2365 string identifier = text( 2366 request.method, " ", 2367 request.ssl ? "https" : "http", "://", 2368 request.host, 2369 (request.port && request.port != defaultPort) ? (":" ~ to!string(request.port)) : "", 2370 request.uri 2371 ); 2372 2373 if(auto res = identifier in population) 2374 return res; 2375 return &defaultResponse; 2376 } 2377 2378 /// Implementation of the ICache interface. We never actually cache anything here since it is all about mock responses, not actually caching real data. 2379 bool cacheResponse(HttpRequestParameters request, HttpResponse response) { 2380 return false; 2381 } 2382 2383 /++ 2384 Convenience method to populate simple responses. For more complex 2385 work, use one of the other overloads where you build complete objects 2386 yourself. 2387 2388 Params: 2389 request = a verb and complete URL to mock as one string. 2390 For example "GET http://example.com/". If you provide only 2391 a partial URL, it will be based on the `baseUrl` you gave 2392 in the `HttpMockProvider` constructor. 2393 2394 responseCode = the HTTP response code, like 200 or 404. 2395 2396 response = the response body as a string. It is assumed 2397 to be of the `defaultResponseContentType` you passed in the 2398 `HttpMockProvider` constructor. 2399 +/ 2400 void populate(string request, int responseCode, string response) { 2401 2402 // FIXME: absolute-ize the URL in the request 2403 2404 HttpResponse r; 2405 r.code = responseCode; 2406 r.codeText = getHttpCodeText(r.code); 2407 2408 r.content = cast(ubyte[]) response; 2409 r.contentText = response; 2410 2411 population[request] = r; 2412 } 2413 2414 version(none) 2415 void populate(string method, string url, HttpResponse response) { 2416 // FIXME 2417 } 2418 2419 private HttpResponse[string] population; 2420 } 2421 2422 // modified from the one in cgi.d to just have the text 2423 private static string getHttpCodeText(int code) pure nothrow @nogc { 2424 switch(code) { 2425 // this module's proprietary extensions 2426 case 0: return null; 2427 case 1: return "request.abort called"; 2428 case 2: return "connection failed"; 2429 case 3: return "server disconnected"; 2430 case 4: return "exception thrown"; // actually should be some other thing 2431 case 5: return "Request timed out"; 2432 2433 // * * * standard ones * * * 2434 2435 // 1xx skipped since they shouldn't happen 2436 2437 // 2438 case 200: return "OK"; 2439 case 201: return "Created"; 2440 case 202: return "Accepted"; 2441 case 203: return "Non-Authoritative Information"; 2442 case 204: return "No Content"; 2443 case 205: return "Reset Content"; 2444 // 2445 case 300: return "Multiple Choices"; 2446 case 301: return "Moved Permanently"; 2447 case 302: return "Found"; 2448 case 303: return "See Other"; 2449 case 307: return "Temporary Redirect"; 2450 case 308: return "Permanent Redirect"; 2451 // 2452 case 400: return "Bad Request"; 2453 case 403: return "Forbidden"; 2454 case 404: return "Not Found"; 2455 case 405: return "Method Not Allowed"; 2456 case 406: return "Not Acceptable"; 2457 case 409: return "Conflict"; 2458 case 410: return "Gone"; 2459 // 2460 case 500: return "Internal Server Error"; 2461 case 501: return "Not Implemented"; 2462 case 502: return "Bad Gateway"; 2463 case 503: return "Service Unavailable"; 2464 // 2465 default: assert(0, "Unsupported http code"); 2466 } 2467 } 2468 2469 2470 /// 2471 struct HttpCookie { 2472 string name; /// 2473 string value; /// 2474 string domain; /// 2475 string path; /// 2476 //SysTime expirationDate; /// 2477 bool secure; /// 2478 bool httpOnly; /// 2479 } 2480 2481 // FIXME: websocket 2482 2483 version(testing) 2484 void main() { 2485 import std.stdio; 2486 auto client = new HttpClient(); 2487 auto request = client.navigateTo(Uri("http://localhost/chunked.php")); 2488 request.send(); 2489 auto request2 = client.navigateTo(Uri("http://dlang.org/")); 2490 request2.send(); 2491 2492 { 2493 auto response = request2.waitForCompletion(); 2494 //write(cast(string) response.content); 2495 } 2496 2497 auto response = request.waitForCompletion(); 2498 write(cast(string) response.content); 2499 2500 writeln(HttpRequest.socketsPerHost); 2501 } 2502 2503 2504 // From sslsocket.d, but this is the maintained version! 2505 version(use_openssl) { 2506 alias SslClientSocket = OpenSslSocket; 2507 2508 // macros in the original C 2509 SSL_METHOD* SSLv23_client_method() { 2510 if(ossllib.SSLv23_client_method) 2511 return ossllib.SSLv23_client_method(); 2512 else 2513 return ossllib.TLS_client_method(); 2514 } 2515 2516 struct SSL {} 2517 struct SSL_CTX {} 2518 struct SSL_METHOD {} 2519 enum SSL_VERIFY_NONE = 0; 2520 2521 struct ossllib { 2522 __gshared static extern(C) { 2523 /* these are only on older openssl versions { */ 2524 int function() SSL_library_init; 2525 void function() SSL_load_error_strings; 2526 SSL_METHOD* function() SSLv23_client_method; 2527 /* } */ 2528 2529 void function(ulong, void*) OPENSSL_init_ssl; 2530 2531 SSL_CTX* function(const SSL_METHOD*) SSL_CTX_new; 2532 SSL* function(SSL_CTX*) SSL_new; 2533 int function(SSL*, int) SSL_set_fd; 2534 int function(SSL*) SSL_connect; 2535 int function(SSL*, const void*, int) SSL_write; 2536 int function(SSL*, void*, int) SSL_read; 2537 @trusted nothrow @nogc int function(SSL*) SSL_shutdown; 2538 void function(SSL*) SSL_free; 2539 void function(SSL_CTX*) SSL_CTX_free; 2540 2541 int function(const SSL*) SSL_pending; 2542 2543 void function(SSL*, int, void*) SSL_set_verify; 2544 2545 void function(SSL*, int, c_long, void*) SSL_ctrl; 2546 2547 SSL_METHOD* function() SSLv3_client_method; 2548 SSL_METHOD* function() TLS_client_method; 2549 2550 void function(SSL_CTX*, void function(SSL*, char* line)) SSL_CTX_set_keylog_callback; 2551 } 2552 } 2553 2554 import core.stdc.config; 2555 2556 struct eallib { 2557 __gshared static extern(C) { 2558 /* these are only on older openssl versions { */ 2559 void function() OpenSSL_add_all_ciphers; 2560 void function() OpenSSL_add_all_digests; 2561 /* } */ 2562 2563 void function(ulong, void*) OPENSSL_init_crypto; 2564 2565 void function(FILE*) ERR_print_errors_fp; 2566 } 2567 } 2568 2569 2570 SSL_CTX* SSL_CTX_new(const SSL_METHOD* a) { 2571 if(ossllib.SSL_CTX_new) 2572 return ossllib.SSL_CTX_new(a); 2573 else throw new Exception("SSL_CTX_new not loaded"); 2574 } 2575 SSL* SSL_new(SSL_CTX* a) { 2576 if(ossllib.SSL_new) 2577 return ossllib.SSL_new(a); 2578 else throw new Exception("SSL_new not loaded"); 2579 } 2580 int SSL_set_fd(SSL* a, int b) { 2581 if(ossllib.SSL_set_fd) 2582 return ossllib.SSL_set_fd(a, b); 2583 else throw new Exception("SSL_set_fd not loaded"); 2584 } 2585 int SSL_connect(SSL* a) { 2586 if(ossllib.SSL_connect) 2587 return ossllib.SSL_connect(a); 2588 else throw new Exception("SSL_connect not loaded"); 2589 } 2590 int SSL_write(SSL* a, const void* b, int c) { 2591 if(ossllib.SSL_write) 2592 return ossllib.SSL_write(a, b, c); 2593 else throw new Exception("SSL_write not loaded"); 2594 } 2595 int SSL_read(SSL* a, void* b, int c) { 2596 if(ossllib.SSL_read) 2597 return ossllib.SSL_read(a, b, c); 2598 else throw new Exception("SSL_read not loaded"); 2599 } 2600 @trusted nothrow @nogc int SSL_shutdown(SSL* a) { 2601 if(ossllib.SSL_shutdown) 2602 return ossllib.SSL_shutdown(a); 2603 assert(0); 2604 } 2605 void SSL_free(SSL* a) { 2606 if(ossllib.SSL_free) 2607 return ossllib.SSL_free(a); 2608 else throw new Exception("SSL_free not loaded"); 2609 } 2610 void SSL_CTX_free(SSL_CTX* a) { 2611 if(ossllib.SSL_CTX_free) 2612 return ossllib.SSL_CTX_free(a); 2613 else throw new Exception("SSL_CTX_free not loaded"); 2614 } 2615 2616 int SSL_pending(const SSL* a) { 2617 if(ossllib.SSL_pending) 2618 return ossllib.SSL_pending(a); 2619 else throw new Exception("SSL_pending not loaded"); 2620 } 2621 void SSL_set_verify(SSL* a, int b, void* c) { 2622 if(ossllib.SSL_set_verify) 2623 return ossllib.SSL_set_verify(a, b, c); 2624 else throw new Exception("SSL_set_verify not loaded"); 2625 } 2626 void SSL_set_tlsext_host_name(SSL* a, const char* b) { 2627 if(ossllib.SSL_ctrl) 2628 return ossllib.SSL_ctrl(a, 55 /*SSL_CTRL_SET_TLSEXT_HOSTNAME*/, 0 /*TLSEXT_NAMETYPE_host_name*/, cast(void*) b); 2629 else throw new Exception("SSL_set_tlsext_host_name not loaded"); 2630 } 2631 2632 SSL_METHOD* SSLv3_client_method() { 2633 if(ossllib.SSLv3_client_method) 2634 return ossllib.SSLv3_client_method(); 2635 else throw new Exception("SSLv3_client_method not loaded"); 2636 } 2637 SSL_METHOD* TLS_client_method() { 2638 if(ossllib.TLS_client_method) 2639 return ossllib.TLS_client_method(); 2640 else throw new Exception("TLS_client_method not loaded"); 2641 } 2642 void ERR_print_errors_fp(FILE* a) { 2643 if(eallib.ERR_print_errors_fp) 2644 return eallib.ERR_print_errors_fp(a); 2645 else throw new Exception("ERR_print_errors_fp not loaded"); 2646 } 2647 2648 extern(C) 2649 void SSL_CTX_keylog_cb_func(SSL_CTX* ctx, void function(SSL*, char*) func) 2650 { 2651 // this isn't in openssl 1.0 and is non-essential, so it is allowed to fail. 2652 if(ossllib.SSL_CTX_set_keylog_callback) 2653 ossllib.SSL_CTX_set_keylog_callback(ctx, func); 2654 //else throw new Exception("SSL_CTX_keylog_cb_func not loaded"); 2655 } 2656 2657 2658 private __gshared void* ossllib_handle; 2659 version(Windows) 2660 private __gshared void* oeaylib_handle; 2661 else 2662 alias oeaylib_handle = ossllib_handle; 2663 version(Posix) 2664 private import core.sys.posix.dlfcn; 2665 else version(Windows) 2666 private import core.sys.windows.windows; 2667 2668 import core.stdc.stdio; 2669 2670 private __gshared Object loadSslMutex = new Object; 2671 private __gshared bool sslLoaded = false; 2672 2673 void loadOpenSsl() { 2674 if(sslLoaded) 2675 return; 2676 synchronized(loadSslMutex) { 2677 2678 version(OSX) { 2679 // newest box 2680 ossllib_handle = dlopen("libssl.1.1.dylib", RTLD_NOW); 2681 // other boxes 2682 if(ossllib_handle is null) 2683 ossllib_handle = dlopen("libssl.dylib", RTLD_NOW); 2684 // old ones like my laptop test 2685 if(ossllib_handle is null) 2686 ossllib_handle = dlopen("/usr/local/opt/openssl/lib/libssl.1.0.0.dylib", RTLD_NOW); 2687 2688 } else version(Posix) { 2689 ossllib_handle = dlopen("libssl.so.1.1", RTLD_NOW); 2690 if(ossllib_handle is null) 2691 ossllib_handle = dlopen("libssl.so", RTLD_NOW); 2692 } else version(Windows) { 2693 ossllib_handle = LoadLibraryW("libssl32.dll"w.ptr); 2694 oeaylib_handle = LoadLibraryW("libeay32.dll"w.ptr); 2695 2696 if(ossllib_handle is null) { 2697 ossllib_handle = LoadLibraryW("ssleay32.dll"w.ptr); 2698 oeaylib_handle = ossllib_handle; 2699 } 2700 } 2701 2702 if(ossllib_handle is null) 2703 throw new Exception("libssl library not found"); 2704 if(oeaylib_handle is null) 2705 throw new Exception("libeay32 library not found"); 2706 2707 foreach(memberName; __traits(allMembers, ossllib)) { 2708 alias t = typeof(__traits(getMember, ossllib, memberName)); 2709 version(Posix) 2710 __traits(getMember, ossllib, memberName) = cast(t) dlsym(ossllib_handle, memberName); 2711 else version(Windows) { 2712 __traits(getMember, ossllib, memberName) = cast(t) GetProcAddress(ossllib_handle, memberName); 2713 } 2714 } 2715 2716 foreach(memberName; __traits(allMembers, eallib)) { 2717 alias t = typeof(__traits(getMember, eallib, memberName)); 2718 version(Posix) 2719 __traits(getMember, eallib, memberName) = cast(t) dlsym(oeaylib_handle, memberName); 2720 else version(Windows) { 2721 __traits(getMember, eallib, memberName) = cast(t) GetProcAddress(oeaylib_handle, memberName); 2722 } 2723 } 2724 2725 2726 if(ossllib.SSL_library_init) 2727 ossllib.SSL_library_init(); 2728 else if(ossllib.OPENSSL_init_ssl) 2729 ossllib.OPENSSL_init_ssl(0, null); 2730 else throw new Exception("couldn't init openssl"); 2731 2732 if(eallib.OpenSSL_add_all_ciphers) { 2733 eallib.OpenSSL_add_all_ciphers(); 2734 if(eallib.OpenSSL_add_all_digests is null) 2735 throw new Exception("no add digests"); 2736 eallib.OpenSSL_add_all_digests(); 2737 } else if(eallib.OPENSSL_init_crypto) 2738 eallib.OPENSSL_init_crypto(0 /*OPENSSL_INIT_ADD_ALL_CIPHERS and ALL_DIGESTS together*/, null); 2739 else throw new Exception("couldn't init crypto openssl"); 2740 2741 if(ossllib.SSL_load_error_strings) 2742 ossllib.SSL_load_error_strings(); 2743 else if(ossllib.OPENSSL_init_ssl) 2744 ossllib.OPENSSL_init_ssl(0x00200000L, null); 2745 else throw new Exception("couldn't load openssl errors"); 2746 2747 sslLoaded = true; 2748 } 2749 } 2750 2751 /+ 2752 // I'm just gonna let the OS clean this up on process termination because otherwise SSL_free 2753 // might have trouble being run from the GC after this module is unloaded. 2754 shared static ~this() { 2755 if(ossllib_handle) { 2756 version(Windows) { 2757 FreeLibrary(oeaylib_handle); 2758 FreeLibrary(ossllib_handle); 2759 } else version(Posix) 2760 dlclose(ossllib_handle); 2761 ossllib_handle = null; 2762 } 2763 ossllib.tupleof = ossllib.tupleof.init; 2764 } 2765 +/ 2766 2767 //pragma(lib, "crypto"); 2768 //pragma(lib, "ssl"); 2769 extern(C) 2770 void write_to_file(SSL* ssl, char* line) 2771 { 2772 import std.stdio; 2773 import std.string; 2774 import std.process : environment; 2775 string logfile = environment.get("SSLKEYLOGFILE"); 2776 if (logfile !is null) 2777 { 2778 auto f = std.stdio.File("/tmp/keyfile", "a+"); 2779 f.writeln(fromStringz(line)); 2780 f.close(); 2781 } 2782 } 2783 2784 class OpenSslSocket : Socket { 2785 private SSL* ssl; 2786 private SSL_CTX* ctx; 2787 private void initSsl(bool verifyPeer, string hostname) { 2788 ctx = SSL_CTX_new(SSLv23_client_method()); 2789 assert(ctx !is null); 2790 debug SSL_CTX_keylog_cb_func(ctx, &write_to_file); 2791 ssl = SSL_new(ctx); 2792 2793 if(hostname.length) 2794 SSL_set_tlsext_host_name(ssl, toStringz(hostname)); 2795 2796 if(!verifyPeer) 2797 SSL_set_verify(ssl, SSL_VERIFY_NONE, null); 2798 SSL_set_fd(ssl, cast(int) this.handle); // on win64 it is necessary to truncate, but the value is never large anyway see http://openssl.6102.n7.nabble.com/Sockets-windows-64-bit-td36169.html 2799 } 2800 2801 bool dataPending() { 2802 return SSL_pending(ssl) > 0; 2803 } 2804 2805 @trusted 2806 override void connect(Address to) { 2807 super.connect(to); 2808 do_ssl_connect(); 2809 } 2810 2811 @trusted 2812 void do_ssl_connect() { 2813 if(SSL_connect(ssl) == -1) { 2814 ERR_print_errors_fp(core.stdc.stdio.stderr); 2815 int i; 2816 //printf("wtf\n"); 2817 //scanf("%d\n", i); 2818 throw new Exception("ssl connect"); 2819 } 2820 } 2821 2822 @trusted 2823 override ptrdiff_t send(scope const(void)[] buf, SocketFlags flags) { 2824 //import std.stdio;writeln(cast(string) buf); 2825 auto retval = SSL_write(ssl, buf.ptr, cast(uint) buf.length); 2826 if(retval == -1) { 2827 ERR_print_errors_fp(core.stdc.stdio.stderr); 2828 int i; 2829 //printf("wtf\n"); 2830 //scanf("%d\n", i); 2831 throw new Exception("ssl send"); 2832 } 2833 return retval; 2834 2835 } 2836 override ptrdiff_t send(scope const(void)[] buf) { 2837 return send(buf, SocketFlags.NONE); 2838 } 2839 @trusted 2840 override ptrdiff_t receive(scope void[] buf, SocketFlags flags) { 2841 auto retval = SSL_read(ssl, buf.ptr, cast(int)buf.length); 2842 if(retval == -1) { 2843 ERR_print_errors_fp(core.stdc.stdio.stderr); 2844 int i; 2845 //printf("wtf\n"); 2846 //scanf("%d\n", i); 2847 throw new Exception("ssl send"); 2848 } 2849 return retval; 2850 } 2851 override ptrdiff_t receive(scope void[] buf) { 2852 return receive(buf, SocketFlags.NONE); 2853 } 2854 2855 this(AddressFamily af, SocketType type = SocketType.STREAM, string hostname = null, bool verifyPeer = true) { 2856 super(af, type); 2857 initSsl(verifyPeer, hostname); 2858 } 2859 2860 override void close() { 2861 if(ssl) SSL_shutdown(ssl); 2862 super.close(); 2863 } 2864 2865 this(socket_t sock, AddressFamily af, string hostname, bool verifyPeer = true) { 2866 super(sock, af); 2867 initSsl(verifyPeer, hostname); 2868 } 2869 2870 void freeSsl() { 2871 if(ssl is null) 2872 return; 2873 SSL_free(ssl); 2874 SSL_CTX_free(ctx); 2875 ssl = null; 2876 } 2877 2878 ~this() { 2879 freeSsl(); 2880 } 2881 } 2882 } 2883 2884 2885 /++ 2886 An experimental component for working with REST apis. Note that it 2887 is a zero-argument template, so to create one, use `new HttpApiClient!()(args..)` 2888 or you will get "HttpApiClient is used as a type" compile errors. 2889 2890 This will probably not work for you yet, and I might change it significantly. 2891 2892 Requires [arsd.jsvar]. 2893 2894 2895 Here's a snippet to create a pull request on GitHub to Phobos: 2896 2897 --- 2898 auto github = new HttpApiClient!()("https://api.github.com/", "your personal api token here"); 2899 2900 // create the arguments object 2901 // see: https://developer.github.com/v3/pulls/#create-a-pull-request 2902 var args = var.emptyObject; 2903 args.title = "My Pull Request"; 2904 args.head = "yourusername:" ~ branchName; 2905 args.base = "master"; 2906 // note it is ["body"] instead of .body because `body` is a D keyword 2907 args["body"] = "My cool PR is opened by the API!"; 2908 args.maintainer_can_modify = true; 2909 2910 /+ 2911 Fun fact, you can also write that: 2912 2913 var args = [ 2914 "title": "My Pull Request".var, 2915 "head": "yourusername:" ~ branchName.var, 2916 "base" : "master".var, 2917 "body" : "My cool PR is opened by the API!".var, 2918 "maintainer_can_modify": true.var 2919 ]; 2920 2921 Note the .var constructor calls in there. If everything is the same type, you actually don't need that, but here since there's strings and bools, D won't allow the literal without explicit constructors to align them all. 2922 +/ 2923 2924 // this translates to `repos/dlang/phobos/pulls` and sends a POST request, 2925 // containing `args` as json, then immediately grabs the json result and extracts 2926 // the value `html_url` from it. `prUrl` is typed `var`, from arsd.jsvar. 2927 auto prUrl = github.rest.repos.dlang.phobos.pulls.POST(args).result.html_url; 2928 2929 writeln("Created: ", prUrl); 2930 --- 2931 2932 Why use this instead of just building the URL? Well, of course you can! This just makes 2933 it a bit more convenient than string concatenation and manages a few headers for you. 2934 2935 Subtypes could potentially add static type checks too. 2936 +/ 2937 class HttpApiClient() { 2938 import arsd.jsvar; 2939 2940 HttpClient httpClient; 2941 2942 alias HttpApiClientType = typeof(this); 2943 2944 string urlBase; 2945 string oauth2Token; 2946 string submittedContentType; 2947 2948 /++ 2949 Params: 2950 2951 urlBase = The base url for the api. Tends to be something like `https://api.example.com/v2/` or similar. 2952 oauth2Token = the authorization token for the service. You'll have to get it from somewhere else. 2953 submittedContentType = the content-type of POST, PUT, etc. bodies. 2954 httpClient = an injected http client, or null if you want to use a default-constructed one 2955 2956 History: 2957 The `httpClient` param was added on December 26, 2020. 2958 +/ 2959 this(string urlBase, string oauth2Token, string submittedContentType = "application/json", HttpClient httpClient = null) { 2960 if(httpClient is null) 2961 this.httpClient = new HttpClient(); 2962 else 2963 this.httpClient = httpClient; 2964 2965 assert(urlBase[0] == 'h'); 2966 assert(urlBase[$-1] == '/'); 2967 2968 this.urlBase = urlBase; 2969 this.oauth2Token = oauth2Token; 2970 this.submittedContentType = submittedContentType; 2971 } 2972 2973 /// 2974 static struct HttpRequestWrapper { 2975 HttpApiClientType apiClient; /// 2976 HttpRequest request; /// 2977 HttpResponse _response; 2978 2979 /// 2980 this(HttpApiClientType apiClient, HttpRequest request) { 2981 this.apiClient = apiClient; 2982 this.request = request; 2983 } 2984 2985 /// Returns the full [HttpResponse] object so you can inspect the headers 2986 @property HttpResponse response() { 2987 if(_response is HttpResponse.init) 2988 _response = request.waitForCompletion(); 2989 return _response; 2990 } 2991 2992 /++ 2993 Returns the parsed JSON from the body of the response. 2994 2995 Throws on non-2xx responses. 2996 +/ 2997 var result() { 2998 return apiClient.throwOnError(response); 2999 } 3000 3001 alias request this; 3002 } 3003 3004 /// 3005 HttpRequestWrapper request(string uri, HttpVerb requestMethod = HttpVerb.GET, ubyte[] bodyBytes = null) { 3006 if(uri[0] == '/') 3007 uri = uri[1 .. $]; 3008 3009 auto u = Uri(uri).basedOn(Uri(urlBase)); 3010 3011 auto req = httpClient.navigateTo(u, requestMethod); 3012 3013 if(oauth2Token.length) 3014 req.requestParameters.headers ~= "Authorization: Bearer " ~ oauth2Token; 3015 req.requestParameters.contentType = submittedContentType; 3016 req.requestParameters.bodyData = bodyBytes; 3017 3018 return HttpRequestWrapper(this, req); 3019 } 3020 3021 /// 3022 var throwOnError(HttpResponse res) { 3023 if(res.code < 200 || res.code >= 300) 3024 throw new Exception(res.codeText ~ " " ~ res.contentText); 3025 3026 var response = var.fromJson(res.contentText); 3027 if(response.errors) { 3028 throw new Exception(response.errors.toJson()); 3029 } 3030 3031 return response; 3032 } 3033 3034 /// 3035 @property RestBuilder rest() { 3036 return RestBuilder(this, null, null); 3037 } 3038 3039 // hipchat.rest.room["Tech Team"].history 3040 // gives: "/room/Tech%20Team/history" 3041 // 3042 // hipchat.rest.room["Tech Team"].history("page", "12) 3043 /// 3044 static struct RestBuilder { 3045 HttpApiClientType apiClient; 3046 string[] pathParts; 3047 string[2][] queryParts; 3048 this(HttpApiClientType apiClient, string[] pathParts, string[2][] queryParts) { 3049 this.apiClient = apiClient; 3050 this.pathParts = pathParts; 3051 this.queryParts = queryParts; 3052 } 3053 3054 RestBuilder _SELF() { 3055 return this; 3056 } 3057 3058 /// The args are so you can call opCall on the returned 3059 /// object, despite @property being broken af in D. 3060 RestBuilder opDispatch(string str, T)(string n, T v) { 3061 return RestBuilder(apiClient, pathParts ~ str, queryParts ~ [n, to!string(v)]); 3062 } 3063 3064 /// 3065 RestBuilder opDispatch(string str)() { 3066 return RestBuilder(apiClient, pathParts ~ str, queryParts); 3067 } 3068 3069 3070 /// 3071 RestBuilder opIndex(string str) { 3072 return RestBuilder(apiClient, pathParts ~ str, queryParts); 3073 } 3074 /// 3075 RestBuilder opIndex(var str) { 3076 return RestBuilder(apiClient, pathParts ~ str.get!string, queryParts); 3077 } 3078 /// 3079 RestBuilder opIndex(int i) { 3080 return RestBuilder(apiClient, pathParts ~ to!string(i), queryParts); 3081 } 3082 3083 /// 3084 RestBuilder opCall(T)(string name, T value) { 3085 return RestBuilder(apiClient, pathParts, queryParts ~ [name, to!string(value)]); 3086 } 3087 3088 /// 3089 string toUri() { 3090 import std.uri; 3091 string result; 3092 foreach(idx, part; pathParts) { 3093 if(idx) 3094 result ~= "/"; 3095 result ~= encodeComponent(part); 3096 } 3097 result ~= "?"; 3098 foreach(idx, part; queryParts) { 3099 if(idx) 3100 result ~= "&"; 3101 result ~= encodeComponent(part[0]); 3102 result ~= "="; 3103 result ~= encodeComponent(part[1]); 3104 } 3105 3106 return result; 3107 } 3108 3109 /// 3110 final HttpRequestWrapper GET() { return _EXECUTE(HttpVerb.GET, this.toUri(), ToBytesResult.init); } 3111 /// ditto 3112 final HttpRequestWrapper DELETE() { return _EXECUTE(HttpVerb.DELETE, this.toUri(), ToBytesResult.init); } 3113 3114 // need to be able to send: JSON, urlencoded, multipart/form-data, and raw stuff. 3115 /// ditto 3116 final HttpRequestWrapper POST(T...)(T t) { return _EXECUTE(HttpVerb.POST, this.toUri(), toBytes(t)); } 3117 /// ditto 3118 final HttpRequestWrapper PATCH(T...)(T t) { return _EXECUTE(HttpVerb.PATCH, this.toUri(), toBytes(t)); } 3119 /// ditto 3120 final HttpRequestWrapper PUT(T...)(T t) { return _EXECUTE(HttpVerb.PUT, this.toUri(), toBytes(t)); } 3121 3122 struct ToBytesResult { 3123 ubyte[] bytes; 3124 string contentType; 3125 } 3126 3127 private ToBytesResult toBytes(T...)(T t) { 3128 import std.conv : to; 3129 static if(T.length == 0) 3130 return ToBytesResult(null, null); 3131 else static if(T.length == 1 && is(T[0] == var)) 3132 return ToBytesResult(cast(ubyte[]) t[0].toJson(), "application/json"); // json data 3133 else static if(T.length == 1 && (is(T[0] == string) || is(T[0] == ubyte[]))) 3134 return ToBytesResult(cast(ubyte[]) t[0], null); // raw data 3135 else static if(T.length == 1 && is(T[0] : FormData)) 3136 return ToBytesResult(t[0].toBytes, t[0].contentType); 3137 else static if(T.length > 1 && T.length % 2 == 0 && is(T[0] == string)) { 3138 // string -> value pairs for a POST request 3139 string answer; 3140 foreach(idx, val; t) { 3141 static if(idx % 2 == 0) { 3142 if(answer.length) 3143 answer ~= "&"; 3144 answer ~= encodeComponent(val); // it had better be a string! lol 3145 answer ~= "="; 3146 } else { 3147 answer ~= encodeComponent(to!string(val)); 3148 } 3149 } 3150 3151 return ToBytesResult(cast(ubyte[]) answer, "application/x-www-form-urlencoded"); 3152 } 3153 else 3154 static assert(0); // FIXME 3155 3156 } 3157 3158 HttpRequestWrapper _EXECUTE(HttpVerb verb, string uri, ubyte[] bodyBytes) { 3159 return apiClient.request(uri, verb, bodyBytes); 3160 } 3161 3162 HttpRequestWrapper _EXECUTE(HttpVerb verb, string uri, ToBytesResult tbr) { 3163 auto r = apiClient.request(uri, verb, tbr.bytes); 3164 if(tbr.contentType !is null) 3165 r.requestParameters.contentType = tbr.contentType; 3166 return r; 3167 } 3168 } 3169 } 3170 3171 3172 // see also: arsd.cgi.encodeVariables 3173 /// Creates a multipart/form-data object that is suitable for file uploads and other kinds of POST 3174 class FormData { 3175 struct MimePart { 3176 string name; 3177 const(void)[] data; 3178 string contentType; 3179 string filename; 3180 } 3181 3182 MimePart[] parts; 3183 3184 /// 3185 void append(string key, in void[] value, string contentType = null, string filename = null) { 3186 parts ~= MimePart(key, value, contentType, filename); 3187 } 3188 3189 private string boundary = "0016e64be86203dd36047610926a"; // FIXME 3190 3191 string contentType() { 3192 return "multipart/form-data; boundary=" ~ boundary; 3193 } 3194 3195 /// 3196 ubyte[] toBytes() { 3197 string data; 3198 3199 foreach(part; parts) { 3200 data ~= "--" ~ boundary ~ "\r\n"; 3201 data ~= "Content-Disposition: form-data; name=\""~part.name~"\""; 3202 if(part.filename !is null) 3203 data ~= "; filename=\""~part.filename~"\""; 3204 data ~= "\r\n"; 3205 if(part.contentType !is null) 3206 data ~= "Content-Type: " ~ part.contentType ~ "\r\n"; 3207 data ~= "\r\n"; 3208 3209 data ~= cast(string) part.data; 3210 3211 data ~= "\r\n"; 3212 } 3213 3214 data ~= "--" ~ boundary ~ "--\r\n"; 3215 3216 return cast(ubyte[]) data; 3217 } 3218 } 3219 3220 private bool bicmp(in ubyte[] item, in char[] search) { 3221 if(item.length != search.length) return false; 3222 3223 foreach(i; 0 .. item.length) { 3224 ubyte a = item[i]; 3225 ubyte b = search[i]; 3226 if(a >= 'A' && a <= 'Z') 3227 a += 32; 3228 //if(b >= 'A' && b <= 'Z') 3229 //b += 32; 3230 if(a != b) 3231 return false; 3232 } 3233 3234 return true; 3235 } 3236 3237 /++ 3238 WebSocket client, based on the browser api, though also with other api options. 3239 3240 --- 3241 import arsd.http2; 3242 3243 void main() { 3244 auto ws = new WebSocket(Uri("ws://....")); 3245 3246 ws.onmessage = (in char[] msg) { 3247 ws.send("a reply"); 3248 }; 3249 3250 ws.connect(); 3251 3252 WebSocket.eventLoop(); 3253 } 3254 --- 3255 3256 Symbol_groups: 3257 foundational = 3258 Used with all API styles. 3259 3260 browser_api = 3261 API based on the standard in the browser. 3262 3263 event_loop_integration = 3264 Integrating with external event loops is done through static functions. You should 3265 call these BEFORE doing anything else with the WebSocket module or class. 3266 3267 $(PITFALL NOT IMPLEMENTED) 3268 --- 3269 WebSocket.setEventLoopProxy(arsd.simpledisplay.EventLoop.proxy.tupleof); 3270 // or something like that. it is not implemented yet. 3271 --- 3272 $(PITFALL NOT IMPLEMENTED) 3273 3274 blocking_api = 3275 The blocking API is best used when you only need basic functionality with a single connection. 3276 3277 --- 3278 WebSocketFrame msg; 3279 do { 3280 // FIXME good demo 3281 } while(msg); 3282 --- 3283 3284 Or to check for blocks before calling: 3285 3286 --- 3287 try_to_process_more: 3288 while(ws.isMessageBuffered()) { 3289 auto msg = ws.waitForNextMessage(); 3290 // process msg 3291 } 3292 if(ws.isDataPending()) { 3293 ws.lowLevelReceive(); 3294 goto try_to_process_more; 3295 } else { 3296 // nothing ready, you can do other things 3297 // or at least sleep a while before trying 3298 // to process more. 3299 if(ws.readyState == WebSocket.OPEN) { 3300 Thread.sleep(1.seconds); 3301 goto try_to_process_more; 3302 } 3303 } 3304 --- 3305 3306 +/ 3307 class WebSocket { 3308 private Uri uri; 3309 private string[string] cookies; 3310 private string origin; 3311 3312 private string host; 3313 private ushort port; 3314 private bool ssl; 3315 3316 // used to decide if we mask outgoing msgs 3317 private bool isClient; 3318 3319 private MonoTime timeoutFromInactivity; 3320 private MonoTime nextPing; 3321 3322 /++ 3323 wss://echo.websocket.org 3324 +/ 3325 /// Group: foundational 3326 this(Uri uri, Config config = Config.init) 3327 //in (uri.scheme == "ws" || uri.scheme == "wss") 3328 in { assert(uri.scheme == "ws" || uri.scheme == "wss"); } do 3329 { 3330 this.uri = uri; 3331 this.config = config; 3332 3333 this.receiveBuffer = new ubyte[](config.initialReceiveBufferSize); 3334 3335 host = uri.host; 3336 ssl = uri.scheme == "wss"; 3337 port = cast(ushort) (uri.port ? uri.port : ssl ? 443 : 80); 3338 3339 if(ssl) { 3340 version(with_openssl) { 3341 loadOpenSsl(); 3342 socket = new SslClientSocket(family(uri.unixSocketPath), SocketType.STREAM, host, defaultVerifyPeer); 3343 } else 3344 throw new Exception("SSL not compiled in"); 3345 } else 3346 socket = new Socket(family(uri.unixSocketPath), SocketType.STREAM); 3347 3348 } 3349 3350 /++ 3351 3352 +/ 3353 /// Group: foundational 3354 void connect() { 3355 this.isClient = true; 3356 if(uri.unixSocketPath) 3357 socket.connect(new UnixAddress(uri.unixSocketPath)); 3358 else 3359 socket.connect(new InternetAddress(host, port)); // FIXME: ipv6 support... 3360 // FIXME: websocket handshake could and really should be async too. 3361 3362 auto uri = this.uri.path.length ? this.uri.path : "/"; 3363 if(this.uri.query.length) { 3364 uri ~= "?"; 3365 uri ~= this.uri.query; 3366 } 3367 3368 // the headers really shouldn't be bigger than this, at least 3369 // the chunks i need to process 3370 ubyte[4096] bufferBacking = void; 3371 ubyte[] buffer = bufferBacking[]; 3372 size_t pos; 3373 3374 void append(in char[][] items...) { 3375 foreach(what; items) { 3376 if((pos + what.length) > buffer.length) { 3377 buffer.length += 4096; 3378 } 3379 buffer[pos .. pos + what.length] = cast(ubyte[]) what[]; 3380 pos += what.length; 3381 } 3382 } 3383 3384 append("GET ", uri, " HTTP/1.1\r\n"); 3385 append("Host: ", this.uri.host, "\r\n"); 3386 3387 append("Upgrade: websocket\r\n"); 3388 append("Connection: Upgrade\r\n"); 3389 append("Sec-WebSocket-Version: 13\r\n"); 3390 3391 // FIXME: randomize this 3392 append("Sec-WebSocket-Key: x3JEHMbDL1EzLkh9GBhXDw==\r\n"); 3393 3394 if(config.protocol.length) 3395 append("Sec-WebSocket-Protocol: ", config.protocol, "\r\n"); 3396 if(config.origin.length) 3397 append("Origin: ", origin, "\r\n"); 3398 3399 foreach(h; config.additionalHeaders) { 3400 append(h); 3401 append("\r\n"); 3402 } 3403 3404 append("\r\n"); 3405 3406 auto remaining = buffer[0 .. pos]; 3407 //import std.stdio; writeln(host, " " , port, " ", cast(string) remaining); 3408 while(remaining.length) { 3409 auto r = socket.send(remaining); 3410 if(r < 0) 3411 throw new Exception(lastSocketError()); 3412 if(r == 0) 3413 throw new Exception("unexpected connection termination"); 3414 remaining = remaining[r .. $]; 3415 } 3416 3417 // the response shouldn't be especially large at this point, just 3418 // headers for the most part. gonna try to get it in the stack buffer. 3419 // then copy stuff after headers, if any, to the frame buffer. 3420 ubyte[] used; 3421 3422 void more() { 3423 auto r = socket.receive(buffer[used.length .. $]); 3424 3425 if(r < 0) 3426 throw new Exception(lastSocketError()); 3427 if(r == 0) 3428 throw new Exception("unexpected connection termination"); 3429 //import std.stdio;writef("%s", cast(string) buffer[used.length .. used.length + r]); 3430 3431 used = buffer[0 .. used.length + r]; 3432 } 3433 3434 more(); 3435 3436 import std.algorithm; 3437 if(!used.startsWith(cast(ubyte[]) "HTTP/1.1 101")) 3438 throw new Exception("didn't get a websocket answer"); 3439 // skip the status line 3440 while(used.length && used[0] != '\n') 3441 used = used[1 .. $]; 3442 3443 if(used.length == 0) 3444 throw new Exception("Remote server disconnected or didn't send enough information"); 3445 3446 if(used.length < 1) 3447 more(); 3448 3449 used = used[1 .. $]; // skip the \n 3450 3451 if(used.length == 0) 3452 more(); 3453 3454 // checks on the protocol from ehaders 3455 bool isWebsocket; 3456 bool isUpgrade; 3457 const(ubyte)[] protocol; 3458 const(ubyte)[] accept; 3459 3460 while(used.length) { 3461 if(used.length >= 2 && used[0] == '\r' && used[1] == '\n') { 3462 used = used[2 .. $]; 3463 break; // all done 3464 } 3465 int idxColon; 3466 while(idxColon < used.length && used[idxColon] != ':') 3467 idxColon++; 3468 if(idxColon == used.length) 3469 more(); 3470 auto idxStart = idxColon + 1; 3471 while(idxStart < used.length && used[idxStart] == ' ') 3472 idxStart++; 3473 if(idxStart == used.length) 3474 more(); 3475 auto idxEnd = idxStart; 3476 while(idxEnd < used.length && used[idxEnd] != '\r') 3477 idxEnd++; 3478 if(idxEnd == used.length) 3479 more(); 3480 3481 auto headerName = used[0 .. idxColon]; 3482 auto headerValue = used[idxStart .. idxEnd]; 3483 3484 // move past this header 3485 used = used[idxEnd .. $]; 3486 // and the \r\n 3487 if(2 <= used.length) 3488 used = used[2 .. $]; 3489 3490 if(headerName.bicmp("upgrade")) { 3491 if(headerValue.bicmp("websocket")) 3492 isWebsocket = true; 3493 } else if(headerName.bicmp("connection")) { 3494 if(headerValue.bicmp("upgrade")) 3495 isUpgrade = true; 3496 } else if(headerName.bicmp("sec-websocket-accept")) { 3497 accept = headerValue; 3498 } else if(headerName.bicmp("sec-websocket-protocol")) { 3499 protocol = headerValue; 3500 } 3501 3502 if(!used.length) { 3503 more(); 3504 } 3505 } 3506 3507 3508 if(!isWebsocket) 3509 throw new Exception("didn't answer as websocket"); 3510 if(!isUpgrade) 3511 throw new Exception("didn't answer as upgrade"); 3512 3513 3514 // FIXME: check protocol if config requested one 3515 // FIXME: check accept for the right hash 3516 3517 receiveBuffer[0 .. used.length] = used[]; 3518 receiveBufferUsedLength = used.length; 3519 3520 readyState_ = OPEN; 3521 3522 if(onopen) 3523 onopen(); 3524 3525 nextPing = MonoTime.currTime + config.pingFrequency.msecs; 3526 timeoutFromInactivity = MonoTime.currTime + config.timeoutFromInactivity; 3527 3528 registerActiveSocket(this); 3529 } 3530 3531 /++ 3532 Is data pending on the socket? Also check [isMessageBuffered] to see if there 3533 is already a message in memory too. 3534 3535 If this returns `true`, you can call [lowLevelReceive], then try [isMessageBuffered] 3536 again. 3537 +/ 3538 /// Group: blocking_api 3539 public bool isDataPending(Duration timeout = 0.seconds) { 3540 static SocketSet readSet; 3541 if(readSet is null) 3542 readSet = new SocketSet(); 3543 3544 version(with_openssl) 3545 if(auto s = cast(SslClientSocket) socket) { 3546 // select doesn't handle the case with stuff 3547 // left in the ssl buffer so i'm checking it separately 3548 if(s.dataPending()) { 3549 return true; 3550 } 3551 } 3552 3553 readSet.add(socket); 3554 3555 //tryAgain: 3556 auto selectGot = Socket.select(readSet, null, null, timeout); 3557 if(selectGot == 0) { /* timeout */ 3558 // timeout 3559 return false; 3560 } else if(selectGot == -1) { /* interrupted */ 3561 return false; 3562 } else { /* ready */ 3563 if(readSet.isSet(socket)) { 3564 return true; 3565 } 3566 } 3567 3568 return false; 3569 } 3570 3571 private void llsend(ubyte[] d) { 3572 if(readyState == CONNECTING) 3573 throw new Exception("WebSocket not connected when trying to send. Did you forget to call connect(); ?"); 3574 //connect(); 3575 while(d.length) { 3576 auto r = socket.send(d); 3577 if(r < 0 && wouldHaveBlocked()) { 3578 import core.thread; 3579 Thread.sleep(1.msecs); 3580 continue; 3581 } 3582 //import core.stdc.errno; import std.stdio; writeln(errno); 3583 if(r <= 0) { 3584 // import std.stdio; writeln(GetLastError()); 3585 throw new Exception("Socket send failed"); 3586 } 3587 d = d[r .. $]; 3588 } 3589 } 3590 3591 private void llclose() { 3592 socket.shutdown(SocketShutdown.SEND); 3593 } 3594 3595 /++ 3596 Waits for more data off the low-level socket and adds it to the pending buffer. 3597 3598 Returns `true` if the connection is still active. 3599 +/ 3600 /// Group: blocking_api 3601 public bool lowLevelReceive() { 3602 if(readyState == CONNECTING) 3603 throw new Exception("WebSocket not connected when trying to receive. Did you forget to call connect(); ?"); 3604 if (receiveBufferUsedLength == receiveBuffer.length) 3605 { 3606 if (receiveBuffer.length == config.maximumReceiveBufferSize) 3607 throw new Exception("Maximum receive buffer size exhausted"); 3608 3609 import std.algorithm : min; 3610 receiveBuffer.length = min(receiveBuffer.length + config.initialReceiveBufferSize, 3611 config.maximumReceiveBufferSize); 3612 } 3613 auto r = socket.receive(receiveBuffer[receiveBufferUsedLength .. $]); 3614 if(r == 0) 3615 return false; 3616 if(r < 0 && wouldHaveBlocked()) 3617 return true; 3618 if(r <= 0) { 3619 //import std.stdio; writeln(WSAGetLastError()); 3620 throw new Exception("Socket receive failed"); 3621 } 3622 receiveBufferUsedLength += r; 3623 return true; 3624 } 3625 3626 private Socket socket; 3627 3628 /* copy/paste section { */ 3629 3630 private int readyState_; 3631 private ubyte[] receiveBuffer; 3632 private size_t receiveBufferUsedLength; 3633 3634 private Config config; 3635 3636 enum CONNECTING = 0; /// Socket has been created. The connection is not yet open. 3637 enum OPEN = 1; /// The connection is open and ready to communicate. 3638 enum CLOSING = 2; /// The connection is in the process of closing. 3639 enum CLOSED = 3; /// The connection is closed or couldn't be opened. 3640 3641 /++ 3642 3643 +/ 3644 /// Group: foundational 3645 static struct Config { 3646 /++ 3647 These control the size of the receive buffer. 3648 3649 It starts at the initial size, will temporarily 3650 balloon up to the maximum size, and will reuse 3651 a buffer up to the likely size. 3652 3653 Anything larger than the maximum size will cause 3654 the connection to be aborted and an exception thrown. 3655 This is to protect you against a peer trying to 3656 exhaust your memory, while keeping the user-level 3657 processing simple. 3658 +/ 3659 size_t initialReceiveBufferSize = 4096; 3660 size_t likelyReceiveBufferSize = 4096; /// ditto 3661 size_t maximumReceiveBufferSize = 10 * 1024 * 1024; /// ditto 3662 3663 /++ 3664 Maximum combined size of a message. 3665 +/ 3666 size_t maximumMessageSize = 10 * 1024 * 1024; 3667 3668 string[string] cookies; /// Cookies to send with the initial request. cookies[name] = value; 3669 string origin; /// Origin URL to send with the handshake, if desired. 3670 string protocol; /// the protocol header, if desired. 3671 3672 /++ 3673 Additional headers to put in the HTTP request. These should be formatted `Name: value`, like for example: 3674 3675 --- 3676 Config config; 3677 config.additionalHeaders ~= "Authorization: Bearer your_auth_token_here"; 3678 --- 3679 3680 History: 3681 Added February 19, 2021 (included in dub version 9.2) 3682 +/ 3683 string[] additionalHeaders; 3684 3685 int pingFrequency = 5000; /// Amount of time (in msecs) of idleness after which to send an automatic ping 3686 3687 /++ 3688 Amount of time to disconnect when there's no activity. Note that automatic pings will keep the connection alive; this timeout only occurs if there's absolutely nothing, including no responses to websocket ping frames. Since the default [pingFrequency] is only seconds, this one minute should never elapse unless the connection is actually dead. 3689 3690 The one thing to keep in mind is if your program is busy and doesn't check input, it might consider this a time out since there's no activity. The reason is that your program was busy rather than a connection failure, but it doesn't care. You should avoid long processing periods anyway though! 3691 3692 History: 3693 Added March 31, 2021 (included in dub version 9.4) 3694 +/ 3695 Duration timeoutFromInactivity = 1.minutes; 3696 } 3697 3698 /++ 3699 Returns one of [CONNECTING], [OPEN], [CLOSING], or [CLOSED]. 3700 +/ 3701 int readyState() { 3702 return readyState_; 3703 } 3704 3705 /++ 3706 Closes the connection, sending a graceful teardown message to the other side. 3707 +/ 3708 /// Group: foundational 3709 void close(int code = 0, string reason = null) 3710 //in (reason.length < 123) 3711 in { assert(reason.length < 123); } do 3712 { 3713 if(readyState_ != OPEN) 3714 return; // it cool, we done 3715 WebSocketFrame wss; 3716 wss.fin = true; 3717 wss.opcode = WebSocketOpcode.close; 3718 wss.data = cast(ubyte[]) reason.dup; 3719 wss.send(&llsend); 3720 3721 readyState_ = CLOSING; 3722 3723 llclose(); 3724 } 3725 3726 /++ 3727 Sends a ping message to the server. This is done automatically by the library if you set a non-zero [Config.pingFrequency], but you can also send extra pings explicitly as well with this function. 3728 +/ 3729 /// Group: foundational 3730 void ping() { 3731 WebSocketFrame wss; 3732 wss.fin = true; 3733 wss.opcode = WebSocketOpcode.ping; 3734 wss.send(&llsend); 3735 } 3736 3737 // automatically handled.... 3738 void pong() { 3739 WebSocketFrame wss; 3740 wss.fin = true; 3741 wss.opcode = WebSocketOpcode.pong; 3742 wss.send(&llsend); 3743 } 3744 3745 /++ 3746 Sends a text message through the websocket. 3747 +/ 3748 /// Group: foundational 3749 void send(in char[] textData) { 3750 WebSocketFrame wss; 3751 wss.fin = true; 3752 wss.masked = this.isClient; 3753 wss.opcode = WebSocketOpcode.text; 3754 wss.data = cast(ubyte[]) textData.dup; 3755 wss.send(&llsend); 3756 } 3757 3758 /++ 3759 Sends a binary message through the websocket. 3760 +/ 3761 /// Group: foundational 3762 void send(in ubyte[] binaryData) { 3763 WebSocketFrame wss; 3764 wss.masked = this.isClient; 3765 wss.fin = true; 3766 wss.opcode = WebSocketOpcode.binary; 3767 wss.data = cast(ubyte[]) binaryData.dup; 3768 wss.send(&llsend); 3769 } 3770 3771 /++ 3772 Waits for and returns the next complete message on the socket. 3773 3774 Note that the onmessage function is still called, right before 3775 this returns. 3776 +/ 3777 /// Group: blocking_api 3778 public WebSocketFrame waitForNextMessage() { 3779 do { 3780 auto m = processOnce(); 3781 if(m.populated) 3782 return m; 3783 } while(lowLevelReceive()); 3784 3785 return WebSocketFrame.init; // FIXME? maybe. 3786 } 3787 3788 /++ 3789 Tells if [waitForNextMessage] would block. 3790 +/ 3791 /// Group: blocking_api 3792 public bool waitForNextMessageWouldBlock() { 3793 checkAgain: 3794 if(isMessageBuffered()) 3795 return false; 3796 if(!isDataPending()) 3797 return true; 3798 while(isDataPending()) 3799 lowLevelReceive(); 3800 goto checkAgain; 3801 } 3802 3803 /++ 3804 Is there a message in the buffer already? 3805 If `true`, [waitForNextMessage] is guaranteed to return immediately. 3806 If `false`, check [isDataPending] as the next step. 3807 +/ 3808 /// Group: blocking_api 3809 public bool isMessageBuffered() { 3810 ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; 3811 auto s = d; 3812 if(d.length) { 3813 auto orig = d; 3814 auto m = WebSocketFrame.read(d); 3815 // that's how it indicates that it needs more data 3816 if(d !is orig) 3817 return true; 3818 } 3819 3820 return false; 3821 } 3822 3823 private ubyte continuingType; 3824 private ubyte[] continuingData; 3825 //private size_t continuingDataLength; 3826 3827 private WebSocketFrame processOnce() { 3828 ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; 3829 //import std.stdio; writeln(d); 3830 auto s = d; 3831 // FIXME: handle continuation frames more efficiently. it should really just reuse the receive buffer. 3832 WebSocketFrame m; 3833 if(d.length) { 3834 auto orig = d; 3835 m = WebSocketFrame.read(d); 3836 // that's how it indicates that it needs more data 3837 if(d is orig) 3838 return WebSocketFrame.init; 3839 m.unmaskInPlace(); 3840 switch(m.opcode) { 3841 case WebSocketOpcode.continuation: 3842 if(continuingData.length + m.data.length > config.maximumMessageSize) 3843 throw new Exception("message size exceeded"); 3844 3845 continuingData ~= m.data; 3846 if(m.fin) { 3847 if(ontextmessage) 3848 ontextmessage(cast(char[]) continuingData); 3849 if(onbinarymessage) 3850 onbinarymessage(continuingData); 3851 3852 continuingData = null; 3853 } 3854 break; 3855 case WebSocketOpcode.text: 3856 if(m.fin) { 3857 if(ontextmessage) 3858 ontextmessage(m.textData); 3859 } else { 3860 continuingType = m.opcode; 3861 //continuingDataLength = 0; 3862 continuingData = null; 3863 continuingData ~= m.data; 3864 } 3865 break; 3866 case WebSocketOpcode.binary: 3867 if(m.fin) { 3868 if(onbinarymessage) 3869 onbinarymessage(m.data); 3870 } else { 3871 continuingType = m.opcode; 3872 //continuingDataLength = 0; 3873 continuingData = null; 3874 continuingData ~= m.data; 3875 } 3876 break; 3877 case WebSocketOpcode.close: 3878 readyState_ = CLOSED; 3879 if(onclose) 3880 onclose(); 3881 3882 unregisterActiveSocket(this); 3883 break; 3884 case WebSocketOpcode.ping: 3885 pong(); 3886 break; 3887 case WebSocketOpcode.pong: 3888 // just really references it is still alive, nbd. 3889 break; 3890 default: // ignore though i could and perhaps should throw too 3891 } 3892 } 3893 3894 if(d.length) { 3895 m.data = m.data.dup(); 3896 } 3897 3898 import core.stdc.string; 3899 memmove(receiveBuffer.ptr, d.ptr, d.length); 3900 receiveBufferUsedLength = d.length; 3901 3902 return m; 3903 } 3904 3905 private void autoprocess() { 3906 // FIXME 3907 do { 3908 processOnce(); 3909 } while(lowLevelReceive()); 3910 } 3911 3912 3913 void delegate() onclose; /// 3914 void delegate() onerror; /// 3915 void delegate(in char[]) ontextmessage; /// 3916 void delegate(in ubyte[]) onbinarymessage; /// 3917 void delegate() onopen; /// 3918 3919 /++ 3920 3921 +/ 3922 /// Group: browser_api 3923 void onmessage(void delegate(in char[]) dg) { 3924 ontextmessage = dg; 3925 } 3926 3927 /// ditto 3928 void onmessage(void delegate(in ubyte[]) dg) { 3929 onbinarymessage = dg; 3930 } 3931 3932 /* } end copy/paste */ 3933 3934 /* 3935 const int bufferedAmount // amount pending 3936 const string extensions 3937 3938 const string protocol 3939 const string url 3940 */ 3941 3942 static { 3943 /++ 3944 3945 +/ 3946 void eventLoop() { 3947 3948 static SocketSet readSet; 3949 3950 if(readSet is null) 3951 readSet = new SocketSet(); 3952 3953 loopExited = false; 3954 3955 outermost: while(!loopExited) { 3956 readSet.reset(); 3957 3958 Duration timeout = 10.seconds; 3959 3960 auto now = MonoTime.currTime; 3961 bool hadAny; 3962 foreach(sock; activeSockets) { 3963 if(now >= sock.timeoutFromInactivity) { 3964 // timeout 3965 if(sock.onerror) 3966 sock.onerror(); 3967 3968 sock.socket.close(); 3969 sock.readyState_ = CLOSED; 3970 unregisterActiveSocket(sock); 3971 continue outermost; 3972 } 3973 3974 if(now >= sock.nextPing) { 3975 sock.ping(); 3976 sock.nextPing = now + sock.config.pingFrequency.msecs; 3977 } 3978 3979 auto timeo = sock.timeoutFromInactivity - now; 3980 if(timeo < timeout) 3981 timeout = timeo; 3982 3983 readSet.add(sock.socket); 3984 hadAny = true; 3985 } 3986 3987 if(!hadAny) 3988 return; 3989 3990 tryAgain: 3991 auto selectGot = Socket.select(readSet, null, null, timeout); 3992 if(selectGot == 0) { /* timeout */ 3993 // timeout 3994 continue; // it will be handled at the top of the loop 3995 } else if(selectGot == -1) { /* interrupted */ 3996 goto tryAgain; 3997 } else { 3998 foreach(sock; activeSockets) { 3999 if(readSet.isSet(sock.socket)) { 4000 sock.timeoutFromInactivity = MonoTime.currTime + sock.config.timeoutFromInactivity; 4001 if(!sock.lowLevelReceive()) { 4002 sock.readyState_ = CLOSED; 4003 unregisterActiveSocket(sock); 4004 continue outermost; 4005 } 4006 while(sock.processOnce().populated) {} 4007 selectGot--; 4008 if(selectGot <= 0) 4009 break; 4010 } 4011 } 4012 } 4013 } 4014 } 4015 4016 private bool loopExited; 4017 /++ 4018 4019 +/ 4020 void exitEventLoop() { 4021 loopExited = true; 4022 } 4023 4024 WebSocket[] activeSockets; 4025 void registerActiveSocket(WebSocket s) { 4026 activeSockets ~= s; 4027 } 4028 void unregisterActiveSocket(WebSocket s) { 4029 foreach(i, a; activeSockets) 4030 if(s is a) { 4031 activeSockets[i] = activeSockets[$-1]; 4032 activeSockets = activeSockets[0 .. $-1]; 4033 break; 4034 } 4035 } 4036 } 4037 } 4038 4039 /++ 4040 Warning: you should call this AFTER websocket.connect or else it might throw on connect because the function sets nonblocking mode and the connect function doesn't handle that well (it throws on the "would block" condition in that function. easier to just do that first) 4041 +/ 4042 template addToSimpledisplayEventLoop() { 4043 import arsd.simpledisplay; 4044 void addToSimpledisplayEventLoop(WebSocket ws, SimpleWindow window) { 4045 4046 void midprocess() { 4047 if(!ws.lowLevelReceive()) { 4048 ws.readyState_ = WebSocket.CLOSED; 4049 WebSocket.unregisterActiveSocket(ws); 4050 return; 4051 } 4052 while(ws.processOnce().populated) {} 4053 } 4054 4055 version(Posix) { 4056 auto reader = new PosixFdReader(&midprocess, ws.socket.handle); 4057 } else version(none) { 4058 if(WSAAsyncSelect(ws.socket.handle, window.hwnd, WM_USER + 150, FD_CLOSE | FD_READ)) 4059 throw new Exception("WSAAsyncSelect"); 4060 4061 window.handleNativeEvent = delegate int(HWND hwnd, UINT msg, WPARAM wParam, LPARAM lParam) { 4062 if(hwnd !is window.impl.hwnd) 4063 return 1; // we don't care... 4064 switch(msg) { 4065 case WM_USER + 150: // socket activity 4066 switch(LOWORD(lParam)) { 4067 case FD_READ: 4068 case FD_CLOSE: 4069 midprocess(); 4070 break; 4071 default: 4072 // nothing 4073 } 4074 break; 4075 default: return 1; // not handled, pass it on 4076 } 4077 return 0; 4078 }; 4079 4080 } else version(Windows) { 4081 ws.socket.blocking = false; // the WSAEventSelect does this anyway and doing it here lets phobos know about it. 4082 //CreateEvent(null, 0, 0, null); 4083 auto event = WSACreateEvent(); 4084 if(!event) { 4085 throw new Exception("WSACreateEvent"); 4086 } 4087 if(WSAEventSelect(ws.socket.handle, event, 1/*FD_READ*/ | (1<<5)/*FD_CLOSE*/)) { 4088 //import std.stdio; writeln(WSAGetLastError()); 4089 throw new Exception("WSAEventSelect"); 4090 } 4091 4092 auto handle = new WindowsHandleReader(&midprocess, event); 4093 4094 /+ 4095 static class Ready {} 4096 4097 Ready thisr = new Ready; 4098 4099 justCommunication.addEventListener((Ready r) { 4100 if(r is thisr) 4101 midprocess(); 4102 }); 4103 4104 import core.thread; 4105 auto thread = new Thread({ 4106 while(true) { 4107 WSAWaitForMultipleEvents(1, &event, true, -1/*WSA_INFINITE*/, false); 4108 justCommunication.postEvent(thisr); 4109 } 4110 }); 4111 thread.isDaemon = true; 4112 thread.start; 4113 +/ 4114 4115 } else static assert(0, "unsupported OS"); 4116 } 4117 } 4118 4119 version(Windows) { 4120 import core.sys.windows.windows; 4121 import core.sys.windows.winsock2; 4122 } 4123 4124 version(none) { 4125 extern(Windows) int WSAAsyncSelect(SOCKET, HWND, uint, int); 4126 enum int FD_CLOSE = 1 << 5; 4127 enum int FD_READ = 1 << 0; 4128 enum int WM_USER = 1024; 4129 } 4130 4131 version(Windows) { 4132 import core.stdc.config; 4133 extern(Windows) 4134 int WSAEventSelect(SOCKET, HANDLE /* to an Event */, c_long); 4135 4136 extern(Windows) 4137 HANDLE WSACreateEvent(); 4138 4139 extern(Windows) 4140 DWORD WSAWaitForMultipleEvents(DWORD, HANDLE*, BOOL, DWORD, BOOL); 4141 } 4142 4143 /* copy/paste from cgi.d */ 4144 public { 4145 enum WebSocketOpcode : ubyte { 4146 continuation = 0, 4147 text = 1, 4148 binary = 2, 4149 // 3, 4, 5, 6, 7 RESERVED 4150 close = 8, 4151 ping = 9, 4152 pong = 10, 4153 // 11,12,13,14,15 RESERVED 4154 } 4155 4156 public struct WebSocketFrame { 4157 private bool populated; 4158 bool fin; 4159 bool rsv1; 4160 bool rsv2; 4161 bool rsv3; 4162 WebSocketOpcode opcode; // 4 bits 4163 bool masked; 4164 ubyte lengthIndicator; // don't set this when building one to send 4165 ulong realLength; // don't use when sending 4166 ubyte[4] maskingKey; // don't set this when sending 4167 ubyte[] data; 4168 4169 static WebSocketFrame simpleMessage(WebSocketOpcode opcode, in void[] data) { 4170 WebSocketFrame msg; 4171 msg.fin = true; 4172 msg.opcode = opcode; 4173 msg.data = cast(ubyte[]) data.dup; // it is mutated below when masked, so need to be cautious and copy it, sigh 4174 4175 return msg; 4176 } 4177 4178 private void send(scope void delegate(ubyte[]) llsend) { 4179 ubyte[64] headerScratch; 4180 int headerScratchPos = 0; 4181 4182 realLength = data.length; 4183 4184 { 4185 ubyte b1; 4186 b1 |= cast(ubyte) opcode; 4187 b1 |= rsv3 ? (1 << 4) : 0; 4188 b1 |= rsv2 ? (1 << 5) : 0; 4189 b1 |= rsv1 ? (1 << 6) : 0; 4190 b1 |= fin ? (1 << 7) : 0; 4191 4192 headerScratch[0] = b1; 4193 headerScratchPos++; 4194 } 4195 4196 { 4197 headerScratchPos++; // we'll set header[1] at the end of this 4198 auto rlc = realLength; 4199 ubyte b2; 4200 b2 |= masked ? (1 << 7) : 0; 4201 4202 assert(headerScratchPos == 2); 4203 4204 if(realLength > 65535) { 4205 // use 64 bit length 4206 b2 |= 0x7f; 4207 4208 // FIXME: double check endinaness 4209 foreach(i; 0 .. 8) { 4210 headerScratch[2 + 7 - i] = rlc & 0x0ff; 4211 rlc >>>= 8; 4212 } 4213 4214 headerScratchPos += 8; 4215 } else if(realLength > 125) { 4216 // use 16 bit length 4217 b2 |= 0x7e; 4218 4219 // FIXME: double check endinaness 4220 foreach(i; 0 .. 2) { 4221 headerScratch[2 + 1 - i] = rlc & 0x0ff; 4222 rlc >>>= 8; 4223 } 4224 4225 headerScratchPos += 2; 4226 } else { 4227 // use 7 bit length 4228 b2 |= realLength & 0b_0111_1111; 4229 } 4230 4231 headerScratch[1] = b2; 4232 } 4233 4234 //assert(!masked, "masking key not properly implemented"); 4235 if(masked) { 4236 import std.random; 4237 foreach(ref item; maskingKey) 4238 item = uniform(ubyte.min, ubyte.max); 4239 headerScratch[headerScratchPos .. headerScratchPos + 4] = maskingKey[]; 4240 headerScratchPos += 4; 4241 4242 // we'll just mask it in place... 4243 int keyIdx = 0; 4244 foreach(i; 0 .. data.length) { 4245 data[i] = data[i] ^ maskingKey[keyIdx]; 4246 if(keyIdx == 3) 4247 keyIdx = 0; 4248 else 4249 keyIdx++; 4250 } 4251 } 4252 4253 //writeln("SENDING ", headerScratch[0 .. headerScratchPos], data); 4254 llsend(headerScratch[0 .. headerScratchPos]); 4255 llsend(data); 4256 } 4257 4258 static WebSocketFrame read(ref ubyte[] d) { 4259 WebSocketFrame msg; 4260 4261 auto orig = d; 4262 4263 WebSocketFrame needsMoreData() { 4264 d = orig; 4265 return WebSocketFrame.init; 4266 } 4267 4268 if(d.length < 2) 4269 return needsMoreData(); 4270 4271 ubyte b = d[0]; 4272 4273 msg.populated = true; 4274 4275 msg.opcode = cast(WebSocketOpcode) (b & 0x0f); 4276 b >>= 4; 4277 msg.rsv3 = b & 0x01; 4278 b >>= 1; 4279 msg.rsv2 = b & 0x01; 4280 b >>= 1; 4281 msg.rsv1 = b & 0x01; 4282 b >>= 1; 4283 msg.fin = b & 0x01; 4284 4285 b = d[1]; 4286 msg.masked = (b & 0b1000_0000) ? true : false; 4287 msg.lengthIndicator = b & 0b0111_1111; 4288 4289 d = d[2 .. $]; 4290 4291 if(msg.lengthIndicator == 0x7e) { 4292 // 16 bit length 4293 msg.realLength = 0; 4294 4295 if(d.length < 2) return needsMoreData(); 4296 4297 foreach(i; 0 .. 2) { 4298 msg.realLength |= d[0] << ((1-i) * 8); 4299 d = d[1 .. $]; 4300 } 4301 } else if(msg.lengthIndicator == 0x7f) { 4302 // 64 bit length 4303 msg.realLength = 0; 4304 4305 if(d.length < 8) return needsMoreData(); 4306 4307 foreach(i; 0 .. 8) { 4308 msg.realLength |= ulong(d[0]) << ((7-i) * 8); 4309 d = d[1 .. $]; 4310 } 4311 } else { 4312 // 7 bit length 4313 msg.realLength = msg.lengthIndicator; 4314 } 4315 4316 if(msg.masked) { 4317 4318 if(d.length < 4) return needsMoreData(); 4319 4320 msg.maskingKey = d[0 .. 4]; 4321 d = d[4 .. $]; 4322 } 4323 4324 if(msg.realLength > d.length) { 4325 return needsMoreData(); 4326 } 4327 4328 msg.data = d[0 .. cast(size_t) msg.realLength]; 4329 d = d[cast(size_t) msg.realLength .. $]; 4330 4331 return msg; 4332 } 4333 4334 void unmaskInPlace() { 4335 if(this.masked) { 4336 int keyIdx = 0; 4337 foreach(i; 0 .. this.data.length) { 4338 this.data[i] = this.data[i] ^ this.maskingKey[keyIdx]; 4339 if(keyIdx == 3) 4340 keyIdx = 0; 4341 else 4342 keyIdx++; 4343 } 4344 } 4345 } 4346 4347 char[] textData() { 4348 return cast(char[]) data; 4349 } 4350 } 4351 } 4352 4353 /+ 4354 so the url params are arguments. it knows the request 4355 internally. other params are properties on the req 4356 4357 names may have different paths... those will just add ForSomething i think. 4358 4359 auto req = api.listMergeRequests 4360 req.page = 10; 4361 4362 or 4363 req.page(1) 4364 .bar("foo") 4365 4366 req.execute(); 4367 4368 4369 everything in the response is nullable access through the 4370 dynamic object, just with property getters there. need to make 4371 it static generated tho 4372 4373 other messages may be: isPresent and getDynamic 4374 4375 4376 AND/OR what about doing it like the rails objects 4377 4378 BroadcastMessage.get(4) 4379 // various properties 4380 4381 // it lists what you updated 4382 4383 BroadcastMessage.foo().bar().put(5) 4384 +/