1 // Copyright 2013-2021, Adam D. Ruppe. 2 3 // FIXME: websocket proxy support 4 // FIXME: ipv6 support 5 6 // FIXME: headers are supposed to be case insensitive. ugh. 7 8 // FIXME: 100 continue. tho we never Expect it so should never happen, never kno, 9 10 /++ 11 This is version 2 of my http/1.1 client implementation. 12 13 14 It has no dependencies for basic operation, but does require OpenSSL 15 libraries (or compatible) to be support HTTPS. This dynamically loaded 16 on-demand (meaning it won't be loaded if you don't use it, but if you do 17 use it, the openssl dynamic libraries must be found in the system search path). 18 19 You can compile with `-version=without_openssl` to entirely disable ssl support. 20 21 http2.d, despite its name, does NOT implement HTTP/2.0, but this 22 shouldn't matter for 99.9% of usage, since all servers will continue 23 to support HTTP/1.1 for a very long time. 24 +/ 25 module arsd.http2; 26 27 /// 28 unittest { 29 import arsd.http2; 30 31 void main() { 32 auto client = new HttpClient(); 33 34 auto request = client.request(Uri("http://dlang.org/")); 35 auto response = request.waitForCompletion(); 36 37 import std.stdio; 38 writeln(response.contentText); 39 writeln(response.code, " ", response.codeText); 40 writeln(response.contentType); 41 } 42 43 version(arsd_http2_integration_test) main(); // exclude from docs 44 } 45 46 // FIXME: I think I want to disable sigpipe here too. 47 48 import std.uri : encodeComponent; 49 50 debug(arsd_http2_verbose) debug=arsd_http2; 51 52 debug(arsd_http2) import std.stdio : writeln; 53 54 version=arsd_http_internal_implementation; 55 56 version(without_openssl) {} 57 else { 58 version=use_openssl; 59 version=with_openssl; 60 version(older_openssl) {} else 61 version=newer_openssl; 62 } 63 64 version(arsd_http_winhttp_implementation) { 65 pragma(lib, "winhttp") 66 import core.sys.windows.winhttp; 67 // FIXME: alter the dub package file too 68 69 // https://github.com/curl/curl/blob/master/lib/vtls/schannel.c 70 // https://docs.microsoft.com/en-us/windows/win32/secauthn/creating-an-schannel-security-context 71 72 73 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpreaddata 74 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpsendrequest 75 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpopenrequest 76 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpconnect 77 } 78 79 80 81 /++ 82 Demonstrates core functionality, using the [HttpClient], 83 [HttpRequest] (returned by [HttpClient.navigateTo|client.navigateTo]), 84 and [HttpResponse] (returned by [HttpRequest.waitForCompletion|request.waitForCompletion]). 85 86 +/ 87 unittest { 88 import arsd.http2; 89 90 void main() { 91 auto client = new HttpClient(); 92 auto request = client.navigateTo(Uri("http://dlang.org/")); 93 auto response = request.waitForCompletion(); 94 95 string returnedHtml = response.contentText; 96 } 97 } 98 99 // FIXME: multipart encoded file uploads needs implementation 100 // future: do web client api stuff 101 102 __gshared bool defaultVerifyPeer = true; // public but intentionally undocumented 103 104 debug import std.stdio; 105 106 import std.socket; 107 import core.time; 108 109 // FIXME: check Transfer-Encoding: gzip always 110 111 version(with_openssl) { 112 //pragma(lib, "crypto"); 113 //pragma(lib, "ssl"); 114 } 115 116 /+ 117 HttpRequest httpRequest(string method, string url, ubyte[] content, string[string] content) { 118 return null; 119 } 120 +/ 121 122 /** 123 auto request = get("http://arsdnet.net/"); 124 request.send(); 125 126 auto response = get("http://arsdnet.net/").waitForCompletion(); 127 */ 128 HttpRequest get(string url) { 129 auto client = new HttpClient(); 130 auto request = client.navigateTo(Uri(url)); 131 return request; 132 } 133 134 /** 135 Do not forget to call `waitForCompletion()` on the returned object! 136 */ 137 HttpRequest post(string url, string[string] req) { 138 auto client = new HttpClient(); 139 ubyte[] bdata; 140 foreach(k, v; req) { 141 if(bdata.length) 142 bdata ~= cast(ubyte[]) "&"; 143 bdata ~= cast(ubyte[]) encodeComponent(k); 144 bdata ~= cast(ubyte[]) "="; 145 bdata ~= cast(ubyte[]) encodeComponent(v); 146 } 147 auto request = client.request(Uri(url), HttpVerb.POST, bdata, "application/x-www-form-urlencoded"); 148 return request; 149 } 150 151 /// gets the text off a url. basic operation only. 152 string getText(string url) { 153 auto request = get(url); 154 auto response = request.waitForCompletion(); 155 return cast(string) response.content; 156 } 157 158 /+ 159 ubyte[] getBinary(string url, string[string] cookies = null) { 160 auto hr = httpRequest("GET", url, null, cookies); 161 if(hr.code != 200) 162 throw new Exception(format("HTTP answered %d instead of 200 on %s", hr.code, url)); 163 return hr.content; 164 } 165 166 /** 167 Gets a textual document, ignoring headers. Throws on non-text or error. 168 */ 169 string get(string url, string[string] cookies = null) { 170 auto hr = httpRequest("GET", url, null, cookies); 171 if(hr.code != 200) 172 throw new Exception(format("HTTP answered %d instead of 200 on %s", hr.code, url)); 173 if(hr.contentType.indexOf("text/") == -1) 174 throw new Exception(hr.contentType ~ " is bad content for conversion to string"); 175 return cast(string) hr.content; 176 177 } 178 179 static import std.uri; 180 181 string post(string url, string[string] args, string[string] cookies = null) { 182 string content; 183 184 foreach(name, arg; args) { 185 if(content.length) 186 content ~= "&"; 187 content ~= std.uri.encode(name) ~ "=" ~ std.uri.encode(arg); 188 } 189 190 auto hr = httpRequest("POST", url, cast(ubyte[]) content, cookies, ["Content-Type: application/x-www-form-urlencoded"]); 191 if(hr.code != 200) 192 throw new Exception(format("HTTP answered %d instead of 200", hr.code)); 193 if(hr.contentType.indexOf("text/") == -1) 194 throw new Exception(hr.contentType ~ " is bad content for conversion to string"); 195 196 return cast(string) hr.content; 197 } 198 199 +/ 200 201 /// 202 struct HttpResponse { 203 /++ 204 The HTTP response code, if the response was completed, or some value < 100 if it was aborted or failed. 205 206 Code 0 - initial value, nothing happened 207 Code 1 - you called request.abort 208 Code 2 - connection refused 209 Code 3 - connection succeeded, but server disconnected early 210 Code 4 - server sent corrupted response (or this code has a bug and processed it wrong) 211 Code 5 - request timed out 212 213 Code >= 100 - a HTTP response 214 +/ 215 int code; 216 string codeText; /// 217 218 string httpVersion; /// 219 220 string statusLine; /// 221 222 string contentType; /// The content type header 223 string location; /// The location header 224 225 /++ 226 227 History: 228 Added December 5, 2020 (version 9.1) 229 +/ 230 bool wasSuccessful() { 231 return code >= 200 && code < 400; 232 } 233 234 /// the charset out of content type, if present. `null` if not. 235 string contentTypeCharset() { 236 auto idx = contentType.indexOf("charset="); 237 if(idx == -1) 238 return null; 239 auto c = contentType[idx + "charset=".length .. $].strip; 240 if(c.length) 241 return c; 242 return null; 243 } 244 245 string[string] cookies; /// Names and values of cookies set in the response. 246 247 string[] headers; /// Array of all headers returned. 248 string[string] headersHash; /// 249 250 ubyte[] content; /// The raw content returned in the response body. 251 string contentText; /// [content], but casted to string (for convenience) 252 253 alias responseText = contentText; // just cuz I do this so often. 254 //alias body = content; 255 256 /++ 257 returns `new Document(this.contentText)`. Requires [arsd.dom]. 258 +/ 259 auto contentDom()() { 260 import arsd.dom; 261 return new Document(this.contentText); 262 263 } 264 265 /++ 266 returns `var.fromJson(this.contentText)`. Requires [arsd.jsvar]. 267 +/ 268 auto contentJson()() { 269 import arsd.jsvar; 270 return var.fromJson(this.contentText); 271 } 272 273 HttpRequestParameters requestParameters; /// 274 275 LinkHeader[] linksStored; 276 bool linksLazilyParsed; 277 278 HttpResponse deepCopy() const { 279 HttpResponse h = cast(HttpResponse) this; 280 h.cookies = h.cookies.dup; 281 h.headers = h.headers.dup; 282 h.headersHash = h.headersHash.dup; 283 h.content = h.content.dup; 284 h.linksStored = h.linksStored.dup; 285 return h; 286 } 287 288 /// Returns links header sorted by "rel" attribute. 289 /// It returns a new array on each call. 290 LinkHeader[string] linksHash() { 291 auto links = this.links(); 292 LinkHeader[string] ret; 293 foreach(link; links) 294 ret[link.rel] = link; 295 return ret; 296 } 297 298 /// Returns the Link header, parsed. 299 LinkHeader[] links() { 300 if(linksLazilyParsed) 301 return linksStored; 302 linksLazilyParsed = true; 303 LinkHeader[] ret; 304 305 auto hdrPtr = "Link" in headersHash; 306 if(hdrPtr is null) 307 return ret; 308 309 auto header = *hdrPtr; 310 311 LinkHeader current; 312 313 while(header.length) { 314 char ch = header[0]; 315 316 if(ch == '<') { 317 // read url 318 header = header[1 .. $]; 319 size_t idx; 320 while(idx < header.length && header[idx] != '>') 321 idx++; 322 current.url = header[0 .. idx]; 323 header = header[idx .. $]; 324 } else if(ch == ';') { 325 // read attribute 326 header = header[1 .. $]; 327 header = header.stripLeft; 328 329 size_t idx; 330 while(idx < header.length && header[idx] != '=') 331 idx++; 332 333 string name = header[0 .. idx]; 334 header = header[idx + 1 .. $]; 335 336 string value; 337 338 if(header.length && header[0] == '"') { 339 // quoted value 340 header = header[1 .. $]; 341 idx = 0; 342 while(idx < header.length && header[idx] != '\"') 343 idx++; 344 value = header[0 .. idx]; 345 header = header[idx .. $]; 346 347 } else if(header.length) { 348 // unquoted value 349 idx = 0; 350 while(idx < header.length && header[idx] != ',' && header[idx] != ' ' && header[idx] != ';') 351 idx++; 352 353 value = header[0 .. idx]; 354 header = header[idx .. $].stripLeft; 355 } 356 357 name = name.toLower; 358 if(name == "rel") 359 current.rel = value; 360 else 361 current.attributes[name] = value; 362 363 } else if(ch == ',') { 364 // start another 365 ret ~= current; 366 current = LinkHeader.init; 367 } else if(ch == ' ' || ch == '\n' || ch == '\r' || ch == '\t') { 368 // ignore 369 } 370 371 if(header.length) 372 header = header[1 .. $]; 373 } 374 375 ret ~= current; 376 377 linksStored = ret; 378 379 return ret; 380 } 381 } 382 383 /// 384 struct LinkHeader { 385 string url; /// 386 string rel; /// 387 string[string] attributes; /// like title, rev, media, whatever attributes 388 } 389 390 import std.string; 391 static import std.algorithm; 392 import std.conv; 393 import std.range; 394 395 396 private AddressFamily family(string unixSocketPath) { 397 if(unixSocketPath.length) 398 return AddressFamily.UNIX; 399 else // FIXME: what about ipv6? 400 return AddressFamily.INET; 401 } 402 403 version(Windows) 404 private class UnixAddress : Address { 405 this(string) { 406 throw new Exception("No unix address support on this system in lib yet :("); 407 } 408 override sockaddr* name() { assert(0); } 409 override const(sockaddr)* name() const { assert(0); } 410 override int nameLen() const { assert(0); } 411 } 412 413 414 // Copy pasta from cgi.d, then stripped down. unix path thing added tho 415 /// 416 struct Uri { 417 alias toString this; // blargh idk a url really is a string, but should it be implicit? 418 419 // scheme//userinfo@host:port/path?query#fragment 420 421 string scheme; /// e.g. "http" in "http://example.com/" 422 string userinfo; /// the username (and possibly a password) in the uri 423 string host; /// the domain name 424 int port; /// port number, if given. Will be zero if a port was not explicitly given 425 string path; /// e.g. "/folder/file.html" in "http://example.com/folder/file.html" 426 string query; /// the stuff after the ? in a uri 427 string fragment; /// the stuff after the # in a uri. 428 429 /// Breaks down a uri string to its components 430 this(string uri) { 431 reparse(uri); 432 } 433 434 private string unixSocketPath = null; 435 /// Indicates it should be accessed through a unix socket instead of regular tcp. Returns new version without modifying this object. 436 Uri viaUnixSocket(string path) const { 437 Uri copy = this; 438 copy.unixSocketPath = path; 439 return copy; 440 } 441 442 /// Goes through a unix socket in the abstract namespace (linux only). Returns new version without modifying this object. 443 version(linux) 444 Uri viaAbstractSocket(string path) const { 445 Uri copy = this; 446 copy.unixSocketPath = "\0" ~ path; 447 return copy; 448 } 449 450 private void reparse(string uri) { 451 // from RFC 3986 452 // the ctRegex triples the compile time and makes ugly errors for no real benefit 453 // it was a nice experiment but just not worth it. 454 // enum ctr = ctRegex!r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?"; 455 /* 456 Captures: 457 0 = whole url 458 1 = scheme, with : 459 2 = scheme, no : 460 3 = authority, with // 461 4 = authority, no // 462 5 = path 463 6 = query string, with ? 464 7 = query string, no ? 465 8 = anchor, with # 466 9 = anchor, no # 467 */ 468 // Yikes, even regular, non-CT regex is also unacceptably slow to compile. 1.9s on my computer! 469 // instead, I will DIY and cut that down to 0.6s on the same computer. 470 /* 471 472 Note that authority is 473 user:password@domain:port 474 where the user:password@ part is optional, and the :port is optional. 475 476 Regex translation: 477 478 Scheme cannot have :, /, ?, or # in it, and must have one or more chars and end in a :. It is optional, but must be first. 479 Authority must start with //, but cannot have any other /, ?, or # in it. It is optional. 480 Path cannot have any ? or # in it. It is optional. 481 Query must start with ? and must not have # in it. It is optional. 482 Anchor must start with # and can have anything else in it to end of string. It is optional. 483 */ 484 485 this = Uri.init; // reset all state 486 487 // empty uri = nothing special 488 if(uri.length == 0) { 489 return; 490 } 491 492 size_t idx; 493 494 scheme_loop: foreach(char c; uri[idx .. $]) { 495 switch(c) { 496 case ':': 497 case '/': 498 case '?': 499 case '#': 500 break scheme_loop; 501 default: 502 } 503 idx++; 504 } 505 506 if(idx == 0 && uri[idx] == ':') { 507 // this is actually a path! we skip way ahead 508 goto path_loop; 509 } 510 511 if(idx == uri.length) { 512 // the whole thing is a path, apparently 513 path = uri; 514 return; 515 } 516 517 if(idx > 0 && uri[idx] == ':') { 518 scheme = uri[0 .. idx]; 519 idx++; 520 } else { 521 // we need to rewind; it found a / but no :, so the whole thing is prolly a path... 522 idx = 0; 523 } 524 525 if(idx + 2 < uri.length && uri[idx .. idx + 2] == "//") { 526 // we have an authority.... 527 idx += 2; 528 529 auto authority_start = idx; 530 authority_loop: foreach(char c; uri[idx .. $]) { 531 switch(c) { 532 case '/': 533 case '?': 534 case '#': 535 break authority_loop; 536 default: 537 } 538 idx++; 539 } 540 541 auto authority = uri[authority_start .. idx]; 542 543 auto idx2 = authority.indexOf("@"); 544 if(idx2 != -1) { 545 userinfo = authority[0 .. idx2]; 546 authority = authority[idx2 + 1 .. $]; 547 } 548 549 if(authority.length && authority[0] == '[') { 550 // ipv6 address special casing 551 idx2 = authority.indexOf(']'); 552 if(idx2 != -1) { 553 auto end = authority[idx2 + 1 .. $]; 554 if(end.length && end[0] == ':') 555 idx2 = idx2 + 1; 556 else 557 idx2 = -1; 558 } 559 } else { 560 idx2 = authority.indexOf(":"); 561 } 562 563 if(idx2 == -1) { 564 port = 0; // 0 means not specified; we should use the default for the scheme 565 host = authority; 566 } else { 567 host = authority[0 .. idx2]; 568 port = to!int(authority[idx2 + 1 .. $]); 569 } 570 } 571 572 path_loop: 573 auto path_start = idx; 574 575 foreach(char c; uri[idx .. $]) { 576 if(c == '?' || c == '#') 577 break; 578 idx++; 579 } 580 581 path = uri[path_start .. idx]; 582 583 if(idx == uri.length) 584 return; // nothing more to examine... 585 586 if(uri[idx] == '?') { 587 idx++; 588 auto query_start = idx; 589 foreach(char c; uri[idx .. $]) { 590 if(c == '#') 591 break; 592 idx++; 593 } 594 query = uri[query_start .. idx]; 595 } 596 597 if(idx < uri.length && uri[idx] == '#') { 598 idx++; 599 fragment = uri[idx .. $]; 600 } 601 602 // uriInvalidated = false; 603 } 604 605 private string rebuildUri() const { 606 string ret; 607 if(scheme.length) 608 ret ~= scheme ~ ":"; 609 if(userinfo.length || host.length) 610 ret ~= "//"; 611 if(userinfo.length) 612 ret ~= userinfo ~ "@"; 613 if(host.length) 614 ret ~= host; 615 if(port) 616 ret ~= ":" ~ to!string(port); 617 618 ret ~= path; 619 620 if(query.length) 621 ret ~= "?" ~ query; 622 623 if(fragment.length) 624 ret ~= "#" ~ fragment; 625 626 // uri = ret; 627 // uriInvalidated = false; 628 return ret; 629 } 630 631 /// Converts the broken down parts back into a complete string 632 string toString() const { 633 // if(uriInvalidated) 634 return rebuildUri(); 635 } 636 637 /// Returns a new absolute Uri given a base. It treats this one as 638 /// relative where possible, but absolute if not. (If protocol, domain, or 639 /// other info is not set, the new one inherits it from the base.) 640 /// 641 /// Browsers use a function like this to figure out links in html. 642 Uri basedOn(in Uri baseUrl) const { 643 Uri n = this; // copies 644 // n.uriInvalidated = true; // make sure we regenerate... 645 646 // userinfo is not inherited... is this wrong? 647 648 // if anything is given in the existing url, we don't use the base anymore. 649 if(n.scheme.empty) { 650 n.scheme = baseUrl.scheme; 651 if(n.host.empty) { 652 n.host = baseUrl.host; 653 if(n.port == 0) { 654 n.port = baseUrl.port; 655 if(n.path.length > 0 && n.path[0] != '/') { 656 auto b = baseUrl.path[0 .. baseUrl.path.lastIndexOf("/") + 1]; 657 if(b.length == 0) 658 b = "/"; 659 n.path = b ~ n.path; 660 } else if(n.path.length == 0) { 661 n.path = baseUrl.path; 662 } 663 } 664 } 665 } 666 667 n.removeDots(); 668 669 // if still basically talking to the same thing, we should inherit the unix path 670 // too since basically the unix path is saying for this service, always use this override. 671 if(n.host == baseUrl.host && n.scheme == baseUrl.scheme && n.port == baseUrl.port) 672 n.unixSocketPath = baseUrl.unixSocketPath; 673 674 return n; 675 } 676 677 void removeDots() { 678 auto parts = this.path.split("/"); 679 string[] toKeep; 680 foreach(part; parts) { 681 if(part == ".") { 682 continue; 683 } else if(part == "..") { 684 //if(toKeep.length > 1) 685 toKeep = toKeep[0 .. $-1]; 686 //else 687 //toKeep = [""]; 688 continue; 689 } else { 690 //if(toKeep.length && toKeep[$-1].length == 0 && part.length == 0) 691 //continue; // skip a `//` situation 692 toKeep ~= part; 693 } 694 } 695 696 auto path = toKeep.join("/"); 697 if(path.length && path[0] != '/') 698 path = "/" ~ path; 699 700 this.path = path; 701 } 702 } 703 704 /* 705 void main(string args[]) { 706 write(post("http://arsdnet.net/bugs.php", ["test" : "hey", "again" : "what"])); 707 } 708 */ 709 710 /// 711 struct BasicAuth { 712 string username; /// 713 string password; /// 714 } 715 716 class ProxyException : Exception { 717 this(string msg) {super(msg); } 718 } 719 720 /** 721 Represents a HTTP request. You usually create these through a [HttpClient]. 722 723 724 --- 725 auto request = new HttpRequest(); 726 // set any properties here 727 728 // synchronous usage 729 auto reply = request.perform(); 730 731 // async usage, type 1: 732 request.send(); 733 request2.send(); 734 735 // wait until the first one is done, with the second one still in-flight 736 auto response = request.waitForCompletion(); 737 738 // async usage, type 2: 739 request.onDataReceived = (HttpRequest hr) { 740 if(hr.state == HttpRequest.State.complete) { 741 // use hr.responseData 742 } 743 }; 744 request.send(); // send, using the callback 745 746 // before terminating, be sure you wait for your requests to finish! 747 748 request.waitForCompletion(); 749 --- 750 */ 751 class HttpRequest { 752 753 /// Automatically follow a redirection? 754 bool followLocation = false; 755 756 this() { 757 } 758 759 /// 760 this(Uri where, HttpVerb method, ICache cache = null, Duration timeout = 10.seconds, string proxy = null) { 761 populateFromInfo(where, method); 762 setTimeout(timeout); 763 this.cache = cache; 764 this.proxy = proxy; 765 } 766 767 /++ 768 Sets the timeout from inactivity on the request. This is the amount of time that passes with no send or receive activity on the request before it fails with "request timed out" error. 769 770 History: 771 Added March 31, 2021 772 +/ 773 void setTimeout(Duration timeout) { 774 this.requestParameters.timeoutFromInactivity = timeout; 775 this.timeoutFromInactivity = MonoTime.currTime + this.requestParameters.timeoutFromInactivity; 776 } 777 778 private MonoTime timeoutFromInactivity; 779 780 private Uri where; 781 782 private ICache cache; 783 784 /++ 785 Proxy to use for this request. It should be a URL or `null`. 786 787 This must be sent before you call `send`. 788 789 History: 790 Added April 12, 2021 (dub v9.5) 791 +/ 792 string proxy; 793 794 /// Final url after any redirections 795 string finalUrl; 796 797 void populateFromInfo(Uri where, HttpVerb method) { 798 auto parts = where.basedOn(this.where); 799 this.where = parts; 800 finalUrl = where.toString(); 801 requestParameters.method = method; 802 requestParameters.unixSocketPath = where.unixSocketPath; 803 requestParameters.host = parts.host; 804 requestParameters.port = cast(ushort) parts.port; 805 requestParameters.ssl = parts.scheme == "https"; 806 if(parts.port == 0) 807 requestParameters.port = requestParameters.ssl ? 443 : 80; 808 requestParameters.uri = parts.path.length ? parts.path : "/"; 809 if(parts.query.length) { 810 requestParameters.uri ~= "?"; 811 requestParameters.uri ~= parts.query; 812 } 813 } 814 815 ~this() { 816 } 817 818 ubyte[] sendBuffer; 819 820 HttpResponse responseData; 821 private HttpClient parentClient; 822 823 size_t bodyBytesSent; 824 size_t bodyBytesReceived; 825 826 State state_; 827 State state() { return state_; } 828 State state(State s) { 829 assert(state_ != State.complete); 830 return state_ = s; 831 } 832 /// Called when data is received. Check the state to see what data is available. 833 void delegate(HttpRequest) onDataReceived; 834 835 enum State { 836 /// The request has not yet been sent 837 unsent, 838 839 /// The send() method has been called, but no data is 840 /// sent on the socket yet because the connection is busy. 841 pendingAvailableConnection, 842 843 /// The headers are being sent now 844 sendingHeaders, 845 846 /// The body is being sent now 847 sendingBody, 848 849 /// The request has been sent but we haven't received any response yet 850 waitingForResponse, 851 852 /// We have received some data and are currently receiving headers 853 readingHeaders, 854 855 /// All headers are available but we're still waiting on the body 856 readingBody, 857 858 /// The request is complete. 859 complete, 860 861 /// The request is aborted, either by the abort() method, or as a result of the server disconnecting 862 aborted 863 } 864 865 /// Sends now and waits for the request to finish, returning the response. 866 HttpResponse perform() { 867 send(); 868 return waitForCompletion(); 869 } 870 871 /// Sends the request asynchronously. 872 void send() { 873 sendPrivate(true); 874 } 875 876 private void sendPrivate(bool advance) { 877 if(state != State.unsent && state != State.aborted) 878 return; // already sent 879 880 if(cache !is null) { 881 auto res = cache.getCachedResponse(this.requestParameters); 882 if(res !is null) { 883 state = State.complete; 884 responseData = (*res).deepCopy(); 885 return; 886 } 887 } 888 889 string headers; 890 891 headers ~= to!string(requestParameters.method); 892 headers ~= " "; 893 if(proxy.length && !requestParameters.ssl) { 894 // if we're doing a http proxy, we need to send a complete, absolute uri 895 // so reconstruct it 896 headers ~= "http://"; 897 headers ~= requestParameters.host; 898 if(requestParameters.port != 80) { 899 headers ~= ":"; 900 headers ~= to!string(requestParameters.port); 901 } 902 } 903 904 headers ~= requestParameters.uri; 905 906 if(requestParameters.useHttp11) 907 headers ~= " HTTP/1.1\r\n"; 908 else 909 headers ~= " HTTP/1.0\r\n"; 910 911 // the whole authority section is supposed to be there, but curl doesn't send if default port 912 // so I'll copy what they do 913 headers ~= "Host: "; 914 headers ~= requestParameters.host; 915 if(requestParameters.port != 80 && requestParameters.port != 443) { 916 headers ~= ":"; 917 headers ~= to!string(requestParameters.port); 918 } 919 headers ~= "\r\n"; 920 921 if(requestParameters.userAgent.length) 922 headers ~= "User-Agent: "~requestParameters.userAgent~"\r\n"; 923 if(requestParameters.contentType.length) 924 headers ~= "Content-Type: "~requestParameters.contentType~"\r\n"; 925 if(requestParameters.authorization.length) 926 headers ~= "Authorization: "~requestParameters.authorization~"\r\n"; 927 if(requestParameters.bodyData.length) 928 headers ~= "Content-Length: "~to!string(requestParameters.bodyData.length)~"\r\n"; 929 if(requestParameters.acceptGzip) 930 headers ~= "Accept-Encoding: gzip\r\n"; 931 if(requestParameters.keepAlive) 932 headers ~= "Connection: keep-alive\r\n"; 933 934 foreach(header; requestParameters.headers) 935 headers ~= header ~ "\r\n"; 936 937 headers ~= "\r\n"; 938 939 sendBuffer = cast(ubyte[]) headers ~ requestParameters.bodyData; 940 941 // import std.stdio; writeln("******* ", sendBuffer); 942 943 responseData = HttpResponse.init; 944 responseData.requestParameters = requestParameters; 945 bodyBytesSent = 0; 946 bodyBytesReceived = 0; 947 state = State.pendingAvailableConnection; 948 949 bool alreadyPending = false; 950 foreach(req; pending) 951 if(req is this) { 952 alreadyPending = true; 953 break; 954 } 955 if(!alreadyPending) { 956 pending ~= this; 957 } 958 959 if(advance) 960 HttpRequest.advanceConnections(); 961 } 962 963 964 /// Waits for the request to finish or timeout, whichever comes first. 965 HttpResponse waitForCompletion() { 966 while(state != State.aborted && state != State.complete) { 967 if(state == State.unsent) { 968 send(); 969 continue; 970 } 971 if(auto err = HttpRequest.advanceConnections()) { 972 switch(err) { 973 case 1: throw new Exception("HttpRequest.advanceConnections returned 1: all connections timed out"); 974 case 2: throw new Exception("HttpRequest.advanceConnections returned 2: nothing to do"); 975 case 3: continue; // EINTR 976 default: throw new Exception("HttpRequest.advanceConnections got err " ~ to!string(err)); 977 } 978 } 979 } 980 981 if(state == State.complete && responseData.code >= 200) 982 if(cache !is null) 983 cache.cacheResponse(this.requestParameters, this.responseData); 984 985 return responseData; 986 } 987 988 /// Aborts this request. 989 void abort() { 990 this.state = State.aborted; 991 this.responseData.code = 1; 992 this.responseData.codeText = "request.abort called"; 993 // the actual cancellation happens in the event loop 994 } 995 996 HttpRequestParameters requestParameters; /// 997 998 version(arsd_http_winhttp_implementation) { 999 public static void resetInternals() { 1000 1001 } 1002 1003 static assert(0, "implementation not finished"); 1004 } 1005 1006 1007 version(arsd_http_internal_implementation) { 1008 private static { 1009 // we manage the actual connections. When a request is made on a particular 1010 // host, we try to reuse connections. We may open more than one connection per 1011 // host to do parallel requests. 1012 // 1013 // The key is the *domain name* and the port. Multiple domains on the same address will have separate connections. 1014 Socket[][string] socketsPerHost; 1015 1016 void loseSocket(string host, ushort port, bool ssl, Socket s) { 1017 import std.string; 1018 auto key = format("http%s://%s:%s", ssl ? "s" : "", host, port); 1019 1020 if(auto list = key in socketsPerHost) { 1021 for(int a = 0; a < (*list).length; a++) { 1022 if((*list)[a] is s) { 1023 1024 for(int b = a; b < (*list).length - 1; b++) 1025 (*list)[b] = (*list)[b+1]; 1026 (*list) = (*list)[0 .. $-1]; 1027 break; 1028 } 1029 } 1030 } 1031 } 1032 1033 Socket getOpenSocketOnHost(string proxy, string host, ushort port, bool ssl, string unixSocketPath) { 1034 Socket openNewConnection() { 1035 Socket socket; 1036 if(ssl) { 1037 version(with_openssl) { 1038 loadOpenSsl(); 1039 socket = new SslClientSocket(family(unixSocketPath), SocketType.STREAM, host, defaultVerifyPeer); 1040 } else 1041 throw new Exception("SSL not compiled in"); 1042 } else 1043 socket = new Socket(family(unixSocketPath), SocketType.STREAM); 1044 1045 // FIXME: connect timeout? 1046 if(unixSocketPath) { 1047 import std.stdio; writeln(cast(ubyte[]) unixSocketPath); 1048 socket.connect(new UnixAddress(unixSocketPath)); 1049 } else { 1050 // FIXME: i should prolly do ipv6 if available too. 1051 if(host.length == 0) // this could arguably also be an in contract since it is user error, but the exception is good enough 1052 throw new Exception("No host given for request"); 1053 if(proxy.length) { 1054 if(proxy.indexOf("//") == -1) 1055 proxy = "http://" ~ proxy; 1056 auto proxyurl = Uri(proxy); 1057 1058 //auto proxyhttps = proxyurl.scheme == "https"; 1059 enum proxyhttps = false; // this isn't properly implemented and might never be necessary anyway so meh 1060 1061 // the precise types here are important to help with overload 1062 // resolution of the devirtualized call! 1063 Address pa = new InternetAddress(proxyurl.host, proxyurl.port ? cast(ushort) proxyurl.port : 80); 1064 1065 debug(arsd_http2) writeln("using proxy ", pa.toString()); 1066 1067 if(proxyhttps) { 1068 socket.connect(pa); 1069 } else { 1070 // the proxy never actually starts TLS, but if the request is tls then we need to CONNECT then upgrade the connection 1071 // using the parent class functions let us bypass the encryption 1072 socket.Socket.connect(pa); 1073 } 1074 1075 string message; 1076 if(ssl) { 1077 auto hostName = host ~ ":" ~ to!string(port); 1078 message = "CONNECT " ~ hostName ~ " HTTP/1.1\r\n"; 1079 message ~= "Host: " ~ hostName ~ "\r\n"; 1080 if(proxyurl.userinfo.length) { 1081 import std.base64; 1082 message ~= "Proxy-Authorization: Basic " ~ Base64.encode(cast(ubyte[]) proxyurl.userinfo) ~ "\r\n"; 1083 } 1084 message ~= "\r\n"; 1085 1086 // FIXME: what if proxy times out? should be reasonably fast too. 1087 if(proxyhttps) { 1088 socket.send(message, SocketFlags.NONE); 1089 } else { 1090 socket.Socket.send(message, SocketFlags.NONE); 1091 } 1092 1093 ubyte[1024] recvBuffer; 1094 // and last time 1095 ptrdiff_t rcvGot; 1096 if(proxyhttps) { 1097 rcvGot = socket.receive(recvBuffer[], SocketFlags.NONE); 1098 // bool verifyPeer = true; 1099 //(cast(OpenSslSocket)socket).freeSsl(); 1100 //(cast(OpenSslSocket)socket).initSsl(verifyPeer, host); 1101 } else { 1102 rcvGot = socket.Socket.receive(recvBuffer[], SocketFlags.NONE); 1103 } 1104 1105 if(rcvGot == -1) 1106 throw new ProxyException("proxy receive error"); 1107 auto got = cast(string) recvBuffer[0 .. rcvGot]; 1108 auto expect = "HTTP/1.1 200"; 1109 if(got.length < expect.length || (got[0 .. expect.length] != expect && got[0 .. expect.length] != "HTTP/1.0 200")) 1110 throw new ProxyException("Proxy rejected request: " ~ got[0 .. expect.length <= got.length ? expect.length : got.length]); 1111 1112 if(proxyhttps) { 1113 //(cast(OpenSslSocket)socket).do_ssl_connect(); 1114 } else { 1115 (cast(OpenSslSocket)socket).do_ssl_connect(); 1116 } 1117 } else { 1118 } 1119 } else { 1120 socket.connect(new InternetAddress(host, port)); 1121 } 1122 } 1123 1124 debug(arsd_http2) writeln("opening to ", host, ":", port, " ", cast(void*) socket); 1125 assert(socket.handle() !is socket_t.init); 1126 return socket; 1127 } 1128 1129 import std.string; 1130 auto key = format("http%s://%s:%s", ssl ? "s" : "", host, port); 1131 1132 if(auto hostListing = key in socketsPerHost) { 1133 // try to find an available socket that is already open 1134 foreach(socket; *hostListing) { 1135 if(socket !in activeRequestOnSocket) { 1136 // let's see if it has closed since we last tried 1137 // e.g. a server timeout or something. If so, we need 1138 // to lose this one and immediately open a new one. 1139 static SocketSet readSet = null; 1140 if(readSet is null) 1141 readSet = new SocketSet(); 1142 readSet.reset(); 1143 assert(socket !is null); 1144 assert(socket.handle() !is socket_t.init, socket is null ? "null" : socket.toString()); 1145 readSet.add(socket); 1146 auto got = Socket.select(readSet, null, null, 5.msecs /* timeout */); 1147 if(got > 0) { 1148 // we can read something off this... but there aren't 1149 // any active requests. Assume it is EOF and open a new one 1150 1151 socket.close(); 1152 loseSocket(host, port, ssl, socket); 1153 goto openNew; 1154 } 1155 return socket; 1156 } 1157 } 1158 1159 // if not too many already open, go ahead and do a new one 1160 if((*hostListing).length < 6) { 1161 auto socket = openNewConnection(); 1162 (*hostListing) ~= socket; 1163 return socket; 1164 } else 1165 return null; // too many, you'll have to wait 1166 } 1167 1168 openNew: 1169 1170 auto socket = openNewConnection(); 1171 socketsPerHost[key] ~= socket; 1172 return socket; 1173 } 1174 1175 // only one request can be active on a given socket (at least HTTP < 2.0) so this is that 1176 HttpRequest[Socket] activeRequestOnSocket; 1177 HttpRequest[] pending; // and these are the requests that are waiting 1178 1179 SocketSet readSet; 1180 SocketSet writeSet; 1181 1182 /+ 1183 Generic event loop registration: 1184 1185 handle, operation (read/write), buffer (on posix it *might* be stack if a select loop), timeout (in real time), callback when op completed. 1186 1187 ....basically Windows style. Then it translates internally. 1188 1189 It should tell the thing if the buffer is reused or not 1190 +/ 1191 1192 1193 /++ 1194 This is made public for rudimentary event loop integration, but is still 1195 basically an internal detail. Try not to use it if you have another way. 1196 1197 This does a single iteration of the internal select()-based processing loop. 1198 1199 1200 Future directions: 1201 I want to merge the internal use of [WebSocket.eventLoop] with this; 1202 [advanceConnections] does just one run on the loop, whereas eventLoop 1203 runs it until all connections are closed. But they'd both process both 1204 pending http requests and active websockets. 1205 1206 After that, I want to be able to integrate in other event loops too. 1207 One might be to simply to reactor callbacks, then perhaps Windows overlapped 1208 i/o (that's just going to be tricky to retrofit into the existing select()-based 1209 code). It could then go fiber just by calling the resume function too. 1210 1211 The hard part is ensuring I keep this file stand-alone while offering these 1212 things. 1213 1214 This `advanceConnections` call will probably continue to work now that it is 1215 public, but it may not be wholly compatible with all the future features; you'd 1216 have to pick either the internal event loop or an external one you integrate, but not 1217 mix them. 1218 1219 History: 1220 This has been included in the library since almost day one, but 1221 it was private until April 13, 2021 (dub v9.5). 1222 1223 Params: 1224 maximumTimeout = the maximum time it will wait in select(). It may return much sooner than this if a connection timed out in the mean time. 1225 automaticallyRetryOnInterruption = internally loop on EINTR. 1226 1227 Returns: 1228 1229 0 = no error, work may remain so you should call `advanceConnections` again when you can 1230 1231 1 = passed `maximumTimeout` reached with no work done, yet requests are still in the queue. You may call `advanceConnections` again. 1232 1233 2 = no work to do, no point calling it again unless you've added new requests. Your program may exit if you have nothing to add since it means everything requested is now done. 1234 1235 3 = EINTR occurred on select(), you should check your interrupt flags if you set a signal handler, then call `advanceConnections` again if you aren't exiting. Only occurs if `automaticallyRetryOnInterruption` is set to `false` (the default when it is called externally). 1236 1237 any other value should be considered a non-recoverable error if you want to be forward compatible as I reserve the right to add more values later. 1238 +/ 1239 public int advanceConnections(Duration maximumTimeout = 10.seconds, bool automaticallyRetryOnInterruption = false) { 1240 if(readSet is null) 1241 readSet = new SocketSet(); 1242 if(writeSet is null) 1243 writeSet = new SocketSet(); 1244 1245 ubyte[2048] buffer; 1246 1247 HttpRequest[16] removeFromPending; 1248 size_t removeFromPendingCount = 0; 1249 1250 bool hadAbortedRequest; 1251 1252 // are there pending requests? let's try to send them 1253 foreach(idx, pc; pending) { 1254 if(removeFromPendingCount == removeFromPending.length) 1255 break; 1256 1257 if(pc.state == HttpRequest.State.aborted) { 1258 removeFromPending[removeFromPendingCount++] = pc; 1259 hadAbortedRequest = true; 1260 continue; 1261 } 1262 1263 Socket socket; 1264 1265 try { 1266 socket = getOpenSocketOnHost(pc.proxy, pc.requestParameters.host, pc.requestParameters.port, pc.requestParameters.ssl, pc.requestParameters.unixSocketPath); 1267 } catch(ProxyException e) { 1268 // connection refused or timed out (I should disambiguate somehow)... 1269 pc.state = HttpRequest.State.aborted; 1270 1271 pc.responseData.code = 2; 1272 pc.responseData.codeText = e.msg ~ " from " ~ pc.proxy; 1273 1274 hadAbortedRequest = true; 1275 1276 removeFromPending[removeFromPendingCount++] = pc; 1277 continue; 1278 1279 } catch(SocketException e) { 1280 // connection refused or timed out (I should disambiguate somehow)... 1281 pc.state = HttpRequest.State.aborted; 1282 1283 pc.responseData.code = 2; 1284 pc.responseData.codeText = pc.proxy.length ? ("connection failed to proxy " ~ pc.proxy) : "connection failed"; 1285 1286 hadAbortedRequest = true; 1287 1288 removeFromPending[removeFromPendingCount++] = pc; 1289 continue; 1290 } 1291 1292 if(socket !is null) { 1293 activeRequestOnSocket[socket] = pc; 1294 assert(pc.sendBuffer.length); 1295 pc.state = State.sendingHeaders; 1296 1297 removeFromPending[removeFromPendingCount++] = pc; 1298 } 1299 } 1300 1301 import std.algorithm : remove; 1302 foreach(rp; removeFromPending[0 .. removeFromPendingCount]) 1303 pending = pending.remove!((a) => a is rp)(); 1304 1305 tryAgain: 1306 1307 Socket[16] inactive; 1308 int inactiveCount = 0; 1309 void killInactives() { 1310 foreach(s; inactive[0 .. inactiveCount]) { 1311 debug(arsd_http2) writeln("removing socket from active list ", cast(void*) s); 1312 activeRequestOnSocket.remove(s); 1313 } 1314 } 1315 1316 1317 readSet.reset(); 1318 writeSet.reset(); 1319 1320 bool hadOne = false; 1321 1322 auto minTimeout = maximumTimeout; 1323 auto now = MonoTime.currTime; 1324 1325 // active requests need to be read or written to 1326 foreach(sock, request; activeRequestOnSocket) { 1327 1328 if(request.state == State.aborted) { 1329 inactive[inactiveCount++] = sock; 1330 sock.close(); 1331 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1332 hadAbortedRequest = true; 1333 continue; 1334 } 1335 1336 // check the other sockets just for EOF, if they close, take them out of our list, 1337 // we'll reopen if needed upon request. 1338 readSet.add(sock); 1339 hadOne = true; 1340 1341 Duration timeo; 1342 if(request.timeoutFromInactivity <= now) 1343 timeo = 0.seconds; 1344 else 1345 timeo = request.timeoutFromInactivity - now; 1346 1347 if(timeo < minTimeout) 1348 minTimeout = timeo; 1349 1350 if(request.state == State.sendingHeaders || request.state == State.sendingBody) { 1351 writeSet.add(sock); 1352 hadOne = true; 1353 } 1354 } 1355 1356 if(!hadOne) { 1357 if(hadAbortedRequest) { 1358 killInactives(); 1359 return 0; // something got aborted, that's progress 1360 } 1361 return 2; // automatic timeout, nothing to do 1362 } 1363 1364 auto selectGot = Socket.select(readSet, writeSet, null, minTimeout); 1365 if(selectGot == 0) { /* timeout */ 1366 now = MonoTime.currTime; 1367 bool anyWorkDone = false; 1368 foreach(sock, request; activeRequestOnSocket) { 1369 1370 if(request.timeoutFromInactivity <= now) { 1371 request.state = HttpRequest.State.aborted; 1372 request.responseData.code = 5; 1373 request.responseData.codeText = "Request timed out"; 1374 1375 inactive[inactiveCount++] = sock; 1376 sock.close(); 1377 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1378 anyWorkDone = true; 1379 } 1380 } 1381 killInactives(); 1382 return anyWorkDone ? 0 : 1; 1383 // return 1; was an error to time out but now im making it on the individual request 1384 } else if(selectGot == -1) { /* interrupted */ 1385 /* 1386 version(Posix) { 1387 import core.stdc.errno; 1388 if(errno != EINTR) 1389 throw new Exception("select error: " ~ to!string(errno)); 1390 } 1391 */ 1392 if(automaticallyRetryOnInterruption) 1393 goto tryAgain; 1394 else 1395 return 3; 1396 } else { /* ready */ 1397 foreach(sock, request; activeRequestOnSocket) { 1398 if(readSet.isSet(sock)) { 1399 keep_going: 1400 request.timeoutFromInactivity = MonoTime.currTime + request.requestParameters.timeoutFromInactivity; 1401 auto got = sock.receive(buffer); 1402 debug(arsd_http2_verbose) writeln("====PACKET ",got,"=====",cast(string)buffer[0 .. got],"===/PACKET==="); 1403 if(got < 0) { 1404 throw new Exception("receive error"); 1405 } else if(got == 0) { 1406 // remote side disconnected 1407 debug(arsd_http2) writeln("remote disconnect"); 1408 if(request.state != State.complete) { 1409 request.state = State.aborted; 1410 1411 request.responseData.code = 3; 1412 request.responseData.codeText = "server disconnected"; 1413 } 1414 inactive[inactiveCount++] = sock; 1415 sock.close(); 1416 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1417 } else { 1418 // data available 1419 bool stillAlive; 1420 1421 try { 1422 stillAlive = request.handleIncomingData(buffer[0 .. got]); 1423 } catch (Exception e) { 1424 request.state = HttpRequest.State.aborted; 1425 request.responseData.code = 4; 1426 request.responseData.codeText = e.msg; 1427 1428 inactive[inactiveCount++] = sock; 1429 sock.close(); 1430 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1431 continue; 1432 } 1433 1434 if(!stillAlive || request.state == HttpRequest.State.complete || request.state == HttpRequest.State.aborted) { 1435 //import std.stdio; writeln(cast(void*) sock, " ", stillAlive, " ", request.state); 1436 inactive[inactiveCount++] = sock; 1437 continue; 1438 // reuse the socket for another pending request, if we can 1439 } 1440 } 1441 1442 if(request.onDataReceived) 1443 request.onDataReceived(request); 1444 1445 version(with_openssl) 1446 if(auto s = cast(SslClientSocket) sock) { 1447 // select doesn't handle the case with stuff 1448 // left in the ssl buffer so i'm checking it separately 1449 if(s.dataPending()) { 1450 goto keep_going; 1451 } 1452 } 1453 } 1454 1455 if(request.state == State.sendingHeaders || request.state == State.sendingBody) 1456 if(writeSet.isSet(sock)) { 1457 request.timeoutFromInactivity = MonoTime.currTime + request.requestParameters.timeoutFromInactivity; 1458 assert(request.sendBuffer.length); 1459 auto sent = sock.send(request.sendBuffer); 1460 debug(arsd_http2_verbose) writeln(cast(void*) sock, "<send>", cast(string) request.sendBuffer, "</send>"); 1461 if(sent <= 0) 1462 throw new Exception("send error " ~ lastSocketError); 1463 request.sendBuffer = request.sendBuffer[sent .. $]; 1464 if(request.sendBuffer.length == 0) { 1465 request.state = State.waitingForResponse; 1466 } 1467 } 1468 } 1469 } 1470 1471 killInactives(); 1472 1473 // we've completed a request, are there any more pending connection? if so, send them now 1474 1475 return 0; 1476 } 1477 } 1478 1479 public static void resetInternals() { 1480 socketsPerHost = null; 1481 activeRequestOnSocket = null; 1482 pending = null; 1483 1484 } 1485 1486 struct HeaderReadingState { 1487 bool justSawLf; 1488 bool justSawCr; 1489 bool atStartOfLine = true; 1490 bool readingLineContinuation; 1491 } 1492 HeaderReadingState headerReadingState; 1493 1494 struct BodyReadingState { 1495 bool isGzipped; 1496 bool isDeflated; 1497 1498 bool isChunked; 1499 int chunkedState; 1500 1501 // used for the chunk size if it is chunked 1502 int contentLengthRemaining; 1503 } 1504 BodyReadingState bodyReadingState; 1505 1506 bool closeSocketWhenComplete; 1507 1508 import std.zlib; 1509 UnCompress uncompress; 1510 1511 const(ubyte)[] leftoverDataFromLastTime; 1512 1513 bool handleIncomingData(scope const ubyte[] dataIn) { 1514 bool stillAlive = true; 1515 debug(arsd_http2) writeln("handleIncomingData, state: ", state); 1516 if(state == State.waitingForResponse) { 1517 state = State.readingHeaders; 1518 headerReadingState = HeaderReadingState.init; 1519 bodyReadingState = BodyReadingState.init; 1520 } 1521 1522 const(ubyte)[] data; 1523 if(leftoverDataFromLastTime.length) 1524 data = leftoverDataFromLastTime ~ dataIn[]; 1525 else 1526 data = dataIn[]; 1527 1528 if(state == State.readingHeaders) { 1529 void parseLastHeader() { 1530 assert(responseData.headers.length); 1531 if(responseData.headers.length == 1) { 1532 responseData.statusLine = responseData.headers[0]; 1533 import std.algorithm; 1534 auto parts = responseData.statusLine.splitter(" "); 1535 responseData.httpVersion = parts.front; 1536 parts.popFront(); 1537 if(parts.empty) 1538 throw new Exception("Corrupted response, bad status line"); 1539 responseData.code = to!int(parts.front()); 1540 parts.popFront(); 1541 responseData.codeText = ""; 1542 while(!parts.empty) { 1543 // FIXME: this sucks! 1544 responseData.codeText ~= parts.front(); 1545 parts.popFront(); 1546 if(!parts.empty) 1547 responseData.codeText ~= " "; 1548 } 1549 } else { 1550 // parse the new header 1551 auto header = responseData.headers[$-1]; 1552 1553 auto colon = header.indexOf(":"); 1554 if(colon == -1) 1555 return; 1556 auto name = header[0 .. colon]; 1557 if(colon + 1 == header.length || colon + 2 == header.length) // assuming a space there 1558 return; // empty header, idk 1559 assert(colon + 2 < header.length, header); 1560 auto value = header[colon + 2 .. $]; // skipping the colon itself and the following space 1561 1562 switch(name) { 1563 case "Connection": 1564 case "connection": 1565 if(value == "close") 1566 closeSocketWhenComplete = true; 1567 break; 1568 case "Content-Type": 1569 case "content-type": 1570 responseData.contentType = value; 1571 break; 1572 case "Location": 1573 case "location": 1574 responseData.location = value; 1575 break; 1576 case "Content-Length": 1577 case "content-length": 1578 bodyReadingState.contentLengthRemaining = to!int(value); 1579 break; 1580 case "Transfer-Encoding": 1581 case "transfer-encoding": 1582 // note that if it is gzipped, it zips first, then chunks the compressed stream. 1583 // so we should always dechunk first, then feed into the decompressor 1584 if(value.strip == "chunked") 1585 bodyReadingState.isChunked = true; 1586 else throw new Exception("Unknown Transfer-Encoding: " ~ value); 1587 break; 1588 case "Content-Encoding": 1589 case "content-encoding": 1590 if(value == "gzip") { 1591 bodyReadingState.isGzipped = true; 1592 uncompress = new UnCompress(); 1593 } else if(value == "deflate") { 1594 bodyReadingState.isDeflated = true; 1595 uncompress = new UnCompress(); 1596 } else throw new Exception("Unknown Content-Encoding: " ~ value); 1597 break; 1598 case "Set-Cookie": 1599 case "set-cookie": 1600 // FIXME handle 1601 break; 1602 default: 1603 // ignore 1604 } 1605 1606 responseData.headersHash[name] = value; 1607 } 1608 } 1609 1610 size_t position = 0; 1611 for(position = 0; position < dataIn.length; position++) { 1612 if(headerReadingState.readingLineContinuation) { 1613 if(data[position] == ' ' || data[position] == '\t') 1614 continue; 1615 headerReadingState.readingLineContinuation = false; 1616 } 1617 1618 if(headerReadingState.atStartOfLine) { 1619 headerReadingState.atStartOfLine = false; 1620 if(data[position] == '\r' || data[position] == '\n') { 1621 // done with headers 1622 if(data[position] == '\r' && (position + 1) < data.length && data[position + 1] == '\n') 1623 position++; 1624 if(this.requestParameters.method == HttpVerb.HEAD) 1625 state = State.complete; 1626 else 1627 state = State.readingBody; 1628 position++; // skip the newline 1629 break; 1630 } else if(data[position] == ' ' || data[position] == '\t') { 1631 // line continuation, ignore all whitespace and collapse it into a space 1632 headerReadingState.readingLineContinuation = true; 1633 responseData.headers[$-1] ~= ' '; 1634 } else { 1635 // new header 1636 if(responseData.headers.length) 1637 parseLastHeader(); 1638 responseData.headers ~= ""; 1639 } 1640 } 1641 1642 if(data[position] == '\r') { 1643 headerReadingState.justSawCr = true; 1644 continue; 1645 } else 1646 headerReadingState.justSawCr = false; 1647 1648 if(data[position] == '\n') { 1649 headerReadingState.justSawLf = true; 1650 headerReadingState.atStartOfLine = true; 1651 continue; 1652 } else 1653 headerReadingState.justSawLf = false; 1654 1655 responseData.headers[$-1] ~= data[position]; 1656 } 1657 1658 if(responseData.headers.length) 1659 parseLastHeader(); 1660 data = data[position .. $]; 1661 } 1662 1663 if(state == State.readingBody) { 1664 if(bodyReadingState.isChunked) { 1665 // read the hex length, stopping at a \r\n, ignoring everything between the new line but after the first non-valid hex character 1666 // read binary data of that length. it is our content 1667 // repeat until a zero sized chunk 1668 // then read footers as headers. 1669 1670 start_over: 1671 for(int a = 0; a < data.length; a++) { 1672 final switch(bodyReadingState.chunkedState) { 1673 case 0: // reading hex 1674 char c = data[a]; 1675 if((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) { 1676 // just keep reading 1677 } else { 1678 int power = 1; 1679 bodyReadingState.contentLengthRemaining = 0; 1680 assert(a != 0, cast(string) data); 1681 for(int b = a-1; b >= 0; b--) { 1682 char cc = data[b]; 1683 if(cc >= 'a' && cc <= 'z') 1684 cc -= 0x20; 1685 int val = 0; 1686 if(cc >= '0' && cc <= '9') 1687 val = cc - '0'; 1688 else 1689 val = cc - 'A' + 10; 1690 1691 assert(val >= 0 && val <= 15, to!string(val)); 1692 bodyReadingState.contentLengthRemaining += power * val; 1693 power *= 16; 1694 } 1695 debug(arsd_http2_verbose) writeln("Chunk length: ", bodyReadingState.contentLengthRemaining); 1696 bodyReadingState.chunkedState = 1; 1697 data = data[a + 1 .. $]; 1698 goto start_over; 1699 } 1700 break; 1701 case 1: // reading until end of line 1702 char c = data[a]; 1703 if(c == '\n') { 1704 if(bodyReadingState.contentLengthRemaining == 0) 1705 bodyReadingState.chunkedState = 5; 1706 else 1707 bodyReadingState.chunkedState = 2; 1708 } 1709 data = data[a + 1 .. $]; 1710 goto start_over; 1711 case 2: // reading data 1712 auto can = a + bodyReadingState.contentLengthRemaining; 1713 if(can > data.length) 1714 can = cast(int) data.length; 1715 1716 auto newData = data[a .. can]; 1717 data = data[can .. $]; 1718 1719 //if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) 1720 // responseData.content ~= cast(ubyte[]) uncompress.uncompress(data[a .. can]); 1721 //else 1722 responseData.content ~= newData; 1723 1724 bodyReadingState.contentLengthRemaining -= newData.length; 1725 debug(arsd_http2_verbose) writeln("clr: ", bodyReadingState.contentLengthRemaining, " " , a, " ", can); 1726 assert(bodyReadingState.contentLengthRemaining >= 0); 1727 if(bodyReadingState.contentLengthRemaining == 0) { 1728 bodyReadingState.chunkedState = 3; 1729 } else { 1730 // will continue grabbing more 1731 } 1732 goto start_over; 1733 case 3: // reading 13/10 1734 assert(data[a] == 13); 1735 bodyReadingState.chunkedState++; 1736 data = data[a + 1 .. $]; 1737 goto start_over; 1738 case 4: // reading 10 at end of packet 1739 assert(data[a] == 10); 1740 data = data[a + 1 .. $]; 1741 bodyReadingState.chunkedState = 0; 1742 goto start_over; 1743 case 5: // reading footers 1744 //goto done; // FIXME 1745 state = State.complete; 1746 1747 bodyReadingState.chunkedState = 0; 1748 1749 while(data[a] != 10) { 1750 a++; 1751 if(a == data.length) 1752 return stillAlive; // in the footer state we're just discarding everything until we're done so this should be ok 1753 } 1754 data = data[a + 1 .. $]; 1755 1756 if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) { 1757 auto n = uncompress.uncompress(responseData.content); 1758 n ~= uncompress.flush(); 1759 responseData.content = cast(ubyte[]) n; 1760 } 1761 1762 // responseData.content ~= cast(ubyte[]) uncompress.flush(); 1763 1764 responseData.contentText = cast(string) responseData.content; 1765 1766 goto done; 1767 } 1768 } 1769 1770 done: 1771 // FIXME 1772 //if(closeSocketWhenComplete) 1773 //socket.close(); 1774 } else { 1775 //if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) 1776 // responseData.content ~= cast(ubyte[]) uncompress.uncompress(data); 1777 //else 1778 responseData.content ~= data; 1779 //assert(data.length <= bodyReadingState.contentLengthRemaining, format("%d <= %d\n%s", data.length, bodyReadingState.contentLengthRemaining, cast(string)data)); 1780 int use = cast(int) data.length; 1781 if(use > bodyReadingState.contentLengthRemaining) 1782 use = bodyReadingState.contentLengthRemaining; 1783 bodyReadingState.contentLengthRemaining -= use; 1784 data = data[use .. $]; 1785 if(bodyReadingState.contentLengthRemaining == 0) { 1786 if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) { 1787 auto n = uncompress.uncompress(responseData.content); 1788 n ~= uncompress.flush(); 1789 responseData.content = cast(ubyte[]) n; 1790 //responseData.content ~= cast(ubyte[]) uncompress.flush(); 1791 } 1792 if(followLocation && responseData.location.length) { 1793 static bool first = true; 1794 //version(DigitalMars) if(!first) asm { int 3; } 1795 populateFromInfo(Uri(responseData.location), HttpVerb.GET); 1796 //import std.stdio; writeln("redirected to ", responseData.location); 1797 first = false; 1798 responseData = HttpResponse.init; 1799 headerReadingState = HeaderReadingState.init; 1800 bodyReadingState = BodyReadingState.init; 1801 state = State.unsent; 1802 stillAlive = false; 1803 sendPrivate(false); 1804 } else { 1805 state = State.complete; 1806 responseData.contentText = cast(string) responseData.content; 1807 // FIXME 1808 //if(closeSocketWhenComplete) 1809 //socket.close(); 1810 } 1811 } 1812 } 1813 } 1814 1815 if(data.length) 1816 leftoverDataFromLastTime = data.dup; 1817 else 1818 leftoverDataFromLastTime = null; 1819 1820 return stillAlive; 1821 } 1822 1823 } 1824 } 1825 1826 /// 1827 struct HttpRequestParameters { 1828 // FIXME: implement these 1829 //Duration timeoutTotal; // the whole request must finish in this time or else it fails,even if data is still trickling in 1830 Duration timeoutFromInactivity; // if there's no activity in this time it dies. basically the socket receive timeout 1831 1832 // debugging 1833 bool useHttp11 = true; /// 1834 bool acceptGzip = true; /// 1835 bool keepAlive = true; /// 1836 1837 // the request itself 1838 HttpVerb method; /// 1839 string host; /// 1840 ushort port; /// 1841 string uri; /// 1842 1843 bool ssl; /// 1844 1845 string userAgent; /// 1846 string authorization; /// 1847 1848 string[string] cookies; /// 1849 1850 string[] headers; /// do not duplicate host, content-length, content-type, or any others that have a specific property 1851 1852 string contentType; /// 1853 ubyte[] bodyData; /// 1854 1855 string unixSocketPath; 1856 } 1857 1858 interface IHttpClient { 1859 1860 } 1861 1862 /// 1863 enum HttpVerb { 1864 /// 1865 GET, 1866 /// 1867 HEAD, 1868 /// 1869 POST, 1870 /// 1871 PUT, 1872 /// 1873 DELETE, 1874 /// 1875 OPTIONS, 1876 /// 1877 TRACE, 1878 /// 1879 CONNECT, 1880 /// 1881 PATCH, 1882 /// 1883 MERGE 1884 } 1885 1886 /++ 1887 HttpClient keeps cookies, location, and some other state to reuse connections, when possible, like a web browser. 1888 You can use it as your entry point to make http requests. 1889 1890 See the example on [arsd.http2#examples]. 1891 +/ 1892 class HttpClient { 1893 /* Protocol restrictions, useful to disable when debugging servers */ 1894 bool useHttp11 = true; /// 1895 bool acceptGzip = true; /// 1896 bool keepAlive = true; /// 1897 1898 /// 1899 @property Uri location() { 1900 return currentUrl; 1901 } 1902 1903 /++ 1904 Default timeout for requests created on this client. 1905 1906 History: 1907 Added March 31, 2021 1908 +/ 1909 Duration defaultTimeout = 10.seconds; 1910 1911 /// High level function that works similarly to entering a url 1912 /// into a browser. 1913 /// 1914 /// Follows locations, updates the current url. 1915 HttpRequest navigateTo(Uri where, HttpVerb method = HttpVerb.GET) { 1916 currentUrl = where.basedOn(currentUrl); 1917 currentDomain = where.host; 1918 1919 auto request = this.request(currentUrl, method); 1920 request.followLocation = true; 1921 1922 return request; 1923 } 1924 1925 /++ 1926 Creates a request without updating the current url state 1927 (but will still save cookies btw... when that is implemented) 1928 +/ 1929 HttpRequest request(Uri uri, HttpVerb method = HttpVerb.GET, ubyte[] bodyData = null, string contentType = null) { 1930 string proxyToUse; 1931 switch(uri.scheme) { 1932 case "http": 1933 proxyToUse = httpProxy; 1934 break; 1935 case "https": 1936 proxyToUse = httpsProxy; 1937 break; 1938 default: 1939 proxyToUse = null; 1940 } 1941 1942 auto request = new HttpRequest(uri, method, cache, defaultTimeout, proxyToUse); 1943 1944 request.requestParameters.userAgent = userAgent; 1945 request.requestParameters.authorization = authorization; 1946 1947 request.requestParameters.useHttp11 = this.useHttp11; 1948 request.requestParameters.acceptGzip = this.acceptGzip; 1949 request.requestParameters.keepAlive = this.keepAlive; 1950 1951 request.requestParameters.bodyData = bodyData; 1952 request.requestParameters.contentType = contentType; 1953 1954 return request; 1955 1956 } 1957 1958 /// ditto 1959 HttpRequest request(Uri uri, FormData fd, HttpVerb method = HttpVerb.POST) { 1960 return request(uri, method, fd.toBytes, fd.contentType); 1961 } 1962 1963 1964 private Uri currentUrl; 1965 private string currentDomain; 1966 private ICache cache; 1967 1968 this(ICache cache = null) { 1969 this.cache = cache; 1970 loadDefaultProxy(); 1971 } 1972 1973 /++ 1974 Loads the system-default proxy. Note that the constructor does this automatically 1975 so you should rarely need to call this explicitly. 1976 1977 The environment variables are used, if present, on all operating systems. 1978 1979 History: 1980 Added April 12, 2021 (included in dub v9.5) 1981 1982 Bugs: 1983 On Windows, it does NOT currently check the IE settings, but I do intend to 1984 implement that in the future. When I do, it will be classified as a bug fix, 1985 NOT a breaking change. 1986 +/ 1987 void loadDefaultProxy() { 1988 import std.process; 1989 httpProxy = environment.get("http_proxy", environment.get("HTTP_PROXY", null)); 1990 httpsProxy = environment.get("https_proxy", environment.get("HTTPS_PROXY", null)); 1991 1992 // FIXME: on Windows, I should use the Internet Explorer proxy settings 1993 } 1994 1995 /++ 1996 Proxies to use for requests. The [HttpClient] constructor will set these to the system values, 1997 then you can reset it to `null` if you want to override and not use the proxy after all, or you 1998 can set it after construction to whatever. 1999 2000 The proxy from the client will be automatically set to the requests performed through it. You can 2001 also override on a per-request basis by creating the request and setting the `proxy` field there 2002 before sending it. 2003 2004 History: 2005 Added April 12, 2021 (included in dub v9.5) 2006 +/ 2007 string httpProxy; 2008 /// ditto 2009 string httpsProxy; 2010 2011 /// 2012 void setCookie(string name, string value, string domain = null) { 2013 if(domain == null) 2014 domain = currentDomain; 2015 2016 cookies[domain][name] = value; 2017 } 2018 2019 /// 2020 void clearCookies(string domain = null) { 2021 if(domain is null) 2022 cookies = null; 2023 else 2024 cookies[domain] = null; 2025 } 2026 2027 // If you set these, they will be pre-filled on all requests made with this client 2028 string userAgent = "D arsd.html2"; /// 2029 string authorization; /// 2030 2031 /* inter-request state */ 2032 string[string][string] cookies; 2033 } 2034 2035 interface ICache { 2036 /++ 2037 The client is about to make the given `request`. It will ALWAYS pass it to the cache object first so you can decide if you want to and can provide a response. You should probably check the appropriate headers to see if you should even attempt to look up on the cache (HttpClient does NOT do this to give maximum flexibility to the cache implementor). 2038 2039 Return null if the cache does not provide. 2040 +/ 2041 const(HttpResponse)* getCachedResponse(HttpRequestParameters request); 2042 2043 /++ 2044 The given request has received the given response. The implementing class needs to decide if it wants to cache or not. Return true if it was added, false if you chose not to. 2045 2046 You may wish to examine headers, etc., in making the decision. The HttpClient will ALWAYS pass a request/response to this. 2047 +/ 2048 bool cacheResponse(HttpRequestParameters request, HttpResponse response); 2049 } 2050 2051 /+ 2052 // / Provides caching behavior similar to a real web browser 2053 class HttpCache : ICache { 2054 const(HttpResponse)* getCachedResponse(HttpRequestParameters request) { 2055 return null; 2056 } 2057 } 2058 2059 // / Gives simple maximum age caching, ignoring the actual http headers 2060 class SimpleCache : ICache { 2061 const(HttpResponse)* getCachedResponse(HttpRequestParameters request) { 2062 return null; 2063 } 2064 } 2065 +/ 2066 2067 /++ 2068 A pseudo-cache to provide a mock server. Construct one of these, 2069 populate it with test responses, and pass it to [HttpClient] to 2070 do a network-free test. 2071 2072 You should populate it with the [populate] method. Any request not 2073 pre-populated will return a "server refused connection" response. 2074 +/ 2075 class HttpMockProvider : ICache { 2076 /+ + 2077 2078 +/ 2079 version(none) 2080 this(Uri baseUrl, string defaultResponseContentType) { 2081 2082 } 2083 2084 this() {} 2085 2086 HttpResponse defaultResponse; 2087 2088 /// Implementation of the ICache interface. Hijacks all requests to return a pre-populated response or "server disconnected". 2089 const(HttpResponse)* getCachedResponse(HttpRequestParameters request) { 2090 import std.conv; 2091 auto defaultPort = request.ssl ? 443 : 80; 2092 string identifier = text( 2093 request.method, " ", 2094 request.ssl ? "https" : "http", "://", 2095 request.host, 2096 (request.port && request.port != defaultPort) ? (":" ~ to!string(request.port)) : "", 2097 request.uri 2098 ); 2099 2100 if(auto res = identifier in population) 2101 return res; 2102 return &defaultResponse; 2103 } 2104 2105 /// Implementation of the ICache interface. We never actually cache anything here since it is all about mock responses, not actually caching real data. 2106 bool cacheResponse(HttpRequestParameters request, HttpResponse response) { 2107 return false; 2108 } 2109 2110 /++ 2111 Convenience method to populate simple responses. For more complex 2112 work, use one of the other overloads where you build complete objects 2113 yourself. 2114 2115 Params: 2116 request = a verb and complete URL to mock as one string. 2117 For example "GET http://example.com/". If you provide only 2118 a partial URL, it will be based on the `baseUrl` you gave 2119 in the `HttpMockProvider` constructor. 2120 2121 responseCode = the HTTP response code, like 200 or 404. 2122 2123 response = the response body as a string. It is assumed 2124 to be of the `defaultResponseContentType` you passed in the 2125 `HttpMockProvider` constructor. 2126 +/ 2127 void populate(string request, int responseCode, string response) { 2128 2129 // FIXME: absolute-ize the URL in the request 2130 2131 HttpResponse r; 2132 r.code = responseCode; 2133 r.codeText = getHttpCodeText(r.code); 2134 2135 r.content = cast(ubyte[]) response; 2136 r.contentText = response; 2137 2138 population[request] = r; 2139 } 2140 2141 version(none) 2142 void populate(string method, string url, HttpResponse response) { 2143 // FIXME 2144 } 2145 2146 private HttpResponse[string] population; 2147 } 2148 2149 // modified from the one in cgi.d to just have the text 2150 private static string getHttpCodeText(int code) pure nothrow @nogc { 2151 switch(code) { 2152 // this module's proprietary extensions 2153 case 0: return null; 2154 case 1: return "request.abort called"; 2155 case 2: return "connection failed"; 2156 case 3: return "server disconnected"; 2157 case 4: return "exception thrown"; // actually should be some other thing 2158 case 5: return "Request timed out"; 2159 2160 // * * * standard ones * * * 2161 2162 // 1xx skipped since they shouldn't happen 2163 2164 // 2165 case 200: return "OK"; 2166 case 201: return "Created"; 2167 case 202: return "Accepted"; 2168 case 203: return "Non-Authoritative Information"; 2169 case 204: return "No Content"; 2170 case 205: return "Reset Content"; 2171 // 2172 case 300: return "Multiple Choices"; 2173 case 301: return "Moved Permanently"; 2174 case 302: return "Found"; 2175 case 303: return "See Other"; 2176 case 307: return "Temporary Redirect"; 2177 case 308: return "Permanent Redirect"; 2178 // 2179 case 400: return "Bad Request"; 2180 case 403: return "Forbidden"; 2181 case 404: return "Not Found"; 2182 case 405: return "Method Not Allowed"; 2183 case 406: return "Not Acceptable"; 2184 case 409: return "Conflict"; 2185 case 410: return "Gone"; 2186 // 2187 case 500: return "Internal Server Error"; 2188 case 501: return "Not Implemented"; 2189 case 502: return "Bad Gateway"; 2190 case 503: return "Service Unavailable"; 2191 // 2192 default: assert(0, "Unsupported http code"); 2193 } 2194 } 2195 2196 2197 /// 2198 struct HttpCookie { 2199 string name; /// 2200 string value; /// 2201 string domain; /// 2202 string path; /// 2203 //SysTime expirationDate; /// 2204 bool secure; /// 2205 bool httpOnly; /// 2206 } 2207 2208 // FIXME: websocket 2209 2210 version(testing) 2211 void main() { 2212 import std.stdio; 2213 auto client = new HttpClient(); 2214 auto request = client.navigateTo(Uri("http://localhost/chunked.php")); 2215 request.send(); 2216 auto request2 = client.navigateTo(Uri("http://dlang.org/")); 2217 request2.send(); 2218 2219 { 2220 auto response = request2.waitForCompletion(); 2221 //write(cast(string) response.content); 2222 } 2223 2224 auto response = request.waitForCompletion(); 2225 write(cast(string) response.content); 2226 2227 writeln(HttpRequest.socketsPerHost); 2228 } 2229 2230 2231 // From sslsocket.d, but this is the maintained version! 2232 version(use_openssl) { 2233 alias SslClientSocket = OpenSslSocket; 2234 2235 // macros in the original C 2236 SSL_METHOD* SSLv23_client_method() { 2237 if(ossllib.SSLv23_client_method) 2238 return ossllib.SSLv23_client_method(); 2239 else 2240 return ossllib.TLS_client_method(); 2241 } 2242 2243 struct SSL {} 2244 struct SSL_CTX {} 2245 struct SSL_METHOD {} 2246 enum SSL_VERIFY_NONE = 0; 2247 2248 struct ossllib { 2249 __gshared static extern(C) { 2250 /* these are only on older openssl versions { */ 2251 int function() SSL_library_init; 2252 void function() SSL_load_error_strings; 2253 SSL_METHOD* function() SSLv23_client_method; 2254 /* } */ 2255 2256 void function(ulong, void*) OPENSSL_init_ssl; 2257 2258 SSL_CTX* function(const SSL_METHOD*) SSL_CTX_new; 2259 SSL* function(SSL_CTX*) SSL_new; 2260 int function(SSL*, int) SSL_set_fd; 2261 int function(SSL*) SSL_connect; 2262 int function(SSL*, const void*, int) SSL_write; 2263 int function(SSL*, void*, int) SSL_read; 2264 @trusted nothrow @nogc int function(SSL*) SSL_shutdown; 2265 void function(SSL*) SSL_free; 2266 void function(SSL_CTX*) SSL_CTX_free; 2267 2268 int function(const SSL*) SSL_pending; 2269 2270 void function(SSL*, int, void*) SSL_set_verify; 2271 2272 void function(SSL*, int, c_long, void*) SSL_ctrl; 2273 2274 SSL_METHOD* function() SSLv3_client_method; 2275 SSL_METHOD* function() TLS_client_method; 2276 2277 } 2278 } 2279 2280 import core.stdc.config; 2281 2282 struct eallib { 2283 __gshared static extern(C) { 2284 /* these are only on older openssl versions { */ 2285 void function() OpenSSL_add_all_ciphers; 2286 void function() OpenSSL_add_all_digests; 2287 /* } */ 2288 2289 void function(ulong, void*) OPENSSL_init_crypto; 2290 2291 void function(FILE*) ERR_print_errors_fp; 2292 } 2293 } 2294 2295 2296 SSL_CTX* SSL_CTX_new(const SSL_METHOD* a) { 2297 if(ossllib.SSL_CTX_new) 2298 return ossllib.SSL_CTX_new(a); 2299 else throw new Exception("SSL_CTX_new not loaded"); 2300 } 2301 SSL* SSL_new(SSL_CTX* a) { 2302 if(ossllib.SSL_new) 2303 return ossllib.SSL_new(a); 2304 else throw new Exception("SSL_new not loaded"); 2305 } 2306 int SSL_set_fd(SSL* a, int b) { 2307 if(ossllib.SSL_set_fd) 2308 return ossllib.SSL_set_fd(a, b); 2309 else throw new Exception("SSL_set_fd not loaded"); 2310 } 2311 int SSL_connect(SSL* a) { 2312 if(ossllib.SSL_connect) 2313 return ossllib.SSL_connect(a); 2314 else throw new Exception("SSL_connect not loaded"); 2315 } 2316 int SSL_write(SSL* a, const void* b, int c) { 2317 if(ossllib.SSL_write) 2318 return ossllib.SSL_write(a, b, c); 2319 else throw new Exception("SSL_write not loaded"); 2320 } 2321 int SSL_read(SSL* a, void* b, int c) { 2322 if(ossllib.SSL_read) 2323 return ossllib.SSL_read(a, b, c); 2324 else throw new Exception("SSL_read not loaded"); 2325 } 2326 @trusted nothrow @nogc int SSL_shutdown(SSL* a) { 2327 if(ossllib.SSL_shutdown) 2328 return ossllib.SSL_shutdown(a); 2329 assert(0); 2330 } 2331 void SSL_free(SSL* a) { 2332 if(ossllib.SSL_free) 2333 return ossllib.SSL_free(a); 2334 else throw new Exception("SSL_free not loaded"); 2335 } 2336 void SSL_CTX_free(SSL_CTX* a) { 2337 if(ossllib.SSL_CTX_free) 2338 return ossllib.SSL_CTX_free(a); 2339 else throw new Exception("SSL_CTX_free not loaded"); 2340 } 2341 2342 int SSL_pending(const SSL* a) { 2343 if(ossllib.SSL_pending) 2344 return ossllib.SSL_pending(a); 2345 else throw new Exception("SSL_pending not loaded"); 2346 } 2347 void SSL_set_verify(SSL* a, int b, void* c) { 2348 if(ossllib.SSL_set_verify) 2349 return ossllib.SSL_set_verify(a, b, c); 2350 else throw new Exception("SSL_set_verify not loaded"); 2351 } 2352 void SSL_set_tlsext_host_name(SSL* a, const char* b) { 2353 if(ossllib.SSL_ctrl) 2354 return ossllib.SSL_ctrl(a, 55 /*SSL_CTRL_SET_TLSEXT_HOSTNAME*/, 0 /*TLSEXT_NAMETYPE_host_name*/, cast(void*) b); 2355 else throw new Exception("SSL_set_tlsext_host_name not loaded"); 2356 } 2357 2358 SSL_METHOD* SSLv3_client_method() { 2359 if(ossllib.SSLv3_client_method) 2360 return ossllib.SSLv3_client_method(); 2361 else throw new Exception("SSLv3_client_method not loaded"); 2362 } 2363 SSL_METHOD* TLS_client_method() { 2364 if(ossllib.TLS_client_method) 2365 return ossllib.TLS_client_method(); 2366 else throw new Exception("TLS_client_method not loaded"); 2367 } 2368 void ERR_print_errors_fp(FILE* a) { 2369 if(eallib.ERR_print_errors_fp) 2370 return eallib.ERR_print_errors_fp(a); 2371 else throw new Exception("ERR_print_errors_fp not loaded"); 2372 } 2373 2374 2375 private __gshared void* ossllib_handle; 2376 version(Windows) 2377 private __gshared void* oeaylib_handle; 2378 else 2379 alias oeaylib_handle = ossllib_handle; 2380 version(Posix) 2381 private import core.sys.posix.dlfcn; 2382 else version(Windows) 2383 private import core.sys.windows.windows; 2384 2385 import core.stdc.stdio; 2386 2387 private __gshared Object loadSslMutex = new Object; 2388 private __gshared bool sslLoaded = false; 2389 2390 void loadOpenSsl() { 2391 if(sslLoaded) 2392 return; 2393 synchronized(loadSslMutex) { 2394 2395 version(OSX) { 2396 // newest box 2397 ossllib_handle = dlopen("libssl.1.1.dylib", RTLD_NOW); 2398 // other boxes 2399 if(ossllib_handle is null) 2400 ossllib_handle = dlopen("libssl.dylib", RTLD_NOW); 2401 // old ones like my laptop test 2402 if(ossllib_handle is null) 2403 ossllib_handle = dlopen("/usr/local/opt/openssl/lib/libssl.1.0.0.dylib", RTLD_NOW); 2404 2405 } else version(Posix) { 2406 ossllib_handle = dlopen("libssl.so.1.1", RTLD_NOW); 2407 if(ossllib_handle is null) 2408 ossllib_handle = dlopen("libssl.so", RTLD_NOW); 2409 } else version(Windows) { 2410 ossllib_handle = LoadLibraryW("libssl32.dll"w.ptr); 2411 oeaylib_handle = LoadLibraryW("libeay32.dll"w.ptr); 2412 } 2413 2414 if(ossllib_handle is null) 2415 throw new Exception("libssl library not found"); 2416 if(oeaylib_handle is null) 2417 throw new Exception("libeay32 library not found"); 2418 2419 foreach(memberName; __traits(allMembers, ossllib)) { 2420 alias t = typeof(__traits(getMember, ossllib, memberName)); 2421 version(Posix) 2422 __traits(getMember, ossllib, memberName) = cast(t) dlsym(ossllib_handle, memberName); 2423 else version(Windows) { 2424 __traits(getMember, ossllib, memberName) = cast(t) GetProcAddress(ossllib_handle, memberName); 2425 } 2426 } 2427 2428 foreach(memberName; __traits(allMembers, eallib)) { 2429 alias t = typeof(__traits(getMember, eallib, memberName)); 2430 version(Posix) 2431 __traits(getMember, eallib, memberName) = cast(t) dlsym(oeaylib_handle, memberName); 2432 else version(Windows) { 2433 __traits(getMember, eallib, memberName) = cast(t) GetProcAddress(oeaylib_handle, memberName); 2434 } 2435 } 2436 2437 2438 if(ossllib.SSL_library_init) 2439 ossllib.SSL_library_init(); 2440 else if(ossllib.OPENSSL_init_ssl) 2441 ossllib.OPENSSL_init_ssl(0, null); 2442 else throw new Exception("couldn't init openssl"); 2443 2444 if(eallib.OpenSSL_add_all_ciphers) { 2445 eallib.OpenSSL_add_all_ciphers(); 2446 if(eallib.OpenSSL_add_all_digests is null) 2447 throw new Exception("no add digests"); 2448 eallib.OpenSSL_add_all_digests(); 2449 } else if(eallib.OPENSSL_init_crypto) 2450 eallib.OPENSSL_init_crypto(0 /*OPENSSL_INIT_ADD_ALL_CIPHERS and ALL_DIGESTS together*/, null); 2451 else throw new Exception("couldn't init crypto openssl"); 2452 2453 if(ossllib.SSL_load_error_strings) 2454 ossllib.SSL_load_error_strings(); 2455 else if(ossllib.OPENSSL_init_ssl) 2456 ossllib.OPENSSL_init_ssl(0x00200000L, null); 2457 else throw new Exception("couldn't load openssl errors"); 2458 2459 sslLoaded = true; 2460 } 2461 } 2462 2463 /+ 2464 // I'm just gonna let the OS clean this up on process termination because otherwise SSL_free 2465 // might have trouble being run from the GC after this module is unloaded. 2466 shared static ~this() { 2467 if(ossllib_handle) { 2468 version(Windows) { 2469 FreeLibrary(oeaylib_handle); 2470 FreeLibrary(ossllib_handle); 2471 } else version(Posix) 2472 dlclose(ossllib_handle); 2473 ossllib_handle = null; 2474 } 2475 ossllib.tupleof = ossllib.tupleof.init; 2476 } 2477 +/ 2478 2479 //pragma(lib, "crypto"); 2480 //pragma(lib, "ssl"); 2481 2482 class OpenSslSocket : Socket { 2483 private SSL* ssl; 2484 private SSL_CTX* ctx; 2485 private void initSsl(bool verifyPeer, string hostname) { 2486 ctx = SSL_CTX_new(SSLv23_client_method()); 2487 assert(ctx !is null); 2488 2489 ssl = SSL_new(ctx); 2490 2491 if(hostname.length) 2492 SSL_set_tlsext_host_name(ssl, toStringz(hostname)); 2493 2494 if(!verifyPeer) 2495 SSL_set_verify(ssl, SSL_VERIFY_NONE, null); 2496 SSL_set_fd(ssl, cast(int) this.handle); // on win64 it is necessary to truncate, but the value is never large anyway see http://openssl.6102.n7.nabble.com/Sockets-windows-64-bit-td36169.html 2497 } 2498 2499 bool dataPending() { 2500 return SSL_pending(ssl) > 0; 2501 } 2502 2503 @trusted 2504 override void connect(Address to) { 2505 super.connect(to); 2506 do_ssl_connect(); 2507 } 2508 2509 @trusted 2510 void do_ssl_connect() { 2511 if(SSL_connect(ssl) == -1) { 2512 ERR_print_errors_fp(core.stdc.stdio.stderr); 2513 int i; 2514 //printf("wtf\n"); 2515 //scanf("%d\n", i); 2516 throw new Exception("ssl connect"); 2517 } 2518 } 2519 2520 @trusted 2521 override ptrdiff_t send(scope const(void)[] buf, SocketFlags flags) { 2522 //import std.stdio;writeln(cast(string) buf); 2523 auto retval = SSL_write(ssl, buf.ptr, cast(uint) buf.length); 2524 if(retval == -1) { 2525 ERR_print_errors_fp(core.stdc.stdio.stderr); 2526 int i; 2527 //printf("wtf\n"); 2528 //scanf("%d\n", i); 2529 throw new Exception("ssl send"); 2530 } 2531 return retval; 2532 2533 } 2534 override ptrdiff_t send(scope const(void)[] buf) { 2535 return send(buf, SocketFlags.NONE); 2536 } 2537 @trusted 2538 override ptrdiff_t receive(scope void[] buf, SocketFlags flags) { 2539 auto retval = SSL_read(ssl, buf.ptr, cast(int)buf.length); 2540 if(retval == -1) { 2541 ERR_print_errors_fp(core.stdc.stdio.stderr); 2542 int i; 2543 //printf("wtf\n"); 2544 //scanf("%d\n", i); 2545 throw new Exception("ssl send"); 2546 } 2547 return retval; 2548 } 2549 override ptrdiff_t receive(scope void[] buf) { 2550 return receive(buf, SocketFlags.NONE); 2551 } 2552 2553 this(AddressFamily af, SocketType type = SocketType.STREAM, string hostname = null, bool verifyPeer = true) { 2554 super(af, type); 2555 initSsl(verifyPeer, hostname); 2556 } 2557 2558 override void close() { 2559 if(ssl) SSL_shutdown(ssl); 2560 super.close(); 2561 } 2562 2563 this(socket_t sock, AddressFamily af, string hostname, bool verifyPeer = true) { 2564 super(sock, af); 2565 initSsl(verifyPeer, hostname); 2566 } 2567 2568 void freeSsl() { 2569 if(ssl is null) 2570 return; 2571 SSL_free(ssl); 2572 SSL_CTX_free(ctx); 2573 ssl = null; 2574 } 2575 2576 ~this() { 2577 freeSsl(); 2578 } 2579 } 2580 } 2581 2582 2583 /++ 2584 An experimental component for working with REST apis. Note that it 2585 is a zero-argument template, so to create one, use `new HttpApiClient!()(args..)` 2586 or you will get "HttpApiClient is used as a type" compile errors. 2587 2588 This will probably not work for you yet, and I might change it significantly. 2589 2590 Requires [arsd.jsvar]. 2591 2592 2593 Here's a snippet to create a pull request on GitHub to Phobos: 2594 2595 --- 2596 auto github = new HttpApiClient!()("https://api.github.com/", "your personal api token here"); 2597 2598 // create the arguments object 2599 // see: https://developer.github.com/v3/pulls/#create-a-pull-request 2600 var args = var.emptyObject; 2601 args.title = "My Pull Request"; 2602 args.head = "yourusername:" ~ branchName; 2603 args.base = "master"; 2604 // note it is ["body"] instead of .body because `body` is a D keyword 2605 args["body"] = "My cool PR is opened by the API!"; 2606 args.maintainer_can_modify = true; 2607 2608 /+ 2609 Fun fact, you can also write that: 2610 2611 var args = [ 2612 "title": "My Pull Request".var, 2613 "head": "yourusername:" ~ branchName.var, 2614 "base" : "master".var, 2615 "body" : "My cool PR is opened by the API!".var, 2616 "maintainer_can_modify": true.var 2617 ]; 2618 2619 Note the .var constructor calls in there. If everything is the same type, you actually don't need that, but here since there's strings and bools, D won't allow the literal without explicit constructors to align them all. 2620 +/ 2621 2622 // this translates to `repos/dlang/phobos/pulls` and sends a POST request, 2623 // containing `args` as json, then immediately grabs the json result and extracts 2624 // the value `html_url` from it. `prUrl` is typed `var`, from arsd.jsvar. 2625 auto prUrl = github.rest.repos.dlang.phobos.pulls.POST(args).result.html_url; 2626 2627 writeln("Created: ", prUrl); 2628 --- 2629 2630 Why use this instead of just building the URL? Well, of course you can! This just makes 2631 it a bit more convenient than string concatenation and manages a few headers for you. 2632 2633 Subtypes could potentially add static type checks too. 2634 +/ 2635 class HttpApiClient() { 2636 import arsd.jsvar; 2637 2638 HttpClient httpClient; 2639 2640 alias HttpApiClientType = typeof(this); 2641 2642 string urlBase; 2643 string oauth2Token; 2644 string submittedContentType; 2645 2646 /++ 2647 Params: 2648 2649 urlBase = The base url for the api. Tends to be something like `https://api.example.com/v2/` or similar. 2650 oauth2Token = the authorization token for the service. You'll have to get it from somewhere else. 2651 submittedContentType = the content-type of POST, PUT, etc. bodies. 2652 httpClient = an injected http client, or null if you want to use a default-constructed one 2653 2654 History: 2655 The `httpClient` param was added on December 26, 2020. 2656 +/ 2657 this(string urlBase, string oauth2Token, string submittedContentType = "application/json", HttpClient httpClient = null) { 2658 if(httpClient is null) 2659 this.httpClient = new HttpClient(); 2660 else 2661 this.httpClient = httpClient; 2662 2663 assert(urlBase[0] == 'h'); 2664 assert(urlBase[$-1] == '/'); 2665 2666 this.urlBase = urlBase; 2667 this.oauth2Token = oauth2Token; 2668 this.submittedContentType = submittedContentType; 2669 } 2670 2671 /// 2672 static struct HttpRequestWrapper { 2673 HttpApiClientType apiClient; /// 2674 HttpRequest request; /// 2675 HttpResponse _response; 2676 2677 /// 2678 this(HttpApiClientType apiClient, HttpRequest request) { 2679 this.apiClient = apiClient; 2680 this.request = request; 2681 } 2682 2683 /// Returns the full [HttpResponse] object so you can inspect the headers 2684 @property HttpResponse response() { 2685 if(_response is HttpResponse.init) 2686 _response = request.waitForCompletion(); 2687 return _response; 2688 } 2689 2690 /++ 2691 Returns the parsed JSON from the body of the response. 2692 2693 Throws on non-2xx responses. 2694 +/ 2695 var result() { 2696 return apiClient.throwOnError(response); 2697 } 2698 2699 alias request this; 2700 } 2701 2702 /// 2703 HttpRequestWrapper request(string uri, HttpVerb requestMethod = HttpVerb.GET, ubyte[] bodyBytes = null) { 2704 if(uri[0] == '/') 2705 uri = uri[1 .. $]; 2706 2707 auto u = Uri(uri).basedOn(Uri(urlBase)); 2708 2709 auto req = httpClient.navigateTo(u, requestMethod); 2710 2711 if(oauth2Token.length) 2712 req.requestParameters.headers ~= "Authorization: Bearer " ~ oauth2Token; 2713 req.requestParameters.contentType = submittedContentType; 2714 req.requestParameters.bodyData = bodyBytes; 2715 2716 return HttpRequestWrapper(this, req); 2717 } 2718 2719 /// 2720 var throwOnError(HttpResponse res) { 2721 if(res.code < 200 || res.code >= 300) 2722 throw new Exception(res.codeText ~ " " ~ res.contentText); 2723 2724 var response = var.fromJson(res.contentText); 2725 if(response.errors) { 2726 throw new Exception(response.errors.toJson()); 2727 } 2728 2729 return response; 2730 } 2731 2732 /// 2733 @property RestBuilder rest() { 2734 return RestBuilder(this, null, null); 2735 } 2736 2737 // hipchat.rest.room["Tech Team"].history 2738 // gives: "/room/Tech%20Team/history" 2739 // 2740 // hipchat.rest.room["Tech Team"].history("page", "12) 2741 /// 2742 static struct RestBuilder { 2743 HttpApiClientType apiClient; 2744 string[] pathParts; 2745 string[2][] queryParts; 2746 this(HttpApiClientType apiClient, string[] pathParts, string[2][] queryParts) { 2747 this.apiClient = apiClient; 2748 this.pathParts = pathParts; 2749 this.queryParts = queryParts; 2750 } 2751 2752 RestBuilder _SELF() { 2753 return this; 2754 } 2755 2756 /// The args are so you can call opCall on the returned 2757 /// object, despite @property being broken af in D. 2758 RestBuilder opDispatch(string str, T)(string n, T v) { 2759 return RestBuilder(apiClient, pathParts ~ str, queryParts ~ [n, to!string(v)]); 2760 } 2761 2762 /// 2763 RestBuilder opDispatch(string str)() { 2764 return RestBuilder(apiClient, pathParts ~ str, queryParts); 2765 } 2766 2767 2768 /// 2769 RestBuilder opIndex(string str) { 2770 return RestBuilder(apiClient, pathParts ~ str, queryParts); 2771 } 2772 /// 2773 RestBuilder opIndex(var str) { 2774 return RestBuilder(apiClient, pathParts ~ str.get!string, queryParts); 2775 } 2776 /// 2777 RestBuilder opIndex(int i) { 2778 return RestBuilder(apiClient, pathParts ~ to!string(i), queryParts); 2779 } 2780 2781 /// 2782 RestBuilder opCall(T)(string name, T value) { 2783 return RestBuilder(apiClient, pathParts, queryParts ~ [name, to!string(value)]); 2784 } 2785 2786 /// 2787 string toUri() { 2788 import std.uri; 2789 string result; 2790 foreach(idx, part; pathParts) { 2791 if(idx) 2792 result ~= "/"; 2793 result ~= encodeComponent(part); 2794 } 2795 result ~= "?"; 2796 foreach(idx, part; queryParts) { 2797 if(idx) 2798 result ~= "&"; 2799 result ~= encodeComponent(part[0]); 2800 result ~= "="; 2801 result ~= encodeComponent(part[1]); 2802 } 2803 2804 return result; 2805 } 2806 2807 /// 2808 final HttpRequestWrapper GET() { return _EXECUTE(HttpVerb.GET, this.toUri(), ToBytesResult.init); } 2809 /// ditto 2810 final HttpRequestWrapper DELETE() { return _EXECUTE(HttpVerb.DELETE, this.toUri(), ToBytesResult.init); } 2811 2812 // need to be able to send: JSON, urlencoded, multipart/form-data, and raw stuff. 2813 /// ditto 2814 final HttpRequestWrapper POST(T...)(T t) { return _EXECUTE(HttpVerb.POST, this.toUri(), toBytes(t)); } 2815 /// ditto 2816 final HttpRequestWrapper PATCH(T...)(T t) { return _EXECUTE(HttpVerb.PATCH, this.toUri(), toBytes(t)); } 2817 /// ditto 2818 final HttpRequestWrapper PUT(T...)(T t) { return _EXECUTE(HttpVerb.PUT, this.toUri(), toBytes(t)); } 2819 2820 struct ToBytesResult { 2821 ubyte[] bytes; 2822 string contentType; 2823 } 2824 2825 private ToBytesResult toBytes(T...)(T t) { 2826 import std.conv : to; 2827 static if(T.length == 0) 2828 return ToBytesResult(null, null); 2829 else static if(T.length == 1 && is(T[0] == var)) 2830 return ToBytesResult(cast(ubyte[]) t[0].toJson(), "application/json"); // json data 2831 else static if(T.length == 1 && (is(T[0] == string) || is(T[0] == ubyte[]))) 2832 return ToBytesResult(cast(ubyte[]) t[0], null); // raw data 2833 else static if(T.length == 1 && is(T[0] : FormData)) 2834 return ToBytesResult(t[0].toBytes, t[0].contentType); 2835 else static if(T.length > 1 && T.length % 2 == 0 && is(T[0] == string)) { 2836 // string -> value pairs for a POST request 2837 string answer; 2838 foreach(idx, val; t) { 2839 static if(idx % 2 == 0) { 2840 if(answer.length) 2841 answer ~= "&"; 2842 answer ~= encodeComponent(val); // it had better be a string! lol 2843 answer ~= "="; 2844 } else { 2845 answer ~= encodeComponent(to!string(val)); 2846 } 2847 } 2848 2849 return ToBytesResult(cast(ubyte[]) answer, "application/x-www-form-urlencoded"); 2850 } 2851 else 2852 static assert(0); // FIXME 2853 2854 } 2855 2856 HttpRequestWrapper _EXECUTE(HttpVerb verb, string uri, ubyte[] bodyBytes) { 2857 return apiClient.request(uri, verb, bodyBytes); 2858 } 2859 2860 HttpRequestWrapper _EXECUTE(HttpVerb verb, string uri, ToBytesResult tbr) { 2861 auto r = apiClient.request(uri, verb, tbr.bytes); 2862 if(tbr.contentType !is null) 2863 r.requestParameters.contentType = tbr.contentType; 2864 return r; 2865 } 2866 } 2867 } 2868 2869 2870 // see also: arsd.cgi.encodeVariables 2871 /// Creates a multipart/form-data object that is suitable for file uploads and other kinds of POST 2872 class FormData { 2873 struct MimePart { 2874 string name; 2875 const(void)[] data; 2876 string contentType; 2877 string filename; 2878 } 2879 2880 MimePart[] parts; 2881 2882 /// 2883 void append(string key, in void[] value, string contentType = null, string filename = null) { 2884 parts ~= MimePart(key, value, contentType, filename); 2885 } 2886 2887 private string boundary = "0016e64be86203dd36047610926a"; // FIXME 2888 2889 string contentType() { 2890 return "multipart/form-data; boundary=" ~ boundary; 2891 } 2892 2893 /// 2894 ubyte[] toBytes() { 2895 string data; 2896 2897 foreach(part; parts) { 2898 data ~= "--" ~ boundary ~ "\r\n"; 2899 data ~= "Content-Disposition: form-data; name=\""~part.name~"\""; 2900 if(part.filename !is null) 2901 data ~= "; filename=\""~part.filename~"\""; 2902 data ~= "\r\n"; 2903 if(part.contentType !is null) 2904 data ~= "Content-Type: " ~ part.contentType ~ "\r\n"; 2905 data ~= "\r\n"; 2906 2907 data ~= cast(string) part.data; 2908 2909 data ~= "\r\n"; 2910 } 2911 2912 data ~= "--" ~ boundary ~ "--\r\n"; 2913 2914 return cast(ubyte[]) data; 2915 } 2916 } 2917 2918 private bool bicmp(in ubyte[] item, in char[] search) { 2919 if(item.length != search.length) return false; 2920 2921 foreach(i; 0 .. item.length) { 2922 ubyte a = item[i]; 2923 ubyte b = search[i]; 2924 if(a >= 'A' && a <= 'Z') 2925 a += 32; 2926 //if(b >= 'A' && b <= 'Z') 2927 //b += 32; 2928 if(a != b) 2929 return false; 2930 } 2931 2932 return true; 2933 } 2934 2935 /++ 2936 WebSocket client, based on the browser api, though also with other api options. 2937 2938 --- 2939 auto ws = new WebSocket(URI("ws://....")); 2940 2941 ws.onmessage = (in char[] msg) { 2942 ws.send("a reply"); 2943 }; 2944 2945 ws.connect(); 2946 2947 WebSocket.eventLoop(); 2948 --- 2949 2950 Symbol_groups: 2951 foundational = 2952 Used with all API styles. 2953 2954 browser_api = 2955 API based on the standard in the browser. 2956 2957 event_loop_integration = 2958 Integrating with external event loops is done through static functions. You should 2959 call these BEFORE doing anything else with the WebSocket module or class. 2960 2961 $(PITFALL NOT IMPLEMENTED) 2962 --- 2963 WebSocket.setEventLoopProxy(arsd.simpledisplay.EventLoop.proxy.tupleof); 2964 // or something like that. it is not implemented yet. 2965 --- 2966 $(PITFALL NOT IMPLEMENTED) 2967 2968 blocking_api = 2969 The blocking API is best used when you only need basic functionality with a single connection. 2970 2971 --- 2972 WebSocketFrame msg; 2973 do { 2974 // FIXME good demo 2975 } while(msg); 2976 --- 2977 2978 Or to check for blocks before calling: 2979 2980 --- 2981 try_to_process_more: 2982 while(ws.isMessageBuffered()) { 2983 auto msg = ws.waitForNextMessage(); 2984 // process msg 2985 } 2986 if(ws.isDataPending()) { 2987 ws.lowLevelReceive(); 2988 goto try_to_process_more; 2989 } else { 2990 // nothing ready, you can do other things 2991 // or at least sleep a while before trying 2992 // to process more. 2993 if(ws.readyState == WebSocket.OPEN) { 2994 Thread.sleep(1.seconds); 2995 goto try_to_process_more; 2996 } 2997 } 2998 --- 2999 3000 +/ 3001 class WebSocket { 3002 private Uri uri; 3003 private string[string] cookies; 3004 private string origin; 3005 3006 private string host; 3007 private ushort port; 3008 private bool ssl; 3009 3010 private MonoTime timeoutFromInactivity; 3011 private MonoTime nextPing; 3012 3013 /++ 3014 wss://echo.websocket.org 3015 +/ 3016 /// Group: foundational 3017 this(Uri uri, Config config = Config.init) 3018 //in (uri.scheme == "ws" || uri.scheme == "wss") 3019 in { assert(uri.scheme == "ws" || uri.scheme == "wss"); } do 3020 { 3021 this.uri = uri; 3022 this.config = config; 3023 3024 this.receiveBuffer = new ubyte[](config.initialReceiveBufferSize); 3025 3026 host = uri.host; 3027 ssl = uri.scheme == "wss"; 3028 port = cast(ushort) (uri.port ? uri.port : ssl ? 443 : 80); 3029 3030 if(ssl) { 3031 version(with_openssl) { 3032 loadOpenSsl(); 3033 socket = new SslClientSocket(family(uri.unixSocketPath), SocketType.STREAM, host, defaultVerifyPeer); 3034 } else 3035 throw new Exception("SSL not compiled in"); 3036 } else 3037 socket = new Socket(family(uri.unixSocketPath), SocketType.STREAM); 3038 3039 } 3040 3041 /++ 3042 3043 +/ 3044 /// Group: foundational 3045 void connect() { 3046 if(uri.unixSocketPath) 3047 socket.connect(new UnixAddress(uri.unixSocketPath)); 3048 else 3049 socket.connect(new InternetAddress(host, port)); // FIXME: ipv6 support... 3050 // FIXME: websocket handshake could and really should be async too. 3051 3052 auto uri = this.uri.path.length ? this.uri.path : "/"; 3053 if(this.uri.query.length) { 3054 uri ~= "?"; 3055 uri ~= this.uri.query; 3056 } 3057 3058 // the headers really shouldn't be bigger than this, at least 3059 // the chunks i need to process 3060 ubyte[4096] bufferBacking = void; 3061 ubyte[] buffer = bufferBacking[]; 3062 size_t pos; 3063 3064 void append(in char[][] items...) { 3065 foreach(what; items) { 3066 if((pos + what.length) > buffer.length) { 3067 buffer.length += 4096; 3068 } 3069 buffer[pos .. pos + what.length] = cast(ubyte[]) what[]; 3070 pos += what.length; 3071 } 3072 } 3073 3074 append("GET ", uri, " HTTP/1.1\r\n"); 3075 append("Host: ", this.uri.host, "\r\n"); 3076 3077 append("Upgrade: websocket\r\n"); 3078 append("Connection: Upgrade\r\n"); 3079 append("Sec-WebSocket-Version: 13\r\n"); 3080 3081 // FIXME: randomize this 3082 append("Sec-WebSocket-Key: x3JEHMbDL1EzLkh9GBhXDw==\r\n"); 3083 3084 if(config.protocol.length) 3085 append("Sec-WebSocket-Protocol: ", config.protocol, "\r\n"); 3086 if(config.origin.length) 3087 append("Origin: ", origin, "\r\n"); 3088 3089 foreach(h; config.additionalHeaders) { 3090 append(h); 3091 append("\r\n"); 3092 } 3093 3094 append("\r\n"); 3095 3096 auto remaining = buffer[0 .. pos]; 3097 //import std.stdio; writeln(host, " " , port, " ", cast(string) remaining); 3098 while(remaining.length) { 3099 auto r = socket.send(remaining); 3100 if(r < 0) 3101 throw new Exception(lastSocketError()); 3102 if(r == 0) 3103 throw new Exception("unexpected connection termination"); 3104 remaining = remaining[r .. $]; 3105 } 3106 3107 // the response shouldn't be especially large at this point, just 3108 // headers for the most part. gonna try to get it in the stack buffer. 3109 // then copy stuff after headers, if any, to the frame buffer. 3110 ubyte[] used; 3111 3112 void more() { 3113 auto r = socket.receive(buffer[used.length .. $]); 3114 3115 if(r < 0) 3116 throw new Exception(lastSocketError()); 3117 if(r == 0) 3118 throw new Exception("unexpected connection termination"); 3119 //import std.stdio;writef("%s", cast(string) buffer[used.length .. used.length + r]); 3120 3121 used = buffer[0 .. used.length + r]; 3122 } 3123 3124 more(); 3125 3126 import std.algorithm; 3127 if(!used.startsWith(cast(ubyte[]) "HTTP/1.1 101")) 3128 throw new Exception("didn't get a websocket answer"); 3129 // skip the status line 3130 while(used.length && used[0] != '\n') 3131 used = used[1 .. $]; 3132 3133 if(used.length == 0) 3134 throw new Exception("Remote server disconnected or didn't send enough information"); 3135 3136 if(used.length < 1) 3137 more(); 3138 3139 used = used[1 .. $]; // skip the \n 3140 3141 if(used.length == 0) 3142 more(); 3143 3144 // checks on the protocol from ehaders 3145 bool isWebsocket; 3146 bool isUpgrade; 3147 const(ubyte)[] protocol; 3148 const(ubyte)[] accept; 3149 3150 while(used.length) { 3151 if(used.length >= 2 && used[0] == '\r' && used[1] == '\n') { 3152 used = used[2 .. $]; 3153 break; // all done 3154 } 3155 int idxColon; 3156 while(idxColon < used.length && used[idxColon] != ':') 3157 idxColon++; 3158 if(idxColon == used.length) 3159 more(); 3160 auto idxStart = idxColon + 1; 3161 while(idxStart < used.length && used[idxStart] == ' ') 3162 idxStart++; 3163 if(idxStart == used.length) 3164 more(); 3165 auto idxEnd = idxStart; 3166 while(idxEnd < used.length && used[idxEnd] != '\r') 3167 idxEnd++; 3168 if(idxEnd == used.length) 3169 more(); 3170 3171 auto headerName = used[0 .. idxColon]; 3172 auto headerValue = used[idxStart .. idxEnd]; 3173 3174 // move past this header 3175 used = used[idxEnd .. $]; 3176 // and the \r\n 3177 if(2 <= used.length) 3178 used = used[2 .. $]; 3179 3180 if(headerName.bicmp("upgrade")) { 3181 if(headerValue.bicmp("websocket")) 3182 isWebsocket = true; 3183 } else if(headerName.bicmp("connection")) { 3184 if(headerValue.bicmp("upgrade")) 3185 isUpgrade = true; 3186 } else if(headerName.bicmp("sec-websocket-accept")) { 3187 accept = headerValue; 3188 } else if(headerName.bicmp("sec-websocket-protocol")) { 3189 protocol = headerValue; 3190 } 3191 3192 if(!used.length) { 3193 more(); 3194 } 3195 } 3196 3197 3198 if(!isWebsocket) 3199 throw new Exception("didn't answer as websocket"); 3200 if(!isUpgrade) 3201 throw new Exception("didn't answer as upgrade"); 3202 3203 3204 // FIXME: check protocol if config requested one 3205 // FIXME: check accept for the right hash 3206 3207 receiveBuffer[0 .. used.length] = used[]; 3208 receiveBufferUsedLength = used.length; 3209 3210 readyState_ = OPEN; 3211 3212 if(onopen) 3213 onopen(); 3214 3215 nextPing = MonoTime.currTime + config.pingFrequency.msecs; 3216 timeoutFromInactivity = MonoTime.currTime + config.timeoutFromInactivity; 3217 3218 registerActiveSocket(this); 3219 } 3220 3221 /++ 3222 Is data pending on the socket? Also check [isMessageBuffered] to see if there 3223 is already a message in memory too. 3224 3225 If this returns `true`, you can call [lowLevelReceive], then try [isMessageBuffered] 3226 again. 3227 +/ 3228 /// Group: blocking_api 3229 public bool isDataPending(Duration timeout = 0.seconds) { 3230 static SocketSet readSet; 3231 if(readSet is null) 3232 readSet = new SocketSet(); 3233 3234 version(with_openssl) 3235 if(auto s = cast(SslClientSocket) socket) { 3236 // select doesn't handle the case with stuff 3237 // left in the ssl buffer so i'm checking it separately 3238 if(s.dataPending()) { 3239 return true; 3240 } 3241 } 3242 3243 readSet.add(socket); 3244 3245 //tryAgain: 3246 auto selectGot = Socket.select(readSet, null, null, timeout); 3247 if(selectGot == 0) { /* timeout */ 3248 // timeout 3249 return false; 3250 } else if(selectGot == -1) { /* interrupted */ 3251 return false; 3252 } else { /* ready */ 3253 if(readSet.isSet(socket)) { 3254 return true; 3255 } 3256 } 3257 3258 return false; 3259 } 3260 3261 private void llsend(ubyte[] d) { 3262 if(readyState == CONNECTING) 3263 throw new Exception("WebSocket not connected when trying to send. Did you forget to call connect(); ?"); 3264 //connect(); 3265 while(d.length) { 3266 auto r = socket.send(d); 3267 if(r <= 0) throw new Exception("Socket send failed"); 3268 d = d[r .. $]; 3269 } 3270 } 3271 3272 private void llclose() { 3273 socket.shutdown(SocketShutdown.SEND); 3274 } 3275 3276 /++ 3277 Waits for more data off the low-level socket and adds it to the pending buffer. 3278 3279 Returns `true` if the connection is still active. 3280 +/ 3281 /// Group: blocking_api 3282 public bool lowLevelReceive() { 3283 if(readyState == CONNECTING) 3284 throw new Exception("WebSocket not connected when trying to receive. Did you forget to call connect(); ?"); 3285 auto r = socket.receive(receiveBuffer[receiveBufferUsedLength .. $]); 3286 if(r == 0) 3287 return false; 3288 if(r <= 0) 3289 throw new Exception("Socket receive failed"); 3290 receiveBufferUsedLength += r; 3291 return true; 3292 } 3293 3294 private Socket socket; 3295 3296 /* copy/paste section { */ 3297 3298 private int readyState_; 3299 private ubyte[] receiveBuffer; 3300 private size_t receiveBufferUsedLength; 3301 3302 private Config config; 3303 3304 enum CONNECTING = 0; /// Socket has been created. The connection is not yet open. 3305 enum OPEN = 1; /// The connection is open and ready to communicate. 3306 enum CLOSING = 2; /// The connection is in the process of closing. 3307 enum CLOSED = 3; /// The connection is closed or couldn't be opened. 3308 3309 /++ 3310 3311 +/ 3312 /// Group: foundational 3313 static struct Config { 3314 /++ 3315 These control the size of the receive buffer. 3316 3317 It starts at the initial size, will temporarily 3318 balloon up to the maximum size, and will reuse 3319 a buffer up to the likely size. 3320 3321 Anything larger than the maximum size will cause 3322 the connection to be aborted and an exception thrown. 3323 This is to protect you against a peer trying to 3324 exhaust your memory, while keeping the user-level 3325 processing simple. 3326 +/ 3327 size_t initialReceiveBufferSize = 4096; 3328 size_t likelyReceiveBufferSize = 4096; /// ditto 3329 size_t maximumReceiveBufferSize = 10 * 1024 * 1024; /// ditto 3330 3331 /++ 3332 Maximum combined size of a message. 3333 +/ 3334 size_t maximumMessageSize = 10 * 1024 * 1024; 3335 3336 string[string] cookies; /// Cookies to send with the initial request. cookies[name] = value; 3337 string origin; /// Origin URL to send with the handshake, if desired. 3338 string protocol; /// the protocol header, if desired. 3339 3340 /++ 3341 Additional headers to put in the HTTP request. These should be formatted `Name: value`, like for example: 3342 3343 --- 3344 Config config; 3345 config.additionalHeaders ~= "Authorization: Bearer your_auth_token_here"; 3346 --- 3347 3348 History: 3349 Added February 19, 2021 (included in dub version 9.2) 3350 +/ 3351 string[] additionalHeaders; 3352 3353 int pingFrequency = 5000; /// Amount of time (in msecs) of idleness after which to send an automatic ping 3354 3355 /++ 3356 Amount of time to disconnect when there's no activity. Note that automatic pings will keep the connection alive; this timeout only occurs if there's absolutely nothing, including no responses to websocket ping frames. Since the default [pingFrequency] is only seconds, this one minute should never elapse unless the connection is actually dead. 3357 3358 The one thing to keep in mind is if your program is busy and doesn't check input, it might consider this a time out since there's no activity. The reason is that your program was busy rather than a connection failure, but it doesn't care. You should avoid long processing periods anyway though! 3359 3360 History: 3361 Added March 31, 2021 (included in dub version 9.4) 3362 +/ 3363 Duration timeoutFromInactivity = 1.minutes; 3364 } 3365 3366 /++ 3367 Returns one of [CONNECTING], [OPEN], [CLOSING], or [CLOSED]. 3368 +/ 3369 int readyState() { 3370 return readyState_; 3371 } 3372 3373 /++ 3374 Closes the connection, sending a graceful teardown message to the other side. 3375 +/ 3376 /// Group: foundational 3377 void close(int code = 0, string reason = null) 3378 //in (reason.length < 123) 3379 in { assert(reason.length < 123); } do 3380 { 3381 if(readyState_ != OPEN) 3382 return; // it cool, we done 3383 WebSocketFrame wss; 3384 wss.fin = true; 3385 wss.opcode = WebSocketOpcode.close; 3386 wss.data = cast(ubyte[]) reason; 3387 wss.send(&llsend); 3388 3389 readyState_ = CLOSING; 3390 3391 llclose(); 3392 } 3393 3394 /++ 3395 Sends a ping message to the server. This is done automatically by the library if you set a non-zero [Config.pingFrequency], but you can also send extra pings explicitly as well with this function. 3396 +/ 3397 /// Group: foundational 3398 void ping() { 3399 WebSocketFrame wss; 3400 wss.fin = true; 3401 wss.opcode = WebSocketOpcode.ping; 3402 wss.send(&llsend); 3403 } 3404 3405 // automatically handled.... 3406 void pong() { 3407 WebSocketFrame wss; 3408 wss.fin = true; 3409 wss.opcode = WebSocketOpcode.pong; 3410 wss.send(&llsend); 3411 } 3412 3413 /++ 3414 Sends a text message through the websocket. 3415 +/ 3416 /// Group: foundational 3417 void send(in char[] textData) { 3418 WebSocketFrame wss; 3419 wss.fin = true; 3420 wss.opcode = WebSocketOpcode.text; 3421 wss.data = cast(ubyte[]) textData; 3422 wss.send(&llsend); 3423 } 3424 3425 /++ 3426 Sends a binary message through the websocket. 3427 +/ 3428 /// Group: foundational 3429 void send(in ubyte[] binaryData) { 3430 WebSocketFrame wss; 3431 wss.fin = true; 3432 wss.opcode = WebSocketOpcode.binary; 3433 wss.data = cast(ubyte[]) binaryData; 3434 wss.send(&llsend); 3435 } 3436 3437 /++ 3438 Waits for and returns the next complete message on the socket. 3439 3440 Note that the onmessage function is still called, right before 3441 this returns. 3442 +/ 3443 /// Group: blocking_api 3444 public WebSocketFrame waitForNextMessage() { 3445 do { 3446 auto m = processOnce(); 3447 if(m.populated) 3448 return m; 3449 } while(lowLevelReceive()); 3450 3451 return WebSocketFrame.init; // FIXME? maybe. 3452 } 3453 3454 /++ 3455 Tells if [waitForNextMessage] would block. 3456 +/ 3457 /// Group: blocking_api 3458 public bool waitForNextMessageWouldBlock() { 3459 checkAgain: 3460 if(isMessageBuffered()) 3461 return false; 3462 if(!isDataPending()) 3463 return true; 3464 while(isDataPending()) 3465 lowLevelReceive(); 3466 goto checkAgain; 3467 } 3468 3469 /++ 3470 Is there a message in the buffer already? 3471 If `true`, [waitForNextMessage] is guaranteed to return immediately. 3472 If `false`, check [isDataPending] as the next step. 3473 +/ 3474 /// Group: blocking_api 3475 public bool isMessageBuffered() { 3476 ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; 3477 auto s = d; 3478 if(d.length) { 3479 auto orig = d; 3480 auto m = WebSocketFrame.read(d); 3481 // that's how it indicates that it needs more data 3482 if(d !is orig) 3483 return true; 3484 } 3485 3486 return false; 3487 } 3488 3489 private ubyte continuingType; 3490 private ubyte[] continuingData; 3491 //private size_t continuingDataLength; 3492 3493 private WebSocketFrame processOnce() { 3494 ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; 3495 //import std.stdio; writeln(d); 3496 auto s = d; 3497 // FIXME: handle continuation frames more efficiently. it should really just reuse the receive buffer. 3498 WebSocketFrame m; 3499 if(d.length) { 3500 auto orig = d; 3501 m = WebSocketFrame.read(d); 3502 // that's how it indicates that it needs more data 3503 if(d is orig) 3504 return WebSocketFrame.init; 3505 m.unmaskInPlace(); 3506 switch(m.opcode) { 3507 case WebSocketOpcode.continuation: 3508 if(continuingData.length + m.data.length > config.maximumMessageSize) 3509 throw new Exception("message size exceeded"); 3510 3511 continuingData ~= m.data; 3512 if(m.fin) { 3513 if(ontextmessage) 3514 ontextmessage(cast(char[]) continuingData); 3515 if(onbinarymessage) 3516 onbinarymessage(continuingData); 3517 3518 continuingData = null; 3519 } 3520 break; 3521 case WebSocketOpcode.text: 3522 if(m.fin) { 3523 if(ontextmessage) 3524 ontextmessage(m.textData); 3525 } else { 3526 continuingType = m.opcode; 3527 //continuingDataLength = 0; 3528 continuingData = null; 3529 continuingData ~= m.data; 3530 } 3531 break; 3532 case WebSocketOpcode.binary: 3533 if(m.fin) { 3534 if(onbinarymessage) 3535 onbinarymessage(m.data); 3536 } else { 3537 continuingType = m.opcode; 3538 //continuingDataLength = 0; 3539 continuingData = null; 3540 continuingData ~= m.data; 3541 } 3542 break; 3543 case WebSocketOpcode.close: 3544 readyState_ = CLOSED; 3545 if(onclose) 3546 onclose(); 3547 3548 unregisterActiveSocket(this); 3549 break; 3550 case WebSocketOpcode.ping: 3551 pong(); 3552 break; 3553 case WebSocketOpcode.pong: 3554 // just really references it is still alive, nbd. 3555 break; 3556 default: // ignore though i could and perhaps should throw too 3557 } 3558 } 3559 3560 if(d.length) { 3561 m.data = m.data.dup(); 3562 } 3563 3564 import core.stdc.string; 3565 memmove(receiveBuffer.ptr, d.ptr, d.length); 3566 receiveBufferUsedLength = d.length; 3567 3568 return m; 3569 } 3570 3571 private void autoprocess() { 3572 // FIXME 3573 do { 3574 processOnce(); 3575 } while(lowLevelReceive()); 3576 } 3577 3578 3579 void delegate() onclose; /// 3580 void delegate() onerror; /// 3581 void delegate(in char[]) ontextmessage; /// 3582 void delegate(in ubyte[]) onbinarymessage; /// 3583 void delegate() onopen; /// 3584 3585 /++ 3586 3587 +/ 3588 /// Group: browser_api 3589 void onmessage(void delegate(in char[]) dg) { 3590 ontextmessage = dg; 3591 } 3592 3593 /// ditto 3594 void onmessage(void delegate(in ubyte[]) dg) { 3595 onbinarymessage = dg; 3596 } 3597 3598 /* } end copy/paste */ 3599 3600 /* 3601 const int bufferedAmount // amount pending 3602 const string extensions 3603 3604 const string protocol 3605 const string url 3606 */ 3607 3608 static { 3609 /++ 3610 3611 +/ 3612 void eventLoop() { 3613 3614 static SocketSet readSet; 3615 3616 if(readSet is null) 3617 readSet = new SocketSet(); 3618 3619 loopExited = false; 3620 3621 outermost: while(!loopExited) { 3622 readSet.reset(); 3623 3624 Duration timeout = 10.seconds; 3625 3626 auto now = MonoTime.currTime; 3627 bool hadAny; 3628 foreach(sock; activeSockets) { 3629 if(now >= sock.timeoutFromInactivity) { 3630 // timeout 3631 if(sock.onerror) 3632 sock.onerror(); 3633 3634 sock.socket.close(); 3635 sock.readyState_ = CLOSED; 3636 unregisterActiveSocket(sock); 3637 continue outermost; 3638 } 3639 3640 if(now >= sock.nextPing) { 3641 sock.ping(); 3642 sock.nextPing = now + sock.config.pingFrequency.msecs; 3643 } 3644 3645 auto timeo = sock.timeoutFromInactivity - now; 3646 if(timeo < timeout) 3647 timeout = timeo; 3648 3649 readSet.add(sock.socket); 3650 hadAny = true; 3651 } 3652 3653 if(!hadAny) 3654 return; 3655 3656 tryAgain: 3657 auto selectGot = Socket.select(readSet, null, null, timeout); 3658 if(selectGot == 0) { /* timeout */ 3659 // timeout 3660 continue; // it will be handled at the top of the loop 3661 } else if(selectGot == -1) { /* interrupted */ 3662 goto tryAgain; 3663 } else { 3664 foreach(sock; activeSockets) { 3665 if(readSet.isSet(sock.socket)) { 3666 sock.timeoutFromInactivity = MonoTime.currTime + sock.config.timeoutFromInactivity; 3667 if(!sock.lowLevelReceive()) { 3668 sock.readyState_ = CLOSED; 3669 unregisterActiveSocket(sock); 3670 continue outermost; 3671 } 3672 while(sock.processOnce().populated) {} 3673 selectGot--; 3674 if(selectGot <= 0) 3675 break; 3676 } 3677 } 3678 } 3679 } 3680 } 3681 3682 private bool loopExited; 3683 /++ 3684 3685 +/ 3686 void exitEventLoop() { 3687 loopExited = true; 3688 } 3689 3690 WebSocket[] activeSockets; 3691 void registerActiveSocket(WebSocket s) { 3692 activeSockets ~= s; 3693 } 3694 void unregisterActiveSocket(WebSocket s) { 3695 foreach(i, a; activeSockets) 3696 if(s is a) { 3697 activeSockets[i] = activeSockets[$-1]; 3698 activeSockets = activeSockets[0 .. $-1]; 3699 break; 3700 } 3701 } 3702 } 3703 } 3704 3705 template addToSimpledisplayEventLoop() { 3706 import arsd.simpledisplay; 3707 void addToSimpledisplayEventLoop(WebSocket ws, SimpleWindow window) { 3708 3709 void midprocess() { 3710 if(!ws.lowLevelReceive()) { 3711 ws.readyState_ = WebSocket.CLOSED; 3712 WebSocket.unregisterActiveSocket(ws); 3713 return; 3714 } 3715 while(ws.processOnce().populated) {} 3716 } 3717 3718 version(linux) { 3719 auto reader = new PosixFdReader(&midprocess, ws.socket.handle); 3720 } else version(Windows) { 3721 auto reader = new WindowsHandleReader(&midprocess, ws.socket.handle); 3722 } else static assert(0, "unsupported OS"); 3723 } 3724 } 3725 3726 3727 /* copy/paste from cgi.d */ 3728 public { 3729 enum WebSocketOpcode : ubyte { 3730 continuation = 0, 3731 text = 1, 3732 binary = 2, 3733 // 3, 4, 5, 6, 7 RESERVED 3734 close = 8, 3735 ping = 9, 3736 pong = 10, 3737 // 11,12,13,14,15 RESERVED 3738 } 3739 3740 public struct WebSocketFrame { 3741 private bool populated; 3742 bool fin; 3743 bool rsv1; 3744 bool rsv2; 3745 bool rsv3; 3746 WebSocketOpcode opcode; // 4 bits 3747 bool masked; 3748 ubyte lengthIndicator; // don't set this when building one to send 3749 ulong realLength; // don't use when sending 3750 ubyte[4] maskingKey; // don't set this when sending 3751 ubyte[] data; 3752 3753 static WebSocketFrame simpleMessage(WebSocketOpcode opcode, void[] data) { 3754 WebSocketFrame msg; 3755 msg.fin = true; 3756 msg.opcode = opcode; 3757 msg.data = cast(ubyte[]) data; 3758 3759 return msg; 3760 } 3761 3762 private void send(scope void delegate(ubyte[]) llsend) { 3763 ubyte[64] headerScratch; 3764 int headerScratchPos = 0; 3765 3766 realLength = data.length; 3767 3768 { 3769 ubyte b1; 3770 b1 |= cast(ubyte) opcode; 3771 b1 |= rsv3 ? (1 << 4) : 0; 3772 b1 |= rsv2 ? (1 << 5) : 0; 3773 b1 |= rsv1 ? (1 << 6) : 0; 3774 b1 |= fin ? (1 << 7) : 0; 3775 3776 headerScratch[0] = b1; 3777 headerScratchPos++; 3778 } 3779 3780 { 3781 headerScratchPos++; // we'll set header[1] at the end of this 3782 auto rlc = realLength; 3783 ubyte b2; 3784 b2 |= masked ? (1 << 7) : 0; 3785 3786 assert(headerScratchPos == 2); 3787 3788 if(realLength > 65535) { 3789 // use 64 bit length 3790 b2 |= 0x7f; 3791 3792 // FIXME: double check endinaness 3793 foreach(i; 0 .. 8) { 3794 headerScratch[2 + 7 - i] = rlc & 0x0ff; 3795 rlc >>>= 8; 3796 } 3797 3798 headerScratchPos += 8; 3799 } else if(realLength > 125) { 3800 // use 16 bit length 3801 b2 |= 0x7e; 3802 3803 // FIXME: double check endinaness 3804 foreach(i; 0 .. 2) { 3805 headerScratch[2 + 1 - i] = rlc & 0x0ff; 3806 rlc >>>= 8; 3807 } 3808 3809 headerScratchPos += 2; 3810 } else { 3811 // use 7 bit length 3812 b2 |= realLength & 0b_0111_1111; 3813 } 3814 3815 headerScratch[1] = b2; 3816 } 3817 3818 //assert(!masked, "masking key not properly implemented"); 3819 if(masked) { 3820 // FIXME: randomize this 3821 headerScratch[headerScratchPos .. headerScratchPos + 4] = maskingKey[]; 3822 headerScratchPos += 4; 3823 3824 // we'll just mask it in place... 3825 int keyIdx = 0; 3826 foreach(i; 0 .. data.length) { 3827 data[i] = data[i] ^ maskingKey[keyIdx]; 3828 if(keyIdx == 3) 3829 keyIdx = 0; 3830 else 3831 keyIdx++; 3832 } 3833 } 3834 3835 //writeln("SENDING ", headerScratch[0 .. headerScratchPos], data); 3836 llsend(headerScratch[0 .. headerScratchPos]); 3837 llsend(data); 3838 } 3839 3840 static WebSocketFrame read(ref ubyte[] d) { 3841 WebSocketFrame msg; 3842 3843 auto orig = d; 3844 3845 WebSocketFrame needsMoreData() { 3846 d = orig; 3847 return WebSocketFrame.init; 3848 } 3849 3850 if(d.length < 2) 3851 return needsMoreData(); 3852 3853 ubyte b = d[0]; 3854 3855 msg.populated = true; 3856 3857 msg.opcode = cast(WebSocketOpcode) (b & 0x0f); 3858 b >>= 4; 3859 msg.rsv3 = b & 0x01; 3860 b >>= 1; 3861 msg.rsv2 = b & 0x01; 3862 b >>= 1; 3863 msg.rsv1 = b & 0x01; 3864 b >>= 1; 3865 msg.fin = b & 0x01; 3866 3867 b = d[1]; 3868 msg.masked = (b & 0b1000_0000) ? true : false; 3869 msg.lengthIndicator = b & 0b0111_1111; 3870 3871 d = d[2 .. $]; 3872 3873 if(msg.lengthIndicator == 0x7e) { 3874 // 16 bit length 3875 msg.realLength = 0; 3876 3877 if(d.length < 2) return needsMoreData(); 3878 3879 foreach(i; 0 .. 2) { 3880 msg.realLength |= d[0] << ((1-i) * 8); 3881 d = d[1 .. $]; 3882 } 3883 } else if(msg.lengthIndicator == 0x7f) { 3884 // 64 bit length 3885 msg.realLength = 0; 3886 3887 if(d.length < 8) return needsMoreData(); 3888 3889 foreach(i; 0 .. 8) { 3890 msg.realLength |= d[0] << ((7-i) * 8); 3891 d = d[1 .. $]; 3892 } 3893 } else { 3894 // 7 bit length 3895 msg.realLength = msg.lengthIndicator; 3896 } 3897 3898 if(msg.masked) { 3899 3900 if(d.length < 4) return needsMoreData(); 3901 3902 msg.maskingKey = d[0 .. 4]; 3903 d = d[4 .. $]; 3904 } 3905 3906 if(msg.realLength > d.length) { 3907 return needsMoreData(); 3908 } 3909 3910 msg.data = d[0 .. cast(size_t) msg.realLength]; 3911 d = d[cast(size_t) msg.realLength .. $]; 3912 3913 return msg; 3914 } 3915 3916 void unmaskInPlace() { 3917 if(this.masked) { 3918 int keyIdx = 0; 3919 foreach(i; 0 .. this.data.length) { 3920 this.data[i] = this.data[i] ^ this.maskingKey[keyIdx]; 3921 if(keyIdx == 3) 3922 keyIdx = 0; 3923 else 3924 keyIdx++; 3925 } 3926 } 3927 } 3928 3929 char[] textData() { 3930 return cast(char[]) data; 3931 } 3932 } 3933 } 3934 3935 /+ 3936 so the url params are arguments. it knows the request 3937 internally. other params are properties on the req 3938 3939 names may have different paths... those will just add ForSomething i think. 3940 3941 auto req = api.listMergeRequests 3942 req.page = 10; 3943 3944 or 3945 req.page(1) 3946 .bar("foo") 3947 3948 req.execute(); 3949 3950 3951 everything in the response is nullable access through the 3952 dynamic object, just with property getters there. need to make 3953 it static generated tho 3954 3955 other messages may be: isPresent and getDynamic 3956 3957 3958 AND/OR what about doing it like the rails objects 3959 3960 BroadcastMessage.get(4) 3961 // various properties 3962 3963 // it lists what you updated 3964 3965 BroadcastMessage.foo().bar().put(5) 3966 +/