1 // Copyright 2013-2020, Adam D. Ruppe. 2 /++ 3 This is version 2 of my http/1.1 client implementation. 4 5 6 It has no dependencies for basic operation, but does require OpenSSL 7 libraries (or compatible) to be support HTTPS. Compile with 8 `-version=with_openssl` to enable such support. 9 10 http2.d, despite its name, does NOT implement HTTP/2.0, but this 11 shouldn't matter for 99.9% of usage, since all servers will continue 12 to support HTTP/1.1 for a very long time. 13 14 +/ 15 module arsd.http2; 16 17 // FIXME: I think I want to disable sigpipe here too. 18 19 import std.uri : encodeComponent; 20 21 debug(arsd_http2_verbose) debug=arsd_http2; 22 23 debug(arsd_http2) import std.stdio : writeln; 24 25 version=arsd_http_internal_implementation; 26 27 version(without_openssl) {} 28 else { 29 version=use_openssl; 30 version=with_openssl; 31 version(older_openssl) {} else 32 version=newer_openssl; 33 } 34 35 version(arsd_http_winhttp_implementation) { 36 pragma(lib, "winhttp") 37 import core.sys.windows.winhttp; 38 // FIXME: alter the dub package file too 39 40 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpreaddata 41 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpsendrequest 42 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpopenrequest 43 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpconnect 44 } 45 46 47 48 /++ 49 Demonstrates core functionality, using the [HttpClient], 50 [HttpRequest] (returned by [HttpClient.navigateTo|client.navigateTo]), 51 and [HttpResponse] (returned by [HttpRequest.waitForCompletion|request.waitForCompletion]). 52 53 +/ 54 unittest { 55 import arsd.http2; 56 57 void main() { 58 auto client = new HttpClient(); 59 auto request = client.navigateTo(Uri("http://dlang.org/")); 60 auto response = request.waitForCompletion(); 61 62 string returnedHtml = response.contentText; 63 } 64 } 65 66 // FIXME: multipart encoded file uploads needs implementation 67 // future: do web client api stuff 68 69 debug import std.stdio; 70 71 import std.socket; 72 import core.time; 73 74 // FIXME: check Transfer-Encoding: gzip always 75 76 version(with_openssl) { 77 //pragma(lib, "crypto"); 78 //pragma(lib, "ssl"); 79 } 80 81 /+ 82 HttpRequest httpRequest(string method, string url, ubyte[] content, string[string] content) { 83 return null; 84 } 85 +/ 86 87 /** 88 auto request = get("http://arsdnet.net/"); 89 request.send(); 90 91 auto response = get("http://arsdnet.net/").waitForCompletion(); 92 */ 93 HttpRequest get(string url) { 94 auto client = new HttpClient(); 95 auto request = client.navigateTo(Uri(url)); 96 return request; 97 } 98 99 /** 100 Do not forget to call `waitForCompletion()` on the returned object! 101 */ 102 HttpRequest post(string url, string[string] req) { 103 auto client = new HttpClient(); 104 ubyte[] bdata; 105 foreach(k, v; req) { 106 if(bdata.length) 107 bdata ~= cast(ubyte[]) "&"; 108 bdata ~= cast(ubyte[]) encodeComponent(k); 109 bdata ~= cast(ubyte[]) "="; 110 bdata ~= cast(ubyte[]) encodeComponent(v); 111 } 112 auto request = client.request(Uri(url), HttpVerb.POST, bdata, "application/x-www-form-urlencoded"); 113 return request; 114 } 115 116 /// gets the text off a url. basic operation only. 117 string getText(string url) { 118 auto request = get(url); 119 auto response = request.waitForCompletion(); 120 return cast(string) response.content; 121 } 122 123 /+ 124 ubyte[] getBinary(string url, string[string] cookies = null) { 125 auto hr = httpRequest("GET", url, null, cookies); 126 if(hr.code != 200) 127 throw new Exception(format("HTTP answered %d instead of 200 on %s", hr.code, url)); 128 return hr.content; 129 } 130 131 /** 132 Gets a textual document, ignoring headers. Throws on non-text or error. 133 */ 134 string get(string url, string[string] cookies = null) { 135 auto hr = httpRequest("GET", url, null, cookies); 136 if(hr.code != 200) 137 throw new Exception(format("HTTP answered %d instead of 200 on %s", hr.code, url)); 138 if(hr.contentType.indexOf("text/") == -1) 139 throw new Exception(hr.contentType ~ " is bad content for conversion to string"); 140 return cast(string) hr.content; 141 142 } 143 144 static import std.uri; 145 146 string post(string url, string[string] args, string[string] cookies = null) { 147 string content; 148 149 foreach(name, arg; args) { 150 if(content.length) 151 content ~= "&"; 152 content ~= std.uri.encode(name) ~ "=" ~ std.uri.encode(arg); 153 } 154 155 auto hr = httpRequest("POST", url, cast(ubyte[]) content, cookies, ["Content-Type: application/x-www-form-urlencoded"]); 156 if(hr.code != 200) 157 throw new Exception(format("HTTP answered %d instead of 200", hr.code)); 158 if(hr.contentType.indexOf("text/") == -1) 159 throw new Exception(hr.contentType ~ " is bad content for conversion to string"); 160 161 return cast(string) hr.content; 162 } 163 164 +/ 165 166 /// 167 struct HttpResponse { 168 int code; /// 169 string codeText; /// 170 171 string httpVersion; /// 172 173 string statusLine; /// 174 175 string contentType; /// The content type header 176 string location; /// The location header 177 178 /// the charset out of content type, if present. `null` if not. 179 string contentTypeCharset() { 180 auto idx = contentType.indexOf("charset="); 181 if(idx == -1) 182 return null; 183 auto c = contentType[idx + "charset=".length .. $].strip; 184 if(c.length) 185 return c; 186 return null; 187 } 188 189 string[string] cookies; /// Names and values of cookies set in the response. 190 191 string[] headers; /// Array of all headers returned. 192 string[string] headersHash; /// 193 194 ubyte[] content; /// The raw content returned in the response body. 195 string contentText; /// [content], but casted to string (for convenience) 196 197 /++ 198 returns `new Document(this.contentText)`. Requires [arsd.dom]. 199 +/ 200 auto contentDom()() { 201 import arsd.dom; 202 return new Document(this.contentText); 203 204 } 205 206 /++ 207 returns `var.fromJson(this.contentText)`. Requires [arsd.jsvar]. 208 +/ 209 auto contentJson()() { 210 import arsd.jsvar; 211 return var.fromJson(this.contentText); 212 } 213 214 HttpRequestParameters requestParameters; /// 215 216 LinkHeader[] linksStored; 217 bool linksLazilyParsed; 218 219 /// Returns links header sorted by "rel" attribute. 220 /// It returns a new array on each call. 221 LinkHeader[string] linksHash() { 222 auto links = this.links(); 223 LinkHeader[string] ret; 224 foreach(link; links) 225 ret[link.rel] = link; 226 return ret; 227 } 228 229 /// Returns the Link header, parsed. 230 LinkHeader[] links() { 231 if(linksLazilyParsed) 232 return linksStored; 233 linksLazilyParsed = true; 234 LinkHeader[] ret; 235 236 auto hdrPtr = "Link" in headersHash; 237 if(hdrPtr is null) 238 return ret; 239 240 auto header = *hdrPtr; 241 242 LinkHeader current; 243 244 while(header.length) { 245 char ch = header[0]; 246 247 if(ch == '<') { 248 // read url 249 header = header[1 .. $]; 250 size_t idx; 251 while(idx < header.length && header[idx] != '>') 252 idx++; 253 current.url = header[0 .. idx]; 254 header = header[idx .. $]; 255 } else if(ch == ';') { 256 // read attribute 257 header = header[1 .. $]; 258 header = header.stripLeft; 259 260 size_t idx; 261 while(idx < header.length && header[idx] != '=') 262 idx++; 263 264 string name = header[0 .. idx]; 265 header = header[idx + 1 .. $]; 266 267 string value; 268 269 if(header.length && header[0] == '"') { 270 // quoted value 271 header = header[1 .. $]; 272 idx = 0; 273 while(idx < header.length && header[idx] != '\"') 274 idx++; 275 value = header[0 .. idx]; 276 header = header[idx .. $]; 277 278 } else if(header.length) { 279 // unquoted value 280 idx = 0; 281 while(idx < header.length && header[idx] != ',' && header[idx] != ' ' && header[idx] != ';') 282 idx++; 283 284 value = header[0 .. idx]; 285 header = header[idx .. $].stripLeft; 286 } 287 288 name = name.toLower; 289 if(name == "rel") 290 current.rel = value; 291 else 292 current.attributes[name] = value; 293 294 } else if(ch == ',') { 295 // start another 296 ret ~= current; 297 current = LinkHeader.init; 298 } else if(ch == ' ' || ch == '\n' || ch == '\r' || ch == '\t') { 299 // ignore 300 } 301 302 header = header[1 .. $]; 303 } 304 305 ret ~= current; 306 307 linksStored = ret; 308 309 return ret; 310 } 311 } 312 313 /// 314 struct LinkHeader { 315 string url; /// 316 string rel; /// 317 string[string] attributes; /// like title, rev, media, whatever attributes 318 } 319 320 import std.string; 321 static import std.algorithm; 322 import std.conv; 323 import std.range; 324 325 326 private AddressFamily family(string unixSocketPath) { 327 if(unixSocketPath.length) 328 return AddressFamily.UNIX; 329 else // FIXME: what about ipv6? 330 return AddressFamily.INET; 331 } 332 333 version(Windows) 334 private class UnixAddress : Address { 335 this(string) { 336 throw new Exception("No unix address support on this system in lib yet :("); 337 } 338 override sockaddr* name() { assert(0); } 339 override const(sockaddr)* name() const { assert(0); } 340 override int nameLen() const { assert(0); } 341 } 342 343 344 // Copy pasta from cgi.d, then stripped down. unix path thing added tho 345 /// 346 struct Uri { 347 alias toString this; // blargh idk a url really is a string, but should it be implicit? 348 349 // scheme//userinfo@host:port/path?query#fragment 350 351 string scheme; /// e.g. "http" in "http://example.com/" 352 string userinfo; /// the username (and possibly a password) in the uri 353 string host; /// the domain name 354 int port; /// port number, if given. Will be zero if a port was not explicitly given 355 string path; /// e.g. "/folder/file.html" in "http://example.com/folder/file.html" 356 string query; /// the stuff after the ? in a uri 357 string fragment; /// the stuff after the # in a uri. 358 359 /// Breaks down a uri string to its components 360 this(string uri) { 361 reparse(uri); 362 } 363 364 private string unixSocketPath = null; 365 /// Indicates it should be accessed through a unix socket instead of regular tcp. Returns new version without modifying this object. 366 Uri viaUnixSocket(string path) const { 367 Uri copy = this; 368 copy.unixSocketPath = path; 369 return copy; 370 } 371 372 /// Goes through a unix socket in the abstract namespace (linux only). Returns new version without modifying this object. 373 version(linux) 374 Uri viaAbstractSocket(string path) const { 375 Uri copy = this; 376 copy.unixSocketPath = "\0" ~ path; 377 return copy; 378 } 379 380 private void reparse(string uri) { 381 // from RFC 3986 382 // the ctRegex triples the compile time and makes ugly errors for no real benefit 383 // it was a nice experiment but just not worth it. 384 // enum ctr = ctRegex!r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?"; 385 /* 386 Captures: 387 0 = whole url 388 1 = scheme, with : 389 2 = scheme, no : 390 3 = authority, with // 391 4 = authority, no // 392 5 = path 393 6 = query string, with ? 394 7 = query string, no ? 395 8 = anchor, with # 396 9 = anchor, no # 397 */ 398 // Yikes, even regular, non-CT regex is also unacceptably slow to compile. 1.9s on my computer! 399 // instead, I will DIY and cut that down to 0.6s on the same computer. 400 /* 401 402 Note that authority is 403 user:password@domain:port 404 where the user:password@ part is optional, and the :port is optional. 405 406 Regex translation: 407 408 Scheme cannot have :, /, ?, or # in it, and must have one or more chars and end in a :. It is optional, but must be first. 409 Authority must start with //, but cannot have any other /, ?, or # in it. It is optional. 410 Path cannot have any ? or # in it. It is optional. 411 Query must start with ? and must not have # in it. It is optional. 412 Anchor must start with # and can have anything else in it to end of string. It is optional. 413 */ 414 415 this = Uri.init; // reset all state 416 417 // empty uri = nothing special 418 if(uri.length == 0) { 419 return; 420 } 421 422 size_t idx; 423 424 scheme_loop: foreach(char c; uri[idx .. $]) { 425 switch(c) { 426 case ':': 427 case '/': 428 case '?': 429 case '#': 430 break scheme_loop; 431 default: 432 } 433 idx++; 434 } 435 436 if(idx == 0 && uri[idx] == ':') { 437 // this is actually a path! we skip way ahead 438 goto path_loop; 439 } 440 441 if(idx == uri.length) { 442 // the whole thing is a path, apparently 443 path = uri; 444 return; 445 } 446 447 if(idx > 0 && uri[idx] == ':') { 448 scheme = uri[0 .. idx]; 449 idx++; 450 } else { 451 // we need to rewind; it found a / but no :, so the whole thing is prolly a path... 452 idx = 0; 453 } 454 455 if(idx + 2 < uri.length && uri[idx .. idx + 2] == "//") { 456 // we have an authority.... 457 idx += 2; 458 459 auto authority_start = idx; 460 authority_loop: foreach(char c; uri[idx .. $]) { 461 switch(c) { 462 case '/': 463 case '?': 464 case '#': 465 break authority_loop; 466 default: 467 } 468 idx++; 469 } 470 471 auto authority = uri[authority_start .. idx]; 472 473 auto idx2 = authority.indexOf("@"); 474 if(idx2 != -1) { 475 userinfo = authority[0 .. idx2]; 476 authority = authority[idx2 + 1 .. $]; 477 } 478 479 idx2 = authority.indexOf(":"); 480 if(idx2 == -1) { 481 port = 0; // 0 means not specified; we should use the default for the scheme 482 host = authority; 483 } else { 484 host = authority[0 .. idx2]; 485 port = to!int(authority[idx2 + 1 .. $]); 486 } 487 } 488 489 path_loop: 490 auto path_start = idx; 491 492 foreach(char c; uri[idx .. $]) { 493 if(c == '?' || c == '#') 494 break; 495 idx++; 496 } 497 498 path = uri[path_start .. idx]; 499 500 if(idx == uri.length) 501 return; // nothing more to examine... 502 503 if(uri[idx] == '?') { 504 idx++; 505 auto query_start = idx; 506 foreach(char c; uri[idx .. $]) { 507 if(c == '#') 508 break; 509 idx++; 510 } 511 query = uri[query_start .. idx]; 512 } 513 514 if(idx < uri.length && uri[idx] == '#') { 515 idx++; 516 fragment = uri[idx .. $]; 517 } 518 519 // uriInvalidated = false; 520 } 521 522 private string rebuildUri() const { 523 string ret; 524 if(scheme.length) 525 ret ~= scheme ~ ":"; 526 if(userinfo.length || host.length) 527 ret ~= "//"; 528 if(userinfo.length) 529 ret ~= userinfo ~ "@"; 530 if(host.length) 531 ret ~= host; 532 if(port) 533 ret ~= ":" ~ to!string(port); 534 535 ret ~= path; 536 537 if(query.length) 538 ret ~= "?" ~ query; 539 540 if(fragment.length) 541 ret ~= "#" ~ fragment; 542 543 // uri = ret; 544 // uriInvalidated = false; 545 return ret; 546 } 547 548 /// Converts the broken down parts back into a complete string 549 string toString() const { 550 // if(uriInvalidated) 551 return rebuildUri(); 552 } 553 554 /// Returns a new absolute Uri given a base. It treats this one as 555 /// relative where possible, but absolute if not. (If protocol, domain, or 556 /// other info is not set, the new one inherits it from the base.) 557 /// 558 /// Browsers use a function like this to figure out links in html. 559 Uri basedOn(in Uri baseUrl) const { 560 Uri n = this; // copies 561 // n.uriInvalidated = true; // make sure we regenerate... 562 563 // userinfo is not inherited... is this wrong? 564 565 // if anything is given in the existing url, we don't use the base anymore. 566 if(n.scheme.empty) { 567 n.scheme = baseUrl.scheme; 568 if(n.host.empty) { 569 n.host = baseUrl.host; 570 if(n.port == 0) { 571 n.port = baseUrl.port; 572 if(n.path.length > 0 && n.path[0] != '/') { 573 auto b = baseUrl.path[0 .. baseUrl.path.lastIndexOf("/") + 1]; 574 if(b.length == 0) 575 b = "/"; 576 n.path = b ~ n.path; 577 } else if(n.path.length == 0) { 578 n.path = baseUrl.path; 579 } 580 } 581 } 582 } 583 584 n.removeDots(); 585 586 // if still basically talking to the same thing, we should inherit the unix path 587 // too since basically the unix path is saying for this service, always use this override. 588 if(n.host == baseUrl.host && n.scheme == baseUrl.scheme && n.port == baseUrl.port) 589 n.unixSocketPath = baseUrl.unixSocketPath; 590 591 return n; 592 } 593 594 void removeDots() { 595 auto parts = this.path.split("/"); 596 string[] toKeep; 597 foreach(part; parts) { 598 if(part == ".") { 599 continue; 600 } else if(part == "..") { 601 toKeep = toKeep[0 .. $-1]; 602 continue; 603 } else { 604 toKeep ~= part; 605 } 606 } 607 608 this.path = toKeep.join("/"); 609 } 610 611 } 612 613 /* 614 void main(string args[]) { 615 write(post("http://arsdnet.net/bugs.php", ["test" : "hey", "again" : "what"])); 616 } 617 */ 618 619 /// 620 struct BasicAuth { 621 string username; /// 622 string password; /// 623 } 624 625 /** 626 When you send something, it creates a request 627 and sends it asynchronously. The request object 628 629 auto request = new HttpRequest(); 630 // set any properties here 631 632 // synchronous usage 633 auto reply = request.perform(); 634 635 // async usage, type 1: 636 request.send(); 637 request2.send(); 638 639 // wait until the first one is done, with the second one still in-flight 640 auto response = request.waitForCompletion(); 641 642 643 // async usage, type 2: 644 request.onDataReceived = (HttpRequest hr) { 645 if(hr.state == HttpRequest.State.complete) { 646 // use hr.responseData 647 } 648 }; 649 request.send(); // send, using the callback 650 651 // before terminating, be sure you wait for your requests to finish! 652 653 request.waitForCompletion(); 654 655 */ 656 class HttpRequest { 657 658 /// Automatically follow a redirection? 659 bool followLocation = false; 660 661 this() { 662 } 663 664 /// 665 this(Uri where, HttpVerb method) { 666 populateFromInfo(where, method); 667 } 668 669 /// Final url after any redirections 670 string finalUrl; 671 672 void populateFromInfo(Uri where, HttpVerb method) { 673 auto parts = where; 674 finalUrl = where.toString(); 675 requestParameters.method = method; 676 requestParameters.unixSocketPath = where.unixSocketPath; 677 requestParameters.host = parts.host; 678 requestParameters.port = cast(ushort) parts.port; 679 requestParameters.ssl = parts.scheme == "https"; 680 if(parts.port == 0) 681 requestParameters.port = requestParameters.ssl ? 443 : 80; 682 requestParameters.uri = parts.path.length ? parts.path : "/"; 683 if(parts.query.length) { 684 requestParameters.uri ~= "?"; 685 requestParameters.uri ~= parts.query; 686 } 687 } 688 689 ~this() { 690 } 691 692 ubyte[] sendBuffer; 693 694 HttpResponse responseData; 695 private HttpClient parentClient; 696 697 size_t bodyBytesSent; 698 size_t bodyBytesReceived; 699 700 State state_; 701 State state() { return state_; } 702 State state(State s) { 703 assert(state_ != State.complete); 704 return state_ = s; 705 } 706 /// Called when data is received. Check the state to see what data is available. 707 void delegate(HttpRequest) onDataReceived; 708 709 enum State { 710 /// The request has not yet been sent 711 unsent, 712 713 /// The send() method has been called, but no data is 714 /// sent on the socket yet because the connection is busy. 715 pendingAvailableConnection, 716 717 /// The headers are being sent now 718 sendingHeaders, 719 720 /// The body is being sent now 721 sendingBody, 722 723 /// The request has been sent but we haven't received any response yet 724 waitingForResponse, 725 726 /// We have received some data and are currently receiving headers 727 readingHeaders, 728 729 /// All headers are available but we're still waiting on the body 730 readingBody, 731 732 /// The request is complete. 733 complete, 734 735 /// The request is aborted, either by the abort() method, or as a result of the server disconnecting 736 aborted 737 } 738 739 /// Sends now and waits for the request to finish, returning the response. 740 HttpResponse perform() { 741 send(); 742 return waitForCompletion(); 743 } 744 745 /// Sends the request asynchronously. 746 void send() { 747 sendPrivate(true); 748 } 749 750 private void sendPrivate(bool advance) { 751 if(state != State.unsent && state != State.aborted) 752 return; // already sent 753 string headers; 754 755 headers ~= to!string(requestParameters.method) ~ " "~requestParameters.uri; 756 if(requestParameters.useHttp11) 757 headers ~= " HTTP/1.1\r\n"; 758 else 759 headers ~= " HTTP/1.0\r\n"; 760 headers ~= "Host: "~requestParameters.host~"\r\n"; 761 if(requestParameters.userAgent.length) 762 headers ~= "User-Agent: "~requestParameters.userAgent~"\r\n"; 763 if(requestParameters.contentType.length) 764 headers ~= "Content-Type: "~requestParameters.contentType~"\r\n"; 765 if(requestParameters.authorization.length) 766 headers ~= "Authorization: "~requestParameters.authorization~"\r\n"; 767 if(requestParameters.bodyData.length) 768 headers ~= "Content-Length: "~to!string(requestParameters.bodyData.length)~"\r\n"; 769 if(requestParameters.acceptGzip) 770 headers ~= "Accept-Encoding: gzip\r\n"; 771 if(requestParameters.keepAlive) 772 headers ~= "Connection: keep-alive\r\n"; 773 774 foreach(header; requestParameters.headers) 775 headers ~= header ~ "\r\n"; 776 777 headers ~= "\r\n"; 778 779 sendBuffer = cast(ubyte[]) headers ~ requestParameters.bodyData; 780 781 // import std.stdio; writeln("******* ", sendBuffer); 782 783 responseData = HttpResponse.init; 784 responseData.requestParameters = requestParameters; 785 bodyBytesSent = 0; 786 bodyBytesReceived = 0; 787 state = State.pendingAvailableConnection; 788 789 bool alreadyPending = false; 790 foreach(req; pending) 791 if(req is this) { 792 alreadyPending = true; 793 break; 794 } 795 if(!alreadyPending) { 796 pending ~= this; 797 } 798 799 if(advance) 800 HttpRequest.advanceConnections(); 801 } 802 803 804 /// Waits for the request to finish or timeout, whichever comes first. 805 HttpResponse waitForCompletion() { 806 while(state != State.aborted && state != State.complete) { 807 if(state == State.unsent) 808 send(); 809 if(auto err = HttpRequest.advanceConnections()) 810 throw new Exception("waitForCompletion got err " ~ to!string(err)); 811 } 812 813 return responseData; 814 } 815 816 /// Aborts this request. 817 void abort() { 818 this.state = State.aborted; 819 // FIXME 820 } 821 822 HttpRequestParameters requestParameters; /// 823 824 version(arsd_http_winhttp_implementation) { 825 public static void resetInternals() { 826 827 } 828 829 static assert(0, "implementation not finished"); 830 } 831 832 833 version(arsd_http_internal_implementation) { 834 private static { 835 // we manage the actual connections. When a request is made on a particular 836 // host, we try to reuse connections. We may open more than one connection per 837 // host to do parallel requests. 838 // 839 // The key is the *domain name* and the port. Multiple domains on the same address will have separate connections. 840 Socket[][string] socketsPerHost; 841 842 void loseSocket(string host, ushort port, bool ssl, Socket s) { 843 import std.string; 844 auto key = format("http%s://%s:%s", ssl ? "s" : "", host, port); 845 846 if(auto list = key in socketsPerHost) { 847 for(int a = 0; a < (*list).length; a++) { 848 if((*list)[a] is s) { 849 850 for(int b = a; b < (*list).length - 1; b++) 851 (*list)[b] = (*list)[b+1]; 852 (*list) = (*list)[0 .. $-1]; 853 break; 854 } 855 } 856 } 857 } 858 859 Socket getOpenSocketOnHost(string host, ushort port, bool ssl, string unixSocketPath) { 860 861 Socket openNewConnection() { 862 Socket socket; 863 if(ssl) { 864 version(with_openssl) 865 socket = new SslClientSocket(family(unixSocketPath), SocketType.STREAM, host); 866 else 867 throw new Exception("SSL not compiled in"); 868 } else 869 socket = new Socket(family(unixSocketPath), SocketType.STREAM); 870 871 if(unixSocketPath) { 872 import std.stdio; writeln(cast(ubyte[]) unixSocketPath); 873 socket.connect(new UnixAddress(unixSocketPath)); 874 } else { 875 // FIXME: i should prolly do ipv6 if available too. 876 socket.connect(new InternetAddress(host, port)); 877 } 878 879 debug(arsd_http2) writeln("opening to ", host, ":", port, " ", cast(void*) socket); 880 assert(socket.handle() !is socket_t.init); 881 return socket; 882 } 883 884 import std.string; 885 auto key = format("http%s://%s:%s", ssl ? "s" : "", host, port); 886 887 if(auto hostListing = key in socketsPerHost) { 888 // try to find an available socket that is already open 889 foreach(socket; *hostListing) { 890 if(socket !in activeRequestOnSocket) { 891 // let's see if it has closed since we last tried 892 // e.g. a server timeout or something. If so, we need 893 // to lose this one and immediately open a new one. 894 static SocketSet readSet = null; 895 if(readSet is null) 896 readSet = new SocketSet(); 897 readSet.reset(); 898 assert(socket.handle() !is socket_t.init, socket is null ? "null" : socket.toString()); 899 readSet.add(socket); 900 auto got = Socket.select(readSet, null, null, 5.msecs /* timeout */); 901 if(got > 0) { 902 // we can read something off this... but there aren't 903 // any active requests. Assume it is EOF and open a new one 904 905 socket.close(); 906 loseSocket(host, port, ssl, socket); 907 goto openNew; 908 } 909 return socket; 910 } 911 } 912 913 // if not too many already open, go ahead and do a new one 914 if((*hostListing).length < 6) { 915 auto socket = openNewConnection(); 916 (*hostListing) ~= socket; 917 return socket; 918 } else 919 return null; // too many, you'll have to wait 920 } 921 922 openNew: 923 924 auto socket = openNewConnection(); 925 socketsPerHost[key] ~= socket; 926 return socket; 927 } 928 929 // only one request can be active on a given socket (at least HTTP < 2.0) so this is that 930 HttpRequest[Socket] activeRequestOnSocket; 931 HttpRequest[] pending; // and these are the requests that are waiting 932 933 SocketSet readSet; 934 SocketSet writeSet; 935 936 937 int advanceConnections() { 938 if(readSet is null) 939 readSet = new SocketSet(); 940 if(writeSet is null) 941 writeSet = new SocketSet(); 942 943 ubyte[2048] buffer; 944 945 HttpRequest[16] removeFromPending; 946 size_t removeFromPendingCount = 0; 947 948 // are there pending requests? let's try to send them 949 foreach(idx, pc; pending) { 950 if(removeFromPendingCount == removeFromPending.length) 951 break; 952 953 if(pc.state == HttpRequest.State.aborted) { 954 removeFromPending[removeFromPendingCount++] = pc; 955 continue; 956 } 957 958 auto socket = getOpenSocketOnHost(pc.requestParameters.host, pc.requestParameters.port, pc.requestParameters.ssl, pc.requestParameters.unixSocketPath); 959 960 if(socket !is null) { 961 activeRequestOnSocket[socket] = pc; 962 assert(pc.sendBuffer.length); 963 pc.state = State.sendingHeaders; 964 965 removeFromPending[removeFromPendingCount++] = pc; 966 } 967 } 968 969 import std.algorithm : remove; 970 foreach(rp; removeFromPending[0 .. removeFromPendingCount]) 971 pending = pending.remove!((a) => a is rp)(); 972 973 readSet.reset(); 974 writeSet.reset(); 975 976 bool hadOne = false; 977 978 // active requests need to be read or written to 979 foreach(sock, request; activeRequestOnSocket) { 980 // check the other sockets just for EOF, if they close, take them out of our list, 981 // we'll reopen if needed upon request. 982 readSet.add(sock); 983 hadOne = true; 984 if(request.state == State.sendingHeaders || request.state == State.sendingBody) { 985 writeSet.add(sock); 986 hadOne = true; 987 } 988 } 989 990 if(!hadOne) 991 return 1; // automatic timeout, nothing to do 992 993 tryAgain: 994 auto selectGot = Socket.select(readSet, writeSet, null, 10.seconds /* timeout */); 995 if(selectGot == 0) { /* timeout */ 996 // timeout 997 return 1; 998 } else if(selectGot == -1) { /* interrupted */ 999 /* 1000 version(Posix) { 1001 import core.stdc.errno; 1002 if(errno != EINTR) 1003 throw new Exception("select error: " ~ to!string(errno)); 1004 } 1005 */ 1006 goto tryAgain; 1007 } else { /* ready */ 1008 Socket[16] inactive; 1009 int inactiveCount = 0; 1010 foreach(sock, request; activeRequestOnSocket) { 1011 if(readSet.isSet(sock)) { 1012 keep_going: 1013 auto got = sock.receive(buffer); 1014 debug(arsd_http2_verbose) writeln("====PACKET ",got,"=====",cast(string)buffer[0 .. got],"===/PACKET==="); 1015 if(got < 0) { 1016 throw new Exception("receive error"); 1017 } else if(got == 0) { 1018 // remote side disconnected 1019 debug(arsd_http2) writeln("remote disconnect"); 1020 if(request.state != State.complete) 1021 request.state = State.aborted; 1022 inactive[inactiveCount++] = sock; 1023 sock.close(); 1024 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1025 } else { 1026 // data available 1027 auto stillAlive = request.handleIncomingData(buffer[0 .. got]); 1028 1029 if(!stillAlive || request.state == HttpRequest.State.complete || request.state == HttpRequest.State.aborted) { 1030 //import std.stdio; writeln(cast(void*) sock, " ", stillAlive, " ", request.state); 1031 inactive[inactiveCount++] = sock; 1032 continue; 1033 // reuse the socket for another pending request, if we can 1034 } 1035 } 1036 1037 if(request.onDataReceived) 1038 request.onDataReceived(request); 1039 1040 version(with_openssl) 1041 if(auto s = cast(SslClientSocket) sock) { 1042 // select doesn't handle the case with stuff 1043 // left in the ssl buffer so i'm checking it separately 1044 if(s.dataPending()) { 1045 goto keep_going; 1046 } 1047 } 1048 } 1049 1050 if(request.state == State.sendingHeaders || request.state == State.sendingBody) 1051 if(writeSet.isSet(sock)) { 1052 assert(request.sendBuffer.length); 1053 auto sent = sock.send(request.sendBuffer); 1054 debug(arsd_http2_verbose) writeln(cast(void*) sock, "<send>", cast(string) request.sendBuffer, "</send>"); 1055 if(sent <= 0) 1056 throw new Exception("send error " ~ lastSocketError); 1057 request.sendBuffer = request.sendBuffer[sent .. $]; 1058 if(request.sendBuffer.length == 0) { 1059 request.state = State.waitingForResponse; 1060 } 1061 } 1062 } 1063 1064 foreach(s; inactive[0 .. inactiveCount]) { 1065 debug(arsd_http2) writeln("removing socket from active list ", cast(void*) s); 1066 activeRequestOnSocket.remove(s); 1067 } 1068 } 1069 1070 // we've completed a request, are there any more pending connection? if so, send them now 1071 1072 return 0; 1073 } 1074 } 1075 1076 public static void resetInternals() { 1077 socketsPerHost = null; 1078 activeRequestOnSocket = null; 1079 pending = null; 1080 1081 } 1082 1083 struct HeaderReadingState { 1084 bool justSawLf; 1085 bool justSawCr; 1086 bool atStartOfLine = true; 1087 bool readingLineContinuation; 1088 } 1089 HeaderReadingState headerReadingState; 1090 1091 struct BodyReadingState { 1092 bool isGzipped; 1093 bool isDeflated; 1094 1095 bool isChunked; 1096 int chunkedState; 1097 1098 // used for the chunk size if it is chunked 1099 int contentLengthRemaining; 1100 } 1101 BodyReadingState bodyReadingState; 1102 1103 bool closeSocketWhenComplete; 1104 1105 import std.zlib; 1106 UnCompress uncompress; 1107 1108 const(ubyte)[] leftoverDataFromLastTime; 1109 1110 bool handleIncomingData(scope const ubyte[] dataIn) { 1111 bool stillAlive = true; 1112 debug(arsd_http2) writeln("handleIncomingData, state: ", state); 1113 if(state == State.waitingForResponse) { 1114 state = State.readingHeaders; 1115 headerReadingState = HeaderReadingState.init; 1116 bodyReadingState = BodyReadingState.init; 1117 } 1118 1119 const(ubyte)[] data; 1120 if(leftoverDataFromLastTime.length) 1121 data = leftoverDataFromLastTime ~ dataIn[]; 1122 else 1123 data = dataIn[]; 1124 1125 if(state == State.readingHeaders) { 1126 void parseLastHeader() { 1127 assert(responseData.headers.length); 1128 if(responseData.headers.length == 1) { 1129 responseData.statusLine = responseData.headers[0]; 1130 import std.algorithm; 1131 auto parts = responseData.statusLine.splitter(" "); 1132 responseData.httpVersion = parts.front; 1133 parts.popFront(); 1134 responseData.code = to!int(parts.front()); 1135 parts.popFront(); 1136 responseData.codeText = ""; 1137 while(!parts.empty) { 1138 // FIXME: this sucks! 1139 responseData.codeText ~= parts.front(); 1140 parts.popFront(); 1141 if(!parts.empty) 1142 responseData.codeText ~= " "; 1143 } 1144 } else { 1145 // parse the new header 1146 auto header = responseData.headers[$-1]; 1147 1148 auto colon = header.indexOf(":"); 1149 if(colon == -1) 1150 return; 1151 auto name = header[0 .. colon]; 1152 if(colon + 1 == header.length || colon + 2 == header.length) // assuming a space there 1153 return; // empty header, idk 1154 assert(colon + 2 < header.length, header); 1155 auto value = header[colon + 2 .. $]; // skipping the colon itself and the following space 1156 1157 switch(name) { 1158 case "Connection": 1159 case "connection": 1160 if(value == "close") 1161 closeSocketWhenComplete = true; 1162 break; 1163 case "Content-Type": 1164 case "content-type": 1165 responseData.contentType = value; 1166 break; 1167 case "Location": 1168 case "location": 1169 responseData.location = value; 1170 break; 1171 case "Content-Length": 1172 case "content-length": 1173 bodyReadingState.contentLengthRemaining = to!int(value); 1174 break; 1175 case "Transfer-Encoding": 1176 case "transfer-encoding": 1177 // note that if it is gzipped, it zips first, then chunks the compressed stream. 1178 // so we should always dechunk first, then feed into the decompressor 1179 if(value.strip == "chunked") 1180 bodyReadingState.isChunked = true; 1181 else throw new Exception("Unknown Transfer-Encoding: " ~ value); 1182 break; 1183 case "Content-Encoding": 1184 case "content-encoding": 1185 if(value == "gzip") { 1186 bodyReadingState.isGzipped = true; 1187 uncompress = new UnCompress(); 1188 } else if(value == "deflate") { 1189 bodyReadingState.isDeflated = true; 1190 uncompress = new UnCompress(); 1191 } else throw new Exception("Unknown Content-Encoding: " ~ value); 1192 break; 1193 case "Set-Cookie": 1194 case "set-cookie": 1195 // FIXME handle 1196 break; 1197 default: 1198 // ignore 1199 } 1200 1201 responseData.headersHash[name] = value; 1202 } 1203 } 1204 1205 size_t position = 0; 1206 for(position = 0; position < dataIn.length; position++) { 1207 if(headerReadingState.readingLineContinuation) { 1208 if(data[position] == ' ' || data[position] == '\t') 1209 continue; 1210 headerReadingState.readingLineContinuation = false; 1211 } 1212 1213 if(headerReadingState.atStartOfLine) { 1214 headerReadingState.atStartOfLine = false; 1215 if(data[position] == '\r' || data[position] == '\n') { 1216 // done with headers 1217 if(data[position] == '\r' && (position + 1) < data.length && data[position + 1] == '\n') 1218 position++; 1219 state = State.readingBody; 1220 position++; // skip the newline 1221 break; 1222 } else if(data[position] == ' ' || data[position] == '\t') { 1223 // line continuation, ignore all whitespace and collapse it into a space 1224 headerReadingState.readingLineContinuation = true; 1225 responseData.headers[$-1] ~= ' '; 1226 } else { 1227 // new header 1228 if(responseData.headers.length) 1229 parseLastHeader(); 1230 responseData.headers ~= ""; 1231 } 1232 } 1233 1234 if(data[position] == '\r') { 1235 headerReadingState.justSawCr = true; 1236 continue; 1237 } else 1238 headerReadingState.justSawCr = false; 1239 1240 if(data[position] == '\n') { 1241 headerReadingState.justSawLf = true; 1242 headerReadingState.atStartOfLine = true; 1243 continue; 1244 } else 1245 headerReadingState.justSawLf = false; 1246 1247 responseData.headers[$-1] ~= data[position]; 1248 } 1249 1250 parseLastHeader(); 1251 data = data[position .. $]; 1252 } 1253 1254 if(state == State.readingBody) { 1255 if(bodyReadingState.isChunked) { 1256 // read the hex length, stopping at a \r\n, ignoring everything between the new line but after the first non-valid hex character 1257 // read binary data of that length. it is our content 1258 // repeat until a zero sized chunk 1259 // then read footers as headers. 1260 1261 start_over: 1262 for(int a = 0; a < data.length; a++) { 1263 final switch(bodyReadingState.chunkedState) { 1264 case 0: // reading hex 1265 char c = data[a]; 1266 if((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) { 1267 // just keep reading 1268 } else { 1269 int power = 1; 1270 bodyReadingState.contentLengthRemaining = 0; 1271 assert(a != 0, cast(string) data); 1272 for(int b = a-1; b >= 0; b--) { 1273 char cc = data[b]; 1274 if(cc >= 'a' && cc <= 'z') 1275 cc -= 0x20; 1276 int val = 0; 1277 if(cc >= '0' && cc <= '9') 1278 val = cc - '0'; 1279 else 1280 val = cc - 'A' + 10; 1281 1282 assert(val >= 0 && val <= 15, to!string(val)); 1283 bodyReadingState.contentLengthRemaining += power * val; 1284 power *= 16; 1285 } 1286 debug(arsd_http2_verbose) writeln("Chunk length: ", bodyReadingState.contentLengthRemaining); 1287 bodyReadingState.chunkedState = 1; 1288 data = data[a + 1 .. $]; 1289 goto start_over; 1290 } 1291 break; 1292 case 1: // reading until end of line 1293 char c = data[a]; 1294 if(c == '\n') { 1295 if(bodyReadingState.contentLengthRemaining == 0) 1296 bodyReadingState.chunkedState = 5; 1297 else 1298 bodyReadingState.chunkedState = 2; 1299 } 1300 data = data[a + 1 .. $]; 1301 goto start_over; 1302 case 2: // reading data 1303 auto can = a + bodyReadingState.contentLengthRemaining; 1304 if(can > data.length) 1305 can = cast(int) data.length; 1306 1307 auto newData = data[a .. can]; 1308 data = data[can .. $]; 1309 1310 //if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) 1311 // responseData.content ~= cast(ubyte[]) uncompress.uncompress(data[a .. can]); 1312 //else 1313 responseData.content ~= newData; 1314 1315 bodyReadingState.contentLengthRemaining -= newData.length; 1316 debug(arsd_http2_verbose) writeln("clr: ", bodyReadingState.contentLengthRemaining, " " , a, " ", can); 1317 assert(bodyReadingState.contentLengthRemaining >= 0); 1318 if(bodyReadingState.contentLengthRemaining == 0) { 1319 bodyReadingState.chunkedState = 3; 1320 } else { 1321 // will continue grabbing more 1322 } 1323 goto start_over; 1324 case 3: // reading 13/10 1325 assert(data[a] == 13); 1326 bodyReadingState.chunkedState++; 1327 data = data[a + 1 .. $]; 1328 goto start_over; 1329 case 4: // reading 10 at end of packet 1330 assert(data[a] == 10); 1331 data = data[a + 1 .. $]; 1332 bodyReadingState.chunkedState = 0; 1333 goto start_over; 1334 case 5: // reading footers 1335 //goto done; // FIXME 1336 state = State.complete; 1337 1338 bodyReadingState.chunkedState = 0; 1339 1340 while(data[a] != 10) 1341 a++; 1342 data = data[a + 1 .. $]; 1343 1344 if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) { 1345 auto n = uncompress.uncompress(responseData.content); 1346 n ~= uncompress.flush(); 1347 responseData.content = cast(ubyte[]) n; 1348 } 1349 1350 // responseData.content ~= cast(ubyte[]) uncompress.flush(); 1351 1352 responseData.contentText = cast(string) responseData.content; 1353 1354 goto done; 1355 } 1356 } 1357 1358 done: 1359 // FIXME 1360 //if(closeSocketWhenComplete) 1361 //socket.close(); 1362 } else { 1363 //if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) 1364 // responseData.content ~= cast(ubyte[]) uncompress.uncompress(data); 1365 //else 1366 responseData.content ~= data; 1367 //assert(data.length <= bodyReadingState.contentLengthRemaining, format("%d <= %d\n%s", data.length, bodyReadingState.contentLengthRemaining, cast(string)data)); 1368 int use = cast(int) data.length; 1369 if(use > bodyReadingState.contentLengthRemaining) 1370 use = bodyReadingState.contentLengthRemaining; 1371 bodyReadingState.contentLengthRemaining -= use; 1372 data = data[use .. $]; 1373 if(bodyReadingState.contentLengthRemaining == 0) { 1374 if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) { 1375 auto n = uncompress.uncompress(responseData.content); 1376 n ~= uncompress.flush(); 1377 responseData.content = cast(ubyte[]) n; 1378 //responseData.content ~= cast(ubyte[]) uncompress.flush(); 1379 } 1380 if(followLocation && responseData.location.length) { 1381 static bool first = true; 1382 //version(DigitalMars) if(!first) asm { int 3; } 1383 populateFromInfo(Uri(responseData.location), HttpVerb.GET); 1384 import std.stdio; writeln("redirected to ", responseData.location); 1385 first = false; 1386 responseData = HttpResponse.init; 1387 headerReadingState = HeaderReadingState.init; 1388 bodyReadingState = BodyReadingState.init; 1389 state = State.unsent; 1390 stillAlive = false; 1391 sendPrivate(false); 1392 } else { 1393 state = State.complete; 1394 responseData.contentText = cast(string) responseData.content; 1395 // FIXME 1396 //if(closeSocketWhenComplete) 1397 //socket.close(); 1398 } 1399 } 1400 } 1401 } 1402 1403 if(data.length) 1404 leftoverDataFromLastTime = data.dup; 1405 else 1406 leftoverDataFromLastTime = null; 1407 1408 return stillAlive; 1409 } 1410 1411 } 1412 } 1413 1414 /// 1415 struct HttpRequestParameters { 1416 // Duration timeout; 1417 1418 // debugging 1419 bool useHttp11 = true; /// 1420 bool acceptGzip = true; /// 1421 bool keepAlive = true; /// 1422 1423 // the request itself 1424 HttpVerb method; /// 1425 string host; /// 1426 ushort port; /// 1427 string uri; /// 1428 1429 bool ssl; /// 1430 1431 string userAgent; /// 1432 string authorization; /// 1433 1434 string[string] cookies; /// 1435 1436 string[] headers; /// do not duplicate host, content-length, content-type, or any others that have a specific property 1437 1438 string contentType; /// 1439 ubyte[] bodyData; /// 1440 1441 string unixSocketPath; 1442 } 1443 1444 interface IHttpClient { 1445 1446 } 1447 1448 /// 1449 enum HttpVerb { 1450 /// 1451 GET, 1452 /// 1453 HEAD, 1454 /// 1455 POST, 1456 /// 1457 PUT, 1458 /// 1459 DELETE, 1460 /// 1461 OPTIONS, 1462 /// 1463 TRACE, 1464 /// 1465 CONNECT, 1466 /// 1467 PATCH, 1468 /// 1469 MERGE 1470 } 1471 1472 /** 1473 Usage: 1474 1475 auto client = new HttpClient("localhost", 80); 1476 // relative links work based on the current url 1477 client.get("foo/bar"); 1478 client.get("baz"); // gets foo/baz 1479 1480 auto request = client.get("rofl"); 1481 auto response = request.waitForCompletion(); 1482 */ 1483 1484 /// HttpClient keeps cookies, location, and some other state to reuse connections, when possible, like a web browser. 1485 class HttpClient { 1486 /* Protocol restrictions, useful to disable when debugging servers */ 1487 bool useHttp11 = true; /// 1488 bool acceptGzip = true; /// 1489 bool keepAlive = true; /// 1490 1491 /// 1492 @property Uri location() { 1493 return currentUrl; 1494 } 1495 1496 /// High level function that works similarly to entering a url 1497 /// into a browser. 1498 /// 1499 /// Follows locations, updates the current url. 1500 HttpRequest navigateTo(Uri where, HttpVerb method = HttpVerb.GET) { 1501 currentUrl = where.basedOn(currentUrl); 1502 currentDomain = where.host; 1503 auto request = new HttpRequest(currentUrl, method); 1504 1505 request.followLocation = true; 1506 1507 request.requestParameters.userAgent = userAgent; 1508 request.requestParameters.authorization = authorization; 1509 1510 request.requestParameters.useHttp11 = this.useHttp11; 1511 request.requestParameters.acceptGzip = this.acceptGzip; 1512 request.requestParameters.keepAlive = this.keepAlive; 1513 1514 return request; 1515 } 1516 1517 /++ 1518 Creates a request without updating the current url state 1519 (but will still save cookies btw) 1520 1521 +/ 1522 HttpRequest request(Uri uri, HttpVerb method = HttpVerb.GET, ubyte[] bodyData = null, string contentType = null) { 1523 auto request = new HttpRequest(uri, method); 1524 1525 request.requestParameters.userAgent = userAgent; 1526 request.requestParameters.authorization = authorization; 1527 1528 request.requestParameters.useHttp11 = this.useHttp11; 1529 request.requestParameters.acceptGzip = this.acceptGzip; 1530 request.requestParameters.keepAlive = this.keepAlive; 1531 1532 request.requestParameters.bodyData = bodyData; 1533 request.requestParameters.contentType = contentType; 1534 1535 return request; 1536 1537 } 1538 1539 /// ditto 1540 HttpRequest request(Uri uri, FormData fd, HttpVerb method = HttpVerb.POST) { 1541 return request(uri, method, fd.toBytes, fd.contentType); 1542 } 1543 1544 1545 private Uri currentUrl; 1546 private string currentDomain; 1547 1548 this(ICache cache = null) { 1549 1550 } 1551 1552 // FIXME: add proxy 1553 // FIXME: some kind of caching 1554 1555 /// 1556 void setCookie(string name, string value, string domain = null) { 1557 if(domain == null) 1558 domain = currentDomain; 1559 1560 cookies[domain][name] = value; 1561 } 1562 1563 /// 1564 void clearCookies(string domain = null) { 1565 if(domain is null) 1566 cookies = null; 1567 else 1568 cookies[domain] = null; 1569 } 1570 1571 // If you set these, they will be pre-filled on all requests made with this client 1572 string userAgent = "D arsd.html2"; /// 1573 string authorization; /// 1574 1575 /* inter-request state */ 1576 string[string][string] cookies; 1577 } 1578 1579 interface ICache { 1580 HttpResponse* getCachedResponse(HttpRequestParameters request); 1581 } 1582 1583 // / Provides caching behavior similar to a real web browser 1584 class HttpCache : ICache { 1585 HttpResponse* getCachedResponse(HttpRequestParameters request) { 1586 return null; 1587 } 1588 } 1589 1590 // / Gives simple maximum age caching, ignoring the actual http headers 1591 class SimpleCache : ICache { 1592 HttpResponse* getCachedResponse(HttpRequestParameters request) { 1593 return null; 1594 } 1595 } 1596 1597 /// 1598 struct HttpCookie { 1599 string name; /// 1600 string value; /// 1601 string domain; /// 1602 string path; /// 1603 //SysTime expirationDate; /// 1604 bool secure; /// 1605 bool httpOnly; /// 1606 } 1607 1608 // FIXME: websocket 1609 1610 version(testing) 1611 void main() { 1612 import std.stdio; 1613 auto client = new HttpClient(); 1614 auto request = client.navigateTo(Uri("http://localhost/chunked.php")); 1615 request.send(); 1616 auto request2 = client.navigateTo(Uri("http://dlang.org/")); 1617 request2.send(); 1618 1619 { 1620 auto response = request2.waitForCompletion(); 1621 //write(cast(string) response.content); 1622 } 1623 1624 auto response = request.waitForCompletion(); 1625 write(cast(string) response.content); 1626 1627 writeln(HttpRequest.socketsPerHost); 1628 } 1629 1630 1631 // From sslsocket.d, but this is the maintained version! 1632 version(use_openssl) { 1633 alias SslClientSocket = OpenSslSocket; 1634 1635 // macros in the original C 1636 SSL_METHOD* SSLv23_client_method() { 1637 if(ossllib.SSLv23_client_method) 1638 return ossllib.SSLv23_client_method(); 1639 else 1640 return ossllib.TLS_client_method(); 1641 } 1642 1643 struct SSL {} 1644 struct SSL_CTX {} 1645 struct SSL_METHOD {} 1646 enum SSL_VERIFY_NONE = 0; 1647 1648 struct ossllib { 1649 __gshared static extern(C) { 1650 /* these are only on older openssl versions { */ 1651 int function() SSL_library_init; 1652 void function() SSL_load_error_strings; 1653 SSL_METHOD* function() SSLv23_client_method; 1654 /* } */ 1655 1656 void function(ulong, void*) OPENSSL_init_ssl; 1657 1658 SSL_CTX* function(const SSL_METHOD*) SSL_CTX_new; 1659 SSL* function(SSL_CTX*) SSL_new; 1660 int function(SSL*, int) SSL_set_fd; 1661 int function(SSL*) SSL_connect; 1662 int function(SSL*, const void*, int) SSL_write; 1663 int function(SSL*, void*, int) SSL_read; 1664 @trusted nothrow @nogc int function(SSL*) SSL_shutdown; 1665 void function(SSL*) SSL_free; 1666 void function(SSL_CTX*) SSL_CTX_free; 1667 1668 int function(const SSL*) SSL_pending; 1669 1670 void function(SSL*, int, void*) SSL_set_verify; 1671 1672 void function(SSL*, int, c_long, void*) SSL_ctrl; 1673 1674 SSL_METHOD* function() SSLv3_client_method; 1675 SSL_METHOD* function() TLS_client_method; 1676 1677 } 1678 } 1679 1680 import core.stdc.config; 1681 1682 struct eallib { 1683 __gshared static extern(C) { 1684 /* these are only on older openssl versions { */ 1685 void function() OpenSSL_add_all_ciphers; 1686 void function() OpenSSL_add_all_digests; 1687 /* } */ 1688 1689 void function(ulong, void*) OPENSSL_init_crypto; 1690 1691 void function(FILE*) ERR_print_errors_fp; 1692 } 1693 } 1694 1695 1696 SSL_CTX* SSL_CTX_new(const SSL_METHOD* a) { 1697 if(ossllib.SSL_CTX_new) 1698 return ossllib.SSL_CTX_new(a); 1699 else throw new Exception("SSL_CTX_new not loaded"); 1700 } 1701 SSL* SSL_new(SSL_CTX* a) { 1702 if(ossllib.SSL_new) 1703 return ossllib.SSL_new(a); 1704 else throw new Exception("SSL_new not loaded"); 1705 } 1706 int SSL_set_fd(SSL* a, int b) { 1707 if(ossllib.SSL_set_fd) 1708 return ossllib.SSL_set_fd(a, b); 1709 else throw new Exception("SSL_set_fd not loaded"); 1710 } 1711 int SSL_connect(SSL* a) { 1712 if(ossllib.SSL_connect) 1713 return ossllib.SSL_connect(a); 1714 else throw new Exception("SSL_connect not loaded"); 1715 } 1716 int SSL_write(SSL* a, const void* b, int c) { 1717 if(ossllib.SSL_write) 1718 return ossllib.SSL_write(a, b, c); 1719 else throw new Exception("SSL_write not loaded"); 1720 } 1721 int SSL_read(SSL* a, void* b, int c) { 1722 if(ossllib.SSL_read) 1723 return ossllib.SSL_read(a, b, c); 1724 else throw new Exception("SSL_read not loaded"); 1725 } 1726 @trusted nothrow @nogc int SSL_shutdown(SSL* a) { 1727 if(ossllib.SSL_shutdown) 1728 return ossllib.SSL_shutdown(a); 1729 assert(0); 1730 } 1731 void SSL_free(SSL* a) { 1732 if(ossllib.SSL_free) 1733 return ossllib.SSL_free(a); 1734 else throw new Exception("SSL_free not loaded"); 1735 } 1736 void SSL_CTX_free(SSL_CTX* a) { 1737 if(ossllib.SSL_CTX_free) 1738 return ossllib.SSL_CTX_free(a); 1739 else throw new Exception("SSL_CTX_free not loaded"); 1740 } 1741 1742 int SSL_pending(const SSL* a) { 1743 if(ossllib.SSL_pending) 1744 return ossllib.SSL_pending(a); 1745 else throw new Exception("SSL_pending not loaded"); 1746 } 1747 void SSL_set_verify(SSL* a, int b, void* c) { 1748 if(ossllib.SSL_set_verify) 1749 return ossllib.SSL_set_verify(a, b, c); 1750 else throw new Exception("SSL_set_verify not loaded"); 1751 } 1752 void SSL_set_tlsext_host_name(SSL* a, const char* b) { 1753 if(ossllib.SSL_ctrl) 1754 return ossllib.SSL_ctrl(a, 55 /*SSL_CTRL_SET_TLSEXT_HOSTNAME*/, 0 /*TLSEXT_NAMETYPE_host_name*/, cast(void*) b); 1755 else throw new Exception("SSL_set_tlsext_host_name not loaded"); 1756 } 1757 1758 SSL_METHOD* SSLv3_client_method() { 1759 if(ossllib.SSLv3_client_method) 1760 return ossllib.SSLv3_client_method(); 1761 else throw new Exception("SSLv3_client_method not loaded"); 1762 } 1763 SSL_METHOD* TLS_client_method() { 1764 if(ossllib.TLS_client_method) 1765 return ossllib.TLS_client_method(); 1766 else throw new Exception("TLS_client_method not loaded"); 1767 } 1768 void ERR_print_errors_fp(FILE* a) { 1769 if(eallib.ERR_print_errors_fp) 1770 return eallib.ERR_print_errors_fp(a); 1771 else throw new Exception("ERR_print_errors_fp not loaded"); 1772 } 1773 1774 1775 private __gshared void* ossllib_handle; 1776 version(Windows) 1777 private __gshared void* oeaylib_handle; 1778 else 1779 alias oeaylib_handle = ossllib_handle; 1780 version(Posix) 1781 private import core.sys.posix.dlfcn; 1782 else version(Windows) 1783 private import core.sys.windows.windows; 1784 1785 import core.stdc.stdio; 1786 1787 shared static this() { 1788 version(Posix) 1789 ossllib_handle = dlopen("libssl.so", RTLD_NOW); 1790 else version(Windows) { 1791 ossllib_handle = LoadLibraryW("libssl32.dll"w.ptr); 1792 oeaylib_handle = LoadLibraryW("libeay32.dll"w.ptr); 1793 } 1794 1795 if(ossllib_handle is null) 1796 throw new Exception("ssl fail open"); 1797 1798 foreach(memberName; __traits(allMembers, ossllib)) { 1799 alias t = typeof(__traits(getMember, ossllib, memberName)); 1800 version(Posix) 1801 __traits(getMember, ossllib, memberName) = cast(t) dlsym(ossllib_handle, memberName); 1802 else version(Windows) { 1803 __traits(getMember, ossllib, memberName) = cast(t) GetProcAddress(ossllib_handle, memberName); 1804 } 1805 } 1806 1807 foreach(memberName; __traits(allMembers, eallib)) { 1808 alias t = typeof(__traits(getMember, eallib, memberName)); 1809 version(Posix) 1810 __traits(getMember, eallib, memberName) = cast(t) dlsym(oeaylib_handle, memberName); 1811 else version(Windows) { 1812 __traits(getMember, eallib, memberName) = cast(t) GetProcAddress(oeaylib_handle, memberName); 1813 } 1814 } 1815 1816 1817 if(ossllib.SSL_library_init) 1818 ossllib.SSL_library_init(); 1819 else if(ossllib.OPENSSL_init_ssl) 1820 ossllib.OPENSSL_init_ssl(0, null); 1821 else throw new Exception("couldn't init openssl"); 1822 1823 if(eallib.OpenSSL_add_all_ciphers) { 1824 eallib.OpenSSL_add_all_ciphers(); 1825 if(eallib.OpenSSL_add_all_digests is null) 1826 throw new Exception("no add digests"); 1827 eallib.OpenSSL_add_all_digests(); 1828 } else if(eallib.OPENSSL_init_crypto) 1829 eallib.OPENSSL_init_crypto(0 /*OPENSSL_INIT_ADD_ALL_CIPHERS and ALL_DIGESTS together*/, null); 1830 else throw new Exception("couldn't init crypto openssl"); 1831 1832 if(ossllib.SSL_load_error_strings) 1833 ossllib.SSL_load_error_strings(); 1834 else if(ossllib.OPENSSL_init_ssl) 1835 ossllib.OPENSSL_init_ssl(0x00200000L, null); 1836 else throw new Exception("couldn't load openssl errors"); 1837 } 1838 1839 /+ 1840 // I'm just gonna let the OS clean this up on process termination because otherwise SSL_free 1841 // might have trouble being run from the GC after this module is unloaded. 1842 shared static ~this() { 1843 if(ossllib_handle) { 1844 version(Windows) { 1845 FreeLibrary(oeaylib_handle); 1846 FreeLibrary(ossllib_handle); 1847 } else version(Posix) 1848 dlclose(ossllib_handle); 1849 ossllib_handle = null; 1850 } 1851 ossllib.tupleof = ossllib.tupleof.init; 1852 } 1853 +/ 1854 1855 //pragma(lib, "crypto"); 1856 //pragma(lib, "ssl"); 1857 1858 class OpenSslSocket : Socket { 1859 private SSL* ssl; 1860 private SSL_CTX* ctx; 1861 private void initSsl(bool verifyPeer, string hostname) { 1862 ctx = SSL_CTX_new(SSLv23_client_method()); 1863 assert(ctx !is null); 1864 1865 ssl = SSL_new(ctx); 1866 1867 if(hostname.length) 1868 SSL_set_tlsext_host_name(ssl, toStringz(hostname)); 1869 1870 if(!verifyPeer) 1871 SSL_set_verify(ssl, SSL_VERIFY_NONE, null); 1872 SSL_set_fd(ssl, cast(int) this.handle); // on win64 it is necessary to truncate, but the value is never large anyway see http://openssl.6102.n7.nabble.com/Sockets-windows-64-bit-td36169.html 1873 } 1874 1875 bool dataPending() { 1876 return SSL_pending(ssl) > 0; 1877 } 1878 1879 @trusted 1880 override void connect(Address to) { 1881 super.connect(to); 1882 if(SSL_connect(ssl) == -1) { 1883 ERR_print_errors_fp(core.stdc.stdio.stderr); 1884 int i; 1885 //printf("wtf\n"); 1886 //scanf("%d\n", i); 1887 throw new Exception("ssl connect"); 1888 } 1889 } 1890 1891 @trusted 1892 override ptrdiff_t send(scope const(void)[] buf, SocketFlags flags) { 1893 //import std.stdio;writeln(cast(string) buf); 1894 auto retval = SSL_write(ssl, buf.ptr, cast(uint) buf.length); 1895 if(retval == -1) { 1896 ERR_print_errors_fp(core.stdc.stdio.stderr); 1897 int i; 1898 //printf("wtf\n"); 1899 //scanf("%d\n", i); 1900 throw new Exception("ssl send"); 1901 } 1902 return retval; 1903 1904 } 1905 override ptrdiff_t send(scope const(void)[] buf) { 1906 return send(buf, SocketFlags.NONE); 1907 } 1908 @trusted 1909 override ptrdiff_t receive(scope void[] buf, SocketFlags flags) { 1910 auto retval = SSL_read(ssl, buf.ptr, cast(int)buf.length); 1911 if(retval == -1) { 1912 ERR_print_errors_fp(core.stdc.stdio.stderr); 1913 int i; 1914 //printf("wtf\n"); 1915 //scanf("%d\n", i); 1916 throw new Exception("ssl send"); 1917 } 1918 return retval; 1919 } 1920 override ptrdiff_t receive(scope void[] buf) { 1921 return receive(buf, SocketFlags.NONE); 1922 } 1923 1924 this(AddressFamily af, SocketType type = SocketType.STREAM, string hostname = null, bool verifyPeer = true) { 1925 super(af, type); 1926 initSsl(verifyPeer, hostname); 1927 } 1928 1929 override void close() { 1930 if(ssl) SSL_shutdown(ssl); 1931 super.close(); 1932 } 1933 1934 this(socket_t sock, AddressFamily af, string hostname) { 1935 super(sock, af); 1936 initSsl(true, hostname); 1937 } 1938 1939 ~this() { 1940 SSL_free(ssl); 1941 SSL_CTX_free(ctx); 1942 ssl = null; 1943 } 1944 } 1945 } 1946 1947 /++ 1948 An experimental component for working with REST apis. Note that it 1949 is a zero-argument template, so to create one, use `new HttpApiClient!()(args..)` 1950 or you will get "HttpApiClient is used as a type" compile errors. 1951 1952 This will probably not work for you yet, and I might change it significantly. 1953 1954 Requires [arsd.jsvar]. 1955 1956 1957 Here's a snippet to create a pull request on GitHub to Phobos: 1958 1959 --- 1960 auto github = new HttpApiClient!()("https://api.github.com/", "your personal api token here"); 1961 1962 // create the arguments object 1963 // see: https://developer.github.com/v3/pulls/#create-a-pull-request 1964 var args = var.emptyObject; 1965 args.title = "My Pull Request"; 1966 args.head = "yourusername:" ~ branchName; 1967 args.base = "master"; 1968 // note it is ["body"] instead of .body because `body` is a D keyword 1969 args["body"] = "My cool PR is opened by the API!"; 1970 args.maintainer_can_modify = true; 1971 1972 // this translates to `repos/dlang/phobos/pulls` and sends a POST request, 1973 // containing `args` as json, then immediately grabs the json result and extracts 1974 // the value `html_url` from it. `prUrl` is typed `var`, from arsd.jsvar. 1975 auto prUrl = github.rest.repos.dlang.phobos.pulls.POST(args).result.html_url; 1976 1977 writeln("Created: ", prUrl); 1978 --- 1979 1980 Why use this instead of just building the URL? Well, of course you can! This just makes 1981 it a bit more convenient than string concatenation and manages a few headers for you. 1982 1983 Subtypes could potentially add static type checks too. 1984 +/ 1985 class HttpApiClient() { 1986 import arsd.jsvar; 1987 1988 HttpClient httpClient; 1989 1990 alias HttpApiClientType = typeof(this); 1991 1992 string urlBase; 1993 string oauth2Token; 1994 string submittedContentType; 1995 1996 /++ 1997 Params: 1998 1999 urlBase = The base url for the api. Tends to be something like `https://api.example.com/v2/` or similar. 2000 oauth2Token = the authorization token for the service. You'll have to get it from somewhere else. 2001 submittedContentType = the content-type of POST, PUT, etc. bodies. 2002 +/ 2003 this(string urlBase, string oauth2Token, string submittedContentType = "application/json") { 2004 httpClient = new HttpClient(); 2005 2006 assert(urlBase[0] == 'h'); 2007 assert(urlBase[$-1] == '/'); 2008 2009 this.urlBase = urlBase; 2010 this.oauth2Token = oauth2Token; 2011 this.submittedContentType = submittedContentType; 2012 } 2013 2014 /// 2015 static struct HttpRequestWrapper { 2016 HttpApiClientType apiClient; /// 2017 HttpRequest request; /// 2018 HttpResponse _response; 2019 2020 /// 2021 this(HttpApiClientType apiClient, HttpRequest request) { 2022 this.apiClient = apiClient; 2023 this.request = request; 2024 } 2025 2026 /// Returns the full [HttpResponse] object so you can inspect the headers 2027 @property HttpResponse response() { 2028 if(_response is HttpResponse.init) 2029 _response = request.waitForCompletion(); 2030 return _response; 2031 } 2032 2033 /++ 2034 Returns the parsed JSON from the body of the response. 2035 2036 Throws on non-2xx responses. 2037 +/ 2038 var result() { 2039 return apiClient.throwOnError(response); 2040 } 2041 2042 alias request this; 2043 } 2044 2045 /// 2046 HttpRequestWrapper request(string uri, HttpVerb requestMethod = HttpVerb.GET, ubyte[] bodyBytes = null) { 2047 if(uri[0] == '/') 2048 uri = uri[1 .. $]; 2049 2050 auto u = Uri(uri).basedOn(Uri(urlBase)); 2051 2052 auto req = httpClient.navigateTo(u, requestMethod); 2053 2054 if(oauth2Token.length) 2055 req.requestParameters.headers ~= "Authorization: Bearer " ~ oauth2Token; 2056 req.requestParameters.contentType = submittedContentType; 2057 req.requestParameters.bodyData = bodyBytes; 2058 2059 return HttpRequestWrapper(this, req); 2060 } 2061 2062 /// 2063 var throwOnError(HttpResponse res) { 2064 if(res.code < 200 || res.code >= 300) 2065 throw new Exception(res.codeText ~ " " ~ res.contentText); 2066 2067 var response = var.fromJson(res.contentText); 2068 if(response.errors) { 2069 throw new Exception(response.errors.toJson()); 2070 } 2071 2072 return response; 2073 } 2074 2075 /// 2076 @property RestBuilder rest() { 2077 return RestBuilder(this, null, null); 2078 } 2079 2080 // hipchat.rest.room["Tech Team"].history 2081 // gives: "/room/Tech%20Team/history" 2082 // 2083 // hipchat.rest.room["Tech Team"].history("page", "12) 2084 /// 2085 static struct RestBuilder { 2086 HttpApiClientType apiClient; 2087 string[] pathParts; 2088 string[2][] queryParts; 2089 this(HttpApiClientType apiClient, string[] pathParts, string[2][] queryParts) { 2090 this.apiClient = apiClient; 2091 this.pathParts = pathParts; 2092 this.queryParts = queryParts; 2093 } 2094 2095 RestBuilder _SELF() { 2096 return this; 2097 } 2098 2099 /// The args are so you can call opCall on the returned 2100 /// object, despite @property being broken af in D. 2101 RestBuilder opDispatch(string str, T)(string n, T v) { 2102 return RestBuilder(apiClient, pathParts ~ str, queryParts ~ [n, to!string(v)]); 2103 } 2104 2105 /// 2106 RestBuilder opDispatch(string str)() { 2107 return RestBuilder(apiClient, pathParts ~ str, queryParts); 2108 } 2109 2110 2111 /// 2112 RestBuilder opIndex(string str) { 2113 return RestBuilder(apiClient, pathParts ~ str, queryParts); 2114 } 2115 /// 2116 RestBuilder opIndex(var str) { 2117 return RestBuilder(apiClient, pathParts ~ str.get!string, queryParts); 2118 } 2119 /// 2120 RestBuilder opIndex(int i) { 2121 return RestBuilder(apiClient, pathParts ~ to!string(i), queryParts); 2122 } 2123 2124 /// 2125 RestBuilder opCall(T)(string name, T value) { 2126 return RestBuilder(apiClient, pathParts, queryParts ~ [name, to!string(value)]); 2127 } 2128 2129 /// 2130 string toUri() { 2131 import std.uri; 2132 string result; 2133 foreach(idx, part; pathParts) { 2134 if(idx) 2135 result ~= "/"; 2136 result ~= encodeComponent(part); 2137 } 2138 result ~= "?"; 2139 foreach(idx, part; queryParts) { 2140 if(idx) 2141 result ~= "&"; 2142 result ~= encodeComponent(part[0]); 2143 result ~= "="; 2144 result ~= encodeComponent(part[1]); 2145 } 2146 2147 return result; 2148 } 2149 2150 /// 2151 final HttpRequestWrapper GET() { return _EXECUTE(HttpVerb.GET, this.toUri(), ToBytesResult.init); } 2152 /// ditto 2153 final HttpRequestWrapper DELETE() { return _EXECUTE(HttpVerb.DELETE, this.toUri(), ToBytesResult.init); } 2154 2155 // need to be able to send: JSON, urlencoded, multipart/form-data, and raw stuff. 2156 /// ditto 2157 final HttpRequestWrapper POST(T...)(T t) { return _EXECUTE(HttpVerb.POST, this.toUri(), toBytes(t)); } 2158 /// ditto 2159 final HttpRequestWrapper PATCH(T...)(T t) { return _EXECUTE(HttpVerb.PATCH, this.toUri(), toBytes(t)); } 2160 /// ditto 2161 final HttpRequestWrapper PUT(T...)(T t) { return _EXECUTE(HttpVerb.PUT, this.toUri(), toBytes(t)); } 2162 2163 struct ToBytesResult { 2164 ubyte[] bytes; 2165 string contentType; 2166 } 2167 2168 private ToBytesResult toBytes(T...)(T t) { 2169 import std.conv : to; 2170 static if(T.length == 0) 2171 return ToBytesResult(null, null); 2172 else static if(T.length == 1 && is(T[0] == var)) 2173 return ToBytesResult(cast(ubyte[]) t[0].toJson(), "application/json"); // json data 2174 else static if(T.length == 1 && (is(T[0] == string) || is(T[0] == ubyte[]))) 2175 return ToBytesResult(cast(ubyte[]) t[0], null); // raw data 2176 else static if(T.length == 1 && is(T[0] : FormData)) 2177 return ToBytesResult(t[0].toBytes, t[0].contentType); 2178 else static if(T.length > 1 && T.length % 2 == 0 && is(T[0] == string)) { 2179 // string -> value pairs for a POST request 2180 string answer; 2181 foreach(idx, val; t) { 2182 static if(idx % 2 == 0) { 2183 if(answer.length) 2184 answer ~= "&"; 2185 answer ~= encodeComponent(val); // it had better be a string! lol 2186 answer ~= "="; 2187 } else { 2188 answer ~= encodeComponent(to!string(val)); 2189 } 2190 } 2191 2192 return ToBytesResult(cast(ubyte[]) answer, "application/x-www-form-urlencoded"); 2193 } 2194 else 2195 static assert(0); // FIXME 2196 2197 } 2198 2199 HttpRequestWrapper _EXECUTE(HttpVerb verb, string uri, ubyte[] bodyBytes) { 2200 return apiClient.request(uri, verb, bodyBytes); 2201 } 2202 2203 HttpRequestWrapper _EXECUTE(HttpVerb verb, string uri, ToBytesResult tbr) { 2204 auto r = apiClient.request(uri, verb, tbr.bytes); 2205 if(tbr.contentType !is null) 2206 r.requestParameters.contentType = tbr.contentType; 2207 return r; 2208 } 2209 } 2210 } 2211 2212 2213 // see also: arsd.cgi.encodeVariables 2214 /// Creates a multipart/form-data object that is suitable for file uploads and other kinds of POST 2215 class FormData { 2216 struct MimePart { 2217 string name; 2218 const(void)[] data; 2219 string contentType; 2220 string filename; 2221 } 2222 2223 MimePart[] parts; 2224 2225 /// 2226 void append(string key, in void[] value, string contentType = null, string filename = null) { 2227 parts ~= MimePart(key, value, contentType, filename); 2228 } 2229 2230 private string boundary = "0016e64be86203dd36047610926a"; // FIXME 2231 2232 string contentType() { 2233 return "multipart/form-data; boundary=" ~ boundary; 2234 } 2235 2236 /// 2237 ubyte[] toBytes() { 2238 string data; 2239 2240 foreach(part; parts) { 2241 data ~= "--" ~ boundary ~ "\r\n"; 2242 data ~= "Content-Disposition: form-data; name=\""~part.name~"\""; 2243 if(part.filename !is null) 2244 data ~= "; filename=\""~part.filename~"\""; 2245 data ~= "\r\n"; 2246 if(part.contentType !is null) 2247 data ~= "Content-Type: " ~ part.contentType ~ "\r\n"; 2248 data ~= "\r\n"; 2249 2250 data ~= cast(string) part.data; 2251 2252 data ~= "\r\n"; 2253 } 2254 2255 data ~= "--" ~ boundary ~ "--\r\n"; 2256 2257 return cast(ubyte[]) data; 2258 } 2259 } 2260 2261 private bool bicmp(in ubyte[] item, in char[] search) { 2262 if(item.length != search.length) return false; 2263 2264 foreach(i; 0 .. item.length) { 2265 ubyte a = item[i]; 2266 ubyte b = search[i]; 2267 if(a >= 'A' && a <= 'Z') 2268 a += 32; 2269 //if(b >= 'A' && b <= 'Z') 2270 //b += 32; 2271 if(a != b) 2272 return false; 2273 } 2274 2275 return true; 2276 } 2277 2278 /++ 2279 WebSocket client, based on the browser api, though also with other api options. 2280 2281 --- 2282 auto ws = new WebSocket(URI("ws://....")); 2283 2284 ws.onmessage = (in char[] msg) { 2285 ws.send("a reply"); 2286 }; 2287 2288 ws.connect(); 2289 2290 WebSocket.eventLoop(); 2291 --- 2292 2293 Symbol_groups: 2294 foundational = 2295 Used with all API styles. 2296 2297 browser_api = 2298 API based on the standard in the browser. 2299 2300 event_loop_integration = 2301 Integrating with external event loops is done through static functions. You should 2302 call these BEFORE doing anything else with the WebSocket module or class. 2303 2304 $(PITFALL NOT IMPLEMENTED) 2305 --- 2306 WebSocket.setEventLoopProxy(arsd.simpledisplay.EventLoop.proxy.tupleof); 2307 // or something like that. it is not implemented yet. 2308 --- 2309 $(PITFALL NOT IMPLEMENTED) 2310 2311 blocking_api = 2312 The blocking API is best used when you only need basic functionality with a single connection. 2313 2314 --- 2315 WebSocketFrame msg; 2316 do { 2317 // FIXME good demo 2318 } while(msg); 2319 --- 2320 2321 Or to check for blocks before calling: 2322 2323 --- 2324 try_to_process_more: 2325 while(ws.isMessageBuffered()) { 2326 auto msg = ws.waitForNextMessage(); 2327 // process msg 2328 } 2329 if(ws.isDataPending()) { 2330 ws.lowLevelReceive(); 2331 goto try_to_process_more; 2332 } else { 2333 // nothing ready, you can do other things 2334 // or at least sleep a while before trying 2335 // to process more. 2336 if(ws.readyState == WebSocket.OPEN) { 2337 Thread.sleep(1.seconds); 2338 goto try_to_process_more; 2339 } 2340 } 2341 --- 2342 2343 +/ 2344 class WebSocket { 2345 private Uri uri; 2346 private string[string] cookies; 2347 private string origin; 2348 2349 private string host; 2350 private ushort port; 2351 private bool ssl; 2352 2353 /++ 2354 wss://echo.websocket.org 2355 +/ 2356 /// Group: foundational 2357 this(Uri uri, Config config = Config.init) 2358 //in (uri.scheme == "ws" || uri.scheme == "wss") 2359 in { assert(uri.scheme == "ws" || uri.scheme == "wss"); } do 2360 { 2361 this.uri = uri; 2362 this.config = config; 2363 2364 this.receiveBuffer = new ubyte[](config.initialReceiveBufferSize); 2365 2366 host = uri.host; 2367 ssl = uri.scheme == "wss"; 2368 port = cast(ushort) (uri.port ? uri.port : ssl ? 443 : 80); 2369 2370 if(ssl) { 2371 version(with_openssl) 2372 socket = new SslClientSocket(family(uri.unixSocketPath), SocketType.STREAM, host); 2373 else 2374 throw new Exception("SSL not compiled in"); 2375 } else 2376 socket = new Socket(family(uri.unixSocketPath), SocketType.STREAM); 2377 2378 } 2379 2380 /++ 2381 2382 +/ 2383 /// Group: foundational 2384 void connect() { 2385 if(uri.unixSocketPath) 2386 socket.connect(new UnixAddress(uri.unixSocketPath)); 2387 else 2388 socket.connect(new InternetAddress(host, port)); // FIXME: ipv6 support... 2389 // FIXME: websocket handshake could and really should be async too. 2390 2391 auto uri = this.uri.path.length ? this.uri.path : "/"; 2392 if(this.uri.query.length) { 2393 uri ~= "?"; 2394 uri ~= this.uri.query; 2395 } 2396 2397 // the headers really shouldn't be bigger than this, at least 2398 // the chunks i need to process 2399 ubyte[4096] buffer; 2400 size_t pos; 2401 2402 void append(in char[][] items...) { 2403 foreach(what; items) { 2404 buffer[pos .. pos + what.length] = cast(ubyte[]) what[]; 2405 pos += what.length; 2406 } 2407 } 2408 2409 append("GET ", uri, " HTTP/1.1\r\n"); 2410 append("Host: ", this.uri.host, "\r\n"); 2411 2412 append("Upgrade: websocket\r\n"); 2413 append("Connection: Upgrade\r\n"); 2414 append("Sec-WebSocket-Version: 13\r\n"); 2415 2416 // FIXME: randomize this 2417 append("Sec-WebSocket-Key: x3JEHMbDL1EzLkh9GBhXDw==\r\n"); 2418 2419 if(config.protocol.length) 2420 append("Sec-WebSocket-Protocol: ", config.protocol, "\r\n"); 2421 if(config.origin.length) 2422 append("Origin: ", origin, "\r\n"); 2423 2424 append("\r\n"); 2425 2426 auto remaining = buffer[0 .. pos]; 2427 //import std.stdio; writeln(host, " " , port, " ", cast(string) remaining); 2428 while(remaining.length) { 2429 auto r = socket.send(remaining); 2430 if(r < 0) 2431 throw new Exception(lastSocketError()); 2432 if(r == 0) 2433 throw new Exception("unexpected connection termination"); 2434 remaining = remaining[r .. $]; 2435 } 2436 2437 // the response shouldn't be especially large at this point, just 2438 // headers for the most part. gonna try to get it in the stack buffer. 2439 // then copy stuff after headers, if any, to the frame buffer. 2440 ubyte[] used; 2441 2442 void more() { 2443 auto r = socket.receive(buffer[used.length .. $]); 2444 2445 if(r < 0) 2446 throw new Exception(lastSocketError()); 2447 if(r == 0) 2448 throw new Exception("unexpected connection termination"); 2449 //import std.stdio;writef("%s", cast(string) buffer[used.length .. used.length + r]); 2450 2451 used = buffer[0 .. used.length + r]; 2452 } 2453 2454 more(); 2455 2456 import std.algorithm; 2457 if(!used.startsWith(cast(ubyte[]) "HTTP/1.1 101")) 2458 throw new Exception("didn't get a websocket answer"); 2459 // skip the status line 2460 while(used.length && used[0] != '\n') 2461 used = used[1 .. $]; 2462 2463 if(used.length == 0) 2464 throw new Exception("wtf"); 2465 2466 if(used.length < 1) 2467 more(); 2468 2469 used = used[1 .. $]; // skip the \n 2470 2471 if(used.length == 0) 2472 more(); 2473 2474 // checks on the protocol from ehaders 2475 bool isWebsocket; 2476 bool isUpgrade; 2477 const(ubyte)[] protocol; 2478 const(ubyte)[] accept; 2479 2480 while(used.length) { 2481 if(used.length >= 2 && used[0] == '\r' && used[1] == '\n') { 2482 used = used[2 .. $]; 2483 break; // all done 2484 } 2485 int idxColon; 2486 while(idxColon < used.length && used[idxColon] != ':') 2487 idxColon++; 2488 if(idxColon == used.length) 2489 more(); 2490 auto idxStart = idxColon + 1; 2491 while(idxStart < used.length && used[idxStart] == ' ') 2492 idxStart++; 2493 if(idxStart == used.length) 2494 more(); 2495 auto idxEnd = idxStart; 2496 while(idxEnd < used.length && used[idxEnd] != '\r') 2497 idxEnd++; 2498 if(idxEnd == used.length) 2499 more(); 2500 2501 auto headerName = used[0 .. idxColon]; 2502 auto headerValue = used[idxStart .. idxEnd]; 2503 2504 // move past this header 2505 used = used[idxEnd .. $]; 2506 // and the \r\n 2507 if(2 <= used.length) 2508 used = used[2 .. $]; 2509 2510 if(headerName.bicmp("upgrade")) { 2511 if(headerValue.bicmp("websocket")) 2512 isWebsocket = true; 2513 } else if(headerName.bicmp("connection")) { 2514 if(headerValue.bicmp("upgrade")) 2515 isUpgrade = true; 2516 } else if(headerName.bicmp("sec-websocket-accept")) { 2517 accept = headerValue; 2518 } else if(headerName.bicmp("sec-websocket-protocol")) { 2519 protocol = headerValue; 2520 } 2521 2522 if(!used.length) { 2523 more(); 2524 } 2525 } 2526 2527 2528 if(!isWebsocket) 2529 throw new Exception("didn't answer as websocket"); 2530 if(!isUpgrade) 2531 throw new Exception("didn't answer as upgrade"); 2532 2533 2534 // FIXME: check protocol if config requested one 2535 // FIXME: check accept for the right hash 2536 2537 receiveBuffer[0 .. used.length] = used[]; 2538 receiveBufferUsedLength = used.length; 2539 2540 readyState_ = OPEN; 2541 2542 if(onopen) 2543 onopen(); 2544 2545 registerActiveSocket(this); 2546 } 2547 2548 /++ 2549 Is data pending on the socket? Also check [isMessageBuffered] to see if there 2550 is already a message in memory too. 2551 2552 If this returns `true`, you can call [lowLevelReceive], then try [isMessageBuffered] 2553 again. 2554 +/ 2555 /// Group: blocking_api 2556 public bool isDataPending(Duration timeout = 0.seconds) { 2557 static SocketSet readSet; 2558 if(readSet is null) 2559 readSet = new SocketSet(); 2560 2561 version(with_openssl) 2562 if(auto s = cast(SslClientSocket) socket) { 2563 // select doesn't handle the case with stuff 2564 // left in the ssl buffer so i'm checking it separately 2565 if(s.dataPending()) { 2566 return true; 2567 } 2568 } 2569 2570 readSet.add(socket); 2571 2572 //tryAgain: 2573 auto selectGot = Socket.select(readSet, null, null, timeout); 2574 if(selectGot == 0) { /* timeout */ 2575 // timeout 2576 return false; 2577 } else if(selectGot == -1) { /* interrupted */ 2578 return false; 2579 } else { /* ready */ 2580 if(readSet.isSet(socket)) { 2581 return true; 2582 } 2583 } 2584 2585 return false; 2586 } 2587 2588 private void llsend(ubyte[] d) { 2589 while(d.length) { 2590 auto r = socket.send(d); 2591 if(r <= 0) throw new Exception("wtf"); 2592 d = d[r .. $]; 2593 } 2594 } 2595 2596 private void llclose() { 2597 socket.shutdown(SocketShutdown.SEND); 2598 } 2599 2600 /++ 2601 Waits for more data off the low-level socket and adds it to the pending buffer. 2602 2603 Returns `true` if the connection is still active. 2604 +/ 2605 /// Group: blocking_api 2606 public bool lowLevelReceive() { 2607 auto r = socket.receive(receiveBuffer[receiveBufferUsedLength .. $]); 2608 if(r == 0) 2609 return false; 2610 if(r <= 0) 2611 throw new Exception("wtf"); 2612 receiveBufferUsedLength += r; 2613 return true; 2614 } 2615 2616 private Socket socket; 2617 2618 /* copy/paste section { */ 2619 2620 private int readyState_; 2621 private ubyte[] receiveBuffer; 2622 private size_t receiveBufferUsedLength; 2623 2624 private Config config; 2625 2626 enum CONNECTING = 0; /// Socket has been created. The connection is not yet open. 2627 enum OPEN = 1; /// The connection is open and ready to communicate. 2628 enum CLOSING = 2; /// The connection is in the process of closing. 2629 enum CLOSED = 3; /// The connection is closed or couldn't be opened. 2630 2631 /++ 2632 2633 +/ 2634 /// Group: foundational 2635 static struct Config { 2636 /++ 2637 These control the size of the receive buffer. 2638 2639 It starts at the initial size, will temporarily 2640 balloon up to the maximum size, and will reuse 2641 a buffer up to the likely size. 2642 2643 Anything larger than the maximum size will cause 2644 the connection to be aborted and an exception thrown. 2645 This is to protect you against a peer trying to 2646 exhaust your memory, while keeping the user-level 2647 processing simple. 2648 +/ 2649 size_t initialReceiveBufferSize = 4096; 2650 size_t likelyReceiveBufferSize = 4096; /// ditto 2651 size_t maximumReceiveBufferSize = 10 * 1024 * 1024; /// ditto 2652 2653 /++ 2654 Maximum combined size of a message. 2655 +/ 2656 size_t maximumMessageSize = 10 * 1024 * 1024; 2657 2658 string[string] cookies; /// Cookies to send with the initial request. cookies[name] = value; 2659 string origin; /// Origin URL to send with the handshake, if desired. 2660 string protocol; /// the protocol header, if desired. 2661 2662 int pingFrequency = 5000; /// Amount of time (in msecs) of idleness after which to send an automatic ping 2663 } 2664 2665 /++ 2666 Returns one of [CONNECTING], [OPEN], [CLOSING], or [CLOSED]. 2667 +/ 2668 int readyState() { 2669 return readyState_; 2670 } 2671 2672 /++ 2673 Closes the connection, sending a graceful teardown message to the other side. 2674 +/ 2675 /// Group: foundational 2676 void close(int code = 0, string reason = null) 2677 //in (reason.length < 123) 2678 in { assert(reason.length < 123); } do 2679 { 2680 if(readyState_ != OPEN) 2681 return; // it cool, we done 2682 WebSocketFrame wss; 2683 wss.fin = true; 2684 wss.opcode = WebSocketOpcode.close; 2685 wss.data = cast(ubyte[]) reason; 2686 wss.send(&llsend); 2687 2688 readyState_ = CLOSING; 2689 2690 llclose(); 2691 } 2692 2693 /++ 2694 Sends a ping message to the server. This is done automatically by the library if you set a non-zero [Config.pingFrequency], but you can also send extra pings explicitly as well with this function. 2695 +/ 2696 /// Group: foundational 2697 void ping() { 2698 WebSocketFrame wss; 2699 wss.fin = true; 2700 wss.opcode = WebSocketOpcode.ping; 2701 wss.send(&llsend); 2702 } 2703 2704 // automatically handled.... 2705 void pong() { 2706 WebSocketFrame wss; 2707 wss.fin = true; 2708 wss.opcode = WebSocketOpcode.pong; 2709 wss.send(&llsend); 2710 } 2711 2712 /++ 2713 Sends a text message through the websocket. 2714 +/ 2715 /// Group: foundational 2716 void send(in char[] textData) { 2717 WebSocketFrame wss; 2718 wss.fin = true; 2719 wss.opcode = WebSocketOpcode.text; 2720 wss.data = cast(ubyte[]) textData; 2721 wss.send(&llsend); 2722 } 2723 2724 /++ 2725 Sends a binary message through the websocket. 2726 +/ 2727 /// Group: foundational 2728 void send(in ubyte[] binaryData) { 2729 WebSocketFrame wss; 2730 wss.fin = true; 2731 wss.opcode = WebSocketOpcode.binary; 2732 wss.data = cast(ubyte[]) binaryData; 2733 wss.send(&llsend); 2734 } 2735 2736 /++ 2737 Waits for and returns the next complete message on the socket. 2738 2739 Note that the onmessage function is still called, right before 2740 this returns. 2741 +/ 2742 /// Group: blocking_api 2743 public WebSocketFrame waitForNextMessage() { 2744 do { 2745 auto m = processOnce(); 2746 if(m.populated) 2747 return m; 2748 } while(lowLevelReceive()); 2749 2750 return WebSocketFrame.init; // FIXME? maybe. 2751 } 2752 2753 /++ 2754 Tells if [waitForNextMessage] would block. 2755 +/ 2756 /// Group: blocking_api 2757 public bool waitForNextMessageWouldBlock() { 2758 checkAgain: 2759 if(isMessageBuffered()) 2760 return false; 2761 if(!isDataPending()) 2762 return true; 2763 while(isDataPending()) 2764 lowLevelReceive(); 2765 goto checkAgain; 2766 } 2767 2768 /++ 2769 Is there a message in the buffer already? 2770 If `true`, [waitForNextMessage] is guaranteed to return immediately. 2771 If `false`, check [isDataPending] as the next step. 2772 +/ 2773 /// Group: blocking_api 2774 public bool isMessageBuffered() { 2775 ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; 2776 auto s = d; 2777 if(d.length) { 2778 auto orig = d; 2779 auto m = WebSocketFrame.read(d); 2780 // that's how it indicates that it needs more data 2781 if(d !is orig) 2782 return true; 2783 } 2784 2785 return false; 2786 } 2787 2788 private ubyte continuingType; 2789 private ubyte[] continuingData; 2790 //private size_t continuingDataLength; 2791 2792 private WebSocketFrame processOnce() { 2793 ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; 2794 auto s = d; 2795 // FIXME: handle continuation frames more efficiently. it should really just reuse the receive buffer. 2796 WebSocketFrame m; 2797 if(d.length) { 2798 auto orig = d; 2799 m = WebSocketFrame.read(d); 2800 // that's how it indicates that it needs more data 2801 if(d is orig) 2802 return WebSocketFrame.init; 2803 m.unmaskInPlace(); 2804 switch(m.opcode) { 2805 case WebSocketOpcode.continuation: 2806 if(continuingData.length + m.data.length > config.maximumMessageSize) 2807 throw new Exception("message size exceeded"); 2808 2809 continuingData ~= m.data; 2810 if(m.fin) { 2811 if(ontextmessage) 2812 ontextmessage(cast(char[]) continuingData); 2813 if(onbinarymessage) 2814 onbinarymessage(continuingData); 2815 2816 continuingData = null; 2817 } 2818 break; 2819 case WebSocketOpcode.text: 2820 if(m.fin) { 2821 if(ontextmessage) 2822 ontextmessage(m.textData); 2823 } else { 2824 continuingType = m.opcode; 2825 //continuingDataLength = 0; 2826 continuingData = null; 2827 continuingData ~= m.data; 2828 } 2829 break; 2830 case WebSocketOpcode.binary: 2831 if(m.fin) { 2832 if(onbinarymessage) 2833 onbinarymessage(m.data); 2834 } else { 2835 continuingType = m.opcode; 2836 //continuingDataLength = 0; 2837 continuingData = null; 2838 continuingData ~= m.data; 2839 } 2840 break; 2841 case WebSocketOpcode.close: 2842 readyState_ = CLOSED; 2843 if(onclose) 2844 onclose(); 2845 2846 unregisterActiveSocket(this); 2847 break; 2848 case WebSocketOpcode.ping: 2849 pong(); 2850 break; 2851 case WebSocketOpcode.pong: 2852 // just really references it is still alive, nbd. 2853 break; 2854 default: // ignore though i could and perhaps should throw too 2855 } 2856 } 2857 receiveBufferUsedLength -= s.length - d.length; 2858 2859 return m; 2860 } 2861 2862 private void autoprocess() { 2863 // FIXME 2864 do { 2865 processOnce(); 2866 } while(lowLevelReceive()); 2867 } 2868 2869 2870 void delegate() onclose; /// 2871 void delegate() onerror; /// 2872 void delegate(in char[]) ontextmessage; /// 2873 void delegate(in ubyte[]) onbinarymessage; /// 2874 void delegate() onopen; /// 2875 2876 /++ 2877 2878 +/ 2879 /// Group: browser_api 2880 void onmessage(void delegate(in char[]) dg) { 2881 ontextmessage = dg; 2882 } 2883 2884 /// ditto 2885 void onmessage(void delegate(in ubyte[]) dg) { 2886 onbinarymessage = dg; 2887 } 2888 2889 /* } end copy/paste */ 2890 2891 /* 2892 const int bufferedAmount // amount pending 2893 const string extensions 2894 2895 const string protocol 2896 const string url 2897 */ 2898 2899 static { 2900 /++ 2901 2902 +/ 2903 void eventLoop() { 2904 2905 static SocketSet readSet; 2906 2907 if(readSet is null) 2908 readSet = new SocketSet(); 2909 2910 loopExited = false; 2911 2912 outermost: while(!loopExited) { 2913 readSet.reset(); 2914 2915 bool hadAny; 2916 foreach(sock; activeSockets) { 2917 readSet.add(sock.socket); 2918 hadAny = true; 2919 } 2920 2921 if(!hadAny) 2922 return; 2923 2924 tryAgain: 2925 auto selectGot = Socket.select(readSet, null, null, 10.seconds /* timeout */); 2926 if(selectGot == 0) { /* timeout */ 2927 // timeout 2928 goto tryAgain; 2929 } else if(selectGot == -1) { /* interrupted */ 2930 goto tryAgain; 2931 } else { 2932 foreach(sock; activeSockets) { 2933 if(readSet.isSet(sock.socket)) { 2934 if(!sock.lowLevelReceive()) { 2935 sock.readyState_ = CLOSED; 2936 unregisterActiveSocket(sock); 2937 continue outermost; 2938 } 2939 while(sock.processOnce().populated) {} 2940 selectGot--; 2941 if(selectGot <= 0) 2942 break; 2943 } 2944 } 2945 } 2946 } 2947 } 2948 2949 private bool loopExited; 2950 /++ 2951 2952 +/ 2953 void exitEventLoop() { 2954 loopExited = true; 2955 } 2956 2957 WebSocket[] activeSockets; 2958 void registerActiveSocket(WebSocket s) { 2959 activeSockets ~= s; 2960 } 2961 void unregisterActiveSocket(WebSocket s) { 2962 foreach(i, a; activeSockets) 2963 if(s is a) { 2964 activeSockets[i] = activeSockets[$-1]; 2965 activeSockets = activeSockets[0 .. $-1]; 2966 break; 2967 } 2968 } 2969 } 2970 } 2971 2972 /* copy/paste from cgi.d */ 2973 public { 2974 enum WebSocketOpcode : ubyte { 2975 continuation = 0, 2976 text = 1, 2977 binary = 2, 2978 // 3, 4, 5, 6, 7 RESERVED 2979 close = 8, 2980 ping = 9, 2981 pong = 10, 2982 // 11,12,13,14,15 RESERVED 2983 } 2984 2985 public struct WebSocketFrame { 2986 private bool populated; 2987 bool fin; 2988 bool rsv1; 2989 bool rsv2; 2990 bool rsv3; 2991 WebSocketOpcode opcode; // 4 bits 2992 bool masked; 2993 ubyte lengthIndicator; // don't set this when building one to send 2994 ulong realLength; // don't use when sending 2995 ubyte[4] maskingKey; // don't set this when sending 2996 ubyte[] data; 2997 2998 static WebSocketFrame simpleMessage(WebSocketOpcode opcode, void[] data) { 2999 WebSocketFrame msg; 3000 msg.fin = true; 3001 msg.opcode = opcode; 3002 msg.data = cast(ubyte[]) data; 3003 3004 return msg; 3005 } 3006 3007 private void send(scope void delegate(ubyte[]) llsend) { 3008 ubyte[64] headerScratch; 3009 int headerScratchPos = 0; 3010 3011 realLength = data.length; 3012 3013 { 3014 ubyte b1; 3015 b1 |= cast(ubyte) opcode; 3016 b1 |= rsv3 ? (1 << 4) : 0; 3017 b1 |= rsv2 ? (1 << 5) : 0; 3018 b1 |= rsv1 ? (1 << 6) : 0; 3019 b1 |= fin ? (1 << 7) : 0; 3020 3021 headerScratch[0] = b1; 3022 headerScratchPos++; 3023 } 3024 3025 { 3026 headerScratchPos++; // we'll set header[1] at the end of this 3027 auto rlc = realLength; 3028 ubyte b2; 3029 b2 |= masked ? (1 << 7) : 0; 3030 3031 assert(headerScratchPos == 2); 3032 3033 if(realLength > 65535) { 3034 // use 64 bit length 3035 b2 |= 0x7f; 3036 3037 // FIXME: double check endinaness 3038 foreach(i; 0 .. 8) { 3039 headerScratch[2 + 7 - i] = rlc & 0x0ff; 3040 rlc >>>= 8; 3041 } 3042 3043 headerScratchPos += 8; 3044 } else if(realLength > 125) { 3045 // use 16 bit length 3046 b2 |= 0x7e; 3047 3048 // FIXME: double check endinaness 3049 foreach(i; 0 .. 2) { 3050 headerScratch[2 + 1 - i] = rlc & 0x0ff; 3051 rlc >>>= 8; 3052 } 3053 3054 headerScratchPos += 2; 3055 } else { 3056 // use 7 bit length 3057 b2 |= realLength & 0b_0111_1111; 3058 } 3059 3060 headerScratch[1] = b2; 3061 } 3062 3063 //assert(!masked, "masking key not properly implemented"); 3064 if(masked) { 3065 // FIXME: randomize this 3066 headerScratch[headerScratchPos .. headerScratchPos + 4] = maskingKey[]; 3067 headerScratchPos += 4; 3068 3069 // we'll just mask it in place... 3070 int keyIdx = 0; 3071 foreach(i; 0 .. data.length) { 3072 data[i] = data[i] ^ maskingKey[keyIdx]; 3073 if(keyIdx == 3) 3074 keyIdx = 0; 3075 else 3076 keyIdx++; 3077 } 3078 } 3079 3080 //writeln("SENDING ", headerScratch[0 .. headerScratchPos], data); 3081 llsend(headerScratch[0 .. headerScratchPos]); 3082 llsend(data); 3083 } 3084 3085 static WebSocketFrame read(ref ubyte[] d) { 3086 WebSocketFrame msg; 3087 3088 auto orig = d; 3089 3090 WebSocketFrame needsMoreData() { 3091 d = orig; 3092 return WebSocketFrame.init; 3093 } 3094 3095 if(d.length < 2) 3096 return needsMoreData(); 3097 3098 ubyte b = d[0]; 3099 3100 msg.populated = true; 3101 3102 msg.opcode = cast(WebSocketOpcode) (b & 0x0f); 3103 b >>= 4; 3104 msg.rsv3 = b & 0x01; 3105 b >>= 1; 3106 msg.rsv2 = b & 0x01; 3107 b >>= 1; 3108 msg.rsv1 = b & 0x01; 3109 b >>= 1; 3110 msg.fin = b & 0x01; 3111 3112 b = d[1]; 3113 msg.masked = (b & 0b1000_0000) ? true : false; 3114 msg.lengthIndicator = b & 0b0111_1111; 3115 3116 d = d[2 .. $]; 3117 3118 if(msg.lengthIndicator == 0x7e) { 3119 // 16 bit length 3120 msg.realLength = 0; 3121 3122 if(d.length < 2) return needsMoreData(); 3123 3124 foreach(i; 0 .. 2) { 3125 msg.realLength |= d[0] << ((1-i) * 8); 3126 d = d[1 .. $]; 3127 } 3128 } else if(msg.lengthIndicator == 0x7f) { 3129 // 64 bit length 3130 msg.realLength = 0; 3131 3132 if(d.length < 8) return needsMoreData(); 3133 3134 foreach(i; 0 .. 8) { 3135 msg.realLength |= d[0] << ((7-i) * 8); 3136 d = d[1 .. $]; 3137 } 3138 } else { 3139 // 7 bit length 3140 msg.realLength = msg.lengthIndicator; 3141 } 3142 3143 if(msg.masked) { 3144 3145 if(d.length < 4) return needsMoreData(); 3146 3147 msg.maskingKey = d[0 .. 4]; 3148 d = d[4 .. $]; 3149 } 3150 3151 if(msg.realLength > d.length) { 3152 return needsMoreData(); 3153 } 3154 3155 msg.data = d[0 .. cast(size_t) msg.realLength]; 3156 d = d[cast(size_t) msg.realLength .. $]; 3157 3158 return msg; 3159 } 3160 3161 void unmaskInPlace() { 3162 if(this.masked) { 3163 int keyIdx = 0; 3164 foreach(i; 0 .. this.data.length) { 3165 this.data[i] = this.data[i] ^ this.maskingKey[keyIdx]; 3166 if(keyIdx == 3) 3167 keyIdx = 0; 3168 else 3169 keyIdx++; 3170 } 3171 } 3172 } 3173 3174 char[] textData() { 3175 return cast(char[]) data; 3176 } 3177 } 3178 } 3179 3180 /+ 3181 so the url params are arguments. it knows the request 3182 internally. other params are properties on the req 3183 3184 names may have different paths... those will just add ForSomething i think. 3185 3186 auto req = api.listMergeRequests 3187 req.page = 10; 3188 3189 or 3190 req.page(1) 3191 .bar("foo") 3192 3193 req.execute(); 3194 3195 3196 everything in the response is nullable access through the 3197 dynamic object, just with property getters there. need to make 3198 it static generated tho 3199 3200 other messages may be: isPresent and getDynamic 3201 3202 3203 AND/OR what about doing it like the rails objects 3204 3205 BroadcastMessage.get(4) 3206 // various properties 3207 3208 // it lists what you updated 3209 3210 BroadcastMessage.foo().bar().put(5) 3211 +/