1 // Copyright 2013-2019, Adam D. Ruppe. 2 /++ 3 This is version 2 of my http/1.1 client implementation. 4 5 6 It has no dependencies for basic operation, but does require OpenSSL 7 libraries (or compatible) to be support HTTPS. Compile with 8 `-version=with_openssl` to enable such support. 9 10 http2.d, despite its name, does NOT implement HTTP/2.0, but this 11 shouldn't matter for 99.9% of usage, since all servers will continue 12 to support HTTP/1.1 for a very long time. 13 14 +/ 15 module arsd.http2; 16 17 import std.uri : encodeComponent; 18 19 debug(arsd_http2_verbose) debug=arsd_http2; 20 21 debug(arsd_http2) import std.stdio : writeln; 22 23 version(without_openssl) {} 24 else { 25 version=use_openssl; 26 version=with_openssl; 27 version(older_openssl) {} else 28 version=newer_openssl; 29 } 30 31 32 33 /++ 34 Demonstrates core functionality, using the [HttpClient], 35 [HttpRequest] (returned by [HttpClient.navigateTo|client.navigateTo]), 36 and [HttpResponse] (returned by [HttpRequest.waitForCompletion|request.waitForCompletion]). 37 38 +/ 39 unittest { 40 import arsd.http2; 41 42 void main() { 43 auto client = new HttpClient(); 44 auto request = client.navigateTo(Uri("http://dlang.org/")); 45 auto response = request.waitForCompletion(); 46 47 string returnedHtml = response.contentText; 48 } 49 } 50 51 // FIXME: multipart encoded file uploads needs implementation 52 // future: do web client api stuff 53 54 debug import std.stdio; 55 56 import std.socket; 57 import core.time; 58 59 // FIXME: check Transfer-Encoding: gzip always 60 61 version(with_openssl) { 62 pragma(lib, "crypto"); 63 pragma(lib, "ssl"); 64 } 65 66 /+ 67 HttpRequest httpRequest(string method, string url, ubyte[] content, string[string] content) { 68 return null; 69 } 70 +/ 71 72 /** 73 auto request = get("http://arsdnet.net/"); 74 request.send(); 75 76 auto response = get("http://arsdnet.net/").waitForCompletion(); 77 */ 78 HttpRequest get(string url) { 79 auto client = new HttpClient(); 80 auto request = client.navigateTo(Uri(url)); 81 return request; 82 } 83 84 /** 85 Do not forget to call `waitForCompletion()` on the returned object! 86 */ 87 HttpRequest post(string url, string[string] req) { 88 auto client = new HttpClient(); 89 ubyte[] bdata; 90 foreach(k, v; req) { 91 if(bdata.length) 92 bdata ~= cast(ubyte[]) "&"; 93 bdata ~= cast(ubyte[]) encodeComponent(k); 94 bdata ~= cast(ubyte[]) "="; 95 bdata ~= cast(ubyte[]) encodeComponent(v); 96 } 97 auto request = client.request(Uri(url), HttpVerb.POST, bdata, "application/x-www-form-urlencoded"); 98 return request; 99 } 100 101 /// gets the text off a url. basic operation only. 102 string getText(string url) { 103 auto request = get(url); 104 auto response = request.waitForCompletion(); 105 return cast(string) response.content; 106 } 107 108 /+ 109 ubyte[] getBinary(string url, string[string] cookies = null) { 110 auto hr = httpRequest("GET", url, null, cookies); 111 if(hr.code != 200) 112 throw new Exception(format("HTTP answered %d instead of 200 on %s", hr.code, url)); 113 return hr.content; 114 } 115 116 /** 117 Gets a textual document, ignoring headers. Throws on non-text or error. 118 */ 119 string get(string url, string[string] cookies = null) { 120 auto hr = httpRequest("GET", url, null, cookies); 121 if(hr.code != 200) 122 throw new Exception(format("HTTP answered %d instead of 200 on %s", hr.code, url)); 123 if(hr.contentType.indexOf("text/") == -1) 124 throw new Exception(hr.contentType ~ " is bad content for conversion to string"); 125 return cast(string) hr.content; 126 127 } 128 129 static import std.uri; 130 131 string post(string url, string[string] args, string[string] cookies = null) { 132 string content; 133 134 foreach(name, arg; args) { 135 if(content.length) 136 content ~= "&"; 137 content ~= std.uri.encode(name) ~ "=" ~ std.uri.encode(arg); 138 } 139 140 auto hr = httpRequest("POST", url, cast(ubyte[]) content, cookies, ["Content-Type: application/x-www-form-urlencoded"]); 141 if(hr.code != 200) 142 throw new Exception(format("HTTP answered %d instead of 200", hr.code)); 143 if(hr.contentType.indexOf("text/") == -1) 144 throw new Exception(hr.contentType ~ " is bad content for conversion to string"); 145 146 return cast(string) hr.content; 147 } 148 149 +/ 150 151 /// 152 struct HttpResponse { 153 int code; /// 154 string codeText; /// 155 156 string httpVersion; /// 157 158 string statusLine; /// 159 160 string contentType; /// The content type header 161 string location; /// The location header 162 163 string[string] cookies; /// Names and values of cookies set in the response. 164 165 string[] headers; /// Array of all headers returned. 166 string[string] headersHash; /// 167 168 ubyte[] content; /// The raw content returned in the response body. 169 string contentText; /// [content], but casted to string (for convenience) 170 171 /++ 172 returns `new Document(this.contentText)`. Requires [arsd.dom]. 173 +/ 174 auto contentDom()() { 175 import arsd.dom; 176 return new Document(this.contentText); 177 178 } 179 180 /++ 181 returns `var.fromJson(this.contentText)`. Requires [arsd.jsvar]. 182 +/ 183 auto contentJson()() { 184 import arsd.jsvar; 185 return var.fromJson(this.contentText); 186 } 187 188 HttpRequestParameters requestParameters; /// 189 190 LinkHeader[] linksStored; 191 bool linksLazilyParsed; 192 193 /// Returns links header sorted by "rel" attribute. 194 /// It returns a new array on each call. 195 LinkHeader[string] linksHash() { 196 auto links = this.links(); 197 LinkHeader[string] ret; 198 foreach(link; links) 199 ret[link.rel] = link; 200 return ret; 201 } 202 203 /// Returns the Link header, parsed. 204 LinkHeader[] links() { 205 if(linksLazilyParsed) 206 return linksStored; 207 linksLazilyParsed = true; 208 LinkHeader[] ret; 209 210 auto hdrPtr = "Link" in headersHash; 211 if(hdrPtr is null) 212 return ret; 213 214 auto header = *hdrPtr; 215 216 LinkHeader current; 217 218 while(header.length) { 219 char ch = header[0]; 220 221 if(ch == '<') { 222 // read url 223 header = header[1 .. $]; 224 size_t idx; 225 while(idx < header.length && header[idx] != '>') 226 idx++; 227 current.url = header[0 .. idx]; 228 header = header[idx .. $]; 229 } else if(ch == ';') { 230 // read attribute 231 header = header[1 .. $]; 232 header = header.stripLeft; 233 234 size_t idx; 235 while(idx < header.length && header[idx] != '=') 236 idx++; 237 238 string name = header[0 .. idx]; 239 header = header[idx + 1 .. $]; 240 241 string value; 242 243 if(header.length && header[0] == '"') { 244 // quoted value 245 header = header[1 .. $]; 246 idx = 0; 247 while(idx < header.length && header[idx] != '\"') 248 idx++; 249 value = header[0 .. idx]; 250 header = header[idx .. $]; 251 252 } else if(header.length) { 253 // unquoted value 254 idx = 0; 255 while(idx < header.length && header[idx] != ',' && header[idx] != ' ' && header[idx] != ';') 256 idx++; 257 258 value = header[0 .. idx]; 259 header = header[idx .. $].stripLeft; 260 } 261 262 name = name.toLower; 263 if(name == "rel") 264 current.rel = value; 265 else 266 current.attributes[name] = value; 267 268 } else if(ch == ',') { 269 // start another 270 ret ~= current; 271 current = LinkHeader.init; 272 } else if(ch == ' ' || ch == '\n' || ch == '\r' || ch == '\t') { 273 // ignore 274 } 275 276 header = header[1 .. $]; 277 } 278 279 ret ~= current; 280 281 linksStored = ret; 282 283 return ret; 284 } 285 } 286 287 /// 288 struct LinkHeader { 289 string url; /// 290 string rel; /// 291 string[string] attributes; /// like title, rev, media, whatever attributes 292 } 293 294 import std.string; 295 static import std.algorithm; 296 import std.conv; 297 import std.range; 298 299 300 301 // Copy pasta from cgi.d, then stripped down 302 /// 303 struct Uri { 304 alias toString this; // blargh idk a url really is a string, but should it be implicit? 305 306 // scheme//userinfo@host:port/path?query#fragment 307 308 string scheme; /// e.g. "http" in "http://example.com/" 309 string userinfo; /// the username (and possibly a password) in the uri 310 string host; /// the domain name 311 int port; /// port number, if given. Will be zero if a port was not explicitly given 312 string path; /// e.g. "/folder/file.html" in "http://example.com/folder/file.html" 313 string query; /// the stuff after the ? in a uri 314 string fragment; /// the stuff after the # in a uri. 315 316 /// Breaks down a uri string to its components 317 this(string uri) { 318 reparse(uri); 319 } 320 321 private void reparse(string uri) { 322 // from RFC 3986 323 // the ctRegex triples the compile time and makes ugly errors for no real benefit 324 // it was a nice experiment but just not worth it. 325 // enum ctr = ctRegex!r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?"; 326 /* 327 Captures: 328 0 = whole url 329 1 = scheme, with : 330 2 = scheme, no : 331 3 = authority, with // 332 4 = authority, no // 333 5 = path 334 6 = query string, with ? 335 7 = query string, no ? 336 8 = anchor, with # 337 9 = anchor, no # 338 */ 339 // Yikes, even regular, non-CT regex is also unacceptably slow to compile. 1.9s on my computer! 340 // instead, I will DIY and cut that down to 0.6s on the same computer. 341 /* 342 343 Note that authority is 344 user:password@domain:port 345 where the user:password@ part is optional, and the :port is optional. 346 347 Regex translation: 348 349 Scheme cannot have :, /, ?, or # in it, and must have one or more chars and end in a :. It is optional, but must be first. 350 Authority must start with //, but cannot have any other /, ?, or # in it. It is optional. 351 Path cannot have any ? or # in it. It is optional. 352 Query must start with ? and must not have # in it. It is optional. 353 Anchor must start with # and can have anything else in it to end of string. It is optional. 354 */ 355 356 this = Uri.init; // reset all state 357 358 // empty uri = nothing special 359 if(uri.length == 0) { 360 return; 361 } 362 363 size_t idx; 364 365 scheme_loop: foreach(char c; uri[idx .. $]) { 366 switch(c) { 367 case ':': 368 case '/': 369 case '?': 370 case '#': 371 break scheme_loop; 372 default: 373 } 374 idx++; 375 } 376 377 if(idx == 0 && uri[idx] == ':') { 378 // this is actually a path! we skip way ahead 379 goto path_loop; 380 } 381 382 if(idx == uri.length) { 383 // the whole thing is a path, apparently 384 path = uri; 385 return; 386 } 387 388 if(idx > 0 && uri[idx] == ':') { 389 scheme = uri[0 .. idx]; 390 idx++; 391 } else { 392 // we need to rewind; it found a / but no :, so the whole thing is prolly a path... 393 idx = 0; 394 } 395 396 if(idx + 2 < uri.length && uri[idx .. idx + 2] == "//") { 397 // we have an authority.... 398 idx += 2; 399 400 auto authority_start = idx; 401 authority_loop: foreach(char c; uri[idx .. $]) { 402 switch(c) { 403 case '/': 404 case '?': 405 case '#': 406 break authority_loop; 407 default: 408 } 409 idx++; 410 } 411 412 auto authority = uri[authority_start .. idx]; 413 414 auto idx2 = authority.indexOf("@"); 415 if(idx2 != -1) { 416 userinfo = authority[0 .. idx2]; 417 authority = authority[idx2 + 1 .. $]; 418 } 419 420 idx2 = authority.indexOf(":"); 421 if(idx2 == -1) { 422 port = 0; // 0 means not specified; we should use the default for the scheme 423 host = authority; 424 } else { 425 host = authority[0 .. idx2]; 426 port = to!int(authority[idx2 + 1 .. $]); 427 } 428 } 429 430 path_loop: 431 auto path_start = idx; 432 433 foreach(char c; uri[idx .. $]) { 434 if(c == '?' || c == '#') 435 break; 436 idx++; 437 } 438 439 path = uri[path_start .. idx]; 440 441 if(idx == uri.length) 442 return; // nothing more to examine... 443 444 if(uri[idx] == '?') { 445 idx++; 446 auto query_start = idx; 447 foreach(char c; uri[idx .. $]) { 448 if(c == '#') 449 break; 450 idx++; 451 } 452 query = uri[query_start .. idx]; 453 } 454 455 if(idx < uri.length && uri[idx] == '#') { 456 idx++; 457 fragment = uri[idx .. $]; 458 } 459 460 // uriInvalidated = false; 461 } 462 463 private string rebuildUri() const { 464 string ret; 465 if(scheme.length) 466 ret ~= scheme ~ ":"; 467 if(userinfo.length || host.length) 468 ret ~= "//"; 469 if(userinfo.length) 470 ret ~= userinfo ~ "@"; 471 if(host.length) 472 ret ~= host; 473 if(port) 474 ret ~= ":" ~ to!string(port); 475 476 ret ~= path; 477 478 if(query.length) 479 ret ~= "?" ~ query; 480 481 if(fragment.length) 482 ret ~= "#" ~ fragment; 483 484 // uri = ret; 485 // uriInvalidated = false; 486 return ret; 487 } 488 489 /// Converts the broken down parts back into a complete string 490 string toString() const { 491 // if(uriInvalidated) 492 return rebuildUri(); 493 } 494 495 /// Returns a new absolute Uri given a base. It treats this one as 496 /// relative where possible, but absolute if not. (If protocol, domain, or 497 /// other info is not set, the new one inherits it from the base.) 498 /// 499 /// Browsers use a function like this to figure out links in html. 500 Uri basedOn(in Uri baseUrl) const { 501 Uri n = this; // copies 502 // n.uriInvalidated = true; // make sure we regenerate... 503 504 // userinfo is not inherited... is this wrong? 505 506 // if anything is given in the existing url, we don't use the base anymore. 507 if(n.scheme.empty) { 508 n.scheme = baseUrl.scheme; 509 if(n.host.empty) { 510 n.host = baseUrl.host; 511 if(n.port == 0) { 512 n.port = baseUrl.port; 513 if(n.path.length > 0 && n.path[0] != '/') { 514 auto b = baseUrl.path[0 .. baseUrl.path.lastIndexOf("/") + 1]; 515 if(b.length == 0) 516 b = "/"; 517 n.path = b ~ n.path; 518 } else if(n.path.length == 0) { 519 n.path = baseUrl.path; 520 } 521 } 522 } 523 } 524 525 n.removeDots(); 526 527 return n; 528 } 529 530 void removeDots() { 531 auto parts = this.path.split("/"); 532 string[] toKeep; 533 foreach(part; parts) { 534 if(part == ".") { 535 continue; 536 } else if(part == "..") { 537 toKeep = toKeep[0 .. $-1]; 538 continue; 539 } else { 540 toKeep ~= part; 541 } 542 } 543 544 this.path = toKeep.join("/"); 545 } 546 547 } 548 549 /* 550 void main(string args[]) { 551 write(post("http://arsdnet.net/bugs.php", ["test" : "hey", "again" : "what"])); 552 } 553 */ 554 555 /// 556 struct BasicAuth { 557 string username; /// 558 string password; /// 559 } 560 561 /** 562 When you send something, it creates a request 563 and sends it asynchronously. The request object 564 565 auto request = new HttpRequest(); 566 // set any properties here 567 568 // synchronous usage 569 auto reply = request.perform(); 570 571 // async usage, type 1: 572 request.send(); 573 request2.send(); 574 575 // wait until the first one is done, with the second one still in-flight 576 auto response = request.waitForCompletion(); 577 578 579 // async usage, type 2: 580 request.onDataReceived = (HttpRequest hr) { 581 if(hr.state == HttpRequest.State.complete) { 582 // use hr.responseData 583 } 584 }; 585 request.send(); // send, using the callback 586 587 // before terminating, be sure you wait for your requests to finish! 588 589 request.waitForCompletion(); 590 591 */ 592 class HttpRequest { 593 594 /// Automatically follow a redirection? 595 bool followLocation = false; 596 597 private static { 598 // we manage the actual connections. When a request is made on a particular 599 // host, we try to reuse connections. We may open more than one connection per 600 // host to do parallel requests. 601 // 602 // The key is the *domain name* and the port. Multiple domains on the same address will have separate connections. 603 Socket[][string] socketsPerHost; 604 605 void loseSocket(string host, ushort port, bool ssl, Socket s) { 606 import std.string; 607 auto key = format("http%s://%s:%s", ssl ? "s" : "", host, port); 608 609 if(auto list = key in socketsPerHost) { 610 for(int a = 0; a < (*list).length; a++) { 611 if((*list)[a] is s) { 612 613 for(int b = a; b < (*list).length - 1; b++) 614 (*list)[b] = (*list)[b+1]; 615 (*list) = (*list)[0 .. $-1]; 616 break; 617 } 618 } 619 } 620 } 621 622 Socket getOpenSocketOnHost(string host, ushort port, bool ssl) { 623 Socket openNewConnection() { 624 Socket socket; 625 if(ssl) { 626 version(with_openssl) 627 socket = new SslClientSocket(AddressFamily.INET, SocketType.STREAM); 628 else 629 throw new Exception("SSL not compiled in"); 630 } else 631 socket = new Socket(AddressFamily.INET, SocketType.STREAM); 632 633 socket.connect(new InternetAddress(host, port)); 634 debug(arsd_http2) writeln("opening to ", host, ":", port, " ", cast(void*) socket); 635 assert(socket.handle() !is socket_t.init); 636 return socket; 637 } 638 639 import std.string; 640 auto key = format("http%s://%s:%s", ssl ? "s" : "", host, port); 641 642 if(auto hostListing = key in socketsPerHost) { 643 // try to find an available socket that is already open 644 foreach(socket; *hostListing) { 645 if(socket !in activeRequestOnSocket) { 646 // let's see if it has closed since we last tried 647 // e.g. a server timeout or something. If so, we need 648 // to lose this one and immediately open a new one. 649 static SocketSet readSet = null; 650 if(readSet is null) 651 readSet = new SocketSet(); 652 readSet.reset(); 653 assert(socket.handle() !is socket_t.init, socket is null ? "null" : socket.toString()); 654 readSet.add(socket); 655 auto got = Socket.select(readSet, null, null, 5.msecs /* timeout */); 656 if(got > 0) { 657 // we can read something off this... but there aren't 658 // any active requests. Assume it is EOF and open a new one 659 660 socket.close(); 661 loseSocket(host, port, ssl, socket); 662 goto openNew; 663 } 664 return socket; 665 } 666 } 667 668 // if not too many already open, go ahead and do a new one 669 if((*hostListing).length < 6) { 670 auto socket = openNewConnection(); 671 (*hostListing) ~= socket; 672 return socket; 673 } else 674 return null; // too many, you'll have to wait 675 } 676 677 openNew: 678 679 auto socket = openNewConnection(); 680 socketsPerHost[key] ~= socket; 681 return socket; 682 } 683 684 // only one request can be active on a given socket (at least HTTP < 2.0) so this is that 685 HttpRequest[Socket] activeRequestOnSocket; 686 HttpRequest[] pending; // and these are the requests that are waiting 687 688 SocketSet readSet; 689 SocketSet writeSet; 690 691 692 int advanceConnections() { 693 if(readSet is null) 694 readSet = new SocketSet(); 695 if(writeSet is null) 696 writeSet = new SocketSet(); 697 698 ubyte[2048] buffer; 699 700 HttpRequest[16] removeFromPending; 701 size_t removeFromPendingCount = 0; 702 703 // are there pending requests? let's try to send them 704 foreach(idx, pc; pending) { 705 if(removeFromPendingCount == removeFromPending.length) 706 break; 707 708 if(pc.state == HttpRequest.State.aborted) { 709 removeFromPending[removeFromPendingCount++] = pc; 710 continue; 711 } 712 713 auto socket = getOpenSocketOnHost(pc.requestParameters.host, pc.requestParameters.port, pc.requestParameters.ssl); 714 715 if(socket !is null) { 716 activeRequestOnSocket[socket] = pc; 717 assert(pc.sendBuffer.length); 718 pc.state = State.sendingHeaders; 719 720 removeFromPending[removeFromPendingCount++] = pc; 721 } 722 } 723 724 import std.algorithm : remove; 725 foreach(rp; removeFromPending[0 .. removeFromPendingCount]) 726 pending = pending.remove!((a) => a is rp)(); 727 728 readSet.reset(); 729 writeSet.reset(); 730 731 bool hadOne = false; 732 733 // active requests need to be read or written to 734 foreach(sock, request; activeRequestOnSocket) { 735 // check the other sockets just for EOF, if they close, take them out of our list, 736 // we'll reopen if needed upon request. 737 readSet.add(sock); 738 hadOne = true; 739 if(request.state == State.sendingHeaders || request.state == State.sendingBody) { 740 writeSet.add(sock); 741 hadOne = true; 742 } 743 } 744 745 if(!hadOne) 746 return 1; // automatic timeout, nothing to do 747 748 tryAgain: 749 auto selectGot = Socket.select(readSet, writeSet, null, 10.seconds /* timeout */); 750 if(selectGot == 0) { /* timeout */ 751 // timeout 752 return 1; 753 } else if(selectGot == -1) { /* interrupted */ 754 /* 755 version(Posix) { 756 import core.stdc.errno; 757 if(errno != EINTR) 758 throw new Exception("select error: " ~ to!string(errno)); 759 } 760 */ 761 goto tryAgain; 762 } else { /* ready */ 763 Socket[16] inactive; 764 int inactiveCount = 0; 765 foreach(sock, request; activeRequestOnSocket) { 766 if(readSet.isSet(sock)) { 767 keep_going: 768 auto got = sock.receive(buffer); 769 debug(arsd_http2_verbose) writeln("====PACKET ",got,"=====",cast(string)buffer[0 .. got],"===/PACKET==="); 770 if(got < 0) { 771 throw new Exception("receive error"); 772 } else if(got == 0) { 773 // remote side disconnected 774 debug(arsd_http2) writeln("remote disconnect"); 775 request.state = State.aborted; 776 inactive[inactiveCount++] = sock; 777 sock.close(); 778 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 779 } else { 780 // data available 781 auto stillAlive = request.handleIncomingData(buffer[0 .. got]); 782 783 if(!stillAlive || request.state == HttpRequest.State.complete || request.state == HttpRequest.State.aborted) { 784 //import std.stdio; writeln(cast(void*) sock, " ", stillAlive, " ", request.state); 785 inactive[inactiveCount++] = sock; 786 continue; 787 // reuse the socket for another pending request, if we can 788 } 789 } 790 791 if(request.onDataReceived) 792 request.onDataReceived(request); 793 794 version(with_openssl) 795 if(auto s = cast(SslClientSocket) sock) { 796 // select doesn't handle the case with stuff 797 // left in the ssl buffer so i'm checking it separately 798 if(s.dataPending()) { 799 goto keep_going; 800 } 801 } 802 } 803 804 if(request.state == State.sendingHeaders || request.state == State.sendingBody) 805 if(writeSet.isSet(sock)) { 806 assert(request.sendBuffer.length); 807 auto sent = sock.send(request.sendBuffer); 808 debug(arsd_http2_verbose) writeln(cast(void*) sock, "<send>", cast(string) request.sendBuffer, "</send>"); 809 if(sent <= 0) 810 throw new Exception("send error " ~ lastSocketError); 811 request.sendBuffer = request.sendBuffer[sent .. $]; 812 if(request.sendBuffer.length == 0) { 813 request.state = State.waitingForResponse; 814 } 815 } 816 } 817 818 foreach(s; inactive[0 .. inactiveCount]) { 819 debug(arsd_http2) writeln("removing socket from active list ", cast(void*) s); 820 activeRequestOnSocket.remove(s); 821 } 822 } 823 824 // we've completed a request, are there any more pending connection? if so, send them now 825 826 return 0; 827 } 828 } 829 830 public static void resetInternals() { 831 socketsPerHost = null; 832 activeRequestOnSocket = null; 833 pending = null; 834 835 } 836 837 struct HeaderReadingState { 838 bool justSawLf; 839 bool justSawCr; 840 bool atStartOfLine = true; 841 bool readingLineContinuation; 842 } 843 HeaderReadingState headerReadingState; 844 845 struct BodyReadingState { 846 bool isGzipped; 847 bool isDeflated; 848 849 bool isChunked; 850 int chunkedState; 851 852 // used for the chunk size if it is chunked 853 int contentLengthRemaining; 854 } 855 BodyReadingState bodyReadingState; 856 857 bool closeSocketWhenComplete; 858 859 import std.zlib; 860 UnCompress uncompress; 861 862 const(ubyte)[] leftoverDataFromLastTime; 863 864 bool handleIncomingData(scope const ubyte[] dataIn) { 865 bool stillAlive = true; 866 debug(arsd_http2) writeln("handleIncomingData, state: ", state); 867 if(state == State.waitingForResponse) { 868 state = State.readingHeaders; 869 headerReadingState = HeaderReadingState.init; 870 bodyReadingState = BodyReadingState.init; 871 } 872 873 const(ubyte)[] data; 874 if(leftoverDataFromLastTime.length) 875 data = leftoverDataFromLastTime ~ dataIn[]; 876 else 877 data = dataIn[]; 878 879 if(state == State.readingHeaders) { 880 void parseLastHeader() { 881 assert(responseData.headers.length); 882 if(responseData.headers.length == 1) { 883 responseData.statusLine = responseData.headers[0]; 884 import std.algorithm; 885 auto parts = responseData.statusLine.splitter(" "); 886 responseData.httpVersion = parts.front; 887 parts.popFront(); 888 responseData.code = to!int(parts.front()); 889 parts.popFront(); 890 responseData.codeText = ""; 891 while(!parts.empty) { 892 // FIXME: this sucks! 893 responseData.codeText ~= parts.front(); 894 parts.popFront(); 895 if(!parts.empty) 896 responseData.codeText ~= " "; 897 } 898 } else { 899 // parse the new header 900 auto header = responseData.headers[$-1]; 901 902 auto colon = header.indexOf(":"); 903 if(colon == -1) 904 return; 905 auto name = header[0 .. colon]; 906 auto value = header[colon + 2 .. $]; // skipping the colon itself and the following space 907 908 switch(name) { 909 case "Connection": 910 case "connection": 911 if(value == "close") 912 closeSocketWhenComplete = true; 913 break; 914 case "Content-Type": 915 case "content-type": 916 responseData.contentType = value; 917 break; 918 case "Location": 919 case "location": 920 responseData.location = value; 921 break; 922 case "Content-Length": 923 case "content-length": 924 bodyReadingState.contentLengthRemaining = to!int(value); 925 break; 926 case "Transfer-Encoding": 927 case "transfer-encoding": 928 // note that if it is gzipped, it zips first, then chunks the compressed stream. 929 // so we should always dechunk first, then feed into the decompressor 930 if(value.strip == "chunked") 931 bodyReadingState.isChunked = true; 932 else throw new Exception("Unknown Transfer-Encoding: " ~ value); 933 break; 934 case "Content-Encoding": 935 case "content-encoding": 936 if(value == "gzip") { 937 bodyReadingState.isGzipped = true; 938 uncompress = new UnCompress(); 939 } else if(value == "deflate") { 940 bodyReadingState.isDeflated = true; 941 uncompress = new UnCompress(); 942 } else throw new Exception("Unknown Content-Encoding: " ~ value); 943 break; 944 case "Set-Cookie": 945 case "set-cookie": 946 // FIXME handle 947 break; 948 default: 949 // ignore 950 } 951 952 responseData.headersHash[name] = value; 953 } 954 } 955 956 size_t position = 0; 957 for(position = 0; position < dataIn.length; position++) { 958 if(headerReadingState.readingLineContinuation) { 959 if(data[position] == ' ' || data[position] == '\t') 960 continue; 961 headerReadingState.readingLineContinuation = false; 962 } 963 964 if(headerReadingState.atStartOfLine) { 965 headerReadingState.atStartOfLine = false; 966 if(data[position] == '\r' || data[position] == '\n') { 967 // done with headers 968 if(data[position] == '\r' && (position + 1) < data.length && data[position + 1] == '\n') 969 position++; 970 state = State.readingBody; 971 position++; // skip the newline 972 break; 973 } else if(data[position] == ' ' || data[position] == '\t') { 974 // line continuation, ignore all whitespace and collapse it into a space 975 headerReadingState.readingLineContinuation = true; 976 responseData.headers[$-1] ~= ' '; 977 } else { 978 // new header 979 if(responseData.headers.length) 980 parseLastHeader(); 981 responseData.headers ~= ""; 982 } 983 } 984 985 if(data[position] == '\r') { 986 headerReadingState.justSawCr = true; 987 continue; 988 } else 989 headerReadingState.justSawCr = false; 990 991 if(data[position] == '\n') { 992 headerReadingState.justSawLf = true; 993 headerReadingState.atStartOfLine = true; 994 continue; 995 } else 996 headerReadingState.justSawLf = false; 997 998 responseData.headers[$-1] ~= data[position]; 999 } 1000 1001 parseLastHeader(); 1002 data = data[position .. $]; 1003 } 1004 1005 if(state == State.readingBody) { 1006 if(bodyReadingState.isChunked) { 1007 // read the hex length, stopping at a \r\n, ignoring everything between the new line but after the first non-valid hex character 1008 // read binary data of that length. it is our content 1009 // repeat until a zero sized chunk 1010 // then read footers as headers. 1011 1012 start_over: 1013 for(int a = 0; a < data.length; a++) { 1014 final switch(bodyReadingState.chunkedState) { 1015 case 0: // reading hex 1016 char c = data[a]; 1017 if((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) { 1018 // just keep reading 1019 } else { 1020 int power = 1; 1021 bodyReadingState.contentLengthRemaining = 0; 1022 assert(a != 0, cast(string) data); 1023 for(int b = a-1; b >= 0; b--) { 1024 char cc = data[b]; 1025 if(cc >= 'a' && cc <= 'z') 1026 cc -= 0x20; 1027 int val = 0; 1028 if(cc >= '0' && cc <= '9') 1029 val = cc - '0'; 1030 else 1031 val = cc - 'A' + 10; 1032 1033 assert(val >= 0 && val <= 15, to!string(val)); 1034 bodyReadingState.contentLengthRemaining += power * val; 1035 power *= 16; 1036 } 1037 debug(arsd_http2_verbose) writeln("Chunk length: ", bodyReadingState.contentLengthRemaining); 1038 bodyReadingState.chunkedState = 1; 1039 data = data[a + 1 .. $]; 1040 goto start_over; 1041 } 1042 break; 1043 case 1: // reading until end of line 1044 char c = data[a]; 1045 if(c == '\n') { 1046 if(bodyReadingState.contentLengthRemaining == 0) 1047 bodyReadingState.chunkedState = 5; 1048 else 1049 bodyReadingState.chunkedState = 2; 1050 } 1051 data = data[a + 1 .. $]; 1052 goto start_over; 1053 case 2: // reading data 1054 auto can = a + bodyReadingState.contentLengthRemaining; 1055 if(can > data.length) 1056 can = cast(int) data.length; 1057 1058 auto newData = data[a .. can]; 1059 data = data[can .. $]; 1060 1061 //if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) 1062 // responseData.content ~= cast(ubyte[]) uncompress.uncompress(data[a .. can]); 1063 //else 1064 responseData.content ~= newData; 1065 1066 bodyReadingState.contentLengthRemaining -= newData.length; 1067 debug(arsd_http2_verbose) writeln("clr: ", bodyReadingState.contentLengthRemaining, " " , a, " ", can); 1068 assert(bodyReadingState.contentLengthRemaining >= 0); 1069 if(bodyReadingState.contentLengthRemaining == 0) { 1070 bodyReadingState.chunkedState = 3; 1071 } else { 1072 // will continue grabbing more 1073 } 1074 goto start_over; 1075 case 3: // reading 13/10 1076 assert(data[a] == 13); 1077 bodyReadingState.chunkedState++; 1078 data = data[a + 1 .. $]; 1079 goto start_over; 1080 case 4: // reading 10 at end of packet 1081 assert(data[a] == 10); 1082 data = data[a + 1 .. $]; 1083 bodyReadingState.chunkedState = 0; 1084 goto start_over; 1085 case 5: // reading footers 1086 //goto done; // FIXME 1087 state = State.complete; 1088 1089 bodyReadingState.chunkedState = 0; 1090 1091 while(data[a] != 10) 1092 a++; 1093 data = data[a + 1 .. $]; 1094 1095 if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) { 1096 auto n = uncompress.uncompress(responseData.content); 1097 n ~= uncompress.flush(); 1098 responseData.content = cast(ubyte[]) n; 1099 } 1100 1101 // responseData.content ~= cast(ubyte[]) uncompress.flush(); 1102 1103 responseData.contentText = cast(string) responseData.content; 1104 1105 goto done; 1106 } 1107 } 1108 1109 done: 1110 // FIXME 1111 //if(closeSocketWhenComplete) 1112 //socket.close(); 1113 } else { 1114 //if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) 1115 // responseData.content ~= cast(ubyte[]) uncompress.uncompress(data); 1116 //else 1117 responseData.content ~= data; 1118 //assert(data.length <= bodyReadingState.contentLengthRemaining, format("%d <= %d\n%s", data.length, bodyReadingState.contentLengthRemaining, cast(string)data)); 1119 int use = cast(int) data.length; 1120 if(use > bodyReadingState.contentLengthRemaining) 1121 use = bodyReadingState.contentLengthRemaining; 1122 bodyReadingState.contentLengthRemaining -= use; 1123 data = data[use .. $]; 1124 if(bodyReadingState.contentLengthRemaining == 0) { 1125 if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) { 1126 auto n = uncompress.uncompress(responseData.content); 1127 n ~= uncompress.flush(); 1128 responseData.content = cast(ubyte[]) n; 1129 //responseData.content ~= cast(ubyte[]) uncompress.flush(); 1130 } 1131 if(followLocation && responseData.location.length) { 1132 static bool first = true; 1133 if(!first) asm { int 3; } 1134 populateFromInfo(Uri(responseData.location), HttpVerb.GET); 1135 import std.stdio; writeln("redirected to ", responseData.location); 1136 first = false; 1137 responseData = HttpResponse.init; 1138 headerReadingState = HeaderReadingState.init; 1139 bodyReadingState = BodyReadingState.init; 1140 state = State.unsent; 1141 stillAlive = false; 1142 sendPrivate(false); 1143 } else { 1144 state = State.complete; 1145 responseData.contentText = cast(string) responseData.content; 1146 // FIXME 1147 //if(closeSocketWhenComplete) 1148 //socket.close(); 1149 } 1150 } 1151 } 1152 } 1153 1154 if(data.length) 1155 leftoverDataFromLastTime = data.dup; 1156 else 1157 leftoverDataFromLastTime = null; 1158 1159 return stillAlive; 1160 } 1161 1162 this() { 1163 } 1164 1165 /// 1166 this(Uri where, HttpVerb method) { 1167 populateFromInfo(where, method); 1168 } 1169 1170 /// Final url after any redirections 1171 string finalUrl; 1172 1173 void populateFromInfo(Uri where, HttpVerb method) { 1174 auto parts = where; 1175 finalUrl = where.toString(); 1176 requestParameters.method = method; 1177 requestParameters.host = parts.host; 1178 requestParameters.port = cast(ushort) parts.port; 1179 requestParameters.ssl = parts.scheme == "https"; 1180 if(parts.port == 0) 1181 requestParameters.port = requestParameters.ssl ? 443 : 80; 1182 requestParameters.uri = parts.path.length ? parts.path : "/"; 1183 if(parts.query.length) { 1184 requestParameters.uri ~= "?"; 1185 requestParameters.uri ~= parts.query; 1186 } 1187 } 1188 1189 ~this() { 1190 } 1191 1192 ubyte[] sendBuffer; 1193 1194 HttpResponse responseData; 1195 private HttpClient parentClient; 1196 1197 size_t bodyBytesSent; 1198 size_t bodyBytesReceived; 1199 1200 State state_; 1201 State state() { return state_; } 1202 State state(State s) { 1203 assert(state_ != State.complete); 1204 return state_ = s; 1205 } 1206 /// Called when data is received. Check the state to see what data is available. 1207 void delegate(HttpRequest) onDataReceived; 1208 1209 enum State { 1210 /// The request has not yet been sent 1211 unsent, 1212 1213 /// The send() method has been called, but no data is 1214 /// sent on the socket yet because the connection is busy. 1215 pendingAvailableConnection, 1216 1217 /// The headers are being sent now 1218 sendingHeaders, 1219 1220 /// The body is being sent now 1221 sendingBody, 1222 1223 /// The request has been sent but we haven't received any response yet 1224 waitingForResponse, 1225 1226 /// We have received some data and are currently receiving headers 1227 readingHeaders, 1228 1229 /// All headers are available but we're still waiting on the body 1230 readingBody, 1231 1232 /// The request is complete. 1233 complete, 1234 1235 /// The request is aborted, either by the abort() method, or as a result of the server disconnecting 1236 aborted 1237 } 1238 1239 /// Sends now and waits for the request to finish, returning the response. 1240 HttpResponse perform() { 1241 send(); 1242 return waitForCompletion(); 1243 } 1244 1245 /// Sends the request asynchronously. 1246 void send() { 1247 sendPrivate(true); 1248 } 1249 1250 private void sendPrivate(bool advance) { 1251 if(state != State.unsent && state != State.aborted) 1252 return; // already sent 1253 string headers; 1254 1255 headers ~= to!string(requestParameters.method) ~ " "~requestParameters.uri; 1256 if(requestParameters.useHttp11) 1257 headers ~= " HTTP/1.1\r\n"; 1258 else 1259 headers ~= " HTTP/1.0\r\n"; 1260 headers ~= "Host: "~requestParameters.host~"\r\n"; 1261 if(requestParameters.userAgent.length) 1262 headers ~= "User-Agent: "~requestParameters.userAgent~"\r\n"; 1263 if(requestParameters.contentType.length) 1264 headers ~= "Content-Type: "~requestParameters.contentType~"\r\n"; 1265 if(requestParameters.authorization.length) 1266 headers ~= "Authorization: "~requestParameters.authorization~"\r\n"; 1267 if(requestParameters.bodyData.length) 1268 headers ~= "Content-Length: "~to!string(requestParameters.bodyData.length)~"\r\n"; 1269 if(requestParameters.acceptGzip) 1270 headers ~= "Accept-Encoding: gzip\r\n"; 1271 1272 foreach(header; requestParameters.headers) 1273 headers ~= header ~ "\r\n"; 1274 1275 headers ~= "\r\n"; 1276 1277 sendBuffer = cast(ubyte[]) headers ~ requestParameters.bodyData; 1278 1279 // import std.stdio; writeln("******* ", sendBuffer); 1280 1281 responseData = HttpResponse.init; 1282 responseData.requestParameters = requestParameters; 1283 bodyBytesSent = 0; 1284 bodyBytesReceived = 0; 1285 state = State.pendingAvailableConnection; 1286 1287 bool alreadyPending = false; 1288 foreach(req; pending) 1289 if(req is this) { 1290 alreadyPending = true; 1291 break; 1292 } 1293 if(!alreadyPending) { 1294 pending ~= this; 1295 } 1296 1297 if(advance) 1298 HttpRequest.advanceConnections(); 1299 } 1300 1301 1302 /// Waits for the request to finish or timeout, whichever comes first. 1303 HttpResponse waitForCompletion() { 1304 while(state != State.aborted && state != State.complete) { 1305 if(state == State.unsent) 1306 send(); 1307 if(auto err = HttpRequest.advanceConnections()) 1308 throw new Exception("waitForCompletion got err " ~ to!string(err)); 1309 } 1310 1311 return responseData; 1312 } 1313 1314 /// Aborts this request. 1315 void abort() { 1316 this.state = State.aborted; 1317 // FIXME 1318 } 1319 1320 HttpRequestParameters requestParameters; /// 1321 } 1322 1323 /// 1324 struct HttpRequestParameters { 1325 // Duration timeout; 1326 1327 // debugging 1328 bool useHttp11 = true; /// 1329 bool acceptGzip = true; /// 1330 1331 // the request itself 1332 HttpVerb method; /// 1333 string host; /// 1334 ushort port; /// 1335 string uri; /// 1336 1337 bool ssl; /// 1338 1339 string userAgent; /// 1340 string authorization; /// 1341 1342 string[string] cookies; /// 1343 1344 string[] headers; /// do not duplicate host, content-length, content-type, or any others that have a specific property 1345 1346 string contentType; /// 1347 ubyte[] bodyData; /// 1348 } 1349 1350 interface IHttpClient { 1351 1352 } 1353 1354 /// 1355 enum HttpVerb { 1356 /// 1357 GET, 1358 /// 1359 HEAD, 1360 /// 1361 POST, 1362 /// 1363 PUT, 1364 /// 1365 DELETE, 1366 /// 1367 OPTIONS, 1368 /// 1369 TRACE, 1370 /// 1371 CONNECT, 1372 /// 1373 PATCH, 1374 /// 1375 MERGE 1376 } 1377 1378 /** 1379 Usage: 1380 1381 auto client = new HttpClient("localhost", 80); 1382 // relative links work based on the current url 1383 client.get("foo/bar"); 1384 client.get("baz"); // gets foo/baz 1385 1386 auto request = client.get("rofl"); 1387 auto response = request.waitForCompletion(); 1388 */ 1389 1390 /// HttpClient keeps cookies, location, and some other state to reuse connections, when possible, like a web browser. 1391 class HttpClient { 1392 /* Protocol restrictions, useful to disable when debugging servers */ 1393 bool useHttp11 = true; /// 1394 bool acceptGzip = true; /// 1395 1396 /// 1397 @property Uri location() { 1398 return currentUrl; 1399 } 1400 1401 /// High level function that works similarly to entering a url 1402 /// into a browser. 1403 /// 1404 /// Follows locations, updates the current url. 1405 HttpRequest navigateTo(Uri where, HttpVerb method = HttpVerb.GET) { 1406 currentUrl = where.basedOn(currentUrl); 1407 currentDomain = where.host; 1408 auto request = new HttpRequest(currentUrl, method); 1409 1410 request.followLocation = true; 1411 1412 request.requestParameters.userAgent = userAgent; 1413 request.requestParameters.authorization = authorization; 1414 1415 request.requestParameters.useHttp11 = this.useHttp11; 1416 request.requestParameters.acceptGzip = this.acceptGzip; 1417 1418 return request; 1419 } 1420 1421 /++ 1422 Creates a request without updating the current url state 1423 (but will still save cookies btw) 1424 1425 +/ 1426 HttpRequest request(Uri uri, HttpVerb method = HttpVerb.GET, ubyte[] bodyData = null, string contentType = null) { 1427 auto request = new HttpRequest(uri, method); 1428 1429 request.requestParameters.userAgent = userAgent; 1430 request.requestParameters.authorization = authorization; 1431 1432 request.requestParameters.useHttp11 = this.useHttp11; 1433 request.requestParameters.acceptGzip = this.acceptGzip; 1434 1435 request.requestParameters.bodyData = bodyData; 1436 request.requestParameters.contentType = contentType; 1437 1438 return request; 1439 1440 } 1441 1442 /// ditto 1443 HttpRequest request(Uri uri, FormData fd, HttpVerb method = HttpVerb.POST) { 1444 return request(uri, method, fd.toBytes, fd.contentType); 1445 } 1446 1447 1448 private Uri currentUrl; 1449 private string currentDomain; 1450 1451 this(ICache cache = null) { 1452 1453 } 1454 1455 // FIXME: add proxy 1456 // FIXME: some kind of caching 1457 1458 /// 1459 void setCookie(string name, string value, string domain = null) { 1460 if(domain == null) 1461 domain = currentDomain; 1462 1463 cookies[domain][name] = value; 1464 } 1465 1466 /// 1467 void clearCookies(string domain = null) { 1468 if(domain is null) 1469 cookies = null; 1470 else 1471 cookies[domain] = null; 1472 } 1473 1474 // If you set these, they will be pre-filled on all requests made with this client 1475 string userAgent = "D arsd.html2"; /// 1476 string authorization; /// 1477 1478 /* inter-request state */ 1479 string[string][string] cookies; 1480 } 1481 1482 interface ICache { 1483 HttpResponse* getCachedResponse(HttpRequestParameters request); 1484 } 1485 1486 // / Provides caching behavior similar to a real web browser 1487 class HttpCache : ICache { 1488 HttpResponse* getCachedResponse(HttpRequestParameters request) { 1489 return null; 1490 } 1491 } 1492 1493 // / Gives simple maximum age caching, ignoring the actual http headers 1494 class SimpleCache : ICache { 1495 HttpResponse* getCachedResponse(HttpRequestParameters request) { 1496 return null; 1497 } 1498 } 1499 1500 /// 1501 struct HttpCookie { 1502 string name; /// 1503 string value; /// 1504 string domain; /// 1505 string path; /// 1506 //SysTime expirationDate; /// 1507 bool secure; /// 1508 bool httpOnly; /// 1509 } 1510 1511 // FIXME: websocket 1512 1513 version(testing) 1514 void main() { 1515 import std.stdio; 1516 auto client = new HttpClient(); 1517 auto request = client.navigateTo(Uri("http://localhost/chunked.php")); 1518 request.send(); 1519 auto request2 = client.navigateTo(Uri("http://dlang.org/")); 1520 request2.send(); 1521 1522 { 1523 auto response = request2.waitForCompletion(); 1524 //write(cast(string) response.content); 1525 } 1526 1527 auto response = request.waitForCompletion(); 1528 write(cast(string) response.content); 1529 1530 writeln(HttpRequest.socketsPerHost); 1531 } 1532 1533 1534 // From sslsocket.d 1535 version(use_openssl) { 1536 alias SslClientSocket = OpenSslSocket; 1537 1538 // macros in the original C 1539 version(newer_openssl) { 1540 void SSL_library_init() { 1541 OPENSSL_init_ssl(0, null); 1542 } 1543 void OpenSSL_add_all_ciphers() { 1544 OPENSSL_init_crypto(0 /*OPENSSL_INIT_ADD_ALL_CIPHERS*/, null); 1545 } 1546 void OpenSSL_add_all_digests() { 1547 OPENSSL_init_crypto(0 /*OPENSSL_INIT_ADD_ALL_DIGESTS*/, null); 1548 } 1549 1550 void SSL_load_error_strings() { 1551 OPENSSL_init_ssl(0x00200000L, null); 1552 } 1553 1554 SSL_METHOD* SSLv23_client_method() { 1555 return TLS_client_method(); 1556 } 1557 } 1558 1559 extern(C) { 1560 version(newer_openssl) {} else { 1561 int SSL_library_init(); 1562 void OpenSSL_add_all_ciphers(); 1563 void OpenSSL_add_all_digests(); 1564 void SSL_load_error_strings(); 1565 SSL_METHOD* SSLv23_client_method(); 1566 } 1567 void OPENSSL_init_ssl(ulong, void*); 1568 void OPENSSL_init_crypto(ulong, void*); 1569 1570 struct SSL {} 1571 struct SSL_CTX {} 1572 struct SSL_METHOD {} 1573 1574 SSL_CTX* SSL_CTX_new(const SSL_METHOD* method); 1575 SSL* SSL_new(SSL_CTX*); 1576 int SSL_set_fd(SSL*, int); 1577 int SSL_connect(SSL*); 1578 int SSL_write(SSL*, const void*, int); 1579 int SSL_read(SSL*, void*, int); 1580 @trusted nothrow @nogc int SSL_shutdown(SSL*); 1581 void SSL_free(SSL*); 1582 void SSL_CTX_free(SSL_CTX*); 1583 1584 int SSL_pending(const SSL*); 1585 1586 void SSL_set_verify(SSL*, int, void*); 1587 enum SSL_VERIFY_NONE = 0; 1588 1589 SSL_METHOD* SSLv3_client_method(); 1590 SSL_METHOD* TLS_client_method(); 1591 1592 void ERR_print_errors_fp(FILE*); 1593 } 1594 1595 import core.stdc.stdio; 1596 1597 shared static this() { 1598 SSL_library_init(); 1599 OpenSSL_add_all_ciphers(); 1600 OpenSSL_add_all_digests(); 1601 SSL_load_error_strings(); 1602 } 1603 1604 pragma(lib, "crypto"); 1605 pragma(lib, "ssl"); 1606 1607 class OpenSslSocket : Socket { 1608 private SSL* ssl; 1609 private SSL_CTX* ctx; 1610 private void initSsl(bool verifyPeer) { 1611 ctx = SSL_CTX_new(SSLv23_client_method()); 1612 assert(ctx !is null); 1613 1614 ssl = SSL_new(ctx); 1615 if(!verifyPeer) 1616 SSL_set_verify(ssl, SSL_VERIFY_NONE, null); 1617 SSL_set_fd(ssl, cast(int) this.handle); // on win64 it is necessary to truncate, but the value is never large anyway see http://openssl.6102.n7.nabble.com/Sockets-windows-64-bit-td36169.html 1618 } 1619 1620 bool dataPending() { 1621 return SSL_pending(ssl) > 0; 1622 } 1623 1624 @trusted 1625 override void connect(Address to) { 1626 super.connect(to); 1627 if(SSL_connect(ssl) == -1) { 1628 ERR_print_errors_fp(core.stdc.stdio.stderr); 1629 int i; 1630 printf("wtf\n"); 1631 scanf("%d\n", i); 1632 throw new Exception("ssl connect"); 1633 } 1634 } 1635 1636 @trusted 1637 override ptrdiff_t send(scope const(void)[] buf, SocketFlags flags) { 1638 //import std.stdio;writeln(cast(string) buf); 1639 auto retval = SSL_write(ssl, buf.ptr, cast(uint) buf.length); 1640 if(retval == -1) { 1641 ERR_print_errors_fp(core.stdc.stdio.stderr); 1642 int i; 1643 printf("wtf\n"); 1644 scanf("%d\n", i); 1645 throw new Exception("ssl send"); 1646 } 1647 return retval; 1648 1649 } 1650 override ptrdiff_t send(scope const(void)[] buf) { 1651 return send(buf, SocketFlags.NONE); 1652 } 1653 @trusted 1654 override ptrdiff_t receive(scope void[] buf, SocketFlags flags) { 1655 auto retval = SSL_read(ssl, buf.ptr, cast(int)buf.length); 1656 if(retval == -1) { 1657 ERR_print_errors_fp(core.stdc.stdio.stderr); 1658 int i; 1659 printf("wtf\n"); 1660 scanf("%d\n", i); 1661 throw new Exception("ssl send"); 1662 } 1663 return retval; 1664 } 1665 override ptrdiff_t receive(scope void[] buf) { 1666 return receive(buf, SocketFlags.NONE); 1667 } 1668 1669 this(AddressFamily af, SocketType type = SocketType.STREAM, bool verifyPeer = true) { 1670 super(af, type); 1671 initSsl(verifyPeer); 1672 } 1673 1674 override void close() { 1675 if(ssl) SSL_shutdown(ssl); 1676 super.close(); 1677 } 1678 1679 this(socket_t sock, AddressFamily af) { 1680 super(sock, af); 1681 initSsl(true); 1682 } 1683 1684 ~this() { 1685 SSL_free(ssl); 1686 SSL_CTX_free(ctx); 1687 ssl = null; 1688 } 1689 } 1690 } 1691 1692 /++ 1693 An experimental component for working with REST apis. Note that it 1694 is a zero-argument template, so to create one, use `new HttpApiClient!()(args..)` 1695 or you will get "HttpApiClient is used as a type" compile errors. 1696 1697 This will probably not work for you yet, and I might change it significantly. 1698 1699 Requires [arsd.jsvar]. 1700 1701 1702 Here's a snippet to create a pull request on GitHub to Phobos: 1703 1704 --- 1705 auto github = new HttpApiClient!()("https://api.github.com/", "your personal api token here"); 1706 1707 // create the arguments object 1708 // see: https://developer.github.com/v3/pulls/#create-a-pull-request 1709 var args = var.emptyObject; 1710 args.title = "My Pull Request"; 1711 args.head = "yourusername:" ~ branchName; 1712 args.base = "master"; 1713 // note it is ["body"] instead of .body because `body` is a D keyword 1714 args["body"] = "My cool PR is opened by the API!"; 1715 args.maintainer_can_modify = true; 1716 1717 // this translates to `repos/dlang/phobos/pulls` and sends a POST request, 1718 // containing `args` as json, then immediately grabs the json result and extracts 1719 // the value `html_url` from it. `prUrl` is typed `var`, from arsd.jsvar. 1720 auto prUrl = github.rest.repos.dlang.phobos.pulls.POST(args).result.html_url; 1721 1722 writeln("Created: ", prUrl); 1723 --- 1724 1725 Why use this instead of just building the URL? Well, of course you can! This just makes 1726 it a bit more convenient than string concatenation and manages a few headers for you. 1727 1728 Subtypes could potentially add static type checks too. 1729 +/ 1730 class HttpApiClient() { 1731 import arsd.jsvar; 1732 1733 HttpClient httpClient; 1734 1735 alias HttpApiClientType = typeof(this); 1736 1737 string urlBase; 1738 string oauth2Token; 1739 string submittedContentType; 1740 1741 /++ 1742 Params: 1743 1744 urlBase = The base url for the api. Tends to be something like `https://api.example.com/v2/` or similar. 1745 oauth2Token = the authorization token for the service. You'll have to get it from somewhere else. 1746 submittedContentType = the content-type of POST, PUT, etc. bodies. 1747 +/ 1748 this(string urlBase, string oauth2Token, string submittedContentType = "application/json") { 1749 httpClient = new HttpClient(); 1750 1751 assert(urlBase[0] == 'h'); 1752 assert(urlBase[$-1] == '/'); 1753 1754 this.urlBase = urlBase; 1755 this.oauth2Token = oauth2Token; 1756 this.submittedContentType = submittedContentType; 1757 } 1758 1759 /// 1760 static struct HttpRequestWrapper { 1761 HttpApiClientType apiClient; /// 1762 HttpRequest request; /// 1763 HttpResponse _response; 1764 1765 /// 1766 this(HttpApiClientType apiClient, HttpRequest request) { 1767 this.apiClient = apiClient; 1768 this.request = request; 1769 } 1770 1771 /// Returns the full [HttpResponse] object so you can inspect the headers 1772 @property HttpResponse response() { 1773 if(_response is HttpResponse.init) 1774 _response = request.waitForCompletion(); 1775 return _response; 1776 } 1777 1778 /++ 1779 Returns the parsed JSON from the body of the response. 1780 1781 Throws on non-2xx responses. 1782 +/ 1783 var result() { 1784 return apiClient.throwOnError(response); 1785 } 1786 1787 alias request this; 1788 } 1789 1790 /// 1791 HttpRequestWrapper request(string uri, HttpVerb requestMethod = HttpVerb.GET, ubyte[] bodyBytes = null) { 1792 if(uri[0] == '/') 1793 uri = uri[1 .. $]; 1794 1795 auto u = Uri(uri).basedOn(Uri(urlBase)); 1796 1797 auto req = httpClient.navigateTo(u, requestMethod); 1798 1799 if(oauth2Token.length) 1800 req.requestParameters.headers ~= "Authorization: Bearer " ~ oauth2Token; 1801 req.requestParameters.contentType = submittedContentType; 1802 req.requestParameters.bodyData = bodyBytes; 1803 1804 return HttpRequestWrapper(this, req); 1805 } 1806 1807 /// 1808 var throwOnError(HttpResponse res) { 1809 if(res.code < 200 || res.code >= 300) 1810 throw new Exception(res.codeText ~ " " ~ res.contentText); 1811 1812 var response = var.fromJson(res.contentText); 1813 if(response.errors) { 1814 throw new Exception(response.errors.toJson()); 1815 } 1816 1817 return response; 1818 } 1819 1820 /// 1821 @property RestBuilder rest() { 1822 return RestBuilder(this, null, null); 1823 } 1824 1825 // hipchat.rest.room["Tech Team"].history 1826 // gives: "/room/Tech%20Team/history" 1827 // 1828 // hipchat.rest.room["Tech Team"].history("page", "12) 1829 /// 1830 static struct RestBuilder { 1831 HttpApiClientType apiClient; 1832 string[] pathParts; 1833 string[2][] queryParts; 1834 this(HttpApiClientType apiClient, string[] pathParts, string[2][] queryParts) { 1835 this.apiClient = apiClient; 1836 this.pathParts = pathParts; 1837 this.queryParts = queryParts; 1838 } 1839 1840 RestBuilder _SELF() { 1841 return this; 1842 } 1843 1844 /// The args are so you can call opCall on the returned 1845 /// object, despite @property being broken af in D. 1846 RestBuilder opDispatch(string str, T)(string n, T v) { 1847 return RestBuilder(apiClient, pathParts ~ str, queryParts ~ [n, to!string(v)]); 1848 } 1849 1850 /// 1851 RestBuilder opDispatch(string str)() { 1852 return RestBuilder(apiClient, pathParts ~ str, queryParts); 1853 } 1854 1855 1856 /// 1857 RestBuilder opIndex(string str) { 1858 return RestBuilder(apiClient, pathParts ~ str, queryParts); 1859 } 1860 /// 1861 RestBuilder opIndex(var str) { 1862 return RestBuilder(apiClient, pathParts ~ str.get!string, queryParts); 1863 } 1864 /// 1865 RestBuilder opIndex(int i) { 1866 return RestBuilder(apiClient, pathParts ~ to!string(i), queryParts); 1867 } 1868 1869 /// 1870 RestBuilder opCall(T)(string name, T value) { 1871 return RestBuilder(apiClient, pathParts, queryParts ~ [name, to!string(value)]); 1872 } 1873 1874 /// 1875 string toUri() { 1876 import std.uri; 1877 string result; 1878 foreach(idx, part; pathParts) { 1879 if(idx) 1880 result ~= "/"; 1881 result ~= encodeComponent(part); 1882 } 1883 result ~= "?"; 1884 foreach(idx, part; queryParts) { 1885 if(idx) 1886 result ~= "&"; 1887 result ~= encodeComponent(part[0]); 1888 result ~= "="; 1889 result ~= encodeComponent(part[1]); 1890 } 1891 1892 return result; 1893 } 1894 1895 /// 1896 final HttpRequestWrapper GET() { return _EXECUTE(HttpVerb.GET, this.toUri(), ToBytesResult.init); } 1897 /// ditto 1898 final HttpRequestWrapper DELETE() { return _EXECUTE(HttpVerb.DELETE, this.toUri(), ToBytesResult.init); } 1899 1900 // need to be able to send: JSON, urlencoded, multipart/form-data, and raw stuff. 1901 /// ditto 1902 final HttpRequestWrapper POST(T...)(T t) { return _EXECUTE(HttpVerb.POST, this.toUri(), toBytes(t)); } 1903 /// ditto 1904 final HttpRequestWrapper PATCH(T...)(T t) { return _EXECUTE(HttpVerb.PATCH, this.toUri(), toBytes(t)); } 1905 /// ditto 1906 final HttpRequestWrapper PUT(T...)(T t) { return _EXECUTE(HttpVerb.PUT, this.toUri(), toBytes(t)); } 1907 1908 struct ToBytesResult { 1909 ubyte[] bytes; 1910 string contentType; 1911 } 1912 1913 private ToBytesResult toBytes(T...)(T t) { 1914 import std.conv : to; 1915 static if(T.length == 0) 1916 return ToBytesResult(null, null); 1917 else static if(T.length == 1 && is(T[0] == var)) 1918 return ToBytesResult(cast(ubyte[]) t[0].toJson(), "application/json"); // json data 1919 else static if(T.length == 1 && (is(T[0] == string) || is(T[0] == ubyte[]))) 1920 return ToBytesResult(cast(ubyte[]) t[0], null); // raw data 1921 else static if(T.length == 1 && is(T[0] : FormData)) 1922 return ToBytesResult(t[0].toBytes, t[0].contentType); 1923 else static if(T.length > 1 && T.length % 2 == 0 && is(T[0] == string)) { 1924 // string -> value pairs for a POST request 1925 string answer; 1926 foreach(idx, val; t) { 1927 static if(idx % 2 == 0) { 1928 if(answer.length) 1929 answer ~= "&"; 1930 answer ~= encodeComponent(val); // it had better be a string! lol 1931 answer ~= "="; 1932 } else { 1933 answer ~= encodeComponent(to!string(val)); 1934 } 1935 } 1936 1937 return ToBytesResult(cast(ubyte[]) answer, "application/x-www-form-urlencoded"); 1938 } 1939 else 1940 static assert(0); // FIXME 1941 1942 } 1943 1944 HttpRequestWrapper _EXECUTE(HttpVerb verb, string uri, ubyte[] bodyBytes) { 1945 return apiClient.request(uri, verb, bodyBytes); 1946 } 1947 1948 HttpRequestWrapper _EXECUTE(HttpVerb verb, string uri, ToBytesResult tbr) { 1949 auto r = apiClient.request(uri, verb, tbr.bytes); 1950 if(tbr.contentType !is null) 1951 r.requestParameters.contentType = tbr.contentType; 1952 return r; 1953 } 1954 } 1955 } 1956 1957 1958 // see also: arsd.cgi.encodeVariables 1959 /// Creates a multipart/form-data object that is suitable for file uploads and other kinds of POST 1960 class FormData { 1961 struct MimePart { 1962 string name; 1963 const(void)[] data; 1964 string contentType; 1965 string filename; 1966 } 1967 1968 MimePart[] parts; 1969 1970 /// 1971 void append(string key, in void[] value, string contentType = null, string filename = null) { 1972 parts ~= MimePart(key, value, contentType, filename); 1973 } 1974 1975 private string boundary = "0016e64be86203dd36047610926a"; // FIXME 1976 1977 string contentType() { 1978 return "multipart/form-data; boundary=" ~ boundary; 1979 } 1980 1981 /// 1982 ubyte[] toBytes() { 1983 string data; 1984 1985 foreach(part; parts) { 1986 data ~= "--" ~ boundary ~ "\r\n"; 1987 data ~= "Content-Disposition: form-data; name=\""~part.name~"\""; 1988 if(part.filename !is null) 1989 data ~= "; filename=\""~part.filename~"\""; 1990 data ~= "\r\n"; 1991 if(part.contentType !is null) 1992 data ~= "Content-Type: " ~ part.contentType ~ "\r\n"; 1993 data ~= "\r\n"; 1994 1995 data ~= cast(string) part.data; 1996 1997 data ~= "\r\n"; 1998 } 1999 2000 data ~= "--" ~ boundary ~ "--\r\n"; 2001 2002 return cast(ubyte[]) data; 2003 } 2004 } 2005