1 /++ 2 OBSOLETE: Old version of my http implementation. Do not use this, instead use [arsd.http2]. 3 4 I no longer work on this, use http2.d instead. 5 +/ 6 /*deprecated*/ module arsd.http; // adrdox apparently loses the comment above with deprecated, i need to fix that over there. 7 8 import std.socket; 9 10 // FIXME: check Transfer-Encoding: gzip always 11 12 version(with_openssl) { 13 pragma(lib, "crypto"); 14 pragma(lib, "ssl"); 15 } 16 17 ubyte[] getBinary(string url, string[string] cookies = null) { 18 auto hr = httpRequest("GET", url, null, cookies); 19 if(hr.code != 200) 20 throw new Exception(format("HTTP answered %d instead of 200 on %s", hr.code, url)); 21 return hr.content; 22 } 23 24 /** 25 Gets a textual document, ignoring headers. Throws on non-text or error. 26 */ 27 string get(string url, string[string] cookies = null) { 28 auto hr = httpRequest("GET", url, null, cookies); 29 if(hr.code != 200) 30 throw new Exception(format("HTTP answered %d instead of 200 on %s", hr.code, url)); 31 if(hr.contentType.indexOf("text/") == -1) 32 throw new Exception(hr.contentType ~ " is bad content for conversion to string"); 33 return cast(string) hr.content; 34 35 } 36 37 static import std.uri; 38 39 string post(string url, string[string] args, string[string] cookies = null) { 40 string content; 41 42 foreach(name, arg; args) { 43 if(content.length) 44 content ~= "&"; 45 content ~= std.uri.encode(name) ~ "=" ~ std.uri.encode(arg); 46 } 47 48 auto hr = httpRequest("POST", url, cast(ubyte[]) content, cookies, ["Content-Type: application/x-www-form-urlencoded"]); 49 if(hr.code != 200) 50 throw new Exception(format("HTTP answered %d instead of 200", hr.code)); 51 if(hr.contentType.indexOf("text/") == -1) 52 throw new Exception(hr.contentType ~ " is bad content for conversion to string"); 53 54 return cast(string) hr.content; 55 } 56 57 struct HttpResponse { 58 int code; 59 string contentType; 60 string[string] cookies; 61 string[] headers; 62 ubyte[] content; 63 } 64 65 import std.string; 66 static import std.algorithm; 67 import std.conv; 68 69 struct UriParts { 70 string original; 71 string method; 72 string host; 73 ushort port; 74 string path; 75 76 bool useHttps; 77 78 this(string uri) { 79 original = uri; 80 81 if(uri[0 .. 8] == "https://") 82 useHttps = true; 83 else 84 if(uri[0..7] != "http://") 85 throw new Exception("You must use an absolute, http or https URL."); 86 87 version(with_openssl) {} else 88 if(useHttps) 89 throw new Exception("openssl support not compiled in try -version=with_openssl"); 90 91 int start = useHttps ? 8 : 7; 92 93 auto posSlash = uri[start..$].indexOf("/"); 94 if(posSlash != -1) 95 posSlash += start; 96 97 if(posSlash == -1) 98 posSlash = uri.length; 99 100 auto posColon = uri[start..$].indexOf(":"); 101 if(posColon != -1) 102 posColon += start; 103 104 if(useHttps) 105 port = 443; 106 else 107 port = 80; 108 109 if(posColon != -1 && posColon < posSlash) { 110 host = uri[start..posColon]; 111 port = to!ushort(uri[posColon+1..posSlash]); 112 } else 113 host = uri[start..posSlash]; 114 115 path = uri[posSlash..$]; 116 if(path == "") 117 path = "/"; 118 } 119 } 120 121 HttpResponse httpRequest(string method, string uri, const(ubyte)[] content = null, string[string] cookies = null, string[] headers = null) { 122 import std.socket; 123 124 auto u = UriParts(uri); 125 // auto f = openNetwork(u.host, u.port); 126 auto f = new TcpSocket(); 127 f.connect(new InternetAddress(u.host, u.port)); 128 129 void delegate(string) write = (string d) { 130 f.send(d); 131 }; 132 133 char[4096] readBuffer; // rawRead actually blocks until it can fill up the whole buffer... which is broken as far as http goes so one char at a time i guess. slow lol 134 char[] delegate() read = () { 135 size_t num = f.receive(readBuffer); 136 return readBuffer[0..num]; 137 }; 138 139 version(with_openssl) { 140 import deimos.openssl.ssl; 141 SSL* ssl; 142 SSL_CTX* ctx; 143 if(u.useHttps) { 144 void sslAssert(bool ret){ 145 if (!ret){ 146 throw new Exception("SSL_ERROR"); 147 } 148 } 149 SSL_library_init(); 150 OpenSSL_add_all_algorithms(); 151 SSL_load_error_strings(); 152 153 ctx = SSL_CTX_new(SSLv3_client_method()); 154 sslAssert(!(ctx is null)); 155 156 ssl = SSL_new(ctx); 157 SSL_set_fd(ssl, f.handle); 158 sslAssert(SSL_connect(ssl) != -1); 159 160 write = (string d) { 161 SSL_write(ssl, d.ptr, cast(uint)d.length); 162 }; 163 164 read = () { 165 auto len = SSL_read(ssl, readBuffer.ptr, readBuffer.length); 166 return readBuffer[0 .. len]; 167 }; 168 } 169 } 170 171 172 HttpResponse response = doHttpRequestOnHelpers(write, read, method, uri, content, cookies, headers, u.useHttps); 173 174 version(with_openssl) { 175 if(u.useHttps) { 176 SSL_free(ssl); 177 SSL_CTX_free(ctx); 178 } 179 } 180 181 return response; 182 } 183 184 /** 185 Executes a generic http request, returning the full result. The correct formatting 186 of the parameters are the caller's responsibility. Content-Length is added automatically, 187 but YOU must give Content-Type! 188 */ 189 HttpResponse doHttpRequestOnHelpers(void delegate(string) write, char[] delegate() read, string method, string uri, const(ubyte)[] content = null, string[string] cookies = null, string[] headers = null, bool https = false) 190 in { 191 assert(method == "POST" || method == "GET"); 192 } 193 do { 194 auto u = UriParts(uri); 195 196 197 198 199 200 write(format("%s %s HTTP/1.1\r\n", method, u.path)); 201 write(format("Host: %s\r\n", u.host)); 202 write(format("Connection: close\r\n")); 203 if(content !is null) 204 write(format("Content-Length: %d\r\n", content.length)); 205 206 if(cookies !is null) { 207 string cookieHeader = "Cookie: "; 208 bool first = true; 209 foreach(k, v; cookies) { 210 if(first) 211 first = false; 212 else 213 cookieHeader ~= "; "; 214 cookieHeader ~= std.uri.encodeComponent(k) ~ "=" ~ std.uri.encodeComponent(v); 215 } 216 217 write(format("%s\r\n", cookieHeader)); 218 } 219 220 if(headers !is null) 221 foreach(header; headers) 222 write(format("%s\r\n", header)); 223 write("\r\n"); 224 if(content !is null) 225 write(cast(string) content); 226 227 228 string buffer; 229 230 string readln() { 231 auto idx = buffer.indexOf("\r\n"); 232 if(idx == -1) { 233 auto more = read(); 234 if(more.length == 0) { // end of file or something 235 auto ret = buffer; 236 buffer = null; 237 return ret; 238 } 239 buffer ~= more; 240 return readln(); 241 } 242 auto ret = buffer[0 .. idx + 2]; // + the \r\n 243 if(idx + 2 < buffer.length) 244 buffer = buffer[idx + 2 .. $]; 245 else 246 buffer = null; 247 return ret; 248 } 249 250 HttpResponse hr; 251 cont: 252 string l = readln(); 253 if(l[0..9] != "HTTP/1.1 ") 254 throw new Exception("Not talking to a http server"); 255 256 hr.code = to!int(l[9..12]); // HTTP/1.1 ### OK 257 258 if(hr.code == 100) { // continue 259 do { 260 l = readln(); 261 } while(l.length > 1); 262 263 goto cont; 264 } 265 266 bool chunked = false; 267 268 auto line = readln(); 269 while(line.length) { 270 if(line.strip.length == 0) 271 break; 272 hr.headers ~= line; 273 if(line.startsWith("Content-Type: ")) 274 hr.contentType = line[14..$-1]; 275 if(line.startsWith("Set-Cookie: ")) { 276 auto hdr = line["Set-Cookie: ".length .. $-1]; 277 auto semi = hdr.indexOf(";"); 278 if(semi != -1) 279 hdr = hdr[0 .. semi]; 280 281 auto equal = hdr.indexOf("="); 282 string name, value; 283 if(equal == -1) { 284 name = hdr; 285 // doesn't this mean erase the cookie? 286 } else { 287 name = hdr[0 .. equal]; 288 value = hdr[equal + 1 .. $]; 289 } 290 291 name = std.uri.decodeComponent(name); 292 value = std.uri.decodeComponent(value); 293 294 hr.cookies[name] = value; 295 } 296 if(line.startsWith("Transfer-Encoding: chunked")) 297 chunked = true; 298 line = readln(); 299 } 300 301 // there might be leftover stuff in the line buffer 302 ubyte[] response = cast(ubyte[]) buffer.dup; 303 auto part = read(); 304 while(part.length) { 305 response ~= part; 306 part = read(); 307 } 308 309 if(chunked) { 310 // read the hex length, stopping at a \r\n, ignoring everything between the new line but after the first non-valid hex character 311 // read binary data of that length. it is our content 312 // repeat until a zero sized chunk 313 // then read footers as headers. 314 315 int state = 0; 316 int size; 317 int start = 0; 318 for(int a = 0; a < response.length; a++) { 319 final switch(state) { 320 case 0: // reading hex 321 char c = response[a]; 322 if((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) { 323 // just keep reading 324 } else { 325 int power = 1; 326 size = 0; 327 for(int b = a-1; b >= start; b--) { 328 char cc = response[b]; 329 if(cc >= 'a' && cc <= 'z') 330 cc -= 0x20; 331 int val = 0; 332 if(cc >= '0' && cc <= '9') 333 val = cc - '0'; 334 else 335 val = cc - 'A' + 10; 336 337 size += power * val; 338 power *= 16; 339 } 340 state++; 341 continue; 342 } 343 break; 344 case 1: // reading until end of line 345 char c = response[a]; 346 if(c == '\n') { 347 if(size == 0) 348 state = 3; 349 else 350 state = 2; 351 } 352 break; 353 case 2: // reading data 354 hr.content ~= response[a..a+size]; 355 a += size; 356 a+= 1; // skipping a 13 10 357 start = a + 1; 358 state = 0; 359 break; 360 case 3: // reading footers 361 goto done; // FIXME 362 } 363 } 364 } else 365 hr.content = response; 366 done: 367 368 return hr; 369 } 370 371 372 /* 373 void main(string args[]) { 374 write(post("http://arsdnet.net/bugs.php", ["test" : "hey", "again" : "what"])); 375 } 376 */ 377 378 version(none): 379 380 struct Url { 381 string url; 382 } 383 384 struct BasicAuth { 385 string username; 386 string password; 387 } 388 389 /* 390 When you send something, it creates a request 391 and sends it asynchronously. The request object 392 393 auto request = new HttpRequest(); 394 // set any properties here 395 396 // synchronous usage 397 auto reply = request.perform(); 398 399 // async usage, type 1: 400 request.send(); 401 request2.send(); 402 403 // wait until the first one is done, with the second one still in-flight 404 auto response = request.waitForCompletion(); 405 406 407 // async usage, type 2: 408 request.onDataReceived = (HttpRequest hr) { 409 if(hr.state == HttpRequest.State.complete) { 410 // use hr.responseData 411 } 412 }; 413 request.send(); // send, using the callback 414 415 // before terminating, be sure you wait for your requests to finish! 416 417 request.waitForCompletion(); 418 419 */ 420 421 class HttpRequest { 422 private static { 423 // we manage the actual connections. When a request is made on a particular 424 // host, we try to reuse connections. We may open more than one connection per 425 // host to do parallel requests. 426 // 427 // The key is the *domain name*. Multiple domains on the same address will have separate connections. 428 Socket[][string] socketsPerHost; 429 430 // only one request can be active on a given socket (at least HTTP < 2.0) so this is that 431 HttpRequest[Socket] activeRequestOnSocket; 432 HttpRequest[] pending; // and these are the requests that are waiting 433 434 SocketSet readSet; 435 436 437 void advanceConnections() { 438 if(readSet is null) 439 readSet = new SocketSet(); 440 441 // are there pending requests? let's try to send them 442 443 readSet.reset(); 444 445 // active requests need to be read or written to 446 foreach(sock, request; activeRequestOnSocket) 447 readSet.add(sock); 448 449 // check the other sockets just for EOF, if they close, take them out of our list, 450 // we'll reopen if needed upon request. 451 452 auto got = Socket.select(readSet, writeSet, null, 10.seconds /* timeout */); 453 if(got == 0) /* timeout */ 454 {} 455 else 456 if(got == -1) /* interrupted */ 457 {} 458 else /* ready */ 459 {} 460 461 // call select(), do what needs to be done 462 // no requests are active, send the ones pending connection now 463 // we've completed a request, are there any more pending connection? if so, send them now 464 465 auto readSet = new SocketSet(); 466 } 467 } 468 469 this() { 470 addConnection(this); 471 } 472 473 ~this() { 474 removeConnection(this); 475 } 476 477 HttpResponse responseData; 478 HttpRequestParameters parameters; 479 private HttpClient parentClient; 480 481 size_t bodyBytesSent; 482 size_t bodyBytesReceived; 483 484 State state; 485 /// Called when data is received. Check the state to see what data is available. 486 void delegate(AsynchronousHttpRequest) onDataReceived; 487 488 enum State { 489 /// The request has not yet been sent 490 unsent, 491 492 /// The send() method has been called, but no data is 493 /// sent on the socket yet because the connection is busy. 494 pendingAvailableConnection, 495 496 /// The headers are being sent now 497 sendingHeaders, 498 499 /// The body is being sent now 500 sendingBody, 501 502 /// The request has been sent but we haven't received any response yet 503 waitingForResponse, 504 505 /// We have received some data and are currently receiving headers 506 readingHeaders, 507 508 /// All headers are available but we're still waiting on the body 509 readingBody, 510 511 /// The request is complete. 512 complete, 513 514 /// The request is aborted, either by the abort() method, or as a result of the server disconnecting 515 aborted 516 } 517 518 /// Sends now and waits for the request to finish, returning the response. 519 HttpResponse perform() { 520 send(); 521 return waitForCompletion(); 522 } 523 524 /// Sends the request asynchronously. 525 void send() { 526 if(state != State.unsent && state != State.aborted) 527 return; // already sent 528 529 responseData = HttpResponse.init; 530 bodyBytesSent = 0; 531 bodyBytesReceived = 0; 532 state = State.pendingAvailableConnection; 533 534 HttpResponse.advanceConnections(); 535 } 536 537 538 /// Waits for the request to finish or timeout, whichever comes furst. 539 HttpResponse waitForCompletion() { 540 while(state != State.aborted && state != State.complete) 541 HttpResponse.advanceConnections(); 542 return responseData; 543 } 544 545 /// Aborts this request. 546 /// Due to the nature of the HTTP protocol, aborting one request will result in all subsequent requests made on this same connection to be aborted as well. 547 void abort() { 548 parentClient.close(); 549 } 550 } 551 552 struct HttpRequestParameters { 553 Duration timeout; 554 555 // debugging 556 bool useHttp11 = true; 557 bool acceptGzip = true; 558 559 // the request itself 560 HttpVerb method; 561 string host; 562 string uri; 563 564 string userAgent; 565 566 string[string] cookies; 567 568 string[] headers; /// do not duplicate host, content-length, content-type, or any others that have a specific property 569 570 string contentType; 571 ubyte[] bodyData; 572 } 573 574 interface IHttpClient { 575 576 } 577 578 enum HttpVerb { GET, HEAD, POST, PUT, DELETE, OPTIONS, TRACE, CONNECT } 579 580 /* 581 Usage: 582 583 auto client = new HttpClient("localhost", 80); 584 // relative links work based on the current url 585 client.get("foo/bar"); 586 client.get("baz"); // gets foo/baz 587 588 auto request = client.get("rofl"); 589 auto response = request.waitForCompletion(); 590 */ 591 592 /// HttpClient keeps cookies, location, and some other state to reuse connections, when possible, like a web browser. 593 class HttpClient { 594 /* Protocol restrictions, useful to disable when debugging servers */ 595 bool useHttp11 = true; 596 bool useGzip = true; 597 598 /// Automatically follow a redirection? 599 bool followLocation = false; 600 601 @property Url location() { 602 return currentUrl; 603 } 604 605 /// High level function that works similarly to entering a url 606 /// into a browser. 607 /// 608 /// Follows locations, updates the current url. 609 AsynchronousHttpRequest navigateTo(Url where) { 610 currentUrl = where.basedOn(currentUrl); 611 assert(0); 612 } 613 614 private Url currentUrl; 615 616 this() { 617 618 } 619 620 this(Url url) { 621 open(url); 622 } 623 624 this(string host, ushort port = 80, bool useSsl = false) { 625 open(host, port); 626 } 627 628 // FIXME: add proxy 629 // FIXME: some kind of caching 630 631 void open(Url url) { 632 633 } 634 635 void open(string host, ushort port = 80, bool useSsl = false) { 636 637 } 638 639 void close() { 640 socket.close(); 641 } 642 643 void setCookie(string name, string value) { 644 645 } 646 647 void clearCookies() { 648 649 } 650 651 HttpResponse sendSynchronously() { 652 auto request = sendAsynchronously(); 653 return request.waitForCompletion(); 654 } 655 656 AsynchronousHttpRequest sendAsynchronously() { 657 658 } 659 660 string method; 661 string host; 662 ushort port; 663 string uri; 664 665 string[] headers; 666 ubyte[] requestBody; 667 668 string userAgent; 669 670 /* inter-request state */ 671 string[string] cookies; 672 } 673 674 // FIXME: websocket