1 /// My old curl wrapper. Use [arsd.http2] instead on newer projects, or [std.net.curl] in Phobos if you don't trust my homemade implementations :) 2 module arsd.curl; 3 4 // see this for info on making a curl.lib on windows: 5 // http://stackoverflow.com/questions/7933845/where-is-curl-lib-for-dmd 6 7 pragma(lib, "curl"); 8 9 import std.string; 10 extern(C) { 11 struct CURL; 12 struct curl_slist; 13 14 alias int CURLcode; 15 alias int CURLoption; 16 17 enum int CURLOPT_URL = 10002; 18 enum int CURLOPT_WRITEFUNCTION = 20011; 19 enum int CURLOPT_WRITEDATA = 10001; 20 enum int CURLOPT_POSTFIELDS = 10015; 21 enum int CURLOPT_POSTFIELDSIZE = 60; 22 enum int CURLOPT_POST = 47; 23 enum int CURLOPT_HTTPHEADER = 10023; 24 enum int CURLOPT_USERPWD = 0x00002715; 25 26 enum int CURLOPT_VERBOSE = 41; 27 28 // enum int CURLOPT_COOKIE = 22; 29 enum int CURLOPT_COOKIEFILE = 10031; 30 enum int CURLOPT_COOKIEJAR = 10082; 31 32 enum int CURLOPT_SSL_VERIFYPEER = 64; 33 34 enum int CURLOPT_FOLLOWLOCATION = 52; 35 36 CURL* curl_easy_init(); 37 void curl_easy_cleanup(CURL* handle); 38 CURLcode curl_easy_perform(CURL* curl); 39 40 void curl_global_init(int flags); 41 42 enum int CURL_GLOBAL_ALL = 0b1111; 43 44 CURLcode curl_easy_setopt(CURL* handle, CURLoption option, ...); 45 curl_slist* curl_slist_append(curl_slist*, const char*); 46 void curl_slist_free_all(curl_slist*); 47 48 // size is size of item, count is how many items 49 size_t write_data(void* buffer, size_t size, size_t count, void* user) { 50 string* str = cast(string*) user; 51 char* data = cast(char*) buffer; 52 53 assert(size == 1); 54 55 *str ~= data[0..count]; 56 57 return count; 58 } 59 60 char* curl_easy_strerror(CURLcode errornum ); 61 } 62 /* 63 struct CurlOptions { 64 string username; 65 string password; 66 } 67 */ 68 69 string getDigestString(string s) { 70 import std.digest.md; 71 import std.digest; 72 auto hash = md5Of(s); 73 auto a = toHexString(hash); 74 return a.idup; 75 } 76 //import std.md5; 77 import std.file; 78 /// this automatically caches to a local file for the given time. it ignores the expires header in favor of your time to keep. 79 version(linux) 80 string cachedCurl(string url, int maxCacheHours) { 81 string res; 82 83 auto cacheFile = "/tmp/arsd-curl-cache-" ~ getDigestString(url); 84 85 import std.datetime; 86 87 if(!std.file.exists(cacheFile) || std.file.timeLastModified(cacheFile) < Clock.currTime() - dur!"hours"(maxCacheHours)) { 88 res = curl(url); 89 std.file.write(cacheFile, res); 90 } else { 91 res = cast(string) std.file.read(cacheFile); 92 } 93 94 return res; 95 } 96 97 98 string curl(string url, string data = null, string contentType = "application/x-www-form-urlencoded") { 99 return curlAuth(url, data, null, null, contentType); 100 } 101 102 string curlCookie(string cookieFile, string url, string data = null, string contentType = "application/x-www-form-urlencoded") { 103 return curlAuth(url, data, null, null, contentType, null, null, cookieFile); 104 } 105 106 string curlAuth(string url, string data = null, string username = null, string password = null, string contentType = "application/x-www-form-urlencoded", string methodOverride = null, string[] customHeaders = null, string cookieJar = null) { 107 CURL* curl = curl_easy_init(); 108 if(curl is null) 109 throw new Exception("curl init"); 110 scope(exit) 111 curl_easy_cleanup(curl); 112 113 string ret; 114 115 int res; 116 117 debug(arsd_curl_verbose) 118 curl_easy_setopt(curl, CURLOPT_VERBOSE, 1); 119 120 res = curl_easy_setopt(curl, CURLOPT_URL, std..string.toStringz(url)); 121 if(res != 0) throw new CurlException(res); 122 if(username !is null) { 123 res = curl_easy_setopt(curl, CURLOPT_USERPWD, std..string.toStringz(username ~ ":" ~ password)); 124 if(res != 0) throw new CurlException(res); 125 } 126 res = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_data); 127 if(res != 0) throw new CurlException(res); 128 res = curl_easy_setopt(curl, CURLOPT_WRITEDATA, &ret); 129 if(res != 0) throw new CurlException(res); 130 131 curl_slist* headers = null; 132 //if(data !is null) 133 // contentType = ""; 134 if(contentType.length) 135 headers = curl_slist_append(headers, toStringz("Content-Type: " ~ contentType)); 136 137 foreach(h; customHeaders) { 138 headers = curl_slist_append(headers, toStringz(h)); 139 } 140 scope(exit) 141 curl_slist_free_all(headers); 142 143 if(data) { 144 res = curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data.ptr); 145 if(res != 0) throw new CurlException(res); 146 res = curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, data.length); 147 if(res != 0) throw new CurlException(res); 148 } 149 150 res = curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); 151 if(res != 0) throw new CurlException(res); 152 153 if(cookieJar !is null) { 154 res = curl_easy_setopt(curl, CURLOPT_COOKIEJAR, toStringz(cookieJar)); 155 if(res != 0) throw new CurlException(res); 156 res = curl_easy_setopt(curl, CURLOPT_COOKIEFILE, toStringz(cookieJar)); 157 if(res != 0) throw new CurlException(res); 158 } else { 159 // just want to enable cookie parsing for location 3xx thingies. 160 // some crappy sites will give you an endless runaround if they can't 161 // place their fucking tracking cookies. 162 res = curl_easy_setopt(curl, CURLOPT_COOKIEFILE, toStringz("lol totally not here")); 163 } 164 165 res = curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); 166 if(res != 0) throw new CurlException(res); 167 //res = curl_easy_setopt(curl, 81, 0); // FIXME verify host 168 //if(res != 0) throw new CurlException(res); 169 170 version(no_curl_follow) {} else { 171 res = curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1); 172 if(res != 0) throw new CurlException(res); 173 } 174 175 if(methodOverride !is null) { 176 switch(methodOverride) { 177 default: assert(0); 178 case "POST": 179 res = curl_easy_setopt(curl, CURLOPT_POST, 1); 180 break; 181 case "GET": 182 //curl_easy_setopt(curl, CURLOPT_POST, 0); 183 break; 184 } 185 } 186 187 auto failure = curl_easy_perform(curl); 188 if(failure != 0) 189 throw new CurlException(failure, "\nURL" ~ url); 190 191 return ret; 192 } 193 194 class CurlException : Exception { 195 this(CURLcode code, string msg = null, string file = __FILE__, int line = __LINE__) { 196 string message = file ~ ":" ~ to!string(line) ~ " (" ~ to!string(code) ~ ") "; 197 198 auto strerror = curl_easy_strerror(code); 199 200 while(*strerror) { 201 message ~= *strerror; 202 strerror++; 203 } 204 205 super(message ~ msg); 206 } 207 } 208 209 210 import std.conv;