00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019 #include "httpClient.h"
00020 #include "httpChannel.h"
00021 #include "config_downloader.h"
00022 #include "filename.h"
00023 #include "config_express.h"
00024 #include "virtualFileSystem.h"
00025 #include "executionEnvironment.h"
00026 #include "httpBasicAuthorization.h"
00027 #include "httpDigestAuthorization.h"
00028
00029 #ifdef HAVE_SSL
00030
00031 #include <openssl/rand.h>
00032 #ifdef REPORT_OPENSSL_ERRORS
00033 #include <openssl/err.h>
00034 #endif
00035
00036
00037 #ifdef X509_NAME
00038 #undef X509_NAME
00039 #endif
00040
00041 bool HTTPClient::_ssl_initialized = false;
00042
00043
00044 X509_STORE *HTTPClient::_x509_store = NULL;
00045
00046
00047
00048
00049
00050
00051
00052 HTTPClient::
00053 HTTPClient() {
00054 _http_version = HTTPEnum::HV_11;
00055 _verify_ssl = verify_ssl ? VS_normal : VS_no_verify;
00056 _ssl_ctx = (SSL_CTX *)NULL;
00057
00058 _proxy = URLSpec(http_proxy, 1);
00059 if (!http_proxy_username.empty()) {
00060 set_username("*proxy", "", http_proxy_username);
00061 }
00062
00063 {
00064
00065 Config::ConfigTable::Symbol http_usernames;
00066 config_downloader.GetAll("http-username", http_usernames);
00067
00068
00069
00070 pset<string> already_read;
00071
00072 Config::ConfigTable::Symbol::iterator si;
00073 for (si = http_usernames.begin(); si != http_usernames.end(); ++si) {
00074 string http_username = (*si).Val();
00075 if (already_read.insert(http_username).second) {
00076 add_http_username(http_username);
00077 }
00078 }
00079 }
00080
00081
00082
00083 if (!_ssl_initialized) {
00084 initialize_ssl();
00085 }
00086 }
00087
00088
00089
00090
00091
00092
00093 HTTPClient::
00094 HTTPClient(const HTTPClient ©) {
00095 _ssl_ctx = (SSL_CTX *)NULL;
00096
00097 (*this) = copy;
00098 }
00099
00100
00101
00102
00103
00104
00105 void HTTPClient::
00106 operator = (const HTTPClient ©) {
00107 _proxy = copy._proxy;
00108 _http_version = copy._http_version;
00109 _verify_ssl = copy._verify_ssl;
00110 _usernames = copy._usernames;
00111 clear_expected_servers();
00112
00113 ExpectedServers::const_iterator ei;
00114 for (ei = copy._expected_servers.begin();
00115 ei != copy._expected_servers.end();
00116 ++ei) {
00117 X509_NAME *orig_name = (*ei);
00118 X509_NAME *new_name = X509_NAME_dup(orig_name);
00119 _expected_servers.push_back(new_name);
00120 }
00121 }
00122
00123
00124
00125
00126
00127
00128 HTTPClient::
00129 ~HTTPClient() {
00130
00131
00132
00133 if (_ssl_ctx != (SSL_CTX *)NULL) {
00134 nassertv(_ssl_ctx->cert_store == _x509_store);
00135 _ssl_ctx->cert_store = NULL;
00136 SSL_CTX_free(_ssl_ctx);
00137 }
00138
00139
00140 clear_expected_servers();
00141 }
00142
00143
00144
00145
00146
00147
00148
00149
00150
00151
00152
00153
00154
00155
00156
00157 void HTTPClient::
00158 set_username(const string &server, const string &realm, const string &username) {
00159 string key = server + ":" + realm;
00160 if (username.empty()) {
00161 _usernames.erase(key);
00162 } else {
00163 _usernames[key] = username;
00164 }
00165 }
00166
00167
00168
00169
00170
00171
00172
00173
00174 string HTTPClient::
00175 get_username(const string &server, const string &realm) const {
00176 string key = server + ":" + realm;
00177 Usernames::const_iterator ui;
00178 ui = _usernames.find(key);
00179 if (ui != _usernames.end()) {
00180 return (*ui).second;
00181 }
00182 return string();
00183 }
00184
00185
00186
00187
00188
00189
00190
00191 string HTTPClient::
00192 get_http_version_string() const {
00193 switch (_http_version) {
00194 case HTTPEnum::HV_09:
00195 return "HTTP/0.9";
00196
00197 case HTTPEnum::HV_10:
00198 return "HTTP/1.0";
00199
00200 case HTTPEnum::HV_11:
00201 return "HTTP/1.1";
00202
00203 case HTTPEnum::HV_other:
00204
00205 return "HTTP/1.1";
00206 }
00207
00208 return "unknown";
00209 }
00210
00211
00212
00213
00214
00215
00216
00217
00218
00219 HTTPEnum::HTTPVersion HTTPClient::
00220 parse_http_version_string(const string &version) {
00221 if (version == "HTTP/1.0") {
00222 return HTTPEnum::HV_10;
00223 } else if (version == "HTTP/1.1") {
00224 return HTTPEnum::HV_11;
00225 } else if (version.substr(0, 6) == "HTTP/0") {
00226 return HTTPEnum::HV_09;
00227 } else {
00228 return HTTPEnum::HV_other;
00229 }
00230 }
00231
00232
00233
00234
00235
00236
00237
00238
00239
00240
00241 bool HTTPClient::
00242 load_certificates(const Filename &filename) {
00243 int result;
00244
00245 if (use_vfs) {
00246 result = load_verify_locations(_ssl_ctx, filename);
00247
00248 } else {
00249 string os_specific = filename.to_os_specific();
00250 result =
00251 SSL_CTX_load_verify_locations(_ssl_ctx, os_specific.c_str(), NULL);
00252 }
00253
00254 if (result <= 0) {
00255 downloader_cat.info()
00256 << "Could not load certificates from " << filename << ".\n";
00257 #ifdef REPORT_OPENSSL_ERRORS
00258 ERR_print_errors_fp(stderr);
00259 #endif
00260 return false;
00261 }
00262
00263 downloader_cat.info()
00264 << "Appending " << result << " SSL certificates from "
00265 << filename << "\n";
00266
00267 return true;
00268 }
00269
00270
00271
00272
00273
00274
00275
00276
00277
00278
00279
00280
00281
00282
00283
00284
00285
00286
00287
00288 bool HTTPClient::
00289 add_expected_server(const string &server_attributes) {
00290 X509_NAME *name = parse_x509_name(server_attributes);
00291 if (name == (X509_NAME *)NULL) {
00292 return false;
00293 }
00294
00295 _expected_servers.push_back(name);
00296 return true;
00297 }
00298
00299
00300
00301
00302
00303
00304
00305 void HTTPClient::
00306 clear_expected_servers() {
00307 for (ExpectedServers::iterator ei = _expected_servers.begin();
00308 ei != _expected_servers.end();
00309 ++ei) {
00310 X509_NAME *name = (*ei);
00311 X509_NAME_free(name);
00312 }
00313 _expected_servers.clear();
00314 }
00315
00316
00317
00318
00319
00320
00321
00322
00323
00324
00325
00326
00327
00328
00329
00330
00331
00332
00333
00334
00335
00336
00337 PT(HTTPChannel) HTTPClient::
00338 make_channel(bool persistent_connection) {
00339 PT(HTTPChannel) doc = new HTTPChannel(this);
00340 doc->set_persistent_connection(persistent_connection);
00341 return doc;
00342 }
00343
00344
00345
00346
00347
00348
00349
00350
00351
00352
00353 PT(HTTPChannel) HTTPClient::
00354 post_form(const URLSpec &url, const string &body) {
00355 PT(HTTPChannel) doc = new HTTPChannel(this);
00356 doc->post_form(url, body);
00357 return doc;
00358 }
00359
00360
00361
00362
00363
00364
00365
00366
00367
00368
00369 PT(HTTPChannel) HTTPClient::
00370 get_document(const URLSpec &url) {
00371 PT(HTTPChannel) doc = new HTTPChannel(this);
00372 doc->get_document(url);
00373 return doc;
00374 }
00375
00376
00377
00378
00379
00380
00381
00382
00383
00384
00385 PT(HTTPChannel) HTTPClient::
00386 get_header(const URLSpec &url) {
00387 PT(HTTPChannel) doc = new HTTPChannel(this);
00388 doc->get_header(url);
00389 return doc;
00390 }
00391
00392
00393
00394
00395
00396
00397
00398
00399 SSL_CTX *HTTPClient::
00400 get_ssl_ctx() {
00401 if (_ssl_ctx != (SSL_CTX *)NULL) {
00402 return _ssl_ctx;
00403 }
00404
00405 _ssl_ctx = SSL_CTX_new(SSLv23_client_method());
00406
00407 #if defined(SSL_097) && !defined(NDEBUG)
00408
00409
00410 if (downloader_cat.is_debug()) {
00411 SSL_CTX_set_msg_callback(_ssl_ctx, ssl_msg_callback);
00412 }
00413 #endif
00414
00415
00416 {
00417
00418 Config::ConfigTable::Symbol expected_servers;
00419 config_downloader.GetAll("expected-ssl-server", expected_servers);
00420
00421
00422
00423 pset<string> already_read;
00424
00425 Config::ConfigTable::Symbol::iterator si;
00426 for (si = expected_servers.begin(); si != expected_servers.end(); ++si) {
00427 string expected_server = (*si).Val();
00428 if (already_read.insert(expected_server).second) {
00429 add_expected_server(expected_server);
00430 }
00431 }
00432 }
00433
00434 if (_x509_store != (X509_STORE *)NULL) {
00435
00436
00437
00438
00439
00440
00441 SSL_CTX_set_cert_store(_ssl_ctx, _x509_store);
00442
00443 } else {
00444
00445
00446 _x509_store = X509_STORE_new();
00447 SSL_CTX_set_cert_store(_ssl_ctx, _x509_store);
00448
00449
00450 Config::ConfigTable::Symbol cert_files;
00451 config_downloader.GetAll("ssl-certificates", cert_files);
00452
00453
00454
00455 pset<string> already_read;
00456
00457 Config::ConfigTable::Symbol::iterator si;
00458 for (si = cert_files.begin(); si != cert_files.end(); ++si) {
00459 string cert_file = (*si).Val();
00460 if (already_read.insert(cert_file).second) {
00461 Filename filename = Filename::from_os_specific(ExecutionEnvironment::expand_string(cert_file));
00462 load_certificates(filename);
00463 }
00464 }
00465 }
00466
00467 return _ssl_ctx;
00468 }
00469
00470
00471
00472
00473
00474
00475
00476
00477
00478 void HTTPClient::
00479 add_http_username(const string &http_username) {
00480 size_t c1 = http_username.find(':');
00481 if (c1 != string::npos) {
00482 size_t c2 = http_username.find(':', c1 + 1);
00483 if (c2 != string::npos) {
00484 size_t c3 = http_username.find(':', c2 + 1);
00485 if (c3 != string::npos) {
00486 size_t c4 = http_username.find(':', c3 + 1);
00487 if (c4 != string::npos) {
00488
00489 downloader_cat.error()
00490 << "Invalid http-username " << http_username << "\n";
00491
00492 } else {
00493
00494 set_username(http_username.substr(0, c1),
00495 http_username.substr(c1 + 1, c2 - (c1 + 1)),
00496 http_username.substr(c2 + 1));
00497 }
00498
00499 } else {
00500
00501 set_username(string(),
00502 http_username.substr(0, c1),
00503 http_username.substr(c1 + 1));
00504 }
00505 } else {
00506
00507 set_username(string(), string(), http_username);
00508 }
00509 } else {
00510
00511 downloader_cat.error()
00512 << "Invalid http-username " << http_username << "\n";
00513 }
00514 }
00515
00516
00517
00518
00519
00520
00521
00522 string HTTPClient::
00523 select_username(const URLSpec &url, bool is_proxy, const string &realm) const {
00524 string username;
00525
00526
00527
00528
00529
00530 if (url.has_username() && !is_proxy) {
00531 username = url.get_username();
00532 }
00533
00534
00535 if (is_proxy) {
00536 if (username.empty()) {
00537
00538 username = get_username("*proxy", realm);
00539 }
00540 if (username.empty()) {
00541
00542 username = get_username("*proxy", string());
00543 }
00544 }
00545 if (username.empty()) {
00546
00547 username = get_username(url.get_server(), realm);
00548 }
00549 if (username.empty()) {
00550
00551 username = get_username(url.get_server(), string());
00552 }
00553 if (username.empty()) {
00554
00555 username = get_username(string(), realm);
00556 }
00557 if (username.empty()) {
00558
00559 username = get_username(string(), string());
00560 }
00561
00562 return username;
00563 }
00564
00565
00566
00567
00568
00569
00570
00571
00572 HTTPAuthorization *HTTPClient::
00573 select_auth(const URLSpec &url, bool is_proxy, const string &last_realm) {
00574 Domains &domains = is_proxy ? _proxy_domains : _www_domains;
00575 string canon = HTTPAuthorization::get_canonical_url(url).get_url();
00576
00577
00578
00579 Domains::const_iterator best_di = domains.end();
00580 size_t longest_length = 0;
00581 Domains::const_iterator di;
00582 for (di = domains.begin(); di != domains.end(); ++di) {
00583 const string &domain = (*di).first;
00584 size_t length = domain.length();
00585 if (domain == canon.substr(0, length)) {
00586
00587 if (length > longest_length) {
00588 best_di = di;
00589 longest_length = length;
00590 }
00591 }
00592 }
00593
00594 if (best_di != domains.end()) {
00595
00596 if (downloader_cat.is_spam()) {
00597 downloader_cat.spam()
00598 << "Choosing domain " << (*best_di).first << " for " << url << "\n";
00599 }
00600 const Realms &realms = (*best_di).second._realms;
00601
00602 Realms::const_iterator ri;
00603 ri = realms.find(last_realm);
00604 if (ri != realms.end()) {
00605 return (*ri).second;
00606 }
00607
00608 if (!realms.empty()) {
00609
00610 return (*realms.begin()).second;
00611 }
00612 }
00613
00614
00615 return NULL;
00616 }
00617
00618
00619
00620
00621
00622
00623
00624
00625
00626
00627
00628 PT(HTTPAuthorization) HTTPClient::
00629 generate_auth(const URLSpec &url, bool is_proxy, const string &challenge) {
00630 HTTPAuthorization::AuthenticationSchemes schemes;
00631 HTTPAuthorization::parse_authentication_schemes(schemes, challenge);
00632
00633 PT(HTTPAuthorization) auth;
00634 HTTPAuthorization::AuthenticationSchemes::iterator si;
00635
00636 si = schemes.find("digest");
00637 if (si != schemes.end()) {
00638 auth = new HTTPDigestAuthorization((*si).second, url, is_proxy);
00639 }
00640
00641 if (auth == (HTTPAuthorization *)NULL || !auth->is_valid()) {
00642 si = schemes.find("basic");
00643 if (si != schemes.end()) {
00644 auth = new HTTPBasicAuthorization((*si).second, url, is_proxy);
00645 }
00646 }
00647
00648 if (auth == (HTTPAuthorization *)NULL || !auth->is_valid()) {
00649 downloader_cat.warning()
00650 << "Don't know how to use any of the server's available authorization schemes:\n";
00651 for (si = schemes.begin(); si != schemes.end(); ++si) {
00652 downloader_cat.warning() << (*si).first << "\n";
00653 }
00654
00655 } else {
00656
00657
00658 Domains &domains = is_proxy ? _proxy_domains : _www_domains;
00659 const vector_string &domain = auth->get_domain();
00660 vector_string::const_iterator si;
00661 for (si = domain.begin(); si != domain.end(); ++si) {
00662 domains[(*si)]._realms[auth->get_realm()] = auth;
00663 }
00664 }
00665
00666 return auth;
00667 }
00668
00669
00670
00671
00672
00673
00674
00675 void HTTPClient::
00676 initialize_ssl() {
00677 #ifdef REPORT_OPENSSL_ERRORS
00678 ERR_load_crypto_strings();
00679 ERR_load_SSL_strings();
00680 #endif
00681 OpenSSL_add_all_algorithms();
00682
00683
00684
00685 if (early_random_seed) {
00686 RAND_status();
00687 }
00688
00689 _ssl_initialized = true;
00690 }
00691
00692
00693
00694
00695
00696
00697
00698
00699
00700
00701
00702
00703
00704 int HTTPClient::
00705 load_verify_locations(SSL_CTX *ctx, const Filename &ca_file) {
00706 VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
00707
00708
00709 string data;
00710 if (!vfs->read_file(ca_file, data)) {
00711
00712 downloader_cat.info()
00713 << "Could not read " << ca_file << ".\n";
00714 return 0;
00715 }
00716
00717 STACK_OF(X509_INFO) *inf;
00718
00719
00720
00721
00722 BIO *mbio = BIO_new_mem_buf((void *)data.data(), data.length());
00723
00724
00725
00726 ERR_clear_error();
00727 inf = PEM_X509_INFO_read_bio(mbio, NULL, NULL, NULL);
00728 BIO_free(mbio);
00729
00730 if (!inf) {
00731
00732 downloader_cat.info()
00733 << "PEM_X509_INFO_read_bio() returned NULL.\n";
00734 #ifdef REPORT_OPENSSL_ERRORS
00735 ERR_print_errors_fp(stderr);
00736 #endif
00737 return 0;
00738 }
00739
00740 if (downloader_cat.is_spam()) {
00741 downloader_cat.spam()
00742 << "PEM_X509_INFO_read_bio() found " << sk_X509_INFO_num(inf)
00743 << " entries.\n";
00744 }
00745
00746
00747 X509_STORE *store = ctx->cert_store;
00748
00749 int count = 0;
00750 int num_entries = sk_X509_INFO_num(inf);
00751 for (int i = 0; i < num_entries; i++) {
00752 X509_INFO *itmp = sk_X509_INFO_value(inf, i);
00753
00754 if (itmp->x509) {
00755 X509_STORE_add_cert(store, itmp->x509);
00756 count++;
00757 if (downloader_cat.is_spam()) {
00758 downloader_cat.spam()
00759 << "Entry " << i << " is x509\n";
00760 }
00761
00762 } else if (itmp->crl) {
00763 X509_STORE_add_crl(store, itmp->crl);
00764 count++;
00765 if (downloader_cat.is_spam()) {
00766 downloader_cat.spam()
00767 << "Entry " << i << " is crl\n";
00768 }
00769
00770 } else {
00771 if (downloader_cat.is_spam()) {
00772 downloader_cat.spam()
00773 << "Entry " << i << " is unknown type\n";
00774 }
00775 }
00776 }
00777 sk_X509_INFO_pop_free(inf, X509_INFO_free);
00778
00779 return count;
00780 }
00781
00782
00783
00784
00785
00786
00787
00788
00789
00790 X509_NAME *HTTPClient::
00791 parse_x509_name(const string &source) {
00792 X509_NAME *result = NULL;
00793
00794 result = X509_NAME_new();
00795 bool added_any = false;
00796
00797 string::const_iterator si;
00798 si = source.begin();
00799 while (si != source.end()) {
00800 if ((*si) == '/') {
00801
00802 ++si;
00803 } else {
00804 string type;
00805 while (si != source.end() && (*si) != '=' && (*si) != '/') {
00806 if ((*si) == '\\') {
00807 ++si;
00808 if (si != source.end()) {
00809 type += (*si);
00810 ++si;
00811 }
00812 } else {
00813 type += (*si);
00814 ++si;
00815 }
00816 }
00817
00818 int nid = OBJ_txt2nid((char *)type.c_str());
00819 if (nid == NID_undef) {
00820 downloader_cat.info()
00821 << "Unknown type " << type << " in X509 name: " << source
00822 << "\n";
00823 X509_NAME_free(result);
00824 return NULL;
00825 }
00826
00827 string value;
00828
00829 if (si != source.end() && (*si) == '=') {
00830 ++si;
00831 while (si != source.end() && (*si) != '/') {
00832 if ((*si) == '\\') {
00833 ++si;
00834 if (si != source.end()) {
00835 value += (*si);
00836 ++si;
00837 }
00838 } else {
00839 value += (*si);
00840 ++si;
00841 }
00842 }
00843 }
00844
00845 if (!value.empty()) {
00846 int add_result =
00847 X509_NAME_add_entry_by_NID(result, nid, V_ASN1_APP_CHOOSE,
00848 (unsigned char *)value.c_str(), -1, -1, 0);
00849 if (!add_result) {
00850 downloader_cat.info()
00851 << "Unable to add " << type << "=" << value << " in X509 name: "
00852 << source << "\n";
00853 X509_NAME_free(result);
00854 return NULL;
00855 }
00856 added_any = true;
00857 }
00858 }
00859 }
00860
00861 if (!added_any) {
00862 downloader_cat.info()
00863 << "Invalid empty X509 name: " << source << "\n";
00864 X509_NAME_free(result);
00865 return NULL;
00866 }
00867
00868 return result;
00869 }
00870
00871 #if defined(SSL_097) && !defined(NDEBUG)
00872
00873
00874
00875
00876
00877
00878 void HTTPClient::
00879 ssl_msg_callback(int write_p, int version, int content_type,
00880 const void *, size_t len, SSL *, void *) {
00881 ostringstream describe;
00882 if (write_p) {
00883 describe << "sent ";
00884 } else {
00885 describe << "received ";
00886 }
00887 switch (version) {
00888 case SSL2_VERSION:
00889 describe << "SSL 2.0 ";
00890 break;
00891
00892 case SSL3_VERSION:
00893 describe << "SSL 3.0 ";
00894 break;
00895
00896 case TLS1_VERSION:
00897 describe << "TLS 1.0 ";
00898 break;
00899
00900 default:
00901 describe << "unknown protocol ";
00902 }
00903
00904 describe << "message: ";
00905
00906 if (version != SSL2_VERSION) {
00907 switch (content_type) {
00908 case 20:
00909 describe << "change cipher spec, ";
00910 break;
00911
00912 case 21:
00913 describe << "alert, ";
00914 break;
00915
00916 case 22:
00917 describe << "handshake, ";
00918 break;
00919
00920 case 23:
00921 describe << "application data, ";
00922 break;
00923
00924 default:
00925 describe << "unknown content type, ";
00926 }
00927 }
00928
00929 describe << len << " bytes.\n";
00930
00931 downloader_cat.debug() << describe.str();
00932 }
00933 #endif // defined(SSL_097) && !defined(NDEBUG)
00934
00935 #endif // HAVE_SSL