view src/IO/http.c @ 4628:e5b39cfdeb0b

complete connect() before moving on to TLS handshake or sending query Johannes reported that mbed TLS didn't like the connect() to be ongoing still when the tls handshake was started on osx or dragonfly. For an earlier version of the fix (which failed), it was basically free to iterate through the IP address list upon connect() failure, so that was implemented, and now I'm artificially breaking this patch into two pieces to commit that separately.
author corvid
date Thu, 07 Jul 2016 14:04:58 +0000
parents cc89abc93e7c
children d4b595bd6fd8
line wrap: on
line source
/*
 * File: http.c
 *
 * Copyright (C) 2000-2007 Jorge Arellano Cid <jcid@dillo.org>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or
 * (at your option) any later version.
 */

/*
 * HTTP connect functions
 */


#include <config.h>

#include <ctype.h>              /* isdigit */
#include <unistd.h>
#include <errno.h>              /* for errno */
#include <stdlib.h>
#include <fcntl.h>
#include <assert.h>
#include <sys/socket.h>         /* for lots of socket stuff */
#include <netinet/in.h>         /* for ntohl and stuff */
#include <arpa/inet.h>          /* for inet_ntop */

#include "IO.h"
#include "iowatch.hh"
#include "tls.h"
#include "Url.h"
#include "../msg.h"
#include "../klist.h"
#include "../dns.h"
#include "../web.hh"
#include "../cookies.h"
#include "../auth.h"
#include "../prefs.h"
#include "../misc.h"

#include "../uicmd.hh"

/* Used to send a message to the bw's status bar */
#define MSG_BW(web, root, ...)                                        \
D_STMT_START {                                                        \
   if (a_Web_valid((web)) && (!(root) || (web)->flags & WEB_RootUrl)) \
      a_UIcmd_set_msg((web)->bw, __VA_ARGS__);                        \
} D_STMT_END

#define _MSG_BW(web, root, ...)

static const int HTTP_SOCKET_USE_PROXY   = 0x1;
static const int HTTP_SOCKET_QUEUED      = 0x2;
static const int HTTP_SOCKET_TO_BE_FREED = 0x4;
static const int HTTP_SOCKET_TLS         = 0x8;
static const int HTTP_SOCKET_IOWATCH_ACTIVE = 0x10;

/* 'web' is just a reference (no need to deallocate it here). */
typedef struct {
   int SockFD;
   uint_t flags;
   DilloWeb *web;          /* reference to client's web structure */
   DilloUrl *url;
   Dlist *addr_list;       /* Holds the DNS answer */
   ChainLink *Info;        /* Used for CCC asynchronous operations */
   char *connected_to;     /* Used for per-server connection limit */
   uint_t connect_port;
   Dstr *https_proxy_reply;
} SocketData_t;

/* Data structures and functions to queue sockets that need to be
 * delayed due to the per host connection limit.
 */
typedef struct {
  char *host;
  uint_t port;
  bool_t https;

  int active_conns;
  int running_the_queue;
  Dlist *queue;
} Server_t;

typedef struct {
   int fd;
   int skey;
} FdMapEntry_t;

static void Http_socket_enqueue(Server_t *srv, SocketData_t* sock);
static Server_t *Http_server_get(const char *host, uint_t port, bool_t https);
static void Http_server_remove(Server_t *srv);
static void Http_connect_socket(ChainLink *Info);
static char *Http_get_connect_str(const DilloUrl *url);
static void Http_send_query(SocketData_t *S);
static void Http_socket_free(int SKey);

/*
 * Local data
 */
static Klist_t *ValidSocks = NULL; /* Active sockets list. It holds pointers to
                                    * SocketData_t structures. */
static DilloUrl *HTTP_Proxy = NULL;
static char *HTTP_Proxy_Auth_base64 = NULL;
static char *HTTP_Language_hdr = NULL;
static Dlist *servers;

/* TODO: If fd_map will stick around in its present form (FDs and SocketData_t)
 * then consider whether having both this and ValidSocks is necessary.
 */
static Dlist *fd_map;

/*
 * Initialize proxy vars and Accept-Language header
 */
int a_Http_init(void)
{
   char *env_proxy = getenv("http_proxy");

   HTTP_Language_hdr = prefs.http_language ?
      dStrconcat("Accept-Language: ", prefs.http_language, "\r\n", NULL) :
      dStrdup("");

   if (env_proxy && strlen(env_proxy))
      HTTP_Proxy = a_Url_new(env_proxy, NULL);
   if (!HTTP_Proxy && prefs.http_proxy)
      HTTP_Proxy = a_Url_dup(prefs.http_proxy);

/*  This allows for storing the proxy password in "user:passwd" format
 * in dillorc, but as this constitutes a security problem, it was disabled.
 *
   if (HTTP_Proxy && prefs.http_proxyuser && strchr(prefs.http_proxyuser, ':'))
      HTTP_Proxy_Auth_base64 = a_Misc_encode_base64(prefs.http_proxyuser);
 */

   servers = dList_new(5);
   fd_map = dList_new(20);

   return 0;
}

/*
 * Tell whether the proxy auth is already set (user:password)
 * Return: 1 Yes, 0 No
 */
int a_Http_proxy_auth(void)
{
   return (HTTP_Proxy_Auth_base64 ? 1 : 0);
}

/*
 * Activate entered proxy password for HTTP.
 */
void a_Http_set_proxy_passwd(const char *str)
{
   char *http_proxyauth = dStrconcat(prefs.http_proxyuser, ":", str, NULL);
   HTTP_Proxy_Auth_base64 = a_Misc_encode_base64(http_proxyauth);
   dFree(http_proxyauth);
}

/*
 * Create and init a new SocketData_t struct, insert into ValidSocks,
 * and return a primary key for it.
 */
static int Http_sock_new(void)
{
   SocketData_t *S = dNew0(SocketData_t, 1);
   S->SockFD = -1;
   return a_Klist_insert(&ValidSocks, S);
}

/*
 * Compare by FD.
 */
static int Http_fd_map_cmp(const void *v1, const void *v2)
{
   int fd = VOIDP2INT(v2);
   const FdMapEntry_t *e = v1;

   return (fd != e->fd);
}

static void Http_fd_map_add_entry(SocketData_t *sd)
{
   FdMapEntry_t *e = dNew0(FdMapEntry_t, 1);
   e->fd = sd->SockFD;
   e->skey = VOIDP2INT(sd->Info->LocalKey);

   if (dList_find_custom(fd_map, INT2VOIDP(e->fd), Http_fd_map_cmp)) {
      MSG_ERR("FD ENTRY ALREADY FOUND FOR %d\n", e->fd);
      assert(0);
   }

   dList_append(fd_map, e);
}

/*
 * Remove and free entry from fd_map.
 */
static void Http_fd_map_remove_entry(int fd)
{
   void *data = dList_find_custom(fd_map, INT2VOIDP(fd), Http_fd_map_cmp);

   if (data) {
      dList_remove_fast(fd_map, data);
      dFree(data);
   } else {
      MSG("FD ENTRY NOT FOUND FOR %d\n", fd);
   }
}

void a_Http_connect_done(int fd, bool_t success)
{
   SocketData_t *sd;
   FdMapEntry_t *fme = dList_find_custom(fd_map, INT2VOIDP(fd),
                                            Http_fd_map_cmp);

   if (fme && (sd = a_Klist_get_data(ValidSocks, fme->skey))) {
      ChainLink *info = sd->Info;
      bool_t valid_web = a_Web_valid(sd->web);

      if (success && valid_web) {
         a_Chain_bfcb(OpSend, info, &sd->SockFD, "FD");
         Http_send_query(sd);
      } else {
         if (valid_web)
            MSG_BW(sd->web, 1, "Could not establish connection.");
         MSG("fd %d is done and failed\n", sd->SockFD);
         dClose(fd);
         Http_socket_free(VOIDP2INT(info->LocalKey)); /* free sd */
         a_Chain_bfcb(OpAbort, info, NULL, "Both");
         dFree(info);
      }
   } else {
      MSG("**** but no luck with fme %p or sd\n", fme);
   }
}

static void Http_socket_activate(Server_t *srv, SocketData_t *sd)
{
   dList_remove(srv->queue, sd);
   sd->flags &= ~HTTP_SOCKET_QUEUED;
   srv->active_conns++;
   sd->connected_to = srv->host;
}

static void Http_connect_queued_sockets(Server_t *srv)
{
   SocketData_t *sd;
   int i;

   srv->running_the_queue++;

   for (i = 0;
        (i < dList_length(srv->queue) &&
         srv->active_conns < prefs.http_max_conns);
        i++) {
      sd = dList_nth_data(srv->queue, i);

      if (sd->flags & HTTP_SOCKET_TO_BE_FREED) {
         dList_remove(srv->queue, sd);
         dFree(sd);
         i--;
      } else {
         int connect_ready = TLS_CONNECT_READY;

         if (sd->flags & HTTP_SOCKET_TLS)
            connect_ready = a_Tls_connect_ready(sd->url);

         if (connect_ready == TLS_CONNECT_NEVER || !a_Web_valid(sd->web)) {
            int SKey = VOIDP2INT(sd->Info->LocalKey);

            Http_socket_free(SKey);
         } else if (connect_ready == TLS_CONNECT_READY) {
            i--;
            Http_socket_activate(srv, sd);
            Http_connect_socket(sd->Info);
         }
      }
   }

   _MSG("Queue http%s://%s:%u len %d\n", srv->https ? "s" : "", srv->host,
        srv->port, dList_length(srv->queue));

   if (--srv->running_the_queue == 0) {
      if (srv->active_conns == 0)
         Http_server_remove(srv);
   }
}

/*
 * Free SocketData_t struct
 */
static void Http_socket_free(int SKey)
{
   SocketData_t *S;

   if ((S = a_Klist_get_data(ValidSocks, SKey))) {
      a_Klist_remove(ValidSocks, SKey);

      if (S->flags & HTTP_SOCKET_IOWATCH_ACTIVE) {
         S->flags &= ~HTTP_SOCKET_IOWATCH_ACTIVE;
         a_IOwatch_remove_fd(S->SockFD, -1);
      }
      dStr_free(S->https_proxy_reply, 1);

      if (S->flags & HTTP_SOCKET_QUEUED) {
         S->flags |= HTTP_SOCKET_TO_BE_FREED;
         a_Url_free(S->url);
      } else {
         if (S->SockFD != -1)
            Http_fd_map_remove_entry(S->SockFD);
         a_Tls_reset_server_state(S->url);
         if (S->connected_to) {
            a_Tls_close_by_fd(S->SockFD);

            Server_t *srv = Http_server_get(S->connected_to, S->connect_port,
                                            (S->flags & HTTP_SOCKET_TLS));
            srv->active_conns--;
            Http_connect_queued_sockets(srv);
         }
         a_Url_free(S->url);
         dFree(S);
      }
   }
}

/*
 * Make the HTTP header's Referer line according to preferences
 * (default is "host" i.e. "scheme://hostname/" )
 */
static char *Http_get_referer(const DilloUrl *url)
{
   char *referer = NULL;

   if (!strcmp(prefs.http_referer, "host")) {
      referer = dStrconcat("Referer: ", URL_SCHEME(url), "://",
                           URL_AUTHORITY(url), "/", "\r\n", NULL);
   } else if (!strcmp(prefs.http_referer, "path")) {
      referer = dStrconcat("Referer: ", URL_SCHEME(url), "://",
                           URL_AUTHORITY(url),
                           URL_PATH_(url) ? URL_PATH(url) : "/", "\r\n", NULL);
   }
   if (!referer)
      referer = dStrdup("");
   _MSG("http, referer='%s'\n", referer);
   return referer;
}

/*
 * Generate Content-Type header value for a POST query.
 */
static Dstr *Http_make_content_type(const DilloUrl *url)
{
   Dstr *dstr;

   if (URL_FLAGS(url) & URL_MultipartEnc) {
      _MSG("submitting multipart/form-data!\n");
      dstr = dStr_new("multipart/form-data; boundary=\"");
      if (URL_DATA(url)->len > 2) {
         /* boundary lines have "--" prepended. Skip that. */
         const char *start = URL_DATA(url)->str + 2;
         char *eol = strchr(start, '\r');
         if (eol)
            dStr_append_l(dstr, start, eol - start);
      } else {
         /* Zero parts; arbitrary boundary */
         dStr_append_c(dstr, '0');
      }
      dStr_append_c(dstr,'"');
   } else {
      dstr = dStr_new("application/x-www-form-urlencoded");
   }
   return dstr;
}

/*
 * Make the http query string
 */
static Dstr *Http_make_query_str(DilloWeb *web, bool_t use_proxy)
{
   char *ptr, *cookies, *referer, *auth;
   const DilloUrl *url = web->url;
   Dstr *query      = dStr_new(""),
        *request_uri = dStr_new(""),
        *proxy_auth = dStr_new("");

   /* BUG: dillo doesn't actually understand application/xml yet */
   const char *accept_hdr_value =
      web->flags & WEB_Image ? "image/png,image/*;q=0.8,*/*;q=0.5" :
      web->flags & WEB_Stylesheet ? "text/css,*/*;q=0.1" :
      "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";

   const char *connection_hdr_val =
      (prefs.http_persistent_conns == TRUE) ? "keep-alive" : "close";

   if (use_proxy) {
      dStr_sprintfa(request_uri, "%s%s",
                    URL_STR(url),
                    (URL_PATH_(url) || URL_QUERY_(url)) ? "" : "/");
      if ((ptr = strrchr(request_uri->str, '#')))
         dStr_truncate(request_uri, ptr - request_uri->str);
      if (HTTP_Proxy_Auth_base64)
         dStr_sprintf(proxy_auth, "Proxy-Authorization: Basic %s\r\n",
                      HTTP_Proxy_Auth_base64);
   } else {
      dStr_sprintfa(request_uri, "%s%s%s%s",
                    URL_PATH(url),
                    URL_QUERY_(url) ? "?" : "",
                    URL_QUERY(url),
                    (URL_PATH_(url) || URL_QUERY_(url)) ? "" : "/");
   }

   cookies = a_Cookies_get_query(url, web->requester);
   auth = a_Auth_get_auth_str(url, request_uri->str);
   referer = Http_get_referer(url);
   if (URL_FLAGS(url) & URL_Post) {
      Dstr *content_type = Http_make_content_type(url);
      dStr_sprintfa(
         query,
         "POST %s HTTP/1.1\r\n"
         "Host: %s\r\n"
         "User-Agent: %s\r\n"
         "Accept: %s\r\n"
         "%s" /* language */
         "Accept-Encoding: gzip, deflate\r\n"
         "%s" /* auth */
         "DNT: 1\r\n"
         "%s" /* proxy auth */
         "%s" /* referer */
         "Connection: %s\r\n"
         "Content-Type: %s\r\n"
         "Content-Length: %ld\r\n"
         "%s" /* cookies */
         "\r\n",
         request_uri->str, URL_AUTHORITY(url), prefs.http_user_agent,
         accept_hdr_value, HTTP_Language_hdr, auth ? auth : "",
         proxy_auth->str, referer, connection_hdr_val, content_type->str,
         (long)URL_DATA(url)->len, cookies);
      dStr_append_l(query, URL_DATA(url)->str, URL_DATA(url)->len);
      dStr_free(content_type, TRUE);
   } else {
      dStr_sprintfa(
         query,
         "GET %s HTTP/1.1\r\n"
         "Host: %s\r\n"
         "User-Agent: %s\r\n"
         "Accept: %s\r\n"
         "%s" /* language */
         "Accept-Encoding: gzip, deflate\r\n"
         "%s" /* auth */
         "DNT: 1\r\n"
         "%s" /* proxy auth */
         "%s" /* referer */
         "Connection: %s\r\n"
         "%s" /* cache control */
         "%s" /* cookies */
         "\r\n",
         request_uri->str, URL_AUTHORITY(url), prefs.http_user_agent,
         accept_hdr_value, HTTP_Language_hdr, auth ? auth : "",
         proxy_auth->str, referer, connection_hdr_val,
         (URL_FLAGS(url) & URL_E2EQuery) ?
            "Pragma: no-cache\r\nCache-Control: no-cache\r\n" : "",
         cookies);
   }
   dFree(referer);
   dFree(cookies);
   dFree(auth);

   dStr_free(request_uri, TRUE);
   dStr_free(proxy_auth, TRUE);
   _MSG("Query: {%s}\n", dStr_printable(query, 8192));
   return query;
}

/*
 * Create and submit the HTTP query to the IO engine
 */
static void Http_send_query(SocketData_t *S)
{
   Dstr *query;
   DataBuf *dbuf;

   /* Create the query */
   query = Http_make_query_str(S->web, S->flags & HTTP_SOCKET_USE_PROXY);
   dbuf = a_Chain_dbuf_new(query->str, query->len, 0);

   MSG_BW(S->web, 1, "Sending query...");

   /* send query */
   a_Chain_bcb(OpSend, S->Info, dbuf, NULL);
   dFree(dbuf);
   dStr_free(query, 1);
}

/*
 * Prepare an HTTPS connection.  If necessary, tunnel it through a proxy.
 * Then perform the TLS handshake.
 */
static void Http_connect_tls(ChainLink *info)
{
   int SKey = VOIDP2INT(info->LocalKey);
   SocketData_t *S = a_Klist_get_data(ValidSocks, SKey);

   if (S->flags & HTTP_SOCKET_USE_PROXY) {
      char *connect_str = Http_get_connect_str(S->url);
      DataBuf *dbuf = a_Chain_dbuf_new(connect_str, strlen(connect_str), 0);

      a_Chain_bfcb(OpSend, info, &S->SockFD, "FD");
      S->https_proxy_reply = dStr_new(NULL);
      a_Chain_bcb(OpSend, info, dbuf, NULL);

      dFree(dbuf);
      dFree(connect_str);
   } else {
      MSG_BW(S->web, 1, "TLS handshake...");
      a_Tls_handshake(S->SockFD, S->url);
   }
}

/*
 * connect() couldn't complete before, but now it's ready, so let's try again.
 */
static void Http_connect_socket_cb(int fd, void *data)
{
   int SKey = VOIDP2INT(data);
   SocketData_t *S = a_Klist_get_data(ValidSocks, SKey);

   if (S) {
      int ret, connect_ret;
      uint_t connect_ret_size = sizeof(connect_ret);

      a_IOwatch_remove_fd(fd, -1); 
      S->flags &= ~HTTP_SOCKET_IOWATCH_ACTIVE;

      ret = getsockopt(S->SockFD, SOL_SOCKET, SO_ERROR, &connect_ret,
                       &connect_ret_size);

      if (ret < 0 || connect_ret != 0) {
         if (ret < 0) {
            MSG("Http_connect_socket_cb getsockopt ERROR: %s.\n",
                dStrerror(errno));
         } else {
            MSG("Http_connect_socket_cb connect ERROR: %s.\n",
                dStrerror(connect_ret));
         }
         a_Http_connect_done(S->SockFD, FALSE);
      } else if (S->flags & HTTP_SOCKET_TLS) {
         Http_connect_tls(S->Info);
      } else {
         a_Http_connect_done(S->SockFD, TRUE);
      }
   }
}

/*
 * This function is called after the DNS succeeds in solving a hostname.
 * Task: Finish socket setup and start connecting the socket.
 */
static void Http_connect_socket(ChainLink *Info)
{
   int i;
   DilloHost *dh;
#ifdef ENABLE_IPV6
   struct sockaddr_in6 name;
#else
   struct sockaddr_in name;
#endif
   socklen_t socket_len = 0;
   SocketData_t *S = a_Klist_get_data(ValidSocks, VOIDP2INT(Info->LocalKey));

   /* TODO: iterate this address list until success, or end-of-list */
   for (i = 0; (dh = dList_nth_data(S->addr_list, i)); ++i) {
      if ((S->SockFD = socket(dh->af, SOCK_STREAM, IPPROTO_TCP)) < 0) {
         MSG("Http_connect_socket socket() ERROR: %s\n", dStrerror(errno));
         continue;
      }
      Http_fd_map_add_entry(S);

      /* set NONBLOCKING and close on exec. */
      fcntl(S->SockFD, F_SETFL, O_NONBLOCK | fcntl(S->SockFD, F_GETFL));
      fcntl(S->SockFD, F_SETFD, FD_CLOEXEC | fcntl(S->SockFD, F_GETFD));

      /* Some OSes require this...  */
      memset(&name, 0, sizeof(name));
      /* Set remaining parms. */
      switch (dh->af) {
      case AF_INET:
      {
         struct sockaddr_in *sin = (struct sockaddr_in *)&name;
         socket_len = sizeof(struct sockaddr_in);
         sin->sin_family = dh->af;
         sin->sin_port = htons(S->connect_port);
         memcpy(&sin->sin_addr, dh->data, (size_t)dh->alen);
         if (a_Web_valid(S->web) && (S->web->flags & WEB_RootUrl))
            MSG("Connecting to %s:%u\n", inet_ntoa(sin->sin_addr),
                S->connect_port);
         break;
      }
#ifdef ENABLE_IPV6
      case AF_INET6:
      {
         char buf[128];
         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&name;
         socket_len = sizeof(struct sockaddr_in6);
         sin6->sin6_family = dh->af;
         sin6->sin6_port = htons(S->connect_port);
         memcpy(&sin6->sin6_addr, dh->data, dh->alen);
         inet_ntop(dh->af, dh->data, buf, sizeof(buf));
         if (a_Web_valid(S->web) && (S->web->flags & WEB_RootUrl))
            MSG("Connecting to [%s]:%u\n", buf, S->connect_port);
         break;
      }
#endif
      } /* switch */
      MSG_BW(S->web, 1, "Contacting host...");

      if (connect(S->SockFD, (struct sockaddr *)&name, socket_len) == 0) {
         /* probably never succeeds immediately on any system */
         if (S->flags & HTTP_SOCKET_TLS) {
            Http_connect_tls(Info);
         } else {
            a_Http_connect_done(S->SockFD, TRUE);
         }
      } else {
         if (errno == EINPROGRESS) {
            a_IOwatch_add_fd(S->SockFD, DIO_WRITE, Http_connect_socket_cb,
                             Info->LocalKey);
            S->flags |= HTTP_SOCKET_IOWATCH_ACTIVE;
         } else {
            MSG("Http_connect_socket connect ERROR: %s\n", dStrerror(errno));
            a_Http_connect_done(S->SockFD, FALSE);
         }
      }
      return;
   }
}

/*
 * Test proxy settings and check the no_proxy domains list
 * Return value: whether to use proxy or not.
 */
static int Http_must_use_proxy(const char *hostname)
{
   char *np, *p, *tok;
   int ret = 0;

   if (HTTP_Proxy) {
      ret = 1;
      if (prefs.no_proxy) {
         size_t host_len = strlen(hostname);

         np = dStrdup(prefs.no_proxy);
         for (p = np; (tok = dStrsep(&p, " "));  ) {
            int start = host_len - strlen(tok);

            if (start >= 0 && dStrAsciiCasecmp(hostname + start, tok) == 0) {
               /* no_proxy token is suffix of host string */
               ret = 0;
               break;
            }
         }
         dFree(np);
      }
   }
   _MSG("Http_must_use_proxy: %s\n  %s\n", hostname, ret ? "YES":"NO");
   return ret;
}

/*
 * Return a new string for the request used to tunnel HTTPS through a proxy.
 */
static char *Http_get_connect_str(const DilloUrl *url)
{
   Dstr *dstr;
   const char *auth1;
   int auth_len;
   char *auth2, *proxy_auth, *retstr;

   dReturn_val_if_fail(Http_must_use_proxy(URL_HOST(url)), NULL);

   dstr = dStr_new("");
   auth1 = URL_AUTHORITY(url);
   auth_len = strlen(auth1);
   if (auth_len > 0 && !isdigit(auth1[auth_len - 1]))
      /* if no port number, add HTTPS port */
      auth2 = dStrconcat(auth1, ":443", NULL);
   else
      auth2 = dStrdup(auth1);
   proxy_auth = HTTP_Proxy_Auth_base64 ?
                   dStrconcat ("Proxy-Authorization: Basic ",
                               HTTP_Proxy_Auth_base64, "\r\n", NULL) :
                   dStrdup("");
   dStr_sprintfa(
      dstr,
      "CONNECT %s HTTP/1.1\r\n"
      "Host: %s\r\n"
      "%s"
      "\r\n",
      auth2,
      auth2,
      proxy_auth);

   dFree(auth2);
   dFree(proxy_auth);
   retstr = dstr->str;
   dStr_free(dstr, 0);
   return retstr;
}

/*
 * Callback function for the DNS resolver.
 * Continue connecting the socket, or abort upon error condition.
 * S->web is checked to assert the operation wasn't aborted while waiting.
 */
static void Http_dns_cb(int Status, Dlist *addr_list, void *data)
{
   int SKey = VOIDP2INT(data);
   bool_t clean_up = TRUE;
   SocketData_t *S;
   Server_t *srv;

   S = a_Klist_get_data(ValidSocks, SKey);
   if (S) {
      const char *host = URL_HOST((S->flags & HTTP_SOCKET_USE_PROXY) ?
                                  HTTP_Proxy : S->url);
      if (a_Web_valid(S->web)) {
         if (Status == 0 && addr_list) {

            /* Successful DNS answer; save the IP */
            S->addr_list = addr_list;
            clean_up = FALSE;
            srv = Http_server_get(host, S->connect_port,
                                 (S->flags & HTTP_SOCKET_TLS));
            Http_socket_enqueue(srv, S);
            Http_connect_queued_sockets(srv);
         } else {
            /* DNS wasn't able to resolve the hostname */
            MSG_BW(S->web, 0, "ERROR: DNS can't resolve %s", host);
         }
      }
      if (clean_up) {
         ChainLink *info = S->Info;

         Http_socket_free(SKey);
         a_Chain_bfcb(OpAbort, info, NULL, "Both");
         dFree(info);
      }
   }
}

/*
 * Asynchronously create a new http connection for 'Url'
 * We'll set some socket parameters; the rest will be set later
 * when the IP is known.
 * ( Data1 = Web structure )
 * Return value: 0 on success, -1 otherwise
 */
static int Http_get(ChainLink *Info, void *Data1)
{
   SocketData_t *S;
   char *hostname;
   const DilloUrl *url;

   S = a_Klist_get_data(ValidSocks, VOIDP2INT(Info->LocalKey));
   /* Reference Web data */
   S->web = Data1;
   /* Reference Info data */
   S->Info = Info;

   /* Proxy support */
   if (Http_must_use_proxy(URL_HOST(S->web->url))) {
      url = HTTP_Proxy;
      S->flags |= HTTP_SOCKET_USE_PROXY;
   } else {
      url = S->web->url;
   }
   hostname = dStrdup(URL_HOST(url));
   S->connect_port = URL_PORT(url);
   S->url = a_Url_dup(S->web->url);
   if (!dStrAsciiCasecmp(URL_SCHEME(S->url), "https"))
      S->flags |= HTTP_SOCKET_TLS;

   /* Let the user know what we'll do */
   MSG_BW(S->web, 1, "DNS resolving %s", hostname);

   /* Let the DNS engine resolve the hostname, and when done,
    * we'll try to connect the socket from the callback function */
   a_Dns_resolve(hostname, Http_dns_cb, Info->LocalKey);

   dFree(hostname);
   return 0;
}

/*
 * Can the old socket's fd be reused for the new socket?
 *
 * NOTE: old and new must come from the same Server_t.
 * This is not built to accept arbitrary sockets.
 */
static bool_t Http_socket_reuse_compatible(SocketData_t *old,
                                           SocketData_t *new)
{
   /*
    * If we are using TLS through a proxy, we need to ensure that old and new
    * are going through to the same host:port.
    */
   if (a_Web_valid(new->web) &&
       ((old->flags & HTTP_SOCKET_TLS) == 0 ||
        (old->flags & HTTP_SOCKET_USE_PROXY) == 0 ||
        ((URL_PORT(old->url) == URL_PORT(new->url)) &&
         !dStrAsciiCasecmp(URL_HOST(old->url), URL_HOST(new->url)))))
      return TRUE;
   return FALSE;
}

/*
 * If any entry in the socket data queue can reuse our connection, set it up
 * and send off a new query.
 */
static void Http_socket_reuse(int SKey)
{
   SocketData_t *new_sd, *old_sd = a_Klist_get_data(ValidSocks, SKey);

   if (old_sd) {
      Server_t *srv = Http_server_get(old_sd->connected_to,
                                      old_sd->connect_port,
                                      (old_sd->flags & HTTP_SOCKET_TLS));
      int i, n = dList_length(srv->queue);

      for (i = 0; i < n; i++) {
         new_sd = dList_nth_data(srv->queue, i);

         if (!(new_sd->flags & HTTP_SOCKET_TO_BE_FREED) &&
             Http_socket_reuse_compatible(old_sd, new_sd)) {
            const bool_t success = TRUE;

            new_sd->SockFD = old_sd->SockFD;

            old_sd->connected_to = NULL;
            srv->active_conns--;
            Http_socket_free(SKey);

            _MSG("Reusing fd %d for %s\n",new_sd->SockFD,URL_STR(new_sd->url));
            Http_socket_activate(srv, new_sd);
            Http_fd_map_add_entry(new_sd);
            a_Http_connect_done(new_sd->SockFD, success);
            return;
         }
      }
      dClose(old_sd->SockFD);
      Http_socket_free(SKey);
   }
}

/*
 * CCC function for the HTTP module
 */
void a_Http_ccc(int Op, int Branch, int Dir, ChainLink *Info,
                void *Data1, void *Data2)
{
   int SKey = VOIDP2INT(Info->LocalKey);
   SocketData_t *sd;
   DataBuf *dbuf;

   dReturn_if_fail( a_Chain_check("a_Http_ccc", Op, Branch, Dir, Info) );

   if (Branch == 1) {
      if (Dir == BCK) {
         /* HTTP query branch */
         switch (Op) {
         case OpStart:
            /* ( Data1 = Web ) */
            SKey = Http_sock_new();
            Info->LocalKey = INT2VOIDP(SKey);
            /* link IO */
            a_Chain_link_new(Info, a_Http_ccc, BCK, a_IO_ccc, 1, 1);
            a_Chain_bcb(OpStart, Info, NULL, NULL);
            /* async. connection */
            Http_get(Info, Data1);
            break;
         case OpEnd:
            /* finished the HTTP query branch */
            a_Chain_bcb(OpEnd, Info, NULL, NULL);
            dFree(Info);
            break;
         case OpAbort:
            _MSG("ABORT 1B\n");
            Http_socket_free(SKey);
            a_Chain_bcb(OpAbort, Info, NULL, NULL);
            dFree(Info);
            break;
         default:
            MSG_WARN("Unused CCC 1B Op %d\n", Op);
            break;
         }
      } else {  /* 1 FWD */
         /* HTTP send-query status branch */
         switch (Op) {
         case OpAbort:
            _MSG("ABORT 1F\n");
            if ((sd = a_Klist_get_data(ValidSocks, SKey)))
               MSG_BW(sd->web, 1, "Can't get %s", URL_STR(sd->url));
            Http_socket_free(SKey);
            a_Chain_fcb(OpAbort, Info, NULL, "Both");
            dFree(Info);
            break;
         default:
            MSG_WARN("Unused CCC 1F Op %d\n", Op);
            break;
         }
      }
   } else if (Branch == 2) {
      if (Dir == FWD) {
         sd = a_Klist_get_data(ValidSocks, SKey);
         assert(sd);
         /* Receiving from server */
         switch (Op) {
         case OpSend:
            if (sd->https_proxy_reply) {
               dbuf = Data1;
               dStr_append(sd->https_proxy_reply, dbuf->Buf);
               if (strstr(sd->https_proxy_reply->str, "\r\n\r\n")) {
                  if (sd->https_proxy_reply->len >= 12 &&
                      sd->https_proxy_reply->str[9] == '2') {
                     /* e.g. "HTTP/1.1 200 Connection established[...]" */
                     MSG("CONNECT through proxy succeeded. Reply:\n%s\n",
                         sd->https_proxy_reply->str);
                     dStr_free(sd->https_proxy_reply, 1);
                     sd->https_proxy_reply = NULL;
                     a_Tls_handshake(sd->SockFD, sd->url);
                  } else {
                     MSG_BW(sd->web, 1, "Can't connect through proxy to %s",
                            URL_HOST(sd->url));
                     MSG("CONNECT through proxy failed. Server sent:\n%s\n",
                         sd->https_proxy_reply->str);
                     Http_socket_free(SKey);
                     a_Chain_bfcb(OpAbort, Info, NULL, "Both");
                     dFree(Info);
                  }
               }
            } else {
               /* Data1 = dbuf */
               a_Chain_fcb(OpSend, Info, Data1, "send_page_2eof");
            }
            break;
         case OpEnd:
            if (sd->https_proxy_reply) {
               MSG("CONNECT through proxy failed. "
                   "Full reply not received:\n%s\n",
                   sd->https_proxy_reply->len ? sd->https_proxy_reply->str :
                   "(nothing)");
               Http_socket_free(SKey);
               a_Chain_bfcb(OpAbort, Info, NULL, "Both");
            } else {
               Http_socket_free(SKey);
               a_Chain_fcb(OpEnd, Info, NULL, NULL);
            }
            dFree(Info);
            break;
         case OpAbort:
            if (sd->https_proxy_reply) {
               MSG("CONNECT through proxy failed. "
                   "Full reply not received:\n%s\n",
                   sd->https_proxy_reply->len ? sd->https_proxy_reply->str :
                   "(nothing)");
            }
            Http_socket_free(SKey);
            a_Chain_fcb(OpAbort, Info, NULL, "Both");
            dFree(Info);
            break;
         default:
            MSG_WARN("Unused CCC 2F Op %d\n", Op);
            break;
         }
      } else {  /* 2 BCK */
         switch (Op) {
         case OpStart:
            a_Chain_link_new(Info, a_Http_ccc, BCK, a_IO_ccc, 2, 2);
            a_Chain_bcb(OpStart, Info, NULL, NULL); /* IORead */
            break;
         case OpSend:
            if (Data2) {
               if (!strcmp(Data2, "FD")) {
                  int fd = *(int*)Data1;
                  FdMapEntry_t *fme = dList_find_custom(fd_map, INT2VOIDP(fd),
                                                        Http_fd_map_cmp);
                  Info->LocalKey = INT2VOIDP(fme->skey);
                  a_Chain_bcb(OpSend, Info, Data1, Data2);
               } else if (!strcmp(Data2, "reply_complete")) {
                  a_Chain_bfcb(OpEnd, Info, NULL, NULL);
                  Http_socket_reuse(SKey);
                  dFree(Info);
               }
            }
            break;
         case OpAbort:
            Http_socket_free(SKey);
            a_Chain_bcb(OpAbort, Info, NULL, NULL);
            dFree(Info);
            break;
         default:
            MSG_WARN("Unused CCC 2B Op %d\n", Op);
            break;
         }
      }
   }
}

/*
 * Add socket data to the queue. Pages/stylesheets/etc. have higher priority
 * than images.
 */
static void Http_socket_enqueue(Server_t *srv, SocketData_t* sock)
{
   sock->flags |= HTTP_SOCKET_QUEUED;

   if ((sock->web->flags & WEB_Image) == 0) {
      int i, n = dList_length(srv->queue);

      for (i = 0; i < n; i++) {
         SocketData_t *curr = dList_nth_data(srv->queue, i);

         if (a_Web_valid(curr->web) && (curr->web->flags & WEB_Image)) {
            dList_insert_pos(srv->queue, sock, i);
            return;
         }
      }
   }
   dList_append(srv->queue, sock);
}

static Server_t *Http_server_get(const char *host, uint_t port, bool_t https)
{
   int i;
   Server_t *srv;

   for (i = 0; i < dList_length(servers); i++) {
      srv = (Server_t*) dList_nth_data(servers, i);

      if (port == srv->port && https == srv->https &&
          !dStrAsciiCasecmp(host, srv->host))
         return srv;
   }

   srv = dNew0(Server_t, 1);
   srv->queue = dList_new(10);
   srv->running_the_queue = 0;
   srv->host = dStrdup(host);
   srv->port = port;
   srv->https = https;
   dList_append(servers, srv);

   return srv;
}

static void Http_server_remove(Server_t *srv)
{
   SocketData_t *sd;

   while ((sd = dList_nth_data(srv->queue, 0))) {
      dList_remove_fast(srv->queue, sd);
      dFree(sd);
   }
   dList_free(srv->queue);
   dList_remove_fast(servers, srv);
   dFree(srv->host);
   dFree(srv);
}

static void Http_servers_remove_all()
{
   Server_t *srv;
   SocketData_t *sd;

   while (dList_length(servers) > 0) {
      srv = (Server_t*) dList_nth_data(servers, 0);
      while ((sd = dList_nth_data(srv->queue, 0))) {
         dList_remove(srv->queue, sd);
         dFree(sd);
      }
      Http_server_remove(srv);
   }
   dList_free(servers);
}

static void Http_fd_map_remove_all()
{
   FdMapEntry_t *fme;
   int i, n = dList_length(fd_map);

   for (i = 0; i < n; i++) {
      fme = (FdMapEntry_t *) dList_nth_data(fd_map, i);
      dFree(fme);
   }
   dList_free(fd_map);
}

/*
 * Deallocate memory used by http module
 * (Call this one at exit time)
 */
void a_Http_freeall(void)
{
   Http_servers_remove_all();
   Http_fd_map_remove_all();
   a_Klist_free(&ValidSocks);
   a_Url_free(HTTP_Proxy);
   dFree(HTTP_Proxy_Auth_base64);
   dFree(HTTP_Language_hdr);
}