static char rcsid[] = "$Id: http.c,v 2.7 2000/01/21 17:37:33 sxw Exp $";
/*
 *  http.c - URL processing for http-specific URLs.  HTTP/1.0 compliant.
 *
 *  DEBUG: section  21, level 1, 5, 9   Common liburl HTTP routines
 *  AUTHOR: Harvest derived
 *
 *  Harvest Indexer http://harvest.sourceforge.net/
 *  -----------------------------------------------
 *
 *  The Harvest Indexer is a continued development of code developed by
 *  the Harvest Project. Development is carried out by numerous individuals
 *  in the Internet community, and is not officially connected with the
 *  original Harvest Project or its funding sources.
 *
 *  Please mail lee@arco.de if you are interested in participating
 *  in the development effort.
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <memory.h>
#include <string.h>
#include <signal.h>
#include <ctype.h>
#include <time.h>
#include <sys/types.h>
#include <sys/time.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <netdb.h>
#include <errno.h>
#include "url.h"
#include "util.h"

#ifdef _HARVEST_AIX_
#include <sys/select.h>
#endif

/* Local variables */
static int read_timeout = 0;
static int do_read ();

struct http_auth {
	char *type;
	char *realm;
	char *username;
	char *passwd;
	char *encoded;
	struct http_auth *next;
};

static struct http_auth *HTTPAuth;
static char *http_make_auth ();
static char *ht_uuencode ();

/*
 *  HTTP/1.0 Status Codes from:
 *     http://info.cern.ch/hypertext/WWW/Protocols/HTTP/HTRESP.html
 *
 *  HTTP/1.1 Status Codes from:
 *     RFC 2068 "Hypertext Transfer Protocol -- HTTP/1.1"
 */
#define HTTP_SUCCESS_STATUS(x) \
	( \
	((x) == 200) ||	/* Success: OK */ \
	((x) == 201) ||	/* Success: CREATED */ \
	((x) == 202) ||	/* Success: Accepted */ \
	((x) == 203) ||	/* Success: Partial Information */ \
	((x) == 204) ||	/* Success: No Response */ \
     	((x) == 205) ||	/* 1.1 Success: Reset Content */ \
     	((x) == 206)   	/* 1.1 Success: Partial Content */ \
	)

#define HTTP_REDIRECTION_STATUS(x) \
	( \
	((x) == 301) ||	/* Redirection: Moved */ \
	((x) == 302) ||	/* Redirection: Found */ \
	((x) == 303) ||	/* Redirection: Method */ \
	((x) == 304) ||	/* Redirection: Not Modified */ \
	((x) == 305)   	/* 1.1 Redirection: Use Proxy */ \
	)

#define HTTP_UNAUTHORIZED_STATUS(x) \
	( \
	((x) == 401) ||	/* Error: Unauthorized */ \
	((x) == 407)   	/* 1.1 Error: Proxy Authentication Required */ \
	)

#define HTTP_ERROR_STATUS(x) \
	( \
	((x) == 400) ||	/* Error: Bad request */ \
	((x) == 402) ||	/* Error: PaymentRequired */ \
	((x) == 403) ||	/* Error: Forbidden */ \
	((x) == 404) ||	/* Error: Not found */ \
	((x) == 405) ||	/* 1.1 Error: Method Not Allowed */ \
	((x) == 406) ||	/* 1.1 Error: Not Acceptable */ \
	((x) == 408) ||	/* 1.1 Error: Request Timeout */ \
	((x) == 409) ||	/* 1.1 Error: Conflict */ \
	((x) == 410) ||	/* 1.1 Error: Gone */ \
	((x) == 411) ||	/* 1.1 Error: Length Required */ \
	((x) == 412) ||	/* 1.1 Error: Precondition Failed */ \
	((x) == 413) ||	/* 1.1 Error: Request Entity Too Large */ \
	((x) == 414) ||	/* 1.1 Error: Request-URI Too Long */ \
	((x) == 415) ||	/* 1.1 Error: Unsupported Media Type */ \
	((x) == 500) ||	/* Error: Internal Error */ \
	((x) == 501) ||	/* Error: Not implemented */ \
	((x) == 502) ||	/* Error: Service temporarily overloaded */ \
	((x) == 503) ||	/* Error: Gateway timeout 503 */ \
	((x) == 504) ||	/* 1.1 Error: Gateway Timeout */ \
	((x) == 505)   	/* 1.1 Error: HTTP Version Not Supported */ \
	)

#define HTTP_VALID_STATUS(x) \
	(  \
	HTTP_SUCCESS_STATUS(x)		|| \
	HTTP_REDIRECTION_STATUS(x)	|| \
	HTTP_ERROR_STATUS(x)		|| \
	HTTP_UNAUTHORIZED_STATUS(x)  	\
	)

/*
 *  get_sockaddr() - create a socket, bind an address to it
 *
 *  Return values:
 *
 *      0       Success
 *      1       DNS errors
 */
int
get_sockaddr (hostname, sa)
char *hostname;
struct sockaddr_in *sa;
{
	Host *H = NULL;

	if ((H = get_host (hostname)) == NULL) {
		errorlog ("Cannot resolve %s\n", hostname);
		return 1;
	}
	memcpy (&(sa->sin_addr.s_addr), H->ipaddr, H->addrlen);
	return 0;
}

/*
 *  http_get() - retrieves the URL and prints into the file up->fp.
 *  Returns non-zero on error; 0 on success.
 *
 *  Return code indicates severity of error (DW 6/9/95):
 *
 *      -1      Indicates HTTP Redirect - may use -1/-2 in future to
 *              denote 301 / 302
 *      1-9     'soft', maybe temporary errors.  Doesn't necessarily
 *              mean the object doens't exist.
 *      10+     'hard' errors from remote HTTPD.  The URL is invalid
 *              or no longer exists
 *
 *  Return codes:
 *      -1      Redirect
 *      0       Success
 *      1       DNS errors (from get_sockaddr())
 *      2       socket()/bind() errors
 *      3       connect() errors
 *      4       network write/read errors
 *      10      HTTP errors
 *
 *
 *  Uses the HTTP/1.0 protocol as described in:
 *      http://info.cern.ch/hypertext/WWW/Protocols/HTTP/HTTP2.html
 *
 *  Patched by Hrvoje Stipetic <hrvoje.stipetic@hck.hr> to support
 *  modification of User-Agent: and From: HTTP headers.  Uses
 *  environment variables HARVEST_USER_AGENT and HARVEST_MAINTAINER_ADDRESS.
 *
 *  Patched by Judith Pluemer <judith@mathematik.uni-osnabrueck.de>
 *  to support incremental gathering on http-Servers. Additionally
 *  the Perl-Scripts uptime.pl and extract.pl are used.
 *
 *  Patched by Simon Wilkinson <sxw@tardis.ed.ac.uk> to remove dependencies
 *  on perl scripts for incremental gatherering
 */
int
http_get (up)
URL *up;
{
	/*
	 *  The following fix was added by Paul Johnson, 27/9/96, to handle
	 *  the end-of-header detection properly.  The original routine
	 *  could not spot a double-new-line (end of header marker) if it
	 *  was split across two messages from the server.  Some HTTP
	 *  servers (notably NCSA 1.5.2) send each line as a separate
	 *  message, and hence break this code.
	 *
	 *  The solution is to add a new flag variable "last_char_was_eol"
	 *  which is set when we see a \n and cleared when we see anything
	 *  else.  If last_char_was_eol and we have a second \n, then we
	 *  have found the end of the header.
	 */

	Buffer *mimebuf = NULL;
	Buffer *reqbuf = NULL;
	Buffer *urlbuf = NULL;
	URL *new_up = NULL;
	char *bufp = NULL;
	char *host = NULL;
	char *http_proxy = getenv ("http_proxy");
	char *accept = getenv ("HARVEST_HTTP_ACCEPT");
	char *p = NULL;
	char *request = 0;
	char *t = NULL;
	char *tmp = NULL;
	char *u = NULL;
	int i;
	int in_http_code = 1;
	int in_http_data = 0;
	int in_http_header = 0;
	int last_char_was_eol = 0;
	int n;
	int nbytes;
	int nw;
	int port = 0;
	int proxy_port = 0;
	int s;
	int x;
	static char buf[BUFSIZ];
	static char junk[128];
	static char newURL[BUFSIZ];
	static char proxy_host[128];
	static char realm[128];
	static char type[128];
	struct sockaddr_in sa;

	Debug (21, 1, ("http_get: http_proxy=%s\n",
		       http_proxy ? http_proxy : "NULL"));

	if (http_proxy) {
		/* FIX: No error checking on sscanf */
		sscanf (http_proxy, "http://%[^:]:%d/", proxy_host,
			&proxy_port);
		host = proxy_host;
		port = proxy_port;
		request = up->url;
	} else {
		host = up->host;
		port = up->port;
		request = up->raw_pathname;	/* spec says need escapes */
	}

	if ((x = get_sockaddr (host, &sa) > 0))
		return x;
	sa.sin_family = AF_INET;
	sa.sin_port = (unsigned short) htons (port);

	if ((s = socket (PF_INET, SOCK_STREAM, 0)) < 0) {
		sprintf (buf, "HTTP socket: %s", host);
		log_errno (buf);
		return 2;
	}
	if (connect (s, (struct sockaddr *) &sa, sizeof (sa)) < 0) {
		sprintf (buf, "HTTP connect: %s:%d [%s]",
			 host, port, inet_ntoa (sa.sin_addr));
		log_errno (buf);
		close (s);
		return 3;
	}
	reqbuf = create_buffer (BUFSIZ);
	sprintf (buf, "GET %s HTTP/1.0\r\n", request);
	add_buffer (reqbuf, buf, strlen (buf));
	if (getenv ("HARVEST_USER_AGENT") != NULL)
		sprintf (buf, "User-Agent: %s (Harvest/%s)\r\n",
			 getenv ("HARVEST_USER_AGENT"), HARVEST_VERSION);
	else
		sprintf (buf, "User-Agent: Harvest/%s\r\n", HARVEST_VERSION);
	add_buffer (reqbuf, buf, strlen (buf));
	if (getenv ("HARVEST_MAINTAINER_ADDRESS") != NULL)
		sprintf (buf, "From: %s\r\n",
			 getenv ("HARVEST_MAINTAINER_ADDRESS"));
	else
		sprintf (buf, "From: %s@%s\r\n",
			 getmylogin (), getfullhostname ());
	add_buffer (reqbuf, buf, strlen (buf));
	/* A minor HTTP/1.1 addition - make Harvest send the Host: header --JLa */
	sprintf (buf, "Host: %s\r\n", up->host);
	add_buffer (reqbuf, buf, strlen (buf));
	/* Send an Accept: Header. This is part of HTTP/1.0 */
	if (accept == NULL) {
		sprintf (buf, "Accept: */*\r\n");
		add_buffer (reqbuf, buf, strlen (buf));
	} else if (accept[0] != '\0') {
		sprintf (buf, "Accept: %s\n", accept);
		add_buffer (reqbuf, buf, strlen (buf));
	}

	urlbuf = create_buffer (BUFSIZ);
	sprintf (buf, "http://%s", up->host);
	add_buffer (urlbuf, buf, strlen (buf));
	if (up->port != 80) {
		sprintf (buf, ":%d", up->port);
		add_buffer (urlbuf, buf, strlen (buf));
	}

	/* FIXME: Is this right, or should we be using the escaped pathname? */
	add_buffer (urlbuf, up->raw_pathname, strlen (up->raw_pathname));
	up->lmt = urldb_getlmt (urlbuf->data);

	if (up->lmt > 0) {
		sprintf (buf, "If-Modified-Since: %s\r\n",
			 mkrfc850 (&(up->lmt)));
		add_buffer (reqbuf, buf, strlen (buf));
	}
#ifdef HTTP_AUTHENTICATION
	if (up->auth_realm) {
		if ((tmp = http_make_auth (up->auth_realm))) {
			sprintf (buf, "Authorization: %s %s\r\n",
				 up->auth_type, tmp);
			add_buffer (reqbuf, buf, strlen (buf));
			xfree (tmp);
		}
	} else
		/*
		 *  Another way to do HTTP authentication.  If they give
		 *  http://user:pw@host:port/url-path
		 *  then use it.
		 */
	if (up->user || up->password) {
		char *xbuf = xmalloc (BUFSIZ);
		sprintf (xbuf, "%s:%s",
			 up->user ? up->user : "",
			 up->password ? up->password : "");
		sprintf (buf, "Authorization: Basic %s\r\n",
			 ht_uuencode (xbuf));
		xfree (xbuf);
		add_buffer (reqbuf, buf, strlen (buf));
	}
#endif
	add_buffer (reqbuf, "\r\n", 2);

	Debug (21, 1, ("http_get: Sending HTTP Request: %s\n", reqbuf->data));
	if (write (s, reqbuf->data, reqbuf->length) != reqbuf->length) {
		sprintf (buf, "HTTP write: %s:%d", up->host, up->port);
		log_errno (buf);
		close (s);
		free_buffer (reqbuf);
		return 4;
	}
	free_buffer (reqbuf);

	/* Now read the HTTP/1.0 response, and write data into a file */
	memset (buf, '\0', BUFSIZ);
	if ((up->fp = fopen (up->filename, "w+")) == NULL) {
		log_errno (up->filename);
		close (s);
		return 4;
	}
	nbytes = 0;
	while (1) {
		read_timeout = 0;
		n = do_read (s, buf, BUFSIZ - 9);	/* need a little extra at end */
		buf[n] = 0;
		Debug (21, 9, ("Read %d bytes: ---BEGIN---\n%s\n---END---\n",
			       n, buf));
		bufp = &buf[0];
		if (n == 0)
			break;	/* nothing left to do */
		if (n < 0) {
			if (read_timeout == 1) {
				char *t = NULL;
				int to = XFER_TIMEOUT;
				if ((t =
				     getenv ("HARVEST_XFER_TIMEOUT")) != NULL)
					to = atol (t);
				errorlog ("HTTP timeout: %s:%d (%d seconds).\n",
					  up->host, up->port, to);
			} else
				errorlog ("HTTP read: %s:%d failed.\n",
					  up->host, up->port);
			close (s);
			fclose (up->fp);
			free_buffer (urlbuf);
			return 4;
		}
		if (in_http_code) {
			in_http_code = 0;
			while (isspace (*bufp) && n > 0)
				bufp++, n--;
			/*
			 * The next line altered by Simon Wilkinson
			 * <sxw@tardis.ed.ac.uk> so Harvest will work with
			 *  HTTP/1.1 compliant servers.
			 */
			if ((strlen (bufp) > 5) &&
			    (memcmp (bufp, "HTTP/1", 6) != 0)) {
				tmp = strchr (bufp, '\n');
				if (tmp != NULL)
					*tmp = '\0';	/* get one line */
				up->http_version = xstrdup ("UNKNOWN");
				Log ("WARNING: Invalid HTTP/1.0 response: %s: %s\n", up->url, bufp);
				in_http_header = 0;
				in_http_data = 1;
			} else {
				if ((strchr (bufp, '3') != NULL) &&
				    (memcmp (strchr (bufp, '3'), "304", 3) ==
				     0)) {
					/*
					 *  We've done a successful If-Modified-Since get
					 *  so we now need to write the file out of the
					 *  production db into where the data is expected
					 */

#ifdef USE_MD5
					/* Pull the MD5 hash from the production db */
					up->md5 = urldb_getmd5 (urlbuf->data);
#endif
					up->http_version = xstrdup ("HTTP/1.0");
					up->http_reason_line =
					    xstrdup ("HTTP/1.0 200 OK\n");

					/*
					 *  Create a fake set of Mime headers for the
					 *  item
					 */
					mimebuf = create_buffer (BUFSIZ);
					sprintf (buf, "HTTP/1.1 200 OK\n");
					add_buffer (mimebuf, buf, strlen (buf));
					sprintf (buf, "Last-Modified: %s\n",
						 mkrfc850 (&(up->lmt)));
					add_buffer (mimebuf, buf, strlen (buf));
					sprintf (buf,
						 "Content-Type: text/x-soif\n");
					add_buffer (mimebuf, buf, strlen (buf));

#if 0 /* kjl/15apr2002 we don't do anything with SOIF yet */
					/*
					 *  Pull the SOIF out of the production database
					 *  and store it into the file
					 */
					urldb_writesoif (urlbuf->data, up->fp);
#endif

					in_http_data = 1;
					break;	/* Don't parse any more of the stuff */
				}
				in_http_header = 1;
				in_http_data = 0;

				up->http_version = xstrdup ("HTTP/1.0");
				bufp += strlen ("HTTP/1.0");
				n -= strlen ("HTTP/1.0");

				while (isspace (*bufp) && n > 0)
					bufp++, n--;

				up->http_status_code = atoi (bufp);
				while (isspace (*bufp) && n > 0)
					bufp++, n--;

				if (!HTTP_VALID_STATUS (up->http_status_code)) {
					errorlog
					    ("Failed HTTP/1.0 transfer for %s: %s (status code: %d)\n",
					     up->url, bufp,
					     up->http_status_code);
					close (s);
					fclose (up->fp);
					free_buffer (urlbuf);
					return 10;
				}
				up->http_reason_line = xstrdup (bufp);
				tmp = strchr (up->http_reason_line, '\n');
				if (tmp != NULL)
					*tmp = '\0';

				if (HTTP_ERROR_STATUS (up->http_status_code)) {
					errorlog
					    ("HTTP/1.0 transfer error for %s: %s\n",
					     up->url, up->http_reason_line);
					close (s);
					fclose (up->fp);
					free_buffer (urlbuf);
					return 10;
				}
				while (*bufp != '\n' && n > 0)
					bufp++, n--;
				bufp++, n--;	/* start of header */
				mimebuf = create_buffer (BUFSIZ);
			}
		}
		if (in_http_header) {
			Debug (21, 9,
			       ("in_http_header, line %d, n=%d\n", __LINE__,
				n));

			for (i = 0; i < n; i++) {
				if (bufp[i] != '\r') {	/* Skip CR chars */
					if (bufp[i] == '\n') {
						if (last_char_was_eol) {
							in_http_header = 0;
							in_http_data = 1;
							Debug (21, 9,
							       ("Found end of header\n"));
							break;
						} else {
							last_char_was_eol = 1;
						}
					} else {
						last_char_was_eol = 0;
					}
				}
			}

			Debug (21, 9, ("Adding %d bytes to mimebuf\n", i));
			add_buffer (mimebuf, bufp, i);
			i = i + 1;
			bufp += i;
			n -= i;
		}
#ifdef FOLLOW_REDIRECTS
		if (in_http_data && mimebuf)
			if (HTTP_REDIRECTION_STATUS (up->http_status_code) &&
			    (p =
			     strstr (mimebuf->data,
				     "Location:")) != (char *) NULL) {
				Debug (21, 1, ("Received HTTP Redirection\n"));
				if (sscanf (p, "%s %s", junk, newURL) != 2)
					goto redirect_fallthru;
				Debug (21, 1, ("New URL: %s\n", newURL));

				if ((new_up =
				     url_open (newURL)) == (URL *) NULL)
					goto redirect_fallthru;
				if (new_up->type != URL_HTTP) {
					url_close (new_up);
					goto redirect_fallthru;
				}
				/*
				 *  copy over URL info from the new URL
				 *  into the original 'up' structure.
				 *  Note that we leave  alone the
				 *  up->filename field.
				 */
				if (up->redir_from_url == (char *) NULL)
					up->redir_from_url = xstrdup (up->url);
				xfree (up->url);
				xfree (up->raw_pathname);
				xfree (up->pathname);
				xfree (up->host);

				up->url = xstrdup (newURL);
				up->type = new_up->type;
				up->raw_pathname =
				    xstrdup (new_up->raw_pathname);
				up->pathname = xstrdup (new_up->pathname);
				up->host = xstrdup (new_up->host);
				up->port = new_up->port;

				Debug (21, 5,
				       ("http_get: redir: up->redir_from   %s\n",
					up->redir_from_url));
				Debug (21, 5,
				       ("http_get: redir: up->type         %d\n",
					up->type));
				Debug (21, 5,
				       ("http_get: redir: up->raw_pathname %s\n",
					up->raw_pathname));
				Debug (21, 5,
				       ("http_get: redir: up->pathname     %s\n",
					up->pathname));
				Debug (21, 5,
				       ("http_get: redir: up->host         %s\n",
					up->host));
				Debug (21, 5,
				       ("http_get: redir: up->port         %d\n",
					up->port));

				url_close (new_up);

				/*
				 *  free memory allocated in this function
				 *  which is going to be re-allocated by the
				 *  recursive function call
				 */
				xfree (up->http_reason_line);
				up->http_reason_line = NULL;
				xfree (up->http_version);
				up->http_version = NULL;
				free_buffer (mimebuf);

				close (s);
				fclose (up->fp);
				up->fp = 0;

				free_buffer (urlbuf);
				return (-1);	/* Return redirect info */
			}
	      redirect_fallthru:
#endif

#ifdef HTTP_AUTHENTICATION
		if (in_http_data && mimebuf) {
			if (!HTTP_UNAUTHORIZED_STATUS (up->http_status_code))
				goto auth_fallthru;

			/*      Loop Detection          */

			Debug (21, 1,
			       ("Got Unauthorized, will try some passwords\n"));

			if (up->auth_realm != (char *) NULL) {
				Log ("http_get: %s UNAUTHORIZED with realm %s\n", up->url, up->auth_realm);
				xfree (up->http_reason_line);
				up->http_reason_line = NULL;
				xfree (up->http_version);
				up->http_version = NULL;
				free_buffer (mimebuf);
				free_buffer (urlbuf);
				fclose (up->fp);
				close (s);
				return 10;	/* 'hard' error from httpd */
			}
			Debug (21, 1, ("Looking for auth line...\n"));
			if ((p = strstr (mimebuf->data, "WWW-Authenticate:")) ||
			    (p = strstr (mimebuf->data, "WWW-authenticate:"))) {
				Debug (21, 1, ("%s\n", p));
				if (sscanf
				    (p, "%s %s %[^\n]", junk, type, realm) != 3)
					goto auth_fallthru;
				Debug (21, 1, ("junk=%s\n", junk));
				Debug (21, 1, ("type=%s\n", type));
				Debug (21, 1, ("realm=%s\n", realm));
				u = realm;
				if ((t = strchr (u, '=')))
					u = t + 1;
				if ((t = strchr (u, '"')))
					u = t + 1;
				if ((t = strrchr (u, '"')))
					*t = '\0';
				xfree (up->auth_type);
				up->auth_type = 0;
				xfree (up->auth_realm);
				up->auth_realm = 0;
				up->auth_realm = xstrdup (u);
				up->auth_type = xstrdup (type);
				Debug (21, 1, ("type=%s\n", up->auth_type));
				Debug (21, 1, ("realm=%s\n", up->auth_realm));

				/*
				 * free memory allocated in this function
				 * which is going to be re-allocated by the
				 * recursive function call
				 */
				xfree (up->http_reason_line);
				up->http_reason_line = NULL;
				xfree (up->http_version);
				up->http_version = NULL;
				free_buffer (mimebuf);
				free_buffer (urlbuf);
				close (s);
				fclose (up->fp);
				return http_get (up);
			} else {
				Debug (21, 1, ("NO AUTH LINE\n"));
			}
		}
	      auth_fallthru:
#endif

		if (in_http_data && n > 0) {
			if ((nw = fwrite (bufp, 1, n, up->fp)) != n)
				log_errno ("HTTP fwrite");
			nbytes += nw;
			if (nbytes > HTTP_MAX_TRANSFER) {
				Log ("http_get: WARNING!: %s has exceeded %d bytes of data, aborting transfer.\n", up->url, HTTP_MAX_TRANSFER);
				break;
			}
		}
	}
	if (mimebuf != NULL) {
		Debug (21, 9,
		       ("Copying mimebuf to up structure, length=%d\n",
			mimebuf->length));
		up->http_mime_hdr = xmalloc (mimebuf->length + 1);
		memcpy (up->http_mime_hdr, mimebuf->data, mimebuf->length);
		up->http_mime_hdr[mimebuf->length] = '\0';
		free_buffer (mimebuf);
	}
	/* Clean up */
	fclose (up->fp);
	close (s);

	free_buffer (urlbuf);

	return (in_http_data == 1 ? 0 : 10);
}

/*
 *  do_read() - read that performs timeout, assumes theSocket is set
 *  for nonblocking I/O .  Based on MyRead() from ccache.
 */
static int
do_read (s, buf, sz)
int s;
char *buf;
int sz;
{
	fd_set readDetect;
	struct timeval timeout;
	int err, readBytes = 0;
	extern int select ();

	/* read until timeout or amount of requested bytes read */
	while (1) {
		char *t = NULL;
		int to = XFER_TIMEOUT;
		if ((t = getenv ("HARVEST_XFER_TIMEOUT")) != NULL)
			to = atol (t);
		FD_ZERO (&readDetect);
		FD_SET (s, &readDetect);
		timeout.tv_sec = to;
		timeout.tv_usec = 0;

		/* wait for data for seconds */
		err = select (s + 1, &readDetect, NULL, NULL, &timeout);
		if (err < 0) {
			if (errno == EINTR)
				continue;
			if (errno == ECONNRESET)
				return (0);
			log_errno ("select");
			return (-1);
		}
		/* timeout on the read */
		if (err == 0) {
			read_timeout = 1;
			return (-1);
		}
		if (FD_ISSET (s, &readDetect)) {
			if ((readBytes = read (s, buf, sz)) < 0)
				if (errno == ECONNRESET)
					return (0);
			break;
		}
	}
	return (readBytes);
}

static void
http_init_auth ()
{
	FILE *fp;
	static char xbuf[4][BUFSIZ];
	char *t = NULL;
	struct http_auth *a = NULL;

	HTTPAuth = NULL;

	if ((t = getenv ("HARVEST_HTTP_AUTHENTICATIONS")) == NULL) {
		Debug (21, 1,
		       ("http_init_auth: No Authentication file passed in the environment.\n"));
		return;
	}
	Debug (21, 1, ("http_init_auth: open %s\n", t));
	if ((fp = fopen (t, "r")) != NULL) {
		while (fscanf (fp, "%s %s %s %s\n",
			       xbuf[0], xbuf[1], xbuf[2], xbuf[3]) == 4) {
			Debug (21, 1,
			       ("http_init_auth: got type=%s realm=%s username=%s passwd=%s\n",
				xbuf[0], xbuf[1], xbuf[2], xbuf[3]));
			rfc1738_unescape (xbuf[1]);
			a = (struct http_auth *)
			    xmalloc (sizeof (struct http_auth));
			a->type = xstrdup (xbuf[0]);
			a->realm = xstrdup (xbuf[1]);
			a->username = xstrdup (xbuf[2]);
			a->passwd = xstrdup (xbuf[3]);
			a->next = 0;
			HTTPAuth = a;
		}
	}
	fclose (fp);
}

/*
 *
 *  ACKNOWLEDGEMENT:
 *       This code is taken from rpem distribution, and was originally
 *       written by Mark Riordan.
 *
 *  routines to convert a buffer of bytes to/from RFC 1113
 *  printable encoding format.
 *
 *  allocates the buffer for the encoded string.  The caller must free it.
 */
static char *
ht_uuencode (inbuf)
char *inbuf;
{
	static char six2pr[64] = {
		'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
		'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
		'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
		'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
		'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'
	};
#define ENC(c) six2pr[c]
	char *outbuf, *s;
	int i;
	int nbytes = strlen (inbuf);

	Debug (21, 5, ("ht_uuencode(): called.  inbuf=%s\n", inbuf));
	s = outbuf = (char *) xmalloc (strlen (inbuf) * 2);
	for (i = 0; i < nbytes; i += 3) {
		*(outbuf++) = ENC (inbuf[0] >> 2);
		*(outbuf++) =
		    ENC (((inbuf[0] << 4) & 060) | ((inbuf[1] >> 4) & 017));
		*(outbuf++) =
		    ENC (((inbuf[1] << 2) & 074) | ((inbuf[2] >> 6) & 03));
		*(outbuf++) = ENC (inbuf[2] & 077);
		inbuf += 3;
	}

	if (i == nbytes + 1) {
		/* There were only 2 bytes in that last group */
		outbuf[-1] = '=';
	} else if (i == nbytes + 2) {
		/* There was only 1 byte in that last group */
		outbuf[-1] = '=';
		outbuf[-2] = '=';
	}
	outbuf[0] = '\0';
	Debug (21, 5, ("ht_uuencode(): returning %s\n", s));
	return s;
}

static char *
http_make_auth (realm)
char *realm;
{
	struct http_auth *a = NULL;
	static char namepw[BUFSIZ];
	static int inited = 0;

	if (!inited) {
		http_init_auth ();
		inited = 1;
	}
	Debug (21, 5, ("http_make_auth: called.  realm=%s\n", realm));
	for (a = HTTPAuth; a; a = a->next) {
		if (!strcasecmp (realm, a->realm)) {
			if (a->encoded == (char *) 0) {
				sprintf (namepw, "%s:%s",
					 a->username ? a->username : "",
					 a->passwd ? a->passwd : "");
				a->encoded = ht_uuencode (namepw);
			}
			return xstrdup (a->encoded);
		}
	}
	Debug (21, 5,
	       ("http_make_auth: No authentication for realm %s\n", realm));
	return 0;
}
