/* pulse_split.C */
/* WU-generator program for pulse_search */

/* Plan:  read tape data, parse headers, interpolate coords, */
/*        assemble new header + 8MB of data */

/* Now includes database support */

class star;
class stars;

#include <stdio.h>
#include <stdlib.h>
#include <math.h>
#include <string.h>


#include <sys/types.h>
#include <sys/stat.h>
#include <sys/statvfs.h>
#include <fcntl.h>
#include <unistd.h>
#include "std_fixes.h"


#include "splitparms.h"
#include "splittypes.h"
#include "timecvt.h"
#include "coordcvt.h"

#include "util.h"
#include "str_util.h"
#include "boinc_db.h"
#include "parse.h"
#include "backend_lib.h"
#include "crypt.h"

#include "sqlapi.h"
#include "db_table.h"
#include "xml_util.h"
#include "sched_config.h"
#include "ap_schema.h"
#include "app_config.h"

#include "sched_util.h"
#include "sched_msgs.h"

#include "setilib.h"

/* $w is declared below.  It is an object of class workunit_header.
 * This class is defined in astropulse/server/db/ap_schema.h, and 
 * its member functions are defined in ap_schema.cpp.
 * The ap_schema.cpp file is generated from the ap_schema.sql file,
 * by running seti_boinc/db/schema_to_class, which calls
 * seti_boinc/db/schema_to_class.awk
 *
 * All of the cfg information will be extracted from the db via
 * a settings object called $active, then placed in the workunit_header
 * $w.  However, a workunit_header contains more information than a
 * settings object does.  (Which is why we don't use $active alone.)
 */
#define SAMP_RATE (w.recorder_cfg->sample_rate)
#define FFT_LEN (w.analysis_cfg->fft_len)
#define DM_CHUNK_LARGE (w.analysis_cfg->dm_chunk_large)
#define DM_CHUNK_SMALL (w.analysis_cfg->dm_chunk_small)
#define FOLD_BUF_BYTES_SHORT (w.analysis_cfg->fold_buf_bytes_short)
#define REMOVE_RADAR (w.analysis_cfg->remove_radar)
#define DM_HIGH (w.analysis_cfg->dm_high)
#define DM_LOW (w.analysis_cfg->dm_low)
#define MAX_COADD (w.analysis_cfg->max_coadd)
/* 5 n log_2 n is the number of floating point
 * operations in a FFT of length n.
 */
#define FFT_OPS (5*FFT_LEN*log(FFT_LEN)/log(2))

int N_BLOCK;  // Number of blocks per workunit; can be set from database if we're running with sci db

/* BOINC params */
/* APP_CONFIG contains the name of the science (informix) database, among other
 * things.  Other parameters are fetched from the science database.
 */
APP_CONFIG ap_config;

/* SCHED_CONFIG is a BOINC class that says how much memory BOINC
 * is allowed to use, how many floating point operations (?),
 * and so on.
 */
SCHED_CONFIG boinc_config;
char *PROJECT_DIR=NULL;
char *triggerfile=NULL;
char *app_name=NULL;

/* The boinc (mysql) database, contains information about workunits that the 
 * server needs to know in order to send a result. (?)
 */
char BOINC_DB_NAME[128];

/* URL that people should send results to.  Probably
 * http://setiathome.berkeley.edu/uploadhandler.cgi
 */
char UPLOAD_URL[256];

/* URL to download the program and workunits from;
 * http://setiathome.berkeley.edu/download
 */
char DOWNLOAD_URL[256];

// I'm not sure what wd is ... ?
char wd[1024];

/* workunit template file, an xml file that tells BOINC about what format
 * the workunit is in.  Contains the name e.g. in.dat
 *
 * Note: the workunit file has two different names.  One is the name that the
 * application (e.g. the astropulse client) uses.  This is e.g. in.dat.
 * To access the file, the application calls a BOINC api that gives it 
 * the file's real name, which is long and complicated.
 * This is a way of pretending that each data file is named the same thing
 * for all runs of the program.  E.g. if there are 23 different data files,
 * the application designer can choose 23 standard names for them.
 */
char WU_TEMPL[1024] = "/home/boincadm/projects/beta/templates/pulse_wu";

/* result template file, an xml file that tells BOINC about the format of
 * the result.  Contains the name e.g. pulse.out.  Referenced only in the
 * create_work function below.  (why?)
 *
 * This file has two different names as well, see above.
 */
char RES_TEMPL[1024] = "templates/pulse_result"; 

/* min_quorum: number of matching results to come back before validator runs ?
 * target_nresults: number of copies initially sent out
 * max_error_results: workunit cancelled if this many errors appear
 * max_success_results: if this many successful results, send no more out
 *              even if validation fails
 * max_total_results: meaningful if smaller than max_success_results
 * 		+ max_error_results
 * // WARNING: I don't think any of these numbers are ever used, but if they are, it would be bad.
 */
const int min_quorum          = 3;
const int target_nresults     = 10;
const int max_error_results   = 10;
const int max_success_results = 10;
const int max_total_results   = 20;

/* splitter_float_to_bits: creates an output data string from an input float array.
 * The data string will be used as input to the client.
 */
void splitter_float_to_bits(std::vector<complex<float> > data, std::string& output);
void check_for_halt();
int check_free_disk_space();
int wait_for_db_wus_ondisk(int appid);
void update_tel_pos(double jd, double ra, double dec);
int get_tel_pos(double jd, double *ra, double *dec);

/* Replace this with: vector<coordinate_t> tel_pos;
 * The class coordinate_t is defined in astropulse/server/db/ap_schema.h.  
 * The three numbers are doubles: time, ra, and dec.  You can use the 
 * default constructor. (?)
 *
 * Note this is referenced later only in get_tel_pos and update_tel_pos,
 * which are supposedly going to be rewriteen by Eric.  So I should
 * coordinate with Eric on this one.
 * // WARNING: This is not up to date
 */
double tel_pos[10][3] = {
                          {0, 0, 0},
                          {0, 0, 0},
                          {0, 0, 0},
                          {0, 0, 0},
                          {0, 0, 0},
                          {0, 0, 0},
                          {0, 0, 0},
                          {0, 0, 0},
                          {0, 0, 0},
                          {0, 0, 0}
                        };

/* ------ Main program ------ */

int main(int argc, char *argv[]) {
  /* datafile: file descriptor for tape file*/
  int datafile;
  /* nohead: Says that we should print only data (no header or end tags) to the output file.
   * no_boinc_db: don't call boinc functions (any function starting "boinc_ ... ")
   *		e.g. telling the boinc server about workunits
   * sci_db_read_only: don't update the sci db (don't insert the current workunit or tape)
   * nth_arg_with_no_dash: keeps track of how many arguments with no dash we have read from
   *		the command line.  (where "dash" means e.g. -no_boinc_db starts with a dash)
   */
  int nohead=0, no_boinc_db=0, sci_db_read_only=0;
  int nth_arg_with_no_dash=0;
  /* pid: fork call ? */
  int pid=0;
  int retval;
  int i;

  /* head: headers from datafile (tape file), will be replaced by dr2_block_t
   * fname: name of datafile (tape file)
   * beam: number of beam to read from the tape file, 0-6
   * polarization: number of polarization to read from the tape file, 0-1
   * infiles: ?
   * wu_template: load WU_TEMPL file into this buffer
   * keyfile: has something to do with RSA
   * wu_dir: the directory in which we will place the finished workunit.
   */
  /* std::vector<tapeheader> head; */
  char fname[1024]={0};
  int beam = -1;
  int polarization = -1;
  char* infiles[1];
  char* wu_template;
  char full_fname[256];
  char keyfile[256], wu_dir[256];
  char query_string[16];

  /* buf: just a buffer for temporary storage of strings.
   * start_jd, end_jd: start and end julian dates for this workunit.
   *		(astropulse only creates one workunit at a time.)
   * start_block: tape block that this workunit starts at (?)
   * w: derived from schema (ap_schema.sql) as described before.
   *    Opens a db via its constructor function, using db_open.
   *    It doesn't check the error return of the db_open call --
   *    it won't exit if no db is available.
   * active: also derived from schema; has a similar structure to
   *     the workunit_header $w.  The idea is that the settings
   *     $active will be used to set the workunit_header $w.
   */
  char buf[256];
  double start_jd, end_jd;
  int start_block=-1;
  /* Execution of this line, with no database, results in 9 messages like
   * "The cursor or statement is not available.
   * -404:0:  DATABASE apb"
   */
  workunit_header w;
  /* This line causes 5 more messages of the same type */
  settings active;

  /* DB_WORKUNIT is a BOINC type, info that gets stored in the BOINC
   *      database.  boinc_wu will become a boinc (mysql) database entry
   * 	  describing this workunit
   * DB_APP is info in the BOINC database about the astropulse application
   * 	  As far as I can tell, its sole purpose is to check that the app
   *      is known, and set boinc_wu.appid
   * R_RSA_PRIVATE_KEY is a code-signing key
   */ 
  DB_WORKUNIT boinc_wu;
  DB_APP boinc_app;
  R_RSA_PRIVATE_KEY key;

  /* Parse command line arguments, except for the first argument
   * which is the name of this program.
   */
  fprintf(stderr, "In pulse_split.C: main: reading command line arguments.\n");
  for (i=1; i<argc; i++) {

    /* If this argument starts with a dash '-' ... */
    if (argv[i][0]=='-') {
      /* ep will be a pointer to the equals sign, if any, 
       * in the current argument 
       */
      char *ep;

      /* Check for flags: -noheader, -no_boinc_db, -sci_db_read_only */
      if (!strncmp(argv[i],"-noheader",std::max(strlen(argv[i]),(size_t)4))) { 
	nohead++; 
      } else if (!strncmp(argv[i],
	    "-no_boinc_db",std::max(strlen(argv[i]),(size_t)5))) { 
	no_boinc_db++; 
      } else if (!strncmp(argv[i],
	    "-sci_db_read_only",std::max(strlen(argv[i]),(size_t)5))) {
	sci_db_read_only++; 

      /* Check for flags with an equal sign: -projectdir, -block,
       * -triggerfile, -app_name
       */
      } else if ((ep=strchr(argv[i],'='))) {
        if (!strncmp(argv[i],"-projectdir",std::max((int)(ep-argv[i]),2))) { 
	  PROJECT_DIR = ep+1; 
	} else if (!strncmp(argv[i], "-block",std::max((int)(ep-argv[i]),2))) { 
	  start_block = atoi(ep+1); 
        /* If there is a flag that has an equal sign but shouldn't,
	 * send an error.
	 */
	} else if(!strncmp(argv[i], "-triggerfile", std::max((int)(ep-argv[i]),2))) {
	  triggerfile = ep+1;
        } else if(!strncmp(argv[i], "-app_name", std::max((int)(ep-argv[i]), 2))) {
          app_name = ep+1;
        } else {
          fprintf(stderr, "In pulse_split.C: main: flag %s is unknown or should not be followed by an equal sign.\n", argv[i]);
	  fprintf(stderr, "Usage: pulse_split datafile beam polarization (-noheader) (-sci_db_read_only) (-no_boinc_db) (-block=nnnn) -projectdir=dir\n -app_name=name");
          exit(EXIT_FAILURE);
        }

      /* If there is an unknown flag, send an error. */
      } else {
	fprintf(stderr, "In pulse_split.C: main: flag %s is unknown.\n", argv[i]);
        fprintf(stderr, "Usage: pulse_split datafile beam polarization (-noheader) (-sci_db_read_only) (-no_boinc_db) (-block=nnnn) -projectdir=dir\n -app_name=name");
        exit(EXIT_FAILURE);
      }

    /* If the argument doesn't start with '-' ...
     */
    } else {
      /* The first argument with no dash should be the fname */
      if(nth_arg_with_no_dash == 0) {
        strlcpy(fname, argv[i], sizeof(fname));
      /* The second argument should be the beam */
      } else if(nth_arg_with_no_dash == 1) {
        beam = atoi(argv[i]);
      /* The third argument should be the polarization */
      } else if(nth_arg_with_no_dash == 2) {
	polarization = atoi(argv[i]);
      /* There should only be three arguments with no dash */
      } else {  
	fprintf(stderr, "In pulse_split.C: main: flag %s is unknown or should be preceded by a dash.\n", argv[i]);
        fprintf(stderr, "Usage: pulse_split.C: main: command-line usage is: pulse_split datafile beam polarization (-noheader) (-sci_db_read_only) (-no_boinc_db) (-block=nnnn) -projectdir=dir -app_name=name\n");
        exit(EXIT_FAILURE);
      }
      /* Now we have read one more argument with a dash */
      nth_arg_with_no_dash++;
    }
  } // finished parsing command line arguments

  /* If we haven't read the fname, beam, and polarization, send error. */
  if(fname[0] == 0 || beam == -1 || polarization == -1) {
    fprintf(stderr, "In pulse_split.C: main: missing datafile, beam, or polarization.\n");
    fprintf(stderr, "Usage: pulse_split datafile beam polarization (-noheader) (-sci_db_read_only) (-no_boinc_db) (-block=nnnn) -projectdir=dir -app_name=name\n");
    exit(EXIT_FAILURE);
  }

  /* We require that the program be called with the -projectdir option. 
   * boinc_db:
   *   extern DB_CONN boinc_db:	boinc/db/boinc_db.C. 
   *   class DB_CONN:		boinc/db/db_base.h;.C
   * print_error:
   *   void DB_CONN::print_error(const char* p): boinc/db/db_base.h;.C
   */
  if (!PROJECT_DIR) {
    boinc_db.print_error("In pulse_split.C: main: No PROJECT_DIR\nYou must use the -projectdir option\n");
    exit(EXIT_FAILURE);
  }

  if(!app_name) {
    boinc_db.print_error("In pulse_split.C: main: No app_name\nYou must use the -app_name option\n");
    exit(EXIT_FAILURE);
  }

  // Default triggerfile
  if(!triggerfile) {
     triggerfile=new char [strlen(PROJECT_DIR)+14+1]; // 14 characters for "/splitter_stop"
    if(!triggerfile) {
      boinc_db.print_error("In pulse_split.C: main: unable to allocate triggerfile name\n");
      exit(EXIT_FAILURE);
    }
    sprintf(triggerfile, "%s/splitter_stop", PROJECT_DIR);
  }
  
  /* Name of the tapefile */
  infiles[0] = fname;

  /* Read the workunit template file astropulse/pulse_wu into the 
   * char array wu_template. 
   * No need to allocate wu_template beforehand, the function
   * does it for us.
   */
  fprintf(stderr, "In pulse_split.C: main: reading the workunit template file.\n");
  read_file_malloc(WU_TEMPL, wu_template);

  datafile = open(fname, O_RDONLY | O_LARGEFILE);

  if(datafile == -1) {
    fprintf(stderr, "Unable to open data file %s\n", fname);
    exit(1);
  }
 
  /* If we are using the boinc database, do the following:
   * 1) Parse the boinc_config
   * 2) Prepare some variables from the config file's info
   * 3) Read the rsa keyfile
   * 4) Check that the astropulse app exists in the BOINC database
   */
  if(no_boinc_db == 0) {
     fprintf(stderr, "In pulse_split.C: main: running with BOINC database.\n");

    /* 1) Read in the BOINC configuration paramters from config.xml
     * in ~boincadm/projects/beta
     *
     * boinc_config is an object of class SCHED_CONFIG, which is
     * defined in boinc/sched/sched_config.h
     *
     * SCHED_CONFIG::parse_file is defined in
     * boinc/sched/sched_config.C
     * It looks for a file called config.xml in the directory
     * PROJECT_DIR, and parses the file into $boinc_config,
     * which is a large and complex data structure.
     */
    fprintf(stderr, "In pulse_split.C: main: parsing the BOINC config file.\n");
    boinc_config.parse_file(PROJECT_DIR);

    /* 2) Set directories/URLs based on the boinc_config information u
     * As far as I can tell, DOWNLOAD_URL and UPLOAD_URL are not used
     * later. (?)
     */
    strlcpy(wu_dir,boinc_config.download_dir,256);
    strlcpy(DOWNLOAD_URL,boinc_config.download_url,256); 
    strlcpy(UPLOAD_URL,boinc_config.upload_url,256); 
    sprintf(keyfile, "%s/upload_private", boinc_config.key_dir);

    /* 3) read_key_file is defined in boinc/lib/crypt.C
     * The $keyfile's name is defined in the boinc config file,
     * config.xml.  This rsa keyfile is read into the $key.
     */
    fprintf(stderr, "In pulse_split.C: main: reading the RSA key file.\n");
    if (read_key_file(keyfile, key)) {
      fprintf(stderr, "Error reading keyfile.\n");
      exit(1);
    }
    
    /* 4) Check that the app exists in the BOINC database */

    /* Open BOINC database
     * open was defined in one of the header files, so this #undef open
     * is necessary to allow us to call boinc_db.open, below.
     */
#undef open

    /* Print hostname, username, password to stderr.  (Why?)
     * All of these are defined in boinc's config.xml file
     */
    fprintf(stderr, "host: %s\nuser: %s\npasswd: %s\n", boinc_config.db_host, 
            boinc_config.db_user, boinc_config.db_passwd);

    /* Recall, boinc_db is DB_CONN instance declared in another file. */
    if (boinc_db.open(boinc_config.db_name,boinc_config.db_host,
        boinc_config.db_user,boinc_config.db_passwd)) {
      //    if (boinc_db.open((char*)BOINC_DB_NAME, "") == -1)
      boinc_db.print_error("boinc_db.open");
      exit(1);
    }

    /* app_name is "AstroPulse" or "astropulse_alpha"; 
     * this creates a string for use in next lines. 
     * Note: or maybe just "astropulse"?
     */
    sprintf(buf, "where name='%s'", app_name);

    /* Check that the app exists */
    fprintf(stderr, "In pulse_split.C: main: checking that application %s exists.\n", app_name);
    if (boinc_app.lookup(buf)) {
      fprintf(stderr, "Error looking up appname\n");
      boinc_db.print_error("boinc_app lookup");
      exit(1);
    }

  /* else ... we are not using the boinc database, print this fact to stdout. */
  } else {
    // We have to find some other way to set the wu_dir.  To do this,
    // we have to produce the object $boinc_config such that it has
    // the required download_dir.
    // Edit: I don't see that we're producing boinc_config here, just wu_dir.
    // Also, it's not clear to me why n = 256 instead of, say, 1.
    strlcpy(wu_dir, ".", 256);

    fprintf(stderr, "In pulse_split.C: main: Running in no_boinc_db mode.\n");
  }

  /* Array of dr2_block_t objects */
  std::vector<dr2_block_t> dr2_block;
 
  /* Access the science database:
   * 1) Parse the science db name from ap_config.xml
   * 2) Try to open the science database
   * 3) Get the active settings
   */

  fprintf(stderr, "In pulse_split.C: main: reading the science database.\n");

  /* 1) Parse the application config file ap_config.xml in the project directory
   * PROJECT_DIR (currently ~boincadm/projects/beta), taking the
   * <scidb_name> tag in that file for the ap_config->scidb_name.
   *
   * ap_config is an object of class APP_CONFIG, which is defined in
   * astropulse/server/db/app_config.h.  It's a pretty simple data
   * structure.
   *  
   * APP_CONFIG::parse_file is defined in both
   * astropulse/server/db/app_config.cpp    and
   * seti_boinc/db/app_config.cpp
   * and I'm not sure which copy is being used here.
   * $retval can be either an error value, or 0.
   */
  fprintf(stderr, "In pulse_split.C: main: parsing the APP_CONFIG file.\n");
  retval = ap_config.parse_file(PROJECT_DIR);
    
  /* If retval is not zero, our call to ap_config.parse_file
   * has failed.
   */
  if (retval) {
    fprintf(stderr, "Fatal error in pulse_split.C: main: Can't parse config file\n");
    exit(EXIT_FAILURE);
  }
  
  /* 2) Try to open the science database.  A database is already open
   * because we opened it when we constructed a workunit_header $w.
   *
   * db_change is defined in astropulse/server/db/ap_schema.h,
   * and (less likely) seti_boinc/db/schema_master.h
   * It opens a database called $ap_config.scidb_name,
   * closing the previously open database if necessary.
   */
  fprintf(stderr, "In pulse_split.C: main: opening the science database.\n");
  if (!db_change(ap_config.scidb_name)) {
    fprintf(stderr,"In pulse_split.C: main: Could not open science database %s\n", ap_config.scidb_name
           );
    if (!sci_db_read_only) exit(EXIT_FAILURE);
  } else {
    fprintf(stderr,"Using science database %s\n", ap_config.scidb_name);
  }

  /* 3) Get the active settings....
   * First, we fetch information from the science database's "settings" table
   * about which configs to use.  (There are 4 types of configs, each of which
   * has its own settings.)
   * Then, using that information, we fetch the config information into the
   * $active object, so called because it contains the active config information.  
   * We will use $active to help create the workunit_header $w.
   */
  fprintf(stderr, "In pulse_split.C: main: fetching the current active settings.\n");
  if (!active.fetch(std::string("where active=1"))) {
    boinc_db.print_error("unable to fetch active settings");
    exit(1);
  }

  fprintf(stderr, "In pulse_split.C: main: fetching individual config settings.\n");

  // sprintf(query_string, "where s4_id=%d", channel_to_receiverid[channel]);
  sprintf(query_string, "where s4_id=%d", channel_to_receiverid[beam_pol_to_channel[bmpol_t(beam, polarization)]]);
  if (!active.receiver_cfg->fetch(query_string)) {
    printf("sql_error_code: %d\n", sql_error_code());
    boinc_db.print_error("unable to fetch receiver config");
    exit(1);
  }
  w.receiver_cfg=active.receiver_cfg;

  if (!active.recorder_cfg->fetch()) {
    boinc_db.print_error("unable to fetch recorder config");
    exit(1);
  }
  w.recorder_cfg=active.recorder_cfg;

  if (!active.splitter_cfg->fetch()) {
    boinc_db.print_error("unable to fetch splitter config");
    exit(1);
  }
  w.splitter_cfg=active.splitter_cfg;

  if (!active.analysis_cfg->fetch()) {
    boinc_db.print_error("unable to fetch analysis config");
    exit(1);
  }
  w.analysis_cfg=active.analysis_cfg;
 
  /* set number of blocks per workunit. */
  N_BLOCK = w.splitter_cfg->blocks_per_wu;

   /* Grab the first chunk of data, to read tape information from the first header */   
  fprintf(stderr, "In pulse_split.C: main: reading first chunk to extract header information.\n");
  retval = seti_StructureDr2Data(datafile, beam, polarization, N_BLOCK, dr2_block, randomize_corrected);
  // retval = seti_StructureDr2Data(datafile, beam, polarization, N_BLOCK, dr2_block);
  if(retval < N_BLOCK) {
    fprintf(stderr, "pulse_split.C: main: Could not allocate or fill array");
    fprintf(stderr, "%d<%d\n",retval,N_BLOCK);
    exit(1);
  }

  /* If we have write access to the science database:
   * 1) Fetch information about this tape, from the science (informix) database
   * 2) If the tape doesn't exist in the database, create a new entry using
   *    information from the file's header.
   */ 
  if(sci_db_read_only == 0) { 

    fprintf(stderr, "In pulse_split.C: main: we have write access to the science database.\n");
 
    /* 1) Fetch info about this tape from the science (informix)
     * database, from a "tape" table.
     */
    sprintf(buf, "where name='%s' and beam=%d and polarization=%d", 
	         dr2_block[0].header.name, beam, polarization);  
    retval=w.tape_info->fetch(std::string(buf));

    /* If old tape: print information about where in the tape we are starting. */
    if (retval) {
      fprintf(stderr, "Tape %s, beam %d, polarization %d: starting at block %ld.\n",
             w.tape_info->name, beam, polarization, w.tape_info->last_block_done);
    /* 2) If new tape: create a new entry in science database's "tape" table */
    } else {
      fprintf(stderr, "In pulse_split.C: main: reading new tape %s: creating new db entry.\n", dr2_block[0].header.name);
      w.tape_info->id = 0;
      strlcpy(w.tape_info->name,dr2_block[0].header.name,sizeof(w.tape_info->name));
      w.tape_info->start_time = dr2_block[0].header.data_time.jd().uval();
      w.tape_info->last_block_time = w.tape_info->start_time;
      w.tape_info->last_block_done = dr2_block[0].header.dataseq;  // WARNING: used to be = 0
      w.tape_info->wu_count = 0;
      w.tape_info->beam = beam;
      w.tape_info->polarization = polarization;

      if (!w.tape_info->insert()) {
        boinc_db.print_error("db_tape_new");
        exit(1);
      }
    } 

  /* else ... sci_db_read_only = 1, we can't write the science database.
   * We don't set the start_time here because we're not going to send
   * the workunit out.  It may or may not matter ... (?) check on this
   */
  } else {
    fprintf(stderr, "In pulse_split.C: main: Running in sci_db_read_only mode.\n");

    w.tape_info->wu_count = 0;
    /* start block is -1 unless we read it from the command line */
    // w.tape_info->last_block_done = start_block + dr2_block[0].header.dataseq;  // What is the relevance of dr2_block[0].etc?
    if(start_block == -1) {
      w.tape_info->last_block_done = dr2_block[0].header.dataseq;
    } else {
      w.tape_info->last_block_done = start_block;
    }

    /* Database info */ 
    fprintf(stderr, "In pulse_split.C: main: tape name is %s\n", dr2_block[0].header.name);
    strcpy(w.tape_info->name, dr2_block[0].header.name);
  }

  fprintf(stderr, "In pulse_split.C: main: Starting at block %ld.\n", w.tape_info->last_block_done);

  /* Seek to the end of the last block done.  If we are starting a new tape, this
   * line will cause us to seek back to the start of the tape.  Which is important,
   * since we just read a chunk already.
   */
  int header_size = dr2_block[0].header.header_size;
  int data_size = dr2_block[0].header.data_size;
  int block_size = header_size + data_size;

/* Old way
  if (lseek64(datafile, (off64_t)w.tape_info->last_block_done*block_size, SEEK_SET) < 0) {
    perror("lseek64");
    exit(1);
  }
*/

  if(lseek64(datafile, (off64_t)0, SEEK_SET) < 0) {
                 // Seek back to the start of the tape, b/c we just read a chunk already.
    perror("lseek64");
    exit(1);
  }

  // WARNING: Will read(datafile, ...) work?  Is datafile the right kind of object to do that?
  // I think so, because they call lseek64(datafile, ...) in the same way
  
  // This will become unequal to header_size if we fail to read a header later on
  int readbytes = header_size;
  // Figure out the dsi for this beam, polarization pair.  This is necessary because
  // there may be two blocks with the same dataseq in the same tape, one for each dsi 
  int channel = beam_pol_to_channel[bmpol_t(beam, polarization)];
  int dsi = channel > 7 ? 0 : 1;
  /* Keep cycling through blocks until this block's dataseq is equal to last_block_done.
   * Note that if this is a new tape, the second condition will already be false.
   */
  while ((readbytes==header_size) && 
             (dr2_block[0].header.dataseq!=w.tape_info->last_block_done)) { 
    
    check_for_halt();


    int nread=0;
    char buffer[header_size];
    readbytes=0;
	// WARNING: I'm not sure what the purpose of this while loop is ... it should only happen 0 or 1 times 
    while ((readbytes!=header_size) &&
          ((nread = read(datafile,buffer,header_size-readbytes)) > 0 )) {
        readbytes+=nread;
    }
    printf("At block #:%d and dsi:%d\n", dr2_block[0].header.dataseq, dr2_block[0].header.dsi);
    if (readbytes == header_size) {
      dr2_block[0].header.populate_from_data(buffer);
      if (dr2_block[0].header.dataseq!=w.tape_info->last_block_done) {
	// WARNING! not sure about this logic.  What happens first time through?  Later?
        // lseek64(datafile,(off64_t)block_size*(w.tape_info->last_block_done-dr2_block[0].header.dataseq)-header_size,SEEK_CUR);
        lseek64(datafile,(off64_t)block_size-header_size,SEEK_CUR);
      } else { // This "else" will happen after the last seek.
        lseek64(datafile, -1*(off_t)header_size,SEEK_CUR);
        fprintf(stderr, "In pulse_split.C: main: found starting point\n");
      }
    }
    if (nread == 0) {
      fprintf(stderr, "In pulse_split.C: Critical: end of file.\n");
      exit(0);
    }
    if (nread < 0) {
      fprintf(stderr, "In pulse_split.C: Critical: file error %d.\n",errno);
      exit(1);
    }
  }

  /* Clear the dr2_block vector in order to read it in again. */ 
  dr2_block.clear();

  /* Get the process id of the process that's running the code.  This will
   * be included in the workunit's name, so that if the same block is done
   * twice, the names won't be the same.  (?)
   */
  pid = getpid();
 
  /* Loop and create multiple workunits until the tape is finished.  
   * Note that the tel_pos vector is
   * used for the whole tape, not for a single workunit.
   */
  fprintf(stderr, "In pulse_split.C: main: beginning loop over data chunks.\n");
  
  check_for_halt();
  /* Main loop */
  while((retval = seti_StructureDr2Data(datafile, beam, polarization, N_BLOCK, dr2_block, randomize_corrected)) == N_BLOCK) {
  // while((retval = seti_StructureDr2Data(datafile, beam, polarization, N_BLOCK, dr2_block)) == N_BLOCK) {

    check_free_disk_space();
    //Wait for ondisk wus in the database to drop below the threshold
    if(!no_boinc_db) wait_for_db_wus_ondisk(boinc_wu.appid);
    check_for_halt();

    for (i=0; i<N_BLOCK; i++) {
      update_tel_pos(dr2_block[i].header.coord_time.jd().uval(), 
		     dr2_block[i].header.ra,
                     dr2_block[i].header.dec);
    }

    /* Coordinates for this wu 
     * Note that data_time.jd() is the jd for the _end_ of the block. 
     * The quantity subtracted is the number of days for the first data block:
     * data_size	bytes per data block (probably = 1024 * 1024)
     *  * 4		samples per byte
     *  / SAMP_RATE	samples per second (probably = 2500000)
     *  / 86400		seconds per day
     *  / 8		samples of raw data per sample of our beam, pol
     *  = days per data block
     * This value appears to be accurate only to 0.01 seconds or so.
     */
    start_jd = dr2_block[0].header.data_time.jd().uval() - data_size*4/(SAMP_RATE*86400*8);

    /* Compute the time at the end of the data block */
    end_jd = dr2_block[N_BLOCK-1].header.data_time.jd().uval();

    /* Use the array of telescope positions, interpolating to find what the
     * telescope position should be at the start of the workunit.  This is
     * necessary because the telescope position is only taken every 5 seconds.
     */
    get_tel_pos(start_jd, &(w.data_desc.start.ra), &(w.data_desc.start.dec));

    get_tel_pos(end_jd, &(w.data_desc.end.ra), &(w.data_desc.end.dec));

    /* Set in.dat header information.  We're going to write the in.dat file. */
    w.id = 0;
    w.data_desc.block_num = w.tape_info->last_block_done;
    w.data_desc.start.time = start_jd;
    w.data_desc.end.time = end_jd;
    /* This used to say:
    w.data_desc.nsamples = (data_size * N_BLOCK)*8/(w.recorder_cfg->bits_per_sample);
	or
    w.data_desc.nsamples = (data_size * N_BLOCK)/(w.recorder_cfg->bits_per_sample);
    Now, data.size() gives the actual number of floats in the data.
    w.data_desc.nsamples = dr2_block[0].data.size() * N_BLOCK / (w.recorder_cfg->bits_per_sample);
    Next change is that the data array is now composed of complex floats, not floats.  So
    we have to get rid of w.recorder_cgf->bits_per_sample.  The number of complex floats _is_ the
    number of samples.
    */
    w.data_desc.nsamples = dr2_block[0].data.size() * N_BLOCK;
    char datestr[256];
    time_t now;
    time(&now);
    strftime(datestr, 256, "%Y%m%d", localtime(&now));
    /* wu filename.  I think we're reusing $fname from previously. */
    sprintf(fname, "ap_%s_B%d_P%d_%.5ld_%s_%.5d.wu", w.tape_info->name, beam, polarization, w.tape_info->wu_count, datestr, pid);
    strcpy(w.name, fname);
    // First argument should be the filename, second argument is the root.
    // second/first should be the full filename
    retval = dir_hier_path(fname,
			   wu_dir, // = boinc_config.download_dir
			   boinc_config.uldl_dir_fanout,
			   full_fname,
			   true
    );
    if (retval) {
	char buf[1024];
	log_messages.printf(SCHED_MSG_LOG::MSG_CRITICAL,"[%s] dir_hier_path() failed: %d\n", fname, retval);
	exit(1);
    }
    fprintf(stderr, "In pulse_split.C: main: next full workunit (output) base filename is %s\n", full_fname);
    fprintf(stderr, "Next block is %ld.  header.dataseq for first block is %ld\n", w.tape_info->last_block_done, dr2_block[0].header.dataseq);

    /* boinc db info
     *
     * boinc_wu.id will be assigned by the database, due to its initial
     *             value of zero.
     * Not sure what boinc_wu.batch is?
     */
    boinc_wu.id = 0;
    boinc_wu.appid = boinc_app.id;
    strcpy(boinc_wu.name, fname);
    boinc_wu.batch = 0;
    // Calculate the number of floating point operations
    boinc_wu.rsc_fpops_est = FFT_LEN;                           // Coadd loops
    boinc_wu.rsc_fpops_est += FFT_OPS+FFT_LEN*6+FFT_LEN/2*4;    // The dechirp function
    boinc_wu.rsc_fpops_est *= 2;                                // Pos/neg dispersion measures
    boinc_wu.rsc_fpops_est *= DM_CHUNK_SMALL;                   // Inner dm loop
    boinc_wu.rsc_fpops_est += FFT_OPS;                          // forward_fft function
    boinc_wu.rsc_fpops_est *= N_BLOCK*data_size/FFT_LEN;        // Raw data loop
    boinc_wu.rsc_fpops_est *= DM_CHUNK_LARGE / DM_CHUNK_SMALL;  // small dm chunk loop
    boinc_wu.rsc_fpops_est += DM_CHUNK_LARGE*FFT_LEN*2*6;       // build_chirp_table function
    boinc_wu.rsc_fpops_est *= (DM_HIGH-DM_LOW)/DM_CHUNK_LARGE;  // Outer two dm loops
    boinc_wu.rsc_fpops_est *= 31; // fudge factor (to compensate for memory transfer?)
    boinc_wu.rsc_fpops_est /= 5; // additional fudge factor, experimentally determined.  JV 4/1/08
    boinc_wu.rsc_fpops_est *= 1.8; // for negative dms  JV  2/10/09

    fprintf(stderr, "boinc_wu.rsc_fpops_est = %f\n", boinc_wu.rsc_fpops_est);
    printf("boinc_wu.rsc_fpops_est = %f\n", boinc_wu.rsc_fpops_est);
    // boinc_wu.rsc_fpops_bound = boinc_wu.rsc_fpops_est*25;  // Revised 07/07/08 EK
    boinc_wu.rsc_fpops_bound = boinc_wu.rsc_fpops_est*10;

/* #if 0 means "don't run this code."
 * We use this instead of commenting out the code, because
 * you can't have comments inside a comment.
 */
#if 0
    boinc_wu.rsc_iops = FFT_LEN;                      // Inner coadd loop if statements (m)
    boinc_wu.rsc_iops += 2;                           // Save fold check
    boinc_wu.rsc_iops += FFT_LEN;                     // coadd function loop
    boinc_wu.rsc_iops *= 2;                           // Pos/neg dispersion measures
    boinc_wu.rsc_iops += 5*FFT_LEN;                   // dechirp function loop/if statements
    boinc_wu.rsc_iops *= DM_CHUNK;                    // Inner dm chunk loop
    boinc_wu.rsc_iops += 2;                           // compare/increment in for loop
    boinc_wu.rsc_iops *= N_BLOCK*data_size/FFT_LEN;    // Raw data loop
    boinc_wu.rsc_iops += 2;                           // compare/increment in for loop
    boinc_wu.rsc_iops += 2*FFT_LEN*DM_CHUNK;          // build_chirp_table
    boinc_wu.rsc_iops *= (DM_HIGH-DM_LOW)/DM_CHUNK;   // Outer dm loop
    //fprintf( stderr, "fpops: %e, iops: %e\n", boinc_wu.rsc_fpops, boinc_wu.rsc_iops);
    //exit(0);
#endif

    /* boinc_wu.rsc_disk_bound: Max. disk space 
     * boinc_wu.need_validate: whether results are ready for validation
     * boinc_wu.canonical_resultid: After validation, chooses one result
     * boinc_wu.canonical_credit: Credit given for the canonical result
     * boinc_wu.delay_bound: Delay until we cut a user off
     * boinc_wu.error_mask: Whether there's been an error
     * boin_wu.file_delete_state: ready to delete the WU and its results 
     */
    boinc_wu.rsc_memory_bound = 160*1024*1024;
    // boinc_wu.rsc_disk_bound = (double)20*(N_BLOCK*block_size + 2*N_BLOCK*1024*1024);  Old calculation, I think it's just wrong ... ?
    boinc_wu.rsc_disk_bound = 64 * 1024 * 1024;
    boinc_wu.need_validate = false;
    boinc_wu.canonical_resultid = 0;
    boinc_wu.canonical_credit = 0;
    boinc_wu.delay_bound = 25*86400;  // 25 days, in seconds
    boinc_wu.error_mask = 0;
    boinc_wu.file_delete_state = 0;
    boinc_wu.assimilate_state = 0;
    // The line below was removed because the compiler says:
    //  error: 'class DB_WORKUNIT' has no member named 'workseq_next
    // boinc_wu.workseq_next = 0;
    boinc_wu.opaque = 0; // this gets set later

    boinc_wu.target_nresults = target_nresults;
    boinc_wu.min_quorum = min_quorum;
    boinc_wu.max_error_results = max_error_results;
    boinc_wu.max_success_results = max_success_results;
    boinc_wu.max_total_results = max_total_results;

    /* "AstroPulse" */
    strcpy(boinc_wu.app_name, app_name);

    /* write to wu file.
     * The $full_fname is defined above; it corresponds to the "real name" of
     * the in.dat file.
     */
    std::ofstream wufile(full_fname);

    if (!wufile) {
      fprintf(stderr, "Error opening wufile for write");
      exit(1);
    }

    /* write header (xml-style) 
     * * Old: std::string encdata=xml_encode_string(raw_data,_binary);
     * The array of floats in each dr2_block[i].data is converted into the
     * string encdata.
     */
    std::string encdata;
    encdata.clear();
    fprintf(stderr, "The length of encdata is %d\n", (int)encdata.size());
    for(int i = 0; i < N_BLOCK; i++) {
      splitter_float_to_bits(dr2_block[i].data, encdata);
    }
    fprintf(stderr, "The length of encdata is %d\n", (int)encdata.size());
    if (nohead == 0) {
      wufile << "<astropulse_workunit>\n";
      /* This must be an overloaded operator */
      wufile << w;
      wufile << "<data length=" << encdata.size() << " encoding=\"binary\">";
    }

    /* write data for one workunit*/
    wufile << encdata;
    if (!wufile) {
      fprintf(stderr, "Error writing wufile.\n");
      fprintf(stderr, "  fname=%s  retval=%d\n", fname, retval);
      exit(1);
    }
    if (nohead == 0) {
      wufile << "</data>\n" ;
      wufile << "</astropulse_workunit>\n";
    }
    wufile.close();

    /* Update databases */
    w.tape_info->wu_count++;
    w.tape_info->last_block_done += N_BLOCK;
    w.tape_info->last_block_time = start_jd;

    if (sci_db_read_only == 0) {
      /* Update science db */

      if (!w.insert()) {
        boinc_db.print_error("db_apworkunit_new");
        exit(1);
      }

      if (!w.tape_info->update()) {
        boinc_db.print_error("db_tape_update");
        exit(1);
      }

      /* connect boinc wu table to ap wu table */
      boinc_wu.opaque = w.id;
    }

    if (!no_boinc_db) {
      /* talk to boinc db */

      if (create_work( 		// dec'd boinc/tools/backend_lib.h
				// def'd boinc/tools/backend_lib.C
            boinc_wu, wu_template, (char*)RES_TEMPL,
		// "key" should not be in line below.  Also, there should
		// be one more argument, a "char* command_line".
                // old vers: wu_dir, (const char**) infiles, 1, key, boinc_config)
	    wu_dir, (const char**) infiles, 1, boinc_config, NULL)
         ) {
        boinc_db.print_error("create_work");
        exit(1);
      }
    }
  
  /* Clear the dr2_block in order to populate it again */
  dr2_block.clear();
  } // End loop over chunks

  close(datafile);

  if (no_boinc_db == 0) boinc_db.close();

}

/* ------- Functions --------- */
/* (mostly ripped from makemap2.C) */

/* splitter_float_to_bits
 *
 * Arguments:
 * data: A vector of floats that serves as input to the function.
 * output: A string that we create from $data.  If $output is
 *	not empty, we append the new string to the old string.
 *
 * This function's output is intended for the astropulse client.
 * The data vector consists of floats, all of which should equal
 * +1 or -1.  We assemble each group of 8 floats into one char.
 * Then the characters are pushed onto the output string, one by one.
 */
void splitter_float_to_bits(std::vector<complex<float> > data, 
                       std::string& output) {
  /* i indexes the data, j indexes two-bit samples within a char */
  unsigned int i, j;
  /* We will write this char to the output string */
  unsigned char c;

  /* i gets incremented at the end of the inner loop */
  for(i=0; i<data.size(); ) {
  c=0;
    for(j=0; j<4; j++) {
      /* Note that the first time we bit shift c = 0, nothing happens. */
      c >>= 2;
      /* We were reversing the real/imaginary bit order.  Why?
       * I believed I stopped this.
       */
      if(data[i].imag() > 0) c |= 0x80;
      if(data[i].real() > 0) c |= 0x40;
      // reverse real/imag:
      // if(data[i].real() > 0) c |= 0x80;
      // if(data[i].imag() > 0) c |= 0x40;
      i += 1;
    }
    output.push_back(c);
  }
} 

void check_for_halt() {
   FILE* TRIGGERFILE = fopen(triggerfile, "r");
   if(TRIGGERFILE) {
            fprintf(stderr, "Trigger file %s exists, exiting.\n", triggerfile);
            fclose(TRIGGERFILE);
            exit(0);
   }
}

int check_free_disk_space() {
      struct statvfs vfsbuf;

      /* check disk free space */
      statvfs("wu_inbox/.",&vfsbuf);
      if (vfsbuf.f_frsize==0) vfsbuf.f_frsize=512;
      if (vfsbuf.f_bavail < (100000*1024/vfsbuf.f_frsize)) {
          fprintf(stderr,"Not enough free disk space in working directory to continue\n");
          exit(EXIT_FAILURE);
      }

        /* check free disk space in download directory tree */
      std::string download_dir(boinc_config.download_dir);
      int waited=0;
      download_dir+="/.";
      /* wait here if the disk is more than N% full */ 
      statvfs(download_dir.c_str(),&vfsbuf);
      if (vfsbuf.f_blocks > vfsbuf.f_bfree*100/ap_config.min_disk_free_pct) {
        fprintf(stderr,"Waiting for free space in download directory\n");
        while (vfsbuf.f_blocks > vfsbuf.f_bfree*100/ap_config.min_disk_free_pct) {
          waited+=10;
          sleep(10);
	  check_for_halt();
          statvfs(download_dir.c_str(),&vfsbuf);
        }
        fprintf(stderr,"Waited %d seconds\n", waited);
        fprintf(stderr,"Continuing...\n");
      }
      return 0;
}

int wait_for_db_wus_ondisk(int appid) {
      int wus_ondisk,rv;

      // The boinc db query below takes a long time.  Until we fix this,
      // we will do it only every 100 times into this routine.  All other
      // calls here will assume that we need to add WUs.  -- jeffc
      static int check_count = 100;
      if (check_count < 100) {
           check_count++;
           return 0;
      } else {
           check_count = 0;
      }

      if (boinc_db.open(boinc_config.db_name,boinc_config.db_host,boinc_config.db_user,boinc_config.db_passwd)) {
            boinc_db.print_error("boinc_db.open");
            fprintf(stderr,"boinc_db.open\n");
            exit(1);
      }
      do {
            DB_RESULT boinc_result;
            char query[1024];
            sprintf(query,"where appid=%d and server_state=2", appid);
            rv=boinc_result.count(wus_ondisk,query);
            if (rv) {
               boinc_db.print_error("boinc_result.count");
               fprintf(stderr,"DB Error, unable to count workunits on disk\n");
               exit(EXIT_FAILURE);
            }
            check_for_halt();
            fprintf(stderr,"%d WUs ondisk\n", wus_ondisk);
            if (wus_ondisk>ap_config.max_wus_ondisk) sleep(600);
      } while (wus_ondisk>ap_config.max_wus_ondisk);

      return 0;
}


/* This function will not be used in this file, it is just here
 * for reference and should be deleted once the splitter is working.
void output_samples(float *data, int i, int buf_pos) {
  int j,k;
  unsigned short s;
  float *p=data;

  for (j=0;j<2;j++) {
    s=0;
    for (k=0; k<8; k++) {
        s >>= 2;
        if (*p>0) s |= 0x8000;
        if (*(p+1)>0) s |= 0x4000;
        p+=2;
    }
    output_buf[i][buf_pos++]=*(char *)(&s);
    output_buf[i][buf_pos++]=*(((char *)(&s))+1);
  }
}*/



#if 0
// We won't call this function no more
int read_chunk(int f, int n_block, unsigned char *raw_data, unsigned char *raw_head) {
  /* Read in N_BLOCK data blocks and headers */
  int i;

  for (i=0; i<n_block; i++) {
    if (read(f, &raw_head[HEADERSIZE*i], HEADERSIZE) != HEADERSIZE) {
      return(-1);
    }

    if (read(f, &raw_data[DATASIZE*i], DATASIZE) != DATASIZE) {
      return(-1);
    }
  }

  return(0);

}
#endif 

void update_tel_pos(double jd, double ra, double dec) {
  /* Adds a point to the telescope position table */
  int i, j;

  if (jd > tel_pos[9][0]) {
    for (i=0; i<9; i++) {
      for (j=0; j<3; j++) {
        tel_pos[i][j] = tel_pos[i+1][j];
      }
    }

    tel_pos[9][0] = jd;
    tel_pos[9][1] = ra;
    tel_pos[9][2] = dec;
  }

}

int get_tel_pos(double jd, double *ra, double *dec) {
  /* Interpolates between points in tel_pos to get position */
  /* at time given by jd. */
  /* Returns (-)1 if it extrapolated to the future (past) */
  int i = 0;
  int offset;
  double jd1, jd2;
  double ra1, ra2;
  double dec1, dec2;

  while ((jd > tel_pos[i][0]) && (i < 10)) i++;

  if (i == 0) {
    offset = 0;
  } else if (i == 10) {
    offset = -2;
  } else if (tel_pos[i-1][0] == 0) {
    offset = 0;
  } else {
    offset = -1;
  }

  jd1 = tel_pos[i+offset][0];
  jd2 = tel_pos[i+offset+1][0];
  ra1 = tel_pos[i+offset][1];
  ra2 = tel_pos[i+offset+1][1];
  dec1 = tel_pos[i+offset][2];
  dec2 = tel_pos[i+offset+1][2];

  *dec = ((dec2 - dec1)/(jd2 - jd1))*(jd - jd1) + dec1;

  if (fabs(ra1 - ra2) > 23) {
    // ra wrapped around
    if (ra1 > ra2) {  ra1 -= 24; } else { ra2 -= 24; }
  }

  *ra = ((ra2 - ra1)/(jd2 - jd1))*(jd - jd1) + ra1;

  if (*ra < 0) { *ra += 24; }

  return(-1*(offset+1));

}



