/* **********************************************************************
 * *                                                                    *
 * *           Generate a Raw Data Product From an INGEST Scan          *
 * *                                                                    *
 * **********************************************************************
 * File: iris/product/raw_c.C
 *
 *       COPYRIGHT (c) 1989, 1996, 1997, 1998, 2004, 2005, 2006 BY
 *              VAISALA INC., WESTFORD MASSACHUSETTS, U.S.A.
 * 
 * THIS SOFTWARE IS FURNISHED UNDER A LICENSE AND MAY BE USED AND  COPIED
 * ONLY  IN  ACCORDANCE WITH  THE  TERMS  OF  SUCH  LICENSE  AND WITH THE
 * INCLUSION OF THE ABOVE COPYRIGHT NOTICE.  THIS SOFTWARE  OR  ANY OTHER
 * COPIES  THEREOF MAY NOT BE PROVIDED OR OTHERWISE MADE AVAILABLE TO ANY
 * OTHER PERSON.  NO TITLE TO AND OWNERSHIP OF  THE  SOFTWARE  IS  HEREBY
 * TRANSFERED.
 */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "sigtypes.h"
#include "dsp.h"
#include "headers.h"
#include "iris_task.h"
#include "ingest.h"
#include "product.h"

#include "user_lib.h"
#include "misc_lib.h"

#include "tdwr_interface.h"

static MESSAGE iraw_pack_data
(const char *ssum_a,           /* Ingest summary path */
 struct ingest_header *sum_a,  /* Ingest summary structure, modified */
 UINT4 idsrc_a,                /* List of data parameters to open on disk */
 UINT4 idcnv_a,                /* List of data parameters to convert */
 char iswpmask_a[MAX_SWEEPS] );

static void insert_raw_bytes
(void *ibuf_a,
 SINT4 ilen_a );

static void raw_convert_plan
(UINT4 iconv_a,
 UINT4 *idata_a,
 UINT4 *idsrc_a,
 UINT4 *idcnv_a );

static SINT4 icnt_c, iraynum_c;
static FILE *filep_c;
static struct raw_prod_block block_c;
char log_str[200];

/* ======================================================================
 * This routine creates, writes, and closes the resulting product file, but
 * makes no changes to the internal INGEST and PRODUCT directories.
 *
 * The raw product is made by collecting together all of the INGEST data
 * for a given volume scan.  The product file is "blocked" so that when it is
 * written to tape the resulting record boundaries will fall at reasonable
 * locations.  The product file consists of:
 *
 *   Record #1    :  A product file header, padded out to TAPE_RECORD_LEN
 *   Record #2    :  The INGEST summary file, padded out to TAPE_RECORD_LEN
 *   Records 3... :  Sweep #1 data
 *                   Sweep #2 data
 *                         .
 *                         .
 * Each sweep consists of an integer number of TAPE_RECORD_LEN sized blocks
 * i.e. a given block does not hold data from more than one sweep.  Each sweep
 * block is also prefixed with a RAW_PROD_BHDR record.
 */
MESSAGE iraw_prod
(const struct product_configuration *pcf_a, /* Product configuration input */
 const char *ssum_a,                        /* Ingest summary path to use */
 struct product_hdr *phdr_a,                /* Header of the product file produced */
 char bfile_a[],                            /* Name of product file produced */
 int  close_flag)
{
  MESSAGE istatus, istatus2;
  SINT4 ii, ij, ik, iw;
  UINT4 idsrc, idcnv;
  void *address;
  SINT2 isweep;
  char iswpmask[MAX_SWEEPS];
  char spath[PATHNAME_SIZE];
  struct ingest_header sum;
  
  
  /* ---------------------------------
   * Open the INGEST summary file.  Copy it to local storage, since we may need
   * to modify the structure if we are modifying the data.  Close the disk file
   * right away.
   */

  strcpy( spath, ssum_a );
  istatus = imapopen( spath, FALSE, &address, &ij, &ik );
  if ( istatus != SS_NORMAL ) return istatus;

  if( ij < INGEST_HEADER_SIZE )
    {
      istatus = imapclose( address, ij, ik );  /* Close summary file */
      return IRIS_INGEST_DATA;

    }

  memmove( &sum, address, INGEST_HEADER_SIZE );
  istatus = imapclose( address, ij, ik );       /* Close summary file */
  if ( istatus != SS_NORMAL ) return istatus;

  /* Set up for possible data conversions.  This may modify the Ingest summary
   * structure list of final data.
   */
  raw_convert_plan( pcf_a->psi.raw.iraw_convert, &sum.tcf.dsp.DataMask.iWord0, &idsrc, 
		    &idcnv );

  /* Make up a mask of the sweeps to be processed by this run.  This consists
   * either of all sweeps for a complete raw product, or the one matching the
   * ingest time for a sweep-by-sweep raw product.
   */
  ik = sum.hdr.iflags;
  if( (pcf_a->psi.raw.iflags & RAW_FLG_SWEEP) != 0 )
    {
      memset( iswpmask, 0, MAX_SWEEPS );
      isweep = pcf_a->psi.raw.isweep;
      iswpmask[ isweep-1 ] = 1;

      /* For sweep-by-sweep RAW products, the overall 'done' flag is set
       * only if the volume scan itself is done, and if we are converting
       * the last of however many sweeps were actually collected.
       */
      
    }
  else
    {
      memset( iswpmask, -1, MAX_SWEEPS );
      sum.hdr.iflags = ik | HD_STRUCTURE_COMPLETE;    /* Full RAW is always 'done' */
    }
  /* Create and open the raw product file
   */

  strcpy( spath, bfile_a );
      
  filep_c = fopen( spath, "w+" );
  if( filep_c == 0 ) return iMessageMod_va( IRIS_RAW_CREATE, 1, spath );

  /* Skip the first two blocks of the file; then write the actual file data.
   * Since the product header and ingest summary are not complete yet, the
   * first two records can not be filled in properly right now.
   */
  if( -1 == fseek( filep_c, 2*TAPE_RECORD_LEN, 0 ) )
    {
      istatus = IRIS_RAW_WRITE;
      goto L910;
    }
  istatus = iraw_pack_data( ssum_a, &sum, idsrc, idcnv, iswpmask );
  if( istatus != SS_NORMAL ) goto L910;

  /* Make up the product header, and fill in the total size based on how much
   * was just written.
   */
  memset( phdr_a, 0, PRODUCT_HDR_SIZE );
  phdr_a->pcf = *pcf_a;

  HeaderFromIngestNoShare( phdr_a, &sum, "tdr_rad1        ", 0 /*Reference height*/ );

  radpos_from_ingest( phdr_a, &sum );

  phdr_a->pcf.psi.raw.isweep = isweep;
  phdr_a->pcf.psi.raw.iflags &= RAW_FLG_SWEEP;

  ii = block_c.hdr.irec;
  phdr_a->hdr.ibytes = TAPE_RECORD_LEN * ii;
  /* Fill in the first two records of the product file with the product header
   * and summary structures.
   */
  if( -1 == fseek( filep_c, 0, 0 ) )
    {
      istatus = IRIS_RAW_WRITE;
      goto L910;
    }

  memset( &block_c, 0, TAPE_RECORD_LEN );
  memmove( &block_c, phdr_a, PRODUCT_HDR_SIZE );
  iw = fwrite( &block_c, TAPE_RECORD_LEN, 1, filep_c );
  if( iw != 1 )
    {
      istatus = IRIS_RAW_WRITE;
      goto L910;
    }

  sum.hdr.iflags = ik & ~HD_STRUCTURE_COMPLETE;
  if( (isweep == sum.tcf.scan.isweeps) || close_flag )
  {
    sum.hdr.iflags = ik | HD_STRUCTURE_COMPLETE;
    Lprintf("setting close flag");
  }
  memset( &block_c, 0, TAPE_RECORD_LEN );
  memmove( &block_c, &sum, INGEST_HEADER_SIZE );
  iw = fwrite( &block_c, TAPE_RECORD_LEN, 1, filep_c );
  if( iw != 1 )
    {
      istatus = IRIS_RAW_WRITE;
      goto L910;
    }

  iw = fclose( filep_c );
  if( iw != 0 )
    {
      istatus = IRIS_RAW_WRITE;
      goto L920;
    }

  return istatus;

  /* Various returns...
   */
L910:  /* File still open */
Lprintf("returned raw_pack bad");
  iw = fclose( filep_c );

L920:  /* File closed, but must be deleted */
  istatus2 = sig_delete_file( spath );
  if( istatus == SS_NORMAL ) istatus = istatus2;

  return istatus;
}

/* ======================================================================
 * Collect sweep and moment files together to make the data portion of a
 * raw product.  Within each sweep, the rays from each parameter are
 * grouped together by ray number, i.e., by the location within the ray
 * pointer table.  This is done so that the data, when written to tape, are
 * grouped together in space and time.  This was thought to be preferable
 * for post-processing routines that read the tapes and directly compute
 * from them.  The order of the rays within each sweep is the same as the
 * order of the pointer table.  This makes it simple to reconstruct INGEST
 * files from the raw product file, since the ray pointer table info
 * is implied.
 * History:
 *   jmh   27 May 1992  Changed extended ray header sizes to not lose the last
 */ 
static MESSAGE iraw_pack_data
( const char *ssum_a,            /* Ingest summary path name */
  struct ingest_header *sum_a,  /* Ingest summary structure, modified */
  UINT4 idsrc_a,                /* List of data parameters to open on disk */
  UINT4 idcnv_a,                /* List of data parameters to convert */
  char iswpmask_a[MAX_SWEEPS] )
{
  
  /* Sized for longest ray holding 16-bit data, with 20 bytes padding for worst
   * case compression failure.
   */
#define  RAYBUF_SZ_P (20+(2*MAX_BINS))
  SINT2 raybuf[ RAYBUF_SZ_P/2 ];

  MESSAGE istatus, istatus2;
  struct ingest_data_file *iaddr[32], *ptr;
  SINT4 isize[32], ichan[32], ibits_bin[32], ii, ij, im,
    iprm, imaxprm, irptr, irbcnt, isweep;
  UINT4 ik;
  struct ingest_data_header dhdr;
  struct ray_header rhdr;
  struct data_convert convert;
  char sfile[160];

#define FREE_MALLOC (123456789)


  /* If no data, then quit right now */
  istatus = SS_NORMAL;
  if( sum_a->tcf.dsp.DataMask.iWord0 == 0 ) return istatus;

  /* Find the highest numbered parameter */
  for ( iprm = 0; iprm< 32; iprm++)
    if( BTEST( sum_a->tcf.dsp.DataMask.iWord0, iprm ) ) imaxprm = iprm;

  memset( &block_c.hdr, 0, RAW_PROD_BHDR_SIZE );
  icnt_c               = 0;                 /* No bytes in this block yet */
  block_c.hdr.irec     = 2;                 /* Data begins on record #2 (org 0) */

  sum_a->icf.idfnum  = 0;                   /* We will rebuild the data file */
  sum_a->icf.idfsize = 0;                   /*  stats from scratch... */

  convert.fvelun_velocity = 
    fNyquistVelocity( sum_a->tcf.dsp.iprf    , sum_a->tcf.dsp.itrig,
		      sum_a->tcf.misc.ilambda, sum_a->tcf.misc.ipolar ) ;

  convert.fvelun_width = 
    fNyquistWidth( sum_a->tcf.dsp.iprf, sum_a->tcf.misc.ilambda,
		   sum_a->tcf.misc.ipolar );

  convert.flambda = ((FLT4)sum_a->tcf.misc.ilambda)/100.0;

  /* Loop through each of the sweeps that make up this task, and which we actually
   * need to run.
   */
  for (isweep = 1; isweep<=sum_a->tcf.scan.isweeps; isweep++ )
    {
      if( !iswpmask_a[isweep-1] ) continue;

      block_c.hdr.isweep = isweep;
      iraynum_c = 0;
            
      /* First open the data files, keeping track of any which are supposed to be
       * present but which in fact are not.  Write out the data headers for each
       * file (whether present or not).
       */
      /* Init these variables once, since they may be accessed in unusual
       * order when params are being converted. */
      for (iprm = 0; iprm<32; iprm++)
	{
	  iaddr[iprm]     = 0;
	  isize[iprm]     = 0;
	  ichan[iprm]     = 0;
	  ibits_bin[iprm] = 0;
	}

      for (iprm=0; iprm<32; iprm++)
	{
	  if( !BTEST( idsrc_a, iprm ) ) continue;
	  strcpy( sfile, ssum_a );
	  strcat( sfile, sIngestEnd( iprm, isweep ) );
	  istatus2 = imapopen( sfile, FALSE, (void**)&iaddr[iprm], &isize[iprm], &ichan[iprm] );

	  memset( &dhdr, 0, INGEST_DATA_HEADER_SIZE );
	  if( istatus2 == SS_NORMAL )
	    {

	      /* If we are converting this parameter, then perform the complete
	       * conversion right now so that everything about this data file
	       * is inserted properly.  We can then close the file itself, and
	       * continue using the converted data...
	       */
	      ik = iprm;
	      if( BTEST( idcnv_a, iprm ) )
		{
		  istatus2 = iconvert_ingest_param( iaddr[iprm], &ik, &ptr, &ij,
						    &convert );

		  if( istatus2 == SS_NORMAL )
		    {
		      /* Make it look like the parameter is really coming from here */
		      iaddr[ik] = ptr;
		      isize[ik] = ij;
		      ichan[ik] = FREE_MALLOC;  /* Reminder of how to free later...*/
		    }
		  else
		    {
		      ik = 0xffffffff;
		      sig_signal( istatus2 );
		    }

		  istatus2 = imapclose( iaddr[iprm], isize[iprm], ichan[iprm] );
		  if( istatus2 != SS_NORMAL ) sig_signal( istatus2 );

		  iaddr[iprm] = 0;         /* No original data will appear */
		}

	      if( ik != 0xffffffff )
		{
		  memmove( &dhdr, iaddr[ik], INGEST_DATA_HEADER_SIZE );
		  ibits_bin[ik] = dhdr.ibits_bin;
		}
	    }
	  else
	    {
	      sig_signal( istatus2 );
	      iaddr[iprm] = 0;
	    }

	  if( dhdr.hdr.id != 0 )    /* Update INGEST summary stats */
	    {
	      sum_a->icf.idfnum  += 1;
	      sum_a->icf.idfsize += dhdr.hdr.ibytes;
	    }
	  insert_raw_bytes( &dhdr, INGEST_DATA_HEADER_SIZE );

         }  /* end for */

      /* Scan through the open parameter files, inserting compressed rays into the
       * final buffer, using the order that they are found in the pointer table.
       */
      /* Insert all rays from pointer tables */
      for (irptr=0; irptr<sum_a->icf.irtotl; irptr++)
	{
	  /* Insert all parameters */
	  for (iprm=0; iprm<=imaxprm; iprm++)
	    {
	      char *pbase, *pray;
	      if( !BTEST( sum_a->tcf.dsp.DataMask.iWord0, iprm ) ) continue;
	      ii = 0; pbase=NULL;

	      if( iaddr[iprm] )
		{
		  pbase = ((char *)iaddr[iprm]) + INGEST_DATA_HEADER_SIZE;
		  ii = *((SINT4*)( pbase + 4*irptr ));
		}

	      if( ii != 0 )    /* Valid ray to insert? */
		{
		  pray =  pbase + 4*sum_a->icf.irtotl + (ii-1);

		  /* Check that the pointer is within the file. */
		  if ((pray >= pbase) &&  (pray < (pbase+isize[iprm])))
		    {
		      memcpy( &rhdr, pray, RAY_HSIZE );
		      if( iprm != DB_XHDR )
			im = RAY_HSIZE + (( ibits_bin[iprm] * rhdr.ibincount ) / 8 );
		      else
			im = RAY_HSIZE + ibits_bin[iprm] / 8;
		      if( im <= (RAYBUF_SZ_P-20) )
			{
			  compress_words( (SINT2 *)pray, (1+im)/2, raybuf, &irbcnt );
			}
		      else
			{
			  istatus = IRIS_INGEST_DATA;
			  compress_words( NULL, 0, raybuf, &irbcnt );
			}
		    }
		  else
		    {
		      istatus = IRIS_INGEST_DATA;
		      compress_words( NULL, 0, raybuf, &irbcnt );
		    }
		}
	      else
		{
		  compress_words( NULL, 0, raybuf, &irbcnt );
		}

	      insert_raw_bytes( raybuf, irbcnt*2 );
	      iraynum_c += 1;
	    }  /* end for */
	}  /* end for */

      /* Pad remainder (if any) of output record for this sweep.
       */
      if( icnt_c != 0 )
	{
	  ii = RAW_PROD_SDATA_SIZE - icnt_c;
	  memset( &raybuf, 0,  MIN(ii, RAYBUF_SZ_P) );
	  while( ii > 0 )
	    {
	      ij = MIN(ii, RAYBUF_SZ_P);
	      insert_raw_bytes( raybuf, ij );
	      ii = ii-ij;
	    }
	}

      /* Close all of this sweep's data files, or the malloc'd memory areas that
       * were created to hold converted data.
       */
      for (iprm=0; iprm<32; iprm++)
	{
	  if( iaddr[iprm] )
	    {
	      if( ichan[iprm] == FREE_MALLOC )
		istatus2 = sig_free( iaddr[iprm] );
	      else
		istatus2 = imapclose( iaddr[iprm], isize[iprm], ichan[iprm] );
	      if( istatus == SS_NORMAL ) istatus = istatus2;
	    }
	}

      }   /* Ends loop on sweep number */

  return istatus;
}

/* ======================================================================
 * Insert an array of bytes into the tape block being assembled so far.
 * Each time a block becomes full, write it out to disk.
 *
 */

static void insert_raw_bytes
(void *ibuf_a,
 SINT4 ilen_a )
{

  SINT4 ii, iw;

  /* Reject arrays that are too long.  Ignore zero length arrays.
   */
  if(( ilen_a < 0) || (ilen_a > RAW_PROD_SDATA_SIZE) )
    {
      sig_signal( IRIS_RAW_BIGRAY );
      return;
    }

  if( ilen_a == 0 ) return;

  /* If the first full ray pointer has not been filled in yet, then do it now.
   */
  if( block_c.hdr.iray_ptr == 0 )
    {
      block_c.hdr.iray_ptr = RAW_PROD_BHDR_SIZE + icnt_c;
      block_c.hdr.iray_num = iraynum_c;
    }

  /* Insert the array of data for the case where it fits entirely in the current
   * record.
   */
  if( (icnt_c + ilen_a) <= RAW_PROD_SDATA_SIZE )
    {
      memcpy( &block_c.ibytes[icnt_c], ibuf_a, ilen_a );
      icnt_c += ilen_a;
      if( icnt_c == RAW_PROD_SDATA_SIZE )
	{
	  iw = fwrite( &block_c, TAPE_RECORD_LEN, 1, filep_c );
	  if( iw != 1 ) goto L190;
	  block_c.hdr.iray_ptr = 0;
	  block_c.hdr.irec    += 1;
	  icnt_c  = 0;
	}
    }    
  /* Insert the array of data for the case where it spills into the next record.
   */
  else
    {
      ii = RAW_PROD_SDATA_SIZE - icnt_c;    /* # bytes for first record */

      memcpy( &block_c.ibytes[icnt_c], ibuf_a, ii );
      iw = fwrite( &block_c, TAPE_RECORD_LEN, 1, filep_c );
      if( iw != 1 ) goto L190;

      block_c.hdr.iray_ptr = 0;
      block_c.hdr.irec    += 1;
      icnt_c = ilen_a - ii;

      memcpy( block_c.ibytes, ((char *)ibuf_a)+ii, icnt_c );
      
    }
  return;

  /* Errors */
L190:
  sig_signal( IRIS_RAW_WRITE );
  
  return;
}
/* ======================================================================
 * Decide whether we will be converting 8-Bit <==> 16-Bit data, and set
 * flags for what will be changed.
 *
 * Entry Parameters
 *   iconv_a : What type of conversion to perform
 *   idata_a : Mask of originally present data
 * Exit Parameters
 *   idata_a : Mask of data that will be present after conversions
 *   idsrc_a : Mask of data to read from disk
 *   idcnv_a : Mask of data to be converted
 */
static void raw_convert_plan
(UINT4 iconv_a,
 UINT4 *idata_a,
 UINT4 *idsrc_a,
 UINT4 *idcnv_a )
{
#define CONV_PRM_NUM (5)

  SINT4 ii, ij;
  SINT4 params[2][CONV_PRM_NUM]=
  {{ DB_DBT , DB_DBZ , DB_VEL , DB_WIDTH , DB_ZDR} ,
   { DB_DBT2, DB_DBZ2, DB_VEL2, DB_WIDTH2, DB_ZDR2} };

  *idsrc_a = *idata_a;           /* Will become source param list */
  *idcnv_a = 0;                  /* Will become params to convert */

  ij = 0;
  if( iconv_a == IRAW_CONVERT_8_TO_16 ) ij = 1;
  if( iconv_a == IRAW_CONVERT_16_TO_8 ) ij = 2;

  if( 0 == ij ) return;

  ij = ij-1;
  for (ii=0; ii<CONV_PRM_NUM; ii++)
    {
      if( BTEST( *idata_a, params[ij][ii] ) )    /* Have source parameter ? */
	{
	  if( BTEST(*idata_a, params[1-ij][ii]) )    /* Yes, Already have dest ? */
	    {
	      *idsrc_a = IBCLR( *idsrc_a, params[ij][ii]);  /* Yes, dont open source */
	    }
	  else
	    {
	      *idata_a = IBSET( *idata_a, params[1-ij][ii] ); /* No, but we will after */
	      *idcnv_a = IBSET( *idcnv_a, params[ij][ii] );   /* converting this one. */
	    }
	  *idata_a = IBCLR( *idata_a, params[ij][ii] );    /* Will not store source */
	}
    }  /* end for (ii=... */
  
  return;
}
