vircam_linearity_analyse.c

00001 /* $Id: vircam_linearity_analyse.c,v 1.63 2010/12/09 13:20:26 jim Exp $
00002  *
00003  * This file is part of the VIRCAM Pipeline
00004  * Copyright (C) 2005 Cambridge Astronomy Survey Unit
00005  *
00006  * This program is free software; you can redistribute it and/or modify
00007  * it under the terms of the GNU General Public License as published by
00008  * the Free Software Foundation; either version 2 of the License, or
00009  * (at your option) any later version.
00010  *
00011  * This program is distributed in the hope that it will be useful,
00012  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00013  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00014  * GNU General Public License for more details.
00015  *
00016  * You should have received a copy of the GNU General Public License
00017  * along with this program; if not, write to the Free Software
00018  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
00019  */
00020 
00021 /*
00022  * $Author: jim $
00023  * $Date: 2010/12/09 13:20:26 $
00024  * $Revision: 1.63 $
00025  * $Name: v1-1-0 $
00026  */
00027 
00028 /* Includes */
00029 
00030 #ifdef HAVE_CONFIG_H
00031 #include <config.h>
00032 #endif
00033 
00034 #include <stdio.h>
00035 #include <cpl.h>
00036 #include <math.h>
00037 
00038 #include "vircam_utils.h"
00039 #include "vircam_pfits.h"
00040 #include "vircam_dfs.h"
00041 #include "vircam_mods.h"
00042 #include "vircam_channel.h"
00043 #include "vircam_stats.h"
00044 #include "vircam_paf.h"
00045 
00046 /* Function prototypes */
00047 
00048 static int vircam_linearity_analyse_create(cpl_plugin *) ;
00049 static int vircam_linearity_analyse_exec(cpl_plugin *) ;
00050 static int vircam_linearity_analyse_destroy(cpl_plugin *) ;
00051 static int vircam_linearity_analyse(cpl_parameterlist *, cpl_frameset *) ;
00052 static int vircam_linearity_analyse_lastbit(int jext, cpl_frameset *framelist,
00053                                             cpl_parameterlist *parlist);
00054 static int vircam_linearity_analyse_save(cpl_frameset *framelist,
00055                                          cpl_parameterlist *parlist);
00056 static int vircam_linearity_analyse_domedark_groups(void);
00057 static double *vircam_linearity_analyse_genstat(vir_fits *fframe, int *bpm,
00058                                                 parquet *p, int np);
00059 static double *vircam_linearity_tweakfac(double **fdata, double *mjd, int nim,
00060                                          int nchan, double *facrng, 
00061                                          double *maxdiff);
00062 static void vircam_mjdsort(double **fdata, double *mjd, int n);
00063 static cpl_table *vircam_linearity_analyse_diagtab_init(int np, int nrows);
00064 static void vircam_linearity_analyse_init(void);
00065 static void vircam_linearity_analyse_tidy(int level);
00066 
00067 /* Static global variables */
00068 
00069 static struct {
00070 
00071     /* Input */
00072 
00073     int     norder;
00074     float   lthr;
00075     float   hthr;
00076     int     maxbpmfr;
00077     int     adjust;
00078     int     diagnostic;
00079     int     extenum;
00080 
00081     /* Output */
00082 
00083     float   linearity;
00084     float   linerror;
00085     float   bad_pixel_stat;
00086     int     bad_pixel_num;
00087     float   facrng;
00088     float   maxdiff;
00089 
00090 } vircam_linearity_analyse_config;
00091 
00092 typedef struct {
00093     cpl_frameset   *darks;
00094     cpl_frameset   *domes;
00095     int            ndarks;
00096     int            ndomes;
00097     float          exptime;
00098     unsigned char  flag;
00099     vir_fits       **proc;
00100 } ddgrp;
00101 
00102 #define OK_FLAG       0
00103 #define SATURATE_FLAG 1
00104 
00105 #define SUBSET 128
00106 #define SUBSET2 (SUBSET/2)
00107 
00108 static struct {
00109     int              *labels;
00110     cpl_frameset     *domelist;
00111     int              ndomes;
00112     cpl_frameset     *darklist;
00113     int              ndarks;
00114     cpl_frameset     *domecheck;
00115     int              ndomecheck;
00116     cpl_frameset     *darkcheck;
00117     int              ndarkcheck;
00118     cpl_frame        *inherit;
00119     cpl_frame        *chanfrm;
00120     vir_tfits        *chantab;
00121     cpl_table        *lchantab;
00122     cpl_array        *bpm_array;
00123     ddgrp            *ddg;
00124     int              nddg;
00125     vir_fits         **flatlist;
00126     int              nflatlist;
00127     cpl_propertylist *plist;
00128     cpl_propertylist *elist;
00129     int              nx;
00130     int              ny;
00131     cpl_propertylist *phupaf;
00132     cpl_table        *diag1;
00133     cpl_table        *diag2;
00134     int              nfdata;
00135     int              nuse;
00136 } ps;
00137 
00138 static int isfirst;
00139 static int dummy;
00140 static float sat;
00141 static cpl_frame *product_frame_chantab = NULL;
00142 static cpl_frame *product_frame_bpm = NULL;
00143 static cpl_frame *product_frame_diag1 = NULL;
00144 static cpl_frame *product_frame_diag2 = NULL;
00145 
00146 static char vircam_linearity_analyse_description[] =
00147 "vircam_linearity_analyse -- VIRCAM linearity mapping recipe.\n\n"
00148 "Form master dark images from the input raw frames and use these to\n"
00149 "dark correct a series of dome flat exposures Using the dark\n"
00150 "corrected dome flat series, work out linearity coefficients for\n"
00151 "each data channel. The program expects the following files in the SOF\n"
00152 "    Tag                   Description\n"
00153 "    -----------------------------------------------------------------------\n"
00154 "    %-21s A list of raw dome flat images\n"
00155 "    %-21s A list of raw dark images\n"
00156 "    %-21s The channel table\n"
00157 "    %-21s A list of raw dome flat images at the monitor exposure time\n"
00158 "    %-21s A list of raw dark images at the monitor exposure time\n"
00159 "The first three of these are required. The last two are only required if"
00160 "the light source monitoring algorithm is to be used"
00161 "\n";
00162 
00284 /* Function code */
00285 
00286 /*---------------------------------------------------------------------------*/
00294 /*---------------------------------------------------------------------------*/
00295 
00296 int cpl_plugin_get_info(cpl_pluginlist *list) {
00297     cpl_recipe  *recipe = cpl_calloc(1,sizeof(*recipe));
00298     cpl_plugin  *plugin = &recipe->interface;
00299     char alldesc[SZ_ALLDESC];
00300     (void)snprintf(alldesc,SZ_ALLDESC,vircam_linearity_analyse_description,
00301                    VIRCAM_LIN_DOME_RAW,VIRCAM_LIN_DARK_RAW,
00302                    VIRCAM_CAL_CHANTAB_INIT,VIRCAM_LIN_DOME_CHECK,
00303                    VIRCAM_LIN_DARK_CHECK);
00304 
00305     cpl_plugin_init(plugin,
00306                     CPL_PLUGIN_API,
00307                     VIRCAM_BINARY_VERSION,
00308                     CPL_PLUGIN_TYPE_RECIPE,
00309                     "vircam_linearity_analyse",
00310                     "VIRCAM linearity analysis recipe",
00311                     alldesc,
00312                     "Jim Lewis",
00313                     "jrl@ast.cam.ac.uk",
00314                     vircam_get_license(),
00315                     vircam_linearity_analyse_create,
00316                     vircam_linearity_analyse_exec,
00317                     vircam_linearity_analyse_destroy);
00318 
00319     cpl_pluginlist_append(list,plugin);
00320 
00321     return(0);
00322 }
00323 
00324 /*---------------------------------------------------------------------------*/
00333 /*---------------------------------------------------------------------------*/
00334 
00335 static int vircam_linearity_analyse_create(cpl_plugin *plugin) {
00336     cpl_recipe      *recipe;
00337     cpl_parameter   *p;
00338 
00339     /* Get the recipe out of the plugin */
00340 
00341     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
00342         recipe = (cpl_recipe *)plugin;
00343     else
00344         return(-1);
00345 
00346     /* Create the parameters list in the cpl_recipe object */
00347 
00348     recipe->parameters = cpl_parameterlist_new();
00349 
00350     /* Fill in the parameters. First the polynomial order */
00351 
00352     p = cpl_parameter_new_range("vircam.vircam_linearity_analyse.norder",
00353                                 CPL_TYPE_INT,
00354                                 "Order of polynomial fit",
00355                                 "vircam.vircam_linearity_analyse",
00356                                 4,1,6);
00357     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"nord");
00358     cpl_parameterlist_append(recipe->parameters,p);
00359 
00360     /* The lower threshold */
00361 
00362     p = cpl_parameter_new_value("vircam.vircam_linearity_analyse.lthr",
00363                                 CPL_TYPE_DOUBLE,
00364                                 "Lower bad pixel threshold",
00365                                 "vircam.vircam_linearity_analyse",8.0);
00366     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"lthr");
00367     cpl_parameterlist_append(recipe->parameters,p);
00368 
00369     /* The upper threshold */
00370 
00371     p = cpl_parameter_new_value("vircam.vircam_linearity_analyse.hthr",
00372                                 CPL_TYPE_DOUBLE,
00373                                 "Upper bad pixel threshold",
00374                                 "vircam.vircam_linearity_analyse",8.0);
00375     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"hthr");
00376     cpl_parameterlist_append(recipe->parameters,p);
00377 
00378     /* The maximum number of frames to be used in forming the bad pixel mask */
00379 
00380     p = cpl_parameter_new_value("vircam.vircam_linearity_analyse.maxbpmfr",
00381                                 CPL_TYPE_INT,
00382                                 "Maximum # frames used in bpm analysis",
00383                                 "vircam.vircam_linearity_analyse",10);
00384     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"maxbpmfr");
00385     cpl_parameterlist_append(recipe->parameters,p);
00386 
00387     /* The flag to allow statistics of the dome flat frames to be adjusted
00388        using a set of monitor exposures */
00389 
00390     p = cpl_parameter_new_value("vircam.vircam_linearity_analyse.adjust",
00391                                 CPL_TYPE_BOOL,
00392                                 "Adjust stats with monitor set",
00393                                 "vircam.vircam_linearity_analyse",1);
00394     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"adjust");
00395     cpl_parameterlist_append(recipe->parameters,p);
00396 
00397     /* The flag to allow diagnostic curves to be written out */
00398 
00399     p = cpl_parameter_new_value("vircam.vircam_linearity_analyse.diagnostic",
00400                                 CPL_TYPE_BOOL,
00401                                 "Write out diagnostic tables",
00402                                 "vircam.vircam_linearity_analyse",0);
00403     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"diagnostic");
00404     cpl_parameterlist_append(recipe->parameters,p);
00405 
00406     /* Extension number of input frames to use */
00407 
00408     p = cpl_parameter_new_range("vircam.vircam_linearity_analyse.extenum",
00409                                 CPL_TYPE_INT,
00410                                 "Extension number to be done, 0 == all",
00411                                 "vircam.vircam_linearity_analyse",
00412                                 1,0,16);
00413     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"ext");
00414     cpl_parameterlist_append(recipe->parameters,p);
00415 
00416     /* Get out of here */
00417 
00418     return(0);
00419 }
00420 
00421 /*---------------------------------------------------------------------------*/
00427 /*---------------------------------------------------------------------------*/
00428 
00429 static int vircam_linearity_analyse_exec(cpl_plugin *plugin) {
00430     cpl_recipe  *recipe;
00431 
00432     /* Get the recipe out of the plugin */
00433 
00434     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
00435         recipe = (cpl_recipe *)plugin;
00436     else
00437         return(-1);
00438 
00439     return(vircam_linearity_analyse(recipe->parameters,recipe->frames));
00440 }
00441 
00442 /*---------------------------------------------------------------------------*/
00448 /*---------------------------------------------------------------------------*/
00449 
00450 static int vircam_linearity_analyse_destroy(cpl_plugin *plugin) {
00451     cpl_recipe *recipe ;
00452 
00453     /* Get the recipe out of the plugin */
00454 
00455     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
00456         recipe = (cpl_recipe *)plugin;
00457     else
00458         return(-1);
00459 
00460     cpl_parameterlist_delete(recipe->parameters);
00461     return(0);
00462 }
00463 
00464 /*---------------------------------------------------------------------------*/
00471 /*---------------------------------------------------------------------------*/
00472 
00473 static int vircam_linearity_analyse(cpl_parameterlist *parlist,
00474                                     cpl_frameset *framelist) {
00475     const char *fctid="vircam_linearity_analyse";
00476     char colname[16];
00477     int i,jst,jfn,j,status,nlab,k,nbad,ngood,krem,n;
00478     int ndarks,ndomes,kk,live,ngood_flats,*bpm,nalloc,adjust,ndit;
00479     long np;
00480     float med,mindit,*exps,badfrac,expt;
00481     unsigned char *rejmask,*rejplus;
00482     double *dexps,**fdata,*d,*mjds,*mjdcheck,mjd,**cdata,*cf,fac;
00483     double facrng,maxdiff,*lindata;
00484     vir_fits **darks,**domes,*test,*outdark,*fframe;
00485     parquet *pp;
00486     cpl_parameter *p;
00487     cpl_propertylist *drs,*plist;
00488     cpl_image *outimage;
00489     cpl_frame *frame;
00490     
00491     /* Check validity of input frameset */
00492 
00493     if (framelist == NULL || cpl_frameset_get_size(framelist) <= 0) {
00494         cpl_msg_error(fctid,"Input framelist NULL or has no input data\n");
00495         return(-1);
00496     }
00497 
00498     /* Check the files in the frameset */
00499 
00500     if (vircam_frameset_fexists(framelist) != VIR_OK) {
00501         cpl_msg_error(fctid,"Input frameset is missing files. Check SOF");
00502         return(-1);
00503     }
00504 
00505     /* Initialise a few things */
00506 
00507     vircam_linearity_analyse_init();
00508 
00509     /* Get the parameters */
00510 
00511     p = cpl_parameterlist_find(parlist,
00512                                "vircam.vircam_linearity_analyse.norder");
00513     vircam_linearity_analyse_config.norder = cpl_parameter_get_int(p);
00514     p = cpl_parameterlist_find(parlist,
00515                                "vircam.vircam_linearity_analyse.lthr");
00516     vircam_linearity_analyse_config.lthr = (float)cpl_parameter_get_double(p);
00517     p = cpl_parameterlist_find(parlist,
00518                                "vircam.vircam_linearity_analyse.hthr");
00519     vircam_linearity_analyse_config.hthr = (float)cpl_parameter_get_double(p);
00520     p = cpl_parameterlist_find(parlist,
00521                                "vircam.vircam_linearity_analyse.maxbpmfr");
00522     vircam_linearity_analyse_config.maxbpmfr = cpl_parameter_get_int(p);
00523     p = cpl_parameterlist_find(parlist,
00524                                "vircam.vircam_linearity_analyse.adjust");
00525     vircam_linearity_analyse_config.adjust = cpl_parameter_get_bool(p);
00526     p = cpl_parameterlist_find(parlist,
00527                                "vircam.vircam_linearity_analyse.diagnostic");
00528     vircam_linearity_analyse_config.diagnostic = cpl_parameter_get_bool(p);
00529     p = cpl_parameterlist_find(parlist,
00530                                "vircam.vircam_linearity_analyse.extenum");
00531     vircam_linearity_analyse_config.extenum = cpl_parameter_get_int(p);
00532 
00533     /* Sort out raw from calib frames */
00534 
00535     if (vircam_dfs_set_groups(framelist) != VIR_OK) {
00536         cpl_msg_error(fctid,"Cannot identify RAW and CALIB frames");
00537         vircam_linearity_analyse_tidy(2);
00538         return(-1);
00539     }
00540 
00541     /* Get framelist labels */
00542 
00543     if ((ps.labels = cpl_frameset_labelise(framelist,vircam_compare_tags,
00544                                            &nlab)) == NULL) {
00545         cpl_msg_error(fctid,"Cannot labelise the input frames");
00546         vircam_linearity_analyse_tidy(2);
00547         return(-1);
00548     }
00549 
00550     /* Get the dome flat frames */
00551 
00552     if ((ps.domelist = vircam_frameset_subgroup(framelist,ps.labels,nlab,
00553                                                 VIRCAM_LIN_DOME_RAW)) == NULL) {
00554         cpl_msg_error(fctid,"Cannot find dome flat frames in input frameset");
00555         vircam_linearity_analyse_tidy(2);
00556         return(-1);
00557     }
00558     ps.ndomes = cpl_frameset_get_size(ps.domelist);
00559     ps.inherit = cpl_frameset_get_first(ps.domelist);
00560 
00561     /* Check to make sure that NDIT == 1 */
00562 
00563     plist = cpl_propertylist_load(cpl_frame_get_filename(cpl_frameset_get_frame(ps.domelist,0)),0);
00564     (void)vircam_pfits_get_ndit(plist,&ndit);
00565     freepropertylist(plist);
00566     if (ndit != 1) {
00567         cpl_msg_error(fctid,"NDIT=%d. Recipe requires that ndit == 1",ndit);
00568         vircam_linearity_analyse_tidy(2);
00569         return(-1);
00570     }
00571 
00572     /* Get the dark frames */
00573 
00574     if ((ps.darklist = vircam_frameset_subgroup(framelist,ps.labels,nlab,
00575                                                 VIRCAM_LIN_DARK_RAW)) == NULL) {
00576         cpl_msg_error(fctid,"Cannot find dark frames in input frameset");
00577         vircam_linearity_analyse_tidy(2);
00578         return(-1);
00579     }
00580     ps.ndarks = cpl_frameset_get_size(ps.darklist);
00581 
00582     /* If you are planning to adjust the stats for the dome frames by using
00583        a monitor exposure set, then read that frameset in now */
00584 
00585     if (vircam_linearity_analyse_config.adjust) {
00586         if ((ps.domecheck = vircam_frameset_subgroup(framelist,ps.labels,nlab,
00587                                                      VIRCAM_LIN_DOME_CHECK)) == NULL) {
00588             cpl_msg_info(fctid,"No monitor frames found in sof. No adjustments made to stats");
00589             vircam_linearity_analyse_config.adjust = 0;
00590             ps.ndomecheck = 0;
00591         } else {
00592             ps.ndomecheck = cpl_frameset_get_size(ps.domecheck);
00593             if ((ps.darkcheck = vircam_frameset_subgroup(framelist,ps.labels,
00594                                                          nlab,VIRCAM_LIN_DARK_CHECK)) == NULL) {
00595                 cpl_msg_info(fctid,"No darks for monitor frames found in sof. No adjustments made to stats");
00596                 vircam_linearity_analyse_config.adjust = 0;
00597                 ps.ndomecheck = 0;
00598                 freeframeset(ps.domecheck);
00599                 ps.ndarkcheck = 0;
00600             } else {
00601                 ps.ndarkcheck = cpl_frameset_get_size(ps.darkcheck);
00602             }
00603         }       
00604     }
00605     
00606     /* Check to see if there is a channel table. If so, then read it */
00607 
00608     if ((ps.chanfrm = vircam_frameset_subgroup_1(framelist,ps.labels,nlab,
00609                                                  VIRCAM_CAL_CHANTAB_INIT)) == NULL) {
00610         cpl_msg_error(fctid,"No initial channel table found");
00611         vircam_linearity_analyse_tidy(2);
00612         return(-1);
00613     }
00614 
00615     /* Group the domes and darks by exposure time */
00616 
00617     if (vircam_linearity_analyse_domedark_groups() != 0) {
00618         vircam_linearity_analyse_tidy(2);
00619         return(-1);
00620     }
00621 
00622     /* See if the number of exposure times is too small for the order of the
00623        polynomial you want to fit. If it is, the adjust the order */
00624 
00625     if (ps.nddg < vircam_linearity_analyse_config.norder+1) {
00626         cpl_msg_warning(fctid,
00627                         "Number of exposure times is too small: %d, order: %d\nTaking fit down to order %d",
00628                         ps.nddg,vircam_linearity_analyse_config.norder,
00629                         ps.nddg-1);
00630         vircam_linearity_analyse_config.norder = ps.nddg - 1;
00631     }
00632 
00633     /* Now, how many image extensions do we want to do? If the extension
00634        number is zero, then we loop for all possible extensions. If it
00635        isn't then we just do the extension specified */
00636 
00637     vircam_exten_range(vircam_linearity_analyse_config.extenum,
00638                        (const cpl_frame *)cpl_frameset_get_frame(ps.ddg[0].darks,0),
00639                        &jst,&jfn);
00640     if (jst == -1 || jfn == -1) {
00641         cpl_msg_error(fctid,"Unable to continue");
00642         vircam_linearity_analyse_tidy(2);
00643         return(-1);
00644     }
00645 
00646     /* Now loop for all the extensions and do the BPM analysis... */
00647 
00648     for (j = jst; j <= jfn; j++) {
00649         cpl_msg_info(fctid,"Beginning BPM work on extension %d",j);
00650         isfirst = (j == jst);
00651         dummy = 0;
00652         vircam_linearity_analyse_config.bad_pixel_stat = 0.0;
00653         vircam_linearity_analyse_config.bad_pixel_num = 0;
00654         vircam_linearity_analyse_config.linerror = 0.0;
00655         vircam_linearity_analyse_config.linearity = 0.0;
00656         vircam_linearity_analyse_config.facrng = 0.0;
00657         vircam_linearity_analyse_config.maxdiff = 0.0;
00658 
00659         /* Get some standard info in case we need it for dummy products */
00660 
00661         test = vircam_fits_load(cpl_frameset_get_first(ps.ddg[0].domes),
00662                                 CPL_TYPE_FLOAT,j);
00663         ps.plist = cpl_propertylist_duplicate(vircam_fits_get_phu(test));
00664         ps.elist = cpl_propertylist_duplicate(vircam_fits_get_ehu(test));
00665         ps.nx = cpl_image_get_size_x(vircam_fits_get_image(test));
00666         ps.ny = cpl_image_get_size_y(vircam_fits_get_image(test));
00667         vircam_fits_delete(test);
00668 
00669         /* Load up the channel table for this detector and verify it. Get 
00670            saturation level for this detector from FITS header */
00671 
00672         ps.chantab = vircam_tfits_load(ps.chanfrm,j);
00673         if (ps.chantab == NULL) {
00674             cpl_msg_error(fctid,"Channel table extension %d failed to load",j);
00675             dummy = 1;
00676             if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
00677                 return(-1);
00678             continue;
00679         } else if (vircam_chantab_verify(vircam_tfits_get_table(ps.chantab)) 
00680                    != VIR_OK) {
00681             cpl_msg_error(fctid,"Channel table extension %d has errors",j);
00682             dummy = 1;
00683             if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
00684                 return(-1);
00685             continue;
00686         }
00687         if (vircam_pfits_get_saturation(vircam_tfits_get_ehu(ps.chantab),
00688                                         &sat) != VIR_OK) {
00689             cpl_msg_error(fctid,"Channel table extension header %d missing saturation info",j);
00690             dummy = 1;
00691             if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
00692                 return(-1);
00693             vircam_linearity_analyse_tidy(1);
00694             continue;
00695         }
00696 
00697         /* Get the channel structure */
00698 
00699         vircam_chan_fill(vircam_tfits_get_table(ps.chantab),&pp,&np);
00700 
00701         /* If doing diagnostics, then create the tables now */
00702 
00703         if (vircam_linearity_analyse_config.diagnostic) {
00704             ps.diag1 = vircam_linearity_analyse_diagtab_init(np,ps.ndomes);
00705             if (vircam_linearity_analyse_config.adjust)
00706                 ps.diag2 = vircam_linearity_analyse_diagtab_init(np,ps.ndomecheck);
00707         }
00708 
00709         /* Check DETLIVE for this extension */
00710 
00711         if (vircam_pfits_get_detlive((const cpl_propertylist *)ps.elist,&live) 
00712             != VIR_OK) {
00713             cpl_msg_error(fctid,"No DET LIVE keyword in this extension");
00714             dummy = 1;
00715             if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0) 
00716                 return(-1);
00717             vircam_linearity_analyse_tidy(1);
00718             continue;
00719         }
00720         if (! live) {
00721             cpl_msg_info(fctid,"Detector flagged dead");
00722             dummy = 1;
00723             if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0) 
00724                 return(-1);
00725             vircam_linearity_analyse_tidy(1);
00726             continue;
00727         }
00728 
00729         /* Get the value of MINDIT so that you can work out the rough level
00730            of the flats before the reset frame was subtracted off */
00731 
00732         if (vircam_pfits_get_mindit((const cpl_propertylist *)ps.elist,
00733                                     &mindit) != VIR_OK) {
00734             cpl_msg_error(fctid,"No value of MINDIT found in extension %d",j);
00735             dummy = 1;
00736             if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
00737                 return(-1);
00738             continue;
00739         }
00740 
00741         /* Get a representative from each exposure group and check if it
00742            is saturated. If it is, then reject the group from further 
00743            analysis */
00744 
00745         ngood = 0;
00746         exps = cpl_malloc(ps.nddg*sizeof(float));
00747         ngood_flats = 0;
00748         for (i = 0; i < ps.nddg; i++) {
00749             test = vircam_fits_load(cpl_frameset_get_first(ps.ddg[i].domes),
00750                                     CPL_TYPE_FLOAT,j);
00751             med = cpl_image_get_median((const cpl_image*)vircam_fits_get_image(test));
00752             med *= (1.0 + mindit/ps.ddg[i].exptime);
00753             if (med > sat) {
00754                 ps.ddg[i].flag = SATURATE_FLAG;
00755             } else {
00756                 ngood++;
00757                 exps[ngood-1] = ps.ddg[i].exptime;
00758                 ngood_flats += ps.ddg[i].ndomes;
00759                 ps.ddg[i].flag = OK_FLAG;
00760             }
00761             vircam_fits_delete(test);
00762         }
00763         exps = cpl_realloc(exps,ngood*sizeof(float));
00764                
00765         /* Are there enough non-saturated exposures for linearity fit? */
00766 
00767         if (ngood <  vircam_linearity_analyse_config.norder+1) {
00768             cpl_msg_info(fctid,"Too few unsaturated flats for linearity fit for extension %d",j);
00769             dummy = 1;
00770             cpl_free(exps);
00771             if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0) 
00772                 return(-1);
00773             continue;
00774         }           
00775 
00776         /* Sort the exposure array */
00777 
00778         vircam_sort(&exps,ngood,1);
00779 
00780         /* Loop for each exposure time. When you have enough flats, then
00781            you can quit this loop */
00782 
00783         ps.nuse = min(vircam_linearity_analyse_config.maxbpmfr,ngood_flats);
00784         ps.nflatlist = 0;
00785         ps.flatlist = cpl_malloc(ps.nuse*sizeof(vir_fits *));
00786         for (i = ngood-1; i >= 0; i--) {
00787             krem = -1;
00788             for (k = 0; k <= ps.nddg; k++) {
00789                 if (ps.ddg[k].exptime == exps[i]) {
00790                     krem = k;
00791                     break;
00792                 }
00793             }
00794 
00795             /* Load the dark frames from this exposure time */
00796 
00797             darks = vircam_fits_load_list(ps.ddg[krem].darks,CPL_TYPE_FLOAT,j);
00798             ndarks = ps.ddg[krem].ndarks;
00799             if (darks == NULL) {
00800                 cpl_msg_error(fctid,"Error loading darks extension %d, exptime %g",
00801                               j,ps.ddg[krem].exptime);
00802                 continue;
00803             }
00804 
00805             /* Form a mean dark for this exposure time. If there is only one
00806                then don't bother, just load up that one frame. */
00807 
00808             if (ndarks == 1) {
00809                 outdark = vircam_fits_duplicate(darks[0]);
00810             } else {
00811                 status = VIR_OK;
00812                 (void)vircam_imcombine(darks,ndarks,1,1,1,5.0,&outimage,
00813                                        &rejmask,&rejplus,&drs,&status);
00814                 freespace(rejmask);
00815                 freespace(rejplus);
00816                 freepropertylist(drs);
00817                 if (status != VIR_OK) {
00818                     cpl_msg_error(fctid,"Dark combine failure extension %d exposure %g",
00819                                   j,ps.ddg[krem].exptime);
00820                     freefitslist(darks,ndarks);
00821                     continue;
00822                 }
00823                 outdark = vircam_fits_wrap(outimage,darks[0],NULL,NULL);
00824             }
00825             freefitslist(darks,ndarks);
00826 
00827             /* Load the flats for this group */
00828 
00829             domes = vircam_fits_load_list(ps.ddg[krem].domes,CPL_TYPE_FLOAT,j);
00830             ndomes = ps.ddg[krem].ndomes;
00831             if (domes == NULL) {
00832                 cpl_msg_error(fctid,"Error loading domes extension %d, exptime %g",
00833                               j,ps.ddg[i].exptime);
00834                 freefits(outdark);
00835                 continue;
00836             }
00837             
00838             /* Now loop for each flat in this group or until you have
00839                filled the flats buffer */
00840 
00841             for (kk = 0; kk < ndomes; kk++) {
00842                 status = VIR_OK;
00843                 vircam_darkcor(domes[kk],outdark,1.0,&status);
00844                 ps.flatlist[ps.nflatlist] = vircam_fits_duplicate(domes[kk]);
00845                 ps.ddg[krem].proc[kk] = ps.flatlist[ps.nflatlist];
00846                 ps.nflatlist++;
00847                 if (ps.nflatlist == ps.nuse)
00848                     break;
00849             }
00850 
00851             /* Tidy up a bit */
00852 
00853             freefitslist(domes,ndomes);
00854             freefits(outdark);
00855 
00856             /* Do we have enough yet? */
00857 
00858             if (ps.nflatlist == ps.nuse) 
00859                 break;
00860         }
00861         freespace(exps);
00862         ps.flatlist = cpl_realloc(ps.flatlist,
00863                                   (ps.nflatlist)*sizeof(vir_fits *));
00864         
00865         /* Generate a bad pixel mask now */
00866 
00867         status = VIR_OK;
00868         (void)vircam_genbpm(ps.flatlist,ps.nflatlist,
00869                             vircam_linearity_analyse_config.lthr,
00870                             vircam_linearity_analyse_config.hthr,
00871                             &(ps.bpm_array),&nbad,&badfrac,&status);
00872         bpm = cpl_array_get_data_int(ps.bpm_array);
00873 
00874         /* Store away some useful info */
00875 
00876         vircam_linearity_analyse_config.bad_pixel_num = nbad;
00877         vircam_linearity_analyse_config.bad_pixel_stat = badfrac;
00878 
00879         /* Right. Free the pointer for flatlist, but don't delete the 
00880            vir_fits structures in the list because they are copied into
00881            the ddg structure */
00882 
00883         freespace(ps.flatlist);
00884         ps.nflatlist = 0;
00885 
00886         /* Get an initial allocation of space to hold the stats */
00887 
00888         nalloc = 16;
00889         fdata = cpl_malloc(nalloc*sizeof(double *));
00890         dexps = cpl_malloc(nalloc*sizeof(double));
00891         mjds = cpl_malloc(nalloc*sizeof(double));
00892 
00893         /* Loop through the ddg structure, missing out any that have
00894            overexposed flats. Work out the stats for the images that 
00895            remain. */
00896 
00897         cpl_msg_info(fctid,"Beginning linearity work on extension %d",j);
00898         ps.nfdata = 0;
00899         outdark = NULL;
00900         for (i = 0; i < ps.nddg; i++) {
00901             if (ps.ddg[i].flag == SATURATE_FLAG)
00902                 continue;
00903             for (k = 0; k < ps.ddg[i].ndomes; k++) {
00904 
00905                 /* If this particular frame wasn't processed, then you need
00906                    to form a dark for this group. */
00907 
00908                 if (ps.ddg[i].proc[k] == NULL) {
00909                     if (outdark == NULL) {
00910 
00911                         /* Load the dark frames from this exposure time */
00912 
00913                         darks = vircam_fits_load_list(ps.ddg[i].darks,
00914                                                       CPL_TYPE_FLOAT,j);
00915                         ndarks = ps.ddg[i].ndarks;
00916                         if (darks == NULL) {
00917                             cpl_msg_error(fctid,"Error loading darks extension %d, exptime %g",
00918                                           j,ps.ddg[i].exptime);
00919                             continue;
00920                         }
00921 
00922                         /* Form a mean dark for this exposure time. If there 
00923                            is only one then don't bother, just load up that 
00924                            one frame. */
00925 
00926                         if (ps.ddg[i].ndarks == 1) {
00927                             outdark = vircam_fits_duplicate(darks[0]);
00928                         } else {
00929                             status = VIR_OK;
00930                             (void)vircam_imcombine(darks,ndarks,1,1,1,5.0,
00931                                                    &outimage,&rejmask,&rejplus,
00932                                                    &drs,&status);
00933                             freespace(rejmask);
00934                             freespace(rejplus);
00935                             freepropertylist(drs);
00936                             if (status != VIR_OK) {
00937                                 cpl_msg_error(fctid,
00938                                               "Dark combine failure extension %d exposure %g",
00939                                               j,ps.ddg[i].exptime);
00940                                 freefitslist(darks,ndarks);
00941                                 continue;
00942                             }
00943                             outdark = vircam_fits_wrap(outimage,darks[0],
00944                                                        NULL,NULL);
00945                         }
00946                         freefitslist(darks,ndarks);
00947                     }
00948 
00949                     /* Load the flat and dark correct it */
00950 
00951                     frame = cpl_frameset_get_frame(ps.ddg[i].domes,k);
00952                     fframe = vircam_fits_load(frame,CPL_TYPE_FLOAT,j);
00953                     vircam_darkcor(fframe,outdark,1.0,&status);
00954 
00955                 /* If this frame has already been corrected, then use it */
00956 
00957                 } else {
00958                     fframe = ps.ddg[i].proc[k];
00959                 }
00960 
00961                 /* Generate the stats for this frame and store it */
00962 
00963                 d = vircam_linearity_analyse_genstat(fframe,bpm,pp,np);
00964                 if (ps.nfdata >= nalloc) {
00965                     nalloc += 16;
00966                     fdata = cpl_realloc(fdata,nalloc*sizeof(double *));
00967                     dexps = cpl_realloc(dexps,nalloc*sizeof(double));
00968                     mjds = cpl_realloc(mjds,nalloc*sizeof(double));
00969                 }
00970                 (void)vircam_pfits_get_mjd(vircam_fits_get_phu(fframe),&mjd);
00971                 mjds[ps.nfdata] = mjd;
00972                 dexps[ps.nfdata] = (double)(ps.ddg[i].exptime);         
00973                 fdata[ps.nfdata] = d;           
00974 
00975                 /* If doing diagnostic curves, then add the relevant
00976                    information to the first table now */
00977 
00978                 if (ps.diag1 != NULL) {
00979                     cpl_table_set_string(ps.diag1,"filename",ps.nfdata,
00980                                          vircam_fits_get_filename(fframe));
00981                     cpl_table_set_double(ps.diag1,"exptime",ps.nfdata,
00982                                          dexps[ps.nfdata]);
00983                     cpl_table_set_double(ps.diag1,"mjd",ps.nfdata,mjd);
00984                     for (n = 1; n <= np; n++) {
00985                         snprintf(colname,16,"rawflux_%02d",n);
00986                         cpl_table_set_double(ps.diag1,colname,ps.nfdata,d[n-1]);
00987                     }
00988                 }
00989                 if (ps.ddg[i].proc[k] != NULL) { 
00990                     freefits(ps.ddg[i].proc[k]);
00991                 } else {
00992                     freefits(fframe);
00993                 }
00994                 ps.nfdata++;
00995             }
00996             freefits(outdark);
00997         }
00998         freefits(outdark);
00999         if (ps.diag1 != NULL) 
01000             cpl_table_set_size(ps.diag1,ps.nfdata);
01001 
01002         /* Now, if we are going to tweak the stats using the monitor exposure
01003            then we should do that now */
01004 
01005         if (vircam_linearity_analyse_config.adjust) {
01006 
01007             /* Get some workspace for the data array */
01008 
01009             cdata = cpl_malloc(ps.ndomecheck*sizeof(double *));
01010 
01011             /* Get the exposure time for the monitor set and make sure
01012                that the set isn't saturated */
01013 
01014             test = vircam_fits_load(cpl_frameset_get_first(ps.domecheck),
01015                                     CPL_TYPE_FLOAT,j);
01016             (void)vircam_pfits_get_exptime(vircam_fits_get_phu(test),&expt);
01017             med = cpl_image_get_median((const cpl_image*)vircam_fits_get_image(test));
01018             med *= (1.0 + mindit/expt);
01019             adjust = 1;
01020             if (med > sat) {
01021                 cpl_msg_info(fctid,"Monitor exposures saturated. No drift adjustment made");
01022                 adjust = 0;
01023             }
01024             vircam_fits_delete(test);
01025 
01026             /* Ok assuming all that's going well, then form a mean dark */
01027 
01028             if (adjust) {
01029 
01030                 darks = vircam_fits_load_list(ps.darkcheck,CPL_TYPE_FLOAT,j);
01031                 ndarks = ps.ndarkcheck;
01032                 if (darks == NULL) {
01033                     cpl_msg_error(fctid,
01034                                   "Error loading check darks extension %d",j);
01035                     continue;
01036                 }
01037 
01038                 /* Form a mean dark for this exposure time. If there 
01039                    is only one then don't bother, just load up that 
01040                    one frame. */
01041 
01042                 if (ndarks == 1) {
01043                     outdark = vircam_fits_duplicate(darks[0]);
01044                 } else {
01045                     status = VIR_OK;
01046                     (void)vircam_imcombine(darks,ndarks,1,1,1,5.0,
01047                                            &outimage,&rejmask,&rejplus,
01048                                            &drs,&status);
01049                     freespace(rejmask);
01050                     freespace(rejplus);
01051                     freepropertylist(drs);
01052                     if (status != VIR_OK) {
01053                         cpl_msg_error(fctid,
01054                                       "Combine failure extension %d monitor",j);
01055                         freefitslist(darks,ndarks);
01056                         continue;
01057                     }
01058                     outdark = vircam_fits_wrap(outimage,darks[0],
01059                                                NULL,NULL);
01060                 }
01061                 freefitslist(darks,ndarks);             
01062 
01063                 /* Now, loop through the monitor domes, dark correct and then
01064                    do the stats */
01065                 
01066                 mjdcheck = cpl_malloc(ps.ndomecheck*sizeof(double));
01067                 for (i = 0; i < ps.ndomecheck; i++) {
01068                     frame = cpl_frameset_get_frame(ps.domecheck,i);
01069                     fframe = vircam_fits_load(frame,CPL_TYPE_FLOAT,j);
01070                     vircam_darkcor(fframe,outdark,1.0,&status);
01071                     d = vircam_linearity_analyse_genstat(fframe,bpm,pp,np);
01072                     cdata[i] = d;
01073                     vircam_pfits_get_mjd(vircam_fits_get_phu(fframe),&mjd);
01074                     vircam_pfits_get_exptime(vircam_fits_get_phu(fframe),&expt);
01075                     mjdcheck[i] = mjd;
01076 
01077                     /* If doing diagnostics then fill in the table now */
01078 
01079                     if (ps.diag2 != NULL) {
01080                         cpl_table_set_string(ps.diag2,"filename",i,
01081                                              vircam_fits_get_filename(fframe));
01082                         cpl_table_set_double(ps.diag2,"exptime",i,(double)expt);
01083                         cpl_table_set_double(ps.diag2,"mjd",i,mjd);
01084                         for (n = 1; n <= np; n++) {
01085                             snprintf(colname,16,"rawflux_%02d",n);
01086                             cpl_table_set_double(ps.diag2,colname,i,
01087                                                  d[n-1]);
01088                             snprintf(colname,16,"linflux_%02d",n);
01089                             cpl_table_set_double(ps.diag2,colname,i,
01090                                                  d[n-1]);
01091                         }
01092                     }
01093                     freefits(fframe);
01094                 }
01095                 freefits(outdark);
01096 
01097                 /* Generate the correction factors now */
01098 
01099                 cf = vircam_linearity_tweakfac(cdata,mjdcheck,ps.ndomecheck,
01100                                                np,&facrng,&maxdiff);
01101                 vircam_linearity_analyse_config.facrng = 100.0*(float)facrng;
01102                 vircam_linearity_analyse_config.maxdiff = 100.0*(float)maxdiff;
01103                 if (ps.diag2 != NULL) {
01104                     for (i = 0; i < ps.ndomecheck; i++) 
01105                         cpl_table_set_double(ps.diag2,"adjust_fac",i,cf[i]);
01106                 }
01107 
01108                 /* Ok, now do the correction for each of the linearity
01109                    sequence frames */
01110 
01111                 for (i = 0; i < ps.nfdata; i++) {
01112                     mjd = mjds[i];
01113                     krem = -1;
01114                     for (k = 0; k < ps.ndomecheck; k++) {
01115                         if (mjd < mjdcheck[k]) {
01116                             krem = k;
01117                             break;
01118                         }
01119                     }
01120                     if (krem == -1) {
01121                         fac = cf[ps.ndomecheck-1];
01122                     } else if (krem == 0) {
01123                         fac = cf[0];
01124                     } else {
01125                         fac = 0.5*(cf[krem -1]  + cf[krem]);
01126                     }
01127                     for (k = 0; k < np; k++)
01128                         fdata[i][k] /= fac;
01129                     if (ps.diag1 != NULL) 
01130                         cpl_table_set_double(ps.diag1,"adjust_fac",i,fac);
01131                 }
01132 
01133                 /* Get rid of some stuff now */
01134 
01135                 freespace2(cdata,ps.ndomecheck);
01136                 freespace(cf);
01137                 freespace(mjdcheck);
01138             }
01139         }
01140 
01141         /* Do some intermediate tidying */
01142         
01143         freespace(mjds);
01144         vircam_chan_free(np,&pp);
01145 
01146         /* Right, there should be no images left in memory now. Do the
01147            linearity analysis now. */
01148 
01149         (void)vircam_genlincur(fdata,ps.nfdata,dexps,(double)mindit,ps.chantab,
01150                                vircam_linearity_analyse_config.norder,
01151                                &(ps.lchantab),&lindata,&status);
01152         if (ps.diag1 != NULL) {
01153             for (i = 0; i < ps.nfdata; i++) {
01154                 for (n = 0; n < np; n++) {
01155                     snprintf(colname,16,"linflux_%02d",n+1);
01156                     cpl_table_set_double(ps.diag1,colname,i,lindata[i*np+n]);
01157                 }
01158             }
01159         }
01160         freespace2(fdata,ps.nfdata);
01161         freespace(dexps);
01162         freespace(lindata);
01163         if (status != VIR_OK) {
01164             cpl_msg_error(fctid,"Linearity curve fit failed extension %d",j);
01165             dummy = 1;
01166             if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
01167                 return(-1);
01168             vircam_linearity_analyse_tidy(1);
01169             continue;
01170         }
01171         
01172         /* Get some QC1 parameters by finding the average fit error and
01173            the average percentage non-linearity */
01174 
01175         vircam_linearity_analyse_config.linearity = 
01176             (float)cpl_table_get_column_mean(ps.lchantab,"lin_10000");
01177         vircam_linearity_analyse_config.linerror = 
01178             (float)cpl_table_get_column_mean(ps.lchantab,"lin_10000_err");
01179 
01180         /* Save new linearity info */
01181 
01182         if (vircam_linearity_analyse_lastbit(j,framelist,parlist) != 0)
01183             return(-1);
01184     }
01185 
01186     /* Get out of here */
01187 
01188     vircam_linearity_analyse_tidy(2);
01189     return(0);
01190 }
01191 
01192 
01193 /*---------------------------------------------------------------------------*/
01200 /*---------------------------------------------------------------------------*/
01201 
01202 static int vircam_linearity_analyse_save(cpl_frameset *framelist,
01203                                          cpl_parameterlist *parlist) {
01204     cpl_propertylist *plist,*elist,*pafprop;
01205     cpl_image *outimg;
01206     const char *outtab = "lchantab.fits";
01207     const char *outbpm = "bpm.fits";
01208     const char *outtabpaf = "lchantab";
01209     const char *outbpmpaf = "bpm";
01210     const char *outdiag1 = "ldiag1.fits";
01211     const char *outdiag2 = "ldiag2.fits";
01212     const char *fctid = "vircam_linearity_analyse_save";
01213     const char *recipeid = "vircam_linearity_analyse";
01214     int nx,ny,nord,*bpm,i;
01215 
01216     /* Do some stuff for the first extension to set up the frame */
01217 
01218     nord = vircam_linearity_analyse_config.norder;
01219     if (isfirst) {
01220 
01221         /* Set up the output frame */
01222 
01223         product_frame_chantab = cpl_frame_new();
01224         cpl_frame_set_filename(product_frame_chantab,outtab);
01225         cpl_frame_set_tag(product_frame_chantab,VIRCAM_PRO_CHANTAB);
01226         cpl_frame_set_type(product_frame_chantab,CPL_FRAME_TYPE_TABLE);
01227         cpl_frame_set_group(product_frame_chantab,CPL_FRAME_GROUP_PRODUCT);
01228         cpl_frame_set_level(product_frame_chantab,CPL_FRAME_LEVEL_FINAL);
01229 
01230         /* Set up the PRO keywords for primary */
01231         
01232         ps.phupaf = vircam_paf_phu_items(ps.plist);
01233         plist = cpl_propertylist_duplicate(ps.plist);
01234         vircam_dfs_set_product_primary_header(plist,product_frame_chantab,
01235                                               framelist,parlist,
01236                                               (char *)recipeid,
01237                                               "PRO-1.15",ps.inherit,0);
01238 
01239         /* Now define the table propertylist and give it an extension name and 
01240            PRO keywords */
01241 
01242         elist = cpl_propertylist_duplicate(ps.elist);
01243         cpl_propertylist_update_float(elist,"ESO DET SATURATION",sat);
01244         vircam_dfs_set_product_exten_header(elist,product_frame_chantab,
01245                                             framelist,parlist,(char *)recipeid,
01246                                             "PRO-1.15",ps.inherit);
01247         
01248         /* Add QC1 info */
01249 
01250         cpl_propertylist_update_float(elist,"ESO QC LINEARITY",
01251                                       vircam_linearity_analyse_config.linearity);
01252         cpl_propertylist_set_comment(elist,"ESO QC LINEARITY",
01253                                      "% non-linearity at 10000 ADU");
01254         cpl_propertylist_update_float(elist,"ESO QC LINERROR",
01255                                       vircam_linearity_analyse_config.linerror);
01256         cpl_propertylist_set_comment(elist,"ESO QC LINERROR",
01257                                      "% error non-linearity at 10000 ADU");
01258         cpl_propertylist_update_float(elist,"ESO QC SCREEN_TOTAL",
01259                                       vircam_linearity_analyse_config.facrng);
01260         cpl_propertylist_set_comment(elist,"ESO QC SCREEN_TOTAL",
01261                                      "total % range in screen variation");
01262         cpl_propertylist_update_float(elist,"ESO QC SCREEN_STEP",
01263                                       vircam_linearity_analyse_config.maxdiff);
01264         cpl_propertylist_set_comment(elist,"ESO QC SCREEN_STEP",
01265                                      "maximum % step in screen variation");
01266         cpl_propertylist_update_int(elist,"ESO PRO DATANCOM",ps.nfdata);
01267 
01268         /* Set up a dummy table if necessary */
01269 
01270         if (dummy == 1) {
01271             vircam_dummy_property(elist);
01272             if (ps.lchantab == NULL) 
01273                 ps.lchantab = vircam_chantab_new(nord,vircam_tfits_get_table(ps.chantab));
01274         }
01275 
01276         /* And finally save the table */
01277  
01278         if (cpl_table_save(ps.lchantab,plist,elist,outtab,CPL_IO_DEFAULT)
01279             != CPL_ERROR_NONE) {
01280             cpl_msg_error(fctid,"Cannot save product table extension");
01281             freepropertylist(plist);
01282             freepropertylist(elist);
01283             return(-1);
01284         }
01285         cpl_frameset_insert(framelist,product_frame_chantab);
01286 
01287         /* Write PAF */
01288 
01289         pafprop = vircam_paf_req_items(elist);
01290         vircam_merge_propertylists(pafprop,ps.phupaf);
01291         vircam_paf_append(pafprop,plist,"ESO INS FILT1 NAME");
01292         vircam_paf_append(pafprop,elist,"ESO PRO CATG");
01293         vircam_paf_append(pafprop,elist,"ESO PRO DATANCOM");
01294         if (vircam_paf_print((char *)outtabpaf,"VIRCAM/vircam_linearity_analyse",
01295                              "QC file",pafprop) != VIR_OK)
01296             cpl_msg_warning(fctid,"Unable to save PAF for linearity table");
01297         cpl_propertylist_delete(pafprop);
01298 
01299         /* Quick tidy */
01300 
01301         freepropertylist(plist);
01302         freepropertylist(elist);
01303 
01304         /* Set up the output bad pixel mask primary */
01305 
01306         product_frame_bpm = cpl_frame_new();
01307         cpl_frame_set_filename(product_frame_bpm,outbpm);
01308         cpl_frame_set_tag(product_frame_bpm,VIRCAM_PRO_BPM);
01309         cpl_frame_set_type(product_frame_bpm,CPL_FRAME_TYPE_IMAGE);
01310         cpl_frame_set_group(product_frame_bpm,CPL_FRAME_GROUP_PRODUCT);
01311         cpl_frame_set_level(product_frame_bpm,CPL_FRAME_LEVEL_FINAL);
01312 
01313         /* Set up the PRO keywords for primary header in the bpm */
01314 
01315         plist = ps.plist;
01316         vircam_dfs_set_product_primary_header(plist,product_frame_bpm,
01317                                               framelist,parlist,
01318                                               (char *)recipeid,"PRO-1.15",
01319                                               ps.inherit,0);
01320 
01321         /* Now save the PHU 'image' */
01322 
01323         if (cpl_image_save(NULL,outbpm,CPL_BPP_8_UNSIGNED,plist,
01324                            CPL_IO_DEFAULT) != CPL_ERROR_NONE) {
01325             cpl_msg_error(fctid,"Cannot save product PHU");
01326             cpl_frame_delete(product_frame_bpm);
01327             return(-1);
01328         }
01329         cpl_frameset_insert(framelist,product_frame_bpm);
01330 
01331         /* Section for saving diagnostic tables. First the linearity
01332            sequence diagnostics */
01333 
01334         if (ps.diag1 != NULL) {
01335 
01336             /* Set up the output frame */
01337 
01338             product_frame_diag1 = cpl_frame_new();
01339             cpl_frame_set_filename(product_frame_diag1,outdiag1);
01340             cpl_frame_set_tag(product_frame_diag1,VIRCAM_PRO_LIN_DIAG1);
01341             cpl_frame_set_type(product_frame_diag1,CPL_FRAME_TYPE_TABLE);
01342             cpl_frame_set_group(product_frame_diag1,CPL_FRAME_GROUP_PRODUCT);
01343             cpl_frame_set_level(product_frame_diag1,CPL_FRAME_LEVEL_FINAL);
01344 
01345             /* Set up the PRO keywords for primary */
01346 
01347             plist = cpl_propertylist_duplicate(ps.plist);
01348             vircam_dfs_set_product_primary_header(plist,product_frame_diag1,
01349                                                   framelist,parlist,
01350                                                   (char *)recipeid,
01351                                                   "PRO-1.15",ps.inherit,0);
01352 
01353             /* Now define the table propertylist and give it an extension name and 
01354                PRO keywords */
01355 
01356             elist = cpl_propertylist_duplicate(ps.elist);
01357             vircam_dfs_set_product_exten_header(elist,product_frame_diag1,
01358                                                 framelist,parlist,
01359                                                 (char *)recipeid,"PRO-1.15",
01360                                                 ps.inherit);
01361 
01362             /* Set up a dummy property if necessary */
01363 
01364             if (dummy == 1) 
01365                 vircam_dummy_property(elist);
01366 
01367             /* And finally save the table */
01368 
01369             if (cpl_table_save(ps.diag1,plist,elist,outdiag1,CPL_IO_DEFAULT)
01370                 != CPL_ERROR_NONE) {
01371                 cpl_msg_error(fctid,"Cannot save product table extension");
01372                 freepropertylist(plist);
01373                 freepropertylist(elist);
01374                 return(-1);
01375             }
01376             cpl_frameset_insert(framelist,product_frame_diag1);
01377             freepropertylist(plist);
01378             freepropertylist(elist);
01379         }
01380 
01381         /* Now the monitor sequence diagnostics */
01382 
01383         if (ps.diag2 != NULL) {
01384 
01385             /* Set up the output frame */
01386 
01387             product_frame_diag2 = cpl_frame_new();
01388             cpl_frame_set_filename(product_frame_diag2,outdiag2);
01389             cpl_frame_set_tag(product_frame_diag2,VIRCAM_PRO_LIN_DIAG2);
01390             cpl_frame_set_type(product_frame_diag2,CPL_FRAME_TYPE_TABLE);
01391             cpl_frame_set_group(product_frame_diag2,CPL_FRAME_GROUP_PRODUCT);
01392             cpl_frame_set_level(product_frame_diag2,CPL_FRAME_LEVEL_FINAL);
01393 
01394             /* Set up the PRO keywords for primary */
01395 
01396             plist = cpl_propertylist_duplicate(ps.plist);
01397             vircam_dfs_set_product_primary_header(plist,product_frame_diag2,
01398                                                   framelist,parlist,
01399                                                   (char *)recipeid,
01400                                                   "PRO-1.15",ps.inherit,0);
01401 
01402             /* Now define the table propertylist and give it an extension name 
01403                and PRO keywords */
01404 
01405             elist = cpl_propertylist_duplicate(ps.elist);
01406             vircam_dfs_set_product_exten_header(elist,product_frame_diag2,
01407                                                 framelist,parlist,
01408                                                 (char *)recipeid,"PRO-1.15",
01409                                                 ps.inherit);
01410 
01411             /* Set up a dummy property if necessary */
01412 
01413             if (dummy == 1) 
01414                 vircam_dummy_property(elist);
01415 
01416             /* And finally save the table */
01417 
01418             if (cpl_table_save(ps.diag2,plist,elist,outdiag2,CPL_IO_DEFAULT)
01419                 != CPL_ERROR_NONE) {
01420                 cpl_msg_error(fctid,"Cannot save product table extension");
01421                 freepropertylist(plist);
01422                 freepropertylist(elist);
01423                 return(-1);
01424             }
01425             cpl_frameset_insert(framelist,product_frame_diag2);
01426             freepropertylist(plist);
01427             freepropertylist(elist);
01428         }
01429 
01430     /* Section for all other extensions */
01431 
01432     } else {
01433 
01434         /* Do the table extension PRO keywords */
01435 
01436         elist = cpl_propertylist_duplicate(ps.elist);
01437         cpl_propertylist_update_float(elist,"ESO DET SATURATION",sat);
01438         vircam_dfs_set_product_exten_header(elist,product_frame_chantab,
01439                                             framelist,parlist,
01440                                             (char *)recipeid,
01441                                             "PRO-1.15",ps.inherit);
01442 
01443         /* Add QC1 info */
01444 
01445         cpl_propertylist_update_float(elist,"ESO QC LINEARITY",
01446                                       vircam_linearity_analyse_config.linearity);
01447         cpl_propertylist_set_comment(elist,"ESO QC LINEARITY",
01448                                      "% non-linearity at 10000 ADU");
01449         cpl_propertylist_update_float(elist,"ESO QC LINERROR",
01450                                       vircam_linearity_analyse_config.linerror);
01451         cpl_propertylist_set_comment(elist,"ESO QC LINERROR",
01452                                      "% error non-linearity at 10000 ADU");
01453         cpl_propertylist_update_float(elist,"ESO QC SCREEN_TOTAL",
01454                                       vircam_linearity_analyse_config.facrng);
01455         cpl_propertylist_set_comment(elist,"ESO QC SCREEN_TOTAL",
01456                                      "total % range in screen variation");
01457         cpl_propertylist_update_float(elist,"ESO QC SCREEN_STEP",
01458                                       vircam_linearity_analyse_config.maxdiff);
01459         cpl_propertylist_set_comment(elist,"ESO QC SCREEN_STEP",
01460                                      "maximum % step in screen variation");
01461         cpl_propertylist_update_int(elist,"ESO PRO DATANCOM",ps.nfdata);
01462 
01463         /* Set up a dummy table if necessary */
01464 
01465         if (dummy == 1) {
01466             vircam_dummy_property(elist);
01467             if (ps.lchantab == NULL) 
01468                 ps.lchantab = vircam_chantab_new(nord,vircam_tfits_get_table(ps.chantab));
01469         }
01470 
01471         /* And finally save the table */
01472  
01473         if (cpl_table_save(ps.lchantab,NULL,elist,outtab,CPL_IO_EXTEND)
01474             != CPL_ERROR_NONE) {
01475             cpl_msg_error(fctid,"Cannot save product table extension");
01476             freepropertylist(elist);
01477             return(-1);
01478         }
01479 
01480         /* Write PAF */
01481 
01482         pafprop = vircam_paf_req_items(elist);
01483         vircam_merge_propertylists(pafprop,ps.phupaf);
01484         vircam_paf_append(pafprop,ps.plist,"ESO INS FILT1 NAME");
01485         vircam_paf_append(pafprop,elist,"ESO PRO CATG");
01486         vircam_paf_append(pafprop,elist,"ESO PRO DATANCOM");
01487         if (vircam_paf_print((char *)outtabpaf,"VIRCAM/vircam_linearity_analyse",
01488                              "QC file",pafprop) != VIR_OK)
01489             cpl_msg_warning(fctid,"Unable to save PAF for BPM");
01490         cpl_propertylist_delete(pafprop);
01491 
01492         /* Quick tidy */
01493 
01494         freepropertylist(elist);
01495 
01496         /* Now the diagnostic tables */
01497 
01498         if (ps.diag1 != NULL) {
01499             elist = cpl_propertylist_duplicate(ps.elist);
01500             vircam_dfs_set_product_exten_header(elist,product_frame_diag1,
01501                                                 framelist,parlist,
01502                                                 (char *)recipeid,"PRO-1.15",
01503                                                 ps.inherit);
01504 
01505             /* Set up a dummy property if necessary */
01506 
01507             if (dummy == 1) 
01508                 vircam_dummy_property(elist);
01509 
01510             /* And finally save the table */
01511 
01512             if (cpl_table_save(ps.diag1,NULL,elist,outdiag1,CPL_IO_EXTEND)
01513                 != CPL_ERROR_NONE) {
01514                 cpl_msg_error(fctid,"Cannot save product table extension");
01515                 freepropertylist(elist);
01516                 return(-1);
01517             }
01518             freepropertylist(elist);
01519         }
01520         if (ps.diag2 != NULL) {
01521             elist = cpl_propertylist_duplicate(ps.elist);
01522             vircam_dfs_set_product_exten_header(elist,product_frame_diag2,
01523                                                 framelist,parlist,
01524                                                 (char *)recipeid,"PRO-1.15",
01525                                                 ps.inherit);
01526 
01527             /* Set up a dummy property if necessary */
01528 
01529             if (dummy == 1) 
01530                 vircam_dummy_property(elist);
01531 
01532             /* And finally save the table */
01533 
01534             if (cpl_table_save(ps.diag2,NULL,elist,outdiag2,CPL_IO_EXTEND)
01535                 != CPL_ERROR_NONE) {
01536                 cpl_msg_error(fctid,"Cannot save product table extension");
01537                 freepropertylist(elist);
01538                 return(-1);
01539             }
01540             freepropertylist(elist);
01541         }
01542 
01543     }
01544 
01545     /* Save the bpm extension now */
01546 
01547     plist = ps.elist;
01548     nx = ps.nx;
01549     ny = ps.ny;
01550     if (dummy && ps.bpm_array == NULL) {
01551         ps.bpm_array = cpl_array_new(nx*ny,CPL_TYPE_INT);
01552         bpm = cpl_array_get_data_int(ps.bpm_array);
01553         for (i = 0; i < nx*ny; i++)
01554             bpm[i] = 0;
01555     }
01556     bpm = cpl_array_get_data_int(ps.bpm_array);
01557     vircam_dfs_set_product_exten_header(plist,product_frame_bpm,
01558                                         framelist,parlist,(char *)recipeid,
01559                                         "PRO-1.15",ps.inherit);
01560     cpl_propertylist_update_float(plist,"ESO QC BAD_PIXEL_STAT",
01561                                   vircam_linearity_analyse_config.bad_pixel_stat);
01562     cpl_propertylist_set_comment(plist,"ESO QC BAD_PIXEL_STAT",
01563                                  "Fraction of pixels that are bad");
01564     cpl_propertylist_update_int(plist,"ESO QC BAD_PIXEL_NUM",
01565                                 vircam_linearity_analyse_config.bad_pixel_num);
01566     cpl_propertylist_set_comment(plist,"ESO QC BAD_PIXEL_NUM",
01567                                  "Number of pixels that are bad");
01568     cpl_propertylist_update_int(plist,"ESO PRO DATANCOM",ps.nuse);
01569     if (dummy)
01570         vircam_dummy_property(plist);
01571     outimg = cpl_image_wrap_int(nx,ny,bpm);
01572     if (cpl_image_save(outimg,outbpm,CPL_BPP_8_UNSIGNED,plist,
01573                        CPL_IO_EXTEND) != CPL_ERROR_NONE) {
01574         cpl_msg_error(fctid,"Cannot save product image extension");
01575         return(-1);
01576     }
01577 
01578     /* Write PAF */
01579 
01580     pafprop = vircam_paf_req_items(plist);
01581     vircam_merge_propertylists(pafprop,ps.phupaf);
01582     vircam_paf_append(pafprop,ps.plist,"ESO INS FILT1 NAME");
01583     vircam_paf_append(pafprop,ps.plist,"ESO PRO CATG");
01584     vircam_paf_append(pafprop,plist,"ESO PRO DATANCOM");
01585     if (vircam_paf_print((char *)outbpmpaf,"VIRCAM/vircam_linearity_analyse",
01586                          "QC file",pafprop) != VIR_OK)
01587         cpl_msg_warning(fctid,"Unable to save PAF for linearity table");
01588     cpl_propertylist_delete(pafprop);
01589 
01590     /* Quick tidy */
01591 
01592     cpl_image_unwrap(outimg);
01593 
01594     return(0);
01595 }
01596 
01597 /*---------------------------------------------------------------------------*/
01605 /*---------------------------------------------------------------------------*/
01606 
01607 static int vircam_linearity_analyse_lastbit(int jext, cpl_frameset *framelist,
01608                                             cpl_parameterlist *parlist) {
01609     int retval;
01610     const char *fctid = "vircam_linearity_analyse_lastbit";
01611 
01612     /* Save the new channel table and bad pixel map */
01613 
01614     cpl_msg_info(fctid,"Saving linearity table and bpm for extension %d",jext);
01615     retval = vircam_linearity_analyse_save(framelist,parlist);
01616     if (retval != 0) {
01617         vircam_linearity_analyse_tidy(2);
01618         return(-1);
01619     }
01620 
01621     /* Do some intermediate tidying */
01622 
01623     vircam_linearity_analyse_tidy(1);
01624     return(0);
01625 }
01626 
01627 /*---------------------------------------------------------------------------*/
01631 /*---------------------------------------------------------------------------*/
01632 
01633 static int vircam_linearity_analyse_domedark_groups(void) {
01634     int i,j,found;
01635     float texp;
01636     cpl_frame *frame;
01637     cpl_propertylist *plist;
01638     const char *fctid = "vircam_linearity_analyse_domedark_groups";
01639 
01640     /* Start by getting the memory for the domedark groups */
01641 
01642     ps.ddg = cpl_calloc(ps.ndomes,sizeof(ddgrp));
01643     ps.nddg = 0;
01644 
01645     /* Loop for each of the dome frames and get its exposure time. If this
01646        doesn't exist, then signal an error and go on */
01647 
01648     for (i = 0; i < ps.ndomes; i++) {
01649         frame = cpl_frameset_get_frame(ps.domelist,i);
01650         plist = cpl_propertylist_load(cpl_frame_get_filename(frame),0);
01651         if (vircam_pfits_get_exptime(plist,&texp) != VIR_OK) {
01652             cpl_msg_warning(fctid,"No exposure time found in %s",
01653                             cpl_frame_get_filename(frame));
01654             cpl_propertylist_delete(plist);
01655             continue;
01656         }
01657         cpl_propertylist_delete(plist);
01658 
01659         /* Search the domedark groups to see if this exposure time has already 
01660            been used. If not, then create a new group. If it has then just add 
01661            this frame to the correct group */
01662 
01663         found = 0;
01664         for (j = 0; j < ps.nddg; j++) {
01665             if (ps.ddg[j].exptime == texp) {
01666                 found = 1;
01667                 break;
01668             }
01669         }
01670         if (found) {
01671             cpl_frameset_insert(ps.ddg[j].domes,cpl_frame_duplicate(frame));
01672             ps.ddg[j].ndomes += 1;
01673         } else {
01674             ps.ddg[ps.nddg].exptime = texp;
01675             ps.ddg[ps.nddg].darks = cpl_frameset_new();
01676             ps.ddg[ps.nddg].domes = cpl_frameset_new();
01677             ps.ddg[ps.nddg].ndarks = 0;
01678             ps.ddg[ps.nddg].ndomes = 1;
01679             ps.ddg[ps.nddg].flag = OK_FLAG;
01680             cpl_frameset_insert(ps.ddg[ps.nddg].domes,
01681                                cpl_frame_duplicate(frame));
01682             ps.nddg += 1;
01683         }
01684     }
01685             
01686     /* Right, now loop through all the darks and get their exposure times */
01687 
01688     for (i = 0; i < ps.ndarks; i++) {
01689         frame = cpl_frameset_get_frame(ps.darklist,i);
01690         plist = cpl_propertylist_load(cpl_frame_get_filename(frame),0);
01691         if (vircam_pfits_get_exptime(plist,&texp) != VIR_OK) {
01692             cpl_msg_warning(fctid,"No exposure time found in %s",
01693                             cpl_frame_get_filename(frame));
01694             cpl_propertylist_delete(plist);
01695             continue;
01696         }
01697         cpl_propertylist_delete(plist);
01698 
01699         /* Search the domedark groups to see if this dark fits into one of
01700            the defined groups. If not, then ignore it. If it does, then
01701            add it into the dark frameset */
01702 
01703         found = 0;
01704         for (j = 0; j < ps.nddg; j++) {
01705             if (ps.ddg[j].exptime == texp) {
01706                 found = 1;
01707                 break;
01708             }
01709         }
01710         if (found) {
01711             cpl_frameset_insert(ps.ddg[j].darks,cpl_frame_duplicate(frame));
01712             ps.ddg[j].ndarks += 1;
01713         }
01714     }
01715 
01716     /* Now go through the domedark groups and ditch any that don't have any
01717        dark frames */
01718 
01719     i = 0;
01720     while (i < ps.nddg) {
01721         if (ps.ddg[i].ndarks == 0) {
01722             cpl_msg_warning(fctid,
01723                             "No dark frames exist for exposure %g\nThrowing these away",
01724                             ps.ddg[i].exptime);
01725             freeframeset(ps.ddg[i].darks);
01726             freeframeset(ps.ddg[i].domes);
01727             for (j = i+1; j < ps.nddg; j++)
01728                 ps.ddg[j-1] = ps.ddg[j];
01729             ps.nddg -= 1;
01730         } else
01731             i++;
01732     }
01733 
01734     /* Allocate some space for vir_fits arrays for processed domes */
01735 
01736     for (i = 0; i < ps.nddg; i++) {
01737         ps.ddg[i].proc = cpl_malloc(ps.ddg[i].ndomes*sizeof(vir_fits *));
01738         for (j = 0; j < ps.ddg[i].ndomes; j++) 
01739             ps.ddg[i].proc[j] = NULL;
01740     }                
01741 
01742     /* Resize the output array and return so long as there is anything
01743        left. If there isn't then signal a major error */
01744 
01745     if (ps.nddg > 0) {
01746         ps.ddg = cpl_realloc(ps.ddg,ps.nddg*sizeof(ddgrp));
01747         return(0);
01748     } else {
01749         cpl_msg_error(fctid,"There are no darks defined for input domes");
01750         return(-1);
01751     }
01752 }
01753 
01754 /*---------------------------------------------------------------------------*/
01763 /*---------------------------------------------------------------------------*/
01764 
01765 static double *vircam_linearity_analyse_genstat(vir_fits *fframe, int *bpm,
01766                                                 parquet *p, int np) {
01767     int i,ist,ifn,jst,jfn,n,jind2,iind2,jj,nx,ii;
01768     parquet *pp;
01769     double *d;
01770     float *tmp,*data;
01771 
01772     /* Get the workspace for the output result */
01773 
01774     d = cpl_malloc(np*sizeof(double));
01775 
01776     /* Get the input data array */
01777 
01778     nx = cpl_image_get_size_x(vircam_fits_get_image(fframe));
01779     data = cpl_image_get_data_float(vircam_fits_get_image(fframe));
01780 
01781     /* Get some workspace for doing the median calculations */
01782 
01783     tmp = cpl_malloc(SUBSET*SUBSET*sizeof(float));
01784 
01785     /* Loop for each channel in the parquet structure */
01786 
01787     for (i = 0; i < np; i++) {
01788         pp = p + i;
01789 
01790         /* Take the central part of the channel */
01791 
01792         ist = ((pp->delta_i)/2 - SUBSET2);
01793         ifn = ist + SUBSET - 1;
01794         jst = ((pp->delta_j)/2 - SUBSET2);
01795         jfn = jst + SUBSET - 1;
01796 
01797         /* Put the data into the workspace and do a median */
01798 
01799         n = 0;
01800         for (jj = jst; jj <= jfn; jj++) {
01801             jind2 = (jj + pp->iymin - 1)*nx;
01802             for (ii = ist; ii <= ifn; ii++) {
01803                 iind2 = jind2 + ii + pp->ixmin - 1;
01804                 if (bpm[iind2] == 0) 
01805                     tmp[n++] = data[iind2];
01806             }
01807         }
01808         d[i] = (double)vircam_med(tmp,NULL,(long)n);
01809     }
01810 
01811     /* Tidy and get out of here */
01812 
01813     freespace(tmp);
01814     return(d);
01815 }
01816 
01817 /*---------------------------------------------------------------------------*/
01829 /*---------------------------------------------------------------------------*/
01830 
01831 static double *vircam_linearity_tweakfac(double **fdata, double *mjd, int nim,
01832                                          int nchan, double *facrng, 
01833                                          double *maxdiff) {
01834     int i,ist,ifn,j;
01835     double *factors,sum,midval,minfac,maxfac;
01836 
01837     /* Get some memory for the output array */
01838 
01839     factors = cpl_malloc(nim*sizeof(double));
01840 
01841     /* First sort the data into order of mjd */
01842 
01843     vircam_mjdsort(fdata,mjd,nim);
01844 
01845     /* Which index is the midpoint? */
01846 
01847     if (nim % 2 == 0) {
01848         ist = nim/2 - 1;
01849         ifn = ist + 1;
01850     } else {
01851         ist = nim/2;
01852         ifn = ist;
01853     }
01854 
01855     /* Loop for each channel */
01856 
01857     for (i = 0; i < nchan; i++) {
01858         
01859         /* Get midpoint value */
01860 
01861         midval = 0.5*(fdata[ist][i] + fdata[ifn][i]);
01862 
01863         /* Now normalise all the ith channels by this value */
01864 
01865         for (j = 0; j < nim; j++) 
01866             fdata[j][i] /= midval;
01867     }
01868 
01869     /* Now loop for each image and average the values for all the channels in
01870        in image */
01871 
01872     *maxdiff = 0.0;
01873     maxfac = 0.0;
01874     minfac = 0.0;
01875     for (j = 0; j < nim; j++) {
01876         sum = 0.0;
01877         for (i = 0; i < nchan; i++)
01878             sum += fdata[j][i];
01879         factors[j] = sum/(double)nchan;
01880         if (j == 0) {
01881             maxfac = factors[j];
01882             minfac = factors[j];
01883         } else {
01884             minfac = min(minfac,factors[j]);
01885             maxfac = max(maxfac,factors[j]);
01886             *maxdiff = max(*maxdiff,fabs(factors[j]-factors[j-1]));
01887         }
01888     }
01889     *facrng = maxfac - minfac;
01890 
01891     /* Get out of here */
01892 
01893     return(factors);
01894 }
01895     
01896 /*---------------------------------------------------------------------------*/
01904 /*---------------------------------------------------------------------------*/
01905 
01906 static void vircam_mjdsort(double **fdata, double *mjd, int n) {
01907     int iii,ii,i,ifin,j;
01908     double tmpmjd,*tmpdata;
01909 
01910 
01911     iii = 2;
01912     while (iii < n)
01913         iii *= 2;
01914     iii = min(n,(3*iii)/4 - 1);
01915 
01916     while (iii > 1) {
01917         iii /= 2;
01918         ifin = n - iii;
01919         for (ii = 0; ii < ifin; ii++) {
01920             i = ii;
01921             j = i + iii;
01922             if (mjd[i] > mjd[j]) {
01923                 tmpmjd = mjd[j];
01924                 tmpdata = fdata[j];
01925                 while (1) {
01926                     mjd[j] = mjd[i];
01927                     fdata[j] = fdata[i];
01928                     j = i;
01929                     i = i - iii;
01930                     if (i < 0 || mjd[0] <= tmpmjd) 
01931                         break;
01932                 }
01933                 mjd[j] = tmpmjd;
01934                 fdata[j] = tmpdata;
01935             }
01936         }
01937     }
01938 }
01939 
01940 /*---------------------------------------------------------------------------*/
01947 /*---------------------------------------------------------------------------*/
01948 
01949 static cpl_table *vircam_linearity_analyse_diagtab_init(int np, int nrows) {
01950     int i;
01951     char colname[16];
01952     cpl_table *t;
01953 
01954     /* Create a new table */
01955 
01956     t = cpl_table_new(nrows);
01957 
01958     /* Add the first few columns */
01959 
01960     cpl_table_new_column(t,"filename",CPL_TYPE_STRING);
01961     cpl_table_new_column(t,"exptime",CPL_TYPE_DOUBLE);
01962     cpl_table_set_column_unit(t,"exptime","seconds");
01963     cpl_table_new_column(t,"mjd",CPL_TYPE_DOUBLE);
01964     cpl_table_set_column_unit(t,"mjd","days");
01965 
01966     /* Add columns for each of the channels' raw median flux and linearised 
01967        median flux */
01968 
01969     for (i = 1; i <= np; i++) {
01970         (void)snprintf(colname,16,"rawflux_%02d",i);
01971         cpl_table_new_column(t,colname,CPL_TYPE_DOUBLE);
01972         cpl_table_set_column_unit(t,colname,"ADU");
01973         (void)snprintf(colname,16,"linflux_%02d",i);
01974         cpl_table_new_column(t,colname,CPL_TYPE_DOUBLE);
01975         cpl_table_set_column_unit(t,colname,"ADU");
01976     }
01977 
01978     /* Finally add the correction factors that were used */
01979 
01980     cpl_table_new_column(t,"adjust_fac",CPL_TYPE_DOUBLE);
01981 
01982     /* Right, get out of here */
01983 
01984     return(t);
01985 }
01986 
01987 /*---------------------------------------------------------------------------*/
01991 /*---------------------------------------------------------------------------*/
01992 
01993 static void vircam_linearity_analyse_init(void) {
01994     ps.labels = NULL;
01995     ps.domelist = NULL;
01996     ps.darklist = NULL;
01997     ps.domecheck = NULL;
01998     ps.darkcheck = NULL;
01999     ps.ndomes = 0;
02000     ps.ndarks = 0;
02001     ps.ndomecheck = 0;
02002     ps.ndarkcheck = 0;
02003     ps.chanfrm = NULL;
02004     ps.chantab = NULL;
02005     ps.lchantab = NULL;
02006     ps.flatlist = NULL;
02007     ps.bpm_array = NULL;
02008     ps.ddg = NULL;
02009     ps.plist = NULL;
02010     ps.elist = NULL;
02011     ps.phupaf = NULL;
02012     ps.diag1 = NULL;
02013     ps.diag2 = NULL;
02014     ps.inherit = NULL;
02015 }
02016 
02017 /*---------------------------------------------------------------------------*/
02021 /*---------------------------------------------------------------------------*/
02022 
02023 static void vircam_linearity_analyse_tidy(int level) {
02024     int i;
02025 
02026     freetfits(ps.chantab);
02027     freearray(ps.bpm_array);
02028     freefitslist(ps.flatlist,ps.nflatlist);
02029     freetable(ps.lchantab);
02030     freepropertylist(ps.plist);
02031     freepropertylist(ps.elist);
02032     freetable(ps.diag1);
02033     freetable(ps.diag2);
02034     if (level == 1)
02035         return;
02036     
02037     freespace(ps.labels);
02038     freeframeset(ps.domelist);
02039     freeframeset(ps.darklist);
02040     freeframeset(ps.domecheck);
02041     freeframeset(ps.darkcheck);
02042     freeframe(ps.chanfrm);
02043     if (ps.ddg != NULL) {
02044         for (i = 0; i < ps.nddg; i++) {
02045             freeframeset(ps.ddg[i].darks);
02046             freeframeset(ps.ddg[i].domes);
02047             freefitslist(ps.ddg[i].proc,ps.ddg[i].ndomes);
02048         }
02049         freespace(ps.ddg);
02050     }
02051     freepropertylist(ps.phupaf);
02052 }
02053 
02056 /* 
02057 
02058 $Log: vircam_linearity_analyse.c,v $
02059 Revision 1.63  2010/12/09 13:20:26  jim
02060 Default polynomial order is now 4
02061 
02062 Revision 1.62  2010/07/02 07:17:35  jim
02063 Fixed typo
02064 
02065 Revision 1.61  2010/06/30 12:42:00  jim
02066 A few fixes to stop compiler compaints
02067 
02068 Revision 1.60  2010/03/09 14:29:55  jim
02069 Now modified ESO PRO DATANCOM to reflect the number of images used in
02070 the analysis
02071 
02072 Revision 1.59  2009/09/09 09:50:21  jim
02073 Modified to try and get headers right
02074 
02075 Revision 1.58  2009/07/13 08:17:57  jim
02076 Fixed bug that meant saturation level wasn't being propogated
02077 
02078 Revision 1.57  2009/06/23 05:22:26  jim
02079 Adds saturation back into the channel table header
02080 
02081 Revision 1.56  2008/12/08 06:32:42  jim
02082 Changed 'missing dark frame' error to warning
02083 
02084 Revision 1.55  2008/12/05 13:28:32  jim
02085 Fixed save routine so that the correct version of PRO CATG is written to the
02086 paf file
02087 
02088 Revision 1.54  2008/10/01 04:59:13  jim
02089 Added call to vircam_frameset_fexists to check input frameset
02090 
02091 Revision 1.53  2008/09/30 11:33:00  jim
02092 Fixed bug where saturation flag wasn't being set to OK.
02093 
02094 Revision 1.52  2008/09/29 11:23:00  jim
02095 Minor fix to docs
02096 
02097 Revision 1.51  2008/01/22 19:47:56  jim
02098 New version to implement new algorithm
02099 
02100 Revision 1.50  2007/11/26 09:58:49  jim
02101 Now fails if given observation files done with NDIT != 1
02102 
02103 Revision 1.49  2007/11/23 18:34:28  jim
02104 fixed memory allocation bug
02105 
02106 Revision 1.48  2007/11/22 12:36:55  jim
02107 Modified to create diagnostic tables
02108 
02109 Revision 1.47  2007/11/20 09:41:13  jim
02110 Added ability to alter dome sequence stats by using the monitoring exposures
02111 
02112 Revision 1.46  2007/11/14 10:42:25  jim
02113 Substantial changes to incorporate new linearity analysis algorithm and to
02114 restrict the amount of memory required to do the analysis (especially
02115 the BPM work)
02116 
02117 Revision 1.45  2007/09/07 13:32:12  jim
02118 uses a sorted framelist to ensure that the correct information is given
02119 to the output product header
02120 
02121 Revision 1.44  2007/09/06 21:37:53  jim
02122 fixed call to vircam_dfs_setup_product_ routines to use the full input
02123 frameset
02124 
02125 Revision 1.43  2007/08/29 09:20:33  jim
02126 Primary header is now derived from the same header that forms the PAF rather
02127 than starting off empty and allowing CPL to copy everything it thinks you
02128 want...
02129 
02130 Revision 1.42  2007/08/23 09:02:03  jim
02131 Modified to check domes for DETLIVE before checking darks
02132 
02133 Revision 1.41  2007/07/09 13:21:55  jim
02134 Modified to use new version of vircam_exten_range
02135 
02136 Revision 1.40  2007/06/13 08:11:27  jim
02137 Modified docs to reflect changes in DFS tags
02138 
02139 Revision 1.39  2007/04/30 09:40:17  jim
02140 Added more stuff to paf files
02141 
02142 Revision 1.38  2007/04/04 10:36:07  jim
02143 Fixed typo preventing output of main PAF. Also modified to use dfs tags
02144 
02145 Revision 1.37  2007/03/29 12:19:38  jim
02146 Little changes to improve documentation
02147 
02148 Revision 1.36  2007/03/01 12:41:49  jim
02149 Modified slightly after code checking
02150 
02151 Revision 1.35  2007/02/19 21:13:04  jim
02152 added bad pixel number QC parameter
02153 
02154 Revision 1.34  2007/02/15 11:54:09  jim
02155 Modified to make a distinction between initial channel table and one that
02156 has the proper linearity information
02157 
02158 Revision 1.33  2007/02/15 06:59:38  jim
02159 Added ability to write QC paf files
02160 
02161 Revision 1.32  2007/02/07 10:12:40  jim
02162 Removed calls to vircam_ndit_correct as this is now no longer necessary
02163 
02164 Revision 1.31  2007/02/06 13:11:12  jim
02165 Fixed entry for PRO dictionary in cpl_dfs_set_product_header
02166 
02167 Revision 1.30  2006/12/13 11:45:36  jim
02168 Fixed scaling of sigma error
02169 
02170 Revision 1.29  2006/12/11 22:47:12  jim
02171 Fixed subtle bug in the way that stats were being done.
02172 
02173 Revision 1.28  2006/11/27 12:15:08  jim
02174 changed calls to cpl_propertylist_append to cpl_propertylist_update
02175 
02176 Revision 1.27  2006/11/10 09:23:46  jim
02177 Fixed save routine so to use a new version of vircam_chantab_new
02178 
02179 Revision 1.26  2006/10/31 10:27:27  jim
02180 Fixed a few bugs and modified to make sure than an extension name appear
02181 in each fits extension
02182 
02183 Revision 1.25  2006/09/09 16:49:40  jim
02184 Header comment update
02185 
02186 Revision 1.24  2006/09/08 09:20:22  jim
02187 major upgrade to main processing routine: to deal with bad input better; to
02188 write out dummy results in the case of failure; to combine raw darks on the
02189 fly for use in dark correction, rather than using master darks;
02190 
02191 Revision 1.23  2006/08/03 13:26:44  jim
02192 fixed another typo
02193 
02194 Revision 1.22  2006/08/03 10:36:32  jim
02195 Fixed typo
02196 
02197 Revision 1.21  2006/06/20 19:06:38  jim
02198 Added correction for ndit. Now adjusts the value of norder if the number
02199 of frames given is too small for the order of polynomial requested
02200 
02201 Revision 1.20  2006/06/15 09:58:58  jim
02202 Minor changes to docs
02203 
02204 Revision 1.19  2006/06/06 13:03:42  jim
02205 Fixed scaling that was causing funny stats
02206 
02207 Revision 1.18  2006/05/27 21:40:06  jim
02208 Bad pixels are now defined by a number of sigma above or below the mean
02209 
02210 Revision 1.17  2006/05/09 09:30:47  jim
02211 Fixed _save routine so that bad pixel mask is saved with unsigned char
02212 data type
02213 
02214 Revision 1.16  2006/05/08 12:32:12  jim
02215 Changed default calling parameters for vircam_imcombine
02216 
02217 Revision 1.15  2006/05/04 11:53:15  jim
02218 Fixed the way the _save routine works to be more consistent with the
02219 standard CPL way of doing things
02220 
02221 Revision 1.14  2006/05/03 12:55:17  jim
02222 Fixed some memory leaks
02223 
02224 Revision 1.13  2006/05/02 13:26:32  jim
02225 fixed bug where the wrong amount of memory was being allocated for the dark
02226 exposure times
02227 
02228 Revision 1.12  2006/05/02 11:36:29  jim
02229 fixed illegal propertylist_delete calls
02230 
02231 Revision 1.11  2006/04/27 09:46:01  jim
02232 Modified DFS frame types to conform to new dictionary
02233 
02234 Revision 1.10  2006/04/25 13:45:57  jim
02235 Fixed to adhere to new calling sequence for vircam_dfs routines
02236 
02237 Revision 1.9  2006/04/24 12:12:59  jim
02238 Fixed --help documentation and sorted out filename extension problem
02239 (.fit -> .fits)
02240 
02241 Revision 1.8  2006/04/20 11:31:34  jim
02242 Added bad pixel masking
02243 
02244 Revision 1.7  2006/03/23 21:18:45  jim
02245 Minor changes mainly to comment headers
02246 
02247 Revision 1.6  2006/03/22 14:02:51  jim
02248 cosmetic changes to keep lint happy
02249 
02250 Revision 1.5  2006/03/22 12:13:51  jim
02251 Modified to use new vircam_mask capability
02252 
02253 Revision 1.4  2006/03/15 10:43:40  jim
02254 Fixed a few things
02255 
02256 Revision 1.3  2006/03/03 14:29:06  jim
02257 Now calls routines with vir_fits.
02258 
02259 Revision 1.2  2006/02/22 10:01:22  jim
02260 Added full documentation
02261 
02262 Revision 1.1  2006/02/18 11:49:58  jim
02263 new file
02264 
02265 
02266 */

Generated on 7 Feb 2011 for VIRCAM Pipeline by  doxygen 1.6.1