OMEGA Pipeline Reference Manual  1.0.6
omega_qcheck.c
1 /* $Id: omega_qcheck.c,v 1.2 2012-01-12 12:05:09 agabasch Exp $
2  *
3  * This file is part of the OMEGA Pipeline
4  * Copyright (C) 2002,2003 European Southern Observatory
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19  */
20 
21 /*
22  * $Author: agabasch $
23  * $Date: 2012-01-12 12:05:09 $
24  * $Revision: 1.2 $
25  * $Name: not supported by cvs2svn $
26  */
27 
28 #ifdef HAVE_CONFIG_H
29 #include <config.h>
30 #endif
31 
32 /*-----------------------------------------------------------------------------
33  Includes
34  -----------------------------------------------------------------------------*/
35 
36 #include "omega_recipe.h"
37 
38 /*----------------------------------------------------------------------------*/
59 /*----------------------------------------------------------------------------*/
62 /*-----------------------------------------------------------------------------
63  Functions prototypes
64  -----------------------------------------------------------------------------*/
65 
66 static int omega_qcheck_create(cpl_plugin *) ;
67 static int omega_qcheck_exec(cpl_plugin *) ;
68 static int omega_qcheck_destroy(cpl_plugin *) ;
69 static int omega_qcheck(cpl_frameset *,cpl_parameterlist *) ;
70 
71 /*-----------------------------------------------------------------------------
72  Private Functions
73  -----------------------------------------------------------------------------*/
74 static void omega_qcheck_init(void);
75 static void omega_qcheck_tidy(void);
76 
77 /* Parameters and QC structure*/
78 static struct {
79  /* Inputs. Parameters */
80  float rejt;
81  int extnum;
82  int niter;
83  int oc;
84  int paf;
85 
86  /* QC parameters */
87  double mean;
88  double median;
89  double stdev;
90 
91 }omega_qcheck_config;
92 
93 /* Input and Output */
94 static struct {
95  cpl_size *labels;
96  cpl_stats *stats;
97  cpl_frameset *domelist;
98  omega_fits *domefits1;
99  cpl_propertylist *eh;
100 
101  /*Products*/
102  char *proname;
103  cpl_table *result;
104 } ps;
105 
106 /* Static variables */
107 
108 /* Static variables */
109 #define RECIPE "omega_qcheck"
110 
111 /*----------------------------------------------------------------------------*/
119 /*----------------------------------------------------------------------------*/
120 int cpl_plugin_get_info(cpl_pluginlist * list)
121 {
122  cpl_recipe * recipe = cpl_calloc(1, sizeof(*recipe)) ;
123  cpl_plugin * plugin = &recipe->interface ;
124 
125  cpl_plugin_init(plugin,
126  CPL_PLUGIN_API,
127  OMEGA_BINARY_VERSION,
128  CPL_PLUGIN_TYPE_RECIPE,
129  "omega_qcheck",
130  "OMEGA - Generates a quick check on the detector responsivity for each chip.",
131  "The recipe takes as input exactly one raw domeflat frame, and \n"
132  "a master bias. Optionally, an overscan correction mode can be set. \n"
133  "The default is to apply no overscan correction.",
134  "Sandra Castro",
135  "scastro@eso.org",
137  omega_qcheck_create,
138  omega_qcheck_exec,
139  omega_qcheck_destroy) ;
140 
141  cpl_pluginlist_append(list, plugin) ;
142 
143  return 0;
144 }
145 
146 /*----------------------------------------------------------------------------*/
155 /*----------------------------------------------------------------------------*/
156 static int omega_qcheck_create(cpl_plugin * plugin)
157 {
158  cpl_recipe * recipe;
159  cpl_parameter * p ;
160 
161  /* Do not create the recipe if an error code is already set */
162  if (cpl_error_get_code() != CPL_ERROR_NONE) {
163  cpl_msg_error(cpl_func, "%s():%d: An error is already set: %s",
164  cpl_func, __LINE__, cpl_error_get_where());
165  return (int)cpl_error_get_code();
166  }
167 
168  if (plugin == NULL) {
169  cpl_msg_error(cpl_func, "Null plugin");
170  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
171  }
172 
173  /* Verify plugin type */
174  if (cpl_plugin_get_type(plugin) != CPL_PLUGIN_TYPE_RECIPE) {
175  cpl_msg_error(cpl_func, "Plugin is not a recipe");
176  cpl_ensure_code(0, (int)CPL_ERROR_TYPE_MISMATCH);
177  }
178 
179  /* Get the recipe */
180  recipe = (cpl_recipe *)plugin;
181 
182  /* Create the parameters list in the cpl_recipe object */
183  recipe->parameters = cpl_parameterlist_new() ;
184 
185  /* Fill the parameters list */
186  p = cpl_parameter_new_value("omega.omega_qcheck.ExtensionNumber",
187  CPL_TYPE_INT,
188  "FITS extension number to load (1 to 32). (-1 == all)",
189  "omega_qcheck",
190  -1) ;
191 
192  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI,"ext") ;
193  cpl_parameterlist_append(recipe->parameters, p) ;
194 
195 
196  p = cpl_parameter_new_range("omega.omega_qcheck.OverscanMethod",
197  CPL_TYPE_INT,
198  "Overscan Correction Method (0 to 6):\n"
199  "0 = no overscan correction;\n"
200  "1 = use median of prescan in X;\n"
201  "2 = use median of overscan in X;\n"
202  "3 = use median on prescan in Y;\n"
203  "4 = use median of overscan in Y;\n"
204  "5 = per-row subtraction of the median of row in prescan regions of X;\n"
205  "6 = per-row subtraction of the median of row in overscan regions of X\n",
206  "omega_qcheck",
207  0, 0, 6);
208 
209  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI,"oc-meth") ;
210  cpl_parameterlist_append(recipe->parameters, p) ;
211 
212  p = cpl_parameter_new_value("omega.omega_qcheck.PAF",
213  CPL_TYPE_BOOL,
214  "Boolean value to create PAF files. 1(Yes), 0(No)",
215  "omega_qcheck",
216  1) ;
217 
218  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI, "paf") ;
219  cpl_parameterlist_append(recipe->parameters, p) ;
220 
221  p = cpl_parameter_new_value("omega.omega_qcheck.rej_threshold",
222  CPL_TYPE_DOUBLE,
223  "The rejection threshold for outlying pixels",
224  "omega_qcheck",
225  5.0) ;
226 
227  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI,"rej-thre") ;
228  cpl_parameterlist_append(recipe->parameters, p) ;
229 
230  p = cpl_parameter_new_value("omega.omega_qcheck.NumberIter",
231  CPL_TYPE_INT,
232  "The maximum number of iterations",
233  "omega_qcheck",
234  5) ;
235 
236  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI,"niter") ;
237  cpl_parameterlist_append(recipe->parameters, p) ;
238 
239 
240 
241  return 0;
242 }
243 
244 /*----------------------------------------------------------------------------*/
250 /*----------------------------------------------------------------------------*/
251 static int omega_qcheck_exec(cpl_plugin * plugin)
252 {
253  cpl_recipe * recipe;
254  int recipe_status;
255 
256  /* Return immediately if an error code is already set */
257  if (cpl_error_get_code() != CPL_ERROR_NONE) {
258  cpl_msg_error(cpl_func, "%s():%d: An error is already set: %s",
259  cpl_func, __LINE__, cpl_error_get_where());
260  return (int)cpl_error_get_code();
261  }
262 
263  if (plugin == NULL) {
264  cpl_msg_error(cpl_func, "Null plugin");
265  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
266  }
267 
268  /* Verify plugin type */
269  if (cpl_plugin_get_type(plugin) != CPL_PLUGIN_TYPE_RECIPE) {
270  cpl_msg_error(cpl_func, "Plugin is not a recipe");
271  cpl_ensure_code(0, (int)CPL_ERROR_TYPE_MISMATCH);
272  }
273 
274  /* Get the recipe */
275  recipe = (cpl_recipe *)plugin;
276 
277  /* Verify parameter and frame lists */
278  if (recipe->parameters == NULL) {
279  cpl_msg_error(cpl_func, "Recipe invoked with NULL parameter list");
280  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
281  }
282  if (recipe->frames == NULL) {
283  cpl_msg_error(cpl_func, "Recipe invoked with NULL frame set");
284  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
285  }
286 
287  /* Invoke the recipe */
288  recipe_status = omega_qcheck(recipe->frames, recipe->parameters);
289 
290  /* Ensure DFS-compliance of the products */
291  if (cpl_dfs_update_product_header(recipe->frames)) {
292  if (!recipe_status) recipe_status = (int)cpl_error_get_code();
293  }
294 
295 
296  return recipe_status;
297 }
298 
299 /*----------------------------------------------------------------------------*/
305 /*----------------------------------------------------------------------------*/
306 static int omega_qcheck_destroy(cpl_plugin * plugin)
307 {
308  cpl_recipe * recipe = (cpl_recipe *)plugin ;
309 
310  if (plugin == NULL) {
311  cpl_msg_error(cpl_func, "Null plugin");
312  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
313  }
314 
315  /* Verify plugin type */
316  if (cpl_plugin_get_type(plugin) != CPL_PLUGIN_TYPE_RECIPE) {
317  cpl_msg_error(cpl_func, "Plugin is not a recipe");
318  cpl_ensure_code(0, (int)CPL_ERROR_TYPE_MISMATCH);
319  }
320 
321  /* Get the recipe */
322  recipe = (cpl_recipe *)plugin;
323 
324  cpl_parameterlist_delete(recipe->parameters);
325 
326  return 0 ;
327 }
328 
329 
330 /*----------------------------------------------------------------------------*/
343 /*----------------------------------------------------------------------------*/
344 
345 static int omega_qcheck(cpl_frameset *set, cpl_parameterlist *pars)
346 {
347 
348  int j,jst,jfn,oscan1,isfirst;
349  cpl_size nlab;
350  cpl_frame *mbias_frame;
351  const cpl_frame *frame1;
352  cpl_image *mbias_image;
353  cpl_image *trim_raw1;
354  cpl_frame *product_frame;
355  cpl_parameter *par;
356  cpl_propertylist *qclist,*alist;
357 
358 
359  /*Start the recipe */
360 
361  if (!pars) {
362  cpl_msg_error (cpl_func, "Parameters list not found");
363  return -1;
364  }
365 
366  if (cpl_frameset_is_empty(set) == 1) {
367  cpl_msg_error (cpl_func, "Frameset not found");
368  return -1;
369  }
370 
371  /* Initialise a few things */
372  omega_qcheck_init();
373 
374  /* Retrieve input parameters */
375  par = cpl_parameterlist_find(pars, "omega.omega_qcheck.ExtensionNumber") ;
376  omega_qcheck_config.extnum = cpl_parameter_get_int(par) ;
377 
378  par = cpl_parameterlist_find(pars, "omega.omega_qcheck.OverscanMethod") ;
379  omega_qcheck_config.oc = cpl_parameter_get_int(par) ;
380 
381  par = cpl_parameterlist_find(pars, "omega.omega_qcheck.NumberIter") ;
382  omega_qcheck_config.niter = cpl_parameter_get_int(par) ;
383 
384  par = cpl_parameterlist_find(pars, "omega.omega_qcheck.rej_threshold") ;
385  omega_qcheck_config.rejt = cpl_parameter_get_double(par) ;
386 
387  par = cpl_parameterlist_find(pars, "omega.omega_qcheck.PAF") ;
388  omega_qcheck_config.paf = cpl_parameter_get_bool(par) ;
389 
390  /* Identify the RAW and CALIB frames in the input frameset */
391  if (oc_dfs_set_groups(set)) {
392  cpl_msg_error(cpl_func, "Cannot identify RAW and CALIB frames") ;
393  omega_qcheck_tidy();
394  return -1 ;
395  }
396 
397 
398  /* Verify the frameset contents. */
399  /*Make sure there is exactly 1 lifetest frames */
400  if ((ps.labels = cpl_frameset_labelise(set,omega_compare_tags,
401  &nlab)) == NULL) {
402  cpl_msg_error(cpl_func,"Cannot labelise the input frameset");
403  omega_qcheck_tidy();
404  return -1;
405  }
406  if ((ps.domelist = omega_frameset_subgroup(set,ps.labels,nlab,
407  LIFETEST_RAW)) == NULL) {
408  cpl_msg_error(cpl_func,"Cannot find dome frame in input frameset");
409  omega_qcheck_tidy();
410  return -1;
411  }
412  if (cpl_frameset_get_size(ps.domelist) < 1) {
413  cpl_msg_error(cpl_func,"Need exactly 1 (%s) frame to run this recipe",LIFETEST_RAW);
414  omega_qcheck_tidy();
415  return -1;
416  }
417 
418  /* Set up some convenience variables */
419  frame1 = cpl_frameset_get_position_const(ps.domelist,0);
420  cpl_msg_info (cpl_func,"Using %s frame: %s",LIFETEST_RAW,cpl_frame_get_filename(frame1));
421 
422  mbias_frame = cpl_frameset_find(set, OMEGA_CALIB_BIAS);
423  if (mbias_frame == NULL) {
424  cpl_msg_error (cpl_func,"No Master Bias is present in frame set");
425  omega_qcheck_tidy();
426  return -1;
427  }
428 
429  cpl_msg_info(cpl_func,"Using %s %s",OMEGA_CALIB_BIAS, cpl_frame_get_filename(mbias_frame));
430 
431  /* Loop for each of the image extensions */
432  omega_extensions(frame1,omega_qcheck_config.extnum,&jst,&jfn);
433  if(omega_qcheck_config.extnum == 0){
434  cpl_msg_error(cpl_func,"Unsupported extension request, %d",omega_qcheck_config.extnum);
435  omega_qcheck_tidy();
436  return -1;
437  }
438 
439  for (j = jst; j <= jfn; j++) {
440  cpl_msg_info(cpl_func,"Working on extension %d",j);
441  isfirst = (j == jst);
442  omega_qcheck_config.mean = 0.0;
443  omega_qcheck_config.median = 0.0;
444  omega_qcheck_config.stdev = 0.0;
445 
446 
447  /* Check that this detector is live*/
448  /*
449  plist = cpl_propertylist_load(cpl_frame_get_filename(frame1),j);
450  omega_pfits_get_detlive(plist,&live);
451  if (! live) {
452  cpl_msg_warning(cpl_func,"First dome image detector not live");
453  save dummy extension
454  freeplist(plist);
455  continue;
456  }
457  freeplist(plist);
458  */
459 
460  /* Check overscan correction method consistency */
461  oscan1 = omega_pfits_get_overscan(mbias_frame, j);
462  if(oscan1 != omega_qcheck_config.oc) {
463  cpl_msg_warning (cpl_func, "Overscan correction mode for Master Bias (oc = %d) differs from "
464  "the one used here (oc = %d)", oscan1, omega_qcheck_config.oc);
465  }
466 
467  /* Load master bias*/
468  mbias_image = cpl_image_load(cpl_frame_get_filename(mbias_frame), CPL_TYPE_FLOAT, 0, j);
469  if (mbias_image == NULL) {
470  cpl_msg_error(cpl_func,"Cannot load MASTER BIAS");
471  omega_qcheck_tidy();
472  return -1;
473  }
474 
475  ps.domefits1 = omega_fits_load(frame1,CPL_TYPE_FLOAT,j);
476 
477  trim_raw1 = omega_trim_oscan_correct(ps.domefits1, omega_qcheck_config.oc);
478  if(trim_raw1 == NULL){
479  cpl_msg_error(cpl_func,"Cannot trim input image");
480  freeimage(mbias_image);
481  omega_qcheck_tidy();
482  return -1;
483  }
484 
485  cpl_image_subtract(trim_raw1, mbias_image);
486  freeimage(mbias_image);
487 
488  /*Calculate statistics iteratively*/
489  ps.stats = omega_iter_stat_opts(trim_raw1,NULL,omega_qcheck_config.rejt,omega_qcheck_config.niter);
490  if(ps.stats != NULL){
491  omega_qcheck_config.mean = cpl_stats_get_mean(ps.stats);
492  omega_qcheck_config.median = cpl_stats_get_median(ps.stats);
493  omega_qcheck_config.stdev = cpl_stats_get_stdev(ps.stats);
494  }
495  else{
496  cpl_msg_warning(cpl_func,"Cannot calculate statistics iteratively");
497  }
498 
499  freestats(ps.stats);
500  freeimage(trim_raw1);
501 
502  /* FIXME: include comparison with REFERENCE frame */
503  /* Save the product */
504  ps.result = cpl_table_new(1);
505  cpl_table_new_column(ps.result, "MEAN", CPL_TYPE_DOUBLE);
506  cpl_table_new_column(ps.result, "MEDIAN", CPL_TYPE_DOUBLE);
507  cpl_table_new_column(ps.result, "STDEV", CPL_TYPE_DOUBLE);
508  cpl_table_set_double(ps.result, "MEAN", 0, omega_qcheck_config.mean);
509  cpl_table_set_double(ps.result, "MEDIAN", 0, omega_qcheck_config.median);
510  cpl_table_set_double(ps.result, "STDEV", 0, omega_qcheck_config.stdev);
511 
512  /* Create QC list */
513  qclist = cpl_propertylist_new();
514  cpl_propertylist_append_double(qclist, "ESO QC QUICK CHECK MEAN",
515  omega_qcheck_config.mean) ;
516  cpl_propertylist_set_comment(qclist,"ESO QC QUICK CHECK MEAN","Mean of difference");
517 
518  cpl_propertylist_append_double(qclist, "ESO QC QUICK CHECK MEDIAN",
519  omega_qcheck_config.median) ;
520  cpl_propertylist_set_comment(qclist,"ESO QC QUICK CHECK MEDIAN","Median of difference");
521 
522  cpl_propertylist_append_double(qclist, "ESO QC QUICK CHECK STDEV",
523  omega_qcheck_config.stdev) ;
524  cpl_propertylist_set_comment(qclist,"ESO QC QUICK CHECK STDEV","Standard deviation of difference");
525 
526  if(isfirst){
527  ps.proname = cpl_sprintf("%s_%s.fits", INSTRUME,LTEST_PROCATG);
528  product_frame = omega_product_frame(ps.proname, LTEST_PROCATG, CPL_FRAME_TYPE_TABLE);
529  }
530 
531 
532  alist=cpl_propertylist_load_regexp(cpl_frame_get_filename(frame1),j,"EXTNAME",0);
533  if(omega_save_table(ps.result,set,pars,alist,qclist,ps.proname,RECIPE,product_frame,
534  NULL,isfirst) == -1){
535  cpl_msg_error(cpl_func, "Cannot save the product");
536  cpl_propertylist_delete(alist);
537  freeplist(qclist);
538  omega_qcheck_tidy();
539  return -1 ;
540  }
541  cpl_propertylist_delete(alist);
542 /*
543  if (omega_qcheck_save(set,pars) == -1) {
544  cpl_msg_error(cpl_func, "Cannot save the product");
545  omega_qcheck_tidy();
546  return -1 ;
547  }
548 */
549 
550  freetable(ps.result);
551  freeplist(qclist);
552  freefits(ps.domefits1);
553  ps.domefits1 = NULL;
554  } /* go back and do next extension */
555 
556 
557  /*Clean up */
558  omega_qcheck_tidy();
559 
560  return 0;
561 
562 }
563 
564 
565 /* Initialize the pointers */
566 static void omega_qcheck_init(void) {
567  ps.labels = NULL;
568  ps.domelist = NULL;
569  ps.domefits1 = NULL;
570  ps.stats = NULL;
571  ps.eh = NULL;
572  ps.proname = NULL;
573  ps.result = NULL;
574 }
575 
576 /* Free any allocated memory */
577 static void omega_qcheck_tidy(void) {
578  freespace(ps.labels);
579  freeframeset(ps.domelist);
580  freefits(ps.domefits1);
581  freestats(ps.stats);
582  freespace(ps.proname);
583  freetable(ps.result);
584 }
585 
int omega_pfits_get_overscan(const cpl_frame *frame, int xn)
Get the DRS keyword of the overscan method used to reduce the image.
Definition: omega_pfits.c:912
cpl_stats * omega_iter_stat_opts(cpl_image *img, cpl_vector *zone, double threshold, int iter)
Compute statistics of an image iteratively.
Definition: omega_stats.c:140
void omega_extensions(const cpl_frame *frame, int inexten, int *out1, int *out2)
Definition: omega_utils.c:348
int omega_save_table(const cpl_table *tbl, cpl_frameset *set, const cpl_parameterlist *pars, cpl_propertylist *alist, const cpl_propertylist *qclist, const char *name, const char *recipe, cpl_frame *frame, const cpl_frame *inherit, int isfirst)
Save a table as a DFS compliant product.
Definition: omega_utils.c:987
int omega_compare_tags(const cpl_frame *frame1, const cpl_frame *frame2)
Definition: omega_utils.c:206
int cpl_plugin_get_info(cpl_pluginlist *list)
Build the list of available plugins, for this module.
Definition: omega_qcheck.c:120
omega_fits * omega_fits_load(const cpl_frame *inframe, cpl_type type, int extnum)
Definition: omega_fits.c:84
cpl_image * omega_trim_oscan_correct(omega_fits *ofits, int oscan)
This method loads a raw image, trims it and (optionally) performs an overscan correction, using the statistics of the pre and overscan regions. This can be done with a single value for the whole image or on a per-row basis.
Definition: omega_trim.c:248
const char * omega_get_license(void)
Get the pipeline copyright and license.
Definition: omega_utils.c:67
cpl_frame * omega_product_frame(const char *filename, const char *tag, cpl_frame_type type)
Setup a frame to save a product.
Definition: omega_utils.c:499
cpl_frameset * omega_frameset_subgroup(cpl_frameset *frameset, cpl_size *labels, cpl_size nlab, const char *tag)
Definition: omega_utils.c:257