OMEGA Pipeline Reference Manual  1.0.6
omega_mflat.c
1 /* $Id: omega_mflat.c,v 1.6 2012-08-30 07:46:52 agabasch Exp $
2  *
3  * This file is part of the OMEGA Pipeline
4  * Copyright (C) 2002,2003 European Southern Observatory
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19  */
20 
21 /*
22  * $Author: agabasch $
23  * $Date: 2012-08-30 07:46:52 $
24  * $Revision: 1.6 $
25  * $Name: not supported by cvs2svn $
26  */
27 
28 #ifdef HAVE_CONFIG_H
29 #include <config.h>
30 #endif
31 
32 /*-----------------------------------------------------------------------------
33  Includes
34  -----------------------------------------------------------------------------*/
35 
36 #include "omega_recipe.h"
37 
38 #include "omega_background.h"
39 #include "omega_bpm.h"
40 #include "omega_flats.h"
41 #include <cpl_errorstate.h>
42 
72 /*-----------------------------------------------------------------------------
73  Functions prototypes
74  -----------------------------------------------------------------------------*/
75 
76 static int omega_mflat_create(cpl_plugin *) ;
77 static int omega_mflat_exec(cpl_plugin *) ;
78 static int omega_mflat_destroy(cpl_plugin *) ;
79 static int omega_mflat(cpl_frameset *,cpl_parameterlist *);
80 
81 /*-----------------------------------------------------------------------------
82  Private Functions
83  -----------------------------------------------------------------------------*/
84 int omega_mflat_combine(cpl_parameterlist *pars, int xn);
85 static void omega_mflat_init(void);
86 static void omega_mflat_tidy(void);
87 
88 /*-----------------------------------------------------------------------------
89  Static structures
90  -----------------------------------------------------------------------------*/
91 
92 static struct {
93  /* Inputs. Parameters */
94  int extnum;
95  int oc;
96  int paf;
97  double sigma;
98 
99  /* Outputs. QC parameters */
100  int bpm;
101  double Mean;
102  double Median;
103  double Stdev;
104  double RawMin;
105  double RawMax;
106  double RawMean;
107  double RawMedian;
108  double RawStdev;
109 
110 }omega_mflat_config;
111 
112 /* Input and Output */
113 static struct {
114 
115  /* Calib frames */
116  const cpl_frame *cframe;
117  const cpl_frame *hframe;
118  const cpl_frame *mbframe;
119  const cpl_frame *mdomefr;
120  const cpl_frame *nskyfr;
121  const cpl_frame *bpframe;
122  cpl_size *labels;
123  cpl_frameset *flatlist;
124  cpl_propertylist *eh;
125  cpl_stats *stats;
126  omega_fits *firstflat;
127 
128  /* Products */
129  cpl_image *bpixels;
130  cpl_image *mflat;
131  cpl_image *mtwilight;
132 
133 }ps;
134 
135 /*-----------------------------------------------------------------------------
136  Static global variables
137  -----------------------------------------------------------------------------*/
138 
139 /* Static variables */
140 #define RECIPE "omega_mflat"
141 
142 static int isfirst;
143 
144 /*----------------------------------------------------------------------------*/
153 /*----------------------------------------------------------------------------*/
154 
155 int cpl_plugin_get_info(cpl_pluginlist * list)
156 {
157  cpl_recipe * recipe = cpl_calloc(1, sizeof(*recipe)) ;
158  cpl_plugin * plugin = &recipe->interface ;
159 
160  cpl_plugin_init(plugin,
161  CPL_PLUGIN_API,
162  OMEGA_BINARY_VERSION,
163  CPL_PLUGIN_TYPE_RECIPE,
164  "omega_mflat",
165  "OMEGA - Create Master Twilight and Master Flat for each chip",
166  "This recipe is used to derive a valid twilight flat frame (Calfile 543, 546) \n"
167  "for one particular chip and filter. The recipe always takes as input \n"
168  "a list of raw twilight flat frames, a master bias and an optional master \n"
169  "dome flat to be used in creating a final Master Flat Field. In addition, a hot \n"
170  "pixel map (and a cold pixel map can be provided, which will be used to create a \n"
171  "bad pixels map. Optionally, an overscan correction mode \n"
172  "can be set. The default is to apply no overscan correction.\n\n"
173  "The raw twilight flats are trimmed and overscan corrected. The data are then \n"
174  "normalized, averaged, and the result is normalized again. Image statistics \n"
175  "are computed for the resulting frames.",
176  "Sandra Castro",
177  "scastro@eso.org",
179  omega_mflat_create,
180  omega_mflat_exec,
181  omega_mflat_destroy) ;
182 
183  cpl_pluginlist_append(list, plugin) ;
184 
185  return 0;
186 }
187 
188 /*
189  * Create the plugin and the parameters list
190  */
191 static int omega_mflat_create(cpl_plugin * plugin)
192 {
193  cpl_recipe * recipe;
194  cpl_parameter * p ;
195  char *path = NULL;
196 
197  /* Do not create the recipe if an error code is already set */
198  if (cpl_error_get_code() != CPL_ERROR_NONE) {
199  cpl_msg_error(cpl_func, "%s():%d: An error is already set: %s",
200  cpl_func, __LINE__, cpl_error_get_where());
201  return (int)cpl_error_get_code();
202  }
203 
204  if (plugin == NULL) {
205  cpl_msg_error(cpl_func, "Null plugin");
206  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
207  }
208 
209  /* Verify plugin type */
210  if (cpl_plugin_get_type(plugin) != CPL_PLUGIN_TYPE_RECIPE) {
211  cpl_msg_error(cpl_func, "Plugin is not a recipe");
212  cpl_ensure_code(0, (int)CPL_ERROR_TYPE_MISMATCH);
213  }
214 
215  /* Get the recipe */
216  recipe = (cpl_recipe *)plugin;
217 
218  /* Create the parameters list in the cpl_recipe object */
219  recipe->parameters = cpl_parameterlist_new() ;
220  if (recipe->parameters == NULL) {
221  cpl_msg_error(cpl_func, "Parameter list allocation failed");
222  cpl_ensure_code(0, (int)CPL_ERROR_ILLEGAL_OUTPUT);
223  }
224 
225 
226  /* Fill the parameters list */
227 
228 /* --------------- General parameters ---------------------------*/
229  p = cpl_parameter_new_value("omega.omega_mflat.ExtensionNumber",
230  CPL_TYPE_INT,
231  "FITS extension number to load (1 to 32). (-1 = all)",
232  "omega_mflat",
233  -1) ;
234 
235  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI,"ext") ;
236  cpl_parameterlist_append(recipe->parameters, p) ;
237 
238 
239  p = cpl_parameter_new_range("omega.omega_mflat.OverscanMethod",
240  CPL_TYPE_INT,
241  "Overscan Correction Method",
242  "omega_mflat",
243  0, 0, 6);
244 
245  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI,"oc-meth") ;
246  cpl_parameterlist_append(recipe->parameters, p) ;
247 
248  p = cpl_parameter_new_value("omega.omega_mflat.PAF",
249  CPL_TYPE_BOOL,
250  "Boolean value to create PAF files. 1(Yes), 0(No)",
251  "omega_mflat",
252  1) ;
253 
254  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI, "paf") ;
255  cpl_parameterlist_append(recipe->parameters, p) ;
256 
257  p = cpl_parameter_new_value("omega.omega_mflat.SigmaClip",
258  CPL_TYPE_DOUBLE,
259  "Sigma Clipping Threshold",
260  "omega_mflat",
261  3.0) ;
262 
263  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI, "sig-clip") ;
264  cpl_parameterlist_append(recipe->parameters, p) ;
265 
266 
267 
268 /*------------------ Parameters to create a Master Flat ------------------ */
269  path = cpl_sprintf("%s", OMEGA_BIN_PATH);
270  p = cpl_parameter_new_value("omega.omega_mflat.BinPath",
271  CPL_TYPE_STRING,
272  "Path to any external executable program.",
273  "omega.BinPath",
274  path);
275 
276  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI, "bin-path");
277  cpl_parameterlist_append(recipe->parameters, p);
278  cpl_free(path);
279 
280  p = cpl_parameter_new_value("omega.omega_mflat.GaussianFiltSize",
281  CPL_TYPE_DOUBLE,
282  "Standard deviation of Gaussian Fourier filter size",
283  "omega_mflat",
284  9.0) ;
285 
286  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI, "gau_filt_sz") ;
287  cpl_parameterlist_append(recipe->parameters, p) ;
288 
289  p = cpl_parameter_new_value("omega.omega_mflat.MirrorXpix",
290  CPL_TYPE_INT,
291  "Width of X region for mirroring edges (FFT continuity)",
292  "omega_mflat",
293  75) ;
294 
295  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI, "mr_xpix") ;
296  cpl_parameterlist_append(recipe->parameters, p) ;
297 
298  p = cpl_parameter_new_value("omega.omega_mflat.MirrorYpix",
299  CPL_TYPE_INT,
300  "Width of Y region for mirroring edges (FFT continuity)",
301  "omega_mflat",
302  150) ;
303 
304  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI, "mr_ypix") ;
305  cpl_parameterlist_append(recipe->parameters, p) ;
306 
307  /* Parameters to create bad pixels map */
308  p = cpl_parameter_new_range("omega.omega_mflat.LowThre",
309  CPL_TYPE_DOUBLE,
310  "Low flagging threshold for cold pixels map",
311  "omega_mflat",
312  0.96, 0.90, 1.00) ;
313 
314  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI, "low") ;
315  cpl_parameterlist_append(recipe->parameters, p) ;
316 
317  p = cpl_parameter_new_range("omega.omega_mflat.HighThre",
318  CPL_TYPE_DOUBLE,
319  "High flagging threshold for cold pixels map",
320  "omega_flat",
321  1.04, 1.00, 1.10) ;
322 
323  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI,"high") ;
324  cpl_parameterlist_append(recipe->parameters, p) ;
325 
326  return 0;
327 }
328 
329 /*
330  * Execute the plugin
331  */
332 static int omega_mflat_exec(cpl_plugin * plugin)
333 {
334  cpl_recipe * recipe;
335  int recipe_status;
336 
337  /* Return immediately if an error code is already set */
338  if (cpl_error_get_code() != CPL_ERROR_NONE) {
339  cpl_msg_error(cpl_func, "%s():%d: An error is already set: %s",
340  cpl_func, __LINE__, cpl_error_get_where());
341  return (int)cpl_error_get_code();
342  }
343 
344  if (plugin == NULL) {
345  cpl_msg_error(cpl_func, "Null plugin");
346  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
347  }
348 
349  /* Verify plugin type */
350  if (cpl_plugin_get_type(plugin) != CPL_PLUGIN_TYPE_RECIPE) {
351  cpl_msg_error(cpl_func, "Plugin is not a recipe");
352  cpl_ensure_code(0, (int)CPL_ERROR_TYPE_MISMATCH);
353  }
354 
355  /* Get the recipe */
356  recipe = (cpl_recipe *)plugin;
357 
358  /* Verify parameter and frame lists */
359  if (recipe->parameters == NULL) {
360  cpl_msg_error(cpl_func, "Recipe invoked with NULL parameter list");
361  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
362  }
363  if (recipe->frames == NULL) {
364  cpl_msg_error(cpl_func, "Recipe invoked with NULL frame set");
365  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
366  }
367 
368  /* Invoke the recipe */
369  recipe_status = omega_mflat(recipe->frames, recipe->parameters);
370 
371  /* Ensure DFS-compliance of the products */
372  if (cpl_dfs_update_product_header(recipe->frames)) {
373  if (!recipe_status) recipe_status = (int)cpl_error_get_code();
374  }
375 
376  return recipe_status;
377 }
378 
379 /*
380  * Destroy the plugin
381  */
382 static int omega_mflat_destroy(cpl_plugin * plugin)
383 {
384  cpl_recipe * recipe;
385 
386  if (plugin == NULL) {
387  cpl_msg_error(cpl_func, "Null plugin");
388  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
389  }
390 
391  /* Verify plugin type */
392  if (cpl_plugin_get_type(plugin) != CPL_PLUGIN_TYPE_RECIPE) {
393  cpl_msg_error(cpl_func, "Plugin is not a recipe");
394  cpl_ensure_code(0, (int)CPL_ERROR_TYPE_MISMATCH);
395  }
396 
397  /* Get the recipe */
398  recipe = (cpl_recipe *)plugin;
399 
400  cpl_parameterlist_delete(recipe->parameters) ;
401 
402  return 0 ;
403 }
404 
405 /*
406  * Implement the recipe
407  */
408 static int omega_mflat(cpl_frameset *set, cpl_parameterlist *pars)
409 {
410  int j,jst,jfn;
411  cpl_size nlab;
412  int nflats = 0;
413  int oscan1 = 0;
414  int status = 1;
415 
416  float bias = BIAS;
417 
418  const char *_id = "omega_mflat";
419  char *outbpm = NULL;
420  char *outtwilight = NULL;
421  char *outmflat = NULL;
422  const cpl_frame *firstframe;
423  const cpl_frame *refframe;
424  cpl_frame *prframe_twil = NULL;
425  cpl_frame *prframe_mflat = NULL;
426  cpl_frame *prframe_bpm = NULL;
427  cpl_parameter *par;
428  cpl_propertylist *qclist;
429  cpl_propertylist *alist;
430  cpl_stats *diffstats = NULL;
431  cpl_errorstate prestate = cpl_errorstate_get();
432 
433 
434  /*Start the recipe */
435 
436  if (!pars) {
437  cpl_msg_error (_id, "Parameters list not found");
438  return -1;
439  }
440 
441  if (cpl_frameset_is_empty(set) == 1) {
442  cpl_msg_error (_id, "Frameset not found");
443  return -1;
444  }
445 
446 /* Retrieve input parameters */
447  par = cpl_parameterlist_find(pars, "omega.omega_mflat.ExtensionNumber") ;
448  omega_mflat_config.extnum = cpl_parameter_get_int(par) ;
449 
450  par = cpl_parameterlist_find(pars, "omega.omega_mflat.OverscanMethod") ;
451  omega_mflat_config.oc = cpl_parameter_get_int(par) ;
452 
453  par = cpl_parameterlist_find(pars, "omega.omega_mflat.SigmaClip") ;
454  omega_mflat_config.sigma = cpl_parameter_get_double(par) ;
455 
456  par = cpl_parameterlist_find(pars, "omega.omega_mflat.PAF") ;
457  omega_mflat_config.paf = cpl_parameter_get_bool(par) ;
458 
459 /* Identify the RAW and CALIB frames in the input frameset */
460  if (oc_dfs_set_groups(set)) {
461  cpl_msg_error(_id, "Cannot identify RAW and CALIB frames") ;
462  return -1 ;
463  }
464 
465  /*Initialized things*/
466  omega_mflat_init();
467 
468 /* Verify the frameset contents. */
469  if ((ps.labels = cpl_frameset_labelise(set,omega_compare_tags,
470  &nlab)) == NULL) {
471  cpl_msg_error(_id,"Cannot labelise the input frameset");
472  omega_mflat_tidy();
473  return -1;
474  }
475  if ((ps.flatlist = omega_frameset_subgroup(set,ps.labels,nlab,
476  MTWIL_RAW)) == NULL) {
477  cpl_msg_error(_id,"Cannot find twilight flats in input frameset");
478  omega_mflat_tidy();
479  return -1;
480  }
481  nflats = cpl_frameset_count_tags (ps.flatlist, MTWIL_RAW);
482  if (nflats < 2) {
483  cpl_msg_error (_id, "Need at least 2 (%s) frames to run "
484  "this recipe", MTWIL_RAW);
485  omega_mflat_tidy();
486  return -1;
487  }
488 
489  cpl_msg_info (_id,"There are %d %s frames in frame set",nflats, MTWIL_RAW);
490 
491  /* Check for calibration frames */
492  /* Master Bias */
493  ps.mbframe = cpl_frameset_find_const(set, OMEGA_CALIB_BIAS);
494  if (ps.mbframe == NULL) {
495  cpl_msg_error(cpl_func,"Cannot find OMEGA_CALIB_BIAS input in frame set");
496  omega_mflat_tidy();
497  return -1;
498  }
499  cpl_msg_info(_id,"Using %s %s",OMEGA_CALIB_BIAS, cpl_frame_get_filename(ps.mbframe));
500 
501  /* Cold Pixels Map */
502  ps.cframe = cpl_frameset_find_const(set, OMEGA_CALIB_CPM);
503  if(ps.cframe == NULL){
504  cpl_msg_error(cpl_func,"Cannot find %s input in frame set",OMEGA_CALIB_CPM);
505  omega_mflat_tidy();
506  return -1;
507  }
508  cpl_msg_info(_id,"Using %s %s",OMEGA_CALIB_CPM, cpl_frame_get_filename(ps.cframe));
509 
510  /* Hot Pixels Map */
511  ps.hframe = cpl_frameset_find_const(set, OMEGA_CALIB_HPM);
512  if(ps.hframe == NULL){
513  cpl_msg_error(cpl_func,"Cannot find %s input in frame set",OMEGA_CALIB_HPM);
514  omega_mflat_tidy();
515  return -1;
516  }
517  cpl_msg_info(_id,"Using %s %s",OMEGA_CALIB_HPM, cpl_frame_get_filename(ps.hframe));
518 
519  /* Optional Master Dome Flat */
520  ps.mdomefr = cpl_frameset_find_const(set, OMEGA_CALIB_DOME);
521  if(ps.mdomefr != NULL) {
522  cpl_msg_info(_id,"Using %s %s", OMEGA_CALIB_DOME, cpl_frame_get_filename(ps.mdomefr));
523  }
524 
525  /* Optional Master Night Sky Flat */
526  ps.nskyfr = cpl_frameset_find_const(set, OMEGA_CALIB_NSKY);
527  if(ps.nskyfr != NULL) {
528  cpl_msg_info(_id,"Using %s %s", OMEGA_CALIB_NSKY, cpl_frame_get_filename(ps.nskyfr));
529  }
530 
531  /* Optional Reference master flat frame */
532  refframe = cpl_frameset_find_const(set, REFMFLAT);
533  if (refframe != NULL)
534  cpl_msg_info(cpl_func,"Using %s for comparison",cpl_frame_get_filename(refframe));
535 
536 
537  /* Get first frame from frame set */
538  firstframe = cpl_frameset_get_position(ps.flatlist, 0);
539 
540  /* Loop for each of the image extensions */
541  omega_extensions(firstframe,omega_mflat_config.extnum,&jst,&jfn);
542  if(omega_mflat_config.extnum == 0){
543  cpl_msg_error(cpl_func,"Unsupported extension request, %d",omega_mflat_config.extnum);
544  omega_mflat_tidy();
545  return -1;
546  }
547 
548  for (j = jst; j <= jfn; j++) {
549 
550  isfirst = (j == jst);
551  cpl_msg_info(_id,"Beginning work on extension %d",j);
552 
553  omega_mflat_config.bpm = 0;
554  omega_mflat_config.Mean = 0.0;
555  omega_mflat_config.Median = 0.0;
556  omega_mflat_config.Stdev = 0.0;
557  omega_mflat_config.RawMin = 0.0;
558  omega_mflat_config.RawMax = 0.0;
559  omega_mflat_config.RawMean = 0.0;
560  omega_mflat_config.RawMedian = 0.0;
561  omega_mflat_config.RawStdev = 0.0;
562 
563  /* Check overscan correction method consistency */
564  if(ps.mbframe != NULL){
565  oscan1 = omega_pfits_get_overscan(ps.mbframe, j);
566  if(oscan1 != omega_mflat_config.oc) {
567  cpl_msg_warning (_id, "Overscan correction mode for Master Bias (oc = %d) differs from "
568  "the one used here (oc = %d)", oscan1, omega_mflat_config.oc);
569  }
570  }
571 
572  /* Load first twilight flat */
573  ps.firstflat = omega_fits_load(firstframe,CPL_TYPE_FLOAT,j);
574  ps.eh = omega_fits_get_ehu(ps.firstflat);
575 
576 
577  /* Call the combining routine */
578  status = omega_mflat_combine(pars, j);
579  if(status == 1){
580  cpl_msg_warning(_id, "Image detector is not live");
581  /* Save dummy product */
582  freefits(ps.firstflat);
583  ps.eh = NULL;
584  continue;
585  }
586  else if(status == -1){
587  cpl_msg_error(_id,"Cannot combine images");
588  freespace(outbpm);
589  freespace(outtwilight);
590  freespace(outmflat);
591  omega_mflat_tidy();
592  return -1;
593  }
594 
595  status = 0;
596 
597  qclist = cpl_propertylist_new();
598 
599 
600  /* Compare with reference frame */
601  if(refframe != NULL){
602  if(omega_compare_reference(ps.mflat,refframe,j,&diffstats) == -1){
603  cpl_msg_warning(cpl_func,"Cannot compare with reference frame");
604  }
605  else{
606  cpl_propertylist_append_double(qclist,"ESO QC DIFF REFMFLAT MEAN",
607  cpl_stats_get_mean(diffstats));
608  cpl_propertylist_set_comment(qclist,"ESO QC DIFF REFMFLAT MEAN",
609  "Mean of difference with reference");
610  cpl_propertylist_append_double(qclist,"ESO QC DIFF REFMFLAT MEDIAN",
611  cpl_stats_get_median(diffstats));
612  cpl_propertylist_set_comment(qclist,"ESO QC DIFF REFMFLAT MEDIAN",
613  "Median of difference with reference");
614  cpl_propertylist_append_double(qclist,"ESO QC DIFF REFMFLAT STDEV",
615  cpl_stats_get_stdev(diffstats));
616  cpl_propertylist_set_comment(qclist,"ESO QC DIFF REFMFLAT STDEV",
617  "Stdev of difference with reference");
618 
619  freestats(diffstats);
620  }
621  }
622 
623  /* Save the MASTER FLAT product */
624  /* Compute statistics on master flat */
625  ps.stats = cpl_stats_new_from_image(ps.mflat, CPL_STATS_ALL);
626  if(ps.stats != NULL) {
627  omega_mflat_config.Mean = cpl_stats_get_mean(ps.stats);
628  omega_mflat_config.Median = cpl_stats_get_median(ps.stats);
629  omega_mflat_config.Stdev = cpl_stats_get_stdev(ps.stats);
630  freestats(ps.stats);
631  }
632 
633  /* Save QC parameters in list */
634  cpl_propertylist_append_double(qclist, "ESO QC MASTER FLAT MEAN",
635  omega_mflat_config.Mean) ;
636  cpl_propertylist_set_comment(qclist,"ESO QC MASTER FLAT MEAN",
637  "Mean value of master flat");
638  cpl_propertylist_append_double(qclist, "ESO QC MASTER FLAT MEDIAN",
639  omega_mflat_config.Median) ;
640  cpl_propertylist_set_comment(qclist,"ESO QC MASTER FLAT MEDIAN",
641  "Median value of master flat");
642  cpl_propertylist_append_double(qclist, "ESO QC MASTER FLAT STDEV",
643  omega_mflat_config.Stdev) ;
644  cpl_propertylist_set_comment(qclist,"ESO QC MASTER FLAT STDEV",
645  "Standard deviation value of master flat");
646 
647  /* Create a new product frame object and define some tags */
648  if(isfirst){
649  outmflat = cpl_sprintf("%s_%s.fits", INSTRUME,MFLAT_PROCATG);
650  prframe_mflat = omega_product_frame(outmflat, MFLAT_PROCATG, CPL_FRAME_TYPE_IMAGE);
651  }
652 
653 
654  alist = cpl_propertylist_new();
655  cpl_propertylist_append_string(alist, "EXTNAME",
656  cpl_propertylist_get_string(ps.eh, "EXTNAME"));
657  cpl_propertylist_set_comment(alist,"EXTNAME", "Extension name");
658 
659  cpl_propertylist_copy_property_regexp(alist, ps.eh, WCS_KEYS, 0);
660 
661  /* Add DRS keywords */
662  cpl_propertylist_update_int(alist, "ESO DRS OVERSCAN METHOD", omega_mflat_config.oc);
663  cpl_propertylist_set_comment(alist, "ESO DRS OVERSCAN METHOD", "overscan correction method");
664 
665  if(omega_save_image(ps.mflat,set,pars,alist,qclist,CPL_BPP_IEEE_FLOAT,outmflat,
666  RECIPE,prframe_mflat,NULL,isfirst) == -1){
667  cpl_msg_error(_id,"Cannot save product %s", MFLAT_PROCATG);
668  freeplist(alist);
669  freeplist(qclist);
670  freespace(outmflat);
671  freespace(outtwilight);
672  freespace(outbpm);
673  omega_mflat_tidy();
674  return -1;;
675  }
676 
677 
678  freeplist(qclist);
679  freeimage(ps.mflat);
680  status = 0;
681 
682  /* Save the MASTER TWILIGHT FLAT product */
683  /* Compute statistics on master twilight flat */
684  ps.stats = cpl_stats_new_from_image(ps.mtwilight, CPL_STATS_ALL);
685  if(ps.stats != NULL) {
686  omega_mflat_config.Mean = cpl_stats_get_mean(ps.stats);
687  omega_mflat_config.Median = cpl_stats_get_median(ps.stats);
688  omega_mflat_config.Stdev = cpl_stats_get_stdev(ps.stats);
689  freestats(ps.stats);
690  }
691 
692  /* Save QC parameters in list */
693  qclist = cpl_propertylist_new();
694  cpl_propertylist_append_double(qclist, "ESO QC MASTER TWILIGHT MEAN",
695  omega_mflat_config.Mean) ;
696  cpl_propertylist_set_comment(qclist,"ESO QC MASTER TWILIGHT MEAN",
697  "Mean value of master twilight flat");
698  cpl_propertylist_append_double(qclist, "ESO QC MASTER TWILIGHT MEDIAN",
699  omega_mflat_config.Median) ;
700  cpl_propertylist_set_comment(qclist,"ESO QC MASTER TWILIGHT MEDIAN",
701  "Median value of master twilight flat");
702  cpl_propertylist_append_double(qclist, "ESO QC MASTER TWILIGHT STDEV",
703  omega_mflat_config.Stdev) ;
704  cpl_propertylist_set_comment(qclist,"ESO QC MASTER TWILIGHT STDEV",
705  "Standard deviation value of master twilight flat");
706 
707  cpl_propertylist_append_double(qclist, "ESO QC RAW TWILIGHT MIN",
708  omega_mflat_config.RawMin);
709  cpl_propertylist_append_double(qclist, "ESO QC RAW TWILIGHT MAX",
710  omega_mflat_config.RawMax);
711  cpl_propertylist_append_double(qclist, "ESO QC RAW TWILIGHT MEAN",
712  omega_mflat_config.RawMean);
713  cpl_propertylist_append_double(qclist, "ESO QC RAW TWILIGHT MEDIAN",
714  omega_mflat_config.RawMedian);
715  cpl_propertylist_append_double(qclist, "ESO QC RAW TWILIGHT STDEV",
716  omega_mflat_config.RawStdev);
717 
718  cpl_propertylist_set_comment(qclist, "ESO QC RAW TWILIGHT MIN",
719  "median value of the raw flat flat having the lowest flux");
720  cpl_propertylist_set_comment(qclist, "ESO QC RAW TWILIGHT MAX",
721  "median value of the raw flat flat having the highest flux");
722  cpl_propertylist_set_comment(qclist, "ESO QC RAW TWILIGHT MEAN",
723  "mean value of all input raw flat flats (ADU)");
724  cpl_propertylist_set_comment(qclist, "ESO QC RAW TWILIGHT MEDIAN",
725  "median value of all input raw flat flats (ADU)");
726  cpl_propertylist_set_comment(qclist, "ESO QC RAW TWILIGHT STDEV",
727  "standard deviation of all input raw flat flats (ADU)");
728 
729  /* Create a new product frame object and define some tags */
730  if(isfirst){
731  outtwilight = cpl_sprintf("%s_%s.fits", INSTRUME,MTWIL_PROCATG);
732  prframe_twil = omega_product_frame(outtwilight, MTWIL_PROCATG, CPL_FRAME_TYPE_IMAGE);
733  }
734 
735  if(omega_save_image(ps.mtwilight,set,pars,alist,qclist,CPL_BPP_IEEE_FLOAT,outtwilight,
736  RECIPE,prframe_twil,NULL,isfirst) == -1){
737  cpl_msg_error(_id,"Cannot save product %s", MTWIL_PROCATG);
738  freeplist(qclist);
739  freeplist(alist);
740  freespace(outtwilight);
741  freespace(outbpm);
742  freespace(outmflat);
743  omega_mflat_tidy();
744  return -1;;
745  }
746 
747  freeplist(qclist);
748  freeimage(ps.mtwilight);
749  status = 0;
750 
751  /* Save the BAD PIXELS MAP product */
752  /* Save QC parameters in list */
753  qclist = cpl_propertylist_new();
754  cpl_propertylist_append_int(qclist, "ESO QC NUMBER BAD PIXELS",omega_mflat_config.bpm);
755  cpl_propertylist_set_comment(qclist,"ESO QC NUMBER BAD PIXELS",
756  "Number of bad pixels");
757 
758  if(isfirst){
759  outbpm = cpl_sprintf("%s_%s.fits", INSTRUME,BPM_PROCATG);
760  prframe_bpm = omega_product_frame(outbpm, BPM_PROCATG, CPL_FRAME_TYPE_IMAGE);
761  }
762 
763  if(omega_save_image(ps.bpixels,set,pars,alist,qclist,CPL_BPP_16_SIGNED,outbpm,
764  RECIPE,prframe_bpm,NULL,isfirst) == -1){
765  cpl_msg_error(_id,"Cannot save product %s", BPM_PROCATG);
766  freeplist(alist);
767  freeplist(qclist);
768  freespace(outbpm);
769  freespace(outtwilight);
770  freespace(outmflat);
771  omega_mflat_tidy();
772  return -1;;
773  }
774 
775  freeplist(qclist);
776  freeplist(alist);
777  freeimage(ps.bpixels);
778  freefits(ps.firstflat);
779  ps.firstflat = NULL;
780  ps.eh = NULL;
781 
782  } /* go and work on next extension */
783 
784  /*Clean up */
785  freespace(outbpm);
786  freespace(outtwilight);
787  freespace(outmflat);
788  omega_mflat_tidy();
789 
790  if(!cpl_errorstate_is_equal(prestate)){
791  cpl_errorstate_dump(prestate,CPL_FALSE,NULL);
792  }
793 
794  return 0;
795 
796 }
797 
798 
809 int omega_mflat_combine(cpl_parameterlist *pars, int xn)
810 {
811 
812  int i,nflats,live, naxis1, naxis2;
813  float bias = BIAS;
814  double gain = 0.0;
815  double median = 1.0;
816  double threshold = 0.0;
817  double *data_scales;
818  const char *_id = "omega_mflat_combine";
819 
820  cpl_error_code code;
821  const cpl_frame *flatfr;
822  cpl_vector *scales;
823  cpl_vector *median_vector = NULL;
824  cpl_image *trim_raw,*dev,*good_float,*good_int,*median_all,*new_image;
825  cpl_image *sum_data,*sum_good;
826  cpl_image *mbias;
827  cpl_image *mdome;
828  cpl_image *nsky;
829  cpl_mask *bpm_map;
830  cpl_imagelist *ilist;
831  cpl_mask *good;
832 
833  nflats = cpl_frameset_get_size(ps.flatlist);
834 
835 
836  /* Check that this detector is live*/
837  omega_pfits_get_detlive(ps.eh,&live);
838  if (! live) {
839  return 1;
840  }
841 
842  /* Load master bias image */
843  if(ps.mbframe != NULL){
844  mbias = cpl_image_load(cpl_frame_get_filename(ps.mbframe), CPL_TYPE_FLOAT,0,xn);
845  if (mbias == NULL) {
846  cpl_msg_warning(_id,"Cannot load image %s", OMEGA_CALIB_BIAS);
847  }
848  }
849 
850  /*Create Bad Pixels Map */
851  bpm_map = makebpm(ps.hframe, ps.cframe, xn);
852  if(bpm_map != NULL){
853  /* cpl_mask_not(bpm_map);*/
854  omega_mflat_config.bpm = cpl_mask_count(bpm_map);
855  }
856 
857  scales = cpl_vector_new(nflats);
858  data_scales = cpl_vector_get_data(scales);
859 
860  median_vector = cpl_vector_new (nflats);
861  cpl_vector_fill(median_vector,0.0);
862 
863  ilist = cpl_imagelist_new();
864  cpl_msg_info (_id,"Doing trim and overscan correction on images");
865 
866  /* Loop through all images in frame list */
867  for (i=0; i< nflats; i++)
868  {
869  flatfr = cpl_frameset_get_position(ps.flatlist, i);
870  if (flatfr == NULL)
871  break;
872  trim_raw = TrimOscanCorrect(flatfr, omega_mflat_config.oc, xn);
873  if(trim_raw == NULL){
874  freevector(scales);
875  freevector(median_vector);
876  freeilist(ilist);
877  freemask(bpm_map);
878  freeimage(mbias);
879  return -1;
880  }
881 
882  if (mbias != NULL)
883  cpl_image_subtract(trim_raw, mbias);
884  else
885  cpl_image_subtract_scalar(trim_raw, bias);
886 
887  if (bpm_map == NULL) {
888  ps.stats = cpl_stats_new_from_image(trim_raw, CPL_STATS_ALL);
889  median = cpl_stats_get_median(ps.stats);
890  }
891  else {
892  cpl_image_reject_from_mask(trim_raw, bpm_map);
893  ps.stats = cpl_stats_new_from_image(trim_raw, CPL_STATS_ALL);
894  median = cpl_stats_get_median(ps.stats);
895  }
896 
897  cpl_vector_set(median_vector, i, median);
898 
899  data_scales[i] = (double)1.0/median;
900  cpl_image_divide_scalar(trim_raw, median);
901 
902  cpl_imagelist_set(ilist, trim_raw, i);
903 
904  freestats(ps.stats);
905 
906  }
907 
908  omega_mflat_config.RawMin = cpl_vector_get_min(median_vector);
909  omega_mflat_config.RawMax = cpl_vector_get_max(median_vector);
910  omega_mflat_config.RawMean = cpl_vector_get_mean(median_vector);
911  omega_mflat_config.RawMedian = cpl_vector_get_median(median_vector);
912  omega_mflat_config.RawStdev = cpl_vector_get_stdev(median_vector);
913 
914  freevector(median_vector);
915 
916  freestats(ps.stats);
917  freeimage(mbias);
918 
919  if (ilist == NULL) {
920  cpl_msg_error(_id,"Error in image list <%s>",cpl_error_get_message());
921  freevector(scales);
922  freeimage(trim_raw);
923  freemask(bpm_map);
924  return -1;
925  }
926 
927  /* Read gain from the header */
928  /* FIXME: or read gain from gain recipe product?? */
929  /* The images have the CONAD as the correct value for
930  * the gain, therefore the following lines should be
931  * changed if the headers are fixed.
932  */
933 
934 // omega_pfits_get_gain(ps.eh, &gain);
935  omega_pfits_get_conad(ps.eh, &gain);
936 
937  /*FIXME: should we avoid having gain=0?*/
938  if (gain < 0.1){
939  omega_pfits_get_gain(ps.eh, &gain);
940  // omega_pfits_get_conad(ps.eh, &gain);
941  if(gain < 0.1) {
942  cpl_msg_info(_id,"Using default value for gain");
943  gain = 2.0;
944  }
945  }
946  else {
947  cpl_msg_info(_id,"Gain value from images is %g", gain);
948  }
949 
950  cpl_msg_info (_id,"Computing the median of all images...");
951  median_all = cpl_imagelist_collapse_median_create(ilist);
952 
953  if (median_all == NULL) {
954  cpl_msg_error (_id,"Cannot take median of imset <%s>",cpl_error_get_message());
955  freevector(scales);
956  freeilist(ilist);
957  freemask(bpm_map);
958  return -1;
959  }
960 
961  naxis1 = cpl_image_get_size_x(median_all);
962  naxis2 = cpl_image_get_size_y(median_all);
963 
964  sum_data = cpl_image_new(naxis1,naxis2,CPL_TYPE_FLOAT);
965  sum_good = cpl_image_new(naxis1,naxis2,CPL_TYPE_FLOAT);
966 
967  /* Clip the outliers from the images*/
968  for (i=0; i< nflats; i++)
969  {
970  trim_raw = cpl_imagelist_get(ilist, i);
971  new_image = cpl_image_duplicate(trim_raw);
972 
973  /* FIXME: setting negative values to 0 to avoid failure of SQRT in next step
974  * when there are negative values in the image. These negative values happen when any
975  * given pixel in the domeflat is smaller than the same pixel in the master_bias image.
976  */
977  cpl_image_threshold(trim_raw, 0, FLT_MAX, 0, 2e20);
978 
979  dev = cpl_image_subtract_create(median_all, trim_raw);
980 
981  code = cpl_image_power(trim_raw, 0.5);
982  if(code != CPL_ERROR_NONE) {
983  cpl_msg_error(_id,"Error in SQRT operation <%s>",cpl_error_get_message());
984  freevector(scales);
985  freeilist(ilist);
986  freeimage(median_all);
987  freeimage(sum_data);
988  freeimage(sum_good);
989  freeimage(new_image);
990  freeimage(dev);
991  freemask(bpm_map);
992  return -1;
993  }
994 
995  cpl_image_divide(dev, trim_raw);
996 
997  threshold = omega_mflat_config.sigma * (sqrt(gain * data_scales[i]));
998 
999  good = cpl_mask_threshold_image_create(dev, -threshold, threshold);
1000 
1001  freeimage(dev);
1002  good_int = cpl_image_new_from_mask(good) ;
1003  freemask(good) ;
1004 
1005  good_float = cpl_image_cast(good_int, CPL_TYPE_FLOAT);
1006  freeimage(good_int);
1007 
1008  cpl_image_multiply(new_image, good_float);
1009  cpl_image_add(sum_data, new_image);
1010 
1011  cpl_image_add(sum_good, good_float);
1012 
1013  freeimage(new_image);
1014  freeimage(good_float);
1015 
1016  }
1017 
1018  freeimage(median_all);
1019  freevector(scales);
1020 
1021  /* Create master twilight flat */
1022 
1023  cpl_image_threshold(sum_good, FLT_EPSILON, FLT_MAX, FLT_EPSILON, FLT_MAX);
1024  ps.mtwilight = cpl_image_divide_create(sum_data, sum_good);
1025  if (ps.mtwilight == NULL) {
1026  cpl_msg_error(_id,"Error in division %s <%s>",MTWIL_PROCATG, cpl_error_get_message());
1027  freeilist(ilist);
1028  freeimage(sum_data);
1029  freeimage(sum_good);
1030  freemask(bpm_map);
1031  return -1;
1032  }
1033 
1034  freeimage(sum_data);
1035  freeimage(sum_good);
1036  freeilist(ilist);
1037 
1038 
1039  /* Create a MASTER FLAT */
1040  if(ps.mdomefr != NULL){
1041  /* Load Master Dome */
1042  mdome = cpl_image_load(cpl_frame_get_filename(ps.mdomefr), CPL_TYPE_FLOAT,0,xn);
1043  if (mdome == NULL){
1044  cpl_msg_warning(_id,"Cannot load image %s", OMEGA_CALIB_DOME);
1045  ps.mflat = cpl_image_duplicate(ps.mtwilight);
1046  }
1047  else{
1048  ps.mflat = omega_make_mflat(mdome,ps.mtwilight,bpm_map,pars);
1049  if(ps.mflat == NULL){
1050  cpl_msg_warning(_id,"Cannot calculate Master Flat Field");
1051  ps.mflat = cpl_image_duplicate(ps.mtwilight);
1052  }
1053  freeimage(mdome);
1054  }
1055  }
1056  else{
1057  cpl_msg_warning(_id,"There is no Master Dome Flat in frame set");
1058  ps.mflat = cpl_image_duplicate(ps.mtwilight);
1059  }
1060 
1061  /* Load Master Night Sky Flat */
1062  if (ps.nskyfr != NULL) {
1063  nsky = cpl_image_load(cpl_frame_get_filename(ps.nskyfr), CPL_TYPE_FLOAT,0,xn);
1064  if (nsky == NULL) {
1065  cpl_msg_warning(_id,"Cannot load image %s", OMEGA_CALIB_NSKY);
1066  }
1067  else{
1068  cpl_image_multiply (ps.mflat, nsky);
1069  freeimage(nsky);
1070  }
1071  }
1072 
1073 
1074  /* Create a BAD PIXELS MAP IMAGE to save later*/
1075  /* cpl_mask_not(bpm_map);*/
1076  if(bpm_map != NULL)
1077  ps.bpixels = cpl_image_new_from_mask(bpm_map);
1078 
1079  omega_mflat_config.bpm = cpl_mask_count(bpm_map);
1080  freemask(bpm_map);
1081 
1082  return 0;
1083 }
1084 
1085 /* Initialize the pointers */
1086 static void omega_mflat_init(void) {
1087  ps.bpixels = NULL;
1088  ps.cframe = NULL;
1089  ps.eh = NULL;
1090  ps.firstflat = NULL;
1091  ps.flatlist = NULL;
1092  ps.hframe = NULL;
1093  ps.labels = NULL;
1094  ps.mbframe = NULL;
1095  ps.mdomefr = NULL;
1096  ps.mflat = NULL;
1097  ps.mtwilight = NULL;
1098  ps.nskyfr = NULL;
1099  ps.stats = NULL;
1100 }
1101 
1102 /* Free any allocated memory */
1103 static void omega_mflat_tidy(void) {
1104  freefits(ps.firstflat);
1105  freeframeset(ps.flatlist);
1106  freeimage(ps.bpixels);
1107  freeimage(ps.mflat);
1108  freeimage(ps.mtwilight);
1109  freespace(ps.labels);
1110  freestats(ps.stats);
1111 }
1112 
int omega_pfits_get_overscan(const cpl_frame *frame, int xn)
Get the DRS keyword of the overscan method used to reduce the image.
Definition: omega_pfits.c:912
cpl_image * omega_make_mflat(cpl_image *dome, cpl_image *twil, cpl_mask *bpmmap, cpl_parameterlist *pars)
It creates a master flat field.
Definition: omega_flats.c:73
int omega_pfits_get_detlive(const cpl_propertylist *plist, int *detlive)
Get the value of DET_LIVE.
Definition: omega_pfits.c:214
void omega_extensions(const cpl_frame *frame, int inexten, int *out1, int *out2)
Definition: omega_utils.c:348
int cpl_plugin_get_info(cpl_pluginlist *list)
Build the list of available plugins, for this module.
void omega_pfits_get_conad(cpl_propertylist *plist, double *conad)
Get the value of the keyword CONAD.
Definition: omega_pfits.c:995
int omega_mflat_combine(cpl_parameterlist *pars, int xn)
Process the twilight flats.
Definition: omega_mflat.c:809
cpl_image * TrimOscanCorrect(const cpl_frame *frame, int oscan, int extn)
This method loads a raw image, trims it and (optionally) performs an overscan correction, using the statistics of the pre and overscan regions. This can be done with a single value for the whole image or on a per-row basis.
Definition: omega_trim.c:87
int omega_compare_tags(const cpl_frame *frame1, const cpl_frame *frame2)
Definition: omega_utils.c:206
cpl_mask * makebpm(const cpl_frame *hot, const cpl_frame *cold, int cxn)
Create a bad pixels map.
Definition: omega_bpm.c:191
omega_fits * omega_fits_load(const cpl_frame *inframe, cpl_type type, int extnum)
Definition: omega_fits.c:84
const char * omega_get_license(void)
Get the pipeline copyright and license.
Definition: omega_utils.c:67
cpl_frame * omega_product_frame(const char *filename, const char *tag, cpl_frame_type type)
Setup a frame to save a product.
Definition: omega_utils.c:499
int omega_compare_reference(const cpl_image *master, const cpl_frame *refframe, int ext, cpl_stats **diffstats)
Definition: omega_utils.c:404
cpl_propertylist * omega_fits_get_ehu(omega_fits *p)
Definition: omega_fits.c:489
cpl_frameset * omega_frameset_subgroup(cpl_frameset *frameset, cpl_size *labels, cpl_size nlab, const char *tag)
Definition: omega_utils.c:257
int omega_save_image(const cpl_image *img, cpl_frameset *set, const cpl_parameterlist *pars, cpl_propertylist *alist, const cpl_propertylist *qclist, cpl_type_bpp bpp, const char *name, const char *recipe, cpl_frame *frame, const cpl_frame *inherit, int isfirst)
Save an image as a DFS compliant product.
Definition: omega_utils.c:916
void omega_pfits_get_gain(cpl_propertylist *plist, double *gain)
Read the gain value from an image.
Definition: omega_pfits.c:952