OMEGA Pipeline Reference Manual  1.0.5
omega_mflat.c
1 /* $Id: omega_mflat.c,v 1.6 2012-08-30 07:46:52 agabasch Exp $
2  *
3  * This file is part of the OMEGA Pipeline
4  * Copyright (C) 2002,2003 European Southern Observatory
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19  */
20 
21 /*
22  * $Author: agabasch $
23  * $Date: 2012-08-30 07:46:52 $
24  * $Revision: 1.6 $
25  * $Name: not supported by cvs2svn $
26  */
27 
28 #ifdef HAVE_CONFIG_H
29 #include <config.h>
30 #endif
31 
32 /*-----------------------------------------------------------------------------
33  Includes
34  -----------------------------------------------------------------------------*/
35 
36 #include "omega_recipe.h"
37 
38 #include "omega_background.h"
39 #include "omega_bpm.h"
40 #include "omega_flats.h"
41 #include <cpl_errorstate.h>
42 
72 /*-----------------------------------------------------------------------------
73  Functions prototypes
74  -----------------------------------------------------------------------------*/
75 
76 static int omega_mflat_create(cpl_plugin *) ;
77 static int omega_mflat_exec(cpl_plugin *) ;
78 static int omega_mflat_destroy(cpl_plugin *) ;
79 static int omega_mflat(cpl_frameset *,cpl_parameterlist *);
80 
81 /*-----------------------------------------------------------------------------
82  Private Functions
83  -----------------------------------------------------------------------------*/
84 int omega_mflat_combine(cpl_parameterlist *pars, int xn);
85 static void omega_mflat_init(void);
86 static void omega_mflat_tidy(void);
87 
88 /*-----------------------------------------------------------------------------
89  Static structures
90  -----------------------------------------------------------------------------*/
91 
92 static struct {
93  /* Inputs. Parameters */
94  int extnum;
95  int oc;
96  int paf;
97  double sigma;
98 
99  /* Outputs. QC parameters */
100  int bpm;
101  double Mean;
102  double Median;
103  double Stdev;
104  double RawMin;
105  double RawMax;
106  double RawMean;
107  double RawMedian;
108  double RawStdev;
109 
110 }omega_mflat_config;
111 
112 /* Input and Output */
113 static struct {
114 
115  /* Calib frames */
116  const cpl_frame *cframe;
117  const cpl_frame *hframe;
118  const cpl_frame *mbframe;
119  const cpl_frame *mdomefr;
120  const cpl_frame *nskyfr;
121  const cpl_frame *bpframe;
122  cpl_size *labels;
123  cpl_frameset *flatlist;
124  cpl_propertylist *eh;
125  cpl_stats *stats;
126  omega_fits *firstflat;
127 
128  /* Products */
129  cpl_image *bpixels;
130  cpl_image *mflat;
131  cpl_image *mtwilight;
132 
133 }ps;
134 
135 /*-----------------------------------------------------------------------------
136  Static global variables
137  -----------------------------------------------------------------------------*/
138 
139 /* Static variables */
140 #define RECIPE "omega_mflat"
141 
142 static int isfirst;
143 static int dummy;
144 
145 /*----------------------------------------------------------------------------*/
154 /*----------------------------------------------------------------------------*/
155 
156 int cpl_plugin_get_info(cpl_pluginlist * list)
157 {
158  cpl_recipe * recipe = cpl_calloc(1, sizeof(*recipe)) ;
159  cpl_plugin * plugin = &recipe->interface ;
160 
161  cpl_plugin_init(plugin,
162  CPL_PLUGIN_API,
163  OMEGA_BINARY_VERSION,
164  CPL_PLUGIN_TYPE_RECIPE,
165  "omega_mflat",
166  "OMEGA - Create Master Twilight and Master Flat for each chip",
167  "This recipe is used to derive a valid twilight flat frame (Calfile 543, 546) \n"
168  "for one particular chip and filter. The recipe always takes as input \n"
169  "a list of raw twilight flat frames, a master bias and an optional master \n"
170  "dome flat to be used in creating a final Master Flat Field. In addition, a hot \n"
171  "pixel map (and a cold pixel map can be provided, which will be used to create a \n"
172  "bad pixels map. Optionally, an overscan correction mode \n"
173  "can be set. The default is to apply no overscan correction.\n\n"
174  "The raw twilight flats are trimmed and overscan corrected. The data are then \n"
175  "normalized, averaged, and the result is normalized again. Image statistics \n"
176  "are computed for the resulting frames.",
177  "Sandra Castro",
178  "scastro@eso.org",
180  omega_mflat_create,
181  omega_mflat_exec,
182  omega_mflat_destroy) ;
183 
184  cpl_pluginlist_append(list, plugin) ;
185 
186  return 0;
187 }
188 
189 /*
190  * Create the plugin and the parameters list
191  */
192 static int omega_mflat_create(cpl_plugin * plugin)
193 {
194  cpl_recipe * recipe;
195  cpl_parameter * p ;
196  char *path = NULL;
197 
198  /* Do not create the recipe if an error code is already set */
199  if (cpl_error_get_code() != CPL_ERROR_NONE) {
200  cpl_msg_error(cpl_func, "%s():%d: An error is already set: %s",
201  cpl_func, __LINE__, cpl_error_get_where());
202  return (int)cpl_error_get_code();
203  }
204 
205  if (plugin == NULL) {
206  cpl_msg_error(cpl_func, "Null plugin");
207  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
208  }
209 
210  /* Verify plugin type */
211  if (cpl_plugin_get_type(plugin) != CPL_PLUGIN_TYPE_RECIPE) {
212  cpl_msg_error(cpl_func, "Plugin is not a recipe");
213  cpl_ensure_code(0, (int)CPL_ERROR_TYPE_MISMATCH);
214  }
215 
216  /* Get the recipe */
217  recipe = (cpl_recipe *)plugin;
218 
219  /* Create the parameters list in the cpl_recipe object */
220  recipe->parameters = cpl_parameterlist_new() ;
221  if (recipe->parameters == NULL) {
222  cpl_msg_error(cpl_func, "Parameter list allocation failed");
223  cpl_ensure_code(0, (int)CPL_ERROR_ILLEGAL_OUTPUT);
224  }
225 
226 
227  /* Fill the parameters list */
228 
229 /* --------------- General parameters ---------------------------*/
230  p = cpl_parameter_new_value("omega.omega_mflat.ExtensionNumber",
231  CPL_TYPE_INT,
232  "FITS extension number to load (1 to 32). (-1 = all)",
233  "omega_mflat",
234  -1) ;
235 
236  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI,"ext") ;
237  cpl_parameterlist_append(recipe->parameters, p) ;
238 
239 
240  p = cpl_parameter_new_range("omega.omega_mflat.OverscanMethod",
241  CPL_TYPE_INT,
242  "Overscan Correction Method",
243  "omega_mflat",
244  0, 0, 6);
245 
246  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI,"oc-meth") ;
247  cpl_parameterlist_append(recipe->parameters, p) ;
248 
249  p = cpl_parameter_new_value("omega.omega_mflat.PAF",
250  CPL_TYPE_BOOL,
251  "Boolean value to create PAF files. 1(Yes), 0(No)",
252  "omega_mflat",
253  1) ;
254 
255  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI, "paf") ;
256  cpl_parameterlist_append(recipe->parameters, p) ;
257 
258  p = cpl_parameter_new_value("omega.omega_mflat.SigmaClip",
259  CPL_TYPE_DOUBLE,
260  "Sigma Clipping Threshold",
261  "omega_mflat",
262  3.0) ;
263 
264  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI, "sig-clip") ;
265  cpl_parameterlist_append(recipe->parameters, p) ;
266 
267 
268 
269 /*------------------ Parameters to create a Master Flat ------------------ */
270  path = cpl_sprintf("%s", OMEGA_BIN_PATH);
271  p = cpl_parameter_new_value("omega.omega_mflat.BinPath",
272  CPL_TYPE_STRING,
273  "Path to any external executable program.",
274  "omega.BinPath",
275  path);
276 
277  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI, "bin-path");
278  cpl_parameterlist_append(recipe->parameters, p);
279  cpl_free(path);
280 
281  p = cpl_parameter_new_value("omega.omega_mflat.GaussianFiltSize",
282  CPL_TYPE_DOUBLE,
283  "Standard deviation of Gaussian Fourier filter size",
284  "omega_mflat",
285  9.0) ;
286 
287  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI, "gau_filt_sz") ;
288  cpl_parameterlist_append(recipe->parameters, p) ;
289 
290  p = cpl_parameter_new_value("omega.omega_mflat.MirrorXpix",
291  CPL_TYPE_INT,
292  "Width of X region for mirroring edges (FFT continuity)",
293  "omega_mflat",
294  75) ;
295 
296  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI, "mr_xpix") ;
297  cpl_parameterlist_append(recipe->parameters, p) ;
298 
299  p = cpl_parameter_new_value("omega.omega_mflat.MirrorYpix",
300  CPL_TYPE_INT,
301  "Width of Y region for mirroring edges (FFT continuity)",
302  "omega_mflat",
303  150) ;
304 
305  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI, "mr_ypix") ;
306  cpl_parameterlist_append(recipe->parameters, p) ;
307 
308  /* Parameters to create bad pixels map */
309  p = cpl_parameter_new_range("omega.omega_mflat.LowThre",
310  CPL_TYPE_DOUBLE,
311  "Low flagging threshold for cold pixels map",
312  "omega_mflat",
313  0.96, 0.90, 1.00) ;
314 
315  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI, "low") ;
316  cpl_parameterlist_append(recipe->parameters, p) ;
317 
318  p = cpl_parameter_new_range("omega.omega_mflat.HighThre",
319  CPL_TYPE_DOUBLE,
320  "High flagging threshold for cold pixels map",
321  "omega_flat",
322  1.04, 1.00, 1.10) ;
323 
324  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI,"high") ;
325  cpl_parameterlist_append(recipe->parameters, p) ;
326 
327  return 0;
328 }
329 
330 /*
331  * Execute the plugin
332  */
333 static int omega_mflat_exec(cpl_plugin * plugin)
334 {
335  cpl_recipe * recipe;
336  int recipe_status;
337 
338  /* Return immediately if an error code is already set */
339  if (cpl_error_get_code() != CPL_ERROR_NONE) {
340  cpl_msg_error(cpl_func, "%s():%d: An error is already set: %s",
341  cpl_func, __LINE__, cpl_error_get_where());
342  return (int)cpl_error_get_code();
343  }
344 
345  if (plugin == NULL) {
346  cpl_msg_error(cpl_func, "Null plugin");
347  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
348  }
349 
350  /* Verify plugin type */
351  if (cpl_plugin_get_type(plugin) != CPL_PLUGIN_TYPE_RECIPE) {
352  cpl_msg_error(cpl_func, "Plugin is not a recipe");
353  cpl_ensure_code(0, (int)CPL_ERROR_TYPE_MISMATCH);
354  }
355 
356  /* Get the recipe */
357  recipe = (cpl_recipe *)plugin;
358 
359  /* Verify parameter and frame lists */
360  if (recipe->parameters == NULL) {
361  cpl_msg_error(cpl_func, "Recipe invoked with NULL parameter list");
362  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
363  }
364  if (recipe->frames == NULL) {
365  cpl_msg_error(cpl_func, "Recipe invoked with NULL frame set");
366  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
367  }
368 
369  /* Invoke the recipe */
370  recipe_status = omega_mflat(recipe->frames, recipe->parameters);
371 
372  /* Ensure DFS-compliance of the products */
373  if (cpl_dfs_update_product_header(recipe->frames)) {
374  if (!recipe_status) recipe_status = (int)cpl_error_get_code();
375  }
376 
377  return recipe_status;
378 }
379 
380 /*
381  * Destroy the plugin
382  */
383 static int omega_mflat_destroy(cpl_plugin * plugin)
384 {
385  cpl_recipe * recipe;
386 
387  if (plugin == NULL) {
388  cpl_msg_error(cpl_func, "Null plugin");
389  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
390  }
391 
392  /* Verify plugin type */
393  if (cpl_plugin_get_type(plugin) != CPL_PLUGIN_TYPE_RECIPE) {
394  cpl_msg_error(cpl_func, "Plugin is not a recipe");
395  cpl_ensure_code(0, (int)CPL_ERROR_TYPE_MISMATCH);
396  }
397 
398  /* Get the recipe */
399  recipe = (cpl_recipe *)plugin;
400 
401  cpl_parameterlist_delete(recipe->parameters) ;
402 
403  return 0 ;
404 }
405 
406 /*
407  * Implement the recipe
408  */
409 static int omega_mflat(cpl_frameset *set, cpl_parameterlist *pars)
410 {
411  int j,jst,jfn;
412  cpl_size nlab;
413  int nflats = 0;
414  int oscan1 = 0;
415  int status = 1;
416 
417  float bias = BIAS;
418 
419  const char *_id = "omega_mflat";
420  char *outbpm = NULL;
421  char *outtwilight = NULL;
422  char *outmflat = NULL;
423  const cpl_frame *firstframe;
424  const cpl_frame *refframe;
425  cpl_frame *prframe_twil = NULL;
426  cpl_frame *prframe_mflat = NULL;
427  cpl_frame *prframe_bpm = NULL;
428  cpl_parameter *par;
429  cpl_propertylist *qclist;
430  cpl_propertylist *alist;
431  cpl_stats *diffstats = NULL;
432  cpl_errorstate prestate = cpl_errorstate_get();
433 
434 
435  /*Start the recipe */
436 
437  if (!pars) {
438  cpl_msg_error (_id, "Parameters list not found");
439  return -1;
440  }
441 
442  if (cpl_frameset_is_empty(set) == 1) {
443  cpl_msg_error (_id, "Frameset not found");
444  return -1;
445  }
446 
447 /* Retrieve input parameters */
448  par = cpl_parameterlist_find(pars, "omega.omega_mflat.ExtensionNumber") ;
449  omega_mflat_config.extnum = cpl_parameter_get_int(par) ;
450 
451  par = cpl_parameterlist_find(pars, "omega.omega_mflat.OverscanMethod") ;
452  omega_mflat_config.oc = cpl_parameter_get_int(par) ;
453 
454  par = cpl_parameterlist_find(pars, "omega.omega_mflat.SigmaClip") ;
455  omega_mflat_config.sigma = cpl_parameter_get_double(par) ;
456 
457  par = cpl_parameterlist_find(pars, "omega.omega_mflat.PAF") ;
458  omega_mflat_config.paf = cpl_parameter_get_bool(par) ;
459 
460 /* Identify the RAW and CALIB frames in the input frameset */
461  if (oc_dfs_set_groups(set)) {
462  cpl_msg_error(_id, "Cannot identify RAW and CALIB frames") ;
463  return -1 ;
464  }
465 
466  /*Initialized things*/
467  omega_mflat_init();
468 
469 /* Verify the frameset contents. */
470  if ((ps.labels = cpl_frameset_labelise(set,omega_compare_tags,
471  &nlab)) == NULL) {
472  cpl_msg_error(_id,"Cannot labelise the input frameset");
473  omega_mflat_tidy();
474  return -1;
475  }
476  if ((ps.flatlist = omega_frameset_subgroup(set,ps.labels,nlab,
477  MTWIL_RAW)) == NULL) {
478  cpl_msg_error(_id,"Cannot find twilight flats in input frameset");
479  omega_mflat_tidy();
480  return -1;
481  }
482  nflats = cpl_frameset_count_tags (ps.flatlist, MTWIL_RAW);
483  if (nflats < 2) {
484  cpl_msg_error (_id, "Need at least 2 (%s) frames to run "
485  "this recipe", MTWIL_RAW);
486  omega_mflat_tidy();
487  return -1;
488  }
489 
490  cpl_msg_info (_id,"There are %d %s frames in frame set",nflats, MTWIL_RAW);
491 
492  /* Check for calibration frames */
493  /* Master Bias */
494  ps.mbframe = cpl_frameset_find_const(set, OMEGA_CALIB_BIAS);
495  if (ps.mbframe == NULL) {
496  cpl_msg_error(cpl_func,"Cannot find OMEGA_CALIB_BIAS input in frame set");
497  omega_mflat_tidy();
498  return -1;
499  }
500  cpl_msg_info(_id,"Using %s %s",OMEGA_CALIB_BIAS, cpl_frame_get_filename(ps.mbframe));
501 
502  /* Cold Pixels Map */
503  ps.cframe = cpl_frameset_find_const(set, OMEGA_CALIB_CPM);
504  if(ps.cframe == NULL){
505  cpl_msg_error(cpl_func,"Cannot find %s input in frame set",OMEGA_CALIB_CPM);
506  omega_mflat_tidy();
507  return -1;
508  }
509  cpl_msg_info(_id,"Using %s %s",OMEGA_CALIB_CPM, cpl_frame_get_filename(ps.cframe));
510 
511  /* Hot Pixels Map */
512  ps.hframe = cpl_frameset_find_const(set, OMEGA_CALIB_HPM);
513  if(ps.hframe == NULL){
514  cpl_msg_error(cpl_func,"Cannot find %s input in frame set",OMEGA_CALIB_HPM);
515  omega_mflat_tidy();
516  return -1;
517  }
518  cpl_msg_info(_id,"Using %s %s",OMEGA_CALIB_HPM, cpl_frame_get_filename(ps.hframe));
519 
520  /* Optional Master Dome Flat */
521  ps.mdomefr = cpl_frameset_find_const(set, OMEGA_CALIB_DOME);
522  if(ps.mdomefr != NULL) {
523  cpl_msg_info(_id,"Using %s %s", OMEGA_CALIB_DOME, cpl_frame_get_filename(ps.mdomefr));
524  }
525 
526  /* Optional Master Night Sky Flat */
527  ps.nskyfr = cpl_frameset_find_const(set, OMEGA_CALIB_NSKY);
528  if(ps.nskyfr != NULL) {
529  cpl_msg_info(_id,"Using %s %s", OMEGA_CALIB_NSKY, cpl_frame_get_filename(ps.nskyfr));
530  }
531 
532  /* Optional Reference master flat frame */
533  refframe = cpl_frameset_find_const(set, REFMFLAT);
534  if (refframe != NULL)
535  cpl_msg_info(cpl_func,"Using %s for comparison",cpl_frame_get_filename(refframe));
536 
537 
538  /* Get first frame from frame set */
539  firstframe = cpl_frameset_get_first(ps.flatlist);
540 
541  /* Loop for each of the image extensions */
542  omega_extensions(firstframe,omega_mflat_config.extnum,&jst,&jfn);
543  if(omega_mflat_config.extnum == 0){
544  cpl_msg_error(cpl_func,"Unsupported extension request, %d",omega_mflat_config.extnum);
545  omega_mflat_tidy();
546  return -1;
547  }
548 
549  for (j = jst; j <= jfn; j++) {
550 
551  isfirst = (j == jst);
552  cpl_msg_info(_id,"Beginning work on extension %d",j);
553 
554  omega_mflat_config.bpm = 0;
555  omega_mflat_config.Mean = 0.0;
556  omega_mflat_config.Median = 0.0;
557  omega_mflat_config.Stdev = 0.0;
558  omega_mflat_config.RawMin = 0.0;
559  omega_mflat_config.RawMax = 0.0;
560  omega_mflat_config.RawMean = 0.0;
561  omega_mflat_config.RawMedian = 0.0;
562  omega_mflat_config.RawStdev = 0.0;
563 
564  /* Check overscan correction method consistency */
565  if(ps.mbframe != NULL){
566  oscan1 = omega_pfits_get_overscan(ps.mbframe, j);
567  if(oscan1 != omega_mflat_config.oc) {
568  cpl_msg_warning (_id, "Overscan correction mode for Master Bias (oc = %d) differs from "
569  "the one used here (oc = %d)", oscan1, omega_mflat_config.oc);
570  }
571  }
572 
573  /* Load first twilight flat */
574  ps.firstflat = omega_fits_load(firstframe,CPL_TYPE_FLOAT,j);
575  ps.eh = omega_fits_get_ehu(ps.firstflat);
576 
577 
578  /* Call the combining routine */
579  status = omega_mflat_combine(pars, j);
580  if(status == 1){
581  cpl_msg_warning(_id, "Image detector is not live");
582  /* Save dummy product */
583  freefits(ps.firstflat);
584  ps.eh = NULL;
585  continue;
586  }
587  else if(status == -1){
588  cpl_msg_error(_id,"Cannot combine images");
589  freespace(outbpm);
590  freespace(outtwilight);
591  freespace(outmflat);
592  omega_mflat_tidy();
593  return -1;
594  }
595 
596  status = 0;
597 
598  qclist = cpl_propertylist_new();
599 
600 
601  /* Compare with reference frame */
602  if(refframe != NULL){
603  if(omega_compare_reference(ps.mflat,refframe,j,&diffstats) == -1){
604  cpl_msg_warning(cpl_func,"Cannot compare with reference frame");
605  }
606  else{
607  cpl_propertylist_append_double(qclist,"ESO QC DIFF REFMFLAT MEAN",
608  cpl_stats_get_mean(diffstats));
609  cpl_propertylist_set_comment(qclist,"ESO QC DIFF REFMFLAT MEAN",
610  "Mean of difference with reference");
611  cpl_propertylist_append_double(qclist,"ESO QC DIFF REFMFLAT MEDIAN",
612  cpl_stats_get_median(diffstats));
613  cpl_propertylist_set_comment(qclist,"ESO QC DIFF REFMFLAT MEDIAN",
614  "Median of difference with reference");
615  cpl_propertylist_append_double(qclist,"ESO QC DIFF REFMFLAT STDEV",
616  cpl_stats_get_stdev(diffstats));
617  cpl_propertylist_set_comment(qclist,"ESO QC DIFF REFMFLAT STDEV",
618  "Stdev of difference with reference");
619 
620  freestats(diffstats);
621  }
622  }
623 
624  /* Save the MASTER FLAT product */
625  /* Compute statistics on master flat */
626  ps.stats = cpl_stats_new_from_image(ps.mflat, CPL_STATS_ALL);
627  if(ps.stats != NULL) {
628  omega_mflat_config.Mean = cpl_stats_get_mean(ps.stats);
629  omega_mflat_config.Median = cpl_stats_get_median(ps.stats);
630  omega_mflat_config.Stdev = cpl_stats_get_stdev(ps.stats);
631  freestats(ps.stats);
632  }
633 
634  /* Save QC parameters in list */
635  cpl_propertylist_append_double(qclist, "ESO QC MASTER FLAT MEAN",
636  omega_mflat_config.Mean) ;
637  cpl_propertylist_set_comment(qclist,"ESO QC MASTER FLAT MEAN",
638  "Mean value of master flat");
639  cpl_propertylist_append_double(qclist, "ESO QC MASTER FLAT MEDIAN",
640  omega_mflat_config.Median) ;
641  cpl_propertylist_set_comment(qclist,"ESO QC MASTER FLAT MEDIAN",
642  "Median value of master flat");
643  cpl_propertylist_append_double(qclist, "ESO QC MASTER FLAT STDEV",
644  omega_mflat_config.Stdev) ;
645  cpl_propertylist_set_comment(qclist,"ESO QC MASTER FLAT STDEV",
646  "Standard deviation value of master flat");
647 
648  /* Create a new product frame object and define some tags */
649  if(isfirst){
650  outmflat = cpl_sprintf("%s_%s.fits", INSTRUME,MFLAT_PROCATG);
651  prframe_mflat = omega_product_frame(outmflat, MFLAT_PROCATG, CPL_FRAME_TYPE_IMAGE);
652  }
653 
654 
655  alist = cpl_propertylist_new();
656  cpl_propertylist_append_string(alist, "EXTNAME",
657  cpl_propertylist_get_string(ps.eh, "EXTNAME"));
658  cpl_propertylist_set_comment(alist,"EXTNAME", "Extension name");
659 
660  cpl_propertylist_copy_property_regexp(alist, ps.eh, WCS_KEYS, 0);
661 
662  /* Add DRS keywords */
663  cpl_propertylist_update_int(alist, "ESO DRS OVERSCAN METHOD", omega_mflat_config.oc);
664  cpl_propertylist_set_comment(alist, "ESO DRS OVERSCAN METHOD", "overscan correction method");
665 
666  if(omega_save_image(ps.mflat,set,pars,alist,qclist,CPL_BPP_IEEE_FLOAT,outmflat,
667  RECIPE,prframe_mflat,NULL,isfirst) == -1){
668  cpl_msg_error(_id,"Cannot save product %s", MFLAT_PROCATG);
669  freeplist(alist);
670  freeplist(qclist);
671  freespace(outmflat);
672  freespace(outtwilight);
673  freespace(outbpm);
674  omega_mflat_tidy();
675  return -1;;
676  }
677 
678 
679  freeplist(qclist);
680  freeimage(ps.mflat);
681  status = 0;
682 
683  /* Save the MASTER TWILIGHT FLAT product */
684  /* Compute statistics on master twilight flat */
685  ps.stats = cpl_stats_new_from_image(ps.mtwilight, CPL_STATS_ALL);
686  if(ps.stats != NULL) {
687  omega_mflat_config.Mean = cpl_stats_get_mean(ps.stats);
688  omega_mflat_config.Median = cpl_stats_get_median(ps.stats);
689  omega_mflat_config.Stdev = cpl_stats_get_stdev(ps.stats);
690  freestats(ps.stats);
691  }
692 
693  /* Save QC parameters in list */
694  qclist = cpl_propertylist_new();
695  cpl_propertylist_append_double(qclist, "ESO QC MASTER TWILIGHT MEAN",
696  omega_mflat_config.Mean) ;
697  cpl_propertylist_set_comment(qclist,"ESO QC MASTER TWILIGHT MEAN",
698  "Mean value of master twilight flat");
699  cpl_propertylist_append_double(qclist, "ESO QC MASTER TWILIGHT MEDIAN",
700  omega_mflat_config.Median) ;
701  cpl_propertylist_set_comment(qclist,"ESO QC MASTER TWILIGHT MEDIAN",
702  "Median value of master twilight flat");
703  cpl_propertylist_append_double(qclist, "ESO QC MASTER TWILIGHT STDEV",
704  omega_mflat_config.Stdev) ;
705  cpl_propertylist_set_comment(qclist,"ESO QC MASTER TWILIGHT STDEV",
706  "Standard deviation value of master twilight flat");
707 
708  cpl_propertylist_append_double(qclist, "ESO QC RAW TWILIGHT MIN",
709  omega_mflat_config.RawMin);
710  cpl_propertylist_append_double(qclist, "ESO QC RAW TWILIGHT MAX",
711  omega_mflat_config.RawMax);
712  cpl_propertylist_append_double(qclist, "ESO QC RAW TWILIGHT MEAN",
713  omega_mflat_config.RawMean);
714  cpl_propertylist_append_double(qclist, "ESO QC RAW TWILIGHT MEDIAN",
715  omega_mflat_config.RawMedian);
716  cpl_propertylist_append_double(qclist, "ESO QC RAW TWILIGHT STDEV",
717  omega_mflat_config.RawStdev);
718 
719  cpl_propertylist_set_comment(qclist, "ESO QC RAW TWILIGHT MIN",
720  "median value of the raw flat flat having the lowest flux");
721  cpl_propertylist_set_comment(qclist, "ESO QC RAW TWILIGHT MAX",
722  "median value of the raw flat flat having the highest flux");
723  cpl_propertylist_set_comment(qclist, "ESO QC RAW TWILIGHT MEAN",
724  "mean value of all input raw flat flats (ADU)");
725  cpl_propertylist_set_comment(qclist, "ESO QC RAW TWILIGHT MEDIAN",
726  "median value of all input raw flat flats (ADU)");
727  cpl_propertylist_set_comment(qclist, "ESO QC RAW TWILIGHT STDEV",
728  "standard deviation of all input raw flat flats (ADU)");
729 
730  /* Create a new product frame object and define some tags */
731  if(isfirst){
732  outtwilight = cpl_sprintf("%s_%s.fits", INSTRUME,MTWIL_PROCATG);
733  prframe_twil = omega_product_frame(outtwilight, MTWIL_PROCATG, CPL_FRAME_TYPE_IMAGE);
734  }
735 
736  if(omega_save_image(ps.mtwilight,set,pars,alist,qclist,CPL_BPP_IEEE_FLOAT,outtwilight,
737  RECIPE,prframe_twil,NULL,isfirst) == -1){
738  cpl_msg_error(_id,"Cannot save product %s", MTWIL_PROCATG);
739  freeplist(qclist);
740  freeplist(alist);
741  freespace(outtwilight);
742  freespace(outbpm);
743  freespace(outmflat);
744  omega_mflat_tidy();
745  return -1;;
746  }
747 
748  freeplist(qclist);
749  freeimage(ps.mtwilight);
750  status = 0;
751 
752  /* Save the BAD PIXELS MAP product */
753  /* Save QC parameters in list */
754  qclist = cpl_propertylist_new();
755  cpl_propertylist_append_int(qclist, "ESO QC NUMBER BAD PIXELS",omega_mflat_config.bpm);
756  cpl_propertylist_set_comment(qclist,"ESO QC NUMBER BAD PIXELS",
757  "Number of bad pixels");
758 
759  if(isfirst){
760  outbpm = cpl_sprintf("%s_%s.fits", INSTRUME,BPM_PROCATG);
761  prframe_bpm = omega_product_frame(outbpm, BPM_PROCATG, CPL_FRAME_TYPE_IMAGE);
762  }
763 
764  if(omega_save_image(ps.bpixels,set,pars,alist,qclist,CPL_BPP_16_SIGNED,outbpm,
765  RECIPE,prframe_bpm,NULL,isfirst) == -1){
766  cpl_msg_error(_id,"Cannot save product %s", BPM_PROCATG);
767  freeplist(alist);
768  freeplist(qclist);
769  freespace(outbpm);
770  freespace(outtwilight);
771  freespace(outmflat);
772  omega_mflat_tidy();
773  return -1;;
774  }
775 
776  freeplist(qclist);
777  freeplist(alist);
778  freeimage(ps.bpixels);
779  freefits(ps.firstflat);
780  ps.firstflat = NULL;
781  ps.eh = NULL;
782 
783  } /* go and work on next extension */
784 
785  /*Clean up */
786  freespace(outbpm);
787  freespace(outtwilight);
788  freespace(outmflat);
789  omega_mflat_tidy();
790 
791  if(!cpl_errorstate_is_equal(prestate)){
792  cpl_errorstate_dump(prestate,CPL_FALSE,NULL);
793  }
794 
795  return 0;
796 
797 }
798 
799 
810 int omega_mflat_combine(cpl_parameterlist *pars, int xn)
811 {
812 
813  int i,nflats,live, naxis1, naxis2;
814  float bias = BIAS;
815  double gain = 0.0;
816  double median = 1.0;
817  double threshold = 0.0;
818  double *data_scales;
819  const char *_id = "omega_mflat_combine";
820 
821  cpl_error_code code;
822  const cpl_frame *flatfr;
823  cpl_vector *scales;
824  cpl_vector *median_vector = NULL;
825  cpl_image *trim_raw,*dev,*good_float,*good_int,*median_all,*new_image;
826  cpl_image *sum_data,*sum_good;
827  cpl_image *mbias;
828  cpl_image *mdome;
829  cpl_image *nsky;
830  cpl_mask *bpm_map;
831  cpl_imagelist *ilist;
832  cpl_mask *good;
833 
834  nflats = cpl_frameset_get_size(ps.flatlist);
835  flatfr = cpl_frameset_get_first_const(ps.flatlist);
836 
837  /* Check that this detector is live*/
838  omega_pfits_get_detlive(ps.eh,&live);
839  if (! live) {
840  return 1;
841  }
842 
843  /* Load master bias image */
844  if(ps.mbframe != NULL){
845  mbias = cpl_image_load(cpl_frame_get_filename(ps.mbframe), CPL_TYPE_FLOAT,0,xn);
846  if (mbias == NULL) {
847  cpl_msg_warning(_id,"Cannot load image %s", OMEGA_CALIB_BIAS);
848  }
849  }
850 
851  /*Create Bad Pixels Map */
852  bpm_map = makebpm(ps.hframe, ps.cframe, xn);
853  if(bpm_map != NULL){
854  /* cpl_mask_not(bpm_map);*/
855  omega_mflat_config.bpm = cpl_mask_count(bpm_map);
856  }
857 
858  scales = cpl_vector_new(nflats);
859  data_scales = cpl_vector_get_data(scales);
860 
861  median_vector = cpl_vector_new (nflats);
862  cpl_vector_fill(median_vector,0.0);
863 
864  ilist = cpl_imagelist_new();
865  cpl_msg_info (_id,"Doing trim and overscan correction on images");
866 
867  /* Loop through all images in frame list */
868  for (i=0; i< nflats; i++)
869  {
870 
871  trim_raw = TrimOscanCorrect(flatfr, omega_mflat_config.oc, xn);
872  if(trim_raw == NULL){
873  freevector(scales);
874  freevector(median_vector);
875  freeilist(ilist);
876  freemask(bpm_map);
877  freeimage(mbias);
878  return -1;
879  }
880 
881  if (mbias != NULL)
882  cpl_image_subtract(trim_raw, mbias);
883  else
884  cpl_image_subtract_scalar(trim_raw, bias);
885 
886  if (bpm_map == NULL) {
887  ps.stats = cpl_stats_new_from_image(trim_raw, CPL_STATS_ALL);
888  median = cpl_stats_get_median(ps.stats);
889  }
890  else {
891  cpl_image_reject_from_mask(trim_raw, bpm_map);
892  ps.stats = cpl_stats_new_from_image(trim_raw, CPL_STATS_ALL);
893  median = cpl_stats_get_median(ps.stats);
894  }
895 
896  cpl_vector_set(median_vector, i, median);
897 
898  data_scales[i] = (double)1.0/median;
899 
900  cpl_image_divide_scalar(trim_raw, median);
901 
902  cpl_imagelist_set(ilist, trim_raw, i);
903 
904 
905  flatfr = cpl_frameset_get_next_const(ps.flatlist);
906  if (flatfr == NULL)
907  break;
908 
909  freestats(ps.stats);
910 
911  }
912 
913  omega_mflat_config.RawMin = cpl_vector_get_min(median_vector);
914  omega_mflat_config.RawMax = cpl_vector_get_max(median_vector);
915  omega_mflat_config.RawMean = cpl_vector_get_mean(median_vector);
916  omega_mflat_config.RawMedian = cpl_vector_get_median(median_vector);
917  omega_mflat_config.RawStdev = cpl_vector_get_stdev(median_vector);
918 
919  freevector(median_vector);
920 
921  freestats(ps.stats);
922  freeimage(mbias);
923 
924  if (ilist == NULL) {
925  cpl_msg_error(_id,"Error in image list <%s>",cpl_error_get_message());
926  freevector(scales);
927  freeimage(trim_raw);
928  freemask(bpm_map);
929  return -1;
930  }
931 
932  /* Read gain from the header */
933  /* FIXME: or read gain from gain recipe product?? */
934  /* The images have the CONAD as the correct value for
935  * the gain, therefore the following lines should be
936  * changed if the headers are fixed.
937  */
938 
939 // omega_pfits_get_gain(ps.eh, &gain);
940  omega_pfits_get_conad(ps.eh, &gain);
941 
942  /*FIXME: should we avoid having gain=0?*/
943  if (gain < 0.1){
944  omega_pfits_get_gain(ps.eh, &gain);
945  // omega_pfits_get_conad(ps.eh, &gain);
946  if(gain < 0.1) {
947  cpl_msg_info(_id,"Using default value for gain");
948  gain = 2.0;
949  }
950  }
951  else {
952  cpl_msg_info(_id,"Gain value from images is %g", gain);
953  }
954 
955  cpl_msg_info (_id,"Computing the median of all images...");
956  median_all = cpl_imagelist_collapse_median_create(ilist);
957 
958  if (median_all == NULL) {
959  cpl_msg_error (_id,"Cannot take median of imset <%s>",cpl_error_get_message());
960  freevector(scales);
961  freeilist(ilist);
962  freemask(bpm_map);
963  return -1;
964  }
965 
966  naxis1 = cpl_image_get_size_x(median_all);
967  naxis2 = cpl_image_get_size_y(median_all);
968 
969  sum_data = cpl_image_new(naxis1,naxis2,CPL_TYPE_FLOAT);
970  sum_good = cpl_image_new(naxis1,naxis2,CPL_TYPE_FLOAT);
971 
972  /* Clip the outliers from the images*/
973  for (i=0; i< nflats; i++)
974  {
975  trim_raw = cpl_imagelist_get(ilist, i);
976  new_image = cpl_image_duplicate(trim_raw);
977 
978  /* FIXME: setting negative values to 0 to avoid failure of SQRT in next step
979  * when there are negative values in the image. These negative values happen when any
980  * given pixel in the domeflat is smaller than the same pixel in the master_bias image.
981  */
982  cpl_image_threshold(trim_raw, 0, FLT_MAX, 0, 2e20);
983 
984  dev = cpl_image_subtract_create(median_all, trim_raw);
985 
986  code = cpl_image_power(trim_raw, 0.5);
987  if(code != CPL_ERROR_NONE) {
988  cpl_msg_error(_id,"Error in SQRT operation <%s>",cpl_error_get_message());
989  freevector(scales);
990  freeilist(ilist);
991  freeimage(median_all);
992  freeimage(sum_data);
993  freeimage(sum_good);
994  freeimage(new_image);
995  freeimage(dev);
996  freemask(bpm_map);
997  return -1;
998  }
999 
1000  cpl_image_divide(dev, trim_raw);
1001 
1002  threshold = omega_mflat_config.sigma * (sqrt(gain * data_scales[i]));
1003 
1004  good = cpl_mask_threshold_image_create(dev, -threshold, threshold);
1005 
1006  freeimage(dev);
1007  good_int = cpl_image_new_from_mask(good) ;
1008  freemask(good) ;
1009 
1010  good_float = cpl_image_cast(good_int, CPL_TYPE_FLOAT);
1011  freeimage(good_int);
1012 
1013  cpl_image_multiply(new_image, good_float);
1014  cpl_image_add(sum_data, new_image);
1015 
1016  cpl_image_add(sum_good, good_float);
1017 
1018  freeimage(new_image);
1019  freeimage(good_float);
1020 
1021  }
1022 
1023  freeimage(median_all);
1024  freevector(scales);
1025 
1026  /* Create master twilight flat */
1027  ps.mtwilight = cpl_image_divide_create(sum_data, sum_good);
1028  if (ps.mtwilight == NULL) {
1029  cpl_msg_error(_id,"Error in division %s <%s>",MTWIL_PROCATG, cpl_error_get_message());
1030  freeilist(ilist);
1031  freeimage(sum_data);
1032  freeimage(sum_good);
1033  freemask(bpm_map);
1034  return -1;
1035  }
1036 
1037  freeimage(sum_data);
1038  freeimage(sum_good);
1039  freeilist(ilist);
1040 
1041 
1042  /* Create a MASTER FLAT */
1043  if(ps.mdomefr != NULL){
1044  /* Load Master Dome */
1045  mdome = cpl_image_load(cpl_frame_get_filename(ps.mdomefr), CPL_TYPE_FLOAT,0,xn);
1046  if (mdome == NULL){
1047  cpl_msg_warning(_id,"Cannot load image %s", OMEGA_CALIB_DOME);
1048  ps.mflat = cpl_image_duplicate(ps.mtwilight);
1049  }
1050  else{
1051  ps.mflat = omega_make_mflat(mdome,ps.mtwilight,bpm_map,pars);
1052  if(ps.mflat == NULL){
1053  cpl_msg_warning(_id,"Cannot calculate Master Flat Field");
1054  ps.mflat = cpl_image_duplicate(ps.mtwilight);
1055  }
1056  freeimage(mdome);
1057  }
1058  }
1059  else{
1060  cpl_msg_warning(_id,"There is no Master Dome Flat in frame set");
1061  ps.mflat = cpl_image_duplicate(ps.mtwilight);
1062  }
1063 
1064  /* Load Master Night Sky Flat */
1065  if (ps.nskyfr != NULL) {
1066  nsky = cpl_image_load(cpl_frame_get_filename(ps.nskyfr), CPL_TYPE_FLOAT,0,xn);
1067  if (nsky == NULL) {
1068  cpl_msg_warning(_id,"Cannot load image %s", OMEGA_CALIB_NSKY);
1069  }
1070  else{
1071  cpl_image_multiply (ps.mflat, nsky);
1072  freeimage(nsky);
1073  }
1074  }
1075 
1076 
1077  /* Create a BAD PIXELS MAP IMAGE to save later*/
1078  /* cpl_mask_not(bpm_map);*/
1079  if(bpm_map != NULL)
1080  ps.bpixels = cpl_image_new_from_mask(bpm_map);
1081 
1082  omega_mflat_config.bpm = cpl_mask_count(bpm_map);
1083  freemask(bpm_map);
1084 
1085  return 0;
1086 }
1087 
1088 /* Initialize the pointers */
1089 static void omega_mflat_init(void) {
1090  ps.bpixels = NULL;
1091  ps.cframe = NULL;
1092  ps.eh = NULL;
1093  ps.firstflat = NULL;
1094  ps.flatlist = NULL;
1095  ps.hframe = NULL;
1096  ps.labels = NULL;
1097  ps.mbframe = NULL;
1098  ps.mdomefr = NULL;
1099  ps.mflat = NULL;
1100  ps.mtwilight = NULL;
1101  ps.nskyfr = NULL;
1102  ps.stats = NULL;
1103 }
1104 
1105 /* Free any allocated memory */
1106 static void omega_mflat_tidy(void) {
1107  freefits(ps.firstflat);
1108  freeframeset(ps.flatlist);
1109  freeimage(ps.bpixels);
1110  freeimage(ps.mflat);
1111  freeimage(ps.mtwilight);
1112  freespace(ps.labels);
1113  freestats(ps.stats);
1114 }
1115