OMEGA Pipeline Reference Manual  1.0.6
omega_mdome.c
1 /* $Id: omega_mdome.c,v 1.7 2012-08-30 07:46:52 agabasch Exp $
2  *
3  * This file is part of the OMEGA Pipeline
4  * Copyright (C) 2002,2003 European Southern Observatory
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19  */
20 
21 /*
22  * $Author: agabasch $
23  * $Date: 2012-08-30 07:46:52 $
24  * $Revision: 1.7 $
25  * $Name: not supported by cvs2svn $
26  */
27 
28 #ifdef HAVE_CONFIG_H
29 #include <config.h>
30 #endif
31 
32 
33 #include "omega_recipe.h"
34 #include "omega_background.h"
35 
36 /*----------------------------------------------------------------------------*/
67 /*-----------------------------------------------------------------------------
68  Functions prototypes
69  -----------------------------------------------------------------------------*/
70 
71 static int omega_mdome_create(cpl_plugin *);
72 static int omega_mdome_exec(cpl_plugin *);
73 static int omega_mdome_destroy(cpl_plugin *);
74 static int omega_mdome(cpl_frameset *, cpl_parameterlist *);
75 
76 /*-----------------------------------------------------------------------------
77  Private Functions
78  -----------------------------------------------------------------------------*/
79 
80 int omega_mdome_combine(cpl_parameterlist *pars, int xn);
81 static void omega_mdome_init(void);
82 static void omega_mdome_tidy(void);
83 
84 /*-----------------------------------------------------------------------------
85  Static structures
86  -----------------------------------------------------------------------------*/
87 
88 static struct {
89  /* Inputs. Parameters */
90  int extnum;
91  int oc;
92  int paf;
93  double sigma;
94  double lthre;
95  double hthre;
96 
97  /* Outputs. QC parameters */
98  int CountColdPixels;
99  double Mean;
100  double Median;
101  double Stdev;
102  double RawMin;
103  double RawMax;
104  double RawMean;
105  double RawMedian;
106  double RawStdev;
107 
108 }omega_mdome_config;
109 
110 /* Input and Output */
111 static struct {
112 
113  cpl_size *labels;
114  const cpl_frame *mbframe;
115  const cpl_frame *cframe;
116  const cpl_frame *hframe;
117  cpl_frameset *domelist;
118  cpl_stats *stats;
119  cpl_propertylist *ph;
120  cpl_propertylist *eh;
121  omega_fits *firstdome;
122 
123  /* Products */
124  cpl_image *mdome;
125  cpl_image *cpixels;
126 
127 }ps;
128 
129 /*-----------------------------------------------------------------------------
130  Static global variables
131  -----------------------------------------------------------------------------*/
132 
133 /* Static variables */
134 #define RECIPE "omega_mdome"
135 
136 static int isfirst;
137 
138 
139 /*----------------------------------------------------------------------------*/
147 /*----------------------------------------------------------------------------*/
148 int cpl_plugin_get_info(cpl_pluginlist * list)
149 {
150  cpl_recipe * recipe = cpl_calloc(1, sizeof(*recipe)) ;
151  cpl_plugin * plugin = &recipe->interface ;
152 
153  cpl_plugin_init(plugin,
154  CPL_PLUGIN_API,
155  OMEGA_BINARY_VERSION,
156  CPL_PLUGIN_TYPE_RECIPE,
157  "omega_mdome",
158  "OMEGA - Create Master Dome Flat for each chip (Calfile 542).",
159  "This recipe is used to derive a valid domeflat frame (Calfile 542) \n"
160  "for one particular chip and filter. The recipe always takes as input \n"
161  "a list of raw domeflat frames, a master bias and a measurement of the \n"
162  "gain of the chip under consideration. In addition, a hot pixel map \n"
163  "(Calfile 522) and a cold pixel map (Calfile 535) can be provided. Optionally, \n"
164  "an overscan correction mode can be set. The default is to apply no overscan \n"
165  "correction.\n\n"
166  "The raw dome flats are trimmed and overscan corrected. The data are then \n"
167  "normalized, averaged, and the result is normalized again. Image statisics \n"
168  "are computed for the resulting frame.",
169  "Sandra Castro",
170  "scastro@eso.org",
172  omega_mdome_create,
173  omega_mdome_exec,
174  omega_mdome_destroy) ;
175 
176  cpl_pluginlist_append(list, plugin) ;
177 
178  return 0;
179 }
180 
181 /*----------------------------------------------------------------------------*/
190 /*----------------------------------------------------------------------------*/
191 static int omega_mdome_create(cpl_plugin * plugin)
192 {
193  cpl_recipe * recipe;
194  cpl_parameter * p ;
195  char *path = NULL;
196 
197  /* Do not create the recipe if an error code is already set */
198  if (cpl_error_get_code() != CPL_ERROR_NONE) {
199  cpl_msg_error(cpl_func, "%s():%d: An error is already set: %s",
200  cpl_func, __LINE__, cpl_error_get_where());
201  return (int)cpl_error_get_code();
202  }
203 
204  if (plugin == NULL) {
205  cpl_msg_error(cpl_func, "Null plugin");
206  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
207  }
208 
209  /* Verify plugin type */
210  if (cpl_plugin_get_type(plugin) != CPL_PLUGIN_TYPE_RECIPE) {
211  cpl_msg_error(cpl_func, "Plugin is not a recipe");
212  cpl_ensure_code(0, (int)CPL_ERROR_TYPE_MISMATCH);
213  }
214 
215  /* Get the recipe */
216  recipe = (cpl_recipe *)plugin;
217 
218  /* Create the parameters list in the cpl_recipe object */
219  recipe->parameters = cpl_parameterlist_new() ;
220 
221  if (recipe->parameters == NULL) {
222  cpl_msg_error(cpl_func, "Parameter list allocation failed");
223  cpl_ensure_code(0, (int)CPL_ERROR_ILLEGAL_OUTPUT);
224  }
225 
226  /* Fill the parameters list */
227 
228  /* General Parameters */
229  p = cpl_parameter_new_value("omega.omega_mdome.ExtensionNumber",
230  CPL_TYPE_INT,
231  "FITS extension number to load (1 to 32). (-1 = all)",
232  "omega_mdome",
233  -1) ;
234 
235  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI,"ext") ;
236  cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
237  cpl_parameterlist_append(recipe->parameters, p) ;
238 
239 
240  p = cpl_parameter_new_range("omega.omega_mdome.OverscanMethod",
241  CPL_TYPE_INT,
242  "Overscan Correction Method",
243  "omega_mdome",
244  0, 0, 6);
245 
246  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI,"oc-meth") ;
247  cpl_parameterlist_append(recipe->parameters, p) ;
248 
249  p = cpl_parameter_new_value("omega.omega_mdome.PAF",
250  CPL_TYPE_BOOL,
251  "Boolean value to create PAF files. 1(Yes), 0(No)",
252  "omega_mdome",
253  1) ;
254 
255  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI, "paf") ;
256  cpl_parameterlist_append(recipe->parameters, p) ;
257 
258  p = cpl_parameter_new_value("omega.omega_mdome.SigmaClip",
259  CPL_TYPE_DOUBLE,
260  "Sigma Clipping Threshold",
261  "omega_mdome",
262  3.0) ;
263 
264  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI, "sig-clip") ;
265  cpl_parameterlist_append(recipe->parameters, p) ;
266 
267 
268 /*------------ Parameters for creating a Cold Pixels Map---------------- */
269  p = cpl_parameter_new_range("omega.omega_mdome.LowThre",
270  CPL_TYPE_DOUBLE,
271  "Low flagging threshold for cold pixels map",
272  "omega_coldpixels",
273  0.93, 0.90, 1.00) ;
274 
275  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI, "low") ;
276  cpl_parameterlist_append(recipe->parameters, p) ;
277 
278  p = cpl_parameter_new_range("omega.omega_mdome.HighThre",
279  CPL_TYPE_DOUBLE,
280  "High flagging threshold for cold pixels map",
281  "omega_coldpixels",
282  1.07, 1.00, 1.10) ;
283 
284  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI,"high") ;
285  cpl_parameterlist_append(recipe->parameters, p) ;
286 
287  p = cpl_parameter_new_range("omega.omega_mdome.BackSize",
288  CPL_TYPE_INT,
289  "Sextractor background mesh size for cold pixels map",
290  "omega_coldpixels",
291  32, 4, 256) ;
292 
293  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI,"backsize") ;
294  cpl_parameterlist_append(recipe->parameters, p) ;
295 
296 /* ------------------ PATH to external executable programs --------------*/
297  path = cpl_sprintf("%s", OMEGA_BIN_PATH);
298  p = cpl_parameter_new_value("omega.omega_mdome.BinPath",
299  CPL_TYPE_STRING,
300  "Path to any external executable program.",
301  "omega.BinPath",
302  path);
303 
304  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI, "bin-path");
305  cpl_parameterlist_append(recipe->parameters, p);
306  cpl_free(path);
307 
308 /* ------------------ Sextractor parameters ------------------------- */
309  path = cpl_sprintf("%s/omega.sex", OMEGA_CONFIG_PATH);
310  p = cpl_parameter_new_value("omega.omega_mdome.SexConfig",
311  CPL_TYPE_STRING,
312  "Path to Sextractor config file.",
313  "omega.Sextractor",
314  path);
315 
316  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI, "sex-config");
317  cpl_parameterlist_append(recipe->parameters, p);
318  cpl_free(path);
319 
320  path = cpl_sprintf("%s/omega.conv", OMEGA_CONFIG_PATH);
321  p = cpl_parameter_new_value("omega.omega_mdome.SexConv",
322  CPL_TYPE_STRING,
323  "Path to Sextractor convolution mask file.",
324  "omega.Sextractor",
325  path);
326 
327  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI,"sex-conv");
328  cpl_parameterlist_append(recipe->parameters, p);
329  cpl_free(path);
330 
331 
332  path = cpl_sprintf("%s/omega.param", OMEGA_CONFIG_PATH);
333  p = cpl_parameter_new_value("omega.omega_mdome.SexParam",
334  CPL_TYPE_STRING,
335  "Path to Sextractor parameters file.",
336  "omega.Sextractor",
337  path);
338 
339  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI, "sex-param");
340  cpl_parameterlist_append(recipe->parameters, p);
341  cpl_free(path);
342 
343  path = cpl_sprintf("%s/omega.nnw", OMEGA_CONFIG_PATH);
344  p = cpl_parameter_new_value("omega.omega_mdome.SexNnw",
345  CPL_TYPE_STRING,
346  "Path to Sextractor neural network config file.",
347  "omega.Sextractor",
348  path);
349 
350  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI,"sex-nnw");
351  cpl_parameterlist_append(recipe->parameters, p);
352  cpl_free(path);
353 
354  p = cpl_parameter_new_value("omega.omega_mdome.BackThreshold",
355  CPL_TYPE_DOUBLE,
356  "Detection threshold for background in image (Sextractor DETECT_THRESH)",
357  "omega_Sextractor",
358  1000.) ;
359 
360  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI, "backthre") ;
361  cpl_parameterlist_append(recipe->parameters, p) ;
362 
363 
364 
365  return 0;
366 }
367 
368 /*----------------------------------------------------------------------------*/
374 /*----------------------------------------------------------------------------*/
375 static int omega_mdome_exec(cpl_plugin * plugin)
376 {
377  cpl_recipe * recipe;
378  int recipe_status;
379 /* cpl_errorstate initial_errorstate = cpl_errorstate_get();*/
380 
381  /* Return immediately if an error code is already set */
382  if (cpl_error_get_code() != CPL_ERROR_NONE) {
383  cpl_msg_error(cpl_func, "%s():%d: An error is already set: %s",
384  cpl_func, __LINE__, cpl_error_get_where());
385  return (int)cpl_error_get_code();
386  }
387 
388  if (plugin == NULL) {
389  cpl_msg_error(cpl_func, "Null plugin");
390  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
391  }
392 
393  /* Verify plugin type */
394  if (cpl_plugin_get_type(plugin) != CPL_PLUGIN_TYPE_RECIPE) {
395  cpl_msg_error(cpl_func, "Plugin is not a recipe");
396  cpl_ensure_code(0, (int)CPL_ERROR_TYPE_MISMATCH);
397  }
398 
399  /* Get the recipe */
400  recipe = (cpl_recipe *)plugin;
401 
402  /* Verify parameter and frame lists */
403  if (recipe->parameters == NULL) {
404  cpl_msg_error(cpl_func, "Recipe invoked with NULL parameter list");
405  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
406  }
407  if (recipe->frames == NULL) {
408  cpl_msg_error(cpl_func, "Recipe invoked with NULL frame set");
409  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
410  }
411 
412  /* Invoke the recipe */
413  recipe_status = omega_mdome(recipe->frames, recipe->parameters);
414 
415  /* Ensure DFS-compliance of the products */
416  if (cpl_dfs_update_product_header(recipe->frames)) {
417  if (!recipe_status) recipe_status = (int)cpl_error_get_code();
418  }
419 
420  /* Dump the error history since recipe execution start.
421  At this point the recipe cannot recover from the error */
422  /* if (!cpl_errorstate_is_equal(initial_errorstate)) {
423  cpl_errorstate_dump(initial_errorstate, CPL_FALSE, NULL);
424  }*/
425 
426  return recipe_status;
427 
428 }
429 
430 /*----------------------------------------------------------------------------*/
436 /*----------------------------------------------------------------------------*/
437 static int omega_mdome_destroy(cpl_plugin * plugin)
438 {
439  cpl_recipe *recipe;
440 
441  if (plugin == NULL) {
442  cpl_msg_error(cpl_func, "Null plugin");
443  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
444  }
445 
446  /* Verify plugin type */
447  if (cpl_plugin_get_type(plugin) != CPL_PLUGIN_TYPE_RECIPE) {
448  cpl_msg_error(cpl_func, "Plugin is not a recipe");
449  cpl_ensure_code(0, (int)CPL_ERROR_TYPE_MISMATCH);
450  }
451 
452  /* Get the recipe */
453  recipe = (cpl_recipe *)plugin;
454 
455  cpl_parameterlist_delete(recipe->parameters);
456 
457  return 0 ;
458 }
459 
460 /*----------------------------------------------------------------------------*/
467 /*----------------------------------------------------------------------------*/
468 static int omega_mdome(cpl_frameset *set, cpl_parameterlist *pars)
469 {
470 
471  int j,jst,jfn;
472  cpl_size nlab;
473  int nflats = 0;
474  int oscan1 = 0;
475  int status = 1;
476 
477  float bias = BIAS;
478 
479  char *outcpm = NULL;
480  char *outmdome = NULL;
481  const char *_id = "omega_mdome";
482  cpl_frame *firstframe = NULL;
483  cpl_frame *prframe_cpm = NULL;
484  cpl_frame *prframe_mdome = NULL;
485  const cpl_frame *refframe = NULL;
486  cpl_parameter *par = NULL;
487  cpl_propertylist *qclist = NULL;
488  cpl_propertylist *alist = NULL;
489  cpl_stats *diffstats = NULL;
490 
491 
492  /*Start the recipe */
493 
494  if (!pars) {
495  cpl_msg_error (_id, "Parameters list not found");
496  return -1;
497  }
498 
499  if (cpl_frameset_is_empty(set) == 1) {
500  cpl_msg_error (_id, "Frameset not found");
501  return -1;
502  }
503 
504 /* Retrieve input parameters */
505  par = cpl_parameterlist_find(pars, "omega.omega_mdome.ExtensionNumber") ;
506  omega_mdome_config.extnum = cpl_parameter_get_int(par) ;
507 
508  par = cpl_parameterlist_find(pars, "omega.omega_mdome.OverscanMethod") ;
509  omega_mdome_config.oc = cpl_parameter_get_int(par) ;
510 
511  par = cpl_parameterlist_find(pars, "omega.omega_mdome.SigmaClip") ;
512  omega_mdome_config.sigma = cpl_parameter_get_double(par) ;
513 
514  par = cpl_parameterlist_find(pars, "omega.omega_mdome.LowThre") ;
515  omega_mdome_config.lthre = cpl_parameter_get_double(par) ;
516 
517  par = cpl_parameterlist_find(pars, "omega.omega_mdome.HighThre") ;
518  omega_mdome_config.hthre = cpl_parameter_get_double(par) ;
519 
520  par = cpl_parameterlist_find(pars, "omega.omega_mdome.PAF") ;
521  omega_mdome_config.paf = cpl_parameter_get_bool(par) ;
522 
523 /* Identify the RAW and CALIB frames in the input frameset */
524  if (oc_dfs_set_groups(set)) {
525  cpl_msg_error(_id, "Cannot identify RAW and CALIB frames") ;
526  return -1 ;
527  }
528 
529  /*Initialized things*/
530  omega_mdome_init();
531 
532 /* Verify the frameset contents. */
533  if ((ps.labels = cpl_frameset_labelise(set,omega_compare_tags,
534  &nlab)) == NULL) {
535  cpl_msg_error(_id,"Cannot labelise the input frameset");
536  omega_mdome_tidy();
537  return -1;
538  }
539  if ((ps.domelist = omega_frameset_subgroup(set,ps.labels,nlab,
540  MDOME_RAW)) == NULL) {
541  cpl_msg_error(_id,"Cannot find dome frames in input frameset");
542  omega_mdome_tidy();
543  return -1;
544  }
545 
546 
547  nflats = cpl_frameset_count_tags(ps.domelist, MDOME_RAW);
548  if (nflats < 2) {
549  cpl_msg_error (_id, "Need at least 2 (%s) frames to run "
550  "this recipe", MDOME_RAW);
551  omega_mdome_tidy();
552  return -1;
553  }
554 
555  cpl_msg_info (_id,"There are %d %s in frame set",nflats, MDOME_RAW);
556 
557  /* Check for calibration frames */
558  /* Master Bias */
559  ps.mbframe = cpl_frameset_find_const(set, OMEGA_CALIB_BIAS);
560  if (ps.mbframe == NULL) {
561  cpl_msg_info(_id,"A Master Bias is not present in frame set. Using default value %f", bias);
562  }
563  else{
564  cpl_msg_info(_id,"Using %s %s",OMEGA_CALIB_BIAS, cpl_frame_get_filename(ps.mbframe));
565  }
566 
567  /* Cold Pixels Map */
568  ps.cframe = cpl_frameset_find_const(set, OMEGA_CALIB_CPM);
569  if(ps.cframe != NULL){
570  cpl_msg_info(_id,"Using %s %s",OMEGA_CALIB_CPM, cpl_frame_get_filename(ps.cframe));
571  }
572 
573  /* Hot Pixels Map */
574  ps.hframe = cpl_frameset_find_const(set, OMEGA_CALIB_HPM);
575  if(ps.hframe != NULL){
576  cpl_msg_info(_id,"Using %s %s",OMEGA_CALIB_HPM, cpl_frame_get_filename(ps.hframe));
577  }
578 
579  /* Optional Reference master dome frame */
580  refframe = cpl_frameset_find_const(set, REFDOME);
581  if (refframe != NULL)
582  cpl_msg_info(cpl_func,"Using %s for comparison",cpl_frame_get_filename(refframe));
583 
584  /* Get first frame from frame set */
585  firstframe = cpl_frameset_get_position(ps.domelist, 0) ;
586 
587  /* Loop for each of the image extensions */
588  omega_exten_range(omega_mdome_config.extnum,&jst,&jfn);
589  if(omega_mdome_config.extnum == 0){
590  cpl_msg_error(cpl_func,"Unsupported extension request, %d",omega_mdome_config.extnum);
591  omega_mdome_tidy();
592  return -1;
593  }
594 
595  /* Check which instrument */
596  if (omega_pfits_check_instrume(firstframe) == 1)
597  sprintf(INSTRUME,"wfi");
598 
599  /* Check which extensions */
600  if(omega_pfits_check_instrume(firstframe) == 1 &&
601  omega_mdome_config.extnum == 0 && jfn == 32)
602  jfn = 8;
603 
604  for (j = jst; j <= jfn; j++) {
605 
606  isfirst = (j == jst);
607  cpl_msg_info(_id,"Beginning work on extension %d",j);
608  omega_mdome_config.CountColdPixels = 0;
609  omega_mdome_config.Mean = 0.0;
610  omega_mdome_config.Median = 0.0;
611  omega_mdome_config.Stdev = 0.0;
612  omega_mdome_config.RawMin = 0.0;
613  omega_mdome_config.RawMax = 0.0;
614  omega_mdome_config.RawMean = 0.0;
615  omega_mdome_config.RawMedian = 0.0;
616  omega_mdome_config.RawStdev = 0.0;
617 
618  /* Check overscan correction method consistency */
619  if(ps.mbframe != NULL){
620  oscan1 = omega_pfits_get_overscan(ps.mbframe, j);
621  if(oscan1 != omega_mdome_config.oc) {
622  cpl_msg_warning (_id, "Overscan correction mode for Master Bias (oc = %d) differs from "
623  "the one used here (oc = %d)", oscan1, omega_mdome_config.oc);
624  }
625  }
626 
627 
628  /* Load first dome flat */
629  ps.firstdome = omega_fits_load(firstframe,CPL_TYPE_FLOAT,j);
630  ps.eh = omega_fits_get_ehu(ps.firstdome);
631 
632  /* Call the combining routine */
633  status = omega_mdome_combine(pars, j);
634  if(status == 1){
635  cpl_msg_warning(_id, "Image detector is not live");
636  /* Save dummy product */
637  freefits(ps.firstdome);
638  ps.eh = NULL;
639  continue;
640  }
641  else if(status == -1){
642  cpl_msg_error(_id,"Cannot combine images");
643  freespace(outcpm);
644  freespace(outmdome);
645  omega_mdome_tidy();
646  return -1;
647  }
648 
649  qclist = cpl_propertylist_new();
650 
651  /* Compare with reference frame */
652  if(refframe != NULL){
653  if(omega_compare_reference(ps.mdome,refframe,j,&diffstats) == -1){
654  cpl_msg_warning(cpl_func,"Cannot compare with reference frame");
655  }
656  else{
657  cpl_propertylist_append_double(qclist,"ESO QC DIFF REFDOME MEAN",
658  cpl_stats_get_mean(diffstats));
659  cpl_propertylist_set_comment(qclist,"ESO QC DIFF REFDOME MEAN",
660  "Mean of difference with reference");
661  cpl_propertylist_append_double(qclist,"ESO QC DIFF REFDOME MEDIAN",
662  cpl_stats_get_median(diffstats));
663  cpl_propertylist_set_comment(qclist,"ESO QC DIFF REFDOME MEDIAN",
664  "Median of difference with reference");
665  cpl_propertylist_append_double(qclist,"ESO QC DIFF REFDOME STDEV",
666  cpl_stats_get_stdev(diffstats));
667  cpl_propertylist_set_comment(qclist,"ESO QC DIFF REFDOME STDEV",
668  "Stdev of difference with reference");
669 
670  freestats(diffstats);
671  }
672  }
673 
674  /* Save the products */
675  /* Save the MASTER DOME product */
676 
677  cpl_propertylist_append_double(qclist, "ESO QC MASTER DOME MEAN",
678  omega_mdome_config.Mean) ;
679  cpl_propertylist_set_comment (qclist, "ESO QC MASTER DOME MEAN",
680  "Mean of master dome flat");
681 
682  cpl_propertylist_append_double(qclist, "ESO QC MASTER DOME MEDIAN",
683  omega_mdome_config.Median) ;
684  cpl_propertylist_set_comment (qclist, "ESO QC MASTER DOME MEDIAN",
685  "Median of master dome flat");
686 
687  cpl_propertylist_append_double(qclist, "ESO QC MASTER DOME STDEV",
688  omega_mdome_config.Stdev) ;
689  cpl_propertylist_set_comment (qclist, "ESO QC MASTER DOME STDEV",
690  "Std Deviation of master dome flat");
691 
692  cpl_propertylist_append_double(qclist, "ESO QC RAW DOME MIN",
693  omega_mdome_config.RawMin);
694  cpl_propertylist_append_double(qclist, "ESO QC RAW DOME MAX",
695  omega_mdome_config.RawMax);
696  cpl_propertylist_append_double(qclist, "ESO QC RAW DOME MEAN",
697  omega_mdome_config.RawMean);
698  cpl_propertylist_append_double(qclist, "ESO QC RAW DOME MEDIAN",
699  omega_mdome_config.RawMedian);
700  cpl_propertylist_append_double(qclist, "ESO QC RAW DOME STDEV",
701  omega_mdome_config.RawStdev);
702 
703  cpl_propertylist_set_comment(qclist, "ESO QC RAW DOME MIN",
704  "median value of the raw dome flat having the lowest flux");
705  cpl_propertylist_set_comment(qclist, "ESO QC RAW DOME MAX",
706  "median value of the raw dome flat having the highest flux");
707  cpl_propertylist_set_comment(qclist, "ESO QC RAW DOME MEAN",
708  "mean value of all input raw dome flats (ADU)");
709  cpl_propertylist_set_comment(qclist, "ESO QC RAW DOME MEDIAN",
710  "median value of all input raw dome flats (ADU)");
711  cpl_propertylist_set_comment(qclist, "ESO QC RAW DOME STDEV",
712  "standard deviation of all input raw dome flats (ADU)");
713 
714  if(isfirst){
715  outmdome = cpl_sprintf("%s_%s.fits", INSTRUME,MDOME_PROCATG);
716  prframe_mdome = omega_product_frame(outmdome, MDOME_PROCATG, CPL_FRAME_TYPE_IMAGE);
717  }
718 
719  alist = cpl_propertylist_new();
720  cpl_propertylist_append_string(alist, "EXTNAME",
721  cpl_propertylist_get_string(ps.eh, "EXTNAME"));
722  cpl_propertylist_set_comment(alist,"EXTNAME", "Extension name");
723 
724  /* Add DRS keywords */
725  cpl_propertylist_update_int(alist, "ESO DRS OVERSCAN METHOD", omega_mdome_config.oc);
726  cpl_propertylist_set_comment(alist, "ESO DRS OVERSCAN METHOD", "overscan correction method");
727 
728  cpl_propertylist_copy_property_regexp(alist, ps.eh, WCS_KEYS, 0);
729 
730  if(omega_save_image(ps.mdome,set,pars,alist,qclist,CPL_BPP_IEEE_FLOAT,outmdome,
731  RECIPE,prframe_mdome,NULL,isfirst) == -1){
732  cpl_msg_error(_id,"Cannot save product %s", MDOME_PROCATG);
733  freeplist(qclist);
734  freeplist(alist);
735  freespace(outcpm);
736  freespace(outmdome);
737  omega_mdome_tidy();
738  return -1;;
739  }
740  freeplist(qclist);
741  freeimage(ps.mdome);
742 
743  /* Save the COLD PIXELS MAP product */
744  qclist = cpl_propertylist_new();
745  cpl_propertylist_append_int(qclist,"ESO QC NUMBER COLD PIXELS",
746  omega_mdome_config.CountColdPixels);
747 
748  cpl_propertylist_set_comment(qclist,"ESO QC NUMBER COLD PIXELS",
749  "Number of cold pixels");
750  if(isfirst){
751  outcpm = cpl_sprintf("%s_%s.fits", INSTRUME,CPM_PROCATG);
752  prframe_cpm = omega_product_frame(outcpm, CPM_PROCATG, CPL_FRAME_TYPE_IMAGE);
753  }
754 
755  if(omega_save_image(ps.cpixels,set,pars,alist,qclist,CPL_BPP_16_SIGNED,outcpm,
756  RECIPE,prframe_cpm,NULL,isfirst) == -1){
757  cpl_msg_error(_id,"Cannot save product %s", CPM_PROCATG);
758  freeplist(qclist);
759  freeplist(alist);
760  freespace(outcpm);
761  freespace(outmdome);
762  omega_mdome_tidy();
763  return -1;;
764  }
765 
766  freeplist(qclist);
767  freeplist(alist);
768  freeimage(ps.cpixels);
769  freefits(ps.firstdome);
770  ps.firstdome = NULL;
771  ps.eh = NULL;
772 
773  } /* go and work on next extension */
774 
775  /*Clean up */
776 
777  freespace(outcpm);
778  freespace(outmdome);
779  omega_mdome_tidy();
780 
781  return 0;
782 
783 }
784 
785 /*----------------------------------------------------------------------------*/
796 /*----------------------------------------------------------------------------*/
797 int omega_mdome_combine(cpl_parameterlist *pars, int xn)
798 {
799 
800  int i,nflats,live, naxis1, naxis2;
801  float bias = BIAS;
802  double gain = 0.0;
803  double median = 1.0;
804  double threshold = 0.0;
805  double *data_scales = NULL;
806  const char *_id = "omega_mdome_combine";
807  const char *mdome_name = "omega_mdome_temp.fits";
808  const char *backname = "omega_mdome_background.fits";
809 
810  cpl_error_code code;
811  const cpl_frame *domefr = NULL;
812  cpl_vector *scales = NULL;
813  cpl_vector *median_vector = NULL;
814  cpl_image *mbias = NULL;
815  cpl_image *trim_raw = NULL, *back_image = NULL, *dev = NULL;
816  cpl_image *good_float = NULL, *good_int = NULL, *median_all = NULL;
817  cpl_image *sum_data = NULL, *sum_good = NULL, *new_image = NULL;
818  cpl_image *norm_image = NULL;
819  cpl_imagelist *ilist = NULL;
820  cpl_mask *good = NULL, *bpm_map = NULL;
821  cpl_mask *pixelmap = NULL;
822 
823  nflats = cpl_frameset_get_size(ps.domelist);
824 
825  /* Check that this detector is live*/
826  omega_pfits_get_detlive(ps.eh,&live);
827  if (! live) {
828  return 1;
829  }
830 
831  /* Load master bias image */
832  if(ps.mbframe != NULL){
833  mbias = cpl_image_load(cpl_frame_get_filename(ps.mbframe), CPL_TYPE_FLOAT,0,xn);
834  if (mbias == NULL) {
835  cpl_msg_warning(_id,"Cannot load image %s", OMEGA_CALIB_BIAS);
836  }
837  }
838 
839  /*Create Mask from Hot and Cold Map */
840  bpm_map = makebpm(ps.hframe, ps.cframe, xn);
841 
842 
843  scales = cpl_vector_new (nflats);
844  median_vector = cpl_vector_new (nflats);
845  cpl_vector_fill(median_vector,0.0);
846 
847  data_scales = cpl_vector_get_data(scales);
848 
849  ilist = cpl_imagelist_new();
850  cpl_msg_info (_id,"Doing trim and overscan correction on images");
851 
852  /* Loop through all images in frame list */
853 
854 
855  for (i=0; i< nflats; i++){
856  domefr = cpl_frameset_get_position(ps.domelist, i) ;
857  if (domefr == NULL)
858  break;
859 
860  trim_raw = TrimOscanCorrect(domefr, omega_mdome_config.oc, xn);
861  if(trim_raw == NULL){
862  freevector(scales);
863  freevector(median_vector);
864  freeilist(ilist);
865  return -1;
866  }
867 
868  if (mbias != NULL)
869  cpl_image_subtract(trim_raw, mbias);
870  else
871  cpl_image_subtract_scalar(trim_raw, bias);
872 
873  if (bpm_map == NULL) {
874  ps.stats = cpl_stats_new_from_image(trim_raw, CPL_STATS_ALL);
875  median = cpl_stats_get_median(ps.stats);
876  }
877  else {
878  cpl_image_reject_from_mask(trim_raw, bpm_map);
879  ps.stats = cpl_stats_new_from_image(trim_raw, CPL_STATS_ALL);
880  median = cpl_stats_get_median(ps.stats);
881  }
882 
883  cpl_vector_set(median_vector, i, median);
884 
885  data_scales[i] = (double)1.0/median;
886 
887  cpl_image_divide_scalar(trim_raw, median);
888 
889  cpl_imagelist_set(ilist, trim_raw, i);
890 
891 
892 
893  freestats(ps.stats);
894 
895  }
896 
897  omega_mdome_config.RawMin = cpl_vector_get_min(median_vector);
898  omega_mdome_config.RawMax = cpl_vector_get_max(median_vector);
899  omega_mdome_config.RawMean = cpl_vector_get_mean(median_vector);
900  omega_mdome_config.RawMedian = cpl_vector_get_median(median_vector);
901  omega_mdome_config.RawStdev = cpl_vector_get_stdev(median_vector);
902 
903  freevector(median_vector);
904 
905  freestats(ps.stats);
906  freeimage(mbias);
907  freemask(bpm_map);
908 
909  if (ilist == NULL) {
910  cpl_msg_error(_id,"Error in image list <%s>",cpl_error_get_message());
911  freevector(scales);
912  freeimage(trim_raw);
913  return -1;
914  }
915 
916 
917  /* Read gain from the header */
918  /* FIXME: or read gain from gain recipe product?? */
919 
920  /* The images have the CONAD as the correct value for
921  * the gain, therefore the following lines should be
922  * changed if the headers are fixed.
923  */
924 
925 // omega_pfits_get_gain(ps.eh, &gain);
926  omega_pfits_get_conad(ps.eh, &gain);
927  /*FIXME: should we avoid having gain=0?*/
928  if (gain < 0.1){
929  omega_pfits_get_gain(ps.eh, &gain);
930 // omega_pfits_get_conad(ps.eh, &gain);
931  if(gain < 0.1) {
932  cpl_msg_info(_id,"Using default value of gain");
933  gain = 2.0;
934  }
935  }
936  else {
937  cpl_msg_info(_id,"Gain value from image is %g", gain);
938  }
939 
940  cpl_msg_info (_id,"Computing the median of all images...");
941  median_all = cpl_imagelist_collapse_median_create(ilist);
942 
943  if (median_all == NULL) {
944  cpl_msg_error (_id,"Cannot take median of list <%s>",cpl_error_get_message());
945  freevector(scales);
946  freeilist(ilist);
947  return -1;
948  }
949 
950  naxis1 = cpl_image_get_size_x(median_all);
951  naxis2 = cpl_image_get_size_y(median_all);
952 
953  sum_data = cpl_image_new(naxis1,naxis2,CPL_TYPE_FLOAT);
954  sum_good = cpl_image_new(naxis1,naxis2,CPL_TYPE_FLOAT);
955 
956  /* Clip the outliers from the images*/
957  for (i=0; i< nflats; i++){
958 
959  trim_raw = cpl_imagelist_get(ilist, i);
960  new_image = cpl_image_duplicate(trim_raw);
961 
962  /* FIXME: setting negative values to 0 to avoid failure of SQRT in next step
963  * when there are negative values in the image. These negative values happen when any
964  * given pixel in the domeflat is smaller than the same pixel in the master_bias image.
965  */
966  cpl_image_threshold(trim_raw, 0, FLT_MAX, 0, 2e20);
967 
968  dev = cpl_image_subtract_create(median_all, trim_raw);
969 
970  code = cpl_image_power(trim_raw, 0.5);
971  if(code != CPL_ERROR_NONE) {
972  cpl_msg_error(_id,"Error in SQRT operation <%s>",cpl_error_get_message());
973  freevector(scales);
974  freeilist(ilist);
975  freeimage(median_all);
976  freeimage(sum_data);
977  freeimage(sum_good);
978  freeimage(new_image);
979  freeimage(dev);
980  return -1;
981  }
982 
983  cpl_image_divide(dev, trim_raw);
984 
985  threshold = omega_mdome_config.sigma * (sqrt(gain * data_scales[i]));
986 
987  good = cpl_mask_threshold_image_create(dev, -threshold, threshold);
988 
989  freeimage(dev);
990  good_int = cpl_image_new_from_mask(good) ;
991  freemask(good) ;
992 
993  good_float = cpl_image_cast(good_int, CPL_TYPE_FLOAT);
994  freeimage(good_int);
995 
996  cpl_image_multiply(new_image, good_float);
997  cpl_image_add(sum_data, new_image);
998 
999  cpl_image_add(sum_good, good_float);
1000 
1001  freeimage(new_image);
1002  freeimage(good_float);
1003 
1004  }
1005 
1006  freeimage(median_all);
1007  freevector(scales);
1008 
1009  /* Create master dome flat */
1010  cpl_image_threshold(sum_good, FLT_EPSILON, FLT_MAX, FLT_EPSILON, FLT_MAX);
1011  ps.mdome = cpl_image_divide_create(sum_data, sum_good);
1012  if (ps.mdome == NULL) {
1013  cpl_msg_error(_id,"Error in division %s <%s>",MDOME_PROCATG, cpl_error_get_message());
1014  freeilist(ilist);
1015  freeimage(sum_data);
1016  freeimage(sum_good);
1017  return -1;
1018  }
1019 
1020  freeimage(sum_data);
1021  freeimage(sum_good);
1022  freeilist(ilist);
1023 
1024  /* Save temporary master dome to disk to use in Sextractor */
1025  cpl_image_save(ps.mdome, mdome_name, BITPIX, NULL, CPL_IO_DEFAULT);
1026 
1027 
1028  /* Create New Cold Pixels Map */
1029  /* Call function to create background image*/
1030  if(omega_create_background(pars, mdome_name, backname) != 0){
1031  cpl_msg_warning(_id,"Error in creating background image");
1032 /* return -1;*/
1033  }
1034 
1035  back_image = cpl_image_load(backname, CPL_TYPE_FLOAT, 0, 0);
1036  if (back_image != NULL){
1037  norm_image = cpl_image_divide_create(ps.mdome, back_image);
1038  freeimage(back_image);
1039  }
1040  else{
1041  cpl_msg_warning(_id,"Could not create normalised image. Cannot load background image.");
1042  norm_image = NULL;
1043  }
1044 
1045  /*Explanation for next steps: the image is normalized to around 1.0, therefore
1046  the bad pixels will be all pixels OUTSIDE the range lthre--hthre. Because of
1047  this, a mask_not needs to be applied to set the bad pixels to CPL_BINARY_1
1048  and count them using cpl_mask_count (which counts CPL_BINARY_1)*/
1049 
1050  /* This cpl function creates a mask of all the pixels within the given interval.
1051  * For example if an image has pixels in the interval 0.0 - 1.0 and the function
1052  * is called like this cpl_mask_threshold_image_create(img,0.5,1.0), all the pixels
1053  * in the range 0.5 to 1.0 will be set to CPL_BINARY_1. The other pixels,
1054  * < 0.5 and > 1.0 will be set to CPL_BINARY_0.
1055  */
1056  pixelmap = cpl_mask_threshold_image_create(norm_image, omega_mdome_config.lthre,
1057  omega_mdome_config.hthre);
1058  cpl_mask_not(pixelmap);
1059 
1060  cpl_msg_info(_id,"Creating pixel map with thresholds: %g,%g",omega_mdome_config.lthre,
1061  omega_mdome_config.hthre);
1062 
1063  omega_mdome_config.CountColdPixels = cpl_mask_count(pixelmap);
1064  if (omega_mdome_config.CountColdPixels == -1) {
1065  cpl_msg_warning(_id,"Pixelmap is NULL. <%s>", cpl_error_get_message());
1066  }
1067 
1068  cpl_msg_info(_id,"Detected %d cold pixels", omega_mdome_config.CountColdPixels);
1069 
1070  freeimage(norm_image);
1071 
1072  /* Convert mask to image to save later */
1073  ps.cpixels = cpl_image_new_from_mask(pixelmap);
1074  freemask(pixelmap);
1075 
1076  /* Compute statistics on master dome flat */
1077  ps.stats = cpl_stats_new_from_image(ps.mdome, CPL_STATS_ALL);
1078  if(ps.stats == NULL && (cpl_error_get_code() == CPL_ERROR_DATA_NOT_FOUND)){
1079  cpl_msg_warning(_id," There are no good pixels in image for doing statistics");
1080  cpl_error_reset();
1081  }
1082  else{
1083  omega_mdome_config.Mean = cpl_stats_get_mean(ps.stats);
1084  omega_mdome_config.Median = cpl_stats_get_median(ps.stats);
1085  omega_mdome_config.Stdev = cpl_stats_get_stdev(ps.stats);
1086  }
1087 
1088  freestats(ps.stats);
1089 
1090 
1091  return 0;
1092 }
1093 
1094 
1095 /* Initialize the pointers */
1096 static void omega_mdome_init(void) {
1097  ps.labels = NULL;
1098  ps.mbframe = NULL;
1099  ps.cframe = NULL;
1100  ps.hframe = NULL;
1101  ps.domelist = NULL;
1102  ps.ph = NULL;
1103  ps.eh = NULL;
1104  ps.stats = NULL;
1105  ps.mdome = NULL;
1106  ps.cpixels = NULL;
1107  ps.firstdome = NULL;
1108 }
1109 
1110 /* Free any allocated memory */
1111 static void omega_mdome_tidy(void) {
1112  freespace(ps.labels);
1113  freeframeset(ps.domelist);
1114  freestats(ps.stats);
1115  freeimage(ps.mdome);
1116  freeimage(ps.cpixels);
1117  freefits(ps.firstdome);
1118 }
int omega_pfits_get_overscan(const cpl_frame *frame, int xn)
Get the DRS keyword of the overscan method used to reduce the image.
Definition: omega_pfits.c:912
int omega_pfits_get_detlive(const cpl_propertylist *plist, int *detlive)
Get the value of DET_LIVE.
Definition: omega_pfits.c:214
void omega_pfits_get_conad(cpl_propertylist *plist, double *conad)
Get the value of the keyword CONAD.
Definition: omega_pfits.c:995
cpl_image * TrimOscanCorrect(const cpl_frame *frame, int oscan, int extn)
This method loads a raw image, trims it and (optionally) performs an overscan correction, using the statistics of the pre and overscan regions. This can be done with a single value for the whole image or on a per-row basis.
Definition: omega_trim.c:87
int omega_compare_tags(const cpl_frame *frame1, const cpl_frame *frame2)
Definition: omega_utils.c:206
cpl_mask * makebpm(const cpl_frame *hot, const cpl_frame *cold, int cxn)
Create a bad pixels map.
Definition: omega_bpm.c:191
int omega_pfits_check_instrume(const cpl_frame *fr)
Check if INSTRUME is WFI or OMEGA.
Definition: omega_pfits.c:785
omega_fits * omega_fits_load(const cpl_frame *inframe, cpl_type type, int extnum)
Definition: omega_fits.c:84
const char * omega_get_license(void)
Get the pipeline copyright and license.
Definition: omega_utils.c:67
void omega_exten_range(int inexten, int *out1, int *out2)
Definition: omega_utils.c:305
cpl_frame * omega_product_frame(const char *filename, const char *tag, cpl_frame_type type)
Setup a frame to save a product.
Definition: omega_utils.c:499
int omega_compare_reference(const cpl_image *master, const cpl_frame *refframe, int ext, cpl_stats **diffstats)
Definition: omega_utils.c:404
cpl_propertylist * omega_fits_get_ehu(omega_fits *p)
Definition: omega_fits.c:489
cpl_frameset * omega_frameset_subgroup(cpl_frameset *frameset, cpl_size *labels, cpl_size nlab, const char *tag)
Definition: omega_utils.c:257
int omega_mdome_combine(cpl_parameterlist *pars, int xn)
Trim and combine input images.
Definition: omega_mdome.c:797
int omega_create_background(cpl_parameterlist *parlist, const char *in_name, const char *out_name)
Creates a background image using Sextractor.
int omega_save_image(const cpl_image *img, cpl_frameset *set, const cpl_parameterlist *pars, cpl_propertylist *alist, const cpl_propertylist *qclist, cpl_type_bpp bpp, const char *name, const char *recipe, cpl_frame *frame, const cpl_frame *inherit, int isfirst)
Save an image as a DFS compliant product.
Definition: omega_utils.c:916
void omega_pfits_get_gain(cpl_propertylist *plist, double *gain)
Read the gain value from an image.
Definition: omega_pfits.c:952
int cpl_plugin_get_info(cpl_pluginlist *list)
Build the list of available plugins, for this module.
Definition: omega_mdome.c:148