SINFONI Pipeline Reference Manual  2.5.2
irplib_mkmaster.c
1 /* $Id: irplib_mkmaster.c,v 1.6 2013-02-27 16:00:51 jtaylor Exp $
2  *
3  * This file is part of the irplib package
4  * Copyright (C) 2002,2003 European Southern Observatory
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1307 USA
19  */
20 
21 /*
22  * $Author: jtaylor $
23  * $Date: 2013-02-27 16:00:51 $
24  * $Revision: 1.6 $
25  * $Name: not supported by cvs2svn $
26  */
27 
28 #ifdef HAVE_CONFIG_H
29 #include <config.h>
30 #endif
31 
32 /*-----------------------------------------------------------------------------
33  Includes
34  -----------------------------------------------------------------------------*/
35 
36 #include <math.h>
37 #include <string.h>
38 #include "irplib_mkmaster.h"
39 
40 /*----------------------------------------------------------------------------*/
44 /*----------------------------------------------------------------------------*/
45 
47 /*---------------------------------------------------------------------------*/
54 /*---------------------------------------------------------------------------*/
55 
56 /*-------------------------------------------------------------------------*/
68 /*--------------------------------------------------------------------------*/
69 static cpl_vector *
70 irplib_imagelist_get_clean_mean_levels(const cpl_imagelist* iml,
71  const double kappa,
72  const int nclip,
73  const double tolerance)
74 {
75 
76  const cpl_image* img=NULL;
77  int size=0;
78  int i=0;
79  cpl_vector* levels=NULL;
80  double* pval=NULL;
81  double mean=0;
82  double stdev=0;
83 
84 
85  cpl_error_ensure(iml != NULL, CPL_ERROR_NULL_INPUT, return(levels),
86  "Null input image list");
87  cpl_error_ensure(kappa >= 0, CPL_ERROR_ILLEGAL_INPUT, return(levels),
88  "Must be kappa>0");
89 
90  size=cpl_imagelist_get_size(iml);
91  levels=cpl_vector_new(size);
92  pval=cpl_vector_get_data(levels);
93 
94  for(i=0;i<size;i++) {
95  img=cpl_imagelist_get_const(iml,i);
96  irplib_ksigma_clip(img,1,1,
97  cpl_image_get_size_x(img),
98  cpl_image_get_size_y(img),
99  nclip,kappa,tolerance,&mean,&stdev);
100  cpl_msg_info(cpl_func,"Ima %d mean level: %g",i+1,mean);
101  pval[i]=mean;
102  }
103 
104 
105  return levels;
106 }
107 
108 /*-------------------------------------------------------------------------*/
116 /*--------------------------------------------------------------------------*/
117 static cpl_error_code
118 irplib_imagelist_subtract_values(cpl_imagelist** iml, cpl_vector* values)
119 {
120 
121  cpl_image* img=NULL;
122  int size=0;
123  int i=0;
124  double* pval=NULL;
125 
126  size=cpl_imagelist_get_size(*iml);
127  pval=cpl_vector_get_data(values);
128 
129  for(i=0;i<size;i++) {
130  img=cpl_imagelist_get(*iml,i);
131  cpl_image_subtract_scalar(img,pval[i]);
132  cpl_imagelist_set(*iml,img,i);
133  }
134 
135  return cpl_error_get_code();
136 }
137 
138 /*---------------------------------------------------------------------------*/
151 /*---------------------------------------------------------------------------*/
152 static double
153 irplib_vector_ksigma(cpl_vector *values,
154  const double klow, const double khigh, int kiter)
155 {
156  cpl_vector *accepted;
157  double mean = 0.0;
158  double sigma = 0.0;
159  double *data = cpl_vector_get_data(values);
160  int n = cpl_vector_get_size(values);
161  int ngood = n;
162  int count = 0;
163  int i;
164 
165  /*
166  * At first iteration the mean is taken as the median, and the
167  * standard deviation relative to this value is computed.
168  */
169 
170  mean = cpl_vector_get_median(values);
171 
172  for (i = 0; i < n; i++) {
173  sigma += (mean - data[i]) * (mean - data[i]);
174  }
175  sigma = sqrt(sigma / (n - 1));
176 
177  while (kiter) {
178  count = 0;
179  for (i = 0; i < ngood; i++) {
180  if (data[i]-mean < khigh*sigma && mean-data[i] < klow*sigma) {
181  data[count] = data[i];
182  ++count;
183  }
184  }
185 
186  if (count == 0) // This cannot happen at first iteration.
187  break; // So we can break: we have already computed a mean.
188 
189  /*
190  * The mean must be computed even if no element was rejected
191  * (count == ngood), because at first iteration median instead
192  * of mean was computed.
193  */
194 
195  accepted = cpl_vector_wrap(count, data);
196  mean = cpl_vector_get_mean(accepted);
197  if(count>1) {
198  sigma = cpl_vector_get_stdev(accepted);
199  }
200  cpl_vector_unwrap(accepted);
201 
202  if (count == ngood) {
203  break;
204  }
205  ngood = count;
206  --kiter;
207  }
208 
209  return mean;
210 }
211 
212 
231 static cpl_image *
232 irplib_imagelist_ksigma_stack(const cpl_imagelist *imlist,
233  double klow, double khigh, int kiter)
234 {
235  int ni, nx, ny, npix;
236  cpl_image *out_ima=NULL;
237  cpl_imagelist *loc_iml=NULL;
238  double *pout_ima=NULL;
239  cpl_image *image=NULL;
240  const double **data=NULL;
241  double *med=NULL;
242  cpl_vector *time_line=NULL;
243 
244  double *ptime_line=NULL;
245  int i, j;
246  double mean_of_medians=0;
247 
248  cpl_error_ensure(imlist != NULL, CPL_ERROR_NULL_INPUT, return(out_ima),
249  "Null input image list");
250 
251  ni = cpl_imagelist_get_size(imlist);
252  loc_iml = cpl_imagelist_duplicate(imlist);
253  image = cpl_imagelist_get(loc_iml, 0);
254  nx = cpl_image_get_size_x(image);
255  ny = cpl_image_get_size_y(image);
256  npix = nx * ny;
257 
258  out_ima = cpl_image_new(nx, ny, CPL_TYPE_DOUBLE);
259  pout_ima = cpl_image_get_data_double(out_ima);
260 
261  time_line = cpl_vector_new(ni);
262 
263  ptime_line = cpl_vector_get_data(time_line);
264 
265  data = cpl_calloc(sizeof(double *), ni);
266  med = cpl_calloc(sizeof(double), ni);
267 
268  for (i = 0; i < ni; i++) {
269  image = cpl_imagelist_get(loc_iml, i);
270  med[i]=cpl_image_get_median(image);
271  cpl_image_subtract_scalar(image,med[i]);
272  data[i] = cpl_image_get_data_double(image);
273  mean_of_medians+=med[i];
274  }
275  mean_of_medians/=ni;
276 
277  for (i = 0; i < npix; i++) {
278  for (j = 0; j < ni; j++) {
279  ptime_line[j] = data[j][i];
280  }
281  pout_ima[i] = irplib_vector_ksigma(time_line, klow, khigh, kiter);
282  }
283 
284  cpl_image_add_scalar(out_ima,mean_of_medians);
285 
286 
287  cpl_free(data);
288  cpl_free(med);
289  cpl_vector_delete(time_line);
290  cpl_imagelist_delete(loc_iml);
291 
292  return out_ima;
293 
294 }
295 
296 
297 
298 
299 /*-------------------------------------------------------------------------*/
311 /*--------------------------------------------------------------------------*/
312 cpl_image*
313 irplib_mkmaster_mean(cpl_imagelist* images,const double kappa, const int nclip, const double tolerance,const double klow,const double khigh,const int niter)
314 {
315 
316  cpl_image* master=NULL;
317  cpl_vector* levels=NULL;
318  double mean=0;
319  cpl_imagelist* iml=NULL;
320 
321  cpl_msg_info(cpl_func,"method mean");
322  iml=cpl_imagelist_duplicate(images);
323  levels=irplib_imagelist_get_clean_mean_levels(iml,kappa,nclip,tolerance);
324  mean=cpl_vector_get_mean(levels);
325  cpl_msg_info(cpl_func,"Master mean level: %g",mean);
326 
327  irplib_imagelist_subtract_values(&iml,levels);
328 
329  master = irplib_imagelist_ksigma_stack(iml,klow,khigh,niter);
330  cpl_image_add_scalar(master,mean);
331 
332  cpl_vector_delete(levels);
333  cpl_imagelist_delete(iml);
334  return master;
335 
336 }
337 
338 /*-------------------------------------------------------------------------*/
350 /*--------------------------------------------------------------------------*/
351 cpl_image*
352 irplib_mkmaster_median(cpl_imagelist* images,const double kappa, const int nclip, const double tolerance)
353 {
354 
355  cpl_image* master=NULL;
356  cpl_vector* levels=NULL;
357  double mean=0;
358  cpl_imagelist* iml=NULL;
359 
360  cpl_msg_info(cpl_func,"method median");
361  iml=cpl_imagelist_duplicate(images);
362  levels=irplib_imagelist_get_clean_mean_levels(iml,kappa,nclip,tolerance);
363 
364  mean=cpl_vector_get_mean(levels);
365  cpl_msg_info(cpl_func,"Master mean level: %g",mean);
366  irplib_imagelist_subtract_values(&iml,levels);
367 
368  master = cpl_imagelist_collapse_median_create(iml);
369 
370  cpl_image_add_scalar(master,mean);
371 
372  cpl_vector_delete(levels);
373  cpl_imagelist_delete(iml);
374 
375  return master;
376 
377 }
378 
379 /* Work in progress */
380 static cpl_error_code
381 irplib_mkmaster_dark_qc(const cpl_imagelist* raw_images,
382  cpl_imagelist* preproc_images,
383  const cpl_parameterlist* parameters,
384  const int pr_num_x, const int pr_num_y,
385  const int pr_box_sx, const int pr_box_sy, const char* recipe_id,
386  cpl_table* qclog) {
387 
388  cpl_ensure_code(qclog !=NULL, CPL_ERROR_NULL_INPUT);
389  cpl_ensure_code(recipe_id !=NULL, CPL_ERROR_NULL_INPUT);
390  cpl_ensure_code(parameters !=NULL, CPL_ERROR_NULL_INPUT);
391 
392  int i = 0;
393  cpl_image* current_dark = 0;
394  if (pr_num_x != 0 && pr_num_y != 0 && pr_box_sx != 0 && pr_box_sy != 0) {
395  for (i = 0; i < cpl_imagelist_get_size(raw_images); i++) {
396  current_dark = cpl_image_duplicate(
397  cpl_imagelist_get_const(preproc_images, i));
398  cpl_msg_info(cpl_func, "Calculating QC parameters on raw dark frame %d",
399  i);
400  /* Here To be defined more general way to qc-log */
401  /* UVES specific stuff: may be this function should not be put in irplib
402  irplib_mdark_region_qc(current_dark, parameters, raw_images, recipe_id,qclog);
403  */
404  /* FIXME: still safe if irplib_mdark_region_qc is commented in? */
405  cpl_image_delete(current_dark);
406  }
407  }
408  return cpl_error_get_code();
409 }
410 
411 /*-------------------------------------------------------------------------*/
421 /*-------------------------------------------------------------------------*/
422 static double
423 irplib_head_get_exptime(const cpl_propertylist * plist) {
424  double result = 0; /* Conversion from electrons to ADUs */
425 
426  result=cpl_propertylist_get_double(plist, "EXPTIME");
427  cpl_ensure_code(result >= 0, CPL_ERROR_ILLEGAL_OUTPUT);
428 
429  return result;
430 }
431 
432 /*-------------------------------------------------------------------------*/
440 /*-------------------------------------------------------------------------*/
441 static cpl_error_code
442 irplib_head_set_exptime(cpl_propertylist *plist, double exptime)
443 {
444  cpl_propertylist_update_double(plist, "EXPTIME", exptime);
445  cpl_propertylist_set_comment(plist, "EXPTIME", "Total integration time");
446 
447  return cpl_error_get_code();
448 }
449 
450 static cpl_imagelist*
451 irplib_mkmaster_dark_fill_imagelist(const cpl_imagelist* raw_images,
452  cpl_propertylist** raw_headers, const cpl_image* master_bias,
453  double* mean_exptime) {
454  /* First process each input image and store the results in a
455  new image list */
456 
457  cpl_imagelist* preproc_images = NULL;
458  int i = 0;
459  cpl_image* current_dark = NULL;
460  double min_exptime = 0;
461  double max_exptime = 0;
462 
463  preproc_images = cpl_imagelist_new();
464  for (i = 0; i < cpl_imagelist_get_size(raw_images); i++) {
465  double exposure_time = 0.0;
466  const cpl_propertylist *current_header;
467 
468  current_dark = cpl_image_duplicate(cpl_imagelist_get_const(raw_images, i));
469  current_header = raw_headers[i];
470 
471  /* Subtract master bias */
472  if (master_bias != NULL) {
473  cpl_msg_info(cpl_func, "Subtracting master bias");
474  cpl_image_subtract(current_dark, master_bias);
475  } else {
476  cpl_msg_info(cpl_func, "Skipping bias subtraction");
477  }
478 
479  exposure_time = irplib_head_get_exptime(current_header);
480 
481  /* Initialize/update min/max exposure time*/
482  if (i == 0 || exposure_time < min_exptime) {
483  min_exptime = exposure_time;
484  }
485  if (i == 0 || exposure_time > max_exptime) {
486  max_exptime = exposure_time;
487  }
488 
489  /* Do not normalize to unit exposure time */
490  /* If this is uncommented, then remember to also calculate the
491  correct master dark exposure time below.
492  irplib_msg("Normalizing from %f s to unit exposure time", exposure_time);
493  check( cpl_image_divide_scalar(current_dark, exposure_time),
494  "Error normalizing dark frame"); */
495 
496  /* Append to imagelist */
497  cpl_imagelist_set(preproc_images, current_dark, i);
498 
499  /* Don't deallocate the image. It will be deallocated when
500  the image list is deallocated */
501  current_dark = NULL;
502  }
503 
504 
505  /* Check exposure times */
506  cpl_msg_info(cpl_func,
507  "Exposure times range from %e s to %e s (%e %% variation)", min_exptime,
508  max_exptime, 100 * (max_exptime - min_exptime) / min_exptime);
509 
510  if ((max_exptime - min_exptime) / min_exptime > .001) {
511  cpl_msg_warning(cpl_func, "Exposure times differ by %e %%",
512  100 * (max_exptime - min_exptime) / min_exptime);
513  }
514 
515  /* compute correct exposure time */
516  *mean_exptime=0.5 * (max_exptime + min_exptime);
517  return preproc_images;
518 }
519 
520 
521 cpl_image *
522 irplib_mdark_process_chip(const cpl_imagelist *raw_images,
523  cpl_propertylist **raw_headers, const cpl_image *master_bias,
524  cpl_propertylist *mdark_header, const cpl_parameterlist *parameters,
525  const char* recipe_id, cpl_table* qclog, const int do_qc,
526  const char* STACK_METHOD, const double STACK_KLOW, const double STACK_KHIGH,
527  const int STACK_NITER,
528  const int pr_num_x, const int pr_num_y,
529  const int pr_box_sx, const int pr_box_sy) {
530  cpl_image *master_dark = NULL; /* Result */
531  cpl_image *current_dark = NULL;
532  cpl_imagelist *preproc_images = NULL;
533  double mean_exptime = 0;
534 
535  /* First process each input image and store the results in a
536  new image list */
537  preproc_images = irplib_mkmaster_dark_fill_imagelist(raw_images, raw_headers,
538  master_bias, &mean_exptime);
539  if (do_qc) {
540  /* Here we should compute QC but a a better way to log it is TBD */
541  irplib_mkmaster_dark_qc(raw_images, preproc_images, parameters, pr_num_x,
542  pr_num_y, pr_box_sx, pr_box_sy, recipe_id, qclog);
543 
544  }
545  /* Get median stack of input darks */
546  if (strcmp(STACK_METHOD, "MEDIAN") == 0) {
547  cpl_msg_info(cpl_func, "Calculating stack median");
548  master_dark = cpl_imagelist_collapse_median_create(preproc_images);
549  } else {
550  cpl_msg_info(cpl_func, "Calculating stack mean");
551  master_dark = irplib_imagelist_ksigma_stack(preproc_images, STACK_KLOW,
552  STACK_KHIGH, STACK_NITER);
553 
554  }
555  irplib_head_set_exptime(mdark_header, mean_exptime );
556 
557  cpl_image_delete(current_dark);
558  cpl_imagelist_delete(preproc_images);
559  if (cpl_error_get_code() != CPL_ERROR_NONE) {
560  cpl_image_delete(master_dark);
561  }
562 
563  return master_dark;
564 }
565