UVES Pipeline Reference Manual  5.4.0
uves_response_efficiency.c
1 /* *
2  * This file is part of the ESO UVES Pipeline *
3  * Copyright (C) 2004,2005 European Southern Observatory *
4  * *
5  * This library is free software; you can redistribute it and/or modify *
6  * it under the terms of the GNU General Public License as published by *
7  * the Free Software Foundation; either version 2 of the License, or *
8  * (at your option) any later version. *
9  * *
10  * This program is distributed in the hope that it will be useful, *
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of *
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
13  * GNU General Public License for more details. *
14  * *
15  * You should have received a copy of the GNU General Public License *
16  * along with this program; if not, write to the Free Software *
17  * Foundation, 51 Franklin St, Fifth Floor, Boston, MA 02111-1307 USA *
18  * */
19 
20 /*
21  * $Author: amodigli $
22  * $Date: 2013-08-08 13:36:46 $
23  * $Revision: 1.39 $
24  * $Name: not supported by cvs2svn $
25  *
26  */
27 
28 /*----------------------------------------------------------------------------*/
32 /*----------------------------------------------------------------------------*/
33 
34 #ifdef HAVE_CONFIG_H
35 # include <config.h>
36 #endif
37 
38 /*-----------------------------------------------------------------------------
39  Includes
40  -----------------------------------------------------------------------------*/
41 
42 #include <uves_response_efficiency.h>
43 #include <uves_response_utils.h>
44 #include <uves_reduce.h>
45 #include <uves_reduce_utils.h>
46 #include <uves_dfs.h>
47 #include <uves_pfits.h>
48 #include <uves_wavecal_utils.h>
49 #include <uves_utils_polynomial.h>
50 #include <uves_utils.h>
51 #include <uves_utils_wrappers.h>
52 #include <uves_utils_cpl.h>
53 #include <uves.h>
54 #include <uves_error.h>
55 #include <uves_msg.h>
56 
57 #include <cpl.h>
58 
59 #include <stdbool.h>
60 
61 /*-----------------------------------------------------------------------------
62  Defines
63  -----------------------------------------------------------------------------*/
64 #define H_BAR 6.626068e-34 /* SI-units */
65 #define PRIMARY_DIA 818 /* Primary diameter (cm) */
66 #define OBSTR_DIA 140 /* Central obstruction diameter (cm) */
67 #define TELESCOPE_EFFECTIVE_AREA \
68  (M_PI * (PRIMARY_DIA * PRIMARY_DIA - OBSTR_DIA * OBSTR_DIA) / 4.0) /* (cm^2) */
69 
98 /*----------------------------------------------------------------------------*/
99 cpl_error_code
100 uves_response_efficiency(const cpl_image *raw_image,
101  const uves_propertylist *raw_header,
102  const uves_propertylist *rotated_header,
103  const cpl_image *master_bias,
104  const uves_propertylist *mbias_header,
105  const cpl_image *master_dark,
106  const uves_propertylist *mdark_header,
107  const cpl_table *ordertable,
108  const polynomial *order_locations,
109  const cpl_table *linetable[3],
110  const uves_propertylist *linetable_header[3],
111  const polynomial *dispersion_relation[3],
112  const cpl_table *flux_table,
113  const cpl_table *atm_extinction,
114  enum uves_chip chip,
115  bool debug_mode,
116  const cpl_parameterlist *parameters,
117  /* Identification */
118  double PACCURACY,
119  /* Output */
120  cpl_table **efficiency,
121  cpl_table **blaze_efficiency)
122 {
123  cpl_image *background = NULL;
124  cpl_image *rebinned_spectrum = NULL;
125  cpl_image *rebinned_noise = NULL;
126  cpl_image *merged_sky = NULL;
127  cpl_image *merged_spectrum = NULL;
128  cpl_image *merged_noise = NULL;
129  cpl_image *reduced_spectrum = NULL;
130  cpl_image *reduced_noise = NULL;
131  cpl_image *reduced_rebinned = NULL;
132  cpl_image *reduced_rebinned_noise = NULL;
133  uves_propertylist *rebinned_header = NULL;
134  uves_propertylist *reduced_header = NULL;
135  polynomial *disprel_1d = NULL;
136 
137  cpl_image *response_orders = NULL;
138  cpl_image *efficiency_spectrum = NULL;
139  cpl_table *central_efficiency = NULL;
140 
141  cpl_table *info_tbl = NULL; /* Local. The info tbl
142  should not be calculated twice
143  in the response recipe. It is
144  calculated in the first extraction */
145 
146  char *ref_obj_id = NULL;
147 
148  double wavestep;
149  double extraction_slit;
150  cpl_image* wave_map=NULL;
151  cpl_image* reduced_rebinned_no_bpm=NULL;
152  cpl_mask* bpm=NULL;
153 
154  /* Set parameters
155  wavestep = 10 * 2/3 * mean(pixelsize)
156 
157  After execution, revert to current value
158  */
159  {
160  double smooth_step;
161 
162  /* wavestep */
163  check( uves_get_parameter(parameters, NULL,
164  make_str(UVES_RESPONSE_ID) "", "reduce.rebin.wavestep",
165  CPL_TYPE_DOUBLE, &wavestep),
166  "Error getting resampling step size");
167 
168  check( smooth_step = cpl_table_get_column_mean(linetable[1], LINETAB_PIXELSIZE),
169  "Error reading mean pixelsize");
170 
171  smooth_step = 10*2*smooth_step/3;
172 
173  /* Cast to non-const is okay. The parameter is reset to its previous value
174  (see below), so there is not net change (unless the reduction fails,
175  in which case parameter list will change).
176  */
177  check( uves_set_parameter((cpl_parameterlist *) parameters,
178  make_str(UVES_RESPONSE_ID) "", "reduce.rebin.wavestep",
179  CPL_TYPE_DOUBLE, &smooth_step),
180  "Error setting resampling step size");
181  }
182 
183  check( uves_reduce(raw_image,
184  raw_header,
185  rotated_header,
186  master_bias,
187  mbias_header,
188  master_dark,
189  mdark_header,
190  NULL,
191  NULL, /* No master flat */
192  ordertable,
193  order_locations,
194  linetable,
195  linetable_header,
196  dispersion_relation,
197  chip,
198  debug_mode,
199  parameters,
200  make_str(UVES_RESPONSE_ID),
201  ".efficiency",
202  /* Output */
203  NULL,
204  NULL,
205  NULL, /* 2d products */
206  NULL, /* Cosmic ray table */
207  &wave_map,
208  &background,
209  NULL,
210  NULL, /* Variance of flat-fielded spectrum */
211  NULL,
212  NULL, /* Don't need these
213  intermediate products */
214  &merged_sky,
215  &rebinned_spectrum,
216  &rebinned_noise,
217  &rebinned_header,
218  &merged_spectrum,
219  &merged_noise,
220  &reduced_header,
221  &reduced_rebinned,
222  &reduced_rebinned_noise,
223  &reduced_spectrum,
224  &reduced_noise,
225  &info_tbl,
226  &extraction_slit, /* not passed on to the caller */
227  NULL),
228  "Could not reduce frame");
229 
230  /* Reset parameter to previous value */
231  {
232  uves_msg_debug("Resetting parameter wavestep = %e", wavestep);
233 
234  /* Cast to non-const is okay. There is no net change in the parameter list (see above). */
235  check( uves_set_parameter((cpl_parameterlist *) parameters,
236  make_str(UVES_RESPONSE_ID) "", "reduce.rebin.wavestep",
237  CPL_TYPE_DOUBLE, &wavestep),
238  "Error resetting resampling step size");
239  }
240 
241 
242  /* Save reduced spectrum */
243  if (debug_mode)
244  {
245  /* Window number doesn't apply. This is middle window minus two other (sky) windows */
246  check( uves_save_image_local("Reduced spectrum (2d)", "reduced",
247  reduced_rebinned, chip, -1, -1, rebinned_header, true),
248  "Error saving reduced spectrum (2d)");
249 
250  check( uves_save_image_local("Reduced spectrum (2d) noise", "errreduced",
251  reduced_rebinned_noise, chip, -1, -1, rebinned_header, true),
252  "Error saving reduced spectrum (2d) noise");
253 
254  check( uves_save_image_local("Reduced spectrum", "merged",
255  reduced_spectrum, chip, -1, -1, reduced_header, true),
256  "Error saving reduced spectrum");
257 
258  check( uves_save_image_local("Reduced spectrum noise", "errmerged",
259  reduced_noise, chip, -1, -1, reduced_header, true),
260  "Error saving reduced spectrum noise");
261  }
262 
263  uves_msg("Dividing by catalogue flux");
264  /*
265  * Calculate 2d response curve (don't scale to unit exposure time, binning, gain, ... )
266  */
267  /* in some cases are flagged too many bad pixels on sky frames used
268  to compute reduced object in linear extraction method, this
269  affecting the number of bad pixels of the reduced_rebinned obj spectrum
270  to fix this we erase the associated bad pixel map
271  */
272  reduced_rebinned_no_bpm=cpl_image_duplicate(reduced_rebinned);
273  bpm=cpl_image_unset_bpm(reduced_rebinned_no_bpm);
274  check( response_orders = uves_calculate_response(reduced_rebinned_no_bpm,
275  rebinned_header,
276  flux_table,
277  raw_header, PACCURACY,
278  false,
279  &ref_obj_id),/* flux/std_flux */
280  "Could not calculate response curve");
281 
282  uves_free_image(&reduced_rebinned_no_bpm);
283  uves_free_mask(&bpm);
284 
285  if (debug_mode)
286  {
287  check( uves_save_image_local("2d response curve", "resp",
288  response_orders, chip, -1, -1, rebinned_header, true),
289  "Error saving 2d response curve");
290  }
291 
292  /*
293  * Extinction correction, exposure time + gain
294  */
295  {
296  int n_traces = cpl_image_get_size_y(merged_spectrum); /* Number of spatial traces */
297 
298  assure( n_traces == 1, CPL_ERROR_ILLEGAL_INPUT,
299  "2d extraction/reduction not supported");
300 
301  check( efficiency_spectrum = uves_normalize_spectrum(response_orders, NULL,
302  /* Spectrum, noise */
303  rebinned_header,
304  raw_header,
305  n_traces,
306  chip,
307  atm_extinction,
308  false, /* Don't divide by binning */
309  NULL), /* Don't need output noise */
310  "Could not normalize spectrum");
311  }
312 
313  /*
314  * 7 x 1 median filter
315  */
316  uves_msg("Applying 7x1 median filter");
317  check( uves_filter_image_median(&efficiency_spectrum, 3, 0, false),
318  "Error applying median filter");
319 
320 
321  uves_msg("Calculating quantum detection efficiency");
322 
323  {
324  int nx, nbins, norders, order;
325  int first_abs_order, last_abs_order, abs_order; /* Absolute order numbers */
326  double dlambda;
327  double average_noise; /* Median of noise of rebinned spectrum */
328  int row = 0; /* Next unused table row */
329 
330  double *efficiency_data; /* For efficiency. cpl_image_get() */
331  double *reduced_noise_data; /* is slow when there are bad pixels */
332 
333  efficiency_data = cpl_image_get_data_double(efficiency_spectrum);
334  reduced_noise_data = cpl_image_get_data_double(reduced_rebinned_noise);
335 
336  nx = cpl_image_get_size_x(raw_image);
337  nbins = cpl_image_get_size_x(efficiency_spectrum);
338  norders = cpl_image_get_size_y(efficiency_spectrum);
339 
340  *efficiency = cpl_table_new(nbins * norders);
341  cpl_table_new_column(*efficiency, "Wave", CPL_TYPE_DOUBLE);
342  cpl_table_new_column(*efficiency, "Eff", CPL_TYPE_DOUBLE);
343  cpl_table_new_column(*efficiency, "Binsize", CPL_TYPE_DOUBLE);
344  cpl_table_new_column(*efficiency, "Order", CPL_TYPE_INT);
345  row = 0;
346 
347  check( first_abs_order = uves_pfits_get_firstabsorder(linetable_header[1]),
348  "Could not read order numbers from line table header");
349  check( last_abs_order = uves_pfits_get_lastabsorder (linetable_header[1]),
350  "Could not read order numbers from line table header");
351 
352  check( dlambda = uves_pfits_get_cdelt1(rebinned_header),
353  "Error reading bin width from header");
354 
355  check( average_noise = cpl_image_get_median(reduced_rebinned_noise),
356  "Error reading median noise level");
357 
358  for (order = 1; order <= norders; order++)
359  {
360  double lambda_start, lambda, lambda_end;
361  double x;
362  int bin;
363 
364  abs_order = uves_absolute_order(first_abs_order, last_abs_order, order);
365 
366  check( lambda_start = uves_pfits_get_wstart(rebinned_header, order),
367  "Error reading start wavelength from header");
368 
369  check( lambda_end = uves_pfits_get_wend(rebinned_header, order),
370  "Error reading end wavelength from header");
371 
372  /* Get 1d dispersion relation for this order
373  f_1d = f(x, m=abs_order)
374 
375  Collapsing a polynomial is slow, so do it
376  only once per order
377  */
378  uves_polynomial_delete(&disprel_1d);
379  check( disprel_1d = uves_polynomial_collapse(dispersion_relation[1],
380  2, /* Independent variable number */
381  abs_order),
382  "Error getting 1d dispersion relation for absolute order #%d", abs_order);
383 
384  x = 1;
385  for (lambda = lambda_start, bin = 1;
386  lambda < lambda_end + 0.5 * dlambda && bin <= nbins;
387  bin++, lambda += dlambda)
388  {
389  double flux;
390  double dldx;
391  double noise; /* Only use positions with low noise
392  (middle of blaze function) */
393 
394  /* flux = cpl_image_get(efficiency_spectrum,
395  bin, order, &pis_rejected);
396  noise = cpl_image_get(reduced_rebinned_noise,
397  bin, order, &pis_rejected); */
398  flux = efficiency_data [(bin-1) + (order-1) * nbins];
399  noise = reduced_noise_data[(bin-1) + (order-1) * nbins];
400 
401 
402  /*
403  * Energy per (time * area * wavelength) =
404  * ((electron counts)/gain) * (hc/lambda) /
405  * (time * area * |dlambda/dx|)
406  *
407  * We already divided by exposure time, gain
408  * We did not multiply by dlambda/dx during rebinning,
409  * so now is the time to do it
410  */
411 
412  /* Solve f(x,m) = m*lambda for x.
413  *
414  * This is equivalent to solving f_1d(x) = m*lambda
415  */
416 
418  disprel_1d,
419  abs_order * lambda, /* right hand side */
420  x, /* guess */
421  1), /* multiplicity */
422  "Could not solve dispersion relation for x "
423  "at (m, lambda) = (%d, %f)", abs_order, lambda);
424 
425 
426 
427  /* For constant absolute order number, m:
428  dl/dx = d (l.m)/dx / m */
429 
431  dispersion_relation[1],
432  x,
433  abs_order, 1) / abs_order),
434  "Could not evaluate dispersion relation");
435 
436  /* Don't make a linear interpolation
437  weight = (lambda - lambda_start) / (lambda_end - lambda_start);
438 
439  check( dldx =
440  fabs(uves_polynomial_derivative_2d(
441  dispersion_relation[1],
442  1 , abs_order,
443  1) / abs_order) * (1 - weight) +
444  fabs(uves_polynomial_derivative_2d(
445  dispersion_relation[1],
446  nx, abs_order, 1) / abs_order) * weight,
447  "Could not evaluate dispersion relation");
448  */
449 
450  flux = flux * 1e16 * 1e17 * H_BAR * SPEED_OF_LIGHT /
451  (dldx * lambda * TELESCOPE_EFFECTIVE_AREA);
452  /* The factor 1e17 accounts for the conversion Joule<-erg
453  * (10^7) and Angstrom->meters (10^10)
454  * The factor 1e16 is to correct for the fact that the
455  * catalogue flux is in units of (10^-16 <standard units>)
456  */
457 
458  if (noise < 3*average_noise)
459  {
460  check(( cpl_table_set_double(*efficiency, "Wave", row, lambda),
461  cpl_table_set_double(*efficiency, "Eff", row, flux),
462  cpl_table_set_double(*efficiency, "Binsize", row, dldx),
463  cpl_table_set_int (*efficiency, "Order", row, order),
464  row++),
465  "Error updating efficiency table row %d", row);
466  }
467  }
468  }
469 
470  /* Remove unused rows of efficiency table */
471  check( cpl_table_set_size(*efficiency, row),
472  "Error setting size of efficiency table to %d rows", row);
473  /* remove negative efficiency points */
474  cpl_table* tmp=cpl_table_duplicate(*efficiency);
475  row=cpl_table_and_selected_double(tmp,"Eff",CPL_GREATER_THAN,0.);
476  uves_free_table(efficiency);
477  *efficiency=cpl_table_extract_selected(tmp);
478  uves_free_table(&tmp);
479 
480 
481  /* Get the "top efficiency" (90% percentile efficiency of middle 20% of each order) */
482 
483  *blaze_efficiency = cpl_table_new(norders);
484  cpl_table_new_column(*blaze_efficiency, "Order", CPL_TYPE_INT);
485  cpl_table_new_column(*blaze_efficiency, "Wave" , CPL_TYPE_DOUBLE);
486  cpl_table_new_column(*blaze_efficiency, "Eff" , CPL_TYPE_DOUBLE);
487  row = 0;
488 
489  for (order = 1; order <= norders; order++)
490  {
491  double lambda_min;
492  double lambda_central_min;
493  double lambda_central;
494  double lambda_central_max;
495  double lambda_max;
496  double top_efficiency;
497 
498  abs_order = uves_absolute_order(first_abs_order, last_abs_order, order);
499 
500 
501  check( lambda_min = uves_pfits_get_wstart(rebinned_header, order),
502  "Error reading bin width from header");
503 
504  check( lambda_max = uves_pfits_get_wend(rebinned_header, order),
505  "Error reading bin width from header");
506 
507  lambda_central_min = lambda_min + 0.4 * (lambda_max - lambda_min);
508  lambda_central = lambda_min + 0.5 * (lambda_max - lambda_min);
509  lambda_central_max = lambda_min + 0.6 * (lambda_max - lambda_min);
510 
511  /* Select rows in this order in range
512  ]lambda_central_min ; lambda_central_max[ */
513  cpl_table_select_all(*efficiency);
514  cpl_table_and_selected_int (*efficiency, "Order",
515  CPL_EQUAL_TO , order);
516  cpl_table_and_selected_double(*efficiency, "Wave" ,
517  CPL_GREATER_THAN, lambda_central_min);
518  cpl_table_and_selected_double(*efficiency, "Wave" ,
519  CPL_LESS_THAN , lambda_central_max);
520 
521  uves_msg_debug("%" CPL_SIZE_FORMAT " bins in central 20 %% range of order #%d",
522  cpl_table_count_selected(*efficiency), order);
523 
524  if ( cpl_table_count_selected(*efficiency) > 0)
525  {
526  uves_free_table(&central_efficiency);
527  central_efficiency = cpl_table_extract_selected(*efficiency);
528 
529  /* Get 90% percentile efficiency */
530  uves_sort_table_1(central_efficiency, "Eff", false); /* Ascending */
531 
532  top_efficiency = cpl_table_get_double(
533  central_efficiency, "Eff",
534  (int) (0.9 * cpl_table_get_nrow(central_efficiency)), NULL);
535  }
536  else
537  {
538  uves_msg_debug("No wavelength bins in central 20%% range of order #%d",
539  order);
540  top_efficiency = 0;
541  }
542 
543  uves_msg("Efficiency(lambda = %.2f A) = %.2f%%",
544  lambda_central, top_efficiency*100);
545 
546  check(( cpl_table_set_int (*blaze_efficiency, "Order", row, order),
547  cpl_table_set_double(*blaze_efficiency, "Wave" , row, lambda_central),
548  cpl_table_set_double(*blaze_efficiency, "Eff" , row, top_efficiency),
549  row++),
550  "Error updating blaze efficiency table");
551  } /* for order */
552  }
553 
554  cleanup:
555  uves_free_image(&background);
556  uves_free_image(&rebinned_spectrum);
557  uves_free_image(&rebinned_noise);
558  uves_free_image(&merged_sky);
559  uves_free_image(&merged_spectrum);
560  uves_free_image(&merged_noise);
561  uves_free_image(&reduced_spectrum);
562  uves_free_image(&reduced_noise);
563  uves_free_image(&reduced_rebinned);
564  uves_free_image(&reduced_rebinned_noise);
565  uves_free_propertylist(&reduced_header);
566  uves_free_propertylist(&rebinned_header);
567  uves_polynomial_delete(&disprel_1d);
568 
569  uves_free_image(&response_orders);
570  uves_free_image(&efficiency_spectrum);
571  uves_free_table(&central_efficiency);
572  uves_free_table(&info_tbl);
573 
574  cpl_free(ref_obj_id);
575 
576  if (cpl_error_get_code() != CPL_ERROR_NONE)
577  {
578  uves_free_table(efficiency);
579  uves_free_table(blaze_efficiency);
580  }
581 
582  return cpl_error_get_code();
583 }