KMOS Pipeline Reference Manual  1.3.0
kmo_illumination.c
00001 /*
00002  * This file is part of the KMOS Pipeline
00003  * Copyright (C) 2002,2003 European Southern Observatory
00004  *
00005  * This program is free software; you can redistribute it and/or modify
00006  * it under the terms of the GNU General Public License as published by
00007  * the Free Software Foundation; either version 2 of the License, or
00008  * (at your option) any later version.
00009  *
00010  * This program is distributed in the hope that it will be useful,
00011  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00012  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00013  * GNU General Public License for more details.
00014  *
00015  * You should have received a copy of the GNU General Public License
00016  * along with this program; if not, write to the Free Software
00017  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
00018  */
00019 
00020 #ifdef HAVE_CONFIG_H
00021 #include <config.h>
00022 #endif
00023 
00024 /*-----------------------------------------------------------------------------
00025  *                              Includes
00026  *----------------------------------------------------------------------------*/
00027 
00028 #include <math.h>
00029 #include <string.h>
00030 
00031 #include <cpl.h>
00032 
00033 #include "kmclipm_priv_splines.h"
00034 
00035 #include "kmo_priv_reconstruct.h"
00036 #include "kmo_priv_functions.h"
00037 #include "kmo_priv_flat.h"
00038 #include "kmo_priv_wave_cal.h"
00039 #include "kmo_functions.h"
00040 #include "kmo_cpl_extensions.h"
00041 #include "kmo_dfs.h"
00042 #include "kmo_error.h"
00043 #include "kmo_constants.h"
00044 #include "kmo_debug.h"
00045 
00046 /*-----------------------------------------------------------------------------
00047  *                          Functions prototypes
00048  *----------------------------------------------------------------------------*/
00049 
00050 static int kmo_illumination_create(cpl_plugin *);
00051 static int kmo_illumination_exec(cpl_plugin *);
00052 static int kmo_illumination_destroy(cpl_plugin *);
00053 static int kmo_illumination(cpl_parameterlist *, cpl_frameset *);
00054 
00055 /*-----------------------------------------------------------------------------
00056  *                          Static variables
00057  *----------------------------------------------------------------------------*/
00058 
00059 static char kmo_illumination_description[] =
00060 "This recipe creates the spatial non-uniformity calibration frame needed for\n"
00061 "all three detectors. It must be called after the kmo_wave_cal-recipe, which\n"
00062 "generates the spectral calibration frame needed in this recipe. As input at\n"
00063 "least a sky, a master dark, a master flat and the spatial and spectral cali-\n"
00064 "bration frames are required.\n"
00065 "The created product, the illumination correction, can be used as input for\n"
00066 "kmo_std_star and kmo_sci_red.\n"
00067 "\n"
00068 "BASIC PARAMETERS:\n"
00069 "-----------------\n"
00070 "--imethod\n"
00071 "The interpolation method used for reconstruction.\n"
00072 "\n"
00073 "--range\n"
00074 "The spectral ranges to combine when collapsing the reconstructed cubes. e.g.\n"
00075 "\"x1_start,x1_end;x2_start,x2_end\" (microns)\n"
00076 "\n"
00077 "ADVANCED PARAMETERS\n"
00078 "-------------------\n"
00079 "--flux\n"
00080 "Specify if flux conservation should be applied.\n"
00081 "\n"
00082 "--add-all\n"
00083 "By default the first FLAT_SKY frame is omitted, since in the\n"
00084 "KMOS_spec_cal_skyflat template this is an acquisition frame to estimate the\n"
00085 "needed exposure time for the subsequent FLAT_SKY frames. If anyway all frames\n"
00086 "should be considered, set this parameter to TRUE.\n"
00087 "\n"
00088 "--neighborhoodRange\n"
00089 "Defines the range to search for neighbors during reconstruction\n"
00090 "\n"
00091 "--b_samples\n"
00092 "The number of samples in spectral direction for the reconstructed cube.\n"
00093 "Ideally this number should be greater than 2048, the detector size.\n"
00094 "\n"
00095 "--b_start\n"
00096 "--b_end\n"
00097 "Used to define manually the start and end wavelength for the reconstructed\n"
00098 "cube. By default the internally defined values are used.\n"
00099 "\n"
00100 "--cmethod\n"
00101 "Following methods of frame combination are available:\n"
00102 "   * 'ksigma' (Default)\n"
00103 "   An iterative sigma clipping. For each position all pixels in the spectrum\n"
00104 "   are examined. If they deviate significantly, they will be rejected according\n"
00105 "   to the conditions:\n"
00106 "       val > mean + stdev * cpos_rej\n"
00107 "   and\n"
00108 "       val < mean - stdev * cneg_rej\n"
00109 "   where --cpos_rej, --cneg_rej and --citer are the corresponding configuration\n"
00110 "   parameters. In the first iteration median and percentile level are used.\n"
00111 "\n"
00112 "   * 'median'\n"
00113 "   At each pixel position the median is calculated.\n"
00114 "\n"
00115 "   * 'average'\n"
00116 "   At each pixel position the average is calculated.\n"
00117 "\n"
00118 "   * 'sum'\n"
00119 "   At each pixel position the sum is calculated.\n"
00120 "\n"
00121 "   * 'min_max'\n"
00122 "   The specified number of minimum and maximum pixel values will be rejected.\n"
00123 "   --cmax and --cmin apply to this method.\n"
00124 "\n"
00125 "--cpos_rej\n"
00126 "--cneg_rej\n"
00127 "--citer\n"
00128 "see --cmethod='ksigma'\n"
00129 "\n"
00130 "--cmax\n"
00131 "--cmin\n"
00132 "see --cmethod='min_max'\n"
00133 "\n"
00134 "--pix_scale\n"
00135 "Change the pixel scale [arcsec]. Default of 0.2\" results into cubes of\n"
00136 "14x14pix, a scale of 0.1\" results into cubes of 28x28pix, etc.\n"
00137 "\n"
00138 "--suppress_extension\n"
00139 "If set to TRUE, the arbitrary filename extensions are supressed. If multiple\n"
00140 "products with the same category are produced, they will be numered consecutively\n"
00141 "starting from 0.\n"
00142 "\n"
00143 "-------------------------------------------------------------------------------\n"
00144 "  Input files:\n"
00145 "\n"
00146 "   DO                    KMOS                                                  \n"
00147 "   category              Type   Explanation                    Required #Frames\n"
00148 "   --------              -----  -----------                    -------- -------\n"
00149 "   FLAT_SKY               F2D   Sky exposures                     Y      1-n   \n"
00150 "                                (at least 3 frames recommended)                \n"
00151 "   MASTER_DARK            F2D   Master dark                       Y       1    \n"
00152 "   MASTER_FLAT            F2D   Master flat                       Y       1    \n"
00153 "   XCAL                   F2D   x calibration frame               Y       1    \n"
00154 "   YCAL                   F2D   y calibration frame               Y       1    \n"
00155 "   LCAL                   F2D   Wavelength calib. frame           Y       1    \n"
00156 "   WAVE_BAND              F2L   Table with start-/end-wavelengths Y       1    \n"
00157 "   FLAT_EDGE              F2L   Table with fitted slitlet edges   N      0,1   \n"
00158 "\n"
00159 "  Output files:\n"
00160 "\n"
00161 "   DO                    KMOS\n"
00162 "   category              Type   Explanation\n"
00163 "   --------              -----  -----------\n"
00164 "   ILLUM_CORR            F2I    Illumination calibration frame   \n"
00165 "   If FLAT_EDGE is provided: \n"
00166 "   SKYFLAT_EDGE          F2L    Frame containing parameters of fitted \n"
00167 "                                slitlets of all IFUs of all detectors\n"
00168 "-------------------------------------------------------------------------------\n"
00169 "\n";
00170 
00171 /*-----------------------------------------------------------------------------
00172  *                              Functions code
00173  *----------------------------------------------------------------------------*/
00174 
00191 int cpl_plugin_get_info(cpl_pluginlist *list)
00192 {
00193     cpl_recipe *recipe = cpl_calloc(1, sizeof *recipe);
00194     cpl_plugin *plugin = &recipe->interface;
00195 
00196     cpl_plugin_init(plugin,
00197                         CPL_PLUGIN_API,
00198                         KMOS_BINARY_VERSION,
00199                         CPL_PLUGIN_TYPE_RECIPE,
00200                         "kmo_illumination",
00201                         "Create a calibration frame to correct spatial "
00202                         "non-uniformity of flatfield.",
00203                         kmo_illumination_description,
00204                         "Alex Agudo Berbel",
00205                         "usd-help@eso.org",
00206                         kmos_get_license(),
00207                         kmo_illumination_create,
00208                         kmo_illumination_exec,
00209                         kmo_illumination_destroy);
00210 
00211     cpl_pluginlist_append(list, plugin);
00212 
00213     return 0;
00214 }
00215 
00223 static int kmo_illumination_create(cpl_plugin *plugin)
00224 {
00225     cpl_recipe *recipe;
00226     cpl_parameter *p;
00227 
00228     /* Check that the plugin is part of a valid recipe */
00229     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00230         recipe = (cpl_recipe *)plugin;
00231     else
00232         return -1;
00233 
00234     /* Create the parameters list in the cpl_recipe object */
00235     recipe->parameters = cpl_parameterlist_new();
00236 
00237     /* Fill the parameters list */
00238     /* --imethod */
00239     p = cpl_parameter_new_value("kmos.kmo_illumination.imethod",
00240                                 CPL_TYPE_STRING,
00241                                 "Method to use for interpolation: "
00242                                 "[\"NN\" (nearest neighbour), "
00243                                 "\"lwNN\" (linear weighted nearest neighbor), "
00244                                 "\"swNN\" (square weighted nearest neighbor), "
00245                                 "\"MS\" (Modified Shepard's method), "
00246                                 "\"CS\" (Cubic spline)]",
00247                                 "kmos.kmo_illumination",
00248                                 "CS");
00249     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "imethod");
00250     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00251     cpl_parameterlist_append(recipe->parameters, p);
00252 
00253     /* --neighborhoodRange */
00254     p = cpl_parameter_new_value("kmos.kmo_illumination.neighborhoodRange",
00255                                 CPL_TYPE_DOUBLE,
00256                                 "Defines the range to search for neighbors. "
00257                                 "in pixels",
00258                                 "kmos.kmo_illumination",
00259                                 1.001);
00260     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "neighborhoodRange");
00261     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00262     cpl_parameterlist_append(recipe->parameters, p);
00263 
00264     /* --range */
00265     p = cpl_parameter_new_value("kmos.kmo_illumination.range",
00266                                 CPL_TYPE_STRING,
00267                                 "The spectral ranges to combine when collapsing "
00268                                 "the reconstructed cubes. e.g. "
00269                                 "\"x1_start,x1_end;x2_start,x2_end\" (microns)",
00270                                 "kmos.kmo_illumination",
00271                                 "");
00272     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "range");
00273     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00274     cpl_parameterlist_append(recipe->parameters, p);
00275 
00276     /* --flux */
00277     p = cpl_parameter_new_value("kmos.kmo_illumination.flux",
00278                                 CPL_TYPE_BOOL,
00279                                 "TRUE: Apply flux conservation. FALSE: otherwise",
00280                                 "kmos.kmo_illumination",
00281                                 FALSE);
00282     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "flux");
00283     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00284     cpl_parameterlist_append(recipe->parameters, p);
00285 
00286     /* --add-all */
00287     p = cpl_parameter_new_value("kmos.kmo_illumination.add-all",
00288                                 CPL_TYPE_BOOL,
00289                                 "FALSE: omit 1st FLAT_SKY frame (acquisition), "
00290                                 "TRUE: don't perform any checks, add them all",
00291                                 "kmos.kmo_illumination",
00292                                 FALSE);
00293     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "add-all");
00294     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00295     cpl_parameterlist_append(recipe->parameters, p);
00296 
00297     /* --pix_scale */
00298     p = cpl_parameter_new_value("kmos.kmo_illumination.pix_scale",
00299                                 CPL_TYPE_DOUBLE,
00300                                 "Change the pixel scale [arcsec]. "
00301                                 "Default of 0.2\" results into cubes of 14x14pix, "
00302                                 "a scale of 0.1\" results into cubes of 28x28pix, "
00303                                 "etc.",
00304                                 "kmos.kmo_illumination",
00305                                 KMOS_PIX_RESOLUTION);
00306     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "pix_scale");
00307     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00308     cpl_parameterlist_append(recipe->parameters, p);
00309 
00310     /* --suppress_extension */
00311     p = cpl_parameter_new_value("kmos.kmo_illumination.suppress_extension",
00312                                 CPL_TYPE_BOOL,
00313                                 "Suppress arbitrary filename extension. "
00314                                 "(TRUE (apply) or FALSE (don't apply)",
00315                                 "kmos.kmo_illumination",
00316                                 FALSE);
00317     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "suppress_extension");
00318     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00319     cpl_parameterlist_append(recipe->parameters, p);
00320 
00321     // add parameters for band-definition
00322     kmo_band_pars_create(recipe->parameters,
00323                          "kmos.kmo_illumination");
00324 
00325     // add parameters for combining
00326     return kmo_combine_pars_create(recipe->parameters,
00327                                    "kmos.kmo_illumination",
00328                                    DEF_REJ_METHOD,
00329                                    FALSE);
00330 }
00331 
00337 static int kmo_illumination_exec(cpl_plugin *plugin)
00338 {
00339     cpl_recipe  *recipe;
00340 
00341     /* Get the recipe out of the plugin */
00342     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00343         recipe = (cpl_recipe *)plugin;
00344     else return -1;
00345 
00346     return kmo_illumination(recipe->parameters, recipe->frames);
00347 }
00348 
00354 static int kmo_illumination_destroy(cpl_plugin *plugin)
00355 {
00356     cpl_recipe *recipe;
00357 
00358     /* Get the recipe out of the plugin */
00359     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00360         recipe = (cpl_recipe *)plugin;
00361     else return -1 ;
00362 
00363     cpl_parameterlist_delete(recipe->parameters);
00364     return 0 ;
00365 }
00366 
00381 static int kmo_illumination(cpl_parameterlist *parlist, cpl_frameset *frameset)
00382 {
00383     int              ret_val                    = 0,
00384                      nr_devices                 = 0,
00385                      ifu_nr                     = 0,
00386                      nx                         = 0,
00387                      ny                         = 0,
00388                      process_noise              = FALSE,
00389                      cmax                       = 0,
00390                      cmin                       = 0,
00391                      citer                      = 0,
00392                      *bounds                    = NULL,
00393                      cnt                        = 0,
00394                      qc_max_dev_id              = 0,
00395                      qc_max_nonunif_id          = 0,
00396                      flux                       = FALSE,
00397                      background                 = FALSE,
00398                      add_all_sky                = FALSE,
00399                      same_exptime               = TRUE,
00400                      suppress_extension         = FALSE,
00401                      has_flat_edge              = FALSE,
00402                      i = 0, j = 0, x = 0, y = 0, ix = 0, iy = 0, det_nr = 0, edgeNr = 0;
00403     const int        *punused_ifus              = NULL;
00404     float            *pbad_pix_mask             = NULL;
00405     double           exptime                    = 0.,
00406                      exptime1                   = 0.,
00407                      exptime2                   = 0.,
00408                      cpos_rej                   = 0.0,
00409                      cneg_rej                   = 0.0,
00410                      neighborhoodRange          = 1.001,
00411                      mean_data                  = 0.0,
00412                      ifu_crpix                  = 0.0,
00413                      ifu_crval                  = 0.0,
00414                      ifu_cdelt                  = 0.0,
00415                      qc_spat_unif               = 0.0,
00416                      qc_max_dev                 = 0.0,
00417                      qc_max_nonunif             = 0.0,
00418                      tmp_stdev                  = 0.0,
00419                      tmp_mean                   = 0.0,
00420                      rotangle                   = 0.0,
00421                      tmp_rotangle               = 0.0,
00422                      rotangle_found             = 0.0,
00423                      pix_scale                  = 0.0;
00424     char             *keyword                   = NULL,
00425                      *fn_lut                    = NULL,
00426                      *suffix                    = NULL,
00427                      *fn_suffix                 = NULL,
00428                      *extname                   = NULL,
00429                      *filter                    = NULL;
00430     const char       *method                    = NULL,
00431                      *cmethod                   = NULL,
00432                      *filter_id_l               = NULL,
00433                      *filter_id                 = NULL,
00434                      *ranges_txt                = NULL;
00435     cpl_array        *calTimestamp              = NULL,
00436                      **unused_ifus_before       = NULL,
00437                      **unused_ifus_after        = NULL;
00438     cpl_frame        *frame                     = NULL,
00439                      *xcalFrame                 = NULL,
00440                      *ycalFrame                 = NULL,
00441                      *lcalFrame                 = NULL;
00442     cpl_frameset     *frameset_sky              = NULL;
00443     cpl_image        *img_in                    = NULL,
00444                      *img_dark                  = NULL,
00445                      *img_dark_noise            = NULL,
00446                      *img_flat                  = NULL,
00447                      *img_flat_noise            = NULL,
00448                      *combined_data             = NULL,
00449                      *combined_noise            = NULL,
00450                      *xcal                      = NULL,
00451                      *ycal                      = NULL,
00452                      *lcal                      = NULL,
00453                      *bad_pix_mask              = NULL,
00454                      *data_ifu                  = NULL,
00455                      *noise_ifu                 = NULL,
00456                      **stored_data_images       = NULL,
00457                      **stored_noise_images      = NULL;
00458     cpl_imagelist    *cube_data                 = NULL,
00459                      *cube_noise                = NULL,
00460                      *detector_in               = NULL,
00461                      **stored_data_cubes        = NULL,
00462                      **stored_noise_cubes       = NULL;
00463     cpl_matrix       **edgepars                 = NULL;
00464     cpl_propertylist *main_header               = NULL,
00465                      *tmp_header                = NULL,
00466                      *sub_header                = NULL,
00467                      **stored_sub_data_headers  = NULL,
00468                      **stored_sub_noise_headers = NULL;
00469     cpl_table        *band_table                = NULL,
00470                      ***edge_table_sky          = NULL,
00471                      **edge_table_flat          = NULL;
00472     cpl_vector       *ranges                    = NULL,
00473                      *identified_slices         = NULL,
00474                      *calAngles                 = NULL,
00475                      **slitlet_ids              = NULL,
00476                      *shift_vec                 = NULL,
00477                      *edge_vec                  = NULL;
00478     main_fits_desc   desc_sky,
00479                      desc_dark,
00480                      desc_flat,
00481                      desc_xcal,
00482                      desc_ycal,
00483                      desc_lcal;
00484     gridDefinition   gd;
00485 
00486     KMO_TRY
00487     {
00488         kmo_init_fits_desc(&desc_sky);
00489         kmo_init_fits_desc(&desc_dark);
00490         kmo_init_fits_desc(&desc_flat);
00491         kmo_init_fits_desc(&desc_xcal);
00492         kmo_init_fits_desc(&desc_ycal);
00493         kmo_init_fits_desc(&desc_lcal);
00494 
00495         /* --- check input --- */
00496         KMO_TRY_ASSURE((parlist != NULL) &&
00497                        (frameset != NULL),
00498                        CPL_ERROR_NULL_INPUT,
00499                        "Not all input data is provided!");
00500 
00501         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, FLAT_SKY) >= 1,
00502                        CPL_ERROR_ILLEGAL_INPUT,
00503                        "One or more FLAT_SKY frames are required!");
00504 
00505         if (cpl_frameset_count_tags(frameset, FLAT_SKY) < 3) {
00506             cpl_msg_warning(cpl_func, "It is recommended to provide at least "
00507                                       "3 FLAT_SKY frames!");
00508         }
00509 
00510         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, MASTER_DARK) == 1,
00511                        CPL_ERROR_ILLEGAL_INPUT,
00512                        "Exactly one MASTER_DARK frame is required!");
00513 
00514         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, MASTER_FLAT) == 1,
00515                        CPL_ERROR_ILLEGAL_INPUT,
00516                        "Exactly one MASTER_FLAT frame is required!");
00517 
00518         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, XCAL) == 1,
00519                        CPL_ERROR_ILLEGAL_INPUT,
00520                        "Exactly one XCAL frame is required!");
00521 
00522         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, YCAL) == 1,
00523                        CPL_ERROR_ILLEGAL_INPUT,
00524                        "Exactly one YCAL frame is required!");
00525 
00526         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, LCAL) == 1,
00527                        CPL_ERROR_ILLEGAL_INPUT,
00528                        "Exactly one LCAL frame is required!");
00529 
00530         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, WAVE_BAND) == 1,
00531                        CPL_ERROR_ILLEGAL_INPUT,
00532                        "Exactly one WAVE_BAND frame is required!");
00533 
00534         KMO_TRY_ASSURE((cpl_frameset_count_tags(frameset, FLAT_EDGE) == 1) ||
00535                        (cpl_frameset_count_tags(frameset, FLAT_EDGE) == 0),
00536                        CPL_ERROR_ILLEGAL_INPUT,
00537                        "Exactly one FLAT_EDGE frame is required!");
00538 
00539         has_flat_edge = cpl_frameset_count_tags(frameset, FLAT_EDGE);
00540 
00541         KMO_TRY_ASSURE(kmo_dfs_set_groups(frameset, "kmo_illumination") == 1,
00542                        CPL_ERROR_ILLEGAL_INPUT,
00543                        "Cannot identify RAW and CALIB frames!");
00544 
00545         /* --- get parameters --- */
00546         cpl_msg_info("", "--- Parameter setup for kmo_illumination ---");
00547 
00548         KMO_TRY_EXIT_IF_NULL(
00549             method = kmo_dfs_get_parameter_string(parlist,
00550                                               "kmos.kmo_illumination.imethod"));
00551 
00552         KMO_TRY_ASSURE((strcmp(method, "NN") == 0) ||
00553                        (strcmp(method, "lwNN") == 0) ||
00554                        (strcmp(method, "swNN") == 0) ||
00555                        (strcmp(method, "MS") == 0) ||
00556                        (strcmp(method, "CS") == 0),
00557                        CPL_ERROR_ILLEGAL_INPUT,
00558                        "method must be either \"NN\", \"lwNN\", "
00559                        "\"swNN\", \"MS\" or \"CS\"!");
00560 
00561         KMO_TRY_EXIT_IF_ERROR(
00562             kmo_dfs_print_parameter_help(parlist,
00563                                         "kmos.kmo_illumination.imethod"));
00564 
00565         neighborhoodRange = kmo_dfs_get_parameter_double(parlist,
00566                 "kmos.kmo_illumination.neighborhoodRange");
00567         KMO_TRY_CHECK_ERROR_STATE();
00568 
00569         KMO_TRY_ASSURE(neighborhoodRange > 0.0,
00570                 CPL_ERROR_ILLEGAL_INPUT,
00571                 "neighborhoodRange must be greater than 0.0");
00572 
00573         KMO_TRY_EXIT_IF_ERROR(
00574             kmo_dfs_print_parameter_help(parlist,
00575                                     "kmos.kmo_illumination.neighborhoodRange"));
00576 
00577         ranges_txt = kmo_dfs_get_parameter_string(parlist,
00578                                                   "kmos.kmo_illumination.range");
00579         KMO_TRY_CHECK_ERROR_STATE();
00580 
00581         KMO_TRY_EXIT_IF_ERROR(
00582             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.range"));
00583 
00584         ranges = kmo_identify_ranges(ranges_txt);
00585         KMO_TRY_CHECK_ERROR_STATE();
00586 
00587         flux = kmo_dfs_get_parameter_bool(parlist,
00588                                           "kmos.kmo_illumination.flux");
00589 
00590         KMO_TRY_ASSURE((flux == 0) ||
00591                        (flux == 1),
00592                        CPL_ERROR_ILLEGAL_INPUT,
00593                        "flux must be either FALSE or TRUE!");
00594 
00595         KMO_TRY_EXIT_IF_ERROR(
00596             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.flux"));
00597 
00598         add_all_sky = kmo_dfs_get_parameter_bool(parlist,
00599                                                  "kmos.kmo_illumination.add-all");
00600 
00601         KMO_TRY_ASSURE((add_all_sky == 0) ||
00602                        (add_all_sky == 1),
00603                        CPL_ERROR_ILLEGAL_INPUT,
00604                        "add_all must be either FALSE or TRUE!");
00605 
00606         KMO_TRY_EXIT_IF_ERROR(
00607             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.add-all"));
00608 
00609         pix_scale = kmo_dfs_get_parameter_double(parlist,
00610                                         "kmos.kmo_illumination.pix_scale");
00611         KMO_TRY_CHECK_ERROR_STATE();
00612         KMO_TRY_EXIT_IF_ERROR(
00613            kmo_dfs_print_parameter_help(parlist,
00614                                        "kmos.kmo_illumination.pix_scale"));
00615         KMO_TRY_ASSURE((pix_scale >= 0.01) &&
00616                        (pix_scale <= 0.4),
00617                        CPL_ERROR_ILLEGAL_INPUT,
00618                        "pix_scale must be between 0.01 and 0.4 (results in cubes "
00619                        "with 7x7 to 280x280 pixels)!");
00620 
00621         suppress_extension = kmo_dfs_get_parameter_bool(parlist,
00622                                           "kmos.kmo_illumination.suppress_extension");
00623         KMO_TRY_CHECK_ERROR_STATE();
00624         KMO_TRY_EXIT_IF_ERROR(
00625             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.suppress_extension"));
00626 
00627         KMO_TRY_ASSURE((suppress_extension == TRUE) || (suppress_extension == FALSE),
00628                        CPL_ERROR_ILLEGAL_INPUT,
00629                        "suppress_extension must be TRUE or FALSE!");
00630 
00631         kmo_band_pars_load(parlist, "kmos.kmo_illumination");
00632 
00633         KMO_TRY_EXIT_IF_ERROR(
00634             kmo_combine_pars_load(parlist,
00635                                   "kmos.kmo_illumination",
00636                                   &cmethod,
00637                                   &cpos_rej,
00638                                   &cneg_rej,
00639                                   &citer,
00640                                   &cmin,
00641                                   &cmax,
00642                                   FALSE));
00643         cpl_msg_info("", "-------------------------------------------");
00644 
00645         // check if filter_id, grating_id and rotator offset match for all
00646         // detectors
00647         KMO_TRY_EXIT_IF_ERROR(
00648             kmo_check_frameset_setup(frameset, FLAT_SKY,
00649                                        TRUE, FALSE, TRUE));
00650         KMO_TRY_EXIT_IF_ERROR(
00651             kmo_check_frame_setup(frameset, FLAT_SKY, XCAL,
00652                                        TRUE, FALSE, TRUE));
00653         KMO_TRY_EXIT_IF_ERROR(
00654             kmo_check_frame_setup(frameset, XCAL, YCAL,
00655                                        TRUE, FALSE, TRUE));
00656         KMO_TRY_EXIT_IF_ERROR(
00657             kmo_check_frame_setup(frameset, XCAL, LCAL,
00658                                        TRUE, FALSE, TRUE));
00659         KMO_TRY_EXIT_IF_ERROR(
00660             kmo_check_frame_setup(frameset, XCAL, MASTER_FLAT,
00661                                        TRUE, FALSE, TRUE));
00662 
00663         KMO_TRY_EXIT_IF_NULL(
00664             frame = kmo_dfs_get_frame(frameset, XCAL));
00665         KMO_TRY_EXIT_IF_NULL(
00666             suffix = kmo_dfs_get_suffix(frame, TRUE, FALSE));
00667 
00668         KMO_TRY_EXIT_IF_ERROR(
00669             kmo_check_frame_setup_md5_xycal(frameset));
00670         KMO_TRY_EXIT_IF_ERROR(
00671             kmo_check_frame_setup_md5(frameset));
00672 
00673         cpl_msg_info("", "Detected instrument setup:   %s", suffix+1);
00674         cpl_msg_info("", "(grating 1, 2 & 3)");
00675 
00676         // check which IFUs are active for all frames
00677         KMO_TRY_EXIT_IF_NULL(
00678             unused_ifus_before = kmo_get_unused_ifus(frameset, 0, 0));
00679 
00680         KMO_TRY_EXIT_IF_NULL(
00681             unused_ifus_after = kmo_duplicate_unused_ifus(unused_ifus_before));
00682 
00683         kmo_print_unused_ifus(unused_ifus_before, FALSE);
00684 
00685         // load desc for MASTER_DARK and check
00686         KMO_TRY_EXIT_IF_NULL(
00687             frame = kmo_dfs_get_frame(frameset, MASTER_DARK));
00688         desc_dark = kmo_identify_fits_header(
00689                     cpl_frame_get_filename(frame));
00690         KMO_TRY_CHECK_ERROR_STATE_MSG("MASTER_DARK frame doesn't seem to "
00691                                       "be in KMOS-format!");
00692         KMO_TRY_ASSURE((desc_dark.nr_ext == 2*KMOS_NR_DETECTORS) &&
00693                        (desc_dark.ex_badpix == FALSE) &&
00694                        (desc_dark.fits_type == f2d_fits) &&
00695                        (desc_dark.frame_type == detector_frame),
00696                        CPL_ERROR_ILLEGAL_INPUT,
00697                        "MASTER_DARK isn't in the correct format!!!");
00698         nx = desc_dark.naxis1;
00699         ny = desc_dark.naxis2;
00700 
00701         // load desc for MASTER_FLAT and check
00702         KMO_TRY_EXIT_IF_NULL(
00703             frame = kmo_dfs_get_frame(frameset, MASTER_FLAT));
00704         desc_flat = kmo_identify_fits_header(cpl_frame_get_filename(frame));
00705         KMO_TRY_CHECK_ERROR_STATE_MSG("MASTER_FLAT frame doesn't seem to "
00706                                       "be in KMOS-format!");
00707         KMO_TRY_ASSURE((desc_flat.nr_ext % (2*KMOS_NR_DETECTORS) == 0) &&
00708                        (desc_flat.ex_badpix == FALSE) &&
00709                        (desc_flat.fits_type == f2d_fits) &&
00710                        (desc_flat.frame_type == detector_frame),
00711                        CPL_ERROR_ILLEGAL_INPUT,
00712                        "MASTER_FLAT isn't in the correct format!!!");
00713 
00714         // load desc for XCAL and check
00715         KMO_TRY_EXIT_IF_NULL(
00716             xcalFrame = kmo_dfs_get_frame(frameset, XCAL));
00717         desc_xcal = kmo_identify_fits_header(cpl_frame_get_filename(xcalFrame));
00718         KMO_TRY_CHECK_ERROR_STATE_MSG("XCAL frame doesn't seem to "
00719                                       "be in KMOS-format!");
00720         KMO_TRY_ASSURE((desc_xcal.nr_ext % KMOS_NR_DETECTORS == 0) &&
00721                        (desc_xcal.ex_badpix == FALSE) &&
00722                        (desc_xcal.fits_type == f2d_fits) &&
00723                        (desc_xcal.frame_type == detector_frame),
00724                        CPL_ERROR_ILLEGAL_INPUT,
00725                        "XCAL isn't in the correct format!!!");
00726         KMO_TRY_ASSURE((desc_xcal.naxis1 == nx) &&
00727                        (desc_xcal.naxis2 == ny),
00728                        CPL_ERROR_ILLEGAL_INPUT,
00729                        "MASTER_DARK and XCAL frame haven't same dimensions! "
00730                        "(x,y): (%d,%d) vs (%d,%d)",
00731                        nx, ny, desc_xcal.naxis1, desc_xcal.naxis2);
00732 
00733         nr_devices = desc_xcal.nr_ext;
00734 
00735         // load desc for YCAL and check
00736         KMO_TRY_EXIT_IF_NULL(
00737             ycalFrame = kmo_dfs_get_frame(frameset, YCAL));
00738         desc_ycal = kmo_identify_fits_header(cpl_frame_get_filename(ycalFrame));
00739         KMO_TRY_CHECK_ERROR_STATE_MSG("YCAL frame doesn't seem to "
00740                                       "be in KMOS-format!");
00741         KMO_TRY_ASSURE((desc_ycal.nr_ext == desc_xcal.nr_ext) &&
00742                        (desc_ycal.ex_badpix == desc_xcal.ex_badpix) &&
00743                        (desc_ycal.fits_type == desc_xcal.fits_type) &&
00744                        (desc_ycal.frame_type == desc_xcal.frame_type),
00745                        CPL_ERROR_ILLEGAL_INPUT,
00746                        "YCAL isn't in the correct format!!!");
00747         KMO_TRY_ASSURE((desc_ycal.naxis1 == desc_xcal.naxis1) &&
00748                        (desc_ycal.naxis2 == desc_xcal.naxis2),
00749                        CPL_ERROR_ILLEGAL_INPUT,
00750                        "MASTER_DARK and YCAL frame haven't same dimensions! "
00751                        "(x,y): (%d,%d) vs (%d,%d)",
00752                        nx, ny, desc_ycal.naxis1, desc_ycal.naxis2);
00753 
00754         // load desc for LCAL and check
00755         KMO_TRY_EXIT_IF_NULL(
00756             lcalFrame = kmo_dfs_get_frame(frameset, LCAL));
00757         desc_lcal = kmo_identify_fits_header(cpl_frame_get_filename(lcalFrame));
00758         KMO_TRY_CHECK_ERROR_STATE_MSG("LCAL frame doesn't seem to "
00759                                       "be in KMOS-format!");
00760         KMO_TRY_ASSURE((desc_lcal.ex_badpix == desc_xcal.ex_badpix) &&
00761                        (desc_lcal.fits_type == desc_xcal.fits_type) &&
00762                        (desc_lcal.frame_type == desc_xcal.frame_type),
00763                        CPL_ERROR_ILLEGAL_INPUT,
00764                        "LCAL isn't in the correct format!!!");
00765         KMO_TRY_ASSURE((desc_lcal.naxis1 == desc_xcal.naxis1) &&
00766                        (desc_lcal.naxis2 == desc_xcal.naxis2),
00767                        CPL_ERROR_ILLEGAL_INPUT,
00768                        "MASTER_DARK and LCAL frame haven't same dimensions! "
00769                        "(x,y): (%d,%d) vs (%d,%d)",
00770                        nx, ny, desc_lcal.naxis1, desc_lcal.naxis2);
00771         KMO_TRY_EXIT_IF_NULL(
00772             tmp_header = kmo_dfs_load_primary_header(frameset, LCAL));
00773 
00774         // load desc for FLAT_SKY and check
00775         nr_devices = KMOS_NR_DETECTORS;
00776         KMO_TRY_EXIT_IF_NULL(
00777             frame = kmo_dfs_get_frame(frameset, FLAT_SKY));
00778 
00779         KMO_TRY_EXIT_IF_NULL(
00780             main_header = kmclipm_propertylist_load(
00781                                          cpl_frame_get_filename(frame), 0));
00782         rotangle = cpl_propertylist_get_double(main_header, ROTANGLE);
00783         KMO_TRY_CHECK_ERROR_STATE_MSG("Cannot retrieve ROTANGLE FITS keyword from sky frame!");
00784         kmclipm_strip_angle(&rotangle);
00785         cpl_propertylist_delete(main_header); main_header = NULL;
00786 
00787         cnt = 1;
00788         while (frame != NULL) {
00789             KMO_TRY_EXIT_IF_NULL(
00790                 main_header = kmclipm_propertylist_load(
00791                                              cpl_frame_get_filename(frame), 0));
00792 
00793             desc_sky = kmo_identify_fits_header(
00794                         cpl_frame_get_filename(frame));
00795             KMO_TRY_CHECK_ERROR_STATE_MSG("FLAT_SKY frame doesn't seem to "
00796                                           "be in KMOS-format!");
00797             KMO_TRY_ASSURE((desc_sky.nr_ext == 3) &&
00798                            (desc_sky.ex_badpix == FALSE) &&
00799                            (desc_sky.fits_type == raw_fits) &&
00800                            (desc_sky.frame_type == detector_frame),
00801                            CPL_ERROR_ILLEGAL_INPUT,
00802                            "FLAT_SKY isn't in the correct format!!!");
00803             KMO_TRY_ASSURE((desc_sky.naxis1 == nx) &&
00804                            (desc_sky.naxis2 == ny) &&
00805                            (desc_sky.nr_ext == nr_devices),
00806                            CPL_ERROR_ILLEGAL_INPUT,
00807                            "MASTER_DARK and FLAT_SKY (no. %d) frame haven't "
00808                            "same dimensions! (x,y): (%d,%d) vs (%d,%d)",
00809                            cnt, nx, ny, desc_flat.naxis1, desc_flat.naxis2);
00810             kmo_free_fits_desc(&desc_sky);
00811             kmo_init_fits_desc(&desc_sky);
00812 
00813             KMO_TRY_ASSURE(
00814                 (kmo_check_lamp(main_header, INS_LAMP1_ST) == FALSE) &&
00815                 (kmo_check_lamp(main_header, INS_LAMP2_ST) == FALSE) &&
00816                 (kmo_check_lamp(main_header, INS_LAMP3_ST) == FALSE) &&
00817                 (kmo_check_lamp(main_header, INS_LAMP4_ST) == FALSE),
00818                 CPL_ERROR_ILLEGAL_INPUT,
00819                 "All lamps must be switched off for the FLAT_SKY frames!");
00820 
00821             // assert that filters have correct IDs and that all detectors of
00822             // all input frames have the same filter set
00823             for (i = 1; i <= KMOS_NR_DETECTORS; i++) {
00824                 // ESO INS FILTi ID
00825                 KMO_TRY_EXIT_IF_NULL(
00826                     keyword = cpl_sprintf("%s%d%s", IFU_FILTID_PREFIX, i, IFU_FILTID_POSTFIX));
00827                 KMO_TRY_EXIT_IF_NULL(
00828                     filter_id = cpl_propertylist_get_string(main_header, keyword));
00829 
00830                 KMO_TRY_EXIT_IF_NULL(
00831                     filter_id_l = cpl_propertylist_get_string(tmp_header, keyword));
00832                 cpl_free(keyword); keyword = NULL;
00833 
00834                 KMO_TRY_ASSURE((strcmp(filter_id, "IZ") == 0) ||
00835                                (strcmp(filter_id, "YJ") == 0) ||
00836                                (strcmp(filter_id, "H") == 0) ||
00837                                (strcmp(filter_id, "K") == 0) ||
00838                                (strcmp(filter_id, "HK") == 0),
00839                                CPL_ERROR_ILLEGAL_INPUT,
00840                                "Filter ID in primary header must be either 'IZ', "
00841                                "'YJ', 'H', 'K' or " "'HK' !");
00842 
00843                 KMO_TRY_ASSURE(strcmp(filter_id, filter_id_l) == 0,
00844                                CPL_ERROR_ILLEGAL_INPUT,
00845                                "Filter IDs must be the same for FLAT_SKY frame"
00846                                " and lcal frame!"
00847                                "Detector No.: %d\n%s: %s\nLCAL: %s\n",
00848                                i, cpl_frame_get_filename(frame),
00849                                filter_id, filter_id_l);
00850 
00851                 // ESO INS GRATi ID
00852                 KMO_TRY_EXIT_IF_NULL(
00853                     keyword = cpl_sprintf("%s%d%s", IFU_GRATID_PREFIX, i, IFU_GRATID_POSTFIX));
00854                 KMO_TRY_EXIT_IF_NULL(
00855                     filter_id = cpl_propertylist_get_string(main_header, keyword));
00856 
00857                 KMO_TRY_EXIT_IF_NULL(
00858                     filter_id_l = cpl_propertylist_get_string(tmp_header, keyword));
00859                 cpl_free(keyword); keyword = NULL;
00860 
00861                 KMO_TRY_ASSURE((strcmp(filter_id, "IZ") == 0) ||
00862                                (strcmp(filter_id, "YJ") == 0) ||
00863                                (strcmp(filter_id, "H") == 0) ||
00864                                (strcmp(filter_id, "K") == 0) ||
00865                                (strcmp(filter_id, "HK") == 0),
00866                                CPL_ERROR_ILLEGAL_INPUT,
00867                                "Grating ID in primary header must be either "
00868                                "'IZ', 'YJ', 'H', 'K' or " "'HK' !");
00869 
00870                 KMO_TRY_ASSURE(strcmp(filter_id, filter_id_l) == 0,
00871                                CPL_ERROR_ILLEGAL_INPUT,
00872                                "Grating IDs must be the same for FLAT_SKY frame"
00873                                " and lcal frame!"
00874                                "Detector No.: %d\n%s: %s\nLCAL: %s\n",
00875                                i, cpl_frame_get_filename(frame),
00876                                filter_id, filter_id_l);
00877 
00878                 tmp_rotangle = cpl_propertylist_get_double(main_header, ROTANGLE);
00879                 KMO_TRY_CHECK_ERROR_STATE_MSG("Cannot retrieve ROTANGLE FITS keyword from sky frame!");
00880                 kmclipm_strip_angle(&tmp_rotangle);
00881                 KMO_TRY_ASSURE((abs(rotangle - tmp_rotangle) < 10.0) ||
00882                                (abs(rotangle - tmp_rotangle) > 360.-10.) ,
00883                         CPL_ERROR_ILLEGAL_INPUT,
00884                         "OCS ROT NAANGLE of sky flat frames differ too much: %f %f",
00885                         rotangle, tmp_rotangle);
00886             }
00887             cpl_propertylist_delete(main_header); main_header = NULL;
00888 
00889             // get next FLAT_SKY frame
00890             frame = kmo_dfs_get_frame(frameset, NULL);
00891             KMO_TRY_CHECK_ERROR_STATE();
00892             cnt++;
00893         }
00894 
00895         cpl_propertylist_delete(tmp_header); tmp_header = NULL;
00896 
00897         //
00898         // noise will be propagated when:
00899         // MASTER_DARK and MASTER_FLAT have noise extensions and if at least
00900         // 2 FLAT_SKY frames are provided.
00901         // Otherwise noise will be ignored.
00902         //
00903         if (desc_dark.ex_noise &&
00904             desc_flat.ex_noise &&
00905             (cpl_frameset_count_tags(frameset, FLAT_SKY) >= 2)) {
00906             process_noise = TRUE;
00907         }
00908 
00909         if (cpl_frameset_count_tags(frameset, FLAT_SKY) == 1) {
00910             cpl_msg_warning(cpl_func, "cmethod is changed to 'average' "
00911                             "since there is only one input frame! (The output "
00912                             "file won't have any noise extensions)");
00913 
00914             cmethod = "average";
00915         }
00916 
00917         //
00918         // Check whether 1st FLAT_SKY should be omitted
00919         //
00920         KMO_TRY_EXIT_IF_NULL(
00921             frameset_sky = cpl_frameset_new());
00922 
00923         if (add_all_sky) {
00924             // just add all FLAT_SKY frames without check
00925             frame = kmo_dfs_get_frame(frameset, FLAT_SKY);
00926             while (frame != NULL) {
00927                 KMO_TRY_EXIT_IF_ERROR(
00928                     cpl_frameset_insert(frameset_sky, cpl_frame_duplicate(frame)));
00929                 frame = kmo_dfs_get_frame(frameset, NULL);
00930             }
00931             cpl_msg_info("", "Add all FLAT_SKY without checking for acquisition frame.");
00932         } else {
00933             // check if 1st FLAT_SKY has different exposure time and whether to omit it
00934             KMO_TRY_EXIT_IF_NULL(
00935                 frame = kmo_dfs_get_frame(frameset, FLAT_SKY));
00936 
00937             if (cpl_frameset_count_tags(frameset, FLAT_SKY) == 1) {
00938                 // just one FLAT_SKY, always add
00939                 KMO_TRY_EXIT_IF_ERROR(
00940                     cpl_frameset_insert(frameset_sky, cpl_frame_duplicate(frame)));
00941                 KMO_TRY_CHECK_ERROR_STATE();
00942             } else {
00943                 // several FLAT_SKY frames, check exptime
00944 
00945                 // get exptime 1
00946                 KMO_TRY_EXIT_IF_NULL(
00947                     main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00948                 exptime1 = cpl_propertylist_get_double(main_header, EXPTIME);
00949                 KMO_TRY_CHECK_ERROR_STATE();
00950                 cpl_propertylist_delete(main_header); main_header = NULL;
00951 
00952                 // get exptime 2
00953                 frame = kmo_dfs_get_frame(frameset, NULL);
00954                 KMO_TRY_EXIT_IF_NULL(
00955                     main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00956                 exptime2 = cpl_propertylist_get_double(main_header, EXPTIME);
00957                 KMO_TRY_CHECK_ERROR_STATE();
00958                 cpl_propertylist_delete(main_header); main_header = NULL;
00959 
00960                 // loop remaining frames
00961                 same_exptime = TRUE;
00962                 frame = kmo_dfs_get_frame(frameset, NULL);
00963                 while (same_exptime && (frame != NULL)) {
00964                     KMO_TRY_EXIT_IF_NULL(
00965                         main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00966                     exptime = cpl_propertylist_get_double(main_header, EXPTIME);
00967                     KMO_TRY_CHECK_ERROR_STATE();
00968                     cpl_propertylist_delete(main_header); main_header = NULL;
00969                     if (fabs(exptime-exptime2) > 0.01) {
00970                         // not same
00971                         same_exptime = FALSE;
00972                     }
00973                     frame = kmo_dfs_get_frame(frameset, NULL);
00974                 }
00975 
00976                 if (same_exptime) {
00977                     // frame [2,n] have same exptime, add them
00978                     frame = kmo_dfs_get_frame(frameset, FLAT_SKY);
00979                     KMO_TRY_EXIT_IF_NULL(
00980                         main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00981                     exptime = cpl_propertylist_get_double(main_header, EXPTIME);
00982                     KMO_TRY_CHECK_ERROR_STATE();
00983                     cpl_propertylist_delete(main_header); main_header = NULL;
00984                     cpl_msg_info("", "Omit FLAT_SKY: %s with EXPTIME of %g sec (acquisition), other frame(s) have EXPTIME of %g sec", cpl_frame_get_filename(frame), exptime, exptime2);
00985                     frame = kmo_dfs_get_frame(frameset, NULL);
00986                     while (frame != NULL) {
00987                         KMO_TRY_EXIT_IF_ERROR(
00988                             cpl_frameset_insert(frameset_sky, cpl_frame_duplicate(frame)));
00989                         frame = kmo_dfs_get_frame(frameset, NULL);
00990                     }
00991                     if (fabs(exptime1-exptime2) < 0.01) {
00992                         cpl_msg_warning("", "The 1st FLAT_SKY has the same exposure time as the following ones. "
00993                                             "It has anyway been omitted since we assume it is an acquisition frame. "
00994                                             "If you want to add it anyway call this recipe with the --add-all parameter");
00995                     }
00996                 } else {
00997                     cpl_msg_error("", "The exposure times of the FLAT_SKY frames don't match!");
00998                     cpl_msg_error("", "We assume that the 1st frame is an acquisition frame and would be omitted.");
00999                     cpl_msg_error("", "The following frames should have the same exposure time if they originate from the same template.");
01000                     cpl_msg_error("", "If you want to reduce them anyway call this recipe with the --add-all parameter");
01001                     frame = kmo_dfs_get_frame(frameset, FLAT_SKY);
01002                     while (frame != NULL) {
01003                         KMO_TRY_EXIT_IF_NULL(
01004                             main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
01005                         exptime = cpl_propertylist_get_double(main_header, EXPTIME);
01006                         KMO_TRY_CHECK_ERROR_STATE();
01007                         cpl_propertylist_delete(main_header); main_header = NULL;
01008                         cpl_msg_error("", "FLAT_SKY: %s, EXPTIME: %g", cpl_frame_get_filename(frame), exptime);
01009                         frame = kmo_dfs_get_frame(frameset, NULL);
01010                     }
01011                     cpl_error_set(cpl_func, CPL_ERROR_ILLEGAL_INPUT);
01012                     KMO_TRY_CHECK_ERROR_STATE();
01013                 }
01014             }
01015         }
01016 
01017         KMO_TRY_EXIT_IF_NULL(
01018             frame = kmo_dfs_get_frame(frameset_sky, FLAT_SKY));
01019         KMO_TRY_EXIT_IF_NULL(
01020             main_header = kmo_dfs_load_primary_header(frameset_sky, FLAT_SKY));
01021         KMO_TRY_EXIT_IF_NULL(
01022             keyword = cpl_sprintf("%s%d%s", IFU_GRATID_PREFIX, 1, IFU_GRATID_POSTFIX));
01023         KMO_TRY_EXIT_IF_NULL(
01024             filter = cpl_sprintf("%s", cpl_propertylist_get_string(main_header, keyword)));
01025         cpl_free(keyword); keyword = NULL;
01026 
01027         //
01028         // set default band-specific ranges for collapsing
01029         //
01030         if (ranges == NULL) {
01031             if (strcmp(filter, "IZ") == 0) {
01032                 ranges_txt = "0.81,1.05";
01033             } else if (strcmp(filter, "YJ") == 0) {
01034                 ranges_txt = "1.025,1.3";
01035             } else if (strcmp(filter, "H") == 0) {
01036                 ranges_txt = "1.5,1.7";
01037             } else if (strcmp(filter, "K") == 0) {
01038                 ranges_txt = "2.1,2.35";
01039             } else if (strcmp(filter, "HK") == 0) {
01040                 ranges_txt = "1.5,1.7;2.1,2.35";
01041 //                ranges_txt = "1.5,1.7";
01042             } else {
01043                 KMO_TRY_ASSURE(1 == 0,
01044                                CPL_ERROR_ILLEGAL_INPUT,
01045                                "We really shouldn't get here...");
01046             }
01047             cpl_msg_info("", "Spectral range to collapse has been set to [%s] um for this band.", ranges_txt);
01048             ranges = kmo_identify_ranges(ranges_txt);
01049             KMO_TRY_CHECK_ERROR_STATE();
01050         }
01051 
01052         // setup grid definition, wavelength start and end points will be set
01053         // in the detector loop
01054         KMO_TRY_EXIT_IF_ERROR(
01055             kmclipm_setup_grid(&gd, method, neighborhoodRange, pix_scale, 0.));
01056 
01057         // create filename for LUT
01058         KMO_TRY_EXIT_IF_NULL(
01059             fn_lut = cpl_sprintf("%s%s", "lut", suffix));
01060 
01061         // extract bounds
01062         KMO_TRY_EXIT_IF_NULL(
01063             tmp_header = kmo_dfs_load_primary_header(frameset, XCAL));
01064         KMO_TRY_EXIT_IF_NULL(
01065             bounds = kmclipm_extract_bounds(tmp_header));
01066         cpl_propertylist_delete(tmp_header); tmp_header = NULL;
01067 
01068         // get timestamps of xcal, ycal & lcal
01069         KMO_TRY_EXIT_IF_NULL(
01070             calTimestamp = kmo_get_timestamps(xcalFrame, ycalFrame, lcalFrame));
01071 
01072         // create arrays to hold reconstructed data and noise cubes and
01073         // their headers
01074         KMO_TRY_EXIT_IF_NULL(
01075             stored_data_cubes = (cpl_imagelist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01076                                                             sizeof(cpl_imagelist*)));
01077         KMO_TRY_EXIT_IF_NULL(
01078             stored_noise_cubes = (cpl_imagelist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01079                                                              sizeof(cpl_imagelist*)));
01080         KMO_TRY_EXIT_IF_NULL(
01081             stored_data_images = (cpl_image**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01082                                                          sizeof(cpl_image*)));
01083         KMO_TRY_EXIT_IF_NULL(
01084             stored_noise_images = (cpl_image**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01085                                                           sizeof(cpl_image*)));
01086         KMO_TRY_EXIT_IF_NULL(
01087             stored_sub_data_headers = (cpl_propertylist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01088                                                                      sizeof(cpl_propertylist*)));
01089         KMO_TRY_EXIT_IF_NULL(
01090             stored_sub_noise_headers = (cpl_propertylist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01091                                                                       sizeof(cpl_propertylist*)));
01092         KMO_TRY_EXIT_IF_NULL(
01093             edge_table_sky = (cpl_table***)cpl_calloc(KMOS_NR_DETECTORS,
01094                                                       sizeof(cpl_table**)));
01095         KMO_TRY_EXIT_IF_NULL(
01096             edge_table_flat = (cpl_table**)cpl_calloc(KMOS_IFUS_PER_DETECTOR,
01097                                                       sizeof(cpl_table*)));
01098         KMO_TRY_EXIT_IF_NULL(
01099             calAngles = cpl_vector_new(3));
01100 
01101         //
01102         // loop through all detectors
01103         //
01104         for (det_nr = 1; det_nr <= nr_devices; det_nr++) {
01105             cpl_msg_info("","Processing detector No. %d", det_nr);
01106 
01107             KMO_TRY_EXIT_IF_NULL(
01108                 detector_in = cpl_imagelist_new());
01109 
01110             // load data of det_nr of all FLAT_SKY frames into an imagelist
01111             KMO_TRY_EXIT_IF_NULL(
01112                 img_in = kmo_dfs_load_image(frameset_sky, FLAT_SKY, det_nr, FALSE, TRUE, NULL));
01113 
01114             cnt = 0;
01115             while (img_in != NULL) {
01116                 cpl_imagelist_set(detector_in, img_in, cnt);
01117                 KMO_TRY_CHECK_ERROR_STATE();
01118 
01119                 /* load same extension of next FLAT_SKY frame*/
01120                 img_in = kmo_dfs_load_image(frameset_sky, NULL, det_nr, FALSE, TRUE, NULL);
01121                 KMO_TRY_CHECK_ERROR_STATE();
01122 
01123                 cnt++;
01124             }
01125 
01126             //
01127             // process imagelist
01128             //
01129 
01130             // combine imagelist (data only) and create noise (stdev of data)
01131             cpl_msg_info("","Combining frames...");
01132             if (process_noise) {
01133                 KMO_TRY_EXIT_IF_ERROR(
01134                     kmclipm_combine_frames(detector_in,
01135                                            NULL,
01136                                            NULL,
01137                                            cmethod,
01138                                            cpos_rej,
01139                                            cneg_rej,
01140                                            citer,
01141                                            cmax,
01142                                            cmin,
01143                                            &combined_data,
01144                                            &combined_noise,
01145                                            -1.0));
01146             } else {
01147                 KMO_TRY_EXIT_IF_ERROR(
01148                     kmclipm_combine_frames(detector_in,
01149                                            NULL,
01150                                            NULL,
01151                                            cmethod,
01152                                            cpos_rej,
01153                                            cneg_rej,
01154                                            citer,
01155                                            cmax,
01156                                            cmin,
01157                                            &combined_data,
01158                                            NULL,
01159                                            -1.0));
01160             }
01161 
01162             if (kmclipm_omit_warning_one_slice > 10) {
01163 // AA: commmented this out: Too unclear for the user, no benefit to know about this number
01164 //                cpl_msg_warning(cpl_func, "Previous warning (number of "
01165 //                                          "identified slices) occured %d times.",
01166 //                                kmclipm_omit_warning_one_slice);
01167                 kmclipm_omit_warning_one_slice = FALSE;
01168             }
01169 
01170             cpl_imagelist_delete(detector_in); detector_in = NULL;
01171 
01172             // load calibration files
01173             KMO_TRY_EXIT_IF_NULL(
01174                 xcal = kmo_dfs_load_cal_image(frameset, XCAL, det_nr, FALSE, rotangle,
01175                                               FALSE, NULL, &rotangle_found, -1, 0, 0));
01176 
01177             KMO_TRY_EXIT_IF_ERROR(
01178                 cpl_vector_set(calAngles, 0, rotangle_found));
01179             KMO_TRY_EXIT_IF_NULL(
01180                 ycal = kmo_dfs_load_cal_image(frameset, YCAL, det_nr, FALSE, rotangle,
01181                                               FALSE, NULL, &rotangle_found, -1, 0, 0));
01182             KMO_TRY_EXIT_IF_ERROR(
01183                 cpl_vector_set(calAngles, 1, rotangle_found));
01184             KMO_TRY_EXIT_IF_NULL(
01185                 lcal = kmo_dfs_load_cal_image(frameset, LCAL, det_nr, FALSE, rotangle,
01186                                               FALSE, NULL, &rotangle_found, -1, 0, 0));
01187             KMO_TRY_EXIT_IF_ERROR(
01188                 cpl_vector_set(calAngles, 2, rotangle_found));
01189 
01190             // load bad pixel mask from XCAL and set NaNs to 0 and all other values to 1
01191             KMO_TRY_EXIT_IF_NULL(
01192                 bad_pix_mask = cpl_image_duplicate(xcal));
01193 
01194             KMO_TRY_EXIT_IF_NULL(
01195                 pbad_pix_mask = cpl_image_get_data_float(bad_pix_mask));
01196             for (x = 0; x < nx; x++) {
01197                 for (y = 0; y < ny; y++) {
01198                     if (isnan(pbad_pix_mask[x+nx*y])) {
01199                         pbad_pix_mask[x+nx*y] = 0.;
01200                     } else {
01201                         pbad_pix_mask[x+nx*y] = 1.;
01202                     }
01203                 }
01204             }
01205             KMO_TRY_CHECK_ERROR_STATE();
01206 
01207             //
01208             // calculate SKYFLAT_EDGE
01209             //
01210             if (has_flat_edge) {
01211                 // get edge-edgepars from FLAT_SKY
01212                 KMO_TRY_EXIT_IF_ERROR(
01213                     kmo_calc_edgepars(combined_data,
01214                                       unused_ifus_after[det_nr-1],
01215                                       bad_pix_mask,
01216                                       det_nr,
01217                                       &slitlet_ids,
01218                                       &edgepars));
01219                 KMO_TRY_CHECK_ERROR_STATE();
01220 
01221                 // copy edgepars to table for saving later on
01222                 KMO_TRY_EXIT_IF_NULL(
01223                     edge_table_sky[det_nr-1] = kmo_edgepars_to_table(slitlet_ids, edgepars));
01224 
01225                 if (edgepars != NULL) {
01226                     for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01227                         cpl_matrix_delete(edgepars[i]); edgepars[i] = NULL;
01228                     }
01229                     cpl_free(edgepars); edgepars = NULL;
01230                 }
01231                 if (slitlet_ids != NULL) {
01232                     for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01233                         cpl_vector_delete(slitlet_ids[i]); slitlet_ids[i] = NULL;
01234                     }
01235                     cpl_free(slitlet_ids); slitlet_ids = NULL;
01236                 }
01237                 KMO_TRY_CHECK_ERROR_STATE();
01238 
01239                 //
01240                 // correlate FLAT_EDGE and SKYFLAT_EDGE
01241                 //
01242 
01243                 // load flat_edge from MASTER_FLAT
01244                 KMO_TRY_EXIT_IF_NULL(
01245                     frame = kmo_dfs_get_frame(frameset, FLAT_EDGE));
01246                 for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01247                     ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01248 
01249                     KMO_TRY_EXIT_IF_NULL(
01250                         punused_ifus = cpl_array_get_data_int_const(unused_ifus_after[det_nr-1]));
01251                     if (punused_ifus[j] == 0) {
01252                         KMO_TRY_EXIT_IF_NULL(
01253                             edge_table_flat[j] = kmclipm_cal_table_load(cpl_frame_get_filename(frame),
01254                                                                         ifu_nr, rotangle, 0, &tmp_rotangle));
01255                     }
01256                 }
01257 
01258                 //
01259                 // calculate shift value
01260                 //
01261 
01262                 KMO_TRY_EXIT_IF_NULL(
01263                     shift_vec = cpl_vector_new(KMOS_IFUS_PER_DETECTOR));
01264                 KMO_TRY_EXIT_IF_NULL(
01265                     edge_vec = cpl_vector_new(2*KMOS_SLITLET_X));
01266 
01267                 // get shift values for each IFU by comparing all edge parameters,
01268                 // rejecting and applying median
01269                 int row = 1024; // middle of frame
01270                 for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01271                     ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01272                     for (edgeNr = 0; edgeNr < 2*KMOS_SLITLET_X; edgeNr++) {
01273                         if (edge_table_flat[j] != NULL) {
01274                             double flatval = kmo_calc_fitted_slitlet_edge(edge_table_flat[j], edgeNr, row);
01275                             double skyval  = kmo_calc_fitted_slitlet_edge(edge_table_sky[det_nr-1][j], edgeNr, row);
01276                             cpl_vector_set(edge_vec, edgeNr, flatval-skyval);
01277                         }
01278                     }
01279 
01280                     // reject deviating edge-differences
01281                     kmclipm_vector *kv = NULL;
01282                     KMO_TRY_EXIT_IF_NULL(
01283                         kv = kmclipm_vector_create(cpl_vector_duplicate(edge_vec)));
01284                     kmclipm_reject_deviant(kv, 3, 3, NULL, NULL);
01285 
01286                     // set shift value for each IFU
01287                     cpl_vector_set(shift_vec, j, kmclipm_vector_get_median(kv, KMCLIPM_ARITHMETIC));
01288                     kmclipm_vector_delete(kv); kv = NULL;
01289                 }
01290                 cpl_vector_delete(edge_vec); edge_vec = NULL;
01291                 KMO_TRY_CHECK_ERROR_STATE();
01292 
01293                 for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01294                     cpl_table_delete(edge_table_flat[j]);
01295                     edge_table_flat[j] = NULL;
01296                 }
01297 
01298                 // take median of all IFU-shift-values
01299                 double shift_val = -cpl_vector_get_median(shift_vec);
01300                 cpl_vector_delete(shift_vec); shift_vec = NULL;
01301 
01302                 cpl_msg_info("", "Shift detector %d by %g pixels.", det_nr, shift_val);
01303 
01304                 int     xdim                = cpl_image_get_size_x(combined_data),
01305                         ydim                = cpl_image_get_size_y(combined_data);
01306                 double  *array_in           = cpl_calloc(xdim, sizeof(double)),
01307                         *array_out          = NULL;
01308                 float   *pcombined_data     = cpl_image_get_data_float(combined_data),
01309                         *pcombined_noise    = NULL;
01310     //            float   *tmpArray           = cpl_calloc(xdim, sizeof(float));
01311                 if (process_noise) {
01312                     pcombined_noise = cpl_image_get_data_float(combined_noise);
01313                 }
01314 
01315                 for (iy = 0; iy < ydim; iy++) {
01316                     // cubic spline
01317                     for (ix = 0; ix < xdim; ix++) {
01318                         array_in[ix] = pcombined_data[ix+iy*xdim];
01319                     }
01320                     array_out = cubicspline_reg_reg(xdim, 0., 1., array_in,
01321                                                     xdim, shift_val, 1.0,
01322                                                     NATURAL);
01323                     for (ix = 0; ix < xdim; ix++) {
01324                       pcombined_data[ix+iy*xdim] = array_out[ix];
01325                     }
01326                     cpl_free(array_out);
01327 
01328     //                // linear
01329     //                for (ix = 1; ix < xdim; ix++) {
01330     //                    tmpArray[ix-1] = (pcombined_data[ix+iy*xdim]-pcombined_data[(ix-1)+iy*xdim])*shift_val +
01331     //                                     pcombined_data[(ix-1)+iy*xdim];
01332     //                }
01333     //                for (ix = 1; ix < xdim; ix++) {
01334     //                    pcombined_data[ix+iy*xdim] = tmpArray[ix];
01335     //                }
01336 
01337                     if (process_noise) {
01338                         // cubic spline
01339                         for (ix = 0; ix < xdim; ix++) {
01340                             array_in[ix] = pcombined_noise[ix+iy*xdim];
01341                         }
01342                         array_out = cubicspline_reg_reg(xdim, 0., 1., array_in,
01343                                                         xdim, shift_val, 1.0,
01344                                                         NATURAL);
01345                         for (ix = 0; ix < xdim; ix++) {
01346                           pcombined_noise[ix+iy*xdim] = array_out[ix];
01347                         }
01348                         cpl_free(array_out);
01349 
01350     //                    // linear
01351     //                    for (ix = 1; ix < xdim; ix++) {
01352     //                        tmpArray[ix-1] = (pcombined_noise[ix+iy*xdim]-pcombined_noise[(ix-1)+iy*xdim])*shift_val +
01353     //                                         pcombined_noise[(ix-1)+iy*xdim];
01354     //                    }
01355     //                    for (ix = 1; ix < xdim; ix++) {
01356     //                        pcombined_noise[ix+iy*xdim] = tmpArray[ix];
01357     //                    }
01358                     }
01359                 }
01360                 cpl_free(array_in); array_in = NULL;
01361             }
01362             //
01363             // reconstruct
01364             //
01365             // load MASTER_DARK and MASTER_FLAT
01366             KMO_TRY_EXIT_IF_NULL(
01367                 img_dark = kmo_dfs_load_image(frameset, MASTER_DARK,
01368                                               det_nr, FALSE, FALSE, NULL));
01369 
01370             if (process_noise) {
01371                 KMO_TRY_EXIT_IF_NULL(
01372                     img_dark_noise = kmo_dfs_load_image(frameset, MASTER_DARK,
01373                                                         det_nr, TRUE, FALSE, NULL));
01374             }
01375 
01376             KMO_TRY_EXIT_IF_NULL(
01377                 img_flat = kmo_dfs_load_cal_image(frameset, MASTER_FLAT, det_nr, FALSE,
01378                                                   rotangle, FALSE, NULL,
01379                                                   &rotangle_found, -1, 0, 0));
01380 
01381             if (process_noise) {
01382                 KMO_TRY_EXIT_IF_NULL(
01383                     img_flat_noise = kmo_dfs_load_cal_image(frameset, MASTER_FLAT, det_nr, TRUE,
01384                                                             rotangle, FALSE, NULL,
01385                                                             &rotangle_found, -1, 0, 0));
01386             }
01387 
01388             // ESO INS FILTi ID
01389             KMO_TRY_EXIT_IF_NULL(
01390                 keyword = cpl_sprintf("%s%d%s", IFU_FILTID_PREFIX, det_nr,
01391                                       IFU_FILTID_POSTFIX));
01392             KMO_TRY_EXIT_IF_NULL(
01393                 filter_id = cpl_propertylist_get_string(main_header, keyword));
01394             cpl_free(keyword); keyword = NULL;
01395 
01396             KMO_TRY_EXIT_IF_NULL(
01397                 band_table = kmo_dfs_load_table(frameset, WAVE_BAND, 1, 0));
01398             KMO_TRY_EXIT_IF_ERROR(
01399                 kmclipm_setup_grid_band_lcal(&gd, filter_id, band_table));
01400             cpl_table_delete(band_table); band_table = NULL;
01401 
01402             cpl_msg_info("","Reconstructing cubes...");
01403             for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01404                 // update sub-header
01405                 ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01406 
01407                 // load raw image and sub-header
01408                 KMO_TRY_EXIT_IF_NULL(
01409                     sub_header = kmo_dfs_load_sub_header(frameset_sky, FLAT_SKY,
01410                                                          det_nr, FALSE));
01411 
01412                 KMO_TRY_EXIT_IF_NULL(
01413                     punused_ifus = cpl_array_get_data_int_const(
01414                                                   unused_ifus_after[det_nr-1]));
01415 
01416                 // check if IFU is valid according to main header keywords &
01417                 // calibration files
01418                 KMO_TRY_EXIT_IF_NULL(
01419                     keyword = cpl_sprintf("%s%d%s", IFU_VALID_PREFIX, ifu_nr,
01420                                           IFU_VALID_POSTFIX));
01421                 KMO_TRY_CHECK_ERROR_STATE();
01422                 ranges_txt = cpl_propertylist_get_string(main_header, keyword);
01423                 cpl_free(keyword); keyword = NULL;
01424 
01425                 if ((cpl_error_get_code() == CPL_ERROR_DATA_NOT_FOUND) &&
01426                     (bounds[2*(ifu_nr-1)] != -1) &&
01427                     (bounds[2*(ifu_nr-1)+1] != -1) &&
01428                     (punused_ifus[j] == 0))
01429                 {
01430                     // IFU is valid
01431                     cpl_error_reset();
01432 
01433                     // calculate WCS
01434                     KMO_TRY_EXIT_IF_ERROR(
01435                         kmo_calc_wcs_gd(main_header, sub_header, ifu_nr, gd));
01436 
01437                     // reconstruct data
01438                     KMO_TRY_EXIT_IF_ERROR(
01439                         kmo_reconstruct_sci_image(ifu_nr,
01440                                                 bounds[2*(ifu_nr-1)],
01441                                                 bounds[2*(ifu_nr-1)+1],
01442                                                 combined_data,
01443                                                 combined_noise,
01444                                                 img_dark,
01445                                                 img_dark_noise,
01446                                                 img_flat,
01447                                                 img_flat_noise,
01448                                                 xcal,
01449                                                 ycal,
01450                                                 lcal,
01451                                                 &gd,
01452                                                 calTimestamp,
01453                                                 calAngles,
01454                                                 fn_lut,
01455                                                 &cube_data,
01456                                                 &cube_noise,
01457                                                 flux,
01458                                                 background,
01459                                                 NULL,
01460                                                 NULL,
01461                                                 NULL));
01462                     KMO_TRY_CHECK_ERROR_STATE();
01463                 } else {
01464                     // IFU is invalid
01465                     cpl_error_reset();
01466                 } // if ((cpl_error_get_code() == CPL_ERROR_DATA_NOT_FOUND) ...
01467 
01468                 // save output
01469                 KMO_TRY_EXIT_IF_NULL(
01470                     extname = kmo_extname_creator(ifu_frame, ifu_nr, EXT_DATA));
01471 
01472                 KMO_TRY_EXIT_IF_ERROR(
01473                     kmclipm_update_property_string(sub_header, EXTNAME,
01474                                                    extname,
01475                                                    "FITS extension name"));
01476 
01477                 cpl_free(extname); extname = NULL;
01478 
01479                 // store cube and sub header into array for later
01480                 stored_data_cubes[ifu_nr - 1] = cube_data;
01481                 stored_sub_data_headers[ifu_nr - 1] = sub_header;
01482 
01483                 if (process_noise) {
01484                     KMO_TRY_EXIT_IF_NULL(
01485                         sub_header = cpl_propertylist_duplicate(
01486                                            stored_sub_data_headers[ifu_nr - 1]));
01487                     KMO_TRY_EXIT_IF_NULL(
01488                         extname = kmo_extname_creator(ifu_frame, ifu_nr,
01489                                                       EXT_NOISE));
01490 
01491                     KMO_TRY_EXIT_IF_ERROR(
01492                         kmclipm_update_property_string(sub_header,
01493                                                 EXTNAME,
01494                                                 extname,
01495                                                 "FITS extension name"));
01496 
01497                     cpl_free(extname); extname = NULL;
01498 
01499                     stored_noise_cubes[ifu_nr - 1] = cube_noise;
01500                     stored_sub_noise_headers[ifu_nr - 1] = sub_header;
01501                 }
01502                 cpl_image_delete(data_ifu); data_ifu = NULL;
01503                 cpl_image_delete(noise_ifu); noise_ifu = NULL;
01504                 cube_data = NULL;
01505                 cube_noise = NULL;
01506             } // for j IFUs
01507 
01508             // free memory
01509             cpl_image_delete(combined_data); combined_data = NULL;
01510             cpl_image_delete(combined_noise); combined_noise = NULL;
01511             cpl_image_delete(xcal); xcal = NULL;
01512             cpl_image_delete(ycal); ycal = NULL;
01513             cpl_image_delete(lcal); lcal = NULL;
01514             cpl_image_delete(img_dark); img_dark = NULL;
01515             cpl_image_delete(img_flat); img_flat = NULL;
01516             cpl_image_delete(bad_pix_mask); bad_pix_mask = NULL;
01517             if (process_noise) {
01518                 cpl_image_delete(img_dark_noise); img_dark_noise = NULL;
01519                 cpl_image_delete(img_flat_noise); img_flat_noise = NULL;
01520             }
01521         } // for nr_devices
01522 
01523         cpl_free(edge_table_flat); edge_table_flat = NULL;
01524 
01525         // collapse cubes using rejection
01526         cpl_msg_info("","Collapsing cubes...");
01527         for (det_nr = 1; det_nr <= nr_devices; det_nr++) {
01528             for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01529                 ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01530 
01531                 KMO_TRY_EXIT_IF_NULL(
01532                     punused_ifus = cpl_array_get_data_int_const(
01533                                                   unused_ifus_after[det_nr-1]));
01534                 if (punused_ifus[j] == 0) {
01535                     if (stored_sub_data_headers[ifu_nr-1] != NULL) {
01536                         // IFU is valid
01537                         ifu_crpix = cpl_propertylist_get_double(stored_sub_data_headers[ifu_nr-1],
01538                                                                 CRPIX3);
01539                         KMO_TRY_CHECK_ERROR_STATE_MSG(
01540                                        "CRPIX3 keyword in FITS-header is missing!");
01541 
01542                         ifu_crval = cpl_propertylist_get_double(stored_sub_data_headers[ifu_nr-1],
01543                                                                 CRVAL3);
01544                         KMO_TRY_CHECK_ERROR_STATE_MSG(
01545                                        "CRVAL3 keyword in FITS-header is missing!");
01546 
01547                         ifu_cdelt = cpl_propertylist_get_double(stored_sub_data_headers[ifu_nr-1],
01548                                                                 CDELT3);
01549                         KMO_TRY_CHECK_ERROR_STATE_MSG(
01550                                        "CDELT3 keyword in FITS-header is missing!");
01551 
01552                         KMO_TRY_EXIT_IF_NULL(
01553                             identified_slices = kmo_identify_slices(ranges,
01554                                                                     ifu_crpix,
01555                                                                     ifu_crval,
01556                                                                     ifu_cdelt,
01557                                                                     gd.l.dim));
01558                     }/* else {
01559                         KMO_TRY_EXIT_IF_NULL(
01560                             identified_slices = cpl_vector_new(gd.l.dim));
01561                         cpl_vector_fill(identified_slices, 1.);
01562                     }*/
01563 
01564                     if (stored_data_cubes[ifu_nr-1] != NULL) {
01565                         KMO_TRY_EXIT_IF_ERROR(
01566                             kmclipm_make_image(stored_data_cubes[ifu_nr-1],
01567                                                stored_noise_cubes[ifu_nr-1],
01568                                                &stored_data_images[ifu_nr-1],
01569                                                &stored_noise_images[ifu_nr-1],
01570                                                identified_slices,
01571                                                cmethod, cpos_rej, cneg_rej,
01572                                                citer, cmax, cmin));
01573                     }
01574                     cpl_vector_delete(identified_slices); identified_slices = NULL;
01575                 } else {
01576                     // IFU is invalid
01577                 }
01578             }
01579         }
01580 
01581         // normalise all IFUs of a detector as a group.
01582         // Calculate mean of each IFU, add up and divide by number of successful
01583         // averaged IFUs.
01584         // Then divide all valid IFUs with mean value
01585         int jj;
01586         for (jj = 0; jj < nr_devices; jj++) {
01587             cnt = 0;
01588             mean_data = 0;
01589             for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01590                 ifu_nr = jj*KMOS_IFUS_PER_DETECTOR + i;
01591                 if (stored_data_images[ifu_nr] != NULL) {
01592                     KMO_TRY_ASSURE(cpl_image_count_rejected(stored_data_images[ifu_nr]) <
01593                                    cpl_image_get_size_x(stored_data_images[ifu_nr])*
01594                                    cpl_image_get_size_y(stored_data_images[ifu_nr]),
01595                                    CPL_ERROR_ILLEGAL_INPUT,
01596                                    "The collapsed, dark-subtracted image contains "
01597                                    "only invalid values! Probably the provided "
01598                                    "FLAT_SKY frames are exactly the same as the "
01599                                    "frames used for MASTER_DARK calculation.");
01600 
01601                     mean_data += cpl_image_get_mean(stored_data_images[ifu_nr]);
01602                     KMO_TRY_CHECK_ERROR_STATE();
01603                     cnt++;
01604                 }
01605 
01606             }
01607             mean_data /= cnt;
01608 
01609             if (mean_data != 0.0) {
01610                 for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01611                     ifu_nr = jj*KMOS_IFUS_PER_DETECTOR + i;
01612                     if (stored_data_images[ifu_nr] != NULL) {
01613                         KMO_TRY_EXIT_IF_ERROR(
01614                             cpl_image_divide_scalar(stored_data_images[ifu_nr],
01615                                                     mean_data));
01616                     }
01617                 }
01618             } else {
01619                 cpl_msg_warning(cpl_func, "Data couldn't be normalised "
01620                                           "(mean = 0.0)!");
01621             }
01622 
01623             if (process_noise) {
01624                 if (mean_data != 0.0) {
01625                     for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01626                         ifu_nr = jj*KMOS_IFUS_PER_DETECTOR + i;
01627                         if (stored_noise_images[ifu_nr] != NULL) {
01628                             KMO_TRY_EXIT_IF_ERROR(
01629                                 cpl_image_divide_scalar(stored_noise_images[ifu_nr],
01630                                                         mean_data));
01631                         }
01632                     }
01633                 } else {
01634                     cpl_msg_warning(cpl_func, "Noise couldn't be normalised "
01635                                               "(mean = 0.0)!");
01636                 }
01637             }
01638         } // end for(jj)
01639 
01640         // calculate qc parameters on normalised data
01641         qc_spat_unif = 0.0;
01642         cnt = 0;
01643         for (i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01644             if (stored_data_images[i] != NULL) {
01645                 tmp_mean = cpl_image_get_mean(stored_data_images[i]);
01646                 tmp_stdev = cpl_image_get_stdev (stored_data_images[i]);
01647 
01648                 qc_spat_unif += pow(tmp_mean-1, 2);
01649                 if (fabs(tmp_mean) > qc_max_dev) {
01650                     qc_max_dev = tmp_mean-1;
01651                     qc_max_dev_id = i+1;
01652                 }
01653                 if (fabs(tmp_stdev) > qc_max_nonunif) {
01654                     qc_max_nonunif = tmp_stdev;
01655                     qc_max_nonunif_id = i+1;
01656                 }
01657                 KMO_TRY_CHECK_ERROR_STATE();
01658                 cnt++;
01659             }
01660         }
01661         qc_spat_unif = sqrt(qc_spat_unif / cnt);
01662 
01663         //
01664         // save data
01665         //
01666 
01667         // update which IFUs are not used
01668         kmo_print_unused_ifus(unused_ifus_after, TRUE);
01669 
01670         KMO_TRY_EXIT_IF_ERROR(
01671             kmo_set_unused_ifus(unused_ifus_after, main_header,
01672                                 "kmo_illumination"));
01673 
01674         cpl_msg_info("","Saving data...");
01675 
01676         KMO_TRY_EXIT_IF_ERROR(
01677             kmclipm_update_property_double(main_header, QC_SPAT_UNIF, qc_spat_unif,
01678                                            "[adu] uniformity of illumination correction"));
01679         KMO_TRY_EXIT_IF_ERROR(
01680             kmclipm_update_property_double(main_header, QC_SPAT_MAX_DEV, qc_max_dev,
01681                                            "[adu] max. deviation from unity"));
01682         KMO_TRY_EXIT_IF_ERROR(
01683             kmclipm_update_property_int(main_header, QC_SPAT_MAX_DEV_ID, qc_max_dev_id,
01684                                         "[] IFU ID with max. dev. from unity"));
01685         KMO_TRY_EXIT_IF_ERROR(
01686             kmclipm_update_property_double(main_header, QC_SPAT_MAX_NONUNIF, qc_max_nonunif,
01687                                            "[adu] max. stdev of illumination corr."));
01688         KMO_TRY_EXIT_IF_ERROR(
01689             kmclipm_update_property_int(main_header, QC_SPAT_MAX_NONUNIF_ID, qc_max_nonunif_id,
01690                                         "[] IFU ID with max. stdev in illum. corr."));
01691 
01692         if (!suppress_extension) {
01693             KMO_TRY_EXIT_IF_NULL(
01694                 fn_suffix = cpl_sprintf("%s", suffix));
01695         } else {
01696             KMO_TRY_EXIT_IF_NULL(
01697                 fn_suffix = cpl_sprintf("%s", ""));
01698         }
01699         KMO_TRY_EXIT_IF_ERROR(
01700             kmo_dfs_save_main_header(frameset, ILLUM_CORR, fn_suffix, frame,
01701                                      main_header, parlist, cpl_func));
01702 
01703         if (has_flat_edge) {
01704             KMO_TRY_EXIT_IF_ERROR(
01705                 kmo_dfs_save_main_header(frameset, SKYFLAT_EDGE, fn_suffix, frame,
01706                                          main_header, parlist, cpl_func));
01707         }
01708 
01709         for (i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01710             KMO_TRY_EXIT_IF_ERROR(
01711                 kmo_dfs_save_image(stored_data_images[i], ILLUM_CORR, fn_suffix,
01712                                    stored_sub_data_headers[i], 0./0.));
01713 
01714             if (process_noise) {
01715                 KMO_TRY_EXIT_IF_ERROR(
01716                     kmo_dfs_save_image(stored_noise_images[i], ILLUM_CORR,
01717                                        fn_suffix, stored_sub_noise_headers[i], 0./0.));
01718             }
01719         }
01720 
01721         for (det_nr = 1; det_nr <= nr_devices; det_nr++) {
01722             for (ifu_nr = 0; ifu_nr < KMOS_IFUS_PER_DETECTOR; ifu_nr++) {
01723                 KMO_TRY_EXIT_IF_ERROR(
01724                     kmclipm_update_property_int(stored_sub_data_headers[(det_nr-1)*KMOS_IFUS_PER_DETECTOR+ifu_nr],
01725                                                 CAL_IFU_NR,
01726                                                 ifu_nr+1+(det_nr-1)*KMOS_IFUS_PER_DETECTOR,
01727                                                 "IFU Number {1..24}"));
01728                 KMO_TRY_EXIT_IF_ERROR(
01729                     kmclipm_update_property_double(
01730                                                 stored_sub_data_headers[(det_nr-1)*KMOS_IFUS_PER_DETECTOR+ifu_nr],
01731                                                 CAL_ROTANGLE,
01732                                                 rotangle_found,
01733                                                 "[deg] Rotator relative to nasmyth"));
01734                 if (has_flat_edge) {
01735                     // save edge-parameters as product
01736                     KMO_TRY_EXIT_IF_ERROR(
01737                         kmo_dfs_save_table(edge_table_sky[det_nr-1][ifu_nr], SKYFLAT_EDGE, fn_suffix,
01738                                            stored_sub_data_headers[(det_nr-1)*KMOS_IFUS_PER_DETECTOR+ifu_nr]));
01739                 }
01740             }
01741         }
01742     }
01743     KMO_CATCH
01744     {
01745         KMO_CATCH_MSG();
01746         ret_val = -1;
01747     }
01748     kmo_free_fits_desc(&desc_sky);
01749     kmo_free_fits_desc(&desc_dark);
01750     kmo_free_fits_desc(&desc_flat);
01751     kmo_free_fits_desc(&desc_xcal);
01752     kmo_free_fits_desc(&desc_ycal);
01753     kmo_free_fits_desc(&desc_lcal);
01754     cpl_image_delete(combined_data); combined_data = NULL;
01755     cpl_image_delete(combined_noise); combined_noise = NULL;
01756     cpl_image_delete(xcal); xcal = NULL;
01757     cpl_image_delete(ycal); ycal = NULL;
01758     cpl_image_delete(lcal); lcal = NULL;
01759     cpl_image_delete(img_dark); img_dark = NULL;
01760     cpl_image_delete(img_dark_noise); img_dark_noise = NULL;
01761     cpl_image_delete(img_flat); img_flat = NULL;
01762     cpl_image_delete(img_flat_noise); img_flat_noise = NULL;
01763     cpl_array_delete(calTimestamp); calTimestamp = NULL;
01764     cpl_free(bounds); bounds = NULL;
01765     kmo_free_unused_ifus(unused_ifus_before); unused_ifus_before = NULL;
01766     kmo_free_unused_ifus(unused_ifus_after); unused_ifus_after = NULL;
01767     cpl_free(fn_lut); fn_lut = NULL;
01768     cpl_free(suffix); suffix = NULL;
01769     cpl_free(fn_suffix); fn_suffix = NULL;
01770     cpl_frameset_delete(frameset_sky); frameset_sky = NULL;
01771     cpl_vector_delete(ranges); ranges = NULL;
01772     cpl_free(filter); filter = NULL;
01773     if (calAngles != NULL) {
01774         cpl_vector_delete(calAngles); calAngles = NULL;
01775     }
01776     cpl_propertylist_delete(main_header); main_header = NULL;
01777     for (i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01778         if (stored_data_cubes != NULL) {
01779             cpl_imagelist_delete(stored_data_cubes[i]);
01780             stored_data_cubes[i] = NULL;
01781         }
01782         if (stored_noise_cubes != NULL) {
01783             cpl_imagelist_delete(stored_noise_cubes[i]);
01784             stored_noise_cubes[i] = NULL;
01785         }
01786         if (stored_data_images != NULL) {
01787             cpl_image_delete(stored_data_images[i]);
01788             stored_data_images[i] = NULL;
01789         }
01790         if (stored_noise_images != NULL) {
01791             cpl_image_delete(stored_noise_images[i]);
01792             stored_noise_images[i] = NULL;
01793         }
01794         if (stored_sub_data_headers != NULL) {
01795             cpl_propertylist_delete(stored_sub_data_headers[i]);
01796             stored_sub_data_headers[i] = NULL;
01797         }
01798         if (stored_sub_noise_headers != NULL) {
01799             cpl_propertylist_delete(stored_sub_noise_headers[i]);
01800             stored_sub_noise_headers[i] = NULL;
01801         }
01802     }
01803     cpl_free(stored_data_cubes); stored_data_cubes = NULL;
01804     cpl_free(stored_noise_cubes); stored_noise_cubes = NULL;
01805     cpl_free(stored_data_images); stored_data_images = NULL;
01806     cpl_free(stored_noise_images); stored_noise_images = NULL;
01807     cpl_free(stored_sub_data_headers); stored_sub_data_headers = NULL;
01808     cpl_free(stored_sub_noise_headers); stored_sub_noise_headers = NULL;
01809     if (edge_table_sky != NULL) {
01810         for (i = 0; i < KMOS_NR_DETECTORS; i++) {
01811             if (edge_table_sky[i] != NULL) {
01812                 for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01813                     cpl_table_delete(edge_table_sky[i][j]);
01814                     edge_table_sky[i][j] = NULL;
01815                 }
01816                 cpl_free(edge_table_sky[i]); edge_table_sky[i] = NULL;
01817             }
01818         }
01819         cpl_free(edge_table_sky); edge_table_sky = NULL;
01820     }
01821     if (edge_table_flat != NULL) {
01822         for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01823             cpl_table_delete(edge_table_flat[j]);
01824             edge_table_flat[j] = NULL;
01825         }
01826         cpl_free(edge_table_flat); edge_table_flat = NULL;
01827     }
01828 
01829     return ret_val;
01830 }
01831