imcore_background.c

00001 /* $Id: imcore_background.c,v 1.15 2010/12/10 12:27:21 jim Exp $
00002  *
00003  * This file is part of the VIRCAM Pipeline
00004  * Copyright (C) 2005 Cambridge Astronomy Survey Unit
00005  *
00006  * This program is free software; you can redistribute it and/or modify
00007  * it under the terms of the GNU General Public License as published by
00008  * the Free Software Foundation; either version 2 of the License, or
00009  * (at your option) any later version.
00010  *
00011  * This program is distributed in the hope that it will be useful,
00012  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00013  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00014  * GNU General Public License for more details.
00015  *
00016  * You should have received a copy of the GNU General Public License
00017  * along with this program; if not, write to the Free Software
00018  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
00019  */
00020 
00021 /*
00022  * $Author: jim $
00023  * $Date: 2010/12/10 12:27:21 $
00024  * $Revision: 1.15 $
00025  * $Name: v1-1-0 $
00026  */
00027 
00028 #include <stdio.h>
00029 #include <stdlib.h>
00030 #include <string.h>
00031 #include <math.h>
00032 
00033 #include <cpl.h>
00034 
00035 #include "floatmath.h"
00036 #include "util.h"
00037 #include "imcore.h"
00038 
00039 static int **hist = NULL;
00040 static int *nnp = NULL;
00041 static int npvx;
00042 static int npvy;
00043 static void tidy(void);
00044 static void sortit (float [], int);
00045 
00048 /*---------------------------------------------------------------------------*/
00079 /*---------------------------------------------------------------------------*/
00080 
00081 extern int imcore_background(ap_t *ap, int nbsize, float nullval) {
00082     float fracx,fracy,skymed,sigma,skymedc,sigmac,avsky,fnbsize,dely,delx;
00083     float t1,t2,dsky,*map,**bvals,*work;
00084     int ifracx,ifracy,nbsizx,nbsizy,nbx,nby,npixstripe,l,i,ll;
00085     int isquare,ilev,j,iclip,mcpix,iloop,irej,nbsizo2,kk,k,iby,ibyp1,ibx,ibxp1;
00086     int *shist,*conf;
00087     unsigned char *mflag;
00088     long nx,ny;
00089 
00090     /* Set up some variables */
00091 
00092     map = ap->indata;
00093     conf = ap->confdata;
00094     mflag = ap->mflag;
00095     nx = ap->lsiz;
00096     ny = ap->csiz;
00097 
00098     /* check to see if nbsize is close to exact divisor */
00099 
00100     fracx = ((float)nx)/((float)nbsize);
00101     fracy = ((float)ny)/((float)nbsize);
00102     ifracx = (int)(fracx + 0.1);
00103     ifracy = (int)(fracy + 0.1);
00104     nbsizx = nx/ifracx;
00105     nbsizy = ny/ifracy;
00106     nbsize = MAX(NINT(0.9*nbsize), MIN(nbsize, MIN(nbsizx,nbsizy)));
00107     nbsize = MIN(nx,MIN(ny,nbsize)); /* trap for small maps */
00108 
00109     /* Divide the map into partitions */
00110 
00111     nbx = nx/nbsize;
00112     nby = ny/nbsize;
00113     npixstripe = nbsize*nx;
00114     npvx = nbx;
00115     npvy = nby;
00116 
00117     /* Get histogram workspace if you can */
00118 
00119     hist = cpl_malloc(nbx*sizeof(int *));
00120     for (l = 0; l < nbx; l++) 
00121         hist[l] = cpl_malloc(MAXHIST*sizeof(int));
00122 
00123     /* Same for background values array */
00124 
00125     bvals = cpl_malloc(nby*sizeof(float *));
00126     for (l = 0; l < nby; l++) 
00127         bvals[l] = cpl_malloc(nbx*sizeof(float));
00128 
00129     /* Store some of this away for use later */
00130 
00131     ap->backmap.nbx = nbx;
00132     ap->backmap.nby = nby;
00133     ap->backmap.nbsize = nbsize;
00134     ap->backmap.bvals = bvals;
00135 
00136     /* Finally a counter array */
00137 
00138     nnp = cpl_malloc(nbx*sizeof(int));
00139 
00140     /* Loop for each row of background squares. Start by initialising
00141        the accumulators and histograms */
00142 
00143     for (l = 0; l < nby; l++) {
00144         memset((char *)nnp,0,nbx*sizeof(*nnp));
00145         for (i = 0; i < nbx; i++) 
00146             memset((char *)hist[i],0,MAXHIST*sizeof(int));
00147 
00148         /* Skim through the data in this stripe. Find out which square each
00149            belongs to and add it it to the relevant histogram */
00150 
00151         ll = l*npixstripe;
00152         for (i = 0; i < npixstripe; i++) {
00153             if (map[ll+i] != nullval && mflag[ll+i] != MF_ZEROCONF &&
00154                 mflag[ll+i] != MF_STUPID_VALUE) {
00155                 isquare = (int)((float)(i % nx)/(float)nbsize);
00156                 isquare = MIN(nbx-1,MAX(0,isquare));
00157                 ilev = MIN(MAXHISTVAL,MAX(MINHISTVAL,NINT(map[i+ll])));
00158                 hist[isquare][ilev-MINHISTVAL] += 1;
00159                 nnp[isquare] += 1;
00160             }
00161         }
00162   
00163         /* but only do background estimation if enough pixels ----------- */
00164 
00165         for (j = 0; j < nbx; j++) {
00166             if (nnp[j] > 0.25*nbsize*nbsize){
00167                 shist = hist[j];
00168                 imcore_medsig(shist,MAXHIST,MINHISTVAL-1,nnp[j],&skymed,&sigma);
00169 
00170                 /* do an iterative 3-sigma upper clip to give a more robust
00171                    estimator */
00172 
00173                 iclip = MAXHISTVAL;
00174                 mcpix = nnp[j];
00175                 skymedc = skymed;
00176                 sigmac = sigma;
00177                 for (iloop = 0; iloop < 3; iloop++) {
00178                     irej = 0;
00179                     for(i = NINT(skymedc+3.0*sigmac); i <= iclip; i++)
00180                         irej += shist[i-MINHISTVAL];
00181                     if (irej == 0)
00182                         break;
00183                     iclip = NINT(skymedc+3.0*sigmac) - 1;
00184                     mcpix = mcpix - irej;
00185                     imcore_medsig(shist,MAXHIST,MINHISTVAL-1,mcpix,&skymedc,
00186                                   &sigmac);
00187                 }
00188                 bvals[l][j] = skymedc;
00189             } else {
00190                 bvals[l][j] = -1000.0;
00191             }
00192         }
00193     }
00194 
00195     /* filter raw background values */
00196 
00197     bfilt(bvals,nbx,nby);
00198 
00199     /* compute average sky level */
00200 
00201     work = cpl_malloc(nbx*nby*sizeof(*work));
00202     k = 0;
00203     for(l = 0; l < nby; l++)
00204         for(j = 0; j < nbx; j++) 
00205             work[k++] = bvals[l][j];
00206     sortit(work,nbx*nby);
00207     avsky = work[(nbx*nby)/2];
00208     freespace(work);
00209 
00210     /* ok now correct map for background variations and put avsky back on */
00211 
00212     nbsizo2 = nbsize/2;
00213     fnbsize = 1.0/((float)nbsize);
00214     for (k = 0; k < ny; k++) {
00215         kk = k*nx;
00216 
00217         /* Nearest background pixel vertically */
00218 
00219         iby = (k + 1 + nbsizo2)/nbsize;
00220         ibyp1 = iby + 1;
00221         iby = MIN(nby,MAX(1,iby));
00222         ibyp1 = MIN(nby,ibyp1);
00223         dely = (k + 1 - nbsize*iby + nbsizo2)*fnbsize;
00224 
00225         for (j = 0; j < nx; j++) {
00226             if (map[kk+j] == nullval) 
00227                 continue;
00228 
00229             /* nearest background pixel across */
00230 
00231             ibx = (j + 1 + nbsizo2)/nbsize;
00232             ibxp1 = ibx + 1;
00233             ibx = MIN(nbx,MAX(1,ibx));
00234             ibxp1 = MIN(nbx,ibxp1);
00235             delx = (j + 1 - nbsize*ibx + nbsizo2)*fnbsize;
00236 
00237             /* bilinear interpolation to find background */
00238             
00239             t1 = (1.0 - dely)*bvals[iby-1][ibx-1] + dely*bvals[ibyp1-1][ibx-1];
00240             t2 = (1.0 - dely)*bvals[iby-1][ibxp1-1] + dely*bvals[ibyp1-1][ibxp1-1];
00241             dsky = avsky - (1.0 - delx)*t1 - delx*t2;
00242             map[kk+j] += dsky;
00243         }
00244     }
00245 
00246     /* Free some workspace */
00247 
00248     tidy();
00249     return(VIR_OK);
00250 }
00251 
00252 /*---------------------------------------------------------------------------*/
00289 /*---------------------------------------------------------------------------*/
00290 
00291 extern int imcore_backstats(ap_t *ap, float nullval, int satonly, 
00292                             float *skymed, float *skysig, float *sat) {
00293     int ilev,iclip,iloop,i,*ihist,isat,iter;
00294     long mpix,npix,k,mcpix,irej,lpix,nx,ny;
00295     float skymedc,sigmac,*map,sata,fac,skyref;
00296     unsigned char *mflag;
00297 
00298     /* Get some info from the ap structure */
00299 
00300     map = ap->indata;
00301     nx = ap->lsiz;
00302     ny = ap->csiz;
00303     mflag = ap->mflag;
00304 
00305     /* Check to make sure there are some non-zero values here */
00306 
00307     ilev = 1;
00308     for (i = 0; i < nx*ny; i++) {
00309         if (map[i] != nullval && mflag[i] != MF_ZEROCONF &&
00310             mflag[i] != MF_STUPID_VALUE) {
00311             ilev = 0;
00312             break;
00313         }
00314     }
00315     if (ilev == 1) {
00316         *skymed = 0.0;
00317         *skysig = 0.0;
00318         *sat = 0.0;
00319         return(VIR_WARN);
00320     }
00321 
00322     /* First, get some workspace for the background histogram */
00323 
00324     ihist = cpl_calloc(MAXHIST,sizeof(*ihist));
00325 
00326     /* Loop for up to 10 iterations. For each iteration we multiply the 
00327        input data by a successively higher power of 2 in order to 
00328        try and deal with data that has very small noise estimates */
00329 
00330     fac = 0.5;
00331     skyref = 0.0;
00332     for (iter = 0; iter <= 9; iter++) {
00333         fac *= 2.0;
00334         if (iter == 1)
00335             skyref = skymedc;
00336         for (k = 0; k < MAXHIST; k++)
00337             ihist[k] = 0;
00338 
00339         /* Now form the histogram of all pixel intensities */
00340 
00341         mpix = 0;
00342         isat = 0;
00343         npix = nx*ny;
00344         for (k = 0; k < npix; k++) {
00345             if (map[k] != nullval && mflag[k] != MF_ZEROCONF &&
00346                 mflag[k] != MF_STUPID_VALUE) {
00347                 ilev = MIN(MAXHISTVAL,MAX(MINHISTVAL,NINT(fac*(map[k]-skyref))));
00348                 ihist[ilev - MINHISTVAL] += 1;
00349                 isat = MAX(isat,ilev);
00350                 mpix++;
00351             }
00352         }
00353         sata = MIN(MAXHISTVAL,MAX(MINSATURATE,0.9*((float)isat))/fac);
00354         lpix = ihist[isat - MINHISTVAL];
00355         while (lpix < mpix/1000 && isat > MINHISTVAL) {
00356             isat--;
00357             lpix += ihist[isat - MINHISTVAL];
00358         }
00359         *sat = ((float)isat)/fac + skyref;
00360         *sat = MIN(MAXHISTVAL,MAX(MINSATURATE,MAX(0.95*(*sat),sata)));
00361 
00362         /* If all you want is the saturation level, then get out of here...*/
00363 
00364         if (satonly) {
00365             freespace(ihist);
00366             return(VIR_OK);
00367         }
00368 
00369         /* Now find the median and sigma */
00370 
00371         imcore_medsig(ihist,MAXHIST,MINHISTVAL-1,mpix,skymed,skysig);
00372 
00373         /* Do an iterative 3-sigma upper clip to give a more robust 
00374            estimator */
00375 
00376         iclip = MAXHISTVAL;
00377         mcpix = mpix;
00378         skymedc = *skymed;
00379         sigmac = *skysig;
00380         for (iloop = 0; iloop < 3; iloop++) {
00381             irej = 0;
00382             for (i = NINT(skymedc+3.0*sigmac); i <= iclip; i++)
00383                 irej += ihist[i - MINHISTVAL];
00384             if (irej == 0)
00385                 break;
00386             iclip = NINT(skymedc+3.0*sigmac)-1;
00387             mcpix = mcpix-irej;
00388             imcore_medsig(ihist,MAXHIST,MINHISTVAL-1,mcpix,&skymedc,&sigmac);
00389         }
00390         if (sigmac > 2.5)
00391             break;
00392     }
00393 
00394     /* Set the final answer */
00395 
00396     *skymed = skymedc/fac + skyref;
00397     *skysig = sigmac/fac;
00398     freespace(ihist);
00399     return(VIR_OK);
00400 }   
00401 
00402 /*---------------------------------------------------------------------------*/
00436 /*---------------------------------------------------------------------------*/
00437 
00438 extern void imcore_backest(ap_t *ap, float x, float y, float *skylev, 
00439                            float *skyrms) {
00440     int i,j,nbx,nby,nbsize,nbsizo2,iby,ibyp1,ibx,ibxp1;
00441     float **bvals,fnbsize,dely,delx,t1,t2;
00442 
00443     /* Define some local variables */
00444 
00445     nbx = ap->backmap.nbx;
00446     nby = ap->backmap.nby;
00447     nbsize = ap->backmap.nbsize;
00448     bvals = ap->backmap.bvals;
00449 
00450     /* Get closest pixel to the input location */
00451 
00452     i = NINT(x);
00453     j = NINT(y);
00454 
00455     /* Now, work out where in the map to do the interpolation */
00456 
00457     nbsizo2 = nbsize/2;
00458     fnbsize = 1.0/((float)nbsize);
00459     iby = (j + nbsizo2)/nbsize;
00460     ibyp1 = iby + 1;
00461     iby = MIN(nby,MAX(1,iby));
00462     ibyp1 = MIN(nby,ibyp1);
00463     dely = (j  - nbsize*iby + nbsizo2)*fnbsize;
00464     ibx = (i + nbsizo2)/nbsize;
00465     ibxp1 = ibx + 1;
00466     ibx = MIN(nbx,MAX(1,ibx));
00467     ibxp1 = MIN(nbx,ibxp1);
00468     delx = (i - nbsize*ibx + nbsizo2)*fnbsize;
00469 
00470     /* Now do a linear interpolation to find the background. Calculate MAD of
00471        the four adjacent background cells as an estimate of the RMS */
00472 
00473     t1 = (1.0 - dely)*bvals[iby-1][ibx-1] + dely*bvals[ibyp1-1][ibx-1];
00474     t2 = (1.0 - dely)*bvals[iby-1][ibxp1-1] + dely*bvals[ibyp1-1][ibxp1-1];
00475     *skylev = (1.0 - delx)*t1 + delx*t2;
00476     *skyrms = 0.25*(fabsf(bvals[iby-1][ibx-1] - *skylev) +
00477                     fabsf(bvals[ibyp1-1][ibx-1] - *skylev) +
00478                     fabsf(bvals[iby-1][ibxp1-1] - *skylev) +
00479                     fabsf(bvals[ibyp1-1][ibxp1-1] - *skylev));
00480 }
00481 
00482 /*---------------------------------------------------------------------------*/
00519 /*---------------------------------------------------------------------------*/
00520 
00521 extern void imcore_medsig(int *shist, int nh, int ist, int itarg, float *med, 
00522                           float *sig) {
00523     int isum, medata;
00524     float ffrac,sigmed;
00525  
00526     /* median */ 
00527 
00528     isum = 0;
00529     medata = ist;
00530     while (isum <= (itarg+1)/2 && (medata-MINHISTVAL) < nh) {
00531         medata++;
00532         isum += shist[medata-MINHISTVAL];
00533     }
00534     if (shist[medata-MINHISTVAL] == 0) {
00535         ffrac = 0.0;
00536     } else {
00537         ffrac = (float)(isum - (itarg+1)/2)/(float)shist[medata-MINHISTVAL];
00538     }
00539     *med = (float)medata - ffrac + 0.5;
00540 
00541     /* sigma */
00542 
00543     isum = 0;
00544     medata = ist;
00545     while (isum <= (itarg+3)/4 && (medata-MINHISTVAL) < nh) {
00546         medata++;
00547         isum += shist[medata-MINHISTVAL];
00548     }
00549     if (shist[medata-MINHISTVAL] == 0) {
00550         ffrac = 0.0;
00551     } else {
00552         ffrac = (float)(isum - (itarg+3)/4)/(float)shist[medata-MINHISTVAL];
00553     }
00554     sigmed = (float)medata - ffrac + 0.5;
00555     *sig = 1.48*(*med - sigmed);
00556     *sig = MAX(0.5,*sig);
00557 }
00558 
00559 /*---------------------------------------------------------------------------*/
00578 /*---------------------------------------------------------------------------*/
00579 
00580 static void sortit (float ia[], int n) {
00581     int i, j, ii, jj, ifin;
00582     float it;
00583  
00584     jj = 4;
00585     while (jj < n) 
00586         jj = 2 * jj;
00587     jj = MIN(n,(3 * jj)/4 - 1);
00588     while (jj > 1) {
00589         jj = jj/2;
00590         ifin = n - jj;
00591         for (ii = 0; ii < ifin; ii++) {
00592             i = ii;
00593             j = i + jj;
00594             if (ia[i] <= ia[j]) 
00595                 continue;
00596             it = ia[j];
00597             do {
00598                 ia[j] = ia[i];
00599                 j = i;
00600                 i = i - jj;
00601                 if (i < 0) 
00602                     break;
00603             } while (ia[i] > it);
00604             ia[j] = it;
00605         }
00606     }
00607     return;
00608 }
00609 
00610 
00611 static void tidy(void) {
00612     int i;
00613  
00614     freespace(nnp);
00615     if (hist != NULL) {
00616         for (i = 0; i < npvx; i++) 
00617             freespace(hist[i]);
00618     }
00619     freespace(hist);
00620     return;
00621 }
00622 
00625 /*
00626 
00627 $Log: imcore_background.c,v $
00628 Revision 1.15  2010/12/10 12:27:21  jim
00629 Fixed bug in backstats to cater for images with high background and low
00630 noise
00631 
00632 Revision 1.14  2010/09/09 12:09:57  jim
00633 Added docs
00634 
00635 Revision 1.13  2009/01/30 08:25:17  jim
00636 fixed a bug imcore_backstats
00637 
00638 Revision 1.12  2009/01/23 12:24:33  jim
00639 Fixed bugs in pixel flagging
00640 
00641 Revision 1.11  2009/01/19 14:34:15  jim
00642 fixed doc
00643 
00644 Revision 1.10  2008/10/28 14:22:50  jim
00645 Fixed bug in imcore_background involving flagging bad pixels
00646 
00647 Revision 1.9  2008/10/13 08:10:35  jim
00648 fixed pixel masking scheme
00649 
00650 Revision 1.8  2007/12/19 13:16:50  jim
00651 Modified the interation step size
00652 
00653 Revision 1.7  2007/04/23 12:51:27  jim
00654 Traps for 0 in confidence map
00655 
00656 Revision 1.6  2007/03/01 12:38:26  jim
00657 Small modifications after a bit of code checking
00658 
00659 Revision 1.5  2006/08/01 11:27:54  jim
00660 Modifications to imcore background estimation and to add ability to
00661 specify the smoothing kernel width
00662 
00663 Revision 1.4  2006/07/11 14:50:27  jim
00664 Modified to do arithmetic in double precision for stats
00665 
00666 Revision 1.3  2006/06/30 21:31:09  jim
00667 MOdifications to background routines and smoothing kernel
00668 
00669 Revision 1.2  2006/04/20 11:15:19  jim
00670 A few minor bugs fixed
00671 
00672 Revision 1.1  2005/09/13 13:25:28  jim
00673 Initial entry after modifications to make cpl compliant
00674 
00675 
00676 */
00677 

Generated on 7 Feb 2011 for VIRCAM Pipeline by  doxygen 1.6.1