truncated_filtering.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "dcd22ba7-244d-4792-98e1-871f5fabae0d",
   "metadata": {},
   "source": [
    "### Notebook containing code for combining truncated object detections with regular detections"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fdf80929",
   "metadata": {},
   "outputs": [],
   "source": [
    "from astropy.io import fits\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import colors\n",
    "from astropy.visualization import make_lupton_rgb\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from astropy.nddata import Cutout2D\n",
    "from astropy.wcs import WCS\n",
    "\n",
    "import cv2\n",
    "from detectron2.structures import BoxMode\n",
    "from astropy.table import Table\n",
    "import glob\n",
    "from astropy.coordinates import SkyCoord  # High-level coordinates\n",
    "from detectron2.config import LazyConfig, get_cfg, instantiate\n",
    "import os\n",
    "import scipy.stats as stats\n",
    "import h5py\n",
    "import json\n",
    "import astropy.units as u\n",
    "from astropy.coordinates import SkyCoord\n",
    "\n",
    "import warnings\n",
    "import time\n",
    "\n",
    "from astropy.wcs import FITSFixedWarning\n",
    "warnings.filterwarnings(\"ignore\", category=FITSFixedWarning)\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "from detectron2.data import detection_utils as utils\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5b2e6c92-97eb-4e40-a139-5b99f34798ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "testmetaf = h5py.File(test_metadatafile, \"r\")\n",
    "test_metadata = testmetaf['metadata_dicts']\n",
    "\n",
    "testf = h5py.File(testfile, \"r\")\n",
    "testims = testf['images']\n",
    "\n",
    "testmetaf_trunc = h5py.File(test_trunc_metadatafile, \"r\")\n",
    "test_trunc_metadata = testmetaf_trunc['metadata_dicts']\n",
    "\n",
    "testf_trunc = h5py.File(testfile_trunc, \"r\")\n",
    "testims_trunc = testf_trunc['images']\n",
    "\n",
    "\n",
    "trainmetaf = h5py.File(train_metadatafile, \"r\")\n",
    "train_metadata = trainmetaf['metadata_dicts']\n",
    "\n",
    "trainf = h5py.File(trainfile, \"r\")\n",
    "trainims = trainf['images']\n",
    "\n",
    "\n",
    "\n",
    "train_ids = []\n",
    "for i in range(len(train_metadata)):\n",
    "    d = json.loads(train_metadata[i])\n",
    "    for a in d['annotations']:\n",
    "        if a['redshift']!=-1:\n",
    "            train_ids.append(a['obj_id'])\n",
    "\n",
    "            \n",
    "test_ids = []\n",
    "for i in range(len(test_metadata)):\n",
    "    d = json.loads(test_metadata[i])\n",
    "    for a in d['annotations']:\n",
    "        if a['redshift']!=-1:\n",
    "            test_ids.append(a['obj_id'])\n",
    "\n",
    "fncat = '/home/shared/hsc/JWST/catalogs/hlsp_jades_jwst_nircam_goods-s-deep_photometry_v2.0_catalog.fits'\n",
    "fnspecz = '/home/shared/hsc/JWST/catalogs/JADES_GOODS_zspec_cleaned.fits'\n",
    "\n",
    "dphot = Table.read(fncat, hdu=2).to_pandas()\n",
    "dspecz = Table.read(fnspecz,hdu=1).to_pandas()\n",
    "dt = Table.read(fncat, hdu=7).to_pandas()\n",
    "            \n",
    "dspecztrain = dspecz.iloc[np.nonzero(np.in1d(dspecz.ID.values,np.unique(train_ids)))]\n",
    "dspecztest = dspecz.iloc[np.nonzero(np.in1d(dspecz.ID.values,np.unique(test_ids)))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8383bd62-c6f7-42be-b253-a3e61bcdf276",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "48a71df9-485c-4ccb-a32b-4dd233b0392f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_198415/2555727190.py:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  dspecztrain['Num_missing']=nmb_train\n",
      "/tmp/ipykernel_198415/2555727190.py:23: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  dspecztest['Num_missing']=nmb_test\n"
     ]
    }
   ],
   "source": [
    "def missing_bands(dspec,dphot_kron):\n",
    "    nmb = []\n",
    "    mbs = []\n",
    "    for idi in dspec.ID.values:\n",
    "        ind = np.where(dphot_kron.ID==idi)\n",
    "        nob=0\n",
    "        mbi=[]\n",
    "        for F in JADES_filters_F:\n",
    "            if dphot_kron[f'{F}_KRON'].values[ind]==0:\n",
    "                nob+=1\n",
    "                mbi.append(JADES_filters_F)\n",
    "        nmb.append(nob)\n",
    "\n",
    "    return nmb,mbs\n",
    "\n",
    "dphot_kron = Table.read(fncat, hdu=8).to_pandas()\n",
    "dphot_size = Table.read(fncat, hdu=3).to_pandas()\n",
    "\n",
    "nmb_train,mbs_train = missing_bands(dspecztrain,dphot_kron)\n",
    "dspecztrain['Num_missing']=nmb_train\n",
    "\n",
    "nmb_test,mbs_test = missing_bands(dspecztest,dphot_kron)\n",
    "dspecztest['Num_missing']=nmb_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "e4cb976c-e59e-46c5-a85d-d91663e85e48",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def outside_box(box,shape):\n",
    "    if box[0]<0:\n",
    "        return True\n",
    "    elif box[1]<0:\n",
    "        return True\n",
    "    elif box[2]>shape[1]:\n",
    "        return True\n",
    "    elif box[3]>shape[0]:\n",
    "        return True\n",
    "    else:\n",
    "        return False\n",
    "\n",
    "\n",
    "## Determines the truncated objects in the original test set\n",
    "truncated = []\n",
    "for i in range(len(test_metadata)):\n",
    "    d = json.loads(test_metadata[i])\n",
    "    shape=(d['height'],d['width'])\n",
    "    boxes = utils.annotations_to_instances(d['annotations'],shape).gt_boxes.tensor.cpu().numpy()\n",
    "    for j,box in enumerate(boxes):\n",
    "        outside = outside_box(box,shape)\n",
    "        if d['annotations'][j]['redshift']!=-1 and outside:\n",
    "            truncated.append((i,j,d['annotations'][j]['obj_id']))\n",
    "            \n",
    "truncated=np.array(truncated)\n",
    "    \n",
    "\n",
    "def get_res(filename):\n",
    "    with open(filename, 'rb') as fp:\n",
    "        data = pickle.load(fp)\n",
    "    return [data[key] for key in data.keys()]\n",
    "    \n",
    "    \n",
    "JADESfilts = ['F090W','F115W','F150W','F200W','F277W','F335M','F356W','F410M','F444W']\n",
    "\n",
    "def get_missfilts(ids):\n",
    "    missfilts = []\n",
    "    for i, idi in enumerate(ids):\n",
    "        cnb=0\n",
    "        mf=[]\n",
    "        for filt in JADESfilts:\n",
    "            if dt[dt['ID']==idi][f'{filt}_KRON'].values[0] ==0:\n",
    "                cnb+=1\n",
    "                mf.append(filt)\n",
    "\n",
    "        missfilts.append(mf)\n",
    "\n",
    "    return missfilts\n",
    "\n",
    "\n",
    "\n",
    "def get_mi(missfilts):\n",
    "    mi=[]\n",
    "    for i,m in enumerate(missfilts):\n",
    "        if m==np.unique(missfilts)[0]:\n",
    "            mi.append(i)\n",
    "    return mi\n",
    "    \n",
    "def get_tot(dtest, truncated, res_name,res_trunc_name):\n",
    "    \n",
    "     \"\"\"Takes as the outputs of running inference on the normal and truncated test set \n",
    "         and returns a single catalog\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        dtest: pandas DataFrame\n",
    "            Dataframe that contains the entire test set of objects \n",
    "            (including those truncated in the original test set of images).  Must have an ID key\n",
    "        truncated: array(int):\n",
    "            Array of test set object IDs that were truncated\n",
    "        res_name: str\n",
    "            path to a dictionary containing zphot, ztrue, object_ids, scores, and pdfs of inferred objects\n",
    "            Assumes the detections have already been matched to a test set catalog and duplicate matches \n",
    "            have been filtered by score\n",
    "        trunc_name: str\n",
    "            path to a dictionary containing zphot, ztrue, object_ids, scores, and pdfs of inferred objects\n",
    "            from the truncated test set. Assumes the detections have already been matched to a test set catalog \n",
    "            and duplicate matches have been filtered by score\n",
    "\n",
    "        \n",
    "        Returns\n",
    "        -------\n",
    "        zts_tot, zps_tot, ids_tot, pdfs_tot\n",
    "        zts_tot : array(float)\n",
    "            Filtered array of true redshifts \n",
    "        zps_tot : array(float)\n",
    "            Filtered array of photometric redshifts\n",
    "        ids_tot : array(float)\n",
    "            Filtered array of object ids\n",
    "        pdfs_tot : 2D array(float)\n",
    "            Filtered array of z PDFs\n",
    "        \"\"\"\n",
    "    \n",
    "    zps,zts,ids,scores,pdfs = get_res(res_name)\n",
    "    zps_trunc,zts_trunc,ids_trunc,scores_trunc,pdfs_trunc = get_res(res_trunc_name)\n",
    "    \n",
    "    #What are the test set objects that are not detected in the original test set of images?\n",
    "    nondects_ids = dtest.ID.values[np.nonzero(np.in1d(dtest.ID.values,ids,invert=True))]\n",
    "    \n",
    "    #How many test set objects that are not detected are truncated?\n",
    "    truncated_ids = np.unique(truncated)\n",
    "    #print(len(np.nonzero(np.in1d(nondects_ids,truncated_ids))[0]))\n",
    "\n",
    "    #How many originally truncated objects that weren't detected, are now detected?\n",
    "    orig_trunc_nondects = nondects_ids[np.nonzero(np.in1d(nondects_ids,truncated_ids))[0]]\n",
    "    \n",
    "    #objects originally detected that were truncated\n",
    "    orig_trunc_dects = ids[np.nonzero(np.in1d(ids,truncated_ids))]\n",
    "    \n",
    "    zts[np.nonzero(np.in1d(ids,truncated_ids)*np.in1d(ids,ids_trunc))] = zts_trunc[np.nonzero(np.in1d(ids_trunc,orig_trunc_dects))]\n",
    "    zps[np.nonzero(np.in1d(ids,truncated_ids)*np.in1d(ids,ids_trunc))] = zps_trunc[np.nonzero(np.in1d(ids_trunc,orig_trunc_dects))]\n",
    "    ids[np.nonzero(np.in1d(ids,truncated_ids)*np.in1d(ids,ids_trunc))] = ids_trunc[np.nonzero(np.in1d(ids_trunc,orig_trunc_dects))]\n",
    "    pdfs[np.nonzero(np.in1d(ids,truncated_ids)*np.in1d(ids,ids_trunc))] = pdfs_trunc[np.nonzero(np.in1d(ids_trunc,orig_trunc_dects))]\n",
    "    \n",
    "    zts_tot = np.concatenate([zts,zts_trunc[np.nonzero(np.in1d(ids_trunc,orig_trunc_nondects))]])\n",
    "    zps_tot = np.concatenate([zps,zps_trunc[np.nonzero(np.in1d(ids_trunc,orig_trunc_nondects))]])\n",
    "    ids_tot = np.concatenate([ids,ids_trunc[np.nonzero(np.in1d(ids_trunc,orig_trunc_nondects))]])\n",
    "    pdfs_tot = np.concatenate([pdfs,pdfs_trunc[np.nonzero(np.in1d(ids_trunc,orig_trunc_nondects))]])\n",
    "        \n",
    "    return zts_tot, zps_tot, ids_tot, pdfs_tot\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "72c3f26b-b3ac-4454-b645-9dec91625f6f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "71ec0eb4-593e-46ea-815f-24e2033081d1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:.conda-ddrailnv]",
   "language": "python",
   "name": "conda-env-.conda-ddrailnv-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}