1_process_starbase_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "numpy     1.15.4\n",
      "watermark 1.8.1\n",
      "pandas    0.23.4\n",
      "ELEANOR LUTZ 2019-07-11 \n",
      "\n",
      "CPython 3.7.1\n",
      "IPython 7.2.0\n",
      "\n",
      "compiler   : MSC v.1900 64 bit (AMD64)\n",
      "system     : Windows\n",
      "release    : 10\n",
      "machine    : AMD64\n",
      "processor  : Intel64 Family 6 Model 63 Stepping 2, GenuineIntel\n",
      "CPU cores  : 12\n",
      "interpreter: 64bit\n"
     ]
    }
   ],
   "source": [
    "# Watermark is not required for this code, but is included for information. \n",
    "import watermark\n",
    "%load_ext watermark\n",
    "%watermark -a \"ELEANOR LUTZ\" -d -v -iv -m"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Source\n",
    "The data used in this Jupyter Notebook is from the [HYG Database version 3](http://www.astronexus.com/hyg) by David Nash. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>hip</th>\n",
       "      <th>hd</th>\n",
       "      <th>hr</th>\n",
       "      <th>gl</th>\n",
       "      <th>bf</th>\n",
       "      <th>proper</th>\n",
       "      <th>ra</th>\n",
       "      <th>dec</th>\n",
       "      <th>dist</th>\n",
       "      <th>...</th>\n",
       "      <th>bayer</th>\n",
       "      <th>flam</th>\n",
       "      <th>con</th>\n",
       "      <th>comp</th>\n",
       "      <th>comp_primary</th>\n",
       "      <th>base</th>\n",
       "      <th>lum</th>\n",
       "      <th>var</th>\n",
       "      <th>var_min</th>\n",
       "      <th>var_max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Sol</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>224700.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000060</td>\n",
       "      <td>1.089009</td>\n",
       "      <td>219.7802</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Psc</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.638290</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>224690.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000283</td>\n",
       "      <td>-19.498840</td>\n",
       "      <td>47.9616</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Cet</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.392283</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3.0</td>\n",
       "      <td>224699.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000335</td>\n",
       "      <td>38.859279</td>\n",
       "      <td>442.4779</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>And</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>386.901132</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>4.0</td>\n",
       "      <td>224707.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000569</td>\n",
       "      <td>-51.893546</td>\n",
       "      <td>134.2282</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Phe</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.366989</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 37 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  hip        hd  hr   gl   bf proper        ra        dec      dist  \\\n",
       "0   0  NaN       NaN NaN  NaN  NaN    Sol  0.000000   0.000000    0.0000   \n",
       "1   1  1.0  224700.0 NaN  NaN  NaN    NaN  0.000060   1.089009  219.7802   \n",
       "2   2  2.0  224690.0 NaN  NaN  NaN    NaN  0.000283 -19.498840   47.9616   \n",
       "3   3  3.0  224699.0 NaN  NaN  NaN    NaN  0.000335  38.859279  442.4779   \n",
       "4   4  4.0  224707.0 NaN  NaN  NaN    NaN  0.000569 -51.893546  134.2282   \n",
       "\n",
       "    ...     bayer  flam  con  comp  comp_primary base         lum  var  \\\n",
       "0   ...       NaN   NaN  NaN     1             0  NaN    1.000000  NaN   \n",
       "1   ...       NaN   NaN  Psc     1             1  NaN    9.638290  NaN   \n",
       "2   ...       NaN   NaN  Cet     1             2  NaN    0.392283  NaN   \n",
       "3   ...       NaN   NaN  And     1             3  NaN  386.901132  NaN   \n",
       "4   ...       NaN   NaN  Phe     1             4  NaN    9.366989  NaN   \n",
       "\n",
       "   var_min  var_max  \n",
       "0      NaN      NaN  \n",
       "1      NaN      NaN  \n",
       "2      NaN      NaN  \n",
       "3      NaN      NaN  \n",
       "4      NaN      NaN  \n",
       "\n",
       "[5 rows x 37 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Tau' 'The' 'Zet' 'Alp' 'Bet' 'Kap-1' 'Eps' 'Gam-3' 'Kap-2' 'Gam' 'Chi'\n",
      " 'Sig' 'Iot' 'Pi' 'Rho' 'Kap' 'Eta' 'Lam-1' 'Bet-1' 'Bet-2' 'Lam' 'Bet-3'\n",
      " 'Lam-2' 'Del' 'Mu' 'Xi' 'Phi-1' 'Omi' 'Nu' 'Phi-2' 'Ups-1' 'Phi-3'\n",
      " 'Ups-2' 'Phi-4' 'Ome' 'Psi-1' 'Ups' 'Psi-2' 'Phi' 'Psi-3' 'Psi' 'Tau-1'\n",
      " 'Tau-2' 'Eta-1' 'Gam-2' 'Eta-2' 'Gam-1' 'Xi-1' 'Pi-1' 'Pi-2' 'Xi-2'\n",
      " 'Iot-1' 'Iot-2' 'Eta-3' 'Rho-1' 'Rho-2' 'Rho-3' 'The-1' 'Tau-3' 'Zet-1'\n",
      " 'Zet-2' 'Tau-4' 'Chi-1' 'Chi-2' 'Chi-3' 'Tau-5' 'Tau-6' 'Tau-7' 'Tau-8'\n",
      " 'Tau-9' 'Ome-1' 'Omi-1' 'Omi-2' 'Ome-2' 'Ups-4' 'Del-1' 'Del-2' 'Del-3'\n",
      " 'The-2' 'Sig-1' 'Sig-2' 'Pi-3' 'Pi-4' 'Pi-5' 'Pi-6' 'Nu-1' 'Nu-2' 'Nu-3'\n",
      " 'Psi-4' 'Psi-5' 'Psi-6' 'Psi-7' 'Psi-8' 'Psi-9' 'Mu-1' 'Mu-2' 'Sig-3'\n",
      " 'Alp-1' 'Alp-2' 'Zet-3' 'Zet-4' 'Eps-1' 'Eps-2']\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>hip</th>\n",
       "      <th>hd</th>\n",
       "      <th>hr</th>\n",
       "      <th>gl</th>\n",
       "      <th>bf</th>\n",
       "      <th>proper</th>\n",
       "      <th>ra</th>\n",
       "      <th>dec</th>\n",
       "      <th>dist</th>\n",
       "      <th>...</th>\n",
       "      <th>flam</th>\n",
       "      <th>con</th>\n",
       "      <th>comp</th>\n",
       "      <th>comp_primary</th>\n",
       "      <th>base</th>\n",
       "      <th>lum</th>\n",
       "      <th>var</th>\n",
       "      <th>var_min</th>\n",
       "      <th>var_max</th>\n",
       "      <th>greek_letters</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>224700.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000060</td>\n",
       "      <td>1.089009</td>\n",
       "      <td>219.7802</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Psc</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.638290</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>224690.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000283</td>\n",
       "      <td>-19.498840</td>\n",
       "      <td>47.9616</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Cet</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.392283</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3.0</td>\n",
       "      <td>224699.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000335</td>\n",
       "      <td>38.859279</td>\n",
       "      <td>442.4779</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>And</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>386.901132</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>4.0</td>\n",
       "      <td>224707.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000569</td>\n",
       "      <td>-51.893546</td>\n",
       "      <td>134.2282</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Phe</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.366989</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>224705.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000665</td>\n",
       "      <td>-40.591202</td>\n",
       "      <td>257.7320</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Phe</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21.998851</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 38 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  hip        hd  hr   gl   bf proper        ra        dec      dist  \\\n",
       "1   1  1.0  224700.0 NaN  NaN  NaN    NaN  0.000060   1.089009  219.7802   \n",
       "2   2  2.0  224690.0 NaN  NaN  NaN    NaN  0.000283 -19.498840   47.9616   \n",
       "3   3  3.0  224699.0 NaN  NaN  NaN    NaN  0.000335  38.859279  442.4779   \n",
       "4   4  4.0  224707.0 NaN  NaN  NaN    NaN  0.000569 -51.893546  134.2282   \n",
       "5   5  5.0  224705.0 NaN  NaN  NaN    NaN  0.000665 -40.591202  257.7320   \n",
       "\n",
       "       ...        flam  con  comp  comp_primary  base         lum  var  \\\n",
       "1      ...         NaN  Psc     1             1   NaN    9.638290  NaN   \n",
       "2      ...         NaN  Cet     1             2   NaN    0.392283  NaN   \n",
       "3      ...         NaN  And     1             3   NaN  386.901132  NaN   \n",
       "4      ...         NaN  Phe     1             4   NaN    9.366989  NaN   \n",
       "5      ...         NaN  Phe     1             5   NaN   21.998851  NaN   \n",
       "\n",
       "   var_min  var_max  greek_letters  \n",
       "1      NaN      NaN            NaN  \n",
       "2      NaN      NaN            NaN  \n",
       "3      NaN      NaN            NaN  \n",
       "4      NaN      NaN            NaN  \n",
       "5      NaN      NaN            NaN  \n",
       "\n",
       "[5 rows x 38 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4307 unique spectral designations\n",
      "14 unique spectral designations\n",
      "['F' 'K' 'B' 'G' 'M' 'A' 'C' 'R' 'O' 'W' 'N' 'S' 'D' 'P']\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>hip</th>\n",
       "      <th>hd</th>\n",
       "      <th>hr</th>\n",
       "      <th>gl</th>\n",
       "      <th>bf</th>\n",
       "      <th>proper</th>\n",
       "      <th>ra</th>\n",
       "      <th>dec</th>\n",
       "      <th>dist</th>\n",
       "      <th>...</th>\n",
       "      <th>comp_primary</th>\n",
       "      <th>base</th>\n",
       "      <th>lum</th>\n",
       "      <th>var</th>\n",
       "      <th>var_min</th>\n",
       "      <th>var_max</th>\n",
       "      <th>greek_letters</th>\n",
       "      <th>spect_desig</th>\n",
       "      <th>color</th>\n",
       "      <th>linecolor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>224700.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000060</td>\n",
       "      <td>1.089009</td>\n",
       "      <td>219.7802</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.638290</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>F</td>\n",
       "      <td>#d4bf94</td>\n",
       "      <td>#d4bf94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>224690.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000283</td>\n",
       "      <td>-19.498840</td>\n",
       "      <td>47.9616</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.392283</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>K</td>\n",
       "      <td>#F99220</td>\n",
       "      <td>#F99220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3.0</td>\n",
       "      <td>224699.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000335</td>\n",
       "      <td>38.859279</td>\n",
       "      <td>442.4779</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>386.901132</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>B</td>\n",
       "      <td>#93C2F1</td>\n",
       "      <td>#93C2F1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>4.0</td>\n",
       "      <td>224707.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000569</td>\n",
       "      <td>-51.893546</td>\n",
       "      <td>134.2282</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.366989</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>F</td>\n",
       "      <td>#d4bf94</td>\n",
       "      <td>#d4bf94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>224705.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000665</td>\n",
       "      <td>-40.591202</td>\n",
       "      <td>257.7320</td>\n",
       "      <td>...</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21.998851</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>G</td>\n",
       "      <td>#FFD423</td>\n",
       "      <td>#FFD423</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 41 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  hip        hd  hr   gl   bf proper        ra        dec      dist  \\\n",
       "1   1  1.0  224700.0 NaN  NaN  NaN    NaN  0.000060   1.089009  219.7802   \n",
       "2   2  2.0  224690.0 NaN  NaN  NaN    NaN  0.000283 -19.498840   47.9616   \n",
       "3   3  3.0  224699.0 NaN  NaN  NaN    NaN  0.000335  38.859279  442.4779   \n",
       "4   4  4.0  224707.0 NaN  NaN  NaN    NaN  0.000569 -51.893546  134.2282   \n",
       "5   5  5.0  224705.0 NaN  NaN  NaN    NaN  0.000665 -40.591202  257.7320   \n",
       "\n",
       "     ...      comp_primary  base         lum  var  var_min var_max  \\\n",
       "1    ...                 1   NaN    9.638290  NaN      NaN     NaN   \n",
       "2    ...                 2   NaN    0.392283  NaN      NaN     NaN   \n",
       "3    ...                 3   NaN  386.901132  NaN      NaN     NaN   \n",
       "4    ...                 4   NaN    9.366989  NaN      NaN     NaN   \n",
       "5    ...                 5   NaN   21.998851  NaN      NaN     NaN   \n",
       "\n",
       "   greek_letters  spect_desig    color  linecolor  \n",
       "1            NaN            F  #d4bf94    #d4bf94  \n",
       "2            NaN            K  #F99220    #F99220  \n",
       "3            NaN            B  #93C2F1    #93C2F1  \n",
       "4            NaN            F  #d4bf94    #d4bf94  \n",
       "5            NaN            G  #FFD423    #FFD423  \n",
       "\n",
       "[5 rows x 41 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "119613 total stars available in database\n",
      "8912 stars visible to the human eye\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('./data/hygdata_v3/hygdata_v3.csv', low_memory=False)\n",
    "display(df.head())\n",
    "\n",
    "# Remove the sun because it doesn't make sense in a star chart\n",
    "df = df[df['proper'] != 'Sol']\n",
    "\n",
    "# Translate plaintext Bayer designations into non-ASCII greek letters\n",
    "greek_dict = {'Alp': u\"α\",'Bet': u\"β\",'Chi': u\"χ\",'Del': u\"δ\",'Eps': u\"ε\",'Eta': u\"η\",\n",
    "              'Gam': u\"γ\",'Iot': u\"ι\",'Kap': u\"κ\",'Lam': u\"λ\",'Mu': u\"μ\",'Nu': u\"ν\",\n",
    "              'Ome': u\"ω\",'Omi': u\"ο\",'Phi': u\"φ\",'Pi': u\"π\",'Psi': u\"ψ\",'Rho': u\"ρ\",\n",
    "              'Sig': u\"σ\",'Tau': u\"τ\",'The': u\"θ\",'Ups': u\"υ\",'Xi': u\"ξ\",'Zet': u\"ζ\"}\n",
    "\n",
    "print(df[pd.notnull(df['bayer'])]['bayer'].unique())\n",
    "def get_greek_letter(n):\n",
    "    if str(n) == 'nan':\n",
    "        return(np.nan)\n",
    "    split = n.split(\"-\")\n",
    "    greek = greek_dict.get(split[0])\n",
    "    if len(split) > 1:\n",
    "        r = greek + split[1]\n",
    "    else: \n",
    "        r = greek\n",
    "    return(r)\n",
    "\n",
    "df['greek_letters'] = df['bayer'].apply(get_greek_letter)\n",
    "display(df.head())\n",
    "\n",
    "print(len(df[pd.notnull(df['spect'])]['spect'].unique()), 'unique spectral designations')\n",
    "def get_first_letter(name):\n",
    "    '''Preprocess spectral designations to remove numbers'''\n",
    "    if str(name) != 'nan':\n",
    "        if len(name) > 1:\n",
    "            if name[0:2] == 'sd':\n",
    "                # remove MK system luminosity class to look just at\n",
    "                # Morgan-Keenan designations\n",
    "                name = name[2::]\n",
    "            alphas = ''.join(c for c in name if c not in '?:!/;.,[]{}()')\n",
    "            return(alphas[0].upper())\n",
    "        else:\n",
    "            return(name.upper())\n",
    "    return(name)\n",
    "    \n",
    "df['dist'].replace(to_replace=100000, value=np.nan, inplace=True)\n",
    "    \n",
    "df['spect_desig'] = df['spect'].apply(get_first_letter)    \n",
    "print(len(df[pd.notnull(df['spect_desig'])]['spect_desig'].unique()), 'unique spectral designations')\n",
    "print(df[pd.notnull(df['spect_desig'])]['spect_desig'].unique())\n",
    "\n",
    "color_dict = { \n",
    "    'O':'#5A90C3', 'B':'#93C2F1', 'A':'#f3e8d3', 'F':'#d4bf94',\n",
    "    'G':'#FFD423', 'K':'#F99220', 'M':'#FF2620',  'L':'#FF2620',\n",
    "    'T':'#FF6199', 'Y':'#6B22FF', \n",
    "    'C':'#979330', 'R':'#979330', 'W':'#979330', 'N':'#979330',\n",
    "    'S':'#979330', 'D':'#979330', 'P':'#979330',\n",
    "    'nan': '#000000' # unknown\n",
    "}\n",
    "df['color'] = df['spect_desig'].replace(to_replace=color_dict)\n",
    "df['color'] = df['color'].replace(to_replace=np.nan, value='#000000')\n",
    "df['linecolor'] = df['color'].replace(['#000000'], ['#f3e8d3']) # beige outline for black NANs\n",
    "\n",
    "display(df.head())\n",
    "df.to_csv('./data/processed/hygdata_processed.csv', index=False)\n",
    "\n",
    "print(len(df), 'total stars available in database')\n",
    "df = df[df['mag'] <= 6.5]\n",
    "print(len(df), 'stars visible to the human eye')\n",
    "df.to_csv('./data/processed/hygdata_processed_mag65.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}