Split mac-setup.sh between elevated and normal tasks

Summary:
I want to easily test env-setup with minimal user-input. The first
pass is to separate what tasks need sudo access and which ones do
not.

In the process, several tasks had to be run over and over again.
These have been put into separate scripts.

I've also fixed homebrew to use the latest installer. And it seems
this installer installs gcc command line tools such that it does
not appear we need to do this anymore.

Issue: https://khanacademy.atlassian.net/browse/INFRA-5864

Test Plan:
Run mac-setup.sh on a clean VM and verify that subsequently setup.sh
and the dev-env in general come up as smoothly as they did before
this refactoring

Reviewers: #devops, boris, davidbraley

Reviewed By: #devops, boris, davidbraley

Subscribers: davidbraley, boris, csilvers

Differential Revision: https://phabricator.khanacademy.org/D68695

(cherry picked from commit 63c9b2f629f112d69ffd91976324bcc8471a87dd)

bin/install-mac-apps.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+# User may be prompted for password so brew can run sudo
+
+# Bail on any errors
+set -e
+
+# TODO(ericbrown): Remove by 6/1/2021 if no complaints (not needed anymore?)
+# Likely need an alternate versions of Casks in order to install chrome-canary
+# Required to install chrome-canary
+#brew tap homebrew/cask-versions
+
+# To install some useful mac apps.
+install_mac_apps() {
+  chosen_apps=() # When the user opts to install a package it will be added to this array.
+
+  mac_apps=(
+    # Browsers
+    firefox firefox-developer-edition google-chrome google-chrome-canary
+    # Tools
+    dropbox google-drive-file-stream iterm2 zoomus
+    # Virtualbox (requires difficult meraki workaround in Catalina++)
+    #virtualbox
+    # Text Editors
+    macvim sublime-text textmate atom
+    # Chat
+    slack
+  )
+
+  mac_apps_str="${mac_apps[@]}"
+  echo "We recommend installing the following apps: ${mac_apps_str}. \n\n"
+
+  read -r -p "Would you like to install [a]ll, [n]one, or [s]ome of the apps? [a/n/s]: " input
+
+  case "$input" in
+      [aA][lL][lL] | [aA])
+          chosen_apps=("${mac_apps[@]}")
+          ;;
+      [sS][oO][mM][eE] | [sS])
+          for app in ${mac_apps[@]}; do
+            if [ "$(get_yn_input "Would you like to install ${app}?" "y")" = "y" ]; then
+              chosen_apps=("${chosen_apps[@]}" "${app}")
+            fi
+          done
+          ;;
+      [nN][oO][nN][eE] | [nN])
+          ;;
+      *)
+          echo "Please choose [a]ll, [n]one, or [s]ome."
+          exit 100
+          ;;
+  esac
+
+  for app in ${chosen_apps[@]}; do
+    if ! brew cask ls $app >/dev/null 2>&1; then
+        echo "$app is not installed, installing $app"
+        brew install --cask $app || warn "Failed to install $app, perhaps it is already installed."
+    else
+        echo "$app already installed"
+    fi
+  done
+}
+
+install_mac_apps

bin/install-mac-gcc.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+set -e
+
+# TODO(ericbrown): It seems homebrew installs tools (for xcode12 for us)
+#     See https://docs.brew.sh/Installation
+# TODO(ericbrown): We only support Catalina (and Big Sur)
+# TODO(ericbrown): This does not work for Big Sur
+# TODO(ericbrown): Can we just use homebrew's gcc?
+# TODO(ericbrown): We are going to start using bottled deps, do we need gcc?
+#     Xcode changes in a rather opinionated way.
+#     Perhaps we should install homebrew gcc (or clang)
+#     Note: Some things in webapp build from source - we need a C compiler
+
+echo
+echo "Checking for Apple command line developer tools..."
+if ! gcc --version >/dev/null 2>&1 || [ ! -s /usr/include/stdio.h ]; then
+    if sw_vers -productVersion | grep -e '^10\.[0-8]$' -e '^10\.[0-8]\.'; then
+        echo "Command line tools are *probably available* for your Mac's OS, but..."
+        echo "why not upgrade your OS right now?"
+        echo "Otherwise, you can always visit developer.apple.com and grab 'em there."
+        exit 1
+    fi
+    if ! gcc --version >/dev/null 2>&1 ; then
+        echo "Installing command line developer tools"
+        # If enter is pressed before its done, not a big deal, but it'll just loop to the same place.
+        echo "You'll want to wait until the xcode install is complete to press Enter again."
+        # Also, how did you get this dotfiles repo in 10.9 without
+        # git auto-triggering the command line tools install process??
+        xcode-select --install
+        exec sh ./mac-setup.sh
+        # If this doesn't work for you, you can find the most recent
+        # version here: https://developer.apple.com/downloads
+    fi
+    if sw_vers -productVersion | grep -q -e '^10\.14\.' && [ ! -s /usr/include/stdio.h ]; then
+        # mac version is Mojave 10.14.*, install SDK headers
+        # The file "macOS_SDK_headers_for_macOS_10.14.pkg" is from
+        # xcode command line tools install
+        if [ -s /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg ]; then
+            # This command isn't guaranteed to work. If it fails, just warn
+            # the user there may be problems and advise they contact
+            # @dev-support if so.
+            if sudo installer -pkg /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg -target / ; then
+                echo "macOS_SDK_headers_for_macOS_10.14 installed"
+            else
+                echo "We're not able to determine if stdio.h is able to be used by compilers correctly on your system."
+                echo "Please reach out to @dev-support if you encounter errors indicating this is a problem while building code or dependencies."
+                echo "You may be able to get more information about the setup by running ${tty_bold}gcc -v${tty_normal}"
+            fi
+        else
+            echo "Updating your command line tools"
+            # If enter is pressed before its done, not a big deal, but it'll just loop to the same place.
+            echo "You'll want to wait until the xcode install is complete to press Enter again."
+            sudo rm -rf /Library/Developer/CommandLineTools
+            xcode-select --install
+            exec sh ./mac-setup.sh
+        fi
+    fi
+else
+    echo "Great, found gcc! (assuming we also have other recent devtools)"
+fi

bin/install-mac-homebrew.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+"""Install or Fix homebrew."""
+
+# This script will prompt for user's password if sudo access is needed
+# TODO(ericbrown): Can we check, install & upgrade apps we know we need/want?
+
+import subprocess
+
+HOMEBREW_INSTALLER = \
+    'https://raw.githubusercontent.com/Homebrew/install/master/install.sh'
+
+print('Checking for mac homebrew')
+
+install_brew = False
+which = subprocess.run(['which', 'brew'], capture_output=True)
+if which.returncode != 0:
+    print('Brew not found, Installing!')
+    install_brew = True
+else:
+    result = subprocess.run(['brew', '--help'], capture_output=True)
+    if result.returncode != 0:
+        print('Brew broken, Re-installing')
+        install_brew = True
+
+if install_brew:
+    # Download installer
+    installer = subprocess.run(['curl', '-fsSL', HOMEBREW_INSTALLER],
+                               stdout=subprocess.PIPE,
+                               stderr=subprocess.STDOUT,
+                               check=True)
+
+    # Validate that we have sudo access (as installer script checks)
+    print("This setup script needs your password to install things as root.")
+    subprocess.run(['sudo', 'sh', '-c', 'echo You have sudo'], check=True)
+
+    # Run downloaded installer
+    result = subprocess.run(['bash'], input=installer.stdout, check=True)
+
+print('Updating (but not upgrading) Homebrew')
+subprocess.run(['brew', 'update'], capture_output=True, check=True)
+
+# Install homebrew-cask, so we can use it manage installing binary/GUI apps
+# brew tap caskroom/cask
+
+# Likely need an alternate versions of Casks in order to install chrome-canary
+# Required to install chrome-canary
+# (Moved to mac-install-apps.sh, but might be needed elsewhere unbeknownst!)
+# subprocess.run(['brew', 'tap', 'brew/cask-versions'], check=True)
+
+# This is where we store our own formula, including a python@2 backport
+subprocess.run(['brew', 'tap', 'khan/repo'], check=True)

bin/install-mac-python2.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+"""Install Khan's python2."""
+
+import argparse
+import subprocess
+import sys
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--force", help="Force install of Khan's python2",
+                    action="store_true")
+args = parser.parse_args()
+
+which = subprocess.run(['which', 'python2'], capture_output=True, text=True)
+if which.stdout.strip() != "/usr/bin/python2":
+    print("Already running a non-system python2.")
+    if not args.force:
+        sys.exit(0)
+
+print("Installing python2 from khan/repo. This may take a few minutes.")
+subprocess.run(['brew', 'install', 'khan/repo/python@2'], check=True)

mac-setup-elevated.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+# We need elevated permissions for a small subset of setup tasks. Isolate these
+# here so that we can test/qa scripts without babysitting them.
+
+# Bail on any errors
+set -e
+
+# The directory to which all repositories will be cloned.
+ROOT=${1-$HOME}
+REPOS_DIR="$ROOT/khan"
+
+# Derived path location constants
+DEVTOOLS_DIR="$REPOS_DIR/devtools"
+
+# Load shared setup functions.
+. "$DEVTOOLS_DIR"/khan-dotfiles/shared-functions.sh
+
+install_protoc() {
+    # If the user has a homebrew version of protobuf installed, uninstall it so
+    # we can manually install our own version in /usr/local.
+    if brew list --formula | grep -q '^protobuf$'; then
+        info "Uninstalling homebrew version of protobuf\n"
+        brew uninstall protobuf
+    fi
+
+    brew install wget
+
+    # The mac and linux installation process is the same from here on out aside
+    # from the platform-dependent zip archive.
+    install_protoc_common https://github.com/protocolbuffers/protobuf/releases/download/v3.4.0/protoc-3.4.0-osx-x86_64.zip
+}
+
+# TODO(ericbrown): Detect pre-requisites (i.e. brew, etc.)
+
+# Run sudo once at the beginning to get the necessary permissions.
+echo "This setup script needs your password to install things as root."
+sudo sh -c 'echo Thanks'
+
+"$DEVTOOLS_DIR"/khan-dotfiles/bin/install-mac-homebrew.py
+
+# It used to be we needed to install xcode-tools, now homebrew does this for us
+#"$DEVTOOLS_DIR"/khan-dotfiles/bin/install-mac-gcc.sh
+
+install_protoc
+
+# We use java for our google cloud dataflow jobs that live in webapp
+# (as well as in khan-linter for linting those jobs)
+install_mac_java
+
+"$DEVTOOLS_DIR"/khan-dotfiles/bin/install-mac-apps.sh