From a6d62af6195d53df7279068d7e681e374556cd7c Mon Sep 17 00:00:00 2001 From: "Eric (iCloud) Brown" Date: Thu, 21 Jan 2021 15:46:11 -0800 Subject: [PATCH] Split mac-setup.sh between elevated and normal tasks Summary: I want to easily test env-setup with minimal user-input. The first pass is to separate what tasks need sudo access and which ones do not. In the process, several tasks had to be run over and over again. These have been put into separate scripts. I've also fixed homebrew to use the latest installer. And it seems this installer installs gcc command line tools such that it does not appear we need to do this anymore. Issue: https://khanacademy.atlassian.net/browse/INFRA-5864 Test Plan: Run mac-setup.sh on a clean VM and verify that subsequently setup.sh and the dev-env in general come up as smoothly as they did before this refactoring Reviewers: #devops, boris, davidbraley Reviewed By: #devops, boris, davidbraley Subscribers: davidbraley, boris, csilvers Differential Revision: https://phabricator.khanacademy.org/D68695 (cherry picked from commit 63c9b2f629f112d69ffd91976324bcc8471a87dd) --- bin/install-mac-apps.sh | 64 +++++ bin/install-mac-gcc.sh | 61 ++++ bin/install-mac-homebrew.py | 51 ++++ bin/install-mac-python2.py | 20 ++ mac-setup-elevated.sh | 51 ++++ mac-setup-normal.sh | 384 +++++++++++++++++++++++++ mac-setup.sh | 554 ++---------------------------------- 7 files changed, 653 insertions(+), 532 deletions(-) create mode 100755 bin/install-mac-apps.sh create mode 100755 bin/install-mac-gcc.sh create mode 100755 bin/install-mac-homebrew.py create mode 100755 bin/install-mac-python2.py create mode 100755 mac-setup-elevated.sh create mode 100755 mac-setup-normal.sh diff --git a/bin/install-mac-apps.sh b/bin/install-mac-apps.sh new file mode 100755 index 0000000..77c06a1 --- /dev/null +++ b/bin/install-mac-apps.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +# User may be prompted for password so brew can run sudo + +# Bail on any errors +set -e + +# TODO(ericbrown): Remove by 6/1/2021 if no complaints (not needed anymore?) +# Likely need an alternate versions of Casks in order to install chrome-canary +# Required to install chrome-canary +#brew tap homebrew/cask-versions + +# To install some useful mac apps. +install_mac_apps() { + chosen_apps=() # When the user opts to install a package it will be added to this array. + + mac_apps=( + # Browsers + firefox firefox-developer-edition google-chrome google-chrome-canary + # Tools + dropbox google-drive-file-stream iterm2 zoomus + # Virtualbox (requires difficult meraki workaround in Catalina++) + #virtualbox + # Text Editors + macvim sublime-text textmate atom + # Chat + slack + ) + + mac_apps_str="${mac_apps[@]}" + echo "We recommend installing the following apps: ${mac_apps_str}. \n\n" + + read -r -p "Would you like to install [a]ll, [n]one, or [s]ome of the apps? [a/n/s]: " input + + case "$input" in + [aA][lL][lL] | [aA]) + chosen_apps=("${mac_apps[@]}") + ;; + [sS][oO][mM][eE] | [sS]) + for app in ${mac_apps[@]}; do + if [ "$(get_yn_input "Would you like to install ${app}?" "y")" = "y" ]; then + chosen_apps=("${chosen_apps[@]}" "${app}") + fi + done + ;; + [nN][oO][nN][eE] | [nN]) + ;; + *) + echo "Please choose [a]ll, [n]one, or [s]ome." + exit 100 + ;; + esac + + for app in ${chosen_apps[@]}; do + if ! brew cask ls $app >/dev/null 2>&1; then + echo "$app is not installed, installing $app" + brew install --cask $app || warn "Failed to install $app, perhaps it is already installed." + else + echo "$app already installed" + fi + done +} + +install_mac_apps diff --git a/bin/install-mac-gcc.sh b/bin/install-mac-gcc.sh new file mode 100755 index 0000000..9215cf5 --- /dev/null +++ b/bin/install-mac-gcc.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +set -e + +# TODO(ericbrown): It seems homebrew installs tools (for xcode12 for us) +# See https://docs.brew.sh/Installation +# TODO(ericbrown): We only support Catalina (and Big Sur) +# TODO(ericbrown): This does not work for Big Sur +# TODO(ericbrown): Can we just use homebrew's gcc? +# TODO(ericbrown): We are going to start using bottled deps, do we need gcc? +# Xcode changes in a rather opinionated way. +# Perhaps we should install homebrew gcc (or clang) +# Note: Some things in webapp build from source - we need a C compiler + +echo +echo "Checking for Apple command line developer tools..." +if ! gcc --version >/dev/null 2>&1 || [ ! -s /usr/include/stdio.h ]; then + if sw_vers -productVersion | grep -e '^10\.[0-8]$' -e '^10\.[0-8]\.'; then + echo "Command line tools are *probably available* for your Mac's OS, but..." + echo "why not upgrade your OS right now?" + echo "Otherwise, you can always visit developer.apple.com and grab 'em there." + exit 1 + fi + if ! gcc --version >/dev/null 2>&1 ; then + echo "Installing command line developer tools" + # If enter is pressed before its done, not a big deal, but it'll just loop to the same place. + echo "You'll want to wait until the xcode install is complete to press Enter again." + # Also, how did you get this dotfiles repo in 10.9 without + # git auto-triggering the command line tools install process?? + xcode-select --install + exec sh ./mac-setup.sh + # If this doesn't work for you, you can find the most recent + # version here: https://developer.apple.com/downloads + fi + if sw_vers -productVersion | grep -q -e '^10\.14\.' && [ ! -s /usr/include/stdio.h ]; then + # mac version is Mojave 10.14.*, install SDK headers + # The file "macOS_SDK_headers_for_macOS_10.14.pkg" is from + # xcode command line tools install + if [ -s /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg ]; then + # This command isn't guaranteed to work. If it fails, just warn + # the user there may be problems and advise they contact + # @dev-support if so. + if sudo installer -pkg /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg -target / ; then + echo "macOS_SDK_headers_for_macOS_10.14 installed" + else + echo "We're not able to determine if stdio.h is able to be used by compilers correctly on your system." + echo "Please reach out to @dev-support if you encounter errors indicating this is a problem while building code or dependencies." + echo "You may be able to get more information about the setup by running ${tty_bold}gcc -v${tty_normal}" + fi + else + echo "Updating your command line tools" + # If enter is pressed before its done, not a big deal, but it'll just loop to the same place. + echo "You'll want to wait until the xcode install is complete to press Enter again." + sudo rm -rf /Library/Developer/CommandLineTools + xcode-select --install + exec sh ./mac-setup.sh + fi + fi +else + echo "Great, found gcc! (assuming we also have other recent devtools)" +fi diff --git a/bin/install-mac-homebrew.py b/bin/install-mac-homebrew.py new file mode 100755 index 0000000..a17c1ab --- /dev/null +++ b/bin/install-mac-homebrew.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Install or Fix homebrew.""" + +# This script will prompt for user's password if sudo access is needed +# TODO(ericbrown): Can we check, install & upgrade apps we know we need/want? + +import subprocess + +HOMEBREW_INSTALLER = \ + 'https://raw.githubusercontent.com/Homebrew/install/master/install.sh' + +print('Checking for mac homebrew') + +install_brew = False +which = subprocess.run(['which', 'brew'], capture_output=True) +if which.returncode != 0: + print('Brew not found, Installing!') + install_brew = True +else: + result = subprocess.run(['brew', '--help'], capture_output=True) + if result.returncode != 0: + print('Brew broken, Re-installing') + install_brew = True + +if install_brew: + # Download installer + installer = subprocess.run(['curl', '-fsSL', HOMEBREW_INSTALLER], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=True) + + # Validate that we have sudo access (as installer script checks) + print("This setup script needs your password to install things as root.") + subprocess.run(['sudo', 'sh', '-c', 'echo You have sudo'], check=True) + + # Run downloaded installer + result = subprocess.run(['bash'], input=installer.stdout, check=True) + +print('Updating (but not upgrading) Homebrew') +subprocess.run(['brew', 'update'], capture_output=True, check=True) + +# Install homebrew-cask, so we can use it manage installing binary/GUI apps +# brew tap caskroom/cask + +# Likely need an alternate versions of Casks in order to install chrome-canary +# Required to install chrome-canary +# (Moved to mac-install-apps.sh, but might be needed elsewhere unbeknownst!) +# subprocess.run(['brew', 'tap', 'brew/cask-versions'], check=True) + +# This is where we store our own formula, including a python@2 backport +subprocess.run(['brew', 'tap', 'khan/repo'], check=True) diff --git a/bin/install-mac-python2.py b/bin/install-mac-python2.py new file mode 100755 index 0000000..dd10c86 --- /dev/null +++ b/bin/install-mac-python2.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +"""Install Khan's python2.""" + +import argparse +import subprocess +import sys + +parser = argparse.ArgumentParser() +parser.add_argument("--force", help="Force install of Khan's python2", + action="store_true") +args = parser.parse_args() + +which = subprocess.run(['which', 'python2'], capture_output=True, text=True) +if which.stdout.strip() != "/usr/bin/python2": + print("Already running a non-system python2.") + if not args.force: + sys.exit(0) + +print("Installing python2 from khan/repo. This may take a few minutes.") +subprocess.run(['brew', 'install', 'khan/repo/python@2'], check=True) diff --git a/mac-setup-elevated.sh b/mac-setup-elevated.sh new file mode 100755 index 0000000..60deeda --- /dev/null +++ b/mac-setup-elevated.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# We need elevated permissions for a small subset of setup tasks. Isolate these +# here so that we can test/qa scripts without babysitting them. + +# Bail on any errors +set -e + +# The directory to which all repositories will be cloned. +ROOT=${1-$HOME} +REPOS_DIR="$ROOT/khan" + +# Derived path location constants +DEVTOOLS_DIR="$REPOS_DIR/devtools" + +# Load shared setup functions. +. "$DEVTOOLS_DIR"/khan-dotfiles/shared-functions.sh + +install_protoc() { + # If the user has a homebrew version of protobuf installed, uninstall it so + # we can manually install our own version in /usr/local. + if brew list --formula | grep -q '^protobuf$'; then + info "Uninstalling homebrew version of protobuf\n" + brew uninstall protobuf + fi + + brew install wget + + # The mac and linux installation process is the same from here on out aside + # from the platform-dependent zip archive. + install_protoc_common https://github.com/protocolbuffers/protobuf/releases/download/v3.4.0/protoc-3.4.0-osx-x86_64.zip +} + +# TODO(ericbrown): Detect pre-requisites (i.e. brew, etc.) + +# Run sudo once at the beginning to get the necessary permissions. +echo "This setup script needs your password to install things as root." +sudo sh -c 'echo Thanks' + +"$DEVTOOLS_DIR"/khan-dotfiles/bin/install-mac-homebrew.py + +# It used to be we needed to install xcode-tools, now homebrew does this for us +#"$DEVTOOLS_DIR"/khan-dotfiles/bin/install-mac-gcc.sh + +install_protoc + +# We use java for our google cloud dataflow jobs that live in webapp +# (as well as in khan-linter for linting those jobs) +install_mac_java + +"$DEVTOOLS_DIR"/khan-dotfiles/bin/install-mac-apps.sh diff --git a/mac-setup-normal.sh b/mac-setup-normal.sh new file mode 100755 index 0000000..d3d926f --- /dev/null +++ b/mac-setup-normal.sh @@ -0,0 +1,384 @@ +#!/bin/bash + +# This script gets no elevated permissions. Do NOT run sudo or anything that +# expects sudo access. + +# Bail on any errors +set -e + +tty_bold=`tput bold` +tty_normal=`tput sgr0` + +# The directory to which all repositories will be cloned. +ROOT=${1-$HOME} +REPOS_DIR="$ROOT/khan" + +# Derived path location constants +DEVTOOLS_DIR="$REPOS_DIR/devtools" + +# Load shared setup functions. +. "$DEVTOOLS_DIR"/khan-dotfiles/shared-functions.sh + +# for printing standard echoish messages +notice () { + printf " $1\n" +} + +# for printing logging messages that *may* be replaced by +# a success/warn/error message +info () { + printf " [ \033[00;34m..\033[0m ] $1" +} + +# for printing prompts that expect user input and will be +# replaced by a success/warn/error message +user () { + printf "\r [ \033[0;33m??\033[0m ] $1 " +} + +# for replacing previous input prompts with success messages +success () { + printf "\r\033[2K [ \033[00;32mOK\033[0m ] $1\n" +} + +# for replacing previous input prompts with warnings +warn () { + printf "\r\033[2K [\033[0;33mWARN\033[0m] $1\n" +} + +# for replacing previous prompts with errors +error () { + printf "\r\033[2K [\033[0;31mFAIL\033[0m] $1\n" +} + +trap exit_warning EXIT # from shared-functions.sh + + +update_path() { + # We need /usr/local/bin to come before /usr/bin on the path, to + # pick up brew files we install. To do this, we just source + # .profile.khan, which does this for us (and the new user). + # (This assumes you're running mac-setup.sh from the khan-dotfiles + # directory.) + . .profile.khan +} + +maybe_generate_ssh_keys () { + # Create a public key if need be. + info "Checking for ssh keys" + mkdir -p ~/.ssh + if [ -s ~/.ssh/id_rsa ] || [ -s ~/.ssh/id_dsa ] + then + success "Found existing ssh keys" + else + ssh-keygen -q -N "" -t rsa -f ~/.ssh/id_rsa + success "Generated an rsa ssh key at ~/.ssh/id_rsa" + fi + return 0 +} + +copy_ssh_key () { + if [ -e ~/.ssh/id_rsa ] + then + pbcopy < ~/.ssh/id_rsa.pub + elif [ -e ~/.ssh/id_dsa ] + then + pbcopy < ~/.ssh/id_dsa.pub + else + error "no ssh public keys found" + exit + fi +} + +register_ssh_keys() { + success "Registering your ssh keys with github\n" + verify_ssh_auth +} + +# checks to see that ssh keys are registered with github +# $1: "true"|"false" to end the auth cycle +verify_ssh_auth () { + ssh_host="git@github.com" + webpage_url="https://github.com/settings/ssh" + instruction="Click 'Add SSH Key', paste into the box, and hit 'Add key'" + + info "Checking for GitHub ssh auth" + if ! ssh -T -v $ssh_host 2>&1 >/dev/null | grep \ + -q -e "Authentication succeeded (publickey)" + then + if [ "$2" == "false" ] # error if auth fails twice in a row + then + error "Still no luck with GitHub ssh auth. Ask a dev!" + ssh_auth_loop $webpage_url "false" + else + # otherwise prompt to upload keys + success "GitHub's ssh auth didn't seem to work\n" + notice "Let's add your public key to GitHub" + info "${tty_bold}${instruction}${tty_normal}\n" + ssh_auth_loop $webpage_url "true" + fi + else + success "GitHub ssh auth succeeded!" + fi +} + +ssh_auth_loop() { + # a convenience function which lets you copy your public key to your clipboard + # open the webpage for the site you're pasting the key into or just bailing + # $1 = ssh key registration url + service_url=$1 + first_run=$2 + if [ "$first_run" == "true" ] + then + notice "1. hit ${tty_bold}o${tty_normal} to open GitHub on the web" + notice "2. hit ${tty_bold}c${tty_normal} to copy your public key to your clipboard" + notice "3. hit ${tty_bold}t${tty_normal} to test ssh auth for GitHub" + notice "☢. hit ${tty_bold}s${tty_normal} to skip ssh setup for GitHub" + ssh_auth_loop $1 "false" + else + user "o|c|t|s) " + read -n1 ssh_option + case $ssh_option in + o|O ) + success "opening GitHub's webpage to register your key!" + open $service_url + ssh_auth_loop $service_url "false" + ;; + c|C ) + success "copying your ssh key to your clipboard" + copy_ssh_key + ssh_auth_loop $service_url "false" + ;; + t|T ) + printf "\r" + verify_ssh_auth "false" + ;; + s|S ) + warn "skipping GitHub ssh registration" + ;; + esac + fi +} + +update_git() { + if ! git --version | grep -q -e 'version 2\.[2-9][0-9]\.'; then + echo "Installing an updated version of git using Homebrew" + echo "Current version is `git --version`" + + if brew ls git >/dev/null 2>&1; then + # If git is already installed via brew, update it + brew upgrade git || true + else + # Otherwise, install via brew + brew install git || true + fi + + # Check git version again + if ! git --version | grep -q -e 'version 2\.[2-9][0-9]\.'; then + if ! brew ls --versions git | grep -q -e 'git 2\.[2-9][0-9]\.' ; then + echo "Error installing git via brew; download and install manually via http://git-scm.com/download/mac. " + read -p "Press enter to continue..." + else + echo "Git has been updated correctly, but will require restarting your terminal to take effect." + fi + fi + fi +} + +install_python2() { + # We only do this if python2 == /usr/bin/python2, which means it's system python + if [ "$(which python2)" != "/usr/bin/python2" ]; then + success "Already running a non-system python2." + return + fi + + info "Installing python2 from khan/repo. This may take a few minutes." + brew install khan/repo/python@2 +} + +install_node() { + if ! which node >/dev/null 2>&1; then + # Install node 12: It's LTS and the latest version supported on + # appengine standard. + brew install node@12 + + # We need this because brew doesn't link /usr/local/bin/node + # by default when installing non-latest node. + brew link --force --overwrite node@12 + fi + # We don't want to force usage of node v12, but we want to make clear we + # don't support anything else. + if ! node --version | grep "v12" >/dev/null ; then + notice "Your version of node is $(node --version). We currently only support v12." + if brew ls --versions node@12 >/dev/null ; then + notice "You do however have node 12 installed." + notice "Consider running:" + else + notice "Consider running:" + notice "\t${tty_bold}brew install node@12${tty_normal}" + fi + notice "\t${tty_bold}brew link --force --overwrite node@12${tty_normal}" + read -p "Press enter to continue..." + fi +} + +install_go() { + if ! has_recent_go; then # has_recent_go is from shared-functions.sh + info "Installing go\n" + if brew ls go >/dev/null 2>&1; then + brew upgrade "go@$DESIRED_GO_VERSION" + else + brew install "go@$DESIRED_GO_VERSION" + fi + + # Brew doesn't link non-latest versions of go on install. This command + # fixes that, telling the system that this is the go executable to use + brew link --force --overwrite "go@$DESIRED_GO_VERSION" + else + success "go already installed" + fi +} + +install_nginx() { + info "Checking for nginx\n" + if ! type nginx >/dev/null 2>&1; then + info "Installing nginx\n" + brew install nginx + else + success "nginx already installed" + fi +} + +install_redis() { + info "Checking for redis\n" + if ! type redis-cli >/dev/null 2>&1; then + info "Installing redis\n" + brew install redis + else + success "redis already installed" + fi + + if ! brew services list | grep redis | grep -q started; then + info "Starting redis service\n" + brew services start redis 2>&1 + else + success "redis service already started" + fi +} + +install_image_utils() { + info "Checking for imagemagick\n" + if ! brew ls imagemagick >/dev/null 2>&1; then + info "Installing imagemagick\n" + brew install imagemagick + else + success "imagemagick already installed" + fi +} + +install_helpful_tools() { + # This installs gtimeout, among a ton of other tools, which we use + # some in our deploy pipeline. + if ! brew ls coreutils >/dev/null 2>&1; then + info "Installing coreutils\n" + brew install coreutils + else + success "coreutils already installed" + fi +} + +install_wget() { + info "Checking for wget\n" + if ! which wget >/dev/null 2>&1; then + info "Installing wget\n" + brew install wget + else + success "wget already installed" + fi +} + +install_openssl() { + info "Checking for openssl\n" + if ! which openssl >/dev/null 2>&1; then + info "Installing openssl\n" + brew install openssl + else + success "openssl already installed" + fi + for source in $(brew --prefix openssl)/lib/*.dylib ; do + dest="/usr/local/lib/$(basename $source)" + # if dest is already a symlink pointing to the correct source, skip it + if [ -h "$dest" -a "$(readlink "$dest")" = "$source" ]; then + : + # else if dest already exists, warn user and skip dotfile + elif [ -e "$dest" ]; then + warn "Not symlinking to $dest because it already exists." + # otherwise, verbosely symlink the file (with --force) + else + info "Symlinking $(basename $source) " + ln -sfvn "$source" "$dest" + fi + done +} + +install_jq() { + info "Checking for jq\n" + if ! which jq >/dev/null 2>&1; then + info "Installing jq\n" + brew install jq + else + success "jq already installed" + fi +} + +install_python_tools() { + # We use various python versions (e.g. internal-service) + # and use Pyenv, pipenv as environment manager + if ! brew ls pyenv >/dev/null 2>&1; then + info "Installing pyenv\n" + brew install pyenv + # At the moment, we depend on MacOS coming with python 2.7. If that + # stops, or we want to align the python versions with the linux + # dotfiles more effectively, we could do it with pyenv: + # `pyenv install 2.7.16 ; pyenv global 2.7.16` + # Because the linux dotfiles do not yet install pyenv, holding off on + # using pyenv to enforce python version until either that happens, or + # MacOs stops including python 2.7 by default. + else + success "pyenv already installed" + fi +} + +install_watchman() { + if ! which watchman >/dev/null 2>&1; then + update "Installing watchman..." + brew install watchman + fi +} + +echo +success "Running Khan mac-setup-normal.sh\n" + +update_path +maybe_generate_ssh_keys +register_ssh_keys +install_wget +install_openssl +install_jq +update_git + +"$DEVTOOLS_DIR"/khan-dotfiles/bin/install-mac-python2.py + +install_node +install_go + +"$DEVTOOLS_DIR"/khan-dotfiles/bin/mac-setup-postgres.py + +install_nginx +install_redis +install_image_utils +install_helpful_tools +install_watchman +install_python_tools + +trap - EXIT diff --git a/mac-setup.sh b/mac-setup.sh index 10ad7f0..f796dac 100755 --- a/mac-setup.sh +++ b/mac-setup.sh @@ -3,8 +3,8 @@ # Bail on any errors set -e -tty_bold=`tput bold` -tty_normal=`tput sgr0` +# TODO(ericbrown): Support --quiet command line argument +# TODO(ericbrown): Use python?? # The directory to which all repositories will be cloned. ROOT=${1-$HOME} @@ -13,547 +13,37 @@ REPOS_DIR="$ROOT/khan" # Derived path location constants DEVTOOLS_DIR="$REPOS_DIR/devtools" -# Load shared setup functions. -. "$DEVTOOLS_DIR"/khan-dotfiles/shared-functions.sh - -# for printing standard echoish messages -notice () { - printf " $1\n" -} - -# for printing logging messages that *may* be replaced by -# a success/warn/error message -info () { - printf " [ \033[00;34m..\033[0m ] $1" -} - -# for printing prompts that expect user input and will be -# replaced by a success/warn/error message -user () { - printf "\r [ \033[0;33m??\033[0m ] $1 " -} - -# for replacing previous input prompts with success messages -success () { - printf "\r\033[2K [ \033[00;32mOK\033[0m ] $1\n" -} - -# for replacing previous input prompts with warnings -warn () { - printf "\r\033[2K [\033[0;33mWARN\033[0m] $1\n" -} - -# for replacing previous prompts with errors -error () { - printf "\r\033[2K [\033[0;31mFAIL\033[0m] $1\n" -} - -trap exit_warning EXIT # from shared-functions.sh - - -update_path() { - # We need /usr/local/bin to come before /usr/bin on the path, to - # pick up brew files we install. To do this, we just source - # .profile.khan, which does this for us (and the new user). - # (This assumes you're running mac-setup.sh from the khan-dotfiles - # directory.) - . .profile.khan -} - -maybe_generate_ssh_keys () { - # Create a public key if need be. - info "Checking for ssh keys" - mkdir -p ~/.ssh - if [ -s ~/.ssh/id_rsa ] || [ -s ~/.ssh/id_dsa ] - then - success "Found existing ssh keys" - else - ssh-keygen -q -N "" -t rsa -f ~/.ssh/id_rsa - success "Generated an rsa ssh key at ~/.ssh/id_rsa" - fi - return 0 -} - -copy_ssh_key () { - if [ -e ~/.ssh/id_rsa ] - then - pbcopy < ~/.ssh/id_rsa.pub - elif [ -e ~/.ssh/id_dsa ] - then - pbcopy < ~/.ssh/id_dsa.pub - else - error "no ssh public keys found" - exit - fi -} - -register_ssh_keys() { - success "Registering your ssh keys with github\n" - verify_ssh_auth -} - -# checks to see that ssh keys are registered with github -# $1: "true"|"false" to end the auth cycle -verify_ssh_auth () { - ssh_host="git@github.com" - webpage_url="https://github.com/settings/ssh" - instruction="Click 'Add SSH Key', paste into the box, and hit 'Add key'" - - info "Checking for GitHub ssh auth" - if ! ssh -T -v $ssh_host 2>&1 >/dev/null | grep \ - -q -e "Authentication succeeded (publickey)" - then - if [ "$2" == "false" ] # error if auth fails twice in a row - then - error "Still no luck with GitHub ssh auth. Ask a dev!" - ssh_auth_loop $webpage_url "false" - else - # otherwise prompt to upload keys - success "GitHub's ssh auth didn't seem to work\n" - notice "Let's add your public key to GitHub" - info "${tty_bold}${instruction}${tty_normal}\n" - ssh_auth_loop $webpage_url "true" - fi - else - success "GitHub ssh auth succeeded!" - fi -} - -ssh_auth_loop() { - # a convenience function which lets you copy your public key to your clipboard - # open the webpage for the site you're pasting the key into or just bailing - # $1 = ssh key registration url - service_url=$1 - first_run=$2 - if [ "$first_run" == "true" ] - then - notice "1. hit ${tty_bold}o${tty_normal} to open GitHub on the web" - notice "2. hit ${tty_bold}c${tty_normal} to copy your public key to your clipboard" - notice "3. hit ${tty_bold}t${tty_normal} to test ssh auth for GitHub" - notice "☢. hit ${tty_bold}s${tty_normal} to skip ssh setup for GitHub" - ssh_auth_loop $1 "false" - else - user "o|c|t|s) " - read -n1 ssh_option - case $ssh_option in - o|O ) - success "opening GitHub's webpage to register your key!" - open $service_url - ssh_auth_loop $service_url "false" - ;; - c|C ) - success "copying your ssh key to your clipboard" - copy_ssh_key - ssh_auth_loop $service_url "false" - ;; - t|T ) - printf "\r" - verify_ssh_auth "false" - ;; - s|S ) - warn "skipping GitHub ssh registration" - ;; - esac - fi -} - -install_gcc() { - info "\nChecking for Apple command line developer tools..." - if ! gcc --version >/dev/null 2>&1 || [ ! -s /usr/include/stdio.h ]; then - if sw_vers -productVersion | grep -e '^10\.[0-8]$' -e '^10\.[0-8]\.'; then - warn "Command line tools are *probably available* for your Mac's OS, but..." - info "why not upgrade your OS right now?\n" - info "Otherwise, you can always visit developer.apple.com and grab 'em there.\n" - exit 1 - fi - if ! gcc --version >/dev/null 2>&1 ; then - success "Installing command line developer tools" - # If enter is pressed before its done, not a big deal, but it'll just loop to the same place. - success "You'll want to wait until the xcode install is complete to press Enter again." - # Also, how did you get this dotfiles repo in 10.9 without - # git auto-triggering the command line tools install process?? - xcode-select --install - exec sh ./mac-setup.sh - # If this doesn't work for you, you can find the most recent - # version here: https://developer.apple.com/downloads - fi - if sw_vers -productVersion | grep -q -e '^10\.14\.' && [ ! -s /usr/include/stdio.h ]; then - # mac version is Mojave 10.14.*, install SDK headers - # The file "macOS_SDK_headers_for_macOS_10.14.pkg" is from - # xcode command line tools install - if [ -s /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg ]; then - # This command isn't guaranteed to work. If it fails, just warn - # the user there may be problems and advise they contact - # @dev-support if so. - if sudo installer -pkg /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg -target / ; then - success "macOS_SDK_headers_for_macOS_10.14 installed" - else - warn "We're not able to determine if stdio.h is able to be used by compilers correctly on your system." - warn "Please reach out to @dev-support if you encounter errors indicating this is a problem while building code or dependencies." - warn "You may be able to get more information about the setup by running ${tty_bold}gcc -v${tty_normal}" - fi - else - success "Updating your command line tools" - # If enter is pressed before its done, not a big deal, but it'll just loop to the same place. - success "You'll want to wait until the xcode install is complete to press Enter again." - sudo rm -rf /Library/Developer/CommandLineTools - xcode-select --install - exec sh ./mac-setup.sh - fi - fi - else - success "Great, found gcc! (assuming we also have other recent devtools)" - fi -} - -install_homebrew() { - info "Checking for mac homebrew" - # If homebrew is already installed, don't do it again. - if ! brew --help >/dev/null 2>&1; then - success "Brew not found. Installing!" - ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" - else - success "Great! Mac homebrew already installed!" - fi - success "Updating (but not upgrading) Homebrew" - brew update > /dev/null - - # Install homebrew-cask, so we can use it manage installing binary/GUI apps - # brew tap caskroom/cask - - # Likely need an alternate versions of Casks in order to install chrome-canary - # Required to install chrome-canary - brew tap homebrew/cask-versions - - # This is where we store our own formula, including a python@2 backport - brew tap khan/repo git@github.com:Khan/homebrew-repo.git - - # Make sure everything is ok. We don't care if we're using an - # obsolete gcc, so instead of looking at the exit code for 'brew - # doctor', we look at its output. The last 'grep', combined with - # the ! at the beginning of this command, causes the overall - # command to fail -- and thus the script to exit -- if brew doctor - # has any errors or warnings after we grep out the stuff we don't - # care about. - ## Commented out for now: too many legit setups have warnings (cf chris). - ## ! brew doctor 2>&1 \ - ## | grep -v -e 'A newer Command Line Tools' \ - ## | grep -v -e 'Your Homebrew is not installed to /usr/local' \ - ## | grep -C1000 -e ^Error -e ^Warning -} - -update_git() { - if ! git --version | grep -q -e 'version 2\.[2-9][0-9]\.'; then - echo "Installing an updated version of git using Homebrew" - echo "Current version is `git --version`" - - if brew ls git >/dev/null 2>&1; then - # If git is already installed via brew, update it - brew upgrade git || true - else - # Otherwise, install via brew - brew install git || true - fi - - # Check git version again - if ! git --version | grep -q -e 'version 2\.[2-9][0-9]\.'; then - if ! brew ls --versions git | grep -q -e 'git 2\.[2-9][0-9]\.' ; then - echo "Error installing git via brew; download and install manually via http://git-scm.com/download/mac. " - read -p "Press enter to continue..." - else - echo "Git has been updated correctly, but will require restarting your terminal to take effect." - fi - fi - fi -} - -install_python2() { - # We only do this if python2 == /usr/bin/python2, which means it's system python - if [ "$(which python2)" != "/usr/bin/python2" ]; then - success "Already running a non-system python2." - return - fi - - info "Installing python2 from khan/repo. This may take a few minutes." - brew install khan/repo/python@2 -} - -install_node() { - if ! which node >/dev/null 2>&1; then - # Install node 12: It's LTS and the latest version supported on - # appengine standard. - brew install node@12 - - # We need this because brew doesn't link /usr/local/bin/node - # by default when installing non-latest node. - brew link --force --overwrite node@12 - fi - # We don't want to force usage of node v12, but we want to make clear we - # don't support anything else. - if ! node --version | grep "v12" >/dev/null ; then - notice "Your version of node is $(node --version). We currently only support v12." - if brew ls --versions node@12 >/dev/null ; then - notice "You do however have node 12 installed." - notice "Consider running:" - else - notice "Consider running:" - notice "\t${tty_bold}brew install node@12${tty_normal}" - fi - notice "\t${tty_bold}brew link --force --overwrite node@12${tty_normal}" - read -p "Press enter to continue..." - fi -} - -install_go() { - if ! has_recent_go; then # has_recent_go is from shared-functions.sh - info "Installing go\n" - if brew ls go >/dev/null 2>&1; then - brew upgrade "go@$DESIRED_GO_VERSION" - else - brew install "go@$DESIRED_GO_VERSION" - fi - - # Brew doesn't link non-latest versions of go on install. This command - # fixes that, telling the system that this is the go executable to use - brew link --force --overwrite "go@$DESIRED_GO_VERSION" - else - success "go already installed" - fi -} - -install_postgresql() { - "$DEVTOOLS_DIR"/khan-dotfiles/bin/mac-setup-postgres.py -} - -install_nginx() { - info "Checking for nginx\n" - if ! type nginx >/dev/null 2>&1; then - info "Installing nginx\n" - brew install nginx - else - success "nginx already installed" - fi -} - -install_redis() { - info "Checking for redis\n" - if ! type redis-cli >/dev/null 2>&1; then - info "Installing redis\n" - brew install redis - else - success "redis already installed" - fi - - if ! brew services list | grep redis | grep -q started; then - info "Starting redis service\n" - brew services start redis 2>&1 - else - success "redis service already started" - fi -} - -install_image_utils() { - info "Checking for imagemagick\n" - if ! brew ls imagemagick >/dev/null 2>&1; then - info "Installing imagemagick\n" - brew install imagemagick - else - success "imagemagick already installed" - fi -} - -install_helpful_tools() { - # This installs gtimeout, among a ton of other tools, which we use - # some in our deploy pipeline. - if ! brew ls coreutils >/dev/null 2>&1; then - info "Installing coreutils\n" - brew install coreutils - else - success "coreutils already installed" - fi -} - -install_wget() { - info "Checking for wget\n" - if ! which wget >/dev/null 2>&1; then - info "Installing wget\n" - brew install wget - else - success "wget already installed" - fi -} - -install_openssl() { - info "Checking for openssl\n" - if ! which openssl >/dev/null 2>&1; then - info "Installing openssl\n" - brew install openssl - else - success "openssl already installed" - fi - for source in $(brew --prefix openssl)/lib/*.dylib ; do - dest="/usr/local/lib/$(basename $source)" - # if dest is already a symlink pointing to the correct source, skip it - if [ -h "$dest" -a "$(readlink "$dest")" = "$source" ]; then - : - # else if dest already exists, warn user and skip dotfile - elif [ -e "$dest" ]; then - warn "Not symlinking to $dest because it already exists." - # otherwise, verbosely symlink the file (with --force) - else - info "Symlinking $(basename $source) " - ln -sfvn "$source" "$dest" - fi - done -} - -install_jq() { - info "Checking for jq\n" - if ! which jq >/dev/null 2>&1; then - info "Installing jq\n" - brew install jq - else - success "jq already installed" - fi -} - -install_protoc() { - # If the user has a homebrew version of protobuf installed, uninstall it so - # we can manually install our own version in /usr/local. - if brew list | grep -q '^protobuf$'; then - info "Uninstalling homebrew version of protobuf\n" - brew uninstall protobuf - fi - - # The mac and linux installation process is the same from here on out aside - # from the platform-dependent zip archive. - install_protoc_common https://github.com/protocolbuffers/protobuf/releases/download/v3.4.0/protoc-3.4.0-osx-x86_64.zip -} - -install_python_tools() { - # We use various python versions (e.g. internal-service) - # and use Pyenv, pipenv as environment manager - if ! brew ls pyenv >/dev/null 2>&1; then - info "Installing pyenv\n" - brew install pyenv - # At the moment, we depend on MacOS coming with python 2.7. If that - # stops, or we want to align the python versions with the linux - # dotfiles more effectively, we could do it with pyenv: - # `pyenv install 2.7.16 ; pyenv global 2.7.16` - # Because the linux dotfiles do not yet install pyenv, holding off on - # using pyenv to enforce python version until either that happens, or - # MacOs stops including python 2.7 by default. - else - success "pyenv already installed" - fi -} - -install_watchman() { - if ! which watchman >/dev/null 2>&1; then - update "Installing watchman..." - brew install watchman - fi -} - -# To install some useful mac apps. -install_mac_apps() { - chosen_apps=() # When the user opts to install a package it will be added to this array. - - mac_apps=( - # Browsers - firefox firefox-developer-edition google-chrome google-chrome-canary - # Tools - dropbox google-drive-file-stream iterm2 virtualbox zoomus - # Text Editors - macvim sublime-text textmate atom - # Chat - slack - ) - - mac_apps_str="${mac_apps[@]}" - info "We recommend installing the following apps: ${mac_apps_str}. \n\n" - - read -r -p "Would you like to install [a]ll, [n]one, or [s]ome of the apps? [a/n/s]: " input - - case "$input" in - [aA][lL][lL] | [aA]) - chosen_apps=("${mac_apps[@]}") - ;; - [sS][oO][mM][eE] | [sS]) - for app in ${mac_apps[@]}; do - if [ "$(get_yn_input "Would you like to install ${app}?" "y")" = "y" ]; then - chosen_apps=("${chosen_apps[@]}" "${app}") - fi - done - ;; - [nN][oO][nN][eE] | [nN]) - ;; - *) - echo "Please choose [a]ll, [n]one, or [s]ome." - exit 100 - ;; - esac - - for app in ${chosen_apps[@]}; do - if ! brew cask ls $app >/dev/null 2>&1; then - info "$app is not installed, installing $app" - brew install --cask $app || warn "Failed to install $app, perhaps it is already installed." - else - success "$app already installed" - fi - done -} - echo -success "Running Khan Installation Script 1.2\n" +echo "Running Khan Installation Script 1.2" if ! sw_vers -productVersion 2>/dev/null | grep -q '10\.1[12345]\.' ; then - warn "Warning: This is only tested up to macOS 10.15 (Catalina).\n" - notice "If you find that this works on a newer version of macOS, " - notice "please update this message.\n" + echo "Warning: This is only tested up to macOS 10.15 (Catalina)." + echo + echo "If you find that this works on a newer version of macOS, " + echo "please update this message." + echo fi -notice "After each statement, either something will open for you to" -notice "interact with, or a script will run for you to use\n" -notice "Press enter when a download/install is completed to go to" -notice "the next step (including this one)" +echo "After each statement, either something will open for you to" +echo "interact with, or a script will run for you to use" +echo +echo "Press enter when a download/install is completed to go to" +echo "the next step (including this one)" if ! echo "$SHELL" | grep -q -e '/bash$' -e '/zsh$' ; then echo - warn "It looks like you're using a shell other than bash or zsh!" - notice "Other shells are not officially supported. Most things" - notice "should work, but dev-support help is not guaranteed." + echo "It looks like you're using a shell other than bash or zsh!" + echo "Other shells are not officially supported. Most things" + echo "should work, but dev-support help is not guaranteed." fi read -p "Press enter to continue..." -# Run sudo once at the beginning to get the necessary permissions. -notice "This setup script needs your password to install things as root." -sudo sh -c 'echo Thanks' +# TODO(ericbrown): Pass command line arguments below +# Note that ensure parsing arguments (above) doesn't hide anything -update_path -maybe_generate_ssh_keys -register_ssh_keys -install_gcc -install_homebrew -install_wget -install_openssl -install_jq -update_git -install_python2 -install_node -install_go -install_postgresql -install_nginx -install_redis -install_image_utils -install_helpful_tools -# We use java for our google cloud dataflow jobs that live in webapp -# (as well as in khan-linter for linting those jobs) -install_mac_java -install_protoc -install_watchman -install_mac_apps -install_python_tools +# Run setup that requires sudo access +"$DEVTOOLS_DIR"/khan-dotfiles/mac-setup-elevated.sh -trap - EXIT +# Run setup that does NOT require sudo access +"$DEVTOOLS_DIR"/khan-dotfiles/mac-setup-normal.sh