diff --git a/code/build.xml b/code/build.xml
index 342e0742..c8948ab1 100644
--- a/code/build.xml
+++ b/code/build.xml
@@ -62,7 +62,7 @@
-
+
@@ -70,17 +70,17 @@
-
+
-
+
-
+
@@ -88,17 +88,17 @@
-
+
-
+
-
+
@@ -106,12 +106,12 @@
-
+
-
+
@@ -119,17 +119,17 @@
-
+
-
+
-
+
@@ -137,17 +137,17 @@
-
+
-
+
-
+
@@ -155,17 +155,17 @@
-
+
-
+
-
+
@@ -173,17 +173,17 @@
-
+
-
+
-
+
@@ -191,12 +191,12 @@
-
+
-
+
@@ -204,12 +204,12 @@
-
+
-
+
@@ -217,12 +217,12 @@
-
+
-
+
@@ -230,12 +230,12 @@
-
+
-
+
@@ -243,17 +243,17 @@
-
+
-
+
-
+
@@ -261,17 +261,17 @@
-
+
-
+
-
+
@@ -279,37 +279,37 @@
-
+
-
+
-
+
-
+
-
+
-
+
-
+
@@ -317,12 +317,12 @@
-
+
-
+
@@ -330,17 +330,17 @@
-
+
-
+
-
+
@@ -348,17 +348,17 @@
-
+
-
+
-
+
diff --git a/intellij/code/.gradle/4.4/fileChanges/last-build.bin b/intellij/code/.gradle/4.4/fileChanges/last-build.bin
new file mode 100644
index 00000000..f76dd238
Binary files /dev/null and b/intellij/code/.gradle/4.4/fileChanges/last-build.bin differ
diff --git a/intellij/code/.gradle/4.4/fileContent/annotation-processors.bin b/intellij/code/.gradle/4.4/fileContent/annotation-processors.bin
new file mode 100644
index 00000000..787051a2
Binary files /dev/null and b/intellij/code/.gradle/4.4/fileContent/annotation-processors.bin differ
diff --git a/intellij/code/.gradle/4.4/fileContent/fileContent.lock b/intellij/code/.gradle/4.4/fileContent/fileContent.lock
new file mode 100644
index 00000000..b8dc939f
Binary files /dev/null and b/intellij/code/.gradle/4.4/fileContent/fileContent.lock differ
diff --git a/intellij/code/.gradle/4.4/fileHashes/fileHashes.bin b/intellij/code/.gradle/4.4/fileHashes/fileHashes.bin
new file mode 100644
index 00000000..d28d750c
Binary files /dev/null and b/intellij/code/.gradle/4.4/fileHashes/fileHashes.bin differ
diff --git a/intellij/code/.gradle/4.4/fileHashes/fileHashes.lock b/intellij/code/.gradle/4.4/fileHashes/fileHashes.lock
new file mode 100644
index 00000000..86d89edf
Binary files /dev/null and b/intellij/code/.gradle/4.4/fileHashes/fileHashes.lock differ
diff --git a/intellij/code/.gradle/4.4/fileHashes/resourceHashesCache.bin b/intellij/code/.gradle/4.4/fileHashes/resourceHashesCache.bin
new file mode 100644
index 00000000..486e5ee2
Binary files /dev/null and b/intellij/code/.gradle/4.4/fileHashes/resourceHashesCache.bin differ
diff --git a/intellij/code/.gradle/4.4/taskHistory/taskHistory.bin b/intellij/code/.gradle/4.4/taskHistory/taskHistory.bin
new file mode 100644
index 00000000..2dcbfef8
Binary files /dev/null and b/intellij/code/.gradle/4.4/taskHistory/taskHistory.bin differ
diff --git a/intellij/code/.gradle/4.4/taskHistory/taskHistory.lock b/intellij/code/.gradle/4.4/taskHistory/taskHistory.lock
new file mode 100644
index 00000000..a91e87a1
Binary files /dev/null and b/intellij/code/.gradle/4.4/taskHistory/taskHistory.lock differ
diff --git a/intellij/code/.gradle/buildOutputCleanup/buildOutputCleanup.lock b/intellij/code/.gradle/buildOutputCleanup/buildOutputCleanup.lock
new file mode 100644
index 00000000..d8f3fa14
Binary files /dev/null and b/intellij/code/.gradle/buildOutputCleanup/buildOutputCleanup.lock differ
diff --git a/intellij/code/.gradle/buildOutputCleanup/cache.properties b/intellij/code/.gradle/buildOutputCleanup/cache.properties
new file mode 100644
index 00000000..c5f68ed5
--- /dev/null
+++ b/intellij/code/.gradle/buildOutputCleanup/cache.properties
@@ -0,0 +1,2 @@
+#Sat Apr 28 15:54:56 KST 2018
+gradle.version=4.4
diff --git a/intellij/code/.gradle/buildOutputCleanup/outputFiles.bin b/intellij/code/.gradle/buildOutputCleanup/outputFiles.bin
new file mode 100644
index 00000000..5c3f6d7a
Binary files /dev/null and b/intellij/code/.gradle/buildOutputCleanup/outputFiles.bin differ
diff --git a/intellij/code/.idea/gradle.xml b/intellij/code/.idea/gradle.xml
new file mode 100644
index 00000000..267d99c5
--- /dev/null
+++ b/intellij/code/.idea/gradle.xml
@@ -0,0 +1,17 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/intellij/code/.idea/misc.xml b/intellij/code/.idea/misc.xml
new file mode 100644
index 00000000..4b495638
--- /dev/null
+++ b/intellij/code/.idea/misc.xml
@@ -0,0 +1,7 @@
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/intellij/code/.idea/workspace.xml b/intellij/code/.idea/workspace.xml
new file mode 100644
index 00000000..d5e57ecf
--- /dev/null
+++ b/intellij/code/.idea/workspace.xml
@@ -0,0 +1,690 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 1524898475722
+
+
+ 1524898475722
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/intellij/code/build.gradle b/intellij/code/build.gradle
new file mode 100644
index 00000000..19d939be
--- /dev/null
+++ b/intellij/code/build.gradle
@@ -0,0 +1,18 @@
+
+apply plugin: 'java'
+
+sourceCompatibility = 1.8
+
+repositories {
+ mavenCentral()
+}
+
+dependencies {
+ implementation 'junit:junit:4.12'
+ implementation 'org.hamcrest:hamcrest-core:1.3'
+ implementation 'org.hamcrest:hamcrest-library:1.3'
+ implementation 'org.apache.commons:commons-math3:3.6'
+ implementation 'org.jfree:jfreechart:1.0.19'
+ implementation 'org.jsoup:jsoup:1.8.3'
+ implementation 'redis.clients:jedis:2.8.0'
+}
diff --git a/intellij/code/gradle/wrapper/gradle-wrapper.properties b/intellij/code/gradle/wrapper/gradle-wrapper.properties
new file mode 100644
index 00000000..490ccddd
--- /dev/null
+++ b/intellij/code/gradle/wrapper/gradle-wrapper.properties
@@ -0,0 +1,6 @@
+#Sat Apr 28 15:57:44 KST 2018
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.4-all.zip
diff --git a/intellij/code/gradlew b/intellij/code/gradlew
new file mode 100644
index 00000000..cccdd3d5
--- /dev/null
+++ b/intellij/code/gradlew
@@ -0,0 +1,172 @@
+#!/usr/bin/env sh
+
+##############################################################################
+##
+## Gradle start up script for UN*X
+##
+##############################################################################
+
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+# Need this for relative symlinks.
+while [ -h "$PRG" ] ; do
+ ls=`ls -ld "$PRG"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '/.*' > /dev/null; then
+ PRG="$link"
+ else
+ PRG=`dirname "$PRG"`"/$link"
+ fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >/dev/null
+APP_HOME="`pwd -P`"
+cd "$SAVED" >/dev/null
+
+APP_NAME="Gradle"
+APP_BASE_NAME=`basename "$0"`
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS=""
+
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD="maximum"
+
+warn () {
+ echo "$*"
+}
+
+die () {
+ echo
+ echo "$*"
+ echo
+ exit 1
+}
+
+# OS specific support (must be 'true' or 'false').
+cygwin=false
+msys=false
+darwin=false
+nonstop=false
+case "`uname`" in
+ CYGWIN* )
+ cygwin=true
+ ;;
+ Darwin* )
+ darwin=true
+ ;;
+ MINGW* )
+ msys=true
+ ;;
+ NONSTOP* )
+ nonstop=true
+ ;;
+esac
+
+CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
+# Determine the Java command to use to start the JVM.
+if [ -n "$JAVA_HOME" ] ; then
+ if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+ # IBM's JDK on AIX uses strange locations for the executables
+ JAVACMD="$JAVA_HOME/jre/sh/java"
+ else
+ JAVACMD="$JAVA_HOME/bin/java"
+ fi
+ if [ ! -x "$JAVACMD" ] ; then
+ die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+ fi
+else
+ JAVACMD="java"
+ which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+fi
+
+# Increase the maximum file descriptors if we can.
+if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
+ MAX_FD_LIMIT=`ulimit -H -n`
+ if [ $? -eq 0 ] ; then
+ if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
+ MAX_FD="$MAX_FD_LIMIT"
+ fi
+ ulimit -n $MAX_FD
+ if [ $? -ne 0 ] ; then
+ warn "Could not set maximum file descriptor limit: $MAX_FD"
+ fi
+ else
+ warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
+ fi
+fi
+
+# For Darwin, add options to specify how the application appears in the dock
+if $darwin; then
+ GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
+fi
+
+# For Cygwin, switch paths to Windows format before running java
+if $cygwin ; then
+ APP_HOME=`cygpath --path --mixed "$APP_HOME"`
+ CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+ JAVACMD=`cygpath --unix "$JAVACMD"`
+
+ # We build the pattern for arguments to be converted via cygpath
+ ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
+ SEP=""
+ for dir in $ROOTDIRSRAW ; do
+ ROOTDIRS="$ROOTDIRS$SEP$dir"
+ SEP="|"
+ done
+ OURCYGPATTERN="(^($ROOTDIRS))"
+ # Add a user-defined pattern to the cygpath arguments
+ if [ "$GRADLE_CYGPATTERN" != "" ] ; then
+ OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
+ fi
+ # Now convert the arguments - kludge to limit ourselves to /bin/sh
+ i=0
+ for arg in "$@" ; do
+ CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
+ CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
+
+ if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
+ eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
+ else
+ eval `echo args$i`="\"$arg\""
+ fi
+ i=$((i+1))
+ done
+ case $i in
+ (0) set -- ;;
+ (1) set -- "$args0" ;;
+ (2) set -- "$args0" "$args1" ;;
+ (3) set -- "$args0" "$args1" "$args2" ;;
+ (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
+ (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
+ (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
+ (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
+ (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
+ (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
+ esac
+fi
+
+# Escape application args
+save () {
+ for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
+ echo " "
+}
+APP_ARGS=$(save "$@")
+
+# Collect all arguments for the java command, following the shell quoting and substitution rules
+eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
+
+# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
+if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
+ cd "$(dirname "$0")"
+fi
+
+exec "$JAVACMD" "$@"
diff --git a/intellij/code/gradlew.bat b/intellij/code/gradlew.bat
new file mode 100644
index 00000000..f9553162
--- /dev/null
+++ b/intellij/code/gradlew.bat
@@ -0,0 +1,84 @@
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto init
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto init
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:init
+@rem Get command-line arguments, handling Windows variants
+
+if not "%OS%" == "Windows_NT" goto win9xME_args
+
+:win9xME_args
+@rem Slurp the command line arguments.
+set CMD_LINE_ARGS=
+set _SKIP=2
+
+:win9xME_args_slurp
+if "x%~1" == "x" goto execute
+
+set CMD_LINE_ARGS=%*
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega
diff --git a/intellij/code/settings.gradle b/intellij/code/settings.gradle
new file mode 100644
index 00000000..5c7c7827
--- /dev/null
+++ b/intellij/code/settings.gradle
@@ -0,0 +1,2 @@
+rootProject.name = 'code'
+
diff --git a/intellij/code/src/main/java/com/allendowney/thinkdast/Card.java b/intellij/code/src/main/java/com/allendowney/thinkdast/Card.java
new file mode 100644
index 00000000..e2dc4978
--- /dev/null
+++ b/intellij/code/src/main/java/com/allendowney/thinkdast/Card.java
@@ -0,0 +1,148 @@
+package com.allendowney.thinkdast;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+
+/**
+ * Represents a playing card.
+ *
+ */
+public class Card implements Comparable {
+
+ // string representations of ranks
+ public static final String[] RANKS = {
+ null, "Ace", "2", "3", "4", "5", "6", "7",
+ "8", "9", "10", "Jack", "Queen", "King"};
+
+ // string representations of suits
+ public static final String[] SUITS = {
+ "Clubs", "Diamonds", "Hearts", "Spades"};
+
+ // rank and suit are instance variables
+ private final int rank;
+ private final int suit;
+
+ /**
+ * Constructs a card of the given rank and suit.
+ */
+ public Card(int rank, int suit) {
+ this.rank = rank;
+ this.suit = suit;
+ }
+
+ /**
+ * Gets the card's rank.
+ */
+ public int getRank() {
+ return this.rank;
+ }
+
+ /**
+ * Gets the card's suit.
+ */
+ public int getSuit() {
+ return this.suit;
+ }
+
+ /**
+ * Returns a string representation of the card.
+ */
+ public String toString() {
+ return RANKS[this.rank] + " of " + SUITS[this.suit];
+ }
+
+ /**
+ * Returns a negative integer if this card comes before
+ * the given card, zero if the two cards are equal, or
+ * a positive integer if this card comes after the card.
+ */
+ public int compareTo(Card that) {
+ if (this.suit < that.suit) {
+ return -1;
+ }
+ if (this.suit > that.suit) {
+ return 1;
+ }
+ if (this.rank < that.rank) {
+ return -1;
+ }
+ if (this.rank > that.rank) {
+ return 1;
+ }
+ return 0;
+ }
+
+ /**
+ * Returns true if the given card has the same
+ * rank AND same suit; otherwise returns false.
+ */
+ public boolean equals(Card that) {
+ return this.rank == that.rank
+ && this.suit == that.suit;
+ }
+
+ /**
+ * Make a List of 52 cards.
+ */
+ public static List makeDeck() {
+ List cards = new ArrayList();
+ for (int suit = 0; suit <= 3; suit++) {
+ for (int rank = 1; rank <= 13; rank++) {
+ Card card = new Card(rank, suit);
+ cards.add(card);
+ }
+ }
+ return cards;
+ }
+
+ /**
+ * Demonstrates how to call the search methods.
+ */
+ public static void main(String[] args) {
+
+ // sort the cards using the natural ordering
+ List cards = makeDeck();
+ Collections.sort(cards);
+ System.out.println(cards.get(0));
+ System.out.println(cards.get(51));
+
+ Comparator comparator = new Comparator() {
+ @Override
+ public int compare(Card card1, Card card2) {
+ if (card1.getSuit() < card2.getSuit()) {
+ return -1;
+ }
+ if (card1.getSuit() > card2.getSuit()) {
+ return 1;
+ }
+ int rank1 = getRankAceHigh(card1);
+ int rank2 = getRankAceHigh(card2);
+
+ if (rank1 < rank2) {
+ return -1;
+ }
+ if (rank1 > rank2) {
+ return 1;
+ }
+ return 0;
+ }
+
+ private int getRankAceHigh(Card card) {
+ int rank = card.getRank();
+ if (rank == 1) {
+ return 14;
+ } else {
+ return rank;
+ }
+ }
+ };
+
+ // sort the cards using an external comparator
+ Collections.sort(cards, comparator);
+ System.out.println(cards.get(0));
+ System.out.println(cards.get(51));
+ }
+}
diff --git a/intellij/code/src/main/java/com/allendowney/thinkdast/HelloJsoup.java b/intellij/code/src/main/java/com/allendowney/thinkdast/HelloJsoup.java
new file mode 100644
index 00000000..77958d6f
--- /dev/null
+++ b/intellij/code/src/main/java/com/allendowney/thinkdast/HelloJsoup.java
@@ -0,0 +1,79 @@
+package com.allendowney.thinkdast;
+
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Example program to list links from a URL.
+ *
+ * From: http://jsoup.org/cookbook/extracting-data/example-list-links
+ */
+public class HelloJsoup {
+
+ public static void main(String[] args) throws IOException {
+
+ String dirname = System.getProperty("user.dir");
+ String filename = "src/main/resources/en.wikipedia.org/wiki/Computer_science";
+ String baseURI = dirname + "/" + filename;
+
+ File input = new File(baseURI);
+ Document doc = Jsoup.parse(input, "UTF-8", baseURI);
+
+ //print("Fetching %s...", url);
+ //Document doc = Jsoup.connect(url).get();
+
+ Element content = doc.getElementById("mw-content-text");
+ Elements paragraphs = content.getElementsByTag("p");
+
+
+ for (Element p : paragraphs) {
+ Elements links = p.select("a[href]");
+ for (Element link : links) {
+ print(" * a: <%s> (%s)", link.attr("href"), trim(link.text(), 35));
+ }
+ break;
+ }
+
+
+// Elements media = doc.select("[src]");
+// Elements imports = doc.select("link[href]");
+//
+// print("\nMedia: (%d)", media.size());
+// for (Element src : media) {
+// if (src.tagName().equals("img")) {
+// print(" * %s: <%s> %sx%s (%s)",
+// src.tagName(), src.attr("abs:src"), src.attr("width"), src.attr("height"),
+// trim(src.attr("alt"), 20));
+// } else {
+// print(" * %s: <%s>", src.tagName(), src.attr("abs:src"));
+// }
+// }
+//
+// print("\nImports: (%d)", imports.size());
+// for (Element link : imports) {
+// print(" * %s <%s> (%s)", link.tagName(),link.attr("abs:href"), link.attr("rel"));
+// }
+//
+// print("\nLinks: (%d)", links.size());
+// for (Element link : links) {
+// print(" * a: <%s> (%s)", link.attr("abs:href"), trim(link.text(), 35));
+// break;
+// }
+ }
+
+ private static void print(String msg, Object... args) {
+ System.out.println(String.format(msg, args));
+ }
+
+ private static String trim(String s, int width) {
+ if (s.length() > width)
+ return s.substring(0, width-1) + ".";
+ else
+ return s;
+ }
+}
diff --git a/intellij/code/src/main/java/com/allendowney/thinkdast/Index.java b/intellij/code/src/main/java/com/allendowney/thinkdast/Index.java
new file mode 100644
index 00000000..47be7803
--- /dev/null
+++ b/intellij/code/src/main/java/com/allendowney/thinkdast/Index.java
@@ -0,0 +1,108 @@
+package com.allendowney.thinkdast;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.HashSet;
+
+import org.jsoup.select.Elements;
+
+/**
+ * Encapsulates a map from search term to set of TermCounter.
+ *
+ * @author downey
+ *
+ */
+public class Index {
+
+ private Map> index = new HashMap>();
+
+ /**
+ * Adds a TermCounter to the set associated with `term`.
+ *
+ * @param term
+ * @param tc
+ */
+ public void add(String term, TermCounter tc) {
+ Set set = get(term);
+
+ // if we're seeing a term for the first time, make a new Set
+ if (set == null) {
+ set = new HashSet();
+ index.put(term, set);
+ }
+ // otherwise we can modify an existing Set
+ set.add(tc);
+ }
+
+ /**
+ * Looks up a search term and returns a set of TermCounters.
+ *
+ * @param term
+ * @return
+ */
+ public Set get(String term) {
+ return index.get(term);
+ }
+
+ /**
+ * Prints the contents of the index.
+ */
+ public void printIndex() {
+ // loop through the search terms
+ for (String term: keySet()) {
+ System.out.println(term);
+
+ // for each term, print the pages where it appears
+ Set tcs = get(term);
+ for (TermCounter tc: tcs) {
+ Integer count = tc.get(term);
+ System.out.println(" " + tc.getLabel() + " " + count);
+ }
+ }
+ }
+
+ /**
+ * Returns the set of terms that have been indexed.
+ *
+ * @return
+ */
+ public Set keySet() {
+ return index.keySet();
+ }
+
+ /**
+ * Add a page to the index.
+ *
+ * @param url URL of the page.
+ * @param paragraphs Collection of elements that should be indexed.
+ */
+ public void indexPage(String url, Elements paragraphs) {
+ // TODO: Your code here
+
+ // make a TermCounter and count the terms in the paragraphs
+
+ // for each term in the TermCounter, add the TermCounter to the index
+ }
+
+ /**
+ * @param args
+ * @throws IOException
+ */
+ public static void main(String[] args) throws IOException {
+
+ WikiFetcher wf = new WikiFetcher();
+ Index indexer = new Index();
+
+ String url = "https://en.wikipedia.org/wiki/Java_(programming_language)";
+ Elements paragraphs = wf.fetchWikipedia(url);
+ indexer.indexPage(url, paragraphs);
+
+ url = "https://en.wikipedia.org/wiki/Programming_language";
+ paragraphs = wf.fetchWikipedia(url);
+ indexer.indexPage(url, paragraphs);
+
+ indexer.printIndex();
+ }
+}
\ No newline at end of file
diff --git a/intellij/code/src/main/java/com/allendowney/thinkdast/IndexTest.java b/intellij/code/src/main/java/com/allendowney/thinkdast/IndexTest.java
new file mode 100644
index 00000000..7b8abb64
--- /dev/null
+++ b/intellij/code/src/main/java/com/allendowney/thinkdast/IndexTest.java
@@ -0,0 +1,47 @@
+package com.allendowney.thinkdast;
+
+import static org.junit.Assert.*;
+import static org.hamcrest.CoreMatchers.*;
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.jsoup.select.Elements;
+import org.junit.Before;
+import org.junit.Test;
+
+public class IndexTest {
+
+ private Index index;
+ private WikiFetcher wf;
+
+ @Before
+ public void setUp() {
+ wf = new WikiFetcher();
+ index = new Index();
+ }
+
+ @Test
+ public void testIndexPage() throws IOException {
+ // add two pages to the index
+ String url = "https://en.wikipedia.org/wiki/Java_(programming_language)";
+ Elements paragraphs = wf.readWikipedia(url);
+ index.indexPage(url, paragraphs);
+
+ url = "https://en.wikipedia.org/wiki/Programming_language";
+ paragraphs = wf.readWikipedia(url);
+ index.indexPage(url, paragraphs);
+
+ // check the results: the word "occur" only appears on one page, twice
+ Set set = index.get("occur");
+ assertThat(set.size(), is(1));
+
+ for (TermCounter tc: set) {
+ // this loop only happens once
+ assertThat(tc.size(), is(4798));
+ assertThat(tc.get("occur"), is(2));
+ assertThat(tc.get("not there"), is(0));
+ }
+ }
+
+}
diff --git a/intellij/code/src/main/java/com/allendowney/thinkdast/JedisIndex.java b/intellij/code/src/main/java/com/allendowney/thinkdast/JedisIndex.java
new file mode 100644
index 00000000..df05e278
--- /dev/null
+++ b/intellij/code/src/main/java/com/allendowney/thinkdast/JedisIndex.java
@@ -0,0 +1,263 @@
+package com.allendowney.thinkdast;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+
+import org.jsoup.select.Elements;
+
+import redis.clients.jedis.Jedis;
+import redis.clients.jedis.Transaction;
+
+/**
+ * Represents a Redis-backed web search index.
+ *
+ */
+public class JedisIndex {
+
+ private Jedis jedis;
+
+ /**
+ * Constructor.
+ *
+ * @param jedis
+ */
+ public JedisIndex(Jedis jedis) {
+ this.jedis = jedis;
+ }
+
+ /**
+ * Returns the Redis key for a given search term.
+ *
+ * @return Redis key.
+ */
+ private String urlSetKey(String term) {
+ return "URLSet:" + term;
+ }
+
+ /**
+ * Returns the Redis key for a URL's TermCounter.
+ *
+ * @return Redis key.
+ */
+ private String termCounterKey(String url) {
+ return "TermCounter:" + url;
+ }
+
+ /**
+ * Checks whether we have a TermCounter for a given URL.
+ *
+ * @param url
+ * @return
+ */
+ public boolean isIndexed(String url) {
+ String redisKey = termCounterKey(url);
+ return jedis.exists(redisKey);
+ }
+
+ /**
+ * Adds a URL to the set associated with `term`.
+ *
+ * @param term
+ * @param tc
+ */
+ public void add(String term, TermCounter tc) {
+ jedis.sadd(urlSetKey(term), tc.getLabel());
+ }
+
+ /**
+ * Looks up a search term and returns a set of URLs.
+ *
+ * @param term
+ * @return Set of URLs.
+ */
+ public Set getURLs(String term) {
+ // FILL THIS IN!
+ return null;
+ }
+
+ /**
+ * Looks up a term and returns a map from URL to count.
+ *
+ * @param term
+ * @return Map from URL to count.
+ */
+ public Map getCounts(String term) {
+ // FILL THIS IN!
+ return null;
+ }
+
+ /**
+ * Returns the number of times the given term appears at the given URL.
+ *
+ * @param url
+ * @param term
+ * @return
+ */
+ public Integer getCount(String url, String term) {
+ // FILL THIS IN!
+ return null;
+ }
+
+ /**
+ * Adds a page to the index.
+ *
+ * @param url URL of the page.
+ * @param paragraphs Collection of elements that should be indexed.
+ */
+ public void indexPage(String url, Elements paragraphs) {
+ // TODO: FILL THIS IN!
+ }
+
+ /**
+ * Prints the contents of the index.
+ *
+ * Should be used for development and testing, not production.
+ */
+ public void printIndex() {
+ // loop through the search terms
+ for (String term: termSet()) {
+ System.out.println(term);
+
+ // for each term, print the pages where it appears
+ Set urls = getURLs(term);
+ for (String url: urls) {
+ Integer count = getCount(url, term);
+ System.out.println(" " + url + " " + count);
+ }
+ }
+ }
+
+ /**
+ * Returns the set of terms that have been indexed.
+ *
+ * Should be used for development and testing, not production.
+ *
+ * @return
+ */
+ public Set termSet() {
+ Set keys = urlSetKeys();
+ Set terms = new HashSet();
+ for (String key: keys) {
+ String[] array = key.split(":");
+ if (array.length < 2) {
+ terms.add("");
+ } else {
+ terms.add(array[1]);
+ }
+ }
+ return terms;
+ }
+
+ /**
+ * Returns URLSet keys for the terms that have been indexed.
+ *
+ * Should be used for development and testing, not production.
+ *
+ * @return
+ */
+ public Set urlSetKeys() {
+ return jedis.keys("URLSet:*");
+ }
+
+ /**
+ * Returns TermCounter keys for the URLS that have been indexed.
+ *
+ * Should be used for development and testing, not production.
+ *
+ * @return
+ */
+ public Set termCounterKeys() {
+ return jedis.keys("TermCounter:*");
+ }
+
+ /**
+ * Deletes all URLSet objects from the database.
+ *
+ * Should be used for development and testing, not production.
+ *
+ * @return
+ */
+ public void deleteURLSets() {
+ Set keys = urlSetKeys();
+ Transaction t = jedis.multi();
+ for (String key: keys) {
+ t.del(key);
+ }
+ t.exec();
+ }
+
+ /**
+ * Deletes all URLSet objects from the database.
+ *
+ * Should be used for development and testing, not production.
+ *
+ * @return
+ */
+ public void deleteTermCounters() {
+ Set keys = termCounterKeys();
+ Transaction t = jedis.multi();
+ for (String key: keys) {
+ t.del(key);
+ }
+ t.exec();
+ }
+
+ /**
+ * Deletes all keys from the database.
+ *
+ * Should be used for development and testing, not production.
+ *
+ * @return
+ */
+ public void deleteAllKeys() {
+ Set keys = jedis.keys("*");
+ Transaction t = jedis.multi();
+ for (String key: keys) {
+ t.del(key);
+ }
+ t.exec();
+ }
+
+ /**
+ * @param args
+ * @throws IOException
+ */
+ public static void main(String[] args) throws IOException {
+ Jedis jedis = JedisMaker.make();
+ JedisIndex index = new JedisIndex(jedis);
+
+ //index.deleteTermCounters();
+ //index.deleteURLSets();
+ //index.deleteAllKeys();
+ loadIndex(index);
+
+ Map map = index.getCounts("the");
+ for (Entry entry: map.entrySet()) {
+ System.out.println(entry);
+ }
+ }
+
+ /**
+ * Stores two pages in the index for testing purposes.
+ *
+ * @return
+ * @throws IOException
+ */
+ private static void loadIndex(JedisIndex index) throws IOException {
+ WikiFetcher wf = new WikiFetcher();
+
+ String url = "https://en.wikipedia.org/wiki/Java_(programming_language)";
+ Elements paragraphs = wf.readWikipedia(url);
+ index.indexPage(url, paragraphs);
+
+ url = "https://en.wikipedia.org/wiki/Programming_language";
+ paragraphs = wf.readWikipedia(url);
+ index.indexPage(url, paragraphs);
+ }
+}
diff --git a/intellij/code/src/main/java/com/allendowney/thinkdast/JedisIndexTest.java b/intellij/code/src/main/java/com/allendowney/thinkdast/JedisIndexTest.java
new file mode 100644
index 00000000..4af80aad
--- /dev/null
+++ b/intellij/code/src/main/java/com/allendowney/thinkdast/JedisIndexTest.java
@@ -0,0 +1,72 @@
+package com.allendowney.thinkdast;
+
+import static org.junit.Assert.*;
+import static org.hamcrest.CoreMatchers.*;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.jsoup.select.Elements;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import redis.clients.jedis.Jedis;
+
+/**
+ * @author downey
+ *
+ */
+public class JedisIndexTest {
+
+ private static String url1, url2;
+ private Jedis jedis;
+ private JedisIndex index;
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @Before
+ public void setUp() throws Exception {
+ jedis = JedisMaker.make();
+ index = new JedisIndex(jedis);
+
+ loadIndex(index);
+ }
+
+ /**
+ * Loads the index with two pages read from files.
+ *
+ * @return
+ * @throws IOException
+ */
+ private static void loadIndex(JedisIndex index) throws IOException {
+ WikiFetcher wf = new WikiFetcher();
+
+ url1 = "https://en.wikipedia.org/wiki/Java_(programming_language)";
+ Elements paragraphs = wf.readWikipedia(url1);
+ index.indexPage(url1, paragraphs);
+
+ url2 = "https://en.wikipedia.org/wiki/Programming_language";
+ paragraphs = wf.readWikipedia(url2);
+ index.indexPage(url2, paragraphs);
+ }
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @After
+ public void tearDown() throws Exception {
+ jedis.close();
+ }
+
+ /**
+ * Test method for {@link JedisIndex#getCounts(java.lang.String)}.
+ */
+ @Test
+ public void testGetCounts() {
+ Map map = index.getCounts("the");
+ assertThat(map.get(url1), is(339));
+ assertThat(map.get(url2), is(264));
+ }
+}
diff --git a/intellij/code/src/main/java/com/allendowney/thinkdast/JedisMaker.java b/intellij/code/src/main/java/com/allendowney/thinkdast/JedisMaker.java
new file mode 100644
index 00000000..76a18aad
--- /dev/null
+++ b/intellij/code/src/main/java/com/allendowney/thinkdast/JedisMaker.java
@@ -0,0 +1,127 @@
+package com.allendowney.thinkdast;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.net.URLDecoder;
+
+import redis.clients.jedis.Jedis;
+
+
+public class JedisMaker {
+
+ /**
+ * Make a Jedis object and authenticate it.
+ *
+ * @return
+ * @throws IOException
+ */
+ public static Jedis make() throws IOException {
+
+ // assemble the directory name
+ String filename = "redis_url.txt";
+ URL fileURL = JedisMaker.class.getClassLoader().getResource(filename);
+ String filepath = URLDecoder.decode(fileURL.getFile(), "UTF-8");
+
+ // open the file
+ StringBuilder sb = new StringBuilder();
+ BufferedReader br;
+ try {
+ br = new BufferedReader(new FileReader(filepath));
+ } catch (FileNotFoundException e1) {
+ System.out.println("File not found: " + filename);
+ printInstructions();
+ return null;
+ }
+
+ // read the file
+ while (true) {
+ String line = br.readLine();
+ if (line == null) break;
+ sb.append(line);
+ }
+ br.close();
+
+ // parse the URL
+ URI uri;
+ try {
+ uri = new URI(sb.toString());
+ } catch (URISyntaxException e) {
+ System.out.println("Reading file: " + filename);
+ System.out.println("It looks like this file does not contain a valid URI.");
+ printInstructions();
+ return null;
+ }
+ String host = uri.getHost();
+ int port = uri.getPort();
+
+ String[] array = uri.getAuthority().split("[:@]");
+ String auth = array[1];
+
+ // connect to the server
+ Jedis jedis = new Jedis(host, port);
+
+ try {
+ jedis.auth(auth);
+ } catch (Exception e) {
+ System.out.println("Trying to connect to " + host);
+ System.out.println("on port " + port);
+ System.out.println("with authcode " + auth);
+ System.out.println("Got exception " + e);
+ printInstructions();
+ return null;
+ }
+ return jedis;
+ }
+
+
+ /**
+ *
+ */
+ private static void printInstructions() {
+ System.out.println("");
+ System.out.println("To connect to RedisToGo, you have to provide a file called");
+ System.out.println("redis_url.txt that contains the URL of your Redis server.");
+ System.out.println("If you select an instance on the RedisToGo web page,");
+ System.out.println("you should see a URL that contains the information you need:");
+ System.out.println("redis://redistogo:AUTH@HOST:PORT");
+ System.out.println("Create a file called redis_url.txt in the src/resources");
+ System.out.println("directory, and paste in the URL.");
+ }
+
+
+ /**
+ * @param args
+ * @throws IOException
+ */
+ public static void main(String[] args) throws IOException {
+
+ Jedis jedis = make();
+
+ // String
+ jedis.set("mykey", "myvalue");
+ String value = jedis.get("mykey");
+ System.out.println("Got value: " + value);
+
+ // Set
+ jedis.sadd("myset", "element1", "element2", "element3");
+ System.out.println("element2 is member: " + jedis.sismember("myset", "element2"));
+
+ // List
+ jedis.rpush("mylist", "element1", "element2", "element3");
+ System.out.println("element at index 1: " + jedis.lindex("mylist", 1));
+
+ // Hash
+ jedis.hset("myhash", "word1", Integer.toString(2));
+ jedis.hincrBy("myhash", "word2", 1);
+ System.out.println("frequency of word1: " + jedis.hget("myhash", "word1"));
+ System.out.println("frequency of word2: " + jedis.hget("myhash", "word2"));
+
+ jedis.close();
+ }
+}
diff --git a/intellij/code/src/main/java/com/allendowney/thinkdast/JedisTermCounter.java b/intellij/code/src/main/java/com/allendowney/thinkdast/JedisTermCounter.java
new file mode 100644
index 00000000..0c2469cf
--- /dev/null
+++ b/intellij/code/src/main/java/com/allendowney/thinkdast/JedisTermCounter.java
@@ -0,0 +1,78 @@
+package com.allendowney.thinkdast;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import org.jsoup.select.Elements;
+
+import redis.clients.jedis.Jedis;
+import redis.clients.jedis.Transaction;
+
+
+/**
+ * Encapsulates a map from search term to frequency (count).
+ *
+ * @author downey
+ *
+ */
+public class JedisTermCounter extends TermCounter {
+
+ public JedisTermCounter(String label) {
+ super(label);
+ }
+
+ /**
+ *
+ * @return
+ */
+ public List