Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implemented an "unmunger" #55

Closed
wants to merge 5 commits into from
Closed

Implemented an "unmunger" #55

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
2ea4841
Implemented an "unmunger" to solve #45
Jun 8, 2020
786b921
Performance improvements for the unmunger
Jun 11, 2020
1dea35c
Moved MungeTable and PasswordChain to resources package
Jun 11, 2020
ebf117e
Added optimization from the old implementation
Jun 12, 2020
5cde668
Redid optimization from previous commit
Jun 12, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,8 @@
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
<configuration>
<source>1.7</source>
<target>1.7</target>
<source>8</source>
<target>8</target>
</configuration>
</plugin>
<plugin>
Expand Down
116 changes: 71 additions & 45 deletions src/main/java/me/gosimple/nbvcxz/matching/DictionaryMatcher.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,10 @@
import me.gosimple.nbvcxz.matching.match.Match;
import me.gosimple.nbvcxz.resources.Configuration;
import me.gosimple.nbvcxz.resources.Dictionary;
import me.gosimple.nbvcxz.resources.MungeTable;
import me.gosimple.nbvcxz.resources.PasswordChain;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.TreeMap;
import java.util.*;

/**
* Look for every part of the password that match an entry in our dictionaries
Expand All @@ -18,69 +17,96 @@
public final class DictionaryMatcher implements PasswordMatcher
{
/**
* Removes all leet substitutions from the password and returns a list of plain text versions.
* Removes all munge substitutions from the password and returns a list of plain text versions.
*
* @param configuration the configuration file used to estimate entropy.
* @param password the password to translate from leet.
* @return a list of all combinations of possible leet translations for the password with all leet removed.
* @param password the password to unmunge.
* @return a list of all combinations of possible unmunged translations for the password with all munges removed.
*/
private static List<String> translateLeet(final Configuration configuration, final String password)
private static List<String> getUnmungedVariations(final Configuration configuration, final String password)
{
final List<String> translations = new ArrayList();
final TreeMap<Integer, Character[]> replacements = new TreeMap<>();
final List<String> translations = new ArrayList<>();
MungeTable mungeTable = configuration.getMungeTable();
PasswordChain chain = new PasswordChain(password);

for (String mungeKey : mungeTable.getKeys()) { // keys sorted largest to smallest
for (int i = 0; i < chain.size(); i++) {
// get the next password part (will be null if it doesn't contain the munge key)
String part = chain.getPartIfContainsKey(i, mungeKey);
if (part != null) {
// split password segment by the munge key
String[] splitParts = mungeTable.getKeyPattern(mungeKey).split(part);

for (int j = 0; j < splitParts.length; j++) {
// index of where the replacements go in the chain
int index = i + j;
String splitPart = splitParts[j];
// check if this substring can be replaced with substitutes, and get the subs if it can
boolean replaceable = mungeTable.isReplaceable(splitPart);
String[] subs = (replaceable ? mungeTable.getSubs(splitPart) : new String[] {splitPart});
// add the replacements back into the chain
if (j == 0) {
chain.replace(index, subs, replaceable);
}
else {
chain.add(index, subs, replaceable);
}

for (int i = 0; i < password.length(); i++)
{
final Character[] replacement = configuration.getLeetTable().get(password.charAt(i));
if (replacement != null)
{
replacements.put(i, replacement);
if (replaceable) {
chain.recordCharsConverted(splitPart.length());
}
}

// do not bother continuing if we're going to replace every single character
if (chain.allReplaced()) {
return translations;
}
}
}
}

// Do not bother continuing if we're going to replace every single character
if(replacements.keySet().size() == password.length())
return translations;

if (replacements.size() > 0)
if (chain.size() > 1)
{
final char[] password_char = new char[password.length()];
for (int i = 0; i < password.length(); i++)
{
password_char[i] = password.charAt(i);
}
replaceAtIndex(replacements, replacements.firstKey(), password_char, translations);
// recursively generate all password permutations using the discovered munges
replaceAtIndex(chain.getParts(), 0, new StringBuilder(), translations);
}

return translations;
}

/**
* Internal function to recursively build the list of un-leet possibilities.
* Internal function to recursively build the list of possible unmunged password permutations.
*
* @param replacements TreeMap of replacement index, and the possible characters at that index to be replaced
* @param current_index internal use for the function
* @param password a Character array of the original password
* @param final_passwords List of the final passwords to be filled
* @param replacements 2D array of possible password replacement substrings
* @param index Next index to generate all permutations at
* @param buffer Buffer used to incrementally build each password permutation
* @param finalPasswords Full list of generated password permutations
*/
private static void replaceAtIndex(final TreeMap<Integer, Character[]> replacements, Integer current_index, final char[] password, final List<String> final_passwords)
private static void replaceAtIndex(final String[][] replacements, int index, StringBuilder buffer, final List<String> finalPasswords)
{
for (final char replacement : replacements.get(current_index))
if (index == replacements.length) {
// reached the end; add the contents of the buffer to the list of password permutations
finalPasswords.add(buffer.toString());
return;
}

// exhaust all of the substring permutations at the current index
for (int i = 0; i < replacements[index].length; i++)
{
password[current_index] = replacement;
if (current_index.equals(replacements.lastKey()))
if (finalPasswords.size() < 100)
{
final_passwords.add(new String(password));
// add the next substring permutation to the buffer
String postfix = replacements[index][i];
buffer.append(postfix);
// recursively build the rest of the string
replaceAtIndex(replacements, index + 1, buffer, finalPasswords);
// backtrack by ignoring the added postfix
buffer.setLength(buffer.length() - postfix.length());
}
else if (final_passwords.size() > 100)
{
// Give up if we've already made 100 replacements
else {
// give up if we've already generated 100 password permutations
return;
}
else
{
replaceAtIndex(replacements, replacements.higherKey(current_index), password, final_passwords);
}
}
}

Expand Down Expand Up @@ -298,7 +324,7 @@ public List<Match> match(final Configuration configuration, final String passwor
}

// Only do unleet if it's different than the regular lower.
final List<String> unleet_list = translateLeet(configuration, lower_part);
final List<String> unleet_list = getUnmungedVariations(configuration, lower_part);
for (final String unleet_part : unleet_list)
{
final Integer unleet_rank = dictionary.getDictonary().get(unleet_part);
Expand Down
17 changes: 7 additions & 10 deletions src/main/java/me/gosimple/nbvcxz/resources/Configuration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
package me.gosimple.nbvcxz.resources;

import me.gosimple.nbvcxz.matching.DictionaryMatcher;
import me.gosimple.nbvcxz.matching.PasswordMatcher;
import me.gosimple.nbvcxz.matching.SpacialMatcher;
import me.gosimple.nbvcxz.matching.YearMatcher;
import me.gosimple.nbvcxz.matching.*;

import java.util.List;
import java.util.Locale;
Expand All @@ -22,7 +19,7 @@ public class Configuration
private final Map<String, Long> guessTypes;
private final List<Dictionary> dictionaries;
private final List<AdjacencyGraph> adjacencyGraphs;
private final Map<Character, Character[]> leetTable;
private final MungeTable mungeTable;
private final Pattern yearPattern;
private final Double minimumEntropy;
private final Locale locale;
Expand All @@ -36,20 +33,20 @@ public class Configuration
* @param guessTypes Map of types of guesses, and associated guesses/sec
* @param dictionaries List of {@link Dictionary} to use for the {@link DictionaryMatcher}
* @param adjacencyGraphs List of adjacency graphs to be used by the {@link SpacialMatcher}
* @param leetTable Leet table for use with {@link DictionaryMatcher}
* @param mungeTable Munge table for use with {@link DictionaryMatcher}
* @param yearPattern Regex {@link Pattern} for use with {@link YearMatcher}
* @param minimumEntropy Minimum entropy value passwords should meet
* @param locale Locale for localized text and feedback
* @param distanceCalc Enable or disable levenshtein distance calculation for dictionary matches
* @param combinationAlgorithmTimeout Timeout for the findBestMatches algorithm
*/
public Configuration(List<PasswordMatcher> passwordMatchers, Map<String, Long> guessTypes, List<Dictionary> dictionaries, List<AdjacencyGraph> adjacencyGraphs, Map<Character, Character[]> leetTable, Pattern yearPattern, Double minimumEntropy, Locale locale, boolean distanceCalc, long combinationAlgorithmTimeout)
public Configuration(List<PasswordMatcher> passwordMatchers, Map<String, Long> guessTypes, List<Dictionary> dictionaries, List<AdjacencyGraph> adjacencyGraphs, MungeTable mungeTable, Pattern yearPattern, Double minimumEntropy, Locale locale, boolean distanceCalc, long combinationAlgorithmTimeout)
{
this.passwordMatchers = passwordMatchers;
this.guessTypes = guessTypes;
this.dictionaries = dictionaries;
this.adjacencyGraphs = adjacencyGraphs;
this.leetTable = leetTable;
this.mungeTable = mungeTable;
this.yearPattern = yearPattern;
this.minimumEntropy = minimumEntropy;
this.locale = locale;
Expand Down Expand Up @@ -94,9 +91,9 @@ public List<AdjacencyGraph> getAdjacencyGraphs()
/**
* @return Leet table for use with {@link DictionaryMatcher}
*/
public Map<Character, Character[]> getLeetTable()
public MungeTable getMungeTable()
{
return leetTable;
return mungeTable;
}

/**
Expand Down
86 changes: 50 additions & 36 deletions src/main/java/me/gosimple/nbvcxz/resources/ConfigurationBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,7 @@
package me.gosimple.nbvcxz.resources;

import me.gosimple.nbvcxz.Nbvcxz;
import me.gosimple.nbvcxz.matching.DateMatcher;
import me.gosimple.nbvcxz.matching.DictionaryMatcher;
import me.gosimple.nbvcxz.matching.PasswordMatcher;
import me.gosimple.nbvcxz.matching.RepeatMatcher;
import me.gosimple.nbvcxz.matching.SeparatorMatcher;
import me.gosimple.nbvcxz.matching.SequenceMatcher;
import me.gosimple.nbvcxz.matching.SpacialMatcher;
import me.gosimple.nbvcxz.matching.YearMatcher;
import me.gosimple.nbvcxz.matching.*;
import me.gosimple.nbvcxz.matching.match.Match;

import java.math.BigDecimal;
Expand All @@ -33,7 +26,7 @@ public class ConfigurationBuilder
private static final List<Dictionary> defaultDictionaries = new ArrayList<>();
private static final List<PasswordMatcher> defaultPasswordMatchers = new ArrayList<>();
private static final List<AdjacencyGraph> defaultAdjacencyGraphs = new ArrayList<>();
private static final Map<Character, Character[]> defaultLeetTable = new HashMap<>();
private static final MungeTable defaultMungeTable = new MungeTable();

static
{
Expand All @@ -56,35 +49,56 @@ public class ConfigurationBuilder
defaultAdjacencyGraphs.add(new AdjacencyGraph("Standard Keypad", AdjacencyGraphUtil.standardKeypad));
defaultAdjacencyGraphs.add(new AdjacencyGraph("Mac Keypad", AdjacencyGraphUtil.macKeypad));

defaultLeetTable.put('4', new Character[]{'a'});
defaultLeetTable.put('@', new Character[]{'a'});
defaultLeetTable.put('8', new Character[]{'b'});
defaultLeetTable.put('(', new Character[]{'c'});
defaultLeetTable.put('{', new Character[]{'c'});
defaultLeetTable.put('[', new Character[]{'c'});
defaultLeetTable.put('<', new Character[]{'c'});
defaultLeetTable.put('3', new Character[]{'e'});
defaultLeetTable.put('9', new Character[]{'g'});
defaultLeetTable.put('6', new Character[]{'g'});
defaultLeetTable.put('&', new Character[]{'g'});
defaultLeetTable.put('#', new Character[]{'h'});
defaultLeetTable.put('!', new Character[]{'i', 'l'});
defaultLeetTable.put('1', new Character[]{'i', 'l'});
defaultLeetTable.put('|', new Character[]{'i', 'l'});
defaultLeetTable.put('0', new Character[]{'o'});
defaultLeetTable.put('$', new Character[]{'s'});
defaultLeetTable.put('5', new Character[]{'s'});
defaultLeetTable.put('+', new Character[]{'t'});
defaultLeetTable.put('7', new Character[]{'t', 'l'});
defaultLeetTable.put('%', new Character[]{'x'});
defaultLeetTable.put('2', new Character[]{'z'});
defaultMungeTable
// simple single character substitutions (mostly leet speak)
.addSub("4", "a")
.addSub("@", "a")
.addSub("8", "b")
.addSub("(", "c")
.addSub("{", "c")
.addSub("[", "c")
.addSub("<", "c", "k", "v")
.addSub(">", "v")
.addSub("3", "e")
.addSub("9", "g", "q")
.addSub("6", "d", "g")
.addSub("&", "g")
.addSub("#", "f", "h")
.addSub("!", "i", "l")
.addSub("1", "i", "l")
.addSub("|", "i", "l")
.addSub("0", "o")
.addSub("$", "s")
.addSub("5", "s")
.addSub("+", "t")
.addSub("7", "t", "l")
.addSub("%", "x")
.addSub("2", "z")
// extra "munged" variations from here: https://en.wikipedia.org/wiki/Munged_password
.addSub("?", "y") // (y = why?)
.addSub("uu", "w")
.addSub("vv", "w")
.addSub("nn", "m")
.addSub("2u", "uu", "w")
.addSub("2v", "vv", "w")
.addSub("2n", "nn", "m")
.addSub("2b", "bb")
.addSub("2d", "dd")
.addSub("2g", "gg")
.addSub("2l", "ll")
.addSub("2p", "pp")
.addSub("2t", "tt")
.addSub("\\/\\/", "w")
.addSub("/\\/\\", "m")
.addSub("|)", "d")
.sort();
}

private List<PasswordMatcher> passwordMatchers;
private Map<String, Long> guessTypes;
private List<Dictionary> dictionaries;
private List<AdjacencyGraph> adjacencyGraphs;
private Map<Character, Character[]> leetTable;
private MungeTable leetTable;
private Pattern yearPattern;
private Double minimumEntropy;
private Locale locale;
Expand Down Expand Up @@ -179,9 +193,9 @@ public static List<AdjacencyGraph> getDefaultAdjacencyGraphs()
/**
* @return The default table of common english leet substitutions
*/
public static Map<Character, Character[]> getDefaultLeetTable()
public static MungeTable getDefaultMungeTable()
{
return defaultLeetTable;
return defaultMungeTable;
}

/**
Expand Down Expand Up @@ -282,7 +296,7 @@ public ConfigurationBuilder setAdjacencyGraphs(List<AdjacencyGraph> adjacencyGra
* @param leetTable Map for leetTable
* @return Builder
*/
public ConfigurationBuilder setLeetTable(Map<Character, Character[]> leetTable)
public ConfigurationBuilder setLeetTable(MungeTable leetTable)
{
this.leetTable = leetTable;
return this;
Expand Down Expand Up @@ -419,7 +433,7 @@ public Configuration createConfiguration()
}
if (leetTable == null)
{
leetTable = getDefaultLeetTable();
leetTable = getDefaultMungeTable();
}
if (yearPattern == null)
{
Expand Down
Loading