Initial commit

Author: AliFlux

.gitignore

@@ -0,0 +1,3 @@
+Raw/*
+*.psd
+*.pdf

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 Ali Ashraf
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,72 @@
+# Dee.py
+
+## A bare-metal deep neural network library for fun learning and experimentation
+
+<p align="center">
+  <img src="promo.png">
+</p>
+
+## So what does it do?
+
+This library allows you to make and train neural networks with any number of layers and neurons pretty easily:
+
+```py
+# Make a network of three hidden layers with 5,3,4,3,5 neurons and 2 output layers
+network = Dee([5, 3, 4, 3, 5], 2)
+
+# train the network with X (input) y (output) and custom parameters
+network.train(X, y, epochs=1000, learningRate=0.01, batchSize=20)
+```
+
+## Purpose
+
+This library was designed for learning and experimentation. The source is almost 200 lines of core code. And you can dig deep, tune parameters, apply custom functions, and visualize the result in no time. There are production-ready deep learning frameworks available such as Theano, Keras, and Tensorflow, but their implementation is pretty complicated and certainly not for the jumpstarters.
+
+## XOR Example
+
+```py
+from dee import Dee
+
+# input data
+X = [
+	[0, 0],
+	[0, 1],
+	[1, 0],
+	[1, 1],	 
+]
+
+# output truth
+y = [0, 1, 1, 0]
+
+# training the network with two hidden layers of 3 neurons, and 2 output layers
+# play with the network and see how the plot changes
+network = Dee([3, 3], 2)
+network.train(X, y, epochs=1000, learningRate=0.01, batchSize=20)
+
+# visualize everything
+network.visualize()
+network.plot2D()
+network.plotLoss()
+```
+
+## Fun Exercises
+
+- Modify the code to use sigmoid or relu instead of tanh
+- Experiment with single batch or single row (stochastic gradient descent)
+- Add regularization to reduce overfitting
+- Optimize the spiral problem to gain a better visual result
+- Try a dataset with multiple classes
+- Make the loss graph update in realtime
+- Use CudaMat to GPUfy the training and save time
+- Apply an advanced optimization function, such as adam (fun is subjective for this one ;)
+
+## Stay In Touch
+
+For latest releases and announcements, check out my site: [aliashraf.net](http://aliashraf.net)
+
+## License
+
+This software is released under the [MIT License](LICENSE). Please read LICENSE for information on the
+software availability and distribution.
+
+Copyright (c) 2018 [Ali Ashraf](http://aliashraf.net)

__pycache__/dee.cpython-36.pyc

@@ -0,0 +1,194 @@
+import numpy as np
+import matplotlib.pyplot as plt
+from visualize_dee import VisualizeDee
+
+# Dee.py core neural network class
+class Dee:
+	
+	# initialization constructor
+	def __init__(self, hiddenLayers, outputNodes=1):
+		self.hiddenLayers = hiddenLayers
+		self.outputNodes = outputNodes
+		self.datasetLength = 0
+		self.W = []
+		self.b = []
+	
+	# standalone prediction function that can give output of any test input
+	def predict(self, x):
+		
+		# basically forward propogation happening here
+		zAll = []
+		aAll = []
+	
+		zAll.append([])
+		aAll.append(np.array(x))
+	
+		WAll = self.W
+		bAll = self.b
+	
+		for j in range(0, len(self.W)):
+			W = WAll[j]
+			b = bAll[j]
+	
+			inputData = aAll[j]
+			
+			z = inputData.dot(W) + b
+			
+			# modify to change activation function
+			a = np.tanh(z)
+	
+			zAll.append(z)
+			aAll.append(a)
+		
+		# probabilities of scores
+		exp_scores = np.exp(zAll[len(zAll) - 1])
+		probs = exp_scores / np.sum(exp_scores, axis=1, keepdims=True)
+		
+		return probs
+	
+	# a helper function that divides dataset into batches
+	@staticmethod
+	def batch(iterable, n=1):
+		l = len(iterable)
+		
+		result = []
+		for ndx in range(0, l, n):
+			result.append(iterable[ndx:min(ndx + n, l)])
+			
+		return result
+	
+	# the core training function
+	def train(self, X, y, epochs = 10000, learningRate=0.1, batchSize = 5):
+		np.random.seed(0)
+		
+		X = np.array(X)
+		y = np.array(y)
+		
+		self.X = X
+		self.y = y
+		self.datasetLength = len(X)
+		numHiddenLayers = len(self.hiddenLayers)
+		numProcessingLayers = numHiddenLayers + 1
+	
+		WAll = []
+		bAll = []
+		
+		# generate dimensions for each layer
+		dimensions = [np.shape(X)[1]]
+		
+		for numNodes in self.hiddenLayers:
+			dimensions.append(numNodes)
+			
+		dimensions.append(self.outputNodes)
+		
+		# initialize to random weights and biases
+		for i in range(0, numProcessingLayers):
+			WAll.append(np.random.randn(dimensions[i], dimensions[i+1]) / np.sqrt(dimensions[i]))
+			bAll.append(np.zeros((1, dimensions[i+1])))
+		
+		self.W = []
+		self.b = []
+		self.loss = []
+		
+		# generate batches of the data
+		XBatches = Dee.batch(X, batchSize)
+		yBatches = Dee.batch(y, batchSize)
+		
+		for i in range(0, epochs):
+			
+			epochActivations = []
+			
+			for x in range(0, len(XBatches)):
+				
+				Xbatch = np.array(XBatches[x])
+				yBatch = np.array(yBatches[x])
+				thisBatchSize = len(Xbatch)
+				
+				zAll = []
+				aAll = []
+		
+				zAll.append([])
+				aAll.append(Xbatch)
+		
+				# forward propogate the network
+				for j in range(0, numProcessingLayers):
+					W = WAll[j]
+					b = bAll[j]
+		
+					inputData = aAll[j]
+					
+					# crunch the numbers and activate the function
+					z = inputData.dot(W) + b
+					a = np.tanh(z)
+		
+					zAll.append(z)
+					aAll.append(a)
+				
+				# calculate errors and probabilities of last layer
+				lastError = zAll[len(zAll) - 1]
+				lastActivation = aAll[len(aAll) - 1]
+				
+				for item in lastActivation:
+					epochActivations.append(item)
+				
+				exp_scores = np.exp(lastError)
+				probs = exp_scores / np.sum(exp_scores, axis=1, keepdims=True)
+				
+				# Backpropagate through the network
+				deltaL = probs
+				deltaL[range(thisBatchSize), yBatch] -= 1
+				
+				dWAll = []
+				dbAll = []
+		
+				lastDelta = deltaL
+				
+				# reverse loop through the layers
+				for j in reversed(range(0, numProcessingLayers)):
+					
+					# calculate & add delta weight and biases
+					dWAll.insert(0, (aAll[j].T).dot(lastDelta))
+					dbAll.insert(0, np.sum(lastDelta, axis=0, keepdims=(j == 0)))
+		
+					lastDelta = lastDelta.dot(WAll[j].T) * (1 - np.power(aAll[j], 2))
+		
+				# apply the weights to the model
+				# the delta is multiplied to the learning rate and adjusted to W/b
+				for j in range(0, numProcessingLayers):
+					WAll[j] += -learningRate * dWAll[j]
+					bAll[j] += -learningRate * dbAll[j]
+			
+			# save the weights and biases
+			self.W = WAll;
+			self.b = bAll;
+			
+			# calculate the errors of this epoch
+			epochActivations = np.array(epochActivations)
+			
+			validActivations = epochActivations[range(self.datasetLength), self.y]
+			
+			# via squared error formula
+			lossNum = 1 - np.mean(np.square(validActivations))
+			self.loss.append(lossNum)
+			
+			# print the progress bar
+			VisualizeDee.printProgress(i, epochs, 30, 0.01, "Epoch: " + str(i) + " Loss: " + str(lossNum))
+		print("\n")
+	
+	# helper functions that visualize data
+	# implemented in dee.py
+	def visualize(self):
+		VisualizeDee.visualize(self)
+			
+	def plot2D(self, colorMap = plt.cm.rainbow, resolution = 0.01, discrete = False, yColumn = 1):
+		VisualizeDee.plot2D(self, colorMap, resolution, discrete, yColumn)
+			
+	def plotLoss(self, color = 'r'):
+		VisualizeDee.plotLoss(self, color)
+	
+
+		
+		
+
+		
+		

__pycache__/visualize_dee.cpython-36.pyc

@@ -0,0 +1,16 @@
+import sklearn
+import sklearn.datasets
+from dee import Dee
+
+# importing moon dataset with 25% noise
+# NOTE: the result will always be different due to random noise
+X, y = sklearn.datasets.make_moons(200, noise=0.25)
+
+# training on three hidden layers with 5,2,5 neurons
+network = Dee([5, 2, 5], 2)
+network.train(X, y, epochs=5000, learningRate=0.01, batchSize=20)
+
+# visualization
+network.visualize()
+network.plot2D()
+network.plotLoss()

dee.py

@@ -0,0 +1,43 @@
+import numpy as np
+import math
+from dee import Dee
+
+# a generic function that makes spirals
+def spiral(offset=0):
+	a = 1
+	offsetR = offset /180 * math.pi
+	phi = np.arange(0, (3) * np.pi, 0.1)
+	x1 = a*phi*np.cos(phi + offsetR)
+	x2 = a*phi*np.sin(phi + offsetR)
+
+	dr = (np.diff(x1)**2 + np.diff(x2)**2)**.5 # segment lengths
+	r = np.zeros_like(x1)
+	r[1:] = np.cumsum(dr) # integrate path
+	r_int = np.linspace(0, r.max(), 50) # regular spaced path
+	x1 = np.interp(r_int, r, x1) # interpolate
+	x2 = np.interp(r_int, r, x2)
+
+	result = np.column_stack((x1,x2))
+	result = np.delete(result, 0, axis=0)
+	
+	return result 
+
+# generating two spirals
+spiralA = spiral()
+spiralB = spiral(offset=180)
+
+# and concatenating them to form X and y
+X = np.concatenate((spiralA, spiralB), axis=0)
+y = np.array([0] * len(spiralA) + [1] * len(spiralB))
+
+# training the network
+# three hidden layers of 10 neurons and 2 output layers
+network = Dee([10, 10, 10], 2)
+network.train(X, y, epochs=15000, learningRate=0.001, batchSize=20)
+
+# visualization
+network.visualize()
+network.plot2D()
+network.plotLoss()
+
+print("Play with the network parameters to get a better result")

example-moons.py

@@ -0,0 +1,39 @@
+from dee import Dee
+
+# XOR function example
+
+# input data
+X = [
+	[0, 0],
+	[0, 1],
+	[1, 0],
+	[1, 1],	 
+]
+
+# output truth
+y = [
+	0,
+	1,
+	1,
+	0,	 
+]
+
+# training the network with two hidden layers of 3 neurons, and 2 output layers
+# play with the network and see how the plot changes
+network = Dee([3, 3], 2)
+network.train(X, y, epochs=2000, learningRate=0.01, batchSize=20)
+
+# predicting the outcomes of a test data: [0, 1]
+print(network.predict([
+	[0, 1],
+])[:, 1])
+
+# and [1, 1]
+print(network.predict([
+	[1, 1],
+])[:, 1])
+
+# network visualizations
+network.visualize()
+network.plot2D()
+network.plotLoss()