remove older commits

Author: tathey1

.Rhistory

@@ -0,0 +1,157 @@
+q()
+q()
+library(mclust)
+library(mclust)
+library(mclust, lib.loc="C:\Users\Thomas Athey\Documents\Labs\Labs\jovo\mclust")
+.libPaths()
+help(.libPaths)
+help(.libPaths)
+.libPaths()
+.libPaths()
+.libPaths("C:\Users\Thomas Athey\Documents\Labs\Labs\jovo\mclust")
+.libPaths("C:/Users/Thomas Athey/Documents/Labs/Labs/jovo/mclust")
+.libPaths
+.libPaths()
+library(mclust)
+install.packages(C:\Users\Thomas Athey\Documents\Labs\Labs\jovo\mclust_5.4.2)
+install.packages("C:\Users\Thomas Athey\Documents\Labs\Labs\jovo\mclust_5.4.2")
+install.packages("C:/Users/Thomas Athey/Documents/Labs/Labs/jovo/mclust_5.4.2")
+.libPaths()
+install.packages("C:\Users\Thomas Athey\Documents\Labs\Labs\jovo\mclust_5.4.2.tar.gz")
+install.packages("C:/Users/Thomas Athey/Documents/Labs/Labs/jovo/mclust_5.4.2.tar.gz")
+install.packages("C:/Users/Thomas Athey/Documents/Labs/Labs/jovo/mclust_5.4.2.tar")
+pwd
+library("mclust", lib.loc="C:/Users/Thomas Athey/Documents/Labs/Labs/jovo/mclust_5.4.2.tar.gz")
+.libPaths
+.libPaths()
+library()
+search()
+library("mclust")
+install.packages("mclust")
+library("mclust")
+plot(BIC)
+plot(BIC)
+library("mclust")
+source('~/Labs/Labs/jovo/link/cluster_init.R')
+source('~/Labs/Labs/jovo/link/cluster_init.R')
+source('~/Labs/Labs/jovo/link/cluster_init.R')
+View(diabetes)
+View(diabetes)
+source('~/Labs/Labs/jovo/link/cluster_init.R')
+source('~/Labs/Labs/jovo/link/cluster_init.R')
+source('~/Labs/Labs/jovo/link/cluster_init.R')
+View(latent)
+source('~/Labs/Labs/jovo/link/cluster_init.R')
+source('~/Labs/Labs/jovo/link/cluster_init.R')
+View(latent)
+View(X)
+source('~/Labs/Labs/jovo/link/cluster_init.R')
+source('~/Labs/Labs/jovo/link/cluster_init.R')
+source('~/Labs/Labs/jovo/link/cluster_init.R')
+source('~/Labs/Labs/jovo/link/cluster_init.R')
+source('~/Labs/Labs/jovo/link/cluster_init.R')
+source('~/Labs/Labs/jovo/mbstructure/demo/sec3.R')
+install.packages("mbstructure")
+install.packages("C:\Users\Thomas Athey\Documents\Labs\Labs\jovo\mbstructure", type="source")
+install.packages("C:/Users/Thomas Athey/Documents/Labs/Labs/jovo/mbstructure", type="source")
+require(devtools)
+library(devtools)
+install.packages("devtools")
+require(devtools)
+devtools::install_github("youngser/mbstructure")
+devtools::install_github("youngser/mbstructure")
+library(mbstructure)
+devtools::install_github("youngser/mbstructure")
+devtools::install(pkg="C:\Users\Thomas Athey\Documents\Labs\Labs\jovo\mbstructure")
+devtools::install(pkg="C:/Users/Thomas Athey/Documents/Labs/Labs/jovo/mbstructure")
+help("Defunct")
+devtools::install_local(path="C:/Users/Thomas Athey/Documents/Labs/Labs/jovo/mbstructure")
+devtools::githubinstall("youngser/mbstructure")
+githubinstall("youngser/mbstructure")
+install_github("youngser/mbstructure")
+list.files()
+install_github("youngser/mbstructure")
+devtools::install_local(path="C:/Users/Thomas Athey/Documents/Labs/Labs/jovo/mbstructure")
+devtools::githubinstall("youngser/mbstructure")
+?install_github
+library(devtools)
+?install_github
+devtools::install_github("youngser/mbstructure")
+sessionInfo()
+help("Defunct")
+setInternet2(use=TRUE)
+setInternet2(NA
+)
+setInternet2(NA)
+list.files()
+rm(list=ls(all=TRUE))
+save.image()
+list.files()
+devtools::install_github("youngser/mbstructure")
+install.packages("remotes")
+install.packages("remotes")
+library(remotes)
+remotes::install_github("youngser/mbstructure")
+help
+R.Version()
+source('~/Labs/Labs/jovo/mbstructure/demo/sec3.R')
+source('~/Labs/Labs/jovo/link/cluster_init.R')
+source('~/Labs/Labs/jovo/mbstructure/demo/sec3.R')
+source('~/Labs/Labs/jovo/mbstructure/demo/sec3.R')
+source('~/Labs/Labs/jovo/link/cluster_init.R')
+source('~/Labs/Labs/jovo/link/cluster_init.R')
+source('~/Labs/Labs/jovo/mbstructure/demo/sec3.R')
+source('~/Labs/Labs/jovo/link/cluster_init.R')
+source('~/Labs/Labs/jovo/mbstructure/demo/sec3.R')
+source('~/Labs/Labs/jovo/link/cluster_init.R')
+source('~/Labs/Labs/jovo/mbstructure/demo/sec3.R')
+source('~/Labs/Labs/jovo/mbstructure/demo/sec3.R')
+source('~/Labs/Labs/jovo/mbstructure/demo/sec3.R')
+View(mc)
+View(vdf.right)
+View(g)
+source('~/Labs/Labs/jovo/mbstructure/demo/sec3.R')
+source('~/Labs/Labs/jovo/mbstructure/demo/sec3.R')
+source('~/Labs/Labs/jovo/mbstructure/demo/sec3.R')
+source('~/Labs/Labs/jovo/mbstructure/demo/sec3.R')
+require(remotes)
+remotes::install_github("youngser/mbstructure")
+source('~/Labs/Labs/jovo/link/cluster_init.R')
+source('~/Labs/Labs/jovo/mbstructure/demo/sec3.R')
+source('~/Labs/Labs/jovo/link/cluster_init.R')
+source('~/Labs/Labs/jovo/mbstructure/demo/sec3.R')
+source('~/Labs/Labs/jovo/link/cluster_init.R')
+0
+source('~/Labs/Labs/jovo/link/cluster_init.R')
+source('~/Labs/Labs/jovo/link/cluster_init.R')
+source('~/Labs/Labs/jovo/link/cluster_init.R')
+source('~/Labs/Labs/jovo/mbstructure/demo/sec3.R')
+source('~/Labs/Labs/jovo/mbstructure/demo/sec3.R')
+source('~/Labs/Labs/jovo/mbstructure/demo/sec3.R')
+summary(mc)
+source('~/Labs/Labs/jovo/mbstructure/demo/sec3.R')
+source('~/Labs/Labs/jovo/mbstructure/demo/sec3.R')
+source('~/Labs/Labs/jovo/link/cluster_init.R')
+source('~/Labs/Labs/jovo/mbstructure/demo/sec3.R')
+system("python Labs/Labs/jovo/mclust_python/drosophila/clust.py")
+system("python Labs/Labs/jovo/mclust_python/drosophila/clust.py")
+source('~/Labs/Labs/jovo/link/cluster_init.R')
+source('~/Labs/Labs/jovo/mbstructure/demo/sec3.R')
+source('~/Labs/Labs/jovo/mbstructure/demo/sec3.R')
+source('~/Labs/Labs/jovo/mbstructure/demo/sec3.R')
+source('~/Labs/Labs/jovo/mbstructure/demo/sec3.R')
+View(Xhat)
+source('~/Labs/Labs/jovo/mbstructure/demo/sec3.R')
+source('~/Labs/Labs/jovo/mbstructure/demo/sec3.R')
+source('~/Labs/Labs/jovo/mbstructure/demo/sec3.R')
+View(Xhat)
+vdf.type
+vdf[[2]]
+vdf$type
+as.character(vdf$type)
+as.numeric(vdf$type)
+write.csv(as.numeric(vdf$type),file='classes.csv')
+source('~/Labs/Labs/jovo/clustering/pyclust/brute_cluster_experiments_mclust.R')
+source('~/Labs/Labs/jovo/clustering/pyclust/brute_cluster_experiments_mclust.R')
+source('~/Labs/Labs/jovo/clustering/pyclust/brute_cluster_experiments_mclust.R')
+source('~/Labs/Labs/jovo/clustering/pyclust/brute_cluster_experiments_mclust.R')

.vscode/settings.json

@@ -0,0 +1,4 @@
+{
+    "python.formatting.provider": "black",
+    "python.pythonPath": "C:\\Users\\Thomas Athey\\Miniconda3\\python.exe"
+}

README.md

@@ -0,0 +1,35 @@
+# pyclust
+This repo reproduces the results in "Pyclust: Automatic Gaussian Mixture Modeling in Python"
+
+
+Directory:
+subset_experiments
+
+These scripts/files reproduce figures 1 and 2
+
+The csv files that start with "idxs_" are the random subsets of the datasets that were used in the experiments.
+
+All other csv files and images can be reproduced with the scripts
+
+First, run pyclust_subset_experiments.py, mclust_subset_experiments.r, and graspyclust_subset_experiments.py. \
+These will generate csv files with the results of each method.
+
+Then, run compare_experiments.py to plot the results.
+
+
+
+complete_experiments
+
+There is a script for each clustering algorithm. \
+Each script reproduces the respective results for Table 2, the Appendix, \
+and in the case of the Drosophila data, Figure 3
+
+
+figure_4
+
+These scripts reproduce figure 4
+
+The data directory contains the synthetic data used for this experiment (created by make_data.py)
+
+The experiments can be reproduced by running pyclust_runtime.py, graspyclust_runtime.py, and mclust_runtime.r. \
+These scripts save the runtime data then show_runtimes.py plots them.

__pycache__/brute_cluster.cpython-37.pyc

@@ -0,0 +1,95 @@
+from graspy.cluster import GaussianCluster
+import numpy as np
+import time
+import pandas as pd
+import matplotlib.pyplot as plt
+import csv
+import random
+
+def brute_graspy_cluster(Ns, x, covariance_types,ks, c_true, savefigs=None,graphList=None):
+    if graphList != None and 'all_bics' in graphList:
+        _, ((ax0,ax1),(ax2,ax3)) = plt.subplots(2,2,sharey='row',sharex='col',figsize=(10,10))
+    titles = ['full','tied','diag','spherical']
+    best_bic = -np.inf
+    for N in Ns:
+        bics = np.zeros([len(ks),len(covariance_types),N])
+        aris = np.zeros([len(ks),len(covariance_types),N])
+        for i in np.arange(N):
+            graspy_gmm = GaussianCluster(min_components=ks[0],max_components=ks[len(ks)-1],covariance_type=covariance_types,random_state=i)
+            c_hat,ari = graspy_gmm.fit_predict(x,y=c_true)
+            bic_values = -graspy_gmm.bic_.values
+            ari_values = graspy_gmm.ari_.values
+            bics[:,:,i] = bic_values
+            aris[:,:,i] = ari_values
+            bic = bic_values.max()
+
+            if bic > best_bic:
+                idx = np.argmax(bic_values)
+                idxs = np.unravel_index(idx,bic_values.shape)
+                best_ari_bic = ari
+                best_bic = bic
+                best_k_bic = ks[idxs[0]]
+                best_cov_bic = titles[3-idxs[1]]
+                best_c_hat_bic = c_hat
+            
+        max_bics = np.amax(bics, axis=2)
+        title = 'N='+str(N)
+        if graphList != None and 'all_bics' in graphList:
+            ax0.plot(np.arange(1,len(ks)+1),max_bics[:,3])
+            ax1.plot(np.arange(1,len(ks)+1),max_bics[:,2],label=title)
+            ax2.plot(np.arange(1,len(ks)+1),max_bics[:,1])
+            ax3.plot(np.arange(1,len(ks)+1),max_bics[:,0])
+
+    if graphList != None and 'best_bic' in graphList:
+        #Plot with best BIC*********************************
+        if c_true is None:
+            best_ari_bic_str = 'NA'
+        else:
+            best_ari_bic_str = '%1.3f'%best_ari_bic
+
+        fig_bestbic = plt.figure(figsize=(8,8))
+        ax_bestbic = fig_bestbic.add_subplot(1,1,1)
+        #ptcolors = [colors[i] for i in best_c_hat_bic]
+        ax_bestbic.scatter(x[:,0],x[:,1],c=best_c_hat_bic)
+        #mncolors = [colors[i] for i in np.arange(best_k_bic)]
+        mncolors = [i for i in np.arange(best_k_bic)]
+        ax_bestbic.set_title("py(agg-gmm) BIC %3.0f from "%best_bic +
+                str(best_cov_bic) + " k=" + str(best_k_bic) +
+                ' ari=' + best_ari_bic_str)# + "iter=" + str(best_iter_bic))
+        ax_bestbic.set_xlabel("First feature")
+        ax_bestbic.set_ylabel("Second feature")
+        if savefigs is not None:
+            plt.savefig(savefigs+'_python_bestbic.jpg')
+
+    if graphList != None and 'all_bics' in graphList:
+        #plot of all BICS*******************************
+        titles = ['full','tied','diag','spherical']
+        #ax0.set_title(titles[0],fontsize=20,fontweight='bold')
+        #ax0.set_ylabel('BIC',fontsize=20)
+        ax0.locator_params(axis='y',tight=True,nbins=4)
+        ax0.set_yticklabels(ax0.get_yticks(),fontsize=14)
+
+        #ax1.set_title(titles[1],fontsize=20,fontweight='bold')
+        legend = ax1.legend(loc='best',title='Number of\nRuns',fontsize=12)
+        plt.setp(legend.get_title(),fontsize=14)
+
+        #ax2.set_title(titles[2],fontsize=20,fontweight='bold')
+        #ax2.set_xlabel('Number of components',fontsize=20)
+        ax2.set_xticks(np.arange(0,21,4))
+        ax2.set_xticklabels(ax2.get_xticks(),fontsize=14)
+        #ax2.set_ylabel('BIC',fontsize=20)
+        ax2.locator_params(axis='y',tight=True,nbins=4)
+        ax2.set_yticklabels(ax2.get_yticks(),fontsize=14)
+
+
+        #ax3.set_title(titles[3],fontsize=20,fontweight='bold')
+        #ax3.set_xlabel('Number of components',fontsize=20)
+        ax3.set_xticks(np.arange(0,21,4))
+        ax3.set_xticklabels(ax3.get_xticks(),fontsize=14)
+
+
+        if savefigs is not None:
+            plt.savefig('.\\figures\\25_6_19_paperv2\\'+savefigs+'_graspy_bicplot2.jpg')
+    plt.show()
+
+    return best_c_hat_bic, best_cov_bic, best_k_bic, best_ari_bic,best_bic