diff --git a/Migration_Project/.vscode/settings.json b/Migration_Project/.vscode/settings.json new file mode 100644 index 0000000..3516cb9 --- /dev/null +++ b/Migration_Project/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.formatting.provider": "autopep8" +} \ No newline at end of file diff --git a/Migration_Project/Rough.py b/Migration_Project/Rough.py new file mode 100644 index 0000000..a95c7b7 --- /dev/null +++ b/Migration_Project/Rough.py @@ -0,0 +1,40 @@ + +# # Entertainment_Booking_System +# event_details/event_type/manager +# event_booked +# customer/event_booked + + + +# # ecommerce +# customer/orders +# orders/lineitem +# part/supplier +# supplier/part + + + +# # Exam Question and Hardcoded values +# users/cart +# users/orders +# orders/lineitem +# products + +tables=["users","orders","lineitem","shoppingCart","products","added"] +tNo={"users":0,"orders":1,"lineitem":2,"shoppingCart":3,"products":4,"added":5} +adj={"lineitem":["orders"], + "orders":["users"], + "users":["shoppingCart"], + "shoppingCart":[], + "added":["shoppingCart","products"], + "products":[] +} +relations={ +"shoppingCart":["users","added"], +"users":["orders"], +"orders":["lineitem"], +"products":["added"], +"lineitem":[], +"added":[] +} + diff --git a/Migration_Project/__pycache__/algo.cpython-310.pyc b/Migration_Project/__pycache__/algo.cpython-310.pyc new file mode 100644 index 0000000..f0507b9 Binary files /dev/null and b/Migration_Project/__pycache__/algo.cpython-310.pyc differ diff --git a/Migration_Project/__pycache__/algo.cpython-39.pyc b/Migration_Project/__pycache__/algo.cpython-39.pyc new file mode 100644 index 0000000..fdb96dc Binary files /dev/null and b/Migration_Project/__pycache__/algo.cpython-39.pyc differ diff --git a/Migration_Project/__pycache__/connection.cpython-310.pyc b/Migration_Project/__pycache__/connection.cpython-310.pyc new file mode 100644 index 0000000..a750158 Binary files /dev/null and b/Migration_Project/__pycache__/connection.cpython-310.pyc differ diff --git a/Migration_Project/__pycache__/connection.cpython-39.pyc b/Migration_Project/__pycache__/connection.cpython-39.pyc new file mode 100644 index 0000000..5ffe861 Binary files /dev/null and b/Migration_Project/__pycache__/connection.cpython-39.pyc differ diff --git a/Migration_Project/__pycache__/embed.cpython-310.pyc b/Migration_Project/__pycache__/embed.cpython-310.pyc new file mode 100644 index 0000000..bb10f80 Binary files /dev/null and b/Migration_Project/__pycache__/embed.cpython-310.pyc differ diff --git a/Migration_Project/__pycache__/embed.cpython-39.pyc b/Migration_Project/__pycache__/embed.cpython-39.pyc new file mode 100644 index 0000000..c2d5fee Binary files /dev/null and b/Migration_Project/__pycache__/embed.cpython-39.pyc differ diff --git a/Migration_Project/__pycache__/embedAndLinking.cpython-310.pyc b/Migration_Project/__pycache__/embedAndLinking.cpython-310.pyc new file mode 100644 index 0000000..6d14fa5 Binary files /dev/null and b/Migration_Project/__pycache__/embedAndLinking.cpython-310.pyc differ diff --git a/Migration_Project/__pycache__/embedAndLinking.cpython-39.pyc b/Migration_Project/__pycache__/embedAndLinking.cpython-39.pyc new file mode 100644 index 0000000..a58e794 Binary files /dev/null and b/Migration_Project/__pycache__/embedAndLinking.cpython-39.pyc differ diff --git a/Migration_Project/__pycache__/extractSchema.cpython-310.pyc b/Migration_Project/__pycache__/extractSchema.cpython-310.pyc new file mode 100644 index 0000000..2925eaa Binary files /dev/null and b/Migration_Project/__pycache__/extractSchema.cpython-310.pyc differ diff --git a/Migration_Project/__pycache__/extractSchema.cpython-39.pyc b/Migration_Project/__pycache__/extractSchema.cpython-39.pyc new file mode 100644 index 0000000..03e4e31 Binary files /dev/null and b/Migration_Project/__pycache__/extractSchema.cpython-39.pyc differ diff --git a/Migration_Project/__pycache__/front.cpython-310.pyc b/Migration_Project/__pycache__/front.cpython-310.pyc new file mode 100644 index 0000000..d8d3258 Binary files /dev/null and b/Migration_Project/__pycache__/front.cpython-310.pyc differ diff --git a/Migration_Project/__pycache__/front.cpython-39.pyc b/Migration_Project/__pycache__/front.cpython-39.pyc new file mode 100644 index 0000000..901531b Binary files /dev/null and b/Migration_Project/__pycache__/front.cpython-39.pyc differ diff --git a/Migration_Project/__pycache__/getEmbeddableData.cpython-310.pyc b/Migration_Project/__pycache__/getEmbeddableData.cpython-310.pyc new file mode 100644 index 0000000..484375c Binary files /dev/null and b/Migration_Project/__pycache__/getEmbeddableData.cpython-310.pyc differ diff --git a/Migration_Project/__pycache__/getEmbeddableData.cpython-39.pyc b/Migration_Project/__pycache__/getEmbeddableData.cpython-39.pyc new file mode 100644 index 0000000..de78f33 Binary files /dev/null and b/Migration_Project/__pycache__/getEmbeddableData.cpython-39.pyc differ diff --git a/Migration_Project/__pycache__/home.cpython-310.pyc b/Migration_Project/__pycache__/home.cpython-310.pyc new file mode 100644 index 0000000..8023c29 Binary files /dev/null and b/Migration_Project/__pycache__/home.cpython-310.pyc differ diff --git a/Migration_Project/__pycache__/home.cpython-39.pyc b/Migration_Project/__pycache__/home.cpython-39.pyc new file mode 100644 index 0000000..5acdfd4 Binary files /dev/null and b/Migration_Project/__pycache__/home.cpython-39.pyc differ diff --git a/Migration_Project/__pycache__/main.cpython-310.pyc b/Migration_Project/__pycache__/main.cpython-310.pyc new file mode 100644 index 0000000..e8ec894 Binary files /dev/null and b/Migration_Project/__pycache__/main.cpython-310.pyc differ diff --git a/Migration_Project/__pycache__/main.cpython-39.pyc b/Migration_Project/__pycache__/main.cpython-39.pyc new file mode 100644 index 0000000..86bde80 Binary files /dev/null and b/Migration_Project/__pycache__/main.cpython-39.pyc differ diff --git a/Migration_Project/__pycache__/migrateTable.cpython-310.pyc b/Migration_Project/__pycache__/migrateTable.cpython-310.pyc new file mode 100644 index 0000000..3bc03dd Binary files /dev/null and b/Migration_Project/__pycache__/migrateTable.cpython-310.pyc differ diff --git a/Migration_Project/__pycache__/migrateTable.cpython-39.pyc b/Migration_Project/__pycache__/migrateTable.cpython-39.pyc new file mode 100644 index 0000000..a683a18 Binary files /dev/null and b/Migration_Project/__pycache__/migrateTable.cpython-39.pyc differ diff --git a/Migration_Project/accessPaths_BT.txt b/Migration_Project/accessPaths_BT.txt new file mode 100644 index 0000000..17d0b51 --- /dev/null +++ b/Migration_Project/accessPaths_BT.txt @@ -0,0 +1,6 @@ +Admin +Player/Contact_Information +Contact_Information/Player +Trainer/Grades +Salary_Structure/Trainer +Team \ No newline at end of file diff --git a/Migration_Project/add.txt b/Migration_Project/add.txt new file mode 100644 index 0000000..083bea2 --- /dev/null +++ b/Migration_Project/add.txt @@ -0,0 +1,4 @@ +users/cart +users/orders +orders/lineitem +products \ No newline at end of file diff --git a/Migration_Project/algo.py b/Migration_Project/algo.py new file mode 100644 index 0000000..bbec6fa --- /dev/null +++ b/Migration_Project/algo.py @@ -0,0 +1,168 @@ +import psycopg2 +import psycopg2.extras +import pymongo +import ssl +from bson.json_util import dumps, loads +import datetime +import json +from queue import Queue + + + +# function to get order of embedding +def getOrder(tables, work) : + n = len(work) + adj = [] + for i in range(n) : + adj.append([]) + indeg = [0]*n + vis = {} + relax = [] + zeroqueue = Queue(maxsize = n+1) + for i in range(n) : + for j in range(n) : + if work[i][j] == 'e' and work[j][i] == "e" : + work[i][j] = '-' + work[j][i] = '-' + # print("embed " + tables[i] + " in " + tables[j] + " simultaneously") + relax.append([i, j, 1]) + for i in range(n) : + for j in range(n) : + if(work[i][j] == 'e') : + indeg[i] = indeg[i] + 1 + adj[j].append(i) + for t in range(n) : + if(indeg[t] == 0) : + zeroqueue.put(t) + + while zeroqueue.qsize() > 0 : + t = zeroqueue.get() + for c in adj[t] : + indeg[c] = indeg[c] - 1 + if(indeg[t] == 0) : + zeroqueue.put(c) + relax.append([c, t]) + for pair in relax : + print("embed " + tables[pair[1]] + " in " + tables[pair[0]]) + return relax + + + +# dfs on access paths +def dfs1(tables,tNo,adj1,work1,currCollec,vis1,curTable): + vis1.add(curTable) + for nxtTable in adj1[curTable]: + if(nxtTable not in vis1): + dfs1(tables,tNo,adj1,work1,currCollec,vis1,nxtTable) + if(nxtTable in currCollec): + work1[tNo[curTable]][tNo[nxtTable]]="l" + else: + work1[tNo[curTable]][tNo[nxtTable]]="e" + + + +# dfs on adjacency list of Relational Schema +def dfs2(tables,tNo,adj2,work2,currCollec,vis2,relations,curTable): + vis2.add(curTable) + if(len(relations[curTable])==0): # not referred - noone is refering curTable + if(len(adj2[curTable])==0): # has 0 FK + currCollec.add(curTable) + elif(len(adj2[curTable])==1): # has 1 FK + nxtTable=adj2[curTable][0] + if(curTable in currCollec): + work2[tNo[curTable]][tNo[nxtTable]]="l" + else: + work2[tNo[nxtTable]][tNo[curTable]]="e" + dfs2(tables,tNo,adj2,work2,currCollec,vis2,relations,nxtTable) + else: # has more than 1 FKs + for nxtTable in adj2[curTable]: + if(nxtTable not in vis2): + dfs2(tables,tNo,adj2,work2,currCollec,vis2,relations,nxtTable) + if(nxtTable in currCollec): + work2[tNo[curTable]][tNo[nxtTable]]="l" + else: + work2[tNo[curTable]][tNo[nxtTable]]="e" + else: # referred + for nxtTable in adj2[curTable]: + if(nxtTable not in vis2): + dfs2(tables,tNo,adj2,work2,currCollec,vis2,relations,nxtTable) + if(nxtTable in currCollec): + work2[tNo[curTable]][tNo[nxtTable]]="l" + else: + work2[tNo[curTable]][tNo[nxtTable]]="e" + return + + + +# function for main algorithm +def algo(tables,tNo,relations,paths): + print(".....Running Algo.....") + + nTables=len(tables) + currCollec = {"dummy"} + + #--------------------------------------------------- Part 1 ---------------------------------------------------- + vis1 = {"dummy"} + work1= [["-"]*nTables for _ in range(nTables)] + + # creating potential collections + for p in paths: + currCollec.add(p[0]) + currCollec.remove("dummy") + vis1.remove("dummy") + + # creates adjacency from access paths + adj1={} + for t in tables: + adj1[t]=[] + + for r in paths: + for i in range(len(r)-1): + adj1[r[i]].append(r[i+1]) + + # print(adj1) + + for t in currCollec: + dfs1(tables,tNo,adj1,work1,currCollec,vis1,t) + + # for x in work1: + # print(x) + + # ------------------------------------------------ Part 2 ------------------------------------------------------- + vis2 = {"dummy"} + work2= [["-"]*nTables for _ in range(nTables)] + + # creates adjacency from schema + adj2={} + for t in tables: + adj2[t]=[] + + # print(adj2) + + for r in relations: + for r1 in relations[r]: + adj2[r1[0]].append(r) + + for t in tables: + dfs2(tables,tNo,adj2,work2,currCollec,vis2,relations,t) + + # for x in work2: + # print(x) + + work = work1 + for i in range(nTables): + for j in range(nTables): + if(work1[i][j]=='-'): + work[i][j]=work2[i][j] + else: + work[i][j]=work1[i][j] + + # for x in work: + # print(x) + + relax = getOrder(tables, work) + + print("------Algo Ends------") + return [currCollec,relax, work, adj2] + + diff --git a/Migration_Project/connection.py b/Migration_Project/connection.py new file mode 100644 index 0000000..8ebc90a --- /dev/null +++ b/Migration_Project/connection.py @@ -0,0 +1,35 @@ +import psycopg2 +import psycopg2.extras +import pymongo +import ssl +from bson.json_util import dumps, loads +import datetime +import json + +class bcolors: + HEADER = '\033[95m' + OKBLUE = '\033[94m' + OKCYAN = '\033[96m' + + OKGREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + +def connectDB(dbName,mongodb_host): + print(f"{bcolors.HEADER}Initializing database connections...{bcolors.ENDC}") + + #Postgres connection + print(f"{bcolors.HEADER}Connecting to PostgreSQL server...{bcolors.ENDC}") + pgsqldb = psycopg2.connect(database=dbName,user="postgres",password="admin") + cursor = pgsqldb.cursor() + print(f"{bcolors.HEADER}Connection to Postgres db succeeded.{bcolors.ENDC}") + + #MongoDB connection + print(f"{bcolors.HEADER}Connecting to MongoDB server...{bcolors.ENDC}") + myClient = pymongo.MongoClient(mongodb_host,tls=True, tlsAllowInvalidCertificates=True) + print(f"{bcolors.HEADER}Connection to MongoDB Server succeeded.{bcolors.ENDC}") + print(f"{bcolors.HEADER}Database connections initialized successfully.{bcolors.ENDC}") + return [myClient,cursor] \ No newline at end of file diff --git a/Migration_Project/embedAndLinking.py b/Migration_Project/embedAndLinking.py new file mode 100644 index 0000000..8e4f2fc --- /dev/null +++ b/Migration_Project/embedAndLinking.py @@ -0,0 +1,256 @@ +import psycopg2 +import pymongo +import ssl +from bson.json_util import dumps, loads +import datetime +import collections +import json +from operator import truediv + + +# main function for embedding and Linking +def embedAndLinking(cursor, schema, tables, tNo, relax, pks, work, adj2, relations): + + # fetches data from postgres + data=[] + for t in tables: + cursor.execute("""select * from \"{}\".\"{}\" ;""".format(schema,t)) + mykeys = [desc[0] for desc in cursor.description] + myresults = cursor.fetchall() + dicted_myresult = [dict(zip(mykeys, myresult)) for myresult in myresults] + data.append(dicted_myresult) + + # embedding data + for e in relax: + data[e[0]] = embed2(data[e[0]], data[e[1]], pks[tables[e[0]]], pks[tables[e[1]]], data, tables[e[0]], tables[e[1]], relations) + + # print(pks) + for i in range(len(work)) : + for j in range(len(work)) : + if (work[i][j] == 'l') : + f = False + for ngh in adj2[tables[i]] : + if tNo[ngh] == j : + f = True + break + if f == True : + work[i][j] = '-' + + for i in range(len(work)): + for j in range(i): + if(work[i][j] == 'l' and work[j][i] == 'l') : + tb = -1 + for t in tables : + fi = False + fj = False + for ngh in adj2[t] : + if (tNo[ngh] == i) : + fi = True + if (tNo[ngh] == j) : + fj = True + if(fi == True and fj == True) : + tb = tNo[t] + break + [data[i], data[j]] = linkManyToMany(data[i], data[j], data[tb], pks[tables[i]], pks[tables[j]], pks[tables[tb]], tables[i],tables[j], tables[tb], relations) + # print("----------------") + # print(data[i]) + # print("----------------") + # print(data[j]) + # print("----------------") + + for i in range(len(work)) : + for j in range(len(work)) : + if(work[i][j] == 'l' and work[j][i] != 'l') : + f = False + for ngh in adj2[tables[i]] : + if tNo[ngh] == j : + f = True + break + if f == False : + data[i] = link(data[i], data[j], pks[tables[i]], pks[tables[j]], tables[i],tables[j], relations) + return data + + + +# function for forward embedding +def embed2(t1, t2, pk1,pk2, data, t1Name, t2Name, relations): + print("embedding "+ t2Name + " in " + t1Name) + refersvia = [] + rel = relations[t2Name] + for referring in rel : + if referring[0] == t1Name : + for key in referring[1] : + refersvia.append(key) + break + + print(refersvia) + print(pk2) + if(len(refersvia) == 0) : + return embed1(t1, t2, pk1,pk2, data, t1Name, t2Name, relations) + + for tt in t1 : + tt[t2Name] = [] + + # f = True + z= 0 + for t in t2 : + z = z + 1 + if(z==500): + print(z) + # if(z > 8000) : + # print("wrong") + # break + # else : + # print(str(z)) + # print("here") + for tt in t1 : + flag = True + for i in range( min(len(refersvia), len(pk2) )) : + # if f == True : + # print(str(tt[refersvia[i]]) + " " + str(t[pk2[i]])) + if(tt[refersvia[i]] != t[pk2[i]]) : + flag = False + # f = False + if(flag == True) : + tt[t2Name].append(t) + # for t in t1 : + # order_id = [] + # for pk in pk2 : + # order_id.append(t[pk]) + # lst = [] + # for lineItem in t2 : + # if lineItem[pk1] == order_id : + # lst.append(lineItem) + # t[t2Name] = lst + + # if(flag==1): + # for order in t2 : + # order_id = order[pk2] + # lst = [] + # for lineItem in t11 : + # if lineItem[pk2] == order_id : + # lst.append(lineItem) + # order[t1Name] = lst + + # print(json.dumps(t1, indent = 4)) + + return t1 + + + +# function for reverse embedding +def embed1(t1, t2, pk1, pk2, data, t1Name, t2Name, relations): + refersvia = [] + rel = relations[t1Name] + for referring in rel : + if referring[0] == t2Name : + for key in referring[1] : + refersvia.append(key) + break + + for t in t1 : + # order_id = t[pk1] + order_id = [] + for pk in pk1 : + order_id.append(t[pk]) + lst = [] + for lineItem in t2 : + flag = True + for i in range(min(len(refersvia), len(order_id)) ) : + if lineItem[refersvia[i]] != order_id[i] : + flag = False + if flag == True : + lst.append(lineItem) + t[t2Name] = lst + + # if(flag==1): + # for order in t2 : + # order_id = order[pk2] + # lst = [] + # for lineItem in t11 : + # if lineItem[pk2] == order_id : + # lst.append(lineItem) + # order[t1Name] = lst + + # print(json.dumps(t1, indent = 4)) + + return t1 + + + +# function for many to many linking +def linkManyToMany(data1, data2, datagum, pk1, pk2, pkgum, t1Name, t2Name, tbName, relations) : + print("linking each other : " + t1Name + " <-> " + t2Name + " via " + tbName) + # print(pk1) + rel1 = relations[t1Name] + for referring in rel1 : + if referring[0] == tbName : + refersvia1 = referring[1] + break + rel2 = relations[t2Name] + for referring in rel2 : + if referring[0] == tbName : + refersvia2 = referring[1] + break + # print(refersvia1) + # print(refersvia2) + for row in data1 : + for key in pk2 : + row[key]= [] # assume simple pk1 + for row in data2 : + for key in pk1 : + row[key] = [] # assume simple pk2 + + for row in datagum : + for row1 in data1 : + f1 = True + for i in range(len(refersvia1)) : + if(row1[pk1[i]] != row[refersvia1[i]]) : + f1 = False + break + if f1 == True : + for i in range(len(refersvia2)) : + row1[pk2[i]].append(row[refersvia2[i]]) + for row1 in data2 : + f1 = True + for i in range(len(refersvia2)) : + if(row1[pk2[i]] != row[refersvia2[i]]) : + f1 = False + break + if f1 == True : + for i in range(len(refersvia1)) : + row1[pk1[i]].append(row[refersvia1[i]]) + return [data1, data2] + + + +# function for single linking +def link(data1, data2, pk1, pk2, t1Name, t2Name, relations) : + print("linking " + t2Name + " in " + t1Name) + + + rel = relations[t1Name] + for referring in rel : + if referring[0] == t2Name : + refersvia = referring[1][0] + break + + indata1 = {} + for row in data1 : + indata1[row[pk1[0]]] = [] + + for row in data2 : + indata1[row[refersvia]].append(row[pk2[0]]) + + for row in data1 : + row[pk2[0]] = indata1[row[pk1[0]]] + + return data1 + # for order in data2 : + # for cust in data1 : + # if cust[pk1] == order[pk1] : + # if pk2 in cust : + # cust[pk2].append(order[pk2]) + # else : + # cust[pk2] = [] + # cust[pk2].append(order[pk2]) \ No newline at end of file diff --git a/Migration_Project/extractSchema.py b/Migration_Project/extractSchema.py new file mode 100644 index 0000000..6bb1e25 --- /dev/null +++ b/Migration_Project/extractSchema.py @@ -0,0 +1,100 @@ + +# # main extract schema function +def extractSchema(cursor,schema): + tables = fetch_table(cursor,schema) + + tNo = getTableNumber(tables) + + pks = fetch_all_PKs(cursor,tables) + + relations = fetch_relations(cursor,tables, schema) + + return [tables,tNo,pks,relations] + + + +# # function to get table numbers +def getTableNumber(tables): + tNo={} + i=0 + for t in tables: + tNo[t]=i + i=i+1 + return tNo + + + +# function to fetch pk of given table +def fetch_PK(cursor,tableName) : + cursor.execute("""SELECT distinct c.column_name, c.data_type + FROM information_schema.table_constraints tc + JOIN information_schema.constraint_column_usage AS ccu USING (constraint_schema, constraint_name) + JOIN information_schema.columns AS c ON c.table_schema = tc.constraint_schema + AND tc.table_name = c.table_name AND ccu.column_name = c.column_name + WHERE constraint_type = 'PRIMARY KEY' and tc.table_name = \'{}\'; + """.format(tableName)) + ans=cursor.fetchall() + pk = [] + for cols in ans : + pk.append(cols[0]) + return pk + + + +# function to fetch pks of all tables +def fetch_all_PKs(cursor,tables) : + ans={} + for i in tables: + ans[i]=[] + for i in tables: + t=i + extract_info=fetch_PK(cursor,t) + ans[t] = extract_info + return ans + + + +# function to fetch fk of given table +def fetch_FK(cursor,tableName, schema): + cursor.execute("""select * + from (select distinct + (select r.relname from pg_class r where r.oid = c.conrelid) as table, + (select array_agg(attname) from pg_attribute + where attrelid = c.conrelid and ARRAY[attnum] <@ c.conkey) as col, + (select r.relname from pg_class r where r.oid = c.confrelid) as ftable + from pg_constraint c + where c.confrelid in (select oid from pg_class where relname = \'{}\')) as imp + where imp.table in (select table_name from information_schema.tables where table_schema = \'{}\') + and imp.ftable in (select table_name from information_schema.tables where table_schema = \'{}\') ; + """.format(tableName, schema, schema)) + ans=cursor.fetchall() + return ans + + + +# function to fetch fks of all tables +def fetch_relations(cursor,tables, schema): + ans={} + for i in tables: + ans[i]=[] + for i in tables: + t=i + extract_info=fetch_FK(cursor,t, schema) + + if(len(extract_info)): + for elements in extract_info: + ans[t].append([elements[0],elements[1]]) + return ans + + + +# function to fetch all the tables +def fetch_table(cursor,schema): + cursor.execute("""SELECT table_name, table_schema + FROM information_schema.tables + WHERE table_schema = \'{}\' AND table_type='BASE TABLE' + """.format(schema)) + myresult = (cursor.fetchall()) + ans=[] + ans=[i[0] for i in myresult] + return ans \ No newline at end of file diff --git a/Migration_Project/home.py b/Migration_Project/home.py new file mode 100644 index 0000000..57583d0 --- /dev/null +++ b/Migration_Project/home.py @@ -0,0 +1,55 @@ +import pywebio.output as pwo +from pywebio.input import * +from home import * +import pywebio.input as pwi +import pandas as pd +import subprocess as sp +import os +from main import * + + +def home_page(): + # import front + import pywebio.output as pwo + from pywebio.input import input + pwo.clear() + pwo.put_markdown(""" # SQL to No-SQL migration""") + + arr = input_group("Fill Following fields to migrate",[ + # input('Connection String', name='connection_string'), + input('Database Name', name='Dbname'), + input('Schema Name',name='schema'), + input('MongoDB Host', name='mongoHost'), + input('MongoDB Name', name='mongo_name'), + file_upload('Choose your file for access path', name='userfile')]) + + userfile=arr['userfile'] + open(userfile['filename'],'wb').write(userfile['content']) + df=pd.read_csv(userfile['filename']) + paths=df.to_string(index=False) + + pwo.clear() + pwo.put_markdown(""" # SQL to No-SQL migration""") + pwo.put_html('

⏳⏳⏳Migrating data please wait...⏳⏳⏳

') + + + isSuccessful=main(arr['Dbname'],arr['schema'],arr['mongoHost'],arr['mongo_name'],paths) + if(isSuccessful==-1): + pwo.clear() + pwo.popup('Error','There is an error.') + img=open("something-went-wrong_f.gif",'rb').read() + pwo.put_image(img) + pwo.put_button('Migrate Another',onclick=home_page) + + else: + pwo.clear() + + pwo.popup('Successful migration','👍🥳🔥Migration done successfully.🔥🥳👍') + img=open("inset_successful_f.gif",'rb').read() + pwo.put_image(img) + pwo.put_button('Migrate Another',onclick=home_page) + while(True): + pass + +if __name__ == '__main__': + home_page() diff --git a/Migration_Project/inset_successful_f.gif b/Migration_Project/inset_successful_f.gif new file mode 100644 index 0000000..53ea923 Binary files /dev/null and b/Migration_Project/inset_successful_f.gif differ diff --git a/Migration_Project/main.py b/Migration_Project/main.py new file mode 100644 index 0000000..4c37f45 --- /dev/null +++ b/Migration_Project/main.py @@ -0,0 +1,117 @@ +import psycopg2 +import psycopg2.extras +import pymongo +import ssl +from bson.json_util import dumps, loads +import datetime +import json +from extractSchema import * +from migrateTable import * +from connection import * +from algo import * +from embedAndLinking import * + +class bcolors: + HEADER = '\033[95m' + OKBLUE = '\033[94m' + OKCYAN = '\033[96m' + + OKGREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + + + +# # fetches access paths from file +def getAccessPaths(file_paths) : + accessPaths = [] + table = file_paths.split('\n' ) + + for lines in table: + lines=lines.strip() + x = lines.split('/') + accessPaths.append(x) + return accessPaths + + +def pre_process(cursor,result,table_schema,table_name): + cursor.execute("""set search_path to \'{}\';""".format(table_schema)) + cursor.execute("SELECT column_name FROM information_schema.columns WHERE table_name = \'{}\' and (data_type='date' or data_type='numeric' or data_type='Decimal' or data_type='time without time zone')".format(table_name)) + ans=[] + ans=cursor.fetchall() + + for check in result: + for col in ans: + check[col[0]]=(str)(check[col[0]]) + + +# main function +def main(dbName, schema, mongodb_host, mongodb_dbname, file_paths): + try: + begin_time = datetime.datetime.now() + print(f"{bcolors.HEADER}Script started at: {begin_time} {bcolors.ENDC}") + + # # DATABASE CONNECTIONS BEGINS + # dbName="Entertainment_Booking_System" + # schema="main_db" + # mongodb_host = "mongodb+srv://akshp:MongoDB005@cluster0.juxn7.mongodb.net/test" + # mongodb_dbname = "pg2mongo" + [myClient,cursor] = connectDB(dbName,mongodb_host) + + # # Get Access Paths + paths = getAccessPaths(file_paths) + # print(paths) + + # # SCHEMA EXTRACTION + [tables, tNo, pks,relations] = extractSchema(cursor,schema) + + # print("Tables are Listed Below:") + # print(tables) + # print() + # print(tNo) + # print() + + # print("Table PKS are Listed Below:") + # print(pks) + # print(json.dumps(pks, indent = 2)) + # print() + + # print("Table Relations are Listed Below:") + # print("referred table : {[referring table, [FK in referring table]], ....}") + # print(relations) + # print(json.dumps(relations, indent = 2)) + # print() + + + # # Algorithm starts + [currCollec, relax, work, adj2] = algo(tables, tNo, relations, paths) + + # print(currCollec) + # print() + # print(relax) + # print() + # print(work) + # print() + + + + # # Embedding and Linking + data = embedAndLinking(cursor, schema, tables, tNo, relax, pks, work, adj2, relations) + + for t in tables: + pre_process(cursor,data[tNo[t]],schema,t) + + # # Data Migration + migrateAll( mongodb_dbname,myClient, schema, cursor, data, tNo, currCollec) + + + end_time = datetime.datetime.now() + print(f"{bcolors.HEADER}Script completed at: {end_time} {bcolors.ENDC}") + print(f"{bcolors.HEADER}Total execution time: {end_time-begin_time} {bcolors.ENDC}") + return 1 + except: + return -1 + diff --git a/Migration_Project/migrateTable.py b/Migration_Project/migrateTable.py new file mode 100644 index 0000000..14eb230 --- /dev/null +++ b/Migration_Project/migrateTable.py @@ -0,0 +1,73 @@ +import psycopg2.extras +import pymongo +import ssl +from bson.json_util import dumps, loads +import datetime +import json + + +class bcolors: + HEADER = '\033[95m' + OKBLUE = '\033[94m' + OKCYAN = '\033[96m' + + OKGREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + + +# preprocess to aviod datatype violation error +def pre_process(cursor, result, table_schema, table_name): + cursor.execute("""set search_path to \'{}\';""".format(table_schema)) + cursor.execute("SELECT column_name FROM information_schema.columns WHERE table_name = \'{}\' and (data_type='date' or data_type='numeric' or data_type='Decimal' or data_type='time without time zone')".format(table_name)) + ans = [] + ans = cursor.fetchall() + + for check in result: + for col in ans: + check[col[0]] = (str)(check[col[0]]) + + +# function to migrate a given tables +def migrate_table(mongodbname,myClient, cursor, data, table_name, table_schema): + + pre_process(cursor, data, table_schema, table_name) + + mydb = myClient[mongodbname] + mycol = mydb[table_name] + + mycol.delete_many({}) # delete existing documents + + x = mycol.insert_many(data) # insert the documents + + return len(x.inserted_ids) + + +# main function to migrate all data from postgres to mongodb +def migrateAll(mongodbname,myClient, schema, cursor, data, tNo, currCollec): + success_count = 0 + fail_count = 0 + total_count = len(currCollec) + for c in currCollec: + try: + print(f"{bcolors.OKCYAN}Processing table: {c}...{bcolors.ENDC}") + inserted_count = migrate_table( + mongodbname,myClient, cursor, data[tNo[c]], c, schema) + print( + f"{bcolors.OKGREEN}Processing table: {c} completed. {inserted_count} documents inserted.{bcolors.ENDC}") + success_count += 1 + except Exception as e: + print(f"{bcolors.FAIL} {e} {bcolors.ENDC}") + fail_count += 1 + + print("") + print("Migration completed.") + + print(f"{bcolors.OKGREEN}{success_count} of {total_count} tables migrated successfully.{bcolors.ENDC}") + + if fail_count > 0: + print( + f"{bcolors.FAIL}Migration of {fail_count} tables failed. See errors above.{bcolors.ENDC}") diff --git a/Migration_Project/something-went-wrong_f.gif b/Migration_Project/something-went-wrong_f.gif new file mode 100644 index 0000000..d17d658 Binary files /dev/null and b/Migration_Project/something-went-wrong_f.gif differ diff --git a/migrate.py b/migrate.py deleted file mode 100644 index 3b37f6e..0000000 --- a/migrate.py +++ /dev/null @@ -1,150 +0,0 @@ -import psycopg2 -import pymongo -import ssl -from bson.json_util import dumps, loads -import datetime - -def fetch_FK(tableName): - - cursor=pgsqldb.cursor() - cursor.execute("""select - (select r.relname from pg_class r where r.oid = c.conrelid) as table, - (select array_agg(attname) from pg_attribute - where attrelid = c.conrelid and ARRAY[attnum] <@ c.conkey) as col, - (select r.relname from pg_class r where r.oid = c.confrelid) as ftable -from pg_constraint c -where c.confrelid = (select oid from pg_class where relname = \'{}\'); -""".format(tableName)) - ans=cursor.fetchall() - - return ans - -def fetch_table(myresult): - ans=[] - ans=[i[0] for i in myresult] - return ans - -def fetch_relations(tables): - ans={} - for i in tables: - ans[i]=[] - for i in tables: - t=i - extract_info=fetch_FK(t) - if(len(extract_info)): - ans[t].append([extract_info[0][0],extract_info[0][1]]) - return ans - - - -class bcolors: - HEADER = '\033[95m' - OKBLUE = '\033[94m' - OKCYAN = '\033[96m' - - OKGREEN = '\033[92m' - WARNING = '\033[93m' - FAIL = '\033[91m' - ENDC = '\033[0m' - BOLD = '\033[1m' - UNDERLINE = '\033[4m' - -begin_time = datetime.datetime.now() -print(f"{bcolors.HEADER}Script started at: {begin_time} {bcolors.ENDC}") -print(f"{bcolors.HEADER}Initializing database connections...{bcolors.ENDC}") - -#Postgres connection -print(f"{bcolors.HEADER}Connecting to PostgreSQL server...{bcolors.ENDC}") - - - - -print("Enter Connection string to connect PostGres SQL") -connection_string=input() -print("Enter Schema Name:") -schema=input() -pgsqldb = psycopg2.connect( - database="Basketball", user='postgres', password='admin', host='127.0.0.1', port= '5432' -) - -print(f"{bcolors.HEADER}Connection to MongoDB Server succeeded.{bcolors.ENDC}") - - -cursor = pgsqldb.cursor() -cursor.execute("select t.table_name,array_agg(c.column_name::text) as columns from information_schema.tables t inner join information_schema.columns c on t.table_name = c.table_name where t.table_schema = \'{}\' and t.table_type= 'BASE TABLE' and c.table_schema = \'{}\' group by t.table_name;".format(schema,schema)) -myresult = (cursor.fetchall()) - -# Get tables -tables=fetch_table(myresult) -print("Tables are Listed Below:") -print(tables) - - -#fetch the FK relations for all table -relations=fetch_relations(tables) -print("First is the Primary Table and Second is a Reference Table if any:") -print(relations) - -# print(ans) -print(f"{bcolors.HEADER}Connecting to MongoDB server...{bcolors.ENDC}") - -mongodb_host = "mongodb+srv://njshah301:*NILAy4564*@cluster0.lyugc.mongodb.net/test" -mongodb_dbname = "mymongodb" - - -myclient = pymongo.MongoClient(mongodb_host,tls=True, tlsAllowInvalidCertificates=True) - - - -print(f"{bcolors.HEADER}Connection to MongoDB Server succeeded.{bcolors.ENDC}") - -print(f"{bcolors.HEADER}Database connections initialized successfully.{bcolors.ENDC}") -mydb=myclient["lab08"] -mycol=mydb["movies"] - -query={"directors": {"$in": [ "King Vidor"]}} -#Start migration - -# print(f"{bcolors.HEADER}Migration started...{bcolors.ENDC}") - - -# if mongodb_dbname in dblist: -# print(f"{bcolors.OKBLUE}The database exists.{bcolors.ENDC}") -# else: -# print(f"{bcolors.WARNING}The database does not exist, it is being created.{bcolors.ENDC}") - -#Function migrate_table -# def migrate_table(db, col_name): -# mycursor = db.cursor(dictionary=True) -# mycursor.execute("SELECT * FROM " + col_name + ";") -# myresult = mycursor.fetchall() - -# mycol = mydb[col_name] - -# if delete_existing_documents: -# #delete all documents in the collection -# mycol.delete_many({}) - -# #insert the documents -# if len(myresult) > 0: -# x = mycol.insert_many(myresult) -# return len(x.inserted_ids) -# else: -# return 0 - -# #Iterate through the list of tables in the schema -# table_list_cursor = pgsqldb.cursor() -# table_list_cursor.execute("SELECT table_name FROM information_schema.tables WHERE table_schema = %s ORDER BY table_name LIMIT 15;", (mysql_schema,)) -# tables = table_list_cursor.fetchall() - -# total_count = len(tables) -# success_count = 0 -# fail_count = 0 - -# for table in tables: -# try: -# print(f"{bcolors.OKCYAN}Processing table: {table[0]}...{bcolors.ENDC}") -# inserted_count = migrate_table(pgsqldb, table[0]) -# print(f"{bcolors.OKGREEN}Processing table: {table[0]} completed. {inserted_count} documents inserted.{bcolors.ENDC}") -# success_count += 1 -# except Exception as e: \ No newline at end of file