1+ from datasets import load_dataset
2+ from datetime import date
3+
4+ # ── config ────────────────────────────────────────────────────────────────────
5+ REPO_ID = "pavanmantha/doctor_patient_conversation"
6+ COLLECTION = "doctor_patient_conversation"
7+ OUTPUT_FILE = "data_sets/source_data.qql"
8+ TODAY = date .today ().isoformat () # e.g. 2026-05-16
9+ BATCH_SIZE = 200 # rows per INSERT block (stays under 33MB)
10+ # ─────────────────────────────────────────────────────────────────────────────
11+
12+
13+ def escape (text : str ) -> str :
14+ """Escape single-quotes inside field values."""
15+ return text .replace ("\\ " , "\\ \\ " ).replace ("'" , "\\ '" )
16+
17+
18+ def build_record (row : dict ) -> str :
19+ description = escape ((row .get ("description" ) or "" ).strip ())
20+ text = escape ((row .get ("conversation" ) or "" ).strip ())
21+ status = escape ((row .get ("status" ) or "" ).strip ())
22+
23+ return (
24+ " {\n "
25+ f" 'description': '{ description } ',\n "
26+ f" 'text': '{ text } ',\n "
27+ f" 'status': '{ status } '\n "
28+ " }"
29+ )
30+
31+
32+ def write_batch (f , batch : list , batch_num : int , collection : str ):
33+ f .write (f"\n -- Batch { batch_num } ({ len (batch )} records)\n " )
34+ f .write (f"INSERT BULK INTO COLLECTION { collection } VALUES [\n " )
35+ for i , record in enumerate (batch ):
36+ is_last = (i == len (batch ) - 1 )
37+ f .write (record )
38+ f .write ("\n " if is_last else ",\n " )
39+ f .write ("]\n " )
40+
41+
42+ def main ():
43+ print (f"Loading dataset from '{ REPO_ID } ' ..." )
44+ ds = load_dataset (REPO_ID , split = "train" )
45+ total = len (ds )
46+ num_batches = (total + BATCH_SIZE - 1 ) // BATCH_SIZE
47+ print (f" -> { total } rows loaded" )
48+ print (f" -> { BATCH_SIZE } rows per INSERT block" )
49+ print (f" -> { num_batches } total batches\n " )
50+
51+ header = f"""\
52+ -- Qdrant Query Language
53+ -- BULK INSERT -- DOCTOR PATIENT CONVERSATION (BATCHED)
54+
55+ -- ============================================================
56+ -- QQL -- Doctor Patient Conversation
57+ -- Collection : { COLLECTION }
58+ -- Source : { REPO_ID }
59+ -- Total rows : { total }
60+ -- Batch size : { BATCH_SIZE }
61+ -- Generated : { TODAY }
62+ -- ============================================================
63+
64+ -- Step 0: Show Collections
65+ SHOW COLLECTIONS
66+
67+ -- Step 1: Create the collection
68+ CREATE COLLECTION { COLLECTION }
69+
70+ -- ============================================================
71+ -- BULK INSERT -- BATCHED (each block <= { BATCH_SIZE } records)
72+ -- ============================================================
73+ """
74+
75+ print (f"Writing { OUTPUT_FILE } ..." )
76+ with open (OUTPUT_FILE , "w" , encoding = "utf-8" ) as f :
77+ f .write (header )
78+
79+ batch = []
80+ batch_num = 1
81+
82+ for i , row in enumerate (ds ):
83+ batch .append (build_record (row ))
84+
85+ if len (batch ) == BATCH_SIZE :
86+ write_batch (f , batch , batch_num , COLLECTION )
87+ print (f" batch { batch_num } written ({ i + 1 } /{ total } rows)" )
88+ batch = []
89+ batch_num += 1
90+
91+ # flush remaining rows
92+ if batch :
93+ write_batch (f , batch , batch_num , COLLECTION )
94+ print (f" batch { batch_num } written ({ total } /{ total } rows)" )
95+
96+ print (f"\n Done. '{ OUTPUT_FILE } ' written -- { total } records across { batch_num } INSERT blocks." )
97+
98+
99+ if __name__ == "__main__" :
100+ main ()
0 commit comments