Skip to content

Commit 342fdbe

Browse files
jeffreyavenjeffreyaven
committed
added benchmark
1 parent 0c0047d commit 342fdbe

File tree

3 files changed

+146
-52
lines changed

3 files changed

+146
-52
lines changed

‎notebooks/aws/aws.ipynb

Lines changed: 53 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@
1515
"metadata": {},
1616
"outputs": [],
1717
"source": [
18-
"region = 'us-east-1'"
18+
"regions = \"'ap-southeast-2', 'us-east-1'\"\n",
19+
"region = 'ap-southeast-2'"
1920
]
2021
},
2122
{
@@ -29,7 +30,7 @@
2930
"split_part(instanceState, '\\n', 3) as instanceState, \n",
3031
"count(*) as num_instances\n",
3132
"FROM aws.ec2.instances \n",
32-
"WHERE region = '$region' \n",
33+
"WHERE region IN ($regions)\n",
3334
"GROUP BY split_part(instanceState, '\\n', 3)"
3435
]
3536
},
@@ -49,9 +50,9 @@
4950
"outputs": [],
5051
"source": [
5152
"%%stackql\n",
52-
"SELECT instanceType, COUNT(*) as num_instances \n",
53+
"SELECT region, instanceType, COUNT(*) as num_instances\n",
5354
"FROM aws.ec2.instances\n",
54-
"WHERE region = '$region'\n",
55+
"WHERE region IN ($regions)\n",
5556
"GROUP BY instanceType"
5657
]
5758
},
@@ -95,10 +96,8 @@
9596
"metadata": {},
9697
"outputs": [],
9798
"source": [
98-
"# get multiple regions asynchronously\n",
99-
"from pystackql import StackQL\n",
100-
"stackql = StackQL(download_dir='/srv/stackql', output='pandas')\n",
101-
"stackql.executeStmt(\"REGISTRY PULL aws\")"
99+
"%%stackql\n",
100+
"select PolicyName, Arn from aws.iam.policies where region = '$region'"
102101
]
103102
},
104103
{
@@ -107,25 +106,27 @@
107106
"metadata": {},
108107
"outputs": [],
109108
"source": [
110-
"# get multiple regions asynchronously\n",
111-
"import nest_asyncio, json\n",
112-
"nest_asyncio.apply()\n",
113-
"import pandas as pd\n",
114-
"\n",
115-
"regions = [\"ap-southeast-2\", \"us-east-1\"]\n",
116-
"\n",
117-
"queries = [\n",
118-
" f\"\"\"\n",
119-
" SELECT '{region}' as region, instanceType, COUNT(*) as num_instances\n",
120-
" FROM aws.ec2.instances\n",
121-
" WHERE region = '{region}'\n",
122-
" GROUP BY instanceType\n",
123-
" \"\"\"\n",
124-
" for region in regions\n",
125-
"]\n",
126-
"\n",
127-
"instances_df = await stackql.executeQueriesAsync(queries)\n",
128-
"instances_df"
109+
"%%stackql\n",
110+
"SELECT region, function_name\n",
111+
"FROM aws.lambda.functions\n",
112+
"WHERE region IN (\n",
113+
"'us-east-1',\n",
114+
"'us-east-2',\n",
115+
"'us-west-1',\n",
116+
"'us-west-2',\n",
117+
"'ap-south-1',\n",
118+
"'ap-northeast-3',\n",
119+
"'ap-northeast-2',\n",
120+
"'ap-southeast-1',\n",
121+
"'ap-southeast-2',\n",
122+
"'ap-northeast-1',\n",
123+
"'ca-central-1',\n",
124+
"'eu-central-1',\n",
125+
"'eu-west-1',\n",
126+
"'eu-west-2',\n",
127+
"'eu-west-3',\n",
128+
"'eu-north-1',\n",
129+
"'sa-east-1')"
129130
]
130131
},
131132
{
@@ -134,8 +135,18 @@
134135
"metadata": {},
135136
"outputs": [],
136137
"source": [
137-
"%%stackql\n",
138-
"select PolicyName, Arn from aws.iam.policies where region = '$region'"
138+
"functions_list = _.to_dict(orient='records')"
139+
]
140+
},
141+
{
142+
"cell_type": "code",
143+
"execution_count": null,
144+
"metadata": {},
145+
"outputs": [],
146+
"source": [
147+
"from pystackql import StackQL\n",
148+
"stackql = StackQL(download_dir='/srv/stackql', app_root='/jupyter/.stackql', output='pandas')\n",
149+
"stackql.executeStmt(\"REGISTRY PULL aws\")"
139150
]
140151
},
141152
{
@@ -144,28 +155,11 @@
144155
"metadata": {},
145156
"outputs": [],
146157
"source": [
147-
"import nest_asyncio, json\n",
158+
"# get all functions across all regions asynchronously using pystackql\n",
159+
"import nest_asyncio\n",
148160
"nest_asyncio.apply()\n",
149-
"import pandas as pd\n",
150-
"\n",
151-
"regions= [\"us-east-1\",\"us-east-2\",\"us-west-1\",\"us-west-2\",\"ap-south-1\",\"ap-northeast-3\",\"ap-northeast-2\",\"ap-southeast-1\",\n",
152-
" \"ap-southeast-2\",\"ap-northeast-1\",\"ca-central-1\",\"eu-central-1\",\"eu-west-1\",\"eu-west-2\",\"eu-west-3\",\"eu-north-1\",\n",
153-
" \"sa-east-1\"]\n",
154-
"\n",
155-
"get_fns = [\n",
156-
" f\"\"\"\n",
157-
" SELECT *\n",
158-
" FROM aws.lambda.functions\n",
159-
" WHERE region = '{region}'\n",
160-
" \"\"\"\n",
161-
" for region in regions\n",
162-
"]\n",
163-
"\n",
164-
"functions_df = await stackql.executeQueriesAsync(get_fns)\n",
165-
"functions_list = functions_df.to_dict(orient='records')\n",
166-
"\n",
167161
"get_fns_details = [\n",
168-
" f\"\"\"\n",
162+
" f'''\n",
169163
" SELECT \n",
170164
" function_name,\n",
171165
" region,\n",
@@ -177,13 +171,21 @@
177171
" FROM aws.lambda.function\n",
178172
" WHERE region = '{function['region']}'\n",
179173
" AND data__Identifier = '{function['function_name']}'\n",
180-
" \"\"\"\n",
174+
" '''\n",
181175
" for function in functions_list\n",
182176
"]\n",
183-
"\n",
184177
"function_details_df = await stackql.executeQueriesAsync(get_fns_details)\n",
185178
"function_details_df"
186179
]
180+
},
181+
{
182+
"cell_type": "code",
183+
"execution_count": null,
184+
"metadata": {},
185+
"outputs": [],
186+
"source": [
187+
"f"
188+
]
187189
}
188190
],
189191
"metadata": {

‎notebooks/aws/benchmarks.ipynb

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": null,
6+
"metadata": {},
7+
"outputs": [],
8+
"source": [
9+
"from pystackql import StackQL\n",
10+
"stackql = StackQL(download_dir='/srv/stackql', execution_concurrency_limit=-1, app_root='/jupyter/.stackql', output='pandas')\n",
11+
"stackql.executeStmt(\"REGISTRY PULL aws\")"
12+
]
13+
},
14+
{
15+
"cell_type": "code",
16+
"execution_count": null,
17+
"metadata": {},
18+
"outputs": [],
19+
"source": [
20+
"regions = [\n",
21+
"'us-east-1',\n",
22+
"'us-east-2',\n",
23+
"'us-west-1',\n",
24+
"'us-west-2',\n",
25+
"'ap-south-1',\n",
26+
"'ap-northeast-3',\n",
27+
"'ap-northeast-2',\n",
28+
"'ap-southeast-1',\n",
29+
"'ap-southeast-2',\n",
30+
"'ap-northeast-1',\n",
31+
"'ca-central-1',\n",
32+
"'eu-central-1',\n",
33+
"'eu-west-1',\n",
34+
"'eu-west-2',\n",
35+
"'eu-west-3',\n",
36+
"'eu-north-1',\n",
37+
"'sa-east-1'\n",
38+
"]"
39+
]
40+
},
41+
{
42+
"cell_type": "code",
43+
"execution_count": null,
44+
"metadata": {},
45+
"outputs": [],
46+
"source": [
47+
"# threading test 1 - stackql native golang threading\n",
48+
"\n",
49+
"regions_list = \"','\".join(regions)\n",
50+
"stackql_query = f\"\"\"\n",
51+
"SELECT region, function_name\n",
52+
"FROM aws.lambda.functions\n",
53+
"WHERE region IN ('{regions_list}')\n",
54+
"\"\"\"\n",
55+
"\n",
56+
"import time\n",
57+
"\n",
58+
"start_time = time.time()\n",
59+
"result = stackql.execute(stackql_query)\n",
60+
"end_time = time.time()\n",
61+
"\n",
62+
"print(f\"Time taken (Go threading): {end_time - start_time} seconds\")"
63+
]
64+
},
65+
{
66+
"cell_type": "code",
67+
"execution_count": null,
68+
"metadata": {},
69+
"outputs": [],
70+
"source": [
71+
"# threading test 2 - python threading model\n",
72+
"\n",
73+
"queries = [f\"SELECT region, function_name FROM aws.lambda.functions WHERE region = '{region}'\" for region in regions]\n",
74+
"\n",
75+
"import nest_asyncio\n",
76+
"nest_asyncio.apply()\n",
77+
"\n",
78+
"start_time = time.time()\n",
79+
"result = await stackql.executeQueriesAsync(queries)\n",
80+
"end_time = time.time()\n",
81+
"print(f\"Time taken (Python async): {end_time - start_time} seconds\")"
82+
]
83+
}
84+
],
85+
"metadata": {
86+
"language_info": {
87+
"name": "python"
88+
}
89+
},
90+
"nbformat": 4,
91+
"nbformat_minor": 2
92+
}

‎scripts/entrypoint.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/sh
22
/srv/stackql/stackql --version
33
echo "starting stackql server..."
4-
nohup /srv/stackql/stackql --http.response.pageLimit=-1 --pgsrv.port=5466 srv &
4+
nohup /srv/stackql/stackql --execution.concurrency.limit=-1 --http.response.pageLimit=-1 --pgsrv.port=5466 srv &
55
echo "stackql server started"
66
start-notebook.sh --NotebookApp.token=''

0 commit comments

Comments
 (0)