Skip to content

Commit ce36a03

Browse files
SoulSongSoulSong
committed
Add Druid implement.
1 parent 238a94e commit ce36a03

File tree

9 files changed

+314
-96
lines changed

9 files changed

+314
-96
lines changed

‎README.MD

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@
22
该工程主要提供了提取SQL文本中table和field信息的样本实例，以及对SQL文进行简单规则检验。核心依赖：
33
- Apache Calcite
44
- SqlParse
5+
- Ali Druid
6+
```text
7+
Ali Druid 提供了比较丰富功能，其整体落地成本更低。相较其他两款解析器，额外添加了sql归一化示例。
8+
```
59

610
为了减少中间表等噪音干扰，目前约定仅统计TABLE_PREFIX指定的前缀表；同理，在SQL文本中表字段仍然会
711
由于大量的别名导致无法将字段与目标表进行有效关联，故仅采集带有TABLE_PREFIX前缀，指定字段来源的filed信息。

‎pom.xml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,12 @@
6464
<artifactId>calcite-core</artifactId>
6565
<version>1.26.0</version>
6666
</dependency>
67+
68+
<dependency>
69+
<groupId>com.alibaba</groupId>
70+
<artifactId>druid</artifactId>
71+
<version>1.2.6</version>
72+
</dependency>
6773
</dependencies>
6874

6975
<build>

‎src/main/java/com/shf/sql/App.java

Lines changed: 39 additions & 47 deletions
Large diffs are not rendered by default.

‎src/main/java/com/shf/sql/helper/SqlExtractHelper.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@ public interface SqlExtractHelper {
2222
*
2323
* @param sql sql文本
2424
* @param sqlInfo sql上下文信息存储
25+
* @return SqlInfo
2526
* @throws Exception e
2627
*/
27-
void extractSqlInfo(String sql, SqlInfo sqlInfo) throws Exception;
28+
SqlInfo extractSqlInfo(String sql, SqlInfo sqlInfo) throws Exception;
2829
}

‎src/main/java/com/shf/sql/helper/SqlFormatter.java

Lines changed: 0 additions & 38 deletions
This file was deleted.

‎src/main/java/com/shf/sql/helper/CalciteHelper.java renamed to ‎src/main/java/com/shf/sql/helper/calcite/CalciteHelper.java

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
1-
package com.shf.sql.helper;
1+
package com.shf.sql.helper.calcite;
22

3+
import com.shf.sql.helper.SqlChecker;
4+
import com.shf.sql.helper.SqlExtractHelper;
5+
import com.shf.sql.helper.SqlInfo;
36
import lombok.extern.slf4j.Slf4j;
47
import org.apache.calcite.avatica.util.Casing;
58
import org.apache.calcite.avatica.util.Quoting;
@@ -76,15 +79,15 @@ public boolean withLimit(String sql, SqlParser.Config config) throws Exception {
7679
}
7780

7881
@Override
79-
public void extractSqlInfo(String sql, SqlInfo sqlInfo) throws Exception {
80-
extractSqlInfo(sql, CONFIG, sqlInfo);
82+
public SqlInfo extractSqlInfo(String sql, SqlInfo sqlInfo) throws Exception {
83+
return extractSqlInfo(sql, CONFIG, sqlInfo);
8184
}
8285

83-
public void extractSqlInfo(String sql, SqlParser.Config config, SqlInfo sqlInfo) throws SqlParseException {
86+
public SqlInfo extractSqlInfo(String sql, SqlParser.Config config, SqlInfo sqlInfo) throws SqlParseException {
8487
SqlParser sqlParser = SqlParser.create(new SourceStringReader(sql), config);
8588
SqlNode sqlNode = sqlParser.parseQuery();
8689
handlerSQL(sqlNode, sqlInfo);
87-
log.info(sqlInfo.toString());
90+
return sqlInfo;
8891
}
8992

9093
private static void handlerSQL(SqlNode sqlNode, SqlInfo sqlInfo) {

‎src/main/java/com/shf/sql/helper/druid/DruidHelper.java

Lines changed: 223 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,223 @@
1+
package com.shf.sql.helper.druid;
2+
3+
import com.alibaba.druid.DbType;
4+
import com.alibaba.druid.proxy.jdbc.DataSourceProxy;
5+
import com.alibaba.druid.sql.SQLUtils;
6+
import com.alibaba.druid.sql.ast.SQLLimit;
7+
import com.alibaba.druid.sql.ast.SQLObject;
8+
import com.alibaba.druid.sql.ast.SQLStatement;
9+
import com.alibaba.druid.sql.ast.expr.SQLIntegerExpr;
10+
import com.alibaba.druid.sql.ast.statement.SQLSelect;
11+
import com.alibaba.druid.sql.ast.statement.SQLSelectQuery;
12+
import com.alibaba.druid.sql.ast.statement.SQLSelectQueryBlock;
13+
import com.alibaba.druid.sql.ast.statement.SQLSelectStatement;
14+
import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
15+
import com.alibaba.druid.util.JdbcConstants;
16+
import com.alibaba.druid.wall.Violation;
17+
import com.alibaba.druid.wall.WallCheckResult;
18+
import com.alibaba.druid.wall.WallConfig;
19+
import com.alibaba.druid.wall.WallProvider;
20+
import com.alibaba.druid.wall.spi.MySqlWallProvider;
21+
import com.alibaba.druid.wall.spi.OracleWallProvider;
22+
import com.alibaba.druid.wall.spi.PGWallProvider;
23+
import com.alibaba.druid.wall.spi.SQLServerWallProvider;
24+
import com.shf.sql.helper.SqlChecker;
25+
import com.shf.sql.helper.SqlExtractHelper;
26+
import com.shf.sql.helper.SqlInfo;
27+
import lombok.extern.slf4j.Slf4j;
28+
import org.apache.commons.collections4.CollectionUtils;
29+
30+
import java.util.HashSet;
31+
import java.util.List;
32+
33+
/**
34+
* description :
35+
* https://github.com/alibaba/druid/wiki/SQL-Parser
36+
*
37+
* @author songhaifeng
38+
* @date 2021/6/2 16:17
39+
*/
40+
@Slf4j
41+
public class DruidHelper implements SqlExtractHelper, FormatHelper, SqlChecker {
42+
private final static DbType DEFAULT_DB_TYPE = JdbcConstants.MYSQL;
43+
private final static WallProvider MYSQL_PROVIDER;
44+
private final static WallProvider ORACLE_PROVIDER;
45+
private final static WallProvider SQL_SERVER_PROVIDER;
46+
private final static WallProvider PG_PROVIDER;
47+
48+
static {
49+
WallConfig mysqlWallConfig = new WallConfig(MySqlWallProvider.DEFAULT_CONFIG_DIR);
50+
mysqlWallConfig.setSelectAllColumnAllow(false);
51+
MYSQL_PROVIDER = new MySqlWallProvider(mysqlWallConfig);
52+
53+
WallConfig oracleWallConfig = new WallConfig(OracleWallProvider.DEFAULT_CONFIG_DIR);
54+
oracleWallConfig.setSelectAllColumnAllow(false);
55+
ORACLE_PROVIDER = new OracleWallProvider(oracleWallConfig);
56+
57+
WallConfig sqlServerWallConfig = new WallConfig(SQLServerWallProvider.DEFAULT_CONFIG_DIR);
58+
sqlServerWallConfig.setSelectAllColumnAllow(false);
59+
SQL_SERVER_PROVIDER = new SQLServerWallProvider(sqlServerWallConfig);
60+
61+
WallConfig pgWallConfig = new WallConfig(PGWallProvider.DEFAULT_CONFIG_DIR);
62+
pgWallConfig.setSelectAllColumnAllow(false);
63+
PG_PROVIDER = new PGWallProvider(pgWallConfig);
64+
}
65+
66+
private DruidHelper() {
67+
}
68+
69+
private static class DruidHelperHolder {
70+
private static final DruidHelper DRUID_HELPER = new DruidHelper();
71+
}
72+
73+
public static DruidHelper getInstance() {
74+
return DruidHelperHolder.DRUID_HELPER;
75+
}
76+
77+
@Override
78+
public SqlInfo extractSqlInfo(String sql, SqlInfo sqlInfo) throws Exception {
79+
return extractSqlInfo(sql, DEFAULT_DB_TYPE, sqlInfo);
80+
}
81+
82+
public SqlInfo extractSqlInfo(String sql, DbType dbType, SqlInfo sqlInfo) throws Exception {
83+
List<SQLStatement> statementList = SQLUtils.parseStatements(sql, dbType);
84+
statementList.forEach(statement -> handleStatement(statement, dbType, sqlInfo));
85+
sqlInfo.setContainsSelectAllColumns(containsSelectAllColumns(checkSql(sql, dbType)));
86+
return sqlInfo;
87+
}
88+
89+
/**
90+
* https://github.com/alibaba/druid/wiki/SchemaStatVisitor
91+
*
92+
* @param sqlStatement
93+
* @param dbType
94+
* @param sqlInfo
95+
*/
96+
private void handleStatement(SQLStatement sqlStatement, DbType dbType, SqlInfo sqlInfo) {
97+
handleSqlSelectStatement((SQLSelectStatement) sqlStatement, dbType, sqlInfo);
98+
}
99+
100+
private void handleSqlSelectStatement(SQLSelectStatement sqlSelectStatement, DbType dbType, SqlInfo sqlInfo) {
101+
SchemaStatVisitor statVisitor = SQLUtils.createSchemaStatVisitor(dbType);
102+
sqlSelectStatement.accept(statVisitor);
103+
if (sqlInfo.isNeedExtractFields()) {
104+
// getColumns contain orderColumns,groupColumns,conditionColumns and relationShip.
105+
statVisitor.getColumns().forEach(column -> {
106+
if (TABLE_PREFIX.stream().anyMatch(prefix -> column.getTable().startsWith(prefix))) {
107+
sqlInfo.getFields().computeIfAbsent(column.getTable(), tmp -> new HashSet<>()).add(column.getName());
108+
}
109+
});
110+
}
111+
// statVisitor.getTables() contains table status. Here only needs tableNames.
112+
if (sqlInfo.isNeedExtractTables()) {
113+
statVisitor.getOriginalTables().forEach(sqlName -> {
114+
if (TABLE_PREFIX.stream().anyMatch(prefix -> sqlName.getSimpleName().startsWith(prefix))) {
115+
sqlInfo.getTables().add(sqlName.getSimpleName());
116+
}
117+
});
118+
}
119+
}
120+
121+
/**
122+
* 目前仅检查select语句的最外层是否包含limit约束，使用默认的Mysql作为语法解析类型
123+
*
124+
* @param sql sql
125+
* @return true 表示包含；反之不包含
126+
* @throws Exception e
127+
*/
128+
@Override
129+
public boolean withLimit(String sql) throws Exception {
130+
return withLimit(sql, DEFAULT_DB_TYPE);
131+
}
132+
133+
/**
134+
* 目前仅检查select语句的最外层是否包含limit约束
135+
*
136+
* @param sql sql
137+
* @param dbType 数据库类型
138+
* @return true 表示包含；反之不包含
139+
* @throws Exception e
140+
*/
141+
public boolean withLimit(String sql, DbType dbType) throws Exception {
142+
List<SQLStatement> statementList = SQLUtils.parseStatements(sql, dbType);
143+
return withLimit(statementList.get(0));
144+
}
145+
146+
/**
147+
* 目前仅检查select语句的最外层是否包含limit约束
148+
*
149+
* @param sqlStatement sqlStatement
150+
* @return true 表示包含；反之不包含
151+
* @throws Exception e
152+
*/
153+
private boolean withLimit(SQLStatement sqlStatement) throws Exception {
154+
List<SQLObject> sqlObjects = sqlStatement.getChildren();
155+
if (CollectionUtils.isNotEmpty(sqlObjects)) {
156+
SQLObject sqlObject = sqlObjects.get(0);
157+
if (sqlObject instanceof SQLSelect) {
158+
SQLSelectQuery sqlSelectQuery = ((SQLSelect) sqlObject).getQuery();
159+
SQLLimit sqlLimit = ((SQLSelectQueryBlock) sqlSelectQuery).getLimit();
160+
if (sqlLimit != null) {
161+
log.debug("start offset : {} ;rowNumber : {}", sqlLimit.getOffset() == null ? 0 : ((SQLIntegerExpr) sqlLimit.getOffset()).getNumber(), ((SQLIntegerExpr) sqlLimit.getRowCount()).getNumber());
162+
return true;
163+
} else {
164+
return false;
165+
}
166+
}
167+
}
168+
return true;
169+
}
170+
171+
@Override
172+
public boolean containsSelectAllColumns(String sql) {
173+
return containsSelectAllColumns(sql, DEFAULT_DB_TYPE);
174+
}
175+
176+
public boolean containsSelectAllColumns(String sql, DbType dbType) {
177+
return containsSelectAllColumns(checkSql(sql, dbType));
178+
}
179+
180+
private boolean containsSelectAllColumns(WallCheckResult checkResult) {
181+
for (Violation violation : checkResult.getViolations()) {
182+
if (1002 == (violation.getErrorCode())) {
183+
return true;
184+
}
185+
}
186+
return false;
187+
}
188+
189+
/**
190+
* {@link com.alibaba.druid.wall.WallFilter#init(DataSourceProxy)}
191+
* refer : https://github.com/alibaba/druid/wiki/%E9%85%8D%E7%BD%AE-wallfilter
192+
*
193+
* @return WallCheckResult
194+
*/
195+
private WallCheckResult checkSql(String sql, DbType dbType) {
196+
WallProvider provider;
197+
switch (dbType) {
198+
case mysql:
199+
case mariadb:
200+
case presto:
201+
provider = MYSQL_PROVIDER;
202+
break;
203+
case oracle:
204+
case ali_oracle:
205+
case oceanbase_oracle:
206+
provider = ORACLE_PROVIDER;
207+
break;
208+
case sqlserver:
209+
case jtds:
210+
provider = SQL_SERVER_PROVIDER;
211+
break;
212+
case postgresql:
213+
case edb:
214+
case polardb:
215+
provider = PG_PROVIDER;
216+
break;
217+
default:
218+
throw new IllegalStateException("dbType not support : " + dbType + ".");
219+
}
220+
return provider.check(sql);
221+
}
222+
223+
}

‎src/main/java/com/shf/sql/helper/druid/FormatHelper.java

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
package com.shf.sql.helper.druid;
2+
3+
import com.alibaba.druid.DbType;
4+
import com.alibaba.druid.sql.visitor.ParameterizedOutputVisitorUtils;
5+
6+
/**
7+
* description :
8+
*
9+
* @author songhaifeng
10+
* @date 2021/6/2 17:19
11+
*/
12+
public interface FormatHelper {
13+
14+
/**
15+
* https://github.com/alibaba/druid/wiki/ParameterizedOutputVisitor
16+
*
17+
* @param sql
18+
* @param dbType
19+
* @return
20+
*/
21+
default String formatParameters(String sql, DbType dbType) {
22+
return ParameterizedOutputVisitorUtils.parameterize(sql, dbType);
23+
}
24+
25+
}

‎src/main/java/com/shf/sql/helper/SqlParseHelper.java renamed to ‎src/main/java/com/shf/sql/helper/sqlparse/SqlParseHelper.java

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
1-
package com.shf.sql.helper;
1+
package com.shf.sql.helper.sqlparse;
22

3+
import com.shf.sql.helper.SqlExtractHelper;
4+
import com.shf.sql.helper.SqlInfo;
35
import lombok.extern.slf4j.Slf4j;
46
import net.sf.jsqlparser.JSQLParserException;
57
import net.sf.jsqlparser.expression.AnalyticExpression;
@@ -52,11 +54,11 @@ public static SqlParseHelper getInstance() {
5254
}
5355

5456
@Override
55-
public void extractSqlInfo(String sql, SqlInfo sqlInfo) throws Exception {
56-
extractSqlInfo(sql, true, sqlInfo);
57+
public SqlInfo extractSqlInfo(String sql, SqlInfo sqlInfo) throws Exception {
58+
return extractSqlInfo(sql, true, sqlInfo);
5759
}
5860

59-
public void extractSqlInfo(String sql, boolean isSingle, SqlInfo sqlInfo) throws JSQLParserException, ParseException {
61+
public SqlInfo extractSqlInfo(String sql, boolean isSingle, SqlInfo sqlInfo) throws JSQLParserException, ParseException {
6062
if (isSingle) {
6163
handlerStatement(parseSingleSql(sql), sqlInfo);
6264
} else {
@@ -65,7 +67,7 @@ public void extractSqlInfo(String sql, boolean isSingle, SqlInfo sqlInfo) throws
6567
handlerStatement(statement, sqlInfo);
6668
});
6769
}
68-
log.info(sqlInfo.toString());
70+
return sqlInfo;
6971
}
7072

7173
private void handlerStatement(Statement statement, SqlInfo sqlInfo) {

0 commit comments

Comments
 (0)