Skip to content

Commit ff6e6bd

Browse files
soumyakanti3578soumyakanti3578
committed
Resolve review comments
1 parent 8836426 commit ff6e6bd

File tree

5 files changed

+55
-98
lines changed

5 files changed

+55
-98
lines changed

‎ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ public class HiveRelFieldTrimmer extends RelFieldTrimmer {
101101

102102
private static final ThreadLocal<ColumnAccessInfo> COLUMN_ACCESS_INFO =
103103
new ThreadLocal<>();
104-
private static final ThreadLocal<Map<RelNode, Table>> VIEW_RELNODE_TO_TABLE = new ThreadLocal<>();
104+
private static final ThreadLocal<Map<RelNode, Table>> REL_TO_TABLE = new ThreadLocal<>();
105105

106106

107107
protected HiveRelFieldTrimmer(boolean fetchStats) {
@@ -158,13 +158,13 @@ public RelNode trim(RelBuilder relBuilder, RelNode root,
158158
try {
159159
// Set local thread variables
160160
COLUMN_ACCESS_INFO.set(columnAccessInfo);
161-
VIEW_RELNODE_TO_TABLE.set(relNodeToTable);
161+
REL_TO_TABLE.set(relNodeToTable);
162162
// Execute pruning
163163
return super.trim(relBuilder, root);
164164
} finally {
165165
// Always remove the local thread variables to avoid leaks
166166
COLUMN_ACCESS_INFO.remove();
167-
VIEW_RELNODE_TO_TABLE.remove();
167+
REL_TO_TABLE.remove();
168168
}
169169
}
170170

@@ -203,20 +203,25 @@ protected RexNode handle(RexFieldAccess fieldAccess) {
203203
}
204204

205205
@Override
206+
protected void preTrim(RelNode rel, ImmutableBitSet fieldsUsed) {
207+
setColumnAccessInfoForViews(rel, fieldsUsed);
208+
}
209+
206210
protected void setColumnAccessInfoForViews(RelNode rel, ImmutableBitSet fieldsUsed) {
207211
final ColumnAccessInfo columnAccessInfo = COLUMN_ACCESS_INFO.get();
208-
final Map<RelNode, Table> relNodeToTableAndProjects = VIEW_RELNODE_TO_TABLE.get();
212+
final Map<RelNode, Table> relToTable = REL_TO_TABLE.get();
209213

210214
// HiveTableScans are handled separately in HiveTableScan's trimFields method.
211215
if (!(rel instanceof HiveTableScan) &&
212216
columnAccessInfo != null &&
213-
relNodeToTableAndProjects != null &&
214-
relNodeToTableAndProjects.containsKey(rel)) {
215-
Table table = relNodeToTableAndProjects.get(rel);
217+
relToTable != null &&
218+
relToTable.containsKey(rel)) {
219+
Table table = relToTable.get(rel);
220+
String tableName = table.getCompleteName();
216221
List<FieldSchema> tableAllCols = table.getAllCols();
217222

218-
for (int i = fieldsUsed.nextSetBit(0); i >= 0; i = fieldsUsed.nextSetBit(i + 1)) {
219-
columnAccessInfo.add(table.getCompleteName(), tableAllCols.get(i).getName());
223+
for (int i : fieldsUsed) {
224+
columnAccessInfo.add(tableName, tableAllCols.get(i).getName());
220225
}
221226
}
222227
}

‎ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/RelFieldTrimmer.java‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -282,7 +282,7 @@ protected final TrimResult dispatchTrimFields(
282282
RelNode rel,
283283
ImmutableBitSet fieldsUsed,
284284
Set<RelDataTypeField> extraFields) {
285-
setColumnAccessInfoForViews(rel, fieldsUsed);
285+
preTrim(rel, fieldsUsed);
286286
final TrimResult trimResult =
287287
trimFieldsDispatcher.invoke(rel, fieldsUsed, extraFields);
288288
final RelNode newRel = trimResult.left;
@@ -1241,15 +1241,12 @@ public TrimResult trimFields(
12411241
}
12421242

12431243
/**
1244-
* Sets columnAccessInfo object for views.
1244+
* Run this method before trimming columns from a relational expression.
12451245
*
12461246
* @param rel RelNode
12471247
* @param fieldsUsed Fields used
12481248
*/
1249-
protected void setColumnAccessInfoForViews(RelNode rel, ImmutableBitSet fieldsUsed) {
1250-
// This method is overridden in child class
1251-
throw new UnsupportedOperationException();
1252-
}
1249+
protected void preTrim(RelNode rel, ImmutableBitSet fieldsUsed) {}
12531250

12541251
//~ Inner Classes ----------------------------------------------------------
12551252

‎ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1577,7 +1577,7 @@ public class CalcitePlannerAction implements Frameworks.PlannerAction<RelNode> {
15771577
private final Map<String, PrunedPartitionList> partitionCache;
15781578
private final Map<String, ColumnStatsList> colStatsCache;
15791579
private final ColumnAccessInfo columnAccessInfo;
1580-
private Map<RelNode, Table> relNodeToTable;
1580+
private final Map<RelNode, Table> relToTable;
15811581
private final QB rootQB;
15821582

15831583
// correlated vars across subqueries within same query needs to have different ID
@@ -1604,6 +1604,7 @@ protected CalcitePlannerAction(
16041604
this.rootQB = rootQB;
16051605
this.colStatsCache = ctx.getOpContext().getColStatsCache();
16061606
this.columnAccessInfo = columnAccessInfo;
1607+
this.relToTable = new HashMap<>();
16071608
}
16081609

16091610
@Override
@@ -1666,7 +1667,7 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu
16661667
HiveRelFactories.HIVE_BUILDER.create(optCluster, null),
16671668
calcitePlan,
16681669
this.columnAccessInfo,
1669-
this.relNodeToTable
1670+
this.relToTable
16701671
);
16711672
}
16721673
perfLogger.perfLogEnd(this.getClass().getName(), PerfLogger.MV_REWRITE_FIELD_TRIMMER);
@@ -4921,10 +4922,7 @@ private RelNode genLogicalPlan(QB qb, boolean outerMostQB,
49214922

49224923
aliasToRel.put(subqAlias, relNode);
49234924
if (qb.getViewToTabSchema().containsKey(subqAlias)) {
4924-
if (this.relNodeToTable == null) {
4925-
this.relNodeToTable = new HashMap<>();
4926-
}
4927-
relNodeToTable.put(relNode, qb.getViewToTabSchema().get(subqAlias));
4925+
relToTable.put(relNode, qb.getViewToTabSchema().get(subqAlias));
49284926
}
49294927
}
49304928

@@ -5045,8 +5043,7 @@ private RelNode genLogicalPlan(QB qb, boolean outerMostQB,
50455043
setQB(qb);
50465044
return srcRel;
50475045
}
5048-
5049-
5046+
50505047
private RelNode genGBHavingLogicalPlan(QB qb, RelNode srcRel) throws SemanticException {
50515048
RelNode gbFilter = null;
50525049
QBParseInfo qbp = getQBParseInfo(qb);

‎ql/src/test/queries/clientpositive/view_top_relnode_not_project_authorization.q‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,12 @@
11
set hive.security.authorization.enabled=true;
22
create table t1 (username string, id int);
33

4-
create view vw_t1 as select distinct username from t1 limit 5;
4+
create view vw_t0 as select distinct username from t1 group by username;
5+
explain cbo select * from vw_t0;
6+
7+
create view vw_t1 as select distinct username from t1 order by username desc limit 5;
58
explain cbo select * from vw_t1;
6-
select * from vw_t1;
79

810
create view vw_t2 as
911
select username from (select username, id from t1 where id > 10 limit 1) x where username > 'a' order by id;
10-
explain cbo select * from vw_t2;
11-
select * from vw_t2;
12-
13-
create view vw_t3 as
14-
select username from (select username, id from t1 where id > 10 limit 10) x where username > 'a' limit 5;
15-
explain cbo select * from vw_t3;
16-
select * from vw_t3;
12+
explain cbo select * from vw_t2;

‎ql/src/test/results/clientpositive/llap/view_top_relnode_not_project_authorization.q.out‎

Lines changed: 28 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,37 @@ POSTHOOK: query: create table t1 (username string, id int)
66
POSTHOOK: type: CREATETABLE
77
POSTHOOK: Output: database:default
88
POSTHOOK: Output: default@t1
9-
PREHOOK: query: create view vw_t1 as select distinct username from t1 limit 5
9+
PREHOOK: query: create view vw_t0 as select distinct username from t1 group by username
10+
PREHOOK: type: CREATEVIEW
11+
PREHOOK: Input: default@t1
12+
PREHOOK: Output: database:default
13+
PREHOOK: Output: default@vw_t0
14+
POSTHOOK: query: create view vw_t0 as select distinct username from t1 group by username
15+
POSTHOOK: type: CREATEVIEW
16+
POSTHOOK: Input: default@t1
17+
POSTHOOK: Output: database:default
18+
POSTHOOK: Output: default@vw_t0
19+
POSTHOOK: Lineage: vw_t0.username SIMPLE [(t1)t1.FieldSchema(name:username, type:string, comment:null), ]
20+
PREHOOK: query: explain cbo select * from vw_t0
21+
PREHOOK: type: QUERY
22+
PREHOOK: Input: default@t1
23+
PREHOOK: Input: default@vw_t0
24+
#### A masked pattern was here ####
25+
POSTHOOK: query: explain cbo select * from vw_t0
26+
POSTHOOK: type: QUERY
27+
POSTHOOK: Input: default@t1
28+
POSTHOOK: Input: default@vw_t0
29+
#### A masked pattern was here ####
30+
CBO PLAN:
31+
HiveAggregate(group=[{0}])
32+
HiveTableScan(table=[[default, t1]], table:alias=[t1])
33+
34+
PREHOOK: query: create view vw_t1 as select distinct username from t1 order by username desc limit 5
1035
PREHOOK: type: CREATEVIEW
1136
PREHOOK: Input: default@t1
1237
PREHOOK: Output: database:default
1338
PREHOOK: Output: default@vw_t1
14-
POSTHOOK: query: create view vw_t1 as select distinct username from t1 limit 5
39+
POSTHOOK: query: create view vw_t1 as select distinct username from t1 order by username desc limit 5
1540
POSTHOOK: type: CREATEVIEW
1641
POSTHOOK: Input: default@t1
1742
POSTHOOK: Output: database:default
@@ -28,21 +53,11 @@ POSTHOOK: Input: default@t1
2853
POSTHOOK: Input: default@vw_t1
2954
#### A masked pattern was here ####
3055
CBO PLAN:
31-
HiveSortLimit(fetch=[5])
56+
HiveSortLimit(sort0=[$0], dir0=[DESC], fetch=[5])
3257
HiveProject(username=[$0])
3358
HiveAggregate(group=[{0}])
3459
HiveTableScan(table=[[default, t1]], table:alias=[t1])
3560

36-
PREHOOK: query: select * from vw_t1
37-
PREHOOK: type: QUERY
38-
PREHOOK: Input: default@t1
39-
PREHOOK: Input: default@vw_t1
40-
#### A masked pattern was here ####
41-
POSTHOOK: query: select * from vw_t1
42-
POSTHOOK: type: QUERY
43-
POSTHOOK: Input: default@t1
44-
POSTHOOK: Input: default@vw_t1
45-
#### A masked pattern was here ####
4661
PREHOOK: query: create view vw_t2 as
4762
select username from (select username, id from t1 where id > 10 limit 1) x where username > 'a' order by id
4863
PREHOOK: type: CREATEVIEW
@@ -74,56 +89,3 @@ HiveFilter(condition=[>($0, _UTF-16LE'a')])
7489
HiveFilter(condition=[>($1, 10)])
7590
HiveTableScan(table=[[default, t1]], table:alias=[t1])
7691

77-
PREHOOK: query: select * from vw_t2
78-
PREHOOK: type: QUERY
79-
PREHOOK: Input: default@t1
80-
PREHOOK: Input: default@vw_t2
81-
#### A masked pattern was here ####
82-
POSTHOOK: query: select * from vw_t2
83-
POSTHOOK: type: QUERY
84-
POSTHOOK: Input: default@t1
85-
POSTHOOK: Input: default@vw_t2
86-
#### A masked pattern was here ####
87-
PREHOOK: query: create view vw_t3 as
88-
select username from (select username, id from t1 where id > 10 limit 10) x where username > 'a' limit 5
89-
PREHOOK: type: CREATEVIEW
90-
PREHOOK: Input: default@t1
91-
PREHOOK: Output: database:default
92-
PREHOOK: Output: default@vw_t3
93-
POSTHOOK: query: create view vw_t3 as
94-
select username from (select username, id from t1 where id > 10 limit 10) x where username > 'a' limit 5
95-
POSTHOOK: type: CREATEVIEW
96-
POSTHOOK: Input: default@t1
97-
POSTHOOK: Output: database:default
98-
POSTHOOK: Output: default@vw_t3
99-
POSTHOOK: Lineage: vw_t3.username SIMPLE [(t1)t1.FieldSchema(name:username, type:string, comment:null), ]
100-
PREHOOK: query: explain cbo select * from vw_t3
101-
PREHOOK: type: QUERY
102-
PREHOOK: Input: default@t1
103-
PREHOOK: Input: default@vw_t3
104-
#### A masked pattern was here ####
105-
POSTHOOK: query: explain cbo select * from vw_t3
106-
POSTHOOK: type: QUERY
107-
POSTHOOK: Input: default@t1
108-
POSTHOOK: Input: default@vw_t3
109-
#### A masked pattern was here ####
110-
CBO PLAN:
111-
HiveSortLimit(fetch=[5])
112-
HiveProject(username=[$0])
113-
HiveFilter(condition=[>($0, _UTF-16LE'a')])
114-
HiveProject(username=[$0])
115-
HiveSortLimit(fetch=[10])
116-
HiveProject(username=[$0])
117-
HiveFilter(condition=[>($1, 10)])
118-
HiveTableScan(table=[[default, t1]], table:alias=[t1])
119-
120-
PREHOOK: query: select * from vw_t3
121-
PREHOOK: type: QUERY
122-
PREHOOK: Input: default@t1
123-
PREHOOK: Input: default@vw_t3
124-
#### A masked pattern was here ####
125-
POSTHOOK: query: select * from vw_t3
126-
POSTHOOK: type: QUERY
127-
POSTHOOK: Input: default@t1
128-
POSTHOOK: Input: default@vw_t3
129-
#### A masked pattern was here ####

0 commit comments

Comments
 (0)