Skip to content

Commit b8fae24

Browse files
zhoneyzhoney
committed
support create olap property key and dynamic create olap table for cassandra
Change-Id: I947c3a96eb7aa67ee9eff5f7fb248e9f4539e91b
1 parent 9f62e6f commit b8fae24

49 files changed

Lines changed: 1009 additions & 38 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎hugegraph-api/src/main/java/com/baidu/hugegraph/api/API.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ public class API {
5757

5858
public static final String ACTION_APPEND = "append";
5959
public static final String ACTION_ELIMINATE = "eliminate";
60+
public static final String ACTION_CLEAR = "clear";
6061

6162
private static final Meter succeedMeter =
6263
MetricsUtil.registerMeter(API.class, "commit-succeed");

‎hugegraph-api/src/main/java/com/baidu/hugegraph/api/graph/VertexAPI.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -423,8 +423,6 @@ private static class JsonVertex extends JsonElement {
423423

424424
@Override
425425
public void checkCreate(boolean isBatch) {
426-
E.checkArgumentNotNull(this.label,
427-
"The label of vertex can't be null");
428426
this.checkUpdate();
429427
}
430428

@@ -446,8 +444,10 @@ public void checkUpdate() {
446444
public Object[] properties() {
447445
Object[] props = API.properties(this.properties);
448446
List<Object> list = new ArrayList<>(Arrays.asList(props));
449-
list.add(T.label);
450-
list.add(this.label);
447+
if (this.label != null) {
448+
list.add(T.label);
449+
list.add(this.label);
450+
}
451451
if (this.id != null) {
452452
list.add(T.id);
453453
list.add(this.id);

‎hugegraph-api/src/main/java/com/baidu/hugegraph/api/schema/PropertyKeyAPI.java‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,10 +98,20 @@ public String update(@Context GraphManager manager,
9898
E.checkArgument(name.equals(jsonPropertyKey.name),
9999
"The name in url(%s) and body(%s) are different",
100100
name, jsonPropertyKey.name);
101+
102+
HugeGraph g = graph(manager, graph);
103+
if (ACTION_CLEAR.equals(action)) {
104+
PropertyKey propertyKey = g.propertyKey(name);
105+
E.checkArgument(propertyKey.readFrequency().olap(),
106+
"Only olap property key can do action clear, " +
107+
"but got '%s'", propertyKey);
108+
g.clearPropertyKey(propertyKey);
109+
return "done";
110+
}
111+
101112
// Parse action parameter
102113
boolean append = checkAndParseAction(action);
103114

104-
HugeGraph g = graph(manager, graph);
105115
PropertyKey.Builder builder = jsonPropertyKey.convert2Builder(g);
106116
PropertyKey propertyKey = append ?
107117
builder.append() :

‎hugegraph-api/src/main/java/com/baidu/hugegraph/auth/HugeGraphAuthProxy.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,12 @@ public void removePropertyKey(Id key) {
176176
this.hugegraph.removePropertyKey(key);
177177
}
178178

179+
@Override
180+
public void clearPropertyKey(PropertyKey propertyKey) {
181+
verifySchemaPermission(HugePermission.DELETE, propertyKey);
182+
this.hugegraph.clearPropertyKey(propertyKey);
183+
}
184+
179185
@Override
180186
public Collection<PropertyKey> propertyKeys() {
181187
Collection<PropertyKey> pkeys = this.hugegraph.propertyKeys();

‎hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraFeatures.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,6 @@ public boolean supportsTtl() {
124124

125125
@Override
126126
public boolean supportsOlapProperties() {
127-
return false;
127+
return true;
128128
}
129129
}

‎hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraSerializer.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,9 @@
3737
import com.baidu.hugegraph.schema.PropertyKey;
3838
import com.baidu.hugegraph.schema.SchemaElement;
3939
import com.baidu.hugegraph.structure.HugeElement;
40+
import com.baidu.hugegraph.structure.HugeIndex;
4041
import com.baidu.hugegraph.structure.HugeProperty;
42+
import com.baidu.hugegraph.structure.HugeVertex;
4143
import com.baidu.hugegraph.type.HugeType;
4244
import com.baidu.hugegraph.type.define.DataType;
4345
import com.baidu.hugegraph.type.define.HugeKeys;
@@ -58,6 +60,16 @@ protected TableBackendEntry newBackendEntry(TableBackendEntry.Row row) {
5860
return new CassandraBackendEntry(row);
5961
}
6062

63+
@Override
64+
protected TableBackendEntry newBackendEntry(HugeIndex index) {
65+
TableBackendEntry backendEntry = newBackendEntry(index.type(),
66+
index.id());
67+
if (index.indexLabel().olap()) {
68+
backendEntry.olap(true);
69+
}
70+
return backendEntry;
71+
}
72+
6173
@Override
6274
protected CassandraBackendEntry convertEntry(BackendEntry backendEntry) {
6375
if (!(backendEntry instanceof CassandraBackendEntry)) {
@@ -130,6 +142,20 @@ protected void parseProperties(HugeElement element,
130142
}
131143
}
132144

145+
@Override
146+
public BackendEntry writeOlapVertex(HugeVertex vertex) {
147+
CassandraBackendEntry entry = newBackendEntry(HugeType.OLAP,
148+
vertex.id());
149+
entry.column(HugeKeys.ID, this.writeId(vertex.id()));
150+
HugeProperty<?> prop = vertex.getProperties().values()
151+
.iterator().next();
152+
PropertyKey pk = prop.propertyKey();
153+
entry.subId(pk.id());
154+
entry.column(HugeKeys.PROPERTY_VALUE,
155+
this.writeProperty(pk, prop.value()));
156+
return entry;
157+
}
158+
133159
@Override
134160
protected Object writeProperty(PropertyKey propertyKey, Object value) {
135161
BytesBuffer buffer = BytesBuffer.allocate(BytesBuffer.BUF_PROPERTY);

‎hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraStore.java‎

Lines changed: 104 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,15 @@
2525
import java.util.Iterator;
2626
import java.util.List;
2727
import java.util.Map;
28+
import java.util.Set;
2829
import java.util.concurrent.ConcurrentHashMap;
2930

3031
import org.slf4j.Logger;
3132

3233
import com.baidu.hugegraph.backend.BackendException;
3334
import com.baidu.hugegraph.backend.id.Id;
3435
import com.baidu.hugegraph.backend.query.Query;
36+
import com.baidu.hugegraph.backend.serializer.MergeIterator;
3537
import com.baidu.hugegraph.backend.store.AbstractBackendStore;
3638
import com.baidu.hugegraph.backend.store.BackendAction;
3739
import com.baidu.hugegraph.backend.store.BackendEntry;
@@ -63,9 +65,9 @@ public abstract class CassandraStore
6365

6466
private final BackendStoreProvider provider;
6567
// TODO: move to parent class
66-
private final Map<HugeType, CassandraTable> tables;
68+
private final Map<String, CassandraTable> tables;
6769

68-
private CassandraSessionPool sessions;
70+
protected CassandraSessionPool sessions;
6971
private HugeConfig conf;
7072

7173
public CassandraStore(final BackendStoreProvider provider,
@@ -95,7 +97,11 @@ private void registerMetaHandlers() {
9597
}
9698

9799
protected void registerTableManager(HugeType type, CassandraTable table) {
98-
this.tables.put(type, table);
100+
this.registerTableManager(type.string(), table);
101+
}
102+
103+
protected void registerTableManager(String name, CassandraTable table) {
104+
this.tables.put(name, table);
99105
}
100106

101107
@Override
@@ -200,6 +206,12 @@ private void mutate(CassandraSessionPool.Session session,
200206

201207
switch (item.action()) {
202208
case INSERT:
209+
// Insert olap entry
210+
if (entry.type() == HugeType.OLAP) {
211+
this.table(this.store + "_" + HugeType.OLAP.string() + "_" + entry.subId().asLong())
212+
.insert(session, entry.row());
213+
break;
214+
}
203215
// Insert entry
204216
if (entry.selfChanged()) {
205217
this.table(entry.type()).insert(session, entry.row());
@@ -220,6 +232,10 @@ private void mutate(CassandraSessionPool.Session session,
220232
}
221233
break;
222234
case APPEND:
235+
if (entry.olap()) {
236+
this.table(this.store + "_" + HugeType.OLAP.string() + "_" + entry.type().string())
237+
.append(session, entry.row());
238+
}
223239
// Append entry
224240
if (entry.selfChanged()) {
225241
this.table(entry.type()).append(session, entry.row());
@@ -248,9 +264,27 @@ private void mutate(CassandraSessionPool.Session session,
248264
@Override
249265
public Iterator<BackendEntry> query(Query query) {
250266
this.checkOpened();
251-
252-
CassandraTable table = this.table(CassandraTable.tableType(query));
253-
return table.query(this.sessions.session(), query);
267+
HugeType type = CassandraTable.tableType(query);
268+
269+
// TODO: move to MergeStore
270+
CassandraTable table = query.olap() ?
271+
this.table(this.store + "_" + HugeType.OLAP.string() + "_" + type.string()) :
272+
this.table(type);
273+
Iterator<BackendEntry> entrys = table.query(this.sessions.session(),
274+
query);
275+
Set<Id> olapPks = query.olapPks();
276+
if (!olapPks.isEmpty()) {
277+
List<Iterator<BackendEntry>> iterators =
278+
new ArrayList<>(olapPks.size());
279+
for (Id pk : olapPks) {
280+
Query q = query.copy();
281+
table = this.table(this.store + "_" + HugeType.OLAP.string() + "_" + pk.asLong());
282+
iterators.add(table.query(this.sessions.session(), q));
283+
}
284+
entrys = new MergeIterator<>(entrys, iterators,
285+
BackendEntry::mergable);
286+
}
287+
return entrys;
254288
}
255289

256290
@Override
@@ -514,7 +548,12 @@ protected void clearTables() {
514548
protected void truncateTables() {
515549
CassandraSessionPool.Session session = this.sessions.session();
516550
for (CassandraTable table : this.tables()) {
517-
table.truncate(session);
551+
System.out.println(table);
552+
if (table.isOlap()) {
553+
table.dropTable(session);
554+
} else {
555+
table.truncate(session);
556+
}
518557
}
519558
}
520559

@@ -524,10 +563,14 @@ protected Collection<CassandraTable> tables() {
524563

525564
@Override
526565
protected final CassandraTable table(HugeType type) {
527-
assert type != null;
528-
CassandraTable table = this.tables.get(type);
566+
return this.table(type.string());
567+
}
568+
569+
protected final CassandraTable table(String name) {
570+
assert name != null;
571+
CassandraTable table = this.tables.get(name);
529572
if (table == null) {
530-
throw new BackendException("Unsupported table type: %s", type);
573+
throw new BackendException("Unsupported table: %s", name);
531574
}
532575
return table;
533576
}
@@ -600,6 +643,12 @@ public long getCounter(HugeType type) {
600643
public boolean isSchemaStore() {
601644
return true;
602645
}
646+
647+
@Override
648+
public void createOlapTable(Id pkId) {
649+
throw new UnsupportedOperationException(
650+
"CassandraSchemaStore.createOlapTable()");
651+
}
603652
}
604653

605654
public static class CassandraGraphStore extends CassandraStore {
@@ -632,6 +681,17 @@ public CassandraGraphStore(BackendStoreProvider provider,
632681
new CassandraTables.ShardIndex(store));
633682
registerTableManager(HugeType.UNIQUE_INDEX,
634683
new CassandraTables.UniqueIndex(store));
684+
685+
registerTableManager(this.store() + "_" + HugeType.OLAP.string() + "_" + HugeType.SECONDARY_INDEX.string(),
686+
new CassandraTables.OlapSecondaryIndex(store));
687+
registerTableManager(this.store() + "_" + HugeType.OLAP.string() + "_" + HugeType.RANGE_INT_INDEX.string(),
688+
new CassandraTables.OlapRangeIntIndex(store));
689+
registerTableManager(this.store() + "_" + HugeType.OLAP.string() + "_" + HugeType.RANGE_LONG_INDEX.string(),
690+
new CassandraTables.OlapRangeLongIndex(store));
691+
registerTableManager(this.store() + "_" + HugeType.OLAP.string() + "_" + HugeType.RANGE_FLOAT_INDEX.string(),
692+
new CassandraTables.OlapRangeFloatIndex(store));
693+
registerTableManager(this.store() + "_" + HugeType.OLAP.string() + "_" + HugeType.RANGE_DOUBLE_INDEX.string(),
694+
new CassandraTables.OlapRangeDoubleIndex(store));
635695
}
636696

637697
@Override
@@ -656,5 +716,39 @@ public long getCounter(HugeType type) {
656716
public boolean isSchemaStore() {
657717
return false;
658718
}
719+
720+
@Override
721+
public void createOlapTable(Id pkId) {
722+
this.initAndRegisterOlapTable(pkId);
723+
}
724+
725+
@Override
726+
public void initAndRegisterOlapTable(Id pkId) {
727+
CassandraTable table = new CassandraTables.Olap(this.store(),
728+
pkId.asLong());
729+
registerTableManager(this.store() + "_" + HugeType.OLAP.string() + "_" + pkId,
730+
table);
731+
this.checkOpened();
732+
CassandraSessionPool.Session session = this.sessions.session();
733+
table.init(session);
734+
}
735+
736+
@Override
737+
public void clearOlapTable(Id pkId) {
738+
CassandraTable table = new CassandraTables.Olap(this.store(),
739+
pkId.asLong());
740+
this.checkOpened();
741+
CassandraSessionPool.Session session = this.sessions.session();
742+
table.truncate(session);
743+
}
744+
745+
@Override
746+
public void removeOlapTable(Id pkId) {
747+
CassandraTable table = new CassandraTables.Olap(this.store(),
748+
pkId.asLong());
749+
this.checkOpened();
750+
CassandraSessionPool.Session session = this.sessions.session();
751+
table.dropTable(session);
752+
}
659753
}
660754
}

‎hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraTable.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -401,7 +401,7 @@ protected Iterator<BackendEntry> results2Entries(Query q, ResultSet r) {
401401
});
402402
}
403403

404-
private static CassandraBackendEntry row2Entry(HugeType type, Row row) {
404+
protected static CassandraBackendEntry row2Entry(HugeType type, Row row) {
405405
CassandraBackendEntry entry = new CassandraBackendEntry(type);
406406

407407
List<Definition> cols = row.getColumnDefinitions().asList();
@@ -656,4 +656,8 @@ public void clear(CassandraSessionPool.Session session) {
656656
public void truncate(CassandraSessionPool.Session session) {
657657
this.truncateTable(session);
658658
}
659+
660+
public boolean isOlap() {
661+
return false;
662+
}
659663
}

0 commit comments

Comments
 (0)