use of herddb.model.DataScanner in project herddb by diennea.
the class IndexScanRangeTest method noIndexOperation.
private void noIndexOperation(String indexType) throws Exception {
String nodeId = "localhost";
try (DBManager manager = new DBManager("localhost", new MemoryMetadataStorageManager(), new MemoryDataStorageManager(), new MemoryCommitLogManager(), null, null)) {
manager.start();
CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
manager.waitForTablespace("tblspace1", 10000);
Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.STRING).column("n1", ColumnTypes.INTEGER).column("n2", ColumnTypes.INTEGER).column("name", ColumnTypes.STRING).primaryKey("id").build();
CreateTableStatement st2 = new CreateTableStatement(table);
manager.executeStatement(st2, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
Index index = Index.builder().onTable(table).type(indexType).column("n1", ColumnTypes.INTEGER).build();
TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,n2,name) values('a',1,5,'n1')", Collections.emptyList());
TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,n2,name) values('b',2,5,'n1')", Collections.emptyList());
TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,n2,name) values('c',2,5,'n2')", Collections.emptyList());
TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,n2,name) values('d',2,5,'n2')", Collections.emptyList());
TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,n2,name) values('e',3,5,'n2')", Collections.emptyList());
// create index, it will be built using existing data
CreateIndexStatement st3 = new CreateIndexStatement(index);
manager.executeStatement(st3, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
{
TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE n1<n2", Collections.emptyList(), true, true, false, -1);
ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
System.out.println("indexOperation:" + scan.getPredicate().getIndexOperation());
assertNull(scan.getPredicate().getIndexOperation());
try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
assertEquals(5, scan1.consume().size());
}
}
}
}
use of herddb.model.DataScanner in project herddb by diennea.
the class IndexScanRangeTest method secondaryIndexPrefixScan.
private void secondaryIndexPrefixScan(String indexType) throws Exception {
String nodeId = "localhost";
try (DBManager manager = new DBManager("localhost", new MemoryMetadataStorageManager(), new MemoryDataStorageManager(), new MemoryCommitLogManager(), null, null)) {
manager.start();
CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
manager.waitForTablespace("tblspace1", 10000);
Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.STRING).column("n1", ColumnTypes.INTEGER).column("n2", ColumnTypes.INTEGER).column("name", ColumnTypes.STRING).primaryKey("id").build();
CreateTableStatement st2 = new CreateTableStatement(table);
manager.executeStatement(st2, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
Index index = Index.builder().onTable(table).type(indexType).column("n1", ColumnTypes.INTEGER).column("n2", ColumnTypes.INTEGER).build();
TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,n2,name) values('a',1,5,'n1')", Collections.emptyList());
TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,n2,name) values('b',2,5,'n1')", Collections.emptyList());
TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,n2,name) values('c',2,5,'n2')", Collections.emptyList());
TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,n2,name) values('d',2,5,'n2')", Collections.emptyList());
TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,n1,n2,name) values('e',3,5,'n2')", Collections.emptyList());
// create index, it will be built using existing data
CreateIndexStatement st3 = new CreateIndexStatement(index);
manager.executeStatement(st3, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
{
TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE n1=2", Collections.emptyList(), true, true, false, -1);
ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexPrefixScan);
try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
assertEquals(3, scan1.consume().size());
}
}
}
}
use of herddb.model.DataScanner in project herddb by diennea.
the class TablespaceReplicasStateTest method test_leader_online_log_available.
@Test
public void test_leader_online_log_available() throws Exception {
ServerConfiguration serverconfig_1 = new ServerConfiguration(folder.newFolder().toPath());
serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
serverconfig_1.set(ServerConfiguration.PROPERTY_PORT, 7867);
serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
serverconfig_1.set(ServerConfiguration.PROPERTY_ENFORCE_LEADERSHIP, false);
ServerConfiguration serverconfig_2 = serverconfig_1.copy().set(ServerConfiguration.PROPERTY_NODEID, "server2").set(ServerConfiguration.PROPERTY_BASEDIR, folder.newFolder().toPath().toAbsolutePath()).set(ServerConfiguration.PROPERTY_PORT, 7868);
try (Server server_1 = new Server(serverconfig_1)) {
server_1.start();
server_1.waitForStandaloneBoot();
Table table = Table.builder().name("t1").column("c", ColumnTypes.INTEGER).primaryKey("c").build();
server_1.getManager().executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
try (Server server_2 = new Server(serverconfig_2)) {
server_2.start();
server_1.getManager().executeStatement(new AlterTableSpaceStatement(TableSpace.DEFAULT, new HashSet<>(Arrays.asList("server1", "server2")), "server1", 2, 0), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
assertTrue(server_2.getManager().waitForTablespace(TableSpace.DEFAULT, 60000, false));
try (DataScanner scan = scan(server_1.getManager(), "SELECT * FROM systablespacereplicastate", Collections.emptyList())) {
List<DataAccessor> results = scan.consume();
assertEquals(2, results.size());
}
try (DataScanner scan = scan(server_1.getManager(), "SELECT * FROM systablespacereplicastate " + "where nodeId='" + server_2.getNodeId() + "' and mode='follower'", Collections.emptyList())) {
assertEquals(1, scan.consume().size());
}
try (DataScanner scan = scan(server_2.getManager(), "SELECT * FROM systablespacereplicastate " + "where nodeId='" + server_2.getNodeId() + "' and mode='follower'", Collections.emptyList())) {
assertEquals(1, scan.consume().size());
}
try (DataScanner scan = scan(server_1.getManager(), "SELECT * FROM systablespacereplicastate " + "where nodeId='" + server_1.getNodeId() + "' and mode='leader'", Collections.emptyList())) {
assertEquals(1, scan.consume().size());
}
server_1.getManager().executeStatement(new AlterTableSpaceStatement(TableSpace.DEFAULT, new HashSet<>(Arrays.asList("server1")), "server1", 1, 0), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
for (int i = 0; i < 100; i++) {
try (DataScanner scan = scan(server_1.getManager(), "SELECT * FROM systablespacereplicastate where nodeId='" + server_2.getNodeId() + "' and mode='stopped'", Collections.emptyList())) {
if (scan.consume().size() > 0) {
break;
}
}
}
try (DataScanner scan = scan(server_1.getManager(), "SELECT * FROM systablespacereplicastate where nodeId='" + server_2.getNodeId() + "' and mode='stopped'", Collections.emptyList())) {
assertEquals(1, scan.consume().size());
}
}
}
}
use of herddb.model.DataScanner in project herddb by diennea.
the class DBManager method executePlan.
public StatementExecutionResult executePlan(ExecutionPlan plan, StatementEvaluationContext context, TransactionContext transactionContext) throws StatementExecutionException {
context.setManager(this);
plan.validateContext(context);
if (plan.mainStatement instanceof ScanStatement) {
DataScanner result = scan((ScanStatement) plan.mainStatement, context, transactionContext);
// transction can be auto generated during the scan
transactionContext = new TransactionContext(result.transactionId);
return executeDataScannerPlan(plan, result, context, transactionContext);
} else if (plan.dataSource != null) {
// INSERT from SELECT
try {
ScanResult data = (ScanResult) executePlan(plan.dataSource, context, transactionContext);
int insertCount = 0;
try {
// transction can be auto generated during the scan
transactionContext = new TransactionContext(data.transactionId);
while (data.dataScanner.hasNext()) {
DataAccessor tuple = data.dataScanner.next();
SQLStatementEvaluationContext tmp_context = new SQLStatementEvaluationContext("--", Arrays.asList(tuple.getValues()));
DMLStatementExecutionResult res = (DMLStatementExecutionResult) executeStatement(plan.mainStatement, tmp_context, transactionContext);
insertCount += res.getUpdateCount();
}
} finally {
data.dataScanner.close();
}
return new DMLStatementExecutionResult(transactionContext.transactionId, insertCount);
} catch (DataScannerException err) {
throw new StatementExecutionException(err);
}
} else if (plan.joinStatements != null) {
List<DataScanner> scanResults = new ArrayList<>();
for (ScanStatement statement : plan.joinStatements) {
DataScanner result = scan(statement, context, transactionContext);
// transction can be auto generated during the scan
transactionContext = new TransactionContext(result.transactionId);
scanResults.add(result);
}
return executeJoinedScansPlan(scanResults, context, transactionContext, plan);
} else if (plan.insertStatements != null) {
int insertCount = 0;
for (InsertStatement insert : plan.insertStatements) {
DMLStatementExecutionResult res = (DMLStatementExecutionResult) executeStatement(insert, context, transactionContext);
// transction can be auto generated during the loop
transactionContext = new TransactionContext(res.transactionId);
insertCount += res.getUpdateCount();
}
return new DMLStatementExecutionResult(transactionContext.transactionId, insertCount);
} else {
return executeStatement(plan.mainStatement, context, transactionContext);
}
}
use of herddb.model.DataScanner in project herddb by diennea.
the class TableManager method scanWithStream.
private DataScanner scanWithStream(ScanStatement statement, StatementEvaluationContext context, Transaction transaction, boolean lockRequired, boolean forWrite) throws StatementExecutionException {
if (transaction != null) {
transaction.increaseRefcount();
}
try {
final TupleComparator comparator = statement.getComparator();
boolean sorted = comparator != null;
boolean sortedByClusteredIndex = comparator != null && comparator.isOnlyPrimaryKeyAndAscending() && keyToPageSortedAscending;
final Projection projection = statement.getProjection();
final boolean applyProjectionDuringScan = projection != null && !sorted;
ScanLimits limits = statement.getLimits();
int maxRows = limits == null ? 0 : limits.computeMaxRows(context);
int offset = limits == null ? 0 : limits.computeOffset(context);
Stream<DataAccessor> result;
Function<Record, DataAccessor> mapper = (Record record) -> {
DataAccessor tuple;
if (applyProjectionDuringScan) {
tuple = projection.map(record.getDataAccessor(table), context);
} else {
tuple = record.getDataAccessor(table);
}
return tuple;
};
Stream<Record> recordsFromTransactionSorted = streamTransactionData(transaction, statement.getPredicate(), context);
Stream<DataAccessor> fromTransactionSorted = recordsFromTransactionSorted != null ? recordsFromTransactionSorted.map(mapper) : null;
if (fromTransactionSorted != null && comparator != null) {
fromTransactionSorted = fromTransactionSorted.sorted(comparator);
}
Stream<DataAccessor> tableData = streamTableData(statement, context, transaction, lockRequired, forWrite).map(mapper);
if (maxRows > 0) {
if (sortedByClusteredIndex) {
// already sorted if needed
if (fromTransactionSorted != null) {
// already sorted from index
tableData = tableData.limit(maxRows + offset);
fromTransactionSorted = fromTransactionSorted.limit(maxRows + offset);
// we need to re-sort after merging the data
result = Stream.concat(fromTransactionSorted, tableData).sorted(comparator);
} else {
// already sorted from index
tableData = tableData.limit(maxRows + offset);
// no need to re-sort
result = tableData;
}
} else if (sorted) {
// need to sort
tableData = tableData.sorted(comparator);
// already sorted if needed
if (fromTransactionSorted != null) {
tableData = tableData.limit(maxRows + offset);
fromTransactionSorted = fromTransactionSorted.limit(maxRows + offset);
// we need to re-sort after merging the data
result = Stream.concat(fromTransactionSorted, tableData).sorted(comparator);
} else {
tableData = tableData.limit(maxRows + offset);
// no need to sort again
result = tableData;
}
} else if (fromTransactionSorted == null) {
result = tableData;
} else {
result = Stream.concat(fromTransactionSorted, tableData);
}
} else {
if (sortedByClusteredIndex) {
// already sorted from index
if (fromTransactionSorted != null) {
tableData = tableData.sorted(comparator);
// fromTransactionSorted is already sorted
// we need to re-sort
result = Stream.concat(fromTransactionSorted, tableData).sorted(comparator);
} else {
result = tableData;
}
} else if (sorted) {
// we need to re-sort
if (fromTransactionSorted != null) {
result = Stream.concat(fromTransactionSorted, tableData).sorted(comparator);
} else {
result = tableData.sorted(comparator);
}
} else if (fromTransactionSorted != null) {
// no need to sort
result = Stream.concat(fromTransactionSorted, tableData);
} else {
result = tableData;
}
}
if (offset > 0) {
result = result.skip(offset);
}
if (maxRows > 0) {
result = result.limit(maxRows);
}
if (!applyProjectionDuringScan && projection != null) {
result = result.map(r -> projection.map(r, context));
}
String[] fieldNames;
Column[] columns;
if (projection != null) {
fieldNames = projection.getFieldNames();
columns = projection.getColumns();
} else {
fieldNames = table.columnNames;
columns = table.columns;
}
return new StreamDataScanner(transaction, fieldNames, columns, result);
} finally {
if (transaction != null) {
transaction.decreaseRefCount();
}
}
}
Aggregations