use of herddb.model.commands.UpdateStatement in project herddb by diennea.
the class RestartTest method recoverTableCreatedInTransaction.
@Test
public void recoverTableCreatedInTransaction() throws Exception {
Path dataPath = folder.newFolder("data").toPath();
Path logsPath = folder.newFolder("logs").toPath();
Path metadataPath = folder.newFolder("metadata").toPath();
Path tmoDir = folder.newFolder("tmoDir").toPath();
Bytes key = Bytes.from_string("k1");
String nodeId = "localhost";
try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath, 64 * 1024 * 1024), tmoDir, null)) {
manager.start();
CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
assertTrue(manager.waitForTablespace("tblspace1", 10000));
;
Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.STRING).column("name", ColumnTypes.STRING).primaryKey("id").build();
long tx = ((TransactionResult) manager.executeStatement(new BeginTransactionStatement("tblspace1"), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION)).getTransactionId();
manager.executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx));
manager.executeStatement(new InsertStatement("tblspace1", table.name, new Record(key, key)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx));
manager.checkpoint();
manager.executeStatement(new CommitTransactionStatement("tblspace1", tx), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
}
try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath, 64 * 1024 * 1024), tmoDir, null)) {
manager.start();
assertTrue(manager.waitForBootOfLocalTablespaces(10000));
DMLStatementExecutionResult executeStatement = (DMLStatementExecutionResult) manager.executeStatement(new UpdateStatement("tblspace1", "t1", new Record(key, key), null), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
assertEquals(1, executeStatement.getUpdateCount());
// on the log there is an UPDATE for a record which is not on the log
}
try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath, 64 * 1024 * 1024), tmoDir, null)) {
manager.start();
assertTrue(manager.waitForBootOfLocalTablespaces(10000));
GetResult result = manager.get(new GetStatement("tblspace1", "t1", key, null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
assertTrue(result.found());
}
}
use of herddb.model.commands.UpdateStatement in project herddb by diennea.
the class BackupRestoreTest method test_backup_restore_with_updates.
/**
* Check that restore a dirty update restores the latest record version
*/
@Test
public void test_backup_restore_with_updates() throws Exception {
ServerConfiguration serverconfig_1 = new ServerConfiguration(folder.newFolder().toPath());
serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
serverconfig_1.set(ServerConfiguration.PROPERTY_PORT, 7867);
serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
serverconfig_1.set(ServerConfiguration.PROPERTY_ENFORCE_LEADERSHIP, false);
/* Disable page compaction (avoid compaction of dirty page) */
serverconfig_1.set(ServerConfiguration.PROPERTY_FILL_PAGE_THRESHOLD, 0.0D);
ServerConfiguration serverconfig_2 = serverconfig_1.copy().set(ServerConfiguration.PROPERTY_NODEID, "server2").set(ServerConfiguration.PROPERTY_BASEDIR, folder.newFolder().toPath().toAbsolutePath()).set(ServerConfiguration.PROPERTY_PORT, 7868);
ClientConfiguration client_configuration = new ClientConfiguration(folder.newFolder().toPath());
client_configuration.set(ClientConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
client_configuration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
client_configuration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
client_configuration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
try (Server server_1 = new Server(serverconfig_1)) {
server_1.start();
server_1.waitForStandaloneBoot();
Table table = Table.builder().name("t1").column("c", ColumnTypes.INTEGER).column("d", ColumnTypes.INTEGER).primaryKey("c").build();
DBManager manager = server_1.getManager();
manager.executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
manager.executeUpdate(new InsertStatement(TableSpace.DEFAULT, table.name, RecordSerializer.makeRecord(table, "c", 1, "d", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
manager.executeUpdate(new InsertStatement(TableSpace.DEFAULT, table.name, RecordSerializer.makeRecord(table, "c", 2, "d", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
manager.executeUpdate(new InsertStatement(TableSpace.DEFAULT, table.name, RecordSerializer.makeRecord(table, "c", 3, "d", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
manager.executeUpdate(new InsertStatement(TableSpace.DEFAULT, table.name, RecordSerializer.makeRecord(table, "c", 4, "d", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
manager.checkpoint();
/* Check that new data isn't in a dirty page */
assertEquals(0, manager.getTableSpaceManager(TableSpace.DEFAULT).getTableManager(table.name).getStats().getDirtypages());
final Record update = RecordSerializer.makeRecord(table, "c", 2, "d", 22);
manager.executeUpdate(new InsertStatement(TableSpace.DEFAULT, table.name, RecordSerializer.makeRecord(table, "c", 5, "d", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
manager.executeUpdate(new UpdateStatement(TableSpace.DEFAULT, table.name, new ConstValueRecordFunction(update.key.data), new ConstValueRecordFunction(update.value.data), null), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
/* Now we have a dirty page in the checkpoint */
manager.checkpoint();
/* Check that a dirty page exists */
assertEquals(1, manager.getTableSpaceManager(TableSpace.DEFAULT).getTableManager(table.name).getStats().getDirtypages());
try (Server server_2 = new Server(serverconfig_2)) {
server_2.start();
try (HDBClient client = new HDBClient(client_configuration);
HDBConnection connection = client.openConnection()) {
assertEquals(5, connection.executeScan(TableSpace.DEFAULT, "SELECT * FROM t1", Collections.emptyList(), 0, 0, 10).consume().size());
ByteArrayOutputStream oo = new ByteArrayOutputStream();
BackupUtils.dumpTableSpace(TableSpace.DEFAULT, 64 * 1024, connection, oo, new ProgressListener() {
});
byte[] backupData = oo.toByteArray();
connection.executeUpdate(TableSpace.DEFAULT, "DELETE FROM t1", 0, false, Collections.emptyList());
assertEquals(0, connection.executeScan(TableSpace.DEFAULT, "SELECT * FROM t1", Collections.emptyList(), 0, 0, 10).consume().size());
BackupUtils.restoreTableSpace("newts", server_2.getNodeId(), connection, new ByteArrayInputStream(backupData), new ProgressListener() {
});
/* No new insert AND no delete... if 5 it did see the new insert but missed the delete! */
assertEquals(5, connection.executeScan("newts", "SELECT * FROM newts.t1", Collections.emptyList(), 0, 0, 10).consume().size());
}
}
}
}
use of herddb.model.commands.UpdateStatement in project herddb by diennea.
the class UpdateOp method executeAsync.
@Override
public CompletableFuture<StatementExecutionResult> executeAsync(TableSpaceManager tableSpaceManager, TransactionContext transactionContext, StatementEvaluationContext context, boolean lockRequired, boolean forWrite) {
StatementExecutionResult input = this.input.execute(tableSpaceManager, transactionContext, context, true, true);
ScanResult downstreamScanResult = (ScanResult) input;
final Table table = tableSpaceManager.getTableManager(tableName).getTable();
long transactionId = transactionContext.transactionId;
List<DMLStatement> statements = new ArrayList<>();
try (DataScanner inputScanner = downstreamScanResult.dataScanner) {
while (inputScanner.hasNext()) {
DataAccessor row = inputScanner.next();
long transactionIdFromScanner = inputScanner.getTransactionId();
if (transactionIdFromScanner > 0 && transactionIdFromScanner != transactionId) {
transactionId = transactionIdFromScanner;
transactionContext = new TransactionContext(transactionId);
}
Bytes key = RecordSerializer.serializeIndexKey(row, table, table.getPrimaryKey());
DMLStatement updateStatement = new UpdateStatement(tableSpace, tableName, new ConstValueRecordFunction(key), this.recordFunction, null).setReturnValues(returnValues);
statements.add(updateStatement);
}
if (statements.isEmpty()) {
return CompletableFuture.completedFuture(new DMLStatementExecutionResult(transactionId, 0, null, null));
}
if (statements.size() == 1) {
return tableSpaceManager.executeStatementAsync(statements.get(0), context, transactionContext);
}
CompletableFuture<StatementExecutionResult> finalResult = new CompletableFuture<>();
AtomicInteger updateCounts = new AtomicInteger();
AtomicReference<Bytes> lastKey = new AtomicReference<>();
AtomicReference<Bytes> lastNewValue = new AtomicReference<>();
class ComputeNext implements BiConsumer<StatementExecutionResult, Throwable> {
int current;
public ComputeNext(int current) {
this.current = current;
}
@Override
public void accept(StatementExecutionResult res, Throwable error) {
if (error != null) {
finalResult.completeExceptionally(error);
return;
}
DMLStatementExecutionResult dml = (DMLStatementExecutionResult) res;
updateCounts.addAndGet(dml.getUpdateCount());
if (returnValues) {
lastKey.set(dml.getKey());
lastNewValue.set(dml.getNewvalue());
}
long newTransactionId = res.transactionId;
if (current == statements.size()) {
DMLStatementExecutionResult finalDMLResult = new DMLStatementExecutionResult(newTransactionId, updateCounts.get(), lastKey.get(), lastNewValue.get());
finalResult.complete(finalDMLResult);
return;
}
DMLStatement nextStatement = statements.get(current);
TransactionContext transactionContext = new TransactionContext(newTransactionId);
CompletableFuture<StatementExecutionResult> nextPromise = tableSpaceManager.executeStatementAsync(nextStatement, context, transactionContext);
nextPromise.whenComplete(new ComputeNext(current + 1));
}
}
DMLStatement firstStatement = statements.get(0);
tableSpaceManager.executeStatementAsync(firstStatement, context, transactionContext).whenComplete(new ComputeNext(1));
return finalResult;
} catch (DataScannerException err) {
throw new StatementExecutionException(err);
}
}
use of herddb.model.commands.UpdateStatement in project herddb by diennea.
the class BackupRestoreTest method test_backup_restore_with_updates.
/**
* Check that restore a dirty update restores the latest record version
*/
@Test
public void test_backup_restore_with_updates() throws Exception {
ServerConfiguration serverconfig_1 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
serverconfig_1.set(ServerConfiguration.PROPERTY_ENFORCE_LEADERSHIP, false);
/* Disable page compaction (avoid compaction of dirty page) */
serverconfig_1.set(ServerConfiguration.PROPERTY_FILL_PAGE_THRESHOLD, 0.0D);
ServerConfiguration serverconfig_2 = serverconfig_1.copy().set(ServerConfiguration.PROPERTY_NODEID, "server2").set(ServerConfiguration.PROPERTY_BASEDIR, folder.newFolder().toPath().toAbsolutePath());
ClientConfiguration client_configuration = new ClientConfiguration(folder.newFolder().toPath());
client_configuration.set(ClientConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
client_configuration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
client_configuration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
client_configuration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
try (Server server_1 = new Server(serverconfig_1)) {
server_1.start();
server_1.waitForStandaloneBoot();
Table table = Table.builder().name("t1").column("c", ColumnTypes.INTEGER).column("d", ColumnTypes.INTEGER).primaryKey("c").build();
DBManager manager = server_1.getManager();
manager.executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
manager.executeUpdate(new InsertStatement(TableSpace.DEFAULT, table.name, RecordSerializer.makeRecord(table, "c", 1, "d", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
manager.executeUpdate(new InsertStatement(TableSpace.DEFAULT, table.name, RecordSerializer.makeRecord(table, "c", 2, "d", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
manager.executeUpdate(new InsertStatement(TableSpace.DEFAULT, table.name, RecordSerializer.makeRecord(table, "c", 3, "d", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
manager.executeUpdate(new InsertStatement(TableSpace.DEFAULT, table.name, RecordSerializer.makeRecord(table, "c", 4, "d", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
manager.checkpoint();
/* Check that new data isn't in a dirty page */
assertEquals(0, manager.getTableSpaceManager(TableSpace.DEFAULT).getTableManager(table.name).getStats().getDirtypages());
final Record update = RecordSerializer.makeRecord(table, "c", 2, "d", 22);
manager.executeUpdate(new InsertStatement(TableSpace.DEFAULT, table.name, RecordSerializer.makeRecord(table, "c", 5, "d", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
manager.executeUpdate(new UpdateStatement(TableSpace.DEFAULT, table.name, new ConstValueRecordFunction(update.key), new ConstValueRecordFunction(update.value), null), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
/* Now we have a dirty page in the checkpoint */
manager.checkpoint();
/* Check that a dirty page exists */
assertEquals(1, manager.getTableSpaceManager(TableSpace.DEFAULT).getTableManager(table.name).getStats().getDirtypages());
try (Server server_2 = new Server(serverconfig_2)) {
server_2.start();
try (HDBClient client = new HDBClient(client_configuration);
HDBConnection connection = client.openConnection()) {
assertEquals(5, connection.executeScan(TableSpace.DEFAULT, "SELECT * FROM t1", true, Collections.emptyList(), 0, 0, 10, true).consume().size());
ByteArrayOutputStream oo = new ByteArrayOutputStream();
BackupUtils.dumpTableSpace(TableSpace.DEFAULT, 64 * 1024, connection, oo, new ProgressListener() {
});
byte[] backupData = oo.toByteArray();
connection.executeUpdate(TableSpace.DEFAULT, "DELETE FROM t1", 0, false, true, Collections.emptyList());
assertEquals(0, connection.executeScan(TableSpace.DEFAULT, "SELECT * FROM t1", true, Collections.emptyList(), 0, 0, 10, true).consume().size());
BackupUtils.restoreTableSpace("newts", server_2.getNodeId(), connection, new ByteArrayInputStream(backupData), new ProgressListener() {
});
/* No new insert AND no delete... if 5 it did see the new insert but missed the delete! */
assertEquals(5, connection.executeScan("newts", "SELECT * FROM newts.t1", true, Collections.emptyList(), 0, 0, 10, true).consume().size());
}
}
}
}
use of herddb.model.commands.UpdateStatement in project herddb by diennea.
the class FlushFileTest method test.
@Test
public void test() throws Exception {
{
Record record = new Record(Bytes.from_string("key1"), Bytes.from_string("0"));
InsertStatement st = new InsertStatement(tableSpace, tableName, record);
assertEquals(1, manager.executeUpdate(st, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
}
String tableUuid = manager.getTableSpaceManager(tableSpace).getTableManager(tableName).getTable().uuid;
assertEquals(0, dataStorageManager.getActualNumberOfPages(tableSpaceUUID, tableUuid));
manager.checkpoint();
assertNotNull(dataStorageManager.readPage(tableSpaceUUID, tableUuid, 1L));
assertEquals(1, dataStorageManager.getActualNumberOfPages(tableSpaceUUID, tableUuid));
{
GetResult result = manager.get(new GetStatement(tableSpace, tableName, Bytes.from_string("key1"), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
assertTrue(result.found());
}
manager.checkpoint();
assertEquals(1, dataStorageManager.getActualNumberOfPages(tableSpaceUUID, tableUuid));
{
Record record = new Record(Bytes.from_string("key1"), Bytes.from_string("5"));
UpdateStatement st = new UpdateStatement(tableSpace, tableName, record, null);
assertEquals(1, manager.executeUpdate(st, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
}
// a new page must be allocated, but the first will be dropped, as it does not contain any useful record
manager.checkpoint();
assertEquals(1, dataStorageManager.getActualNumberOfPages(tableSpaceUUID, tableUuid));
{
Record record = new Record(Bytes.from_string("key1"), Bytes.from_string("6"));
UpdateStatement st = new UpdateStatement(tableSpace, tableName, record, null);
assertEquals(1, manager.executeUpdate(st, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
}
{
Record record = new Record(Bytes.from_string("key1"), Bytes.from_string("7"));
UpdateStatement st = new UpdateStatement(tableSpace, tableName, record, null);
assertEquals(1, manager.executeUpdate(st, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
}
// only a new page must be allocated, not two more, but the prev page will be dropped, as it does not contain any useful record
manager.checkpoint();
assertEquals(1, dataStorageManager.getActualNumberOfPages(tableSpaceUUID, tableUuid));
{
DeleteStatement st = new DeleteStatement(tableSpace, tableName, Bytes.from_string("key1"), null);
assertEquals(1, manager.executeUpdate(st, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
GetResult result = manager.get(new GetStatement(tableSpace, tableName, Bytes.from_string("key1"), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
assertFalse(result.found());
}
// a delete does not trigger new pages in this case, no more record will lead to no more active page
manager.checkpoint();
assertEquals(0, dataStorageManager.getActualNumberOfPages(tableSpaceUUID, tableUuid));
{
assertEquals(1, manager.executeUpdate(new InsertStatement(tableSpace, tableName, new Record(Bytes.from_string("key2"), Bytes.from_string("50"))), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
assertEquals(1, manager.executeUpdate(new InsertStatement(tableSpace, tableName, new Record(Bytes.from_string("key3"), Bytes.from_string("60"))), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
}
manager.checkpoint();
assertEquals(1, dataStorageManager.getActualNumberOfPages(tableSpaceUUID, tableUuid));
{
DeleteStatement st = new DeleteStatement(tableSpace, tableName, Bytes.from_string("key2"), null);
assertEquals(1, manager.executeUpdate(st, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
}
// a new page, containg the key3 record is needed
manager.checkpoint();
assertEquals(1, dataStorageManager.getActualNumberOfPages(tableSpaceUUID, tableUuid));
}
Aggregations