use of herddb.server.ServerConfiguration in project herddb by diennea.
the class BookKeeperCommitLogTest method testFollowEmptyLedgerBookieDown.
@Test
public void testFollowEmptyLedgerBookieDown() throws Exception {
BookieId secondBookie = testEnv.startNewBookie();
final String tableSpaceUUID = UUID.randomUUID().toString();
final String name = TableSpace.DEFAULT;
final String nodeid = "nodeid";
ServerConfiguration serverConfiguration = newServerConfigurationWithAutoPort();
try (ZookeeperMetadataStorageManager man = new ZookeeperMetadataStorageManager(testEnv.getAddress(), testEnv.getTimeout(), testEnv.getPath());
BookkeeperCommitLogManager logManager = new BookkeeperCommitLogManager(man, serverConfiguration, NullStatsLogger.INSTANCE)) {
logManager.setEnsemble(2);
logManager.setWriteQuorumSize(2);
logManager.setAckQuorumSize(2);
man.start();
logManager.start();
try (BookkeeperCommitLog writer = logManager.createCommitLog(tableSpaceUUID, name, nodeid)) {
writer.startWriting(1);
// create a ledger, up to 0.14.x no "logical" write happens, so Bookies are not aware of the
// the ledger
// stop one bookie
testEnv.stopBookie(secondBookie);
// but since 0.15.0 we are now writing a NOOP entry at the beginning of the ledger in order to workaround this issue.
try (BookkeeperCommitLog reader = logManager.createCommitLog(tableSpaceUUID, name, nodeid)) {
reader.recovery(LogSequenceNumber.START_OF_TIME, (a, b) -> {
}, false);
}
}
}
}
use of herddb.server.ServerConfiguration in project herddb by diennea.
the class BookKeeperCommitLogTest method testBookieFailureSyncWrites.
@Test
public void testBookieFailureSyncWrites() throws Exception {
final String tableSpaceUUID = UUID.randomUUID().toString();
final String name = TableSpace.DEFAULT;
final String nodeid = "nodeid";
ServerConfiguration serverConfiguration = newServerConfigurationWithAutoPort();
try (ZookeeperMetadataStorageManager man = new ZookeeperMetadataStorageManager(testEnv.getAddress(), testEnv.getTimeout(), testEnv.getPath());
BookkeeperCommitLogManager logManager = new BookkeeperCommitLogManager(man, serverConfiguration, NullStatsLogger.INSTANCE)) {
man.start();
logManager.start();
try (BookkeeperCommitLog writer = logManager.createCommitLog(tableSpaceUUID, name, nodeid)) {
writer.startWriting(1);
writer.log(LogEntryFactory.beginTransaction(1), true).getLogSequenceNumber();
this.testEnv.pauseBookie();
// this is deemed to fail and we will close the log
TestUtils.assertThrows(LogNotAvailableException.class, () -> writer.log(LogEntryFactory.beginTransaction(2), true).getLogSequenceNumber());
assertTrue(writer.isFailed());
// this one will fail as well
TestUtils.assertThrows(LogNotAvailableException.class, () -> writer.log(LogEntryFactory.beginTransaction(2), true).getLogSequenceNumber());
// no way to recover this instance of BookkeeperCommitLog
// we will bounce the leader and it will restart with a full recovery
// in a production env you do not have only one bookie, and the failure
// of a single bookie will be handled with an ensemble change
this.testEnv.resumeBookie();
TestUtils.assertThrows(LogNotAvailableException.class, () -> writer.log(LogEntryFactory.beginTransaction(2), true).getLogSequenceNumber());
}
// check expected reads
try (BookkeeperCommitLog reader = logManager.createCommitLog(tableSpaceUUID, name, nodeid)) {
List<Map.Entry<LogSequenceNumber, LogEntry>> list = new ArrayList<>();
reader.recovery(LogSequenceNumber.START_OF_TIME, (lsn, entry) -> {
if (entry.type != LogEntryType.NOOP) {
list.add(new AbstractMap.SimpleImmutableEntry<>(lsn, entry));
}
}, false);
assertEquals(1, list.size());
assertTrue(list.get(0).getKey().after(LogSequenceNumber.START_OF_TIME));
}
}
}
use of herddb.server.ServerConfiguration in project herddb by diennea.
the class ChangeRoleTest method testChangeRoleAndReleaseMemory.
@Test
public void testChangeRoleAndReleaseMemory() throws Exception {
ServerConfiguration serverconfig_1 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
serverconfig_1.set(ServerConfiguration.PROPERTY_ENFORCE_LEADERSHIP, false);
// disabled
serverconfig_1.set(ServerConfiguration.PROPERTY_BOOKKEEPER_MAX_IDLE_TIME, 0);
ServerConfiguration serverconfig_2 = serverconfig_1.copy().set(ServerConfiguration.PROPERTY_NODEID, "server2").set(ServerConfiguration.PROPERTY_BASEDIR, folder.newFolder().toPath().toAbsolutePath());
try (Server server_1 = new Server(serverconfig_1);
Server server_2 = new Server(serverconfig_2)) {
server_1.start();
server_1.waitForStandaloneBoot();
server_2.start();
MemoryManager server2MemoryManager = server_2.getManager().getMemoryManager();
Table table = Table.builder().name("t1").column("c", ColumnTypes.INTEGER).column("s", ColumnTypes.INTEGER).primaryKey("c").build();
Index index = Index.builder().onTable(table).type(Index.TYPE_BRIN).column("s", ColumnTypes.STRING).build();
server_1.getManager().executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
server_1.getManager().executeStatement(new CreateIndexStatement(index), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 1, "s", "1")), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 2, "s", "2")), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 3, "s", "3")), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 4, "s", "4")), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
// set forcibly server2 as new follower
server_1.getManager().executeStatement(new AlterTableSpaceStatement(TableSpace.DEFAULT, new HashSet<>(Arrays.asList("server1", "server2")), "server1", 1, 0), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
assertEquals(0, server2MemoryManager.getDataPageReplacementPolicy().size());
assertEquals(0, server2MemoryManager.getIndexPageReplacementPolicy().size());
assertEquals(0, server2MemoryManager.getPKPageReplacementPolicy().size());
server_2.waitForTableSpaceBoot(TableSpace.DEFAULT, false);
assertEquals(1, server2MemoryManager.getDataPageReplacementPolicy().size());
assertEquals(1, server2MemoryManager.getIndexPageReplacementPolicy().size());
assertEquals(1, server2MemoryManager.getPKPageReplacementPolicy().size());
// stop tablespace on server2
server_1.getManager().executeStatement(new AlterTableSpaceStatement(TableSpace.DEFAULT, new HashSet<>(Arrays.asList("server1")), "server1", 1, 0), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
server_2.getManager().triggerActivator(ActivatorRunRequest.FULL);
// wait for tablespace manager to be deallocated
herddb.utils.TestUtils.waitForCondition(() -> {
TableSpaceManager tableSpaceManager = server_2.getManager().getTableSpaceManager(TableSpace.DEFAULT);
return tableSpaceManager == null;
}, herddb.utils.TestUtils.NOOP, 100);
// memory must have been totally released
assertEquals(0, server2MemoryManager.getDataPageReplacementPolicy().size());
assertEquals(0, server2MemoryManager.getIndexPageReplacementPolicy().size());
assertEquals(0, server2MemoryManager.getPKPageReplacementPolicy().size());
// start tablespace on server2, as let it become leader
server_1.getManager().executeStatement(new AlterTableSpaceStatement(TableSpace.DEFAULT, new HashSet<>(Arrays.asList("server1", "server2")), "server2", 1, 0), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
server_2.waitForTableSpaceBoot(TableSpace.DEFAULT, true);
assertTrue("unexpected value " + server2MemoryManager.getDataPageReplacementPolicy().size(), server2MemoryManager.getDataPageReplacementPolicy().size() >= 1);
assertEquals(1, server2MemoryManager.getIndexPageReplacementPolicy().size());
assertEquals(1, server2MemoryManager.getPKPageReplacementPolicy().size());
server_1.getManager().executeStatement(new AlterTableSpaceStatement(TableSpace.DEFAULT, new HashSet<>(Arrays.asList("server1")), "server1", 1, 0), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
// wait for tablespace manager to be deallocated
herddb.utils.TestUtils.waitForCondition(() -> {
TableSpaceManager tableSpaceManager = server_2.getManager().getTableSpaceManager(TableSpace.DEFAULT);
return tableSpaceManager == null;
}, herddb.utils.TestUtils.NOOP, 100);
// memory must have been totally released again
assertEquals(0, server2MemoryManager.getDataPageReplacementPolicy().size());
assertEquals(0, server2MemoryManager.getIndexPageReplacementPolicy().size());
assertEquals(0, server2MemoryManager.getPKPageReplacementPolicy().size());
}
}
use of herddb.server.ServerConfiguration in project herddb by diennea.
the class SimpleCDCTest method testBasicCaptureDataChange.
@Test
public void testBasicCaptureDataChange() throws Exception {
ServerConfiguration serverconfig_1 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
ClientConfiguration client_configuration = new ClientConfiguration(folder.newFolder().toPath());
client_configuration.set(ClientConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
client_configuration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
client_configuration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
client_configuration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
try (Server server_1 = new Server(serverconfig_1)) {
server_1.start();
server_1.waitForStandaloneBoot();
Table table = Table.builder().name("t1").column("c", ColumnTypes.INTEGER).column("d", ColumnTypes.INTEGER).primaryKey("c").build();
server_1.getManager().executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 1, "d", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 2, "d", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
long tx = TestUtils.beginTransaction(server_1.getManager(), TableSpace.DEFAULT);
server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 3, "d", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx));
server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 4, "d", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx));
List<ChangeDataCapture.Mutation> mutations = new ArrayList<>();
try (final ChangeDataCapture cdc = new ChangeDataCapture(server_1.getManager().getTableSpaceManager(TableSpace.DEFAULT).getTableSpaceUUID(), client_configuration, new ChangeDataCapture.MutationListener() {
@Override
public void accept(ChangeDataCapture.Mutation mutation) {
LOG.log(Level.INFO, "mutation " + mutation);
assertTrue(mutation.getTimestamp() > 0);
assertNotNull(mutation.getLogSequenceNumber());
assertNotNull(mutation.getTable());
mutations.add(mutation);
}
}, LogSequenceNumber.START_OF_TIME, new InMemoryTableHistoryStorage())) {
cdc.start();
cdc.run();
// we are missing the last entry, because it is not confirmed yet on BookKeeper at this point
// also the mutations in the transaction are not visible
assertEquals(3, mutations.size());
// commit the transaction
TestUtils.commitTransaction(server_1.getManager(), TableSpace.DEFAULT, tx);
server_1.getManager().executeUpdate(new UpdateStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 4, "d", 2), null), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
cdc.run();
assertEquals(5, mutations.size());
server_1.getManager().executeStatement(new AlterTableStatement(Arrays.asList(Column.column("e", ColumnTypes.INTEGER)), Collections.emptyList(), Collections.emptyList(), null, table.name, TableSpace.DEFAULT, null, Collections.emptyList(), Collections.emptyList()), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
cdc.run();
assertEquals(6, mutations.size());
// transaction to be rolled back
long tx2 = TestUtils.beginTransaction(server_1.getManager(), TableSpace.DEFAULT);
server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 30, "d", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx2));
server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 31, "d", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx2));
TestUtils.roolbackTransaction(server_1.getManager(), TableSpace.DEFAULT, tx2);
// nothing is to be sent to CDC
cdc.run();
assertEquals(7, mutations.size());
server_1.getManager().executeUpdate(new DeleteStatement(TableSpace.DEFAULT, "t1", Bytes.from_int(1), null), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
cdc.run();
assertEquals(7, mutations.size());
// close the server...close the ledger, now we can read the last mutation
server_1.close();
cdc.run();
assertEquals(8, mutations.size());
int i = 0;
assertEquals(ChangeDataCapture.MutationType.CREATE_TABLE, mutations.get(i++).getMutationType());
assertEquals(ChangeDataCapture.MutationType.INSERT, mutations.get(i++).getMutationType());
assertEquals(ChangeDataCapture.MutationType.INSERT, mutations.get(i++).getMutationType());
assertEquals(ChangeDataCapture.MutationType.INSERT, mutations.get(i++).getMutationType());
assertEquals(ChangeDataCapture.MutationType.INSERT, mutations.get(i++).getMutationType());
assertEquals(ChangeDataCapture.MutationType.UPDATE, mutations.get(i++).getMutationType());
assertEquals(ChangeDataCapture.MutationType.ALTER_TABLE, mutations.get(i++).getMutationType());
assertEquals(ChangeDataCapture.MutationType.DELETE, mutations.get(i++).getMutationType());
}
}
}
use of herddb.server.ServerConfiguration in project herddb by diennea.
the class SimpleCDCTest method testBasicCaptureDataChangeWithRestart.
@Test
public void testBasicCaptureDataChangeWithRestart() throws Exception {
ServerConfiguration serverconfig_1 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
ClientConfiguration client_configuration = new ClientConfiguration(folder.newFolder().toPath());
client_configuration.set(ClientConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
client_configuration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
client_configuration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
client_configuration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
try (Server server_1 = new Server(serverconfig_1)) {
server_1.start();
server_1.waitForStandaloneBoot();
Table table = Table.builder().name("t1").column("c", ColumnTypes.INTEGER).column("d", ColumnTypes.INTEGER).primaryKey("c").build();
server_1.getManager().executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 1, "d", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 2, "d", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 3, "d", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 4, "d", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
InMemoryTableHistoryStorage tableHistoryStorage = new InMemoryTableHistoryStorage();
LogSequenceNumber currentPosition = LogSequenceNumber.START_OF_TIME;
List<ChangeDataCapture.Mutation> mutations = new ArrayList<>();
currentPosition = performOneCDCStep(client_configuration, server_1, tableHistoryStorage, currentPosition, mutations);
// we are missing the last entry, because it is not confirmed yet on BookKeeper at this point
assertEquals(4, mutations.size());
server_1.getManager().executeUpdate(new UpdateStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 4, "d", 2), null), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
currentPosition = performOneCDCStep(client_configuration, server_1, tableHistoryStorage, currentPosition, mutations);
assertEquals(5, mutations.size());
server_1.getManager().executeStatement(new AlterTableStatement(Arrays.asList(Column.column("e", ColumnTypes.INTEGER)), Collections.emptyList(), Collections.emptyList(), null, table.name, TableSpace.DEFAULT, null, Collections.emptyList(), Collections.emptyList()), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
currentPosition = performOneCDCStep(client_configuration, server_1, tableHistoryStorage, currentPosition, mutations);
assertEquals(6, mutations.size());
server_1.getManager().executeUpdate(new DeleteStatement(TableSpace.DEFAULT, "t1", Bytes.from_int(1), null), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
currentPosition = performOneCDCStep(client_configuration, server_1, tableHistoryStorage, currentPosition, mutations);
assertEquals(7, mutations.size());
// close the server...close the ledger, now we can read the last mutation
server_1.close();
currentPosition = performOneCDCStep(client_configuration, server_1, tableHistoryStorage, currentPosition, mutations);
assertEquals(8, mutations.size());
int i = 0;
assertEquals(ChangeDataCapture.MutationType.CREATE_TABLE, mutations.get(i++).getMutationType());
assertEquals(ChangeDataCapture.MutationType.INSERT, mutations.get(i++).getMutationType());
assertEquals(ChangeDataCapture.MutationType.INSERT, mutations.get(i++).getMutationType());
assertEquals(ChangeDataCapture.MutationType.INSERT, mutations.get(i++).getMutationType());
assertEquals(ChangeDataCapture.MutationType.INSERT, mutations.get(i++).getMutationType());
assertEquals(ChangeDataCapture.MutationType.UPDATE, mutations.get(i++).getMutationType());
assertEquals(ChangeDataCapture.MutationType.ALTER_TABLE, mutations.get(i++).getMutationType());
assertEquals(ChangeDataCapture.MutationType.DELETE, mutations.get(i++).getMutationType());
}
}
Aggregations