use of herddb.server.ServerConfiguration in project herddb by diennea.
the class RetryOnLeaderChangedTest method testKillLeader.
private void testKillLeader(Consumer<HDBConnection> operation) throws Exception {
TestStatsProvider statsProvider = new TestStatsProvider();
ServerConfiguration serverconfig_1 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
ServerConfiguration serverconfig_2 = serverconfig_1.copy().set(ServerConfiguration.PROPERTY_NODEID, "server2").set(ServerConfiguration.PROPERTY_BASEDIR, folder.newFolder().toPath().toAbsolutePath());
try (Server server_1 = new Server(serverconfig_1)) {
server_1.start();
server_1.waitForStandaloneBoot();
try (Server server_2 = new Server(serverconfig_2)) {
server_2.start();
TestUtils.execute(server_1.getManager(), "CREATE TABLESPACE 'ttt','leader:" + server_2.getNodeId() + "','expectedreplicacount:2'", Collections.emptyList());
// wait for server_2 to wake up
for (int i = 0; i < 40; i++) {
TableSpaceManager tableSpaceManager2 = server_2.getManager().getTableSpaceManager("ttt");
if (tableSpaceManager2 != null && tableSpaceManager2.isLeader()) {
break;
}
Thread.sleep(500);
}
assertTrue(server_2.getManager().getTableSpaceManager("ttt") != null && server_2.getManager().getTableSpaceManager("ttt").isLeader());
// wait for server_1 to announce as follower
waitClusterStatus(server_1.getManager(), server_1.getNodeId(), "follower");
ClientConfiguration clientConfiguration = new ClientConfiguration();
clientConfiguration.set(ClientConfiguration.PROPERTY_MODE, ClientConfiguration.PROPERTY_MODE_CLUSTER);
clientConfiguration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
clientConfiguration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
clientConfiguration.set(ClientConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
clientConfiguration.set(ClientConfiguration.PROPERTY_MAX_OPERATION_RETRY_COUNT, 1000);
StatsLogger logger = statsProvider.getStatsLogger("ds");
try (HDBClient client1 = new HDBClient(clientConfiguration, logger)) {
try (HDBConnection connection = client1.openConnection()) {
// create table and insert data
connection.executeUpdate(TableSpace.DEFAULT, "CREATE TABLE ttt.t1(k1 int primary key, n1 int)", TransactionContext.NOTRANSACTION_ID, false, false, Collections.emptyList());
connection.executeUpdate(TableSpace.DEFAULT, "INSERT INTO ttt.t1(k1,n1) values(1,1)", TransactionContext.NOTRANSACTION_ID, false, false, Collections.emptyList());
connection.executeUpdate(TableSpace.DEFAULT, "INSERT INTO ttt.t1(k1,n1) values(2,1)", TransactionContext.NOTRANSACTION_ID, false, false, Collections.emptyList());
connection.executeUpdate(TableSpace.DEFAULT, "INSERT INTO ttt.t1(k1,n1) values(3,1)", TransactionContext.NOTRANSACTION_ID, false, false, Collections.emptyList());
// use client2, so that it opens a connection to current leader
try (ScanResultSet scan = connection.executeScan(TableSpace.DEFAULT, "SELECT * FROM ttt.t1", false, Collections.emptyList(), TransactionContext.NOTRANSACTION_ID, 0, 0, true)) {
assertEquals(3, scan.consume().size());
}
// change leader
switchLeader(server_1.getNodeId(), null, server_1.getManager());
// make server_2 (current leader) die
server_2.close();
// perform operation, it will eventually succeed, because
// the client will automatically wait for the new leader to be up
operation.accept(connection);
}
}
}
}
}
use of herddb.server.ServerConfiguration in project herddb by diennea.
the class JAASKerberosTest method test.
@Test
public void test() throws Exception {
ServerConfiguration serverConfig = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
serverConfig.set(ServerConfiguration.PROPERTY_HOST, "localhost");
try (Server server = new Server(serverConfig)) {
server.start();
try (HDBClient client = new HDBClient(new ClientConfiguration(folder.newFolder().toPath()));
HDBConnection connection = client.openConnection()) {
client.setClientSideMetadataProvider(new StaticClientSideMetadataProvider(server));
long resultCreateTable = connection.executeUpdate(TableSpace.DEFAULT, "CREATE TABLE mytable (id string primary key, n1 long, n2 integer)", 0, false, true, Collections.emptyList()).updateCount;
Assert.assertEquals(1, resultCreateTable);
Assert.assertEquals(1, connection.executeUpdate(TableSpace.DEFAULT, "INSERT INTO mytable (id,n1,n2) values(?,?,?)", 0, false, true, Arrays.asList("test_0", 1, 2)).updateCount);
Map<RawString, Object> newValue = connection.executeUpdate(TableSpace.DEFAULT, "UPDATE mytable set n1= n1+1 where id=?", 0, true, true, Arrays.asList("test_0")).newvalue;
assertEquals(Long.valueOf(2), newValue.get(RawString.of("n1")));
}
} finally {
System.clearProperty("java.security.auth.login.config");
}
}
use of herddb.server.ServerConfiguration in project herddb by diennea.
the class MultipleConcurrentUpdatesTest method performTest.
private void performTest(boolean useTransactions, long checkPointPeriod, boolean withIndexes) throws Exception {
Path baseDir = folder.newFolder().toPath();
ServerConfiguration serverConfiguration = newServerConfigurationWithAutoPort(baseDir);
serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_LOGICAL_PAGE_SIZE, 10 * 1024);
serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_DATA_MEMORY, 1024 * 1024 / 4);
serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_PK_MEMORY, 1024 * 1024);
serverConfiguration.set(ServerConfiguration.PROPERTY_CHECKPOINT_PERIOD, checkPointPeriod);
serverConfiguration.set(ServerConfiguration.PROPERTY_DATADIR, folder.newFolder().getAbsolutePath());
serverConfiguration.set(ServerConfiguration.PROPERTY_LOGDIR, folder.newFolder().getAbsolutePath());
ConcurrentHashMap<String, Long> expectedValue = new ConcurrentHashMap<>();
try (Server server = new Server(serverConfiguration)) {
server.start();
server.waitForStandaloneBoot();
ClientConfiguration clientConfiguration = new ClientConfiguration(folder.newFolder().toPath());
try (HDBClient client = new HDBClient(clientConfiguration);
HDBConnection connection = client.openConnection()) {
client.setClientSideMetadataProvider(new StaticClientSideMetadataProvider(server));
long resultCreateTable = connection.executeUpdate(TableSpace.DEFAULT, "CREATE TABLE mytable (id string primary key, n1 long, n2 integer)", 0, false, true, Collections.emptyList()).updateCount;
Assert.assertEquals(1, resultCreateTable);
if (withIndexes) {
long resultCreateIndex = connection.executeUpdate(TableSpace.DEFAULT, "CREATE INDEX theindex ON mytable (n1 long)", 0, false, true, Collections.emptyList()).updateCount;
Assert.assertEquals(1, resultCreateIndex);
}
long tx = connection.beginTransaction(TableSpace.DEFAULT);
for (int i = 0; i < TABLESIZE; i++) {
connection.executeUpdate(TableSpace.DEFAULT, "INSERT INTO mytable (id,n1,n2) values(?,?,?)", tx, false, true, Arrays.asList("test_" + i, 1, 2));
expectedValue.put("test_" + i, 1L);
}
connection.commitTransaction(TableSpace.DEFAULT, tx);
ExecutorService threadPool = Executors.newFixedThreadPool(THREADPOLSIZE);
try {
List<Future> futures = new ArrayList<>();
AtomicLong updates = new AtomicLong();
AtomicLong skipped = new AtomicLong();
AtomicLong gets = new AtomicLong();
for (int i = 0; i < TABLESIZE * MULTIPLIER; i++) {
futures.add(threadPool.submit(new Runnable() {
@Override
public void run() {
try {
boolean update = ThreadLocalRandom.current().nextBoolean();
int k = ThreadLocalRandom.current().nextInt(TABLESIZE);
long value = ThreadLocalRandom.current().nextInt(TABLESIZE);
long transactionId;
String key = "test_" + k;
Long actual = expectedValue.remove(key);
if (actual == null) {
// another thread working on this entry, skip
skipped.incrementAndGet();
return;
}
if (update) {
updates.incrementAndGet();
DMLResult updateResult = connection.executeUpdate(TableSpace.DEFAULT, "UPDATE mytable set n1=? WHERE id=?", useTransactions ? TransactionContext.AUTOTRANSACTION_ID : TransactionContext.NOTRANSACTION_ID, false, true, Arrays.asList(value, "test_" + k));
long count = updateResult.updateCount;
transactionId = updateResult.transactionId;
if (count <= 0) {
throw new RuntimeException("not updated ?");
}
} else {
gets.incrementAndGet();
GetResult res = connection.executeGet(TableSpace.DEFAULT, "SELECT * FROM mytable where id=?", useTransactions ? TransactionContext.AUTOTRANSACTION_ID : TransactionContext.NOTRANSACTION_ID, true, Arrays.asList("test_" + k));
if (res.data == null) {
throw new RuntimeException("not found?");
}
if (!res.data.get(N1).equals(actual)) {
throw new RuntimeException("unspected value " + res.data + ", expected: " + actual);
}
transactionId = res.transactionId;
// value did not change actually
value = actual;
}
if (useTransactions) {
if (transactionId <= 0) {
throw new RuntimeException("no transaction ?");
}
connection.commitTransaction(TableSpace.DEFAULT, transactionId);
}
expectedValue.put(key, value);
} catch (Exception err) {
throw new RuntimeException(err);
}
}
}));
}
for (Future f : futures) {
f.get();
}
System.out.println("stats::updates:" + updates);
System.out.println("stats::get:" + gets);
System.out.println("stats::skipped:" + skipped);
assertTrue(updates.get() > 0);
assertTrue(gets.get() > 0);
List<String> erroredKeys = new ArrayList<>();
for (Map.Entry<String, Long> entry : expectedValue.entrySet()) {
GetResult res = connection.executeGet(TableSpace.DEFAULT, "SELECT n1 FROM mytable where id=?", TransactionContext.NOTRANSACTION_ID, true, Arrays.asList(entry.getKey()));
assertNotNull(res.data);
if (!entry.getValue().equals(res.data.get(N1))) {
if (!entry.getValue().equals(res.data.get(N1))) {
System.out.println("expected value " + res.data.get(N1) + ", but got " + Long.valueOf(entry.getValue()) + " for key " + entry.getKey());
erroredKeys.add(entry.getKey());
}
}
}
assertTrue(erroredKeys.isEmpty());
TableManagerStats stats = server.getManager().getTableSpaceManager(TableSpace.DEFAULT).getTableManager("mytable").getStats();
System.out.println("stats::tablesize:" + stats.getTablesize());
System.out.println("stats::dirty records:" + stats.getDirtyrecords());
System.out.println("stats::unload count:" + stats.getUnloadedPagesCount());
System.out.println("stats::load count:" + stats.getLoadedPagesCount());
System.out.println("stats::buffers used mem:" + stats.getBuffersUsedMemory());
assertTrue(stats.getUnloadedPagesCount() > 0);
assertEquals(TABLESIZE, stats.getTablesize());
} finally {
threadPool.shutdown();
threadPool.awaitTermination(1, TimeUnit.MINUTES);
}
}
}
// restart and recovery
try (Server server = new Server(serverConfiguration)) {
server.start();
server.waitForTableSpaceBoot(TableSpace.DEFAULT, 300000, true);
ClientConfiguration clientConfiguration = new ClientConfiguration(folder.newFolder().toPath());
try (HDBClient client = new HDBClient(clientConfiguration);
HDBConnection connection = client.openConnection()) {
client.setClientSideMetadataProvider(new StaticClientSideMetadataProvider(server));
for (Map.Entry<String, Long> entry : expectedValue.entrySet()) {
GetResult res = connection.executeGet(TableSpace.DEFAULT, "SELECT n1 FROM mytable where id=?", TransactionContext.NOTRANSACTION_ID, true, Arrays.asList(entry.getKey()));
assertNotNull(res.data);
assertEquals(entry.getValue(), res.data.get(N1));
}
}
}
}
use of herddb.server.ServerConfiguration in project herddb by diennea.
the class DirectMultipleConcurrentUpdatesSuite method performTest.
protected void performTest(boolean useTransactions, long checkPointPeriod, boolean withIndexes, boolean uniqueIndexes) throws Exception {
Path baseDir = folder.newFolder().toPath();
ServerConfiguration serverConfiguration = newServerConfigurationWithAutoPort(baseDir);
serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_LOGICAL_PAGE_SIZE, 10 * 1024);
serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_DATA_MEMORY, 1024 * 1024 / 4);
serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_PK_MEMORY, 1024 * 1024);
serverConfiguration.set(ServerConfiguration.PROPERTY_CHECKPOINT_PERIOD, checkPointPeriod);
serverConfiguration.set(ServerConfiguration.PROPERTY_DATADIR, folder.newFolder().getAbsolutePath());
serverConfiguration.set(ServerConfiguration.PROPERTY_LOGDIR, folder.newFolder().getAbsolutePath());
ConcurrentHashMap<String, Long> expectedValue = new ConcurrentHashMap<>();
try (Server server = new Server(serverConfiguration)) {
server.start();
server.waitForStandaloneBoot();
DBManager manager = server.getManager();
execute(manager, "CREATE TABLE mytable (id string primary key, n1 long, n2 integer)", Collections.emptyList());
if (withIndexes) {
if (uniqueIndexes) {
// use n1 + id in order to not have collisions and lock timeouts
execute(manager, "CREATE UNIQUE INDEX theindex ON mytable (n1, id)", Collections.emptyList());
} else {
execute(manager, "CREATE INDEX theindex ON mytable (n1)", Collections.emptyList());
}
}
long tx = TestUtils.beginTransaction(manager, TableSpace.DEFAULT);
for (int i = 0; i < TABLESIZE; i++) {
TestUtils.executeUpdate(manager, "INSERT INTO mytable (id,n1,n2) values(?,?,?)", Arrays.asList("test_" + i, 1, 2), new TransactionContext(tx));
expectedValue.put("test_" + i, 1L);
}
TestUtils.commitTransaction(manager, TableSpace.DEFAULT, tx);
ExecutorService threadPool = Executors.newFixedThreadPool(THREADPOLSIZE);
try {
List<Future> futures = new ArrayList<>();
AtomicLong updates = new AtomicLong();
AtomicLong skipped = new AtomicLong();
AtomicLong gets = new AtomicLong();
for (int i = 0; i < TABLESIZE * MULTIPLIER; i++) {
futures.add(threadPool.submit(new Runnable() {
@Override
public void run() {
try {
boolean update = ThreadLocalRandom.current().nextBoolean();
int k = ThreadLocalRandom.current().nextInt(TABLESIZE);
long value = ThreadLocalRandom.current().nextInt(TABLESIZE);
long transactionId;
String key = "test_" + k;
Long actual = expectedValue.remove(key);
if (actual == null) {
// another thread working on this entry, skip
skipped.incrementAndGet();
return;
}
if (update) {
updates.incrementAndGet();
DMLStatementExecutionResult updateResult = TestUtils.executeUpdate(manager, "UPDATE mytable set n1=? WHERE id=?", Arrays.asList(value, "test_" + k), new TransactionContext(useTransactions ? TransactionContext.AUTOTRANSACTION_ID : TransactionContext.NOTRANSACTION_ID));
long count = updateResult.getUpdateCount();
transactionId = updateResult.transactionId;
if (count <= 0) {
throw new RuntimeException("not updated ?");
}
} else {
gets.incrementAndGet();
DataScanner res = TestUtils.scanKeepReadLocks(manager, "SELECT * FROM mytable where id=?", Arrays.asList("test_" + k), new TransactionContext(useTransactions ? TransactionContext.AUTOTRANSACTION_ID : TransactionContext.NOTRANSACTION_ID));
if (!res.hasNext()) {
throw new RuntimeException("not found?");
}
res.close();
transactionId = res.getTransactionId();
// value did not change actually
value = actual;
}
if (useTransactions) {
if (transactionId <= 0) {
throw new RuntimeException("no transaction ?");
}
commitTransaction(manager, TableSpace.DEFAULT, transactionId);
}
expectedValue.put(key, value);
} catch (Exception err) {
throw new RuntimeException(err);
}
}
}));
}
for (Future f : futures) {
f.get();
}
System.out.println("stats::updates:" + updates);
System.out.println("stats::get:" + gets);
assertTrue(updates.get() > 0);
assertTrue(gets.get() > 0);
List<String> erroredKeys = new ArrayList<>();
for (Map.Entry<String, Long> entry : expectedValue.entrySet()) {
List<DataAccessor> records;
DataAccessor data;
try (DataScanner res = scan(manager, "SELECT n1 FROM mytable where id=?", Arrays.asList(entry.getKey()))) {
records = res.consume();
data = records.get(0);
}
assertEquals(1, records.size());
if (!entry.getValue().equals(data.get("n1"))) {
System.out.println("expected value " + data.get("n1") + ", but got " + Long.valueOf(entry.getValue()) + " for key " + entry.getKey());
erroredKeys.add(entry.getKey());
}
}
assertTrue(erroredKeys.isEmpty());
TableManagerStats stats = server.getManager().getTableSpaceManager(TableSpace.DEFAULT).getTableManager("mytable").getStats();
System.out.println("stats::tablesize:" + stats.getTablesize());
System.out.println("stats::dirty records:" + stats.getDirtyrecords());
System.out.println("stats::unload count:" + stats.getUnloadedPagesCount());
System.out.println("stats::load count:" + stats.getLoadedPagesCount());
System.out.println("stats::buffers used mem:" + stats.getBuffersUsedMemory());
// assertTrue(stats.getUnloadedPagesCount() > 0);
assertEquals(TABLESIZE, stats.getTablesize());
} finally {
threadPool.shutdown();
threadPool.awaitTermination(1, TimeUnit.MINUTES);
}
}
// restart and recovery
try (Server server = new Server(serverConfiguration)) {
server.start();
server.waitForTableSpaceBoot(TableSpace.DEFAULT, 300000, true);
DBManager manager = server.getManager();
List<String> erroredKeys = new ArrayList<>();
for (Map.Entry<String, Long> entry : expectedValue.entrySet()) {
List<DataAccessor> records;
DataAccessor data;
try (DataScanner res = scan(manager, "SELECT n1 FROM mytable where id=?", Arrays.asList(entry.getKey()))) {
records = res.consume();
data = records.get(0);
}
assertEquals(1, records.size());
if (!entry.getValue().equals(data.get("n1"))) {
System.out.println("expected value " + data.get("n1") + ", but got " + Long.valueOf(entry.getValue()) + " for key " + entry.getKey());
erroredKeys.add(entry.getKey());
}
}
assertTrue(erroredKeys.isEmpty());
}
}
use of herddb.server.ServerConfiguration in project herddb by diennea.
the class MultiDMLOnSameRecordTest method testWithIndexSeek.
@Test
public void testWithIndexSeek() throws Exception {
Path baseDir = folder.newFolder().toPath();
ServerConfiguration serverConfiguration = newServerConfigurationWithAutoPort(baseDir);
serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_LOGICAL_PAGE_SIZE, 10 * 1024);
serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_DATA_MEMORY, 1024 * 1024 / 4);
serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_PK_MEMORY, 1024 * 1024);
serverConfiguration.set(ServerConfiguration.PROPERTY_CHECKPOINT_PERIOD, 0);
serverConfiguration.set(ServerConfiguration.PROPERTY_DATADIR, folder.newFolder().getAbsolutePath());
serverConfiguration.set(ServerConfiguration.PROPERTY_LOGDIR, folder.newFolder().getAbsolutePath());
try (Server server = new Server(serverConfiguration)) {
server.start();
server.waitForStandaloneBoot();
DBManager manager = server.getManager();
execute(manager, "CREATE TABLE mytable (id string primary key, n1 long, k2 string)", Collections.emptyList());
execute(manager, "CREATE INDEX tt ON mytable (k2)", Collections.emptyList());
ExecutorService threadPool = Executors.newFixedThreadPool(THREADPOLSIZE);
try {
List<Future> futures = new ArrayList<>();
AtomicLong updates = new AtomicLong();
AtomicLong deletes = new AtomicLong();
AtomicLong inserts = new AtomicLong();
AtomicLong duplicatePkErrors = new AtomicLong();
for (int i = 0; i < TESTSIZE; i++) {
futures.add(threadPool.submit(new Runnable() {
@Override
public void run() {
try {
int k = ThreadLocalRandom.current().nextInt(10);
long value = ThreadLocalRandom.current().nextInt(100) + 1000;
String key = "test_" + k;
boolean update = ThreadLocalRandom.current().nextBoolean();
boolean insert = ThreadLocalRandom.current().nextBoolean();
boolean delete = ThreadLocalRandom.current().nextBoolean();
if (update) {
updates.incrementAndGet();
execute(manager, "UPDATE mytable set n1=? WHERE k2=?", Arrays.asList(value, key));
} else if (insert) {
inserts.incrementAndGet();
try {
execute(manager, "INSERT INTO mytable(n1, id, k2) values(?,?,?)", Arrays.asList(value, key, key));
} catch (DuplicatePrimaryKeyException ok) {
duplicatePkErrors.incrementAndGet();
}
} else if (delete) {
deletes.incrementAndGet();
execute(manager, "DELETE FROM mytable WHERE k2=?", Arrays.asList(key));
}
} catch (Exception err) {
throw new RuntimeException(err);
}
}
}));
}
for (Future f : futures) {
f.get();
}
System.out.println("stats::updates:" + updates);
System.out.println("stats::deletes:" + deletes);
System.out.println("stats::inserts:" + inserts);
System.out.println("stats::duplicatePkErrors:" + duplicatePkErrors);
TableManagerStats stats = server.getManager().getTableSpaceManager(TableSpace.DEFAULT).getTableManager("mytable").getStats();
System.out.println("stats::tablesize:" + stats.getTablesize());
System.out.println("stats::dirty records:" + stats.getDirtyrecords());
System.out.println("stats::unload count:" + stats.getUnloadedPagesCount());
System.out.println("stats::load count:" + stats.getLoadedPagesCount());
System.out.println("stats::buffers used mem:" + stats.getBuffersUsedMemory());
} finally {
threadPool.shutdown();
threadPool.awaitTermination(1, TimeUnit.MINUTES);
}
}
// restart and recovery
try (Server server = new Server(serverConfiguration)) {
server.start();
server.waitForTableSpaceBoot(TableSpace.DEFAULT, 300000, true);
}
}
Aggregations