use of herddb.core.DBManager in project herddb by diennea.
the class SecondaryNonUniqueIndexAccessSuite method createIndexOnTableWithData.
@Test
public void createIndexOnTableWithData() throws Exception {
String nodeId = "localhost";
try (DBManager manager = new DBManager("localhost", new MemoryMetadataStorageManager(), new MemoryDataStorageManager(), new MemoryCommitLogManager(), null, null)) {
manager.start();
CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
manager.waitForTablespace("tblspace1", 10000);
Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.STRING).column("name", ColumnTypes.STRING).primaryKey("id").build();
CreateTableStatement st2 = new CreateTableStatement(table);
manager.executeStatement(st2, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
Index index = Index.builder().onTable(table).type(indexType).column("name", ColumnTypes.STRING).build();
TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('a','n1')", Collections.emptyList());
TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('b','n1')", Collections.emptyList());
TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('c','n1')", Collections.emptyList());
TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('d','n2')", Collections.emptyList());
TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('e','n2')", Collections.emptyList());
// create index, it will be built using existing data
CreateIndexStatement st3 = new CreateIndexStatement(index);
manager.executeStatement(st3, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE name='n1'", Collections.emptyList(), true, true, false, -1);
ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexSeek);
try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
assertEquals(3, scan1.consume().size());
}
}
}
use of herddb.core.DBManager in project herddb by diennea.
the class DirectMultipleConcurrentUpdatesSuite method performTest.
protected void performTest(boolean useTransactions, long checkPointPeriod, boolean withIndexes, boolean uniqueIndexes) throws Exception {
Path baseDir = folder.newFolder().toPath();
ServerConfiguration serverConfiguration = newServerConfigurationWithAutoPort(baseDir);
serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_LOGICAL_PAGE_SIZE, 10 * 1024);
serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_DATA_MEMORY, 1024 * 1024 / 4);
serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_PK_MEMORY, 1024 * 1024);
serverConfiguration.set(ServerConfiguration.PROPERTY_CHECKPOINT_PERIOD, checkPointPeriod);
serverConfiguration.set(ServerConfiguration.PROPERTY_DATADIR, folder.newFolder().getAbsolutePath());
serverConfiguration.set(ServerConfiguration.PROPERTY_LOGDIR, folder.newFolder().getAbsolutePath());
ConcurrentHashMap<String, Long> expectedValue = new ConcurrentHashMap<>();
try (Server server = new Server(serverConfiguration)) {
server.start();
server.waitForStandaloneBoot();
DBManager manager = server.getManager();
execute(manager, "CREATE TABLE mytable (id string primary key, n1 long, n2 integer)", Collections.emptyList());
if (withIndexes) {
if (uniqueIndexes) {
// use n1 + id in order to not have collisions and lock timeouts
execute(manager, "CREATE UNIQUE INDEX theindex ON mytable (n1, id)", Collections.emptyList());
} else {
execute(manager, "CREATE INDEX theindex ON mytable (n1)", Collections.emptyList());
}
}
long tx = TestUtils.beginTransaction(manager, TableSpace.DEFAULT);
for (int i = 0; i < TABLESIZE; i++) {
TestUtils.executeUpdate(manager, "INSERT INTO mytable (id,n1,n2) values(?,?,?)", Arrays.asList("test_" + i, 1, 2), new TransactionContext(tx));
expectedValue.put("test_" + i, 1L);
}
TestUtils.commitTransaction(manager, TableSpace.DEFAULT, tx);
ExecutorService threadPool = Executors.newFixedThreadPool(THREADPOLSIZE);
try {
List<Future> futures = new ArrayList<>();
AtomicLong updates = new AtomicLong();
AtomicLong skipped = new AtomicLong();
AtomicLong gets = new AtomicLong();
for (int i = 0; i < TABLESIZE * MULTIPLIER; i++) {
futures.add(threadPool.submit(new Runnable() {
@Override
public void run() {
try {
boolean update = ThreadLocalRandom.current().nextBoolean();
int k = ThreadLocalRandom.current().nextInt(TABLESIZE);
long value = ThreadLocalRandom.current().nextInt(TABLESIZE);
long transactionId;
String key = "test_" + k;
Long actual = expectedValue.remove(key);
if (actual == null) {
// another thread working on this entry, skip
skipped.incrementAndGet();
return;
}
if (update) {
updates.incrementAndGet();
DMLStatementExecutionResult updateResult = TestUtils.executeUpdate(manager, "UPDATE mytable set n1=? WHERE id=?", Arrays.asList(value, "test_" + k), new TransactionContext(useTransactions ? TransactionContext.AUTOTRANSACTION_ID : TransactionContext.NOTRANSACTION_ID));
long count = updateResult.getUpdateCount();
transactionId = updateResult.transactionId;
if (count <= 0) {
throw new RuntimeException("not updated ?");
}
} else {
gets.incrementAndGet();
DataScanner res = TestUtils.scanKeepReadLocks(manager, "SELECT * FROM mytable where id=?", Arrays.asList("test_" + k), new TransactionContext(useTransactions ? TransactionContext.AUTOTRANSACTION_ID : TransactionContext.NOTRANSACTION_ID));
if (!res.hasNext()) {
throw new RuntimeException("not found?");
}
res.close();
transactionId = res.getTransactionId();
// value did not change actually
value = actual;
}
if (useTransactions) {
if (transactionId <= 0) {
throw new RuntimeException("no transaction ?");
}
commitTransaction(manager, TableSpace.DEFAULT, transactionId);
}
expectedValue.put(key, value);
} catch (Exception err) {
throw new RuntimeException(err);
}
}
}));
}
for (Future f : futures) {
f.get();
}
System.out.println("stats::updates:" + updates);
System.out.println("stats::get:" + gets);
assertTrue(updates.get() > 0);
assertTrue(gets.get() > 0);
List<String> erroredKeys = new ArrayList<>();
for (Map.Entry<String, Long> entry : expectedValue.entrySet()) {
List<DataAccessor> records;
DataAccessor data;
try (DataScanner res = scan(manager, "SELECT n1 FROM mytable where id=?", Arrays.asList(entry.getKey()))) {
records = res.consume();
data = records.get(0);
}
assertEquals(1, records.size());
if (!entry.getValue().equals(data.get("n1"))) {
System.out.println("expected value " + data.get("n1") + ", but got " + Long.valueOf(entry.getValue()) + " for key " + entry.getKey());
erroredKeys.add(entry.getKey());
}
}
assertTrue(erroredKeys.isEmpty());
TableManagerStats stats = server.getManager().getTableSpaceManager(TableSpace.DEFAULT).getTableManager("mytable").getStats();
System.out.println("stats::tablesize:" + stats.getTablesize());
System.out.println("stats::dirty records:" + stats.getDirtyrecords());
System.out.println("stats::unload count:" + stats.getUnloadedPagesCount());
System.out.println("stats::load count:" + stats.getLoadedPagesCount());
System.out.println("stats::buffers used mem:" + stats.getBuffersUsedMemory());
// assertTrue(stats.getUnloadedPagesCount() > 0);
assertEquals(TABLESIZE, stats.getTablesize());
} finally {
threadPool.shutdown();
threadPool.awaitTermination(1, TimeUnit.MINUTES);
}
}
// restart and recovery
try (Server server = new Server(serverConfiguration)) {
server.start();
server.waitForTableSpaceBoot(TableSpace.DEFAULT, 300000, true);
DBManager manager = server.getManager();
List<String> erroredKeys = new ArrayList<>();
for (Map.Entry<String, Long> entry : expectedValue.entrySet()) {
List<DataAccessor> records;
DataAccessor data;
try (DataScanner res = scan(manager, "SELECT n1 FROM mytable where id=?", Arrays.asList(entry.getKey()))) {
records = res.consume();
data = records.get(0);
}
assertEquals(1, records.size());
if (!entry.getValue().equals(data.get("n1"))) {
System.out.println("expected value " + data.get("n1") + ", but got " + Long.valueOf(entry.getValue()) + " for key " + entry.getKey());
erroredKeys.add(entry.getKey());
}
}
assertTrue(erroredKeys.isEmpty());
}
}
use of herddb.core.DBManager in project herddb by diennea.
the class MultiDMLOnSameRecordTest method testWithIndexSeek.
@Test
public void testWithIndexSeek() throws Exception {
Path baseDir = folder.newFolder().toPath();
ServerConfiguration serverConfiguration = newServerConfigurationWithAutoPort(baseDir);
serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_LOGICAL_PAGE_SIZE, 10 * 1024);
serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_DATA_MEMORY, 1024 * 1024 / 4);
serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_PK_MEMORY, 1024 * 1024);
serverConfiguration.set(ServerConfiguration.PROPERTY_CHECKPOINT_PERIOD, 0);
serverConfiguration.set(ServerConfiguration.PROPERTY_DATADIR, folder.newFolder().getAbsolutePath());
serverConfiguration.set(ServerConfiguration.PROPERTY_LOGDIR, folder.newFolder().getAbsolutePath());
try (Server server = new Server(serverConfiguration)) {
server.start();
server.waitForStandaloneBoot();
DBManager manager = server.getManager();
execute(manager, "CREATE TABLE mytable (id string primary key, n1 long, k2 string)", Collections.emptyList());
execute(manager, "CREATE INDEX tt ON mytable (k2)", Collections.emptyList());
ExecutorService threadPool = Executors.newFixedThreadPool(THREADPOLSIZE);
try {
List<Future> futures = new ArrayList<>();
AtomicLong updates = new AtomicLong();
AtomicLong deletes = new AtomicLong();
AtomicLong inserts = new AtomicLong();
AtomicLong duplicatePkErrors = new AtomicLong();
for (int i = 0; i < TESTSIZE; i++) {
futures.add(threadPool.submit(new Runnable() {
@Override
public void run() {
try {
int k = ThreadLocalRandom.current().nextInt(10);
long value = ThreadLocalRandom.current().nextInt(100) + 1000;
String key = "test_" + k;
boolean update = ThreadLocalRandom.current().nextBoolean();
boolean insert = ThreadLocalRandom.current().nextBoolean();
boolean delete = ThreadLocalRandom.current().nextBoolean();
if (update) {
updates.incrementAndGet();
execute(manager, "UPDATE mytable set n1=? WHERE k2=?", Arrays.asList(value, key));
} else if (insert) {
inserts.incrementAndGet();
try {
execute(manager, "INSERT INTO mytable(n1, id, k2) values(?,?,?)", Arrays.asList(value, key, key));
} catch (DuplicatePrimaryKeyException ok) {
duplicatePkErrors.incrementAndGet();
}
} else if (delete) {
deletes.incrementAndGet();
execute(manager, "DELETE FROM mytable WHERE k2=?", Arrays.asList(key));
}
} catch (Exception err) {
throw new RuntimeException(err);
}
}
}));
}
for (Future f : futures) {
f.get();
}
System.out.println("stats::updates:" + updates);
System.out.println("stats::deletes:" + deletes);
System.out.println("stats::inserts:" + inserts);
System.out.println("stats::duplicatePkErrors:" + duplicatePkErrors);
TableManagerStats stats = server.getManager().getTableSpaceManager(TableSpace.DEFAULT).getTableManager("mytable").getStats();
System.out.println("stats::tablesize:" + stats.getTablesize());
System.out.println("stats::dirty records:" + stats.getDirtyrecords());
System.out.println("stats::unload count:" + stats.getUnloadedPagesCount());
System.out.println("stats::load count:" + stats.getLoadedPagesCount());
System.out.println("stats::buffers used mem:" + stats.getBuffersUsedMemory());
} finally {
threadPool.shutdown();
threadPool.awaitTermination(1, TimeUnit.MINUTES);
}
}
// restart and recovery
try (Server server = new Server(serverConfiguration)) {
server.start();
server.waitForTableSpaceBoot(TableSpace.DEFAULT, 300000, true);
}
}
use of herddb.core.DBManager in project herddb by diennea.
the class MultiDMLOnSameRecordTest method testWithPrimaryKeyIndexSeek.
@Test
public void testWithPrimaryKeyIndexSeek() throws Exception {
Path baseDir = folder.newFolder().toPath();
ServerConfiguration serverConfiguration = newServerConfigurationWithAutoPort(baseDir);
serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_LOGICAL_PAGE_SIZE, 10 * 1024);
serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_DATA_MEMORY, 1024 * 1024 / 4);
serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_PK_MEMORY, 1024 * 1024);
serverConfiguration.set(ServerConfiguration.PROPERTY_CHECKPOINT_PERIOD, 0);
serverConfiguration.set(ServerConfiguration.PROPERTY_DATADIR, folder.newFolder().getAbsolutePath());
serverConfiguration.set(ServerConfiguration.PROPERTY_LOGDIR, folder.newFolder().getAbsolutePath());
try (Server server = new Server(serverConfiguration)) {
server.start();
server.waitForStandaloneBoot();
DBManager manager = server.getManager();
execute(manager, "CREATE TABLE mytable (id string primary key, n1 long, n2 integer)", Collections.emptyList());
ExecutorService threadPool = Executors.newFixedThreadPool(THREADPOLSIZE);
try {
List<Future> futures = new ArrayList<>();
AtomicLong updates = new AtomicLong();
AtomicLong deletes = new AtomicLong();
AtomicLong inserts = new AtomicLong();
AtomicLong duplicatePkErrors = new AtomicLong();
for (int i = 0; i < TESTSIZE; i++) {
futures.add(threadPool.submit(new Runnable() {
@Override
public void run() {
try {
int k = ThreadLocalRandom.current().nextInt(10);
long value = ThreadLocalRandom.current().nextInt(100) + 1000;
String key = "test_" + k;
boolean update = ThreadLocalRandom.current().nextBoolean();
boolean insert = ThreadLocalRandom.current().nextBoolean();
boolean delete = ThreadLocalRandom.current().nextBoolean();
if (update) {
System.err.println("do " + Thread.currentThread() + " update on " + Bytes.from_string(key));
updates.incrementAndGet();
execute(manager, "UPDATE mytable set n1=? WHERE id=?", Arrays.asList(value, key));
} else if (insert) {
System.err.println("do " + Thread.currentThread() + " insert on " + Bytes.from_string(key));
inserts.incrementAndGet();
try {
execute(manager, "INSERT INTO mytable(n1, id) values(?,?)", Arrays.asList(value, key));
} catch (DuplicatePrimaryKeyException ok) {
duplicatePkErrors.incrementAndGet();
}
} else if (delete) {
System.err.println("do " + Thread.currentThread() + " delete on " + Bytes.from_string(key));
deletes.incrementAndGet();
execute(manager, "DELETE FROM mytable WHERE id=?", Arrays.asList(key));
}
} catch (Exception err) {
throw new RuntimeException(err);
}
}
}));
}
for (Future f : futures) {
f.get();
}
System.out.println("stats::updates:" + updates);
System.out.println("stats::deletes:" + deletes);
System.out.println("stats::inserts:" + inserts);
System.out.println("stats::duplicatePkErrors:" + duplicatePkErrors);
TableManagerStats stats = server.getManager().getTableSpaceManager(TableSpace.DEFAULT).getTableManager("mytable").getStats();
System.out.println("stats::tablesize:" + stats.getTablesize());
System.out.println("stats::dirty records:" + stats.getDirtyrecords());
System.out.println("stats::unload count:" + stats.getUnloadedPagesCount());
System.out.println("stats::load count:" + stats.getLoadedPagesCount());
System.out.println("stats::buffers used mem:" + stats.getBuffersUsedMemory());
} finally {
threadPool.shutdown();
threadPool.awaitTermination(1, TimeUnit.MINUTES);
}
}
// restart and recovery
try (Server server = new Server(serverConfiguration)) {
server.start();
server.waitForTableSpaceBoot(TableSpace.DEFAULT, 300000, true);
}
}
use of herddb.core.DBManager in project herddb by diennea.
the class IndexCreationTest method caseSensitivity.
@Test
public void caseSensitivity() throws Exception {
String nodeId = "localhost";
try (DBManager manager = new DBManager("localhost", new MemoryMetadataStorageManager(), new MemoryDataStorageManager(), new MemoryCommitLogManager(), null, null)) {
manager.start();
CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tbl1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
manager.waitForTablespace("tbl1", 10000);
// Table create uppercase, index create uppercase
{
execute(manager, "CREATE TABLE tbl1.TABLE_1 (TABLE_ID BIGINT NOT NULL, FIELD INT NOT NULL, PRIMARY KEY (TABLE_ID))", Collections.emptyList());
execute(manager, "CREATE INDEX TABLE_1_INDEX ON tbl1.TABLE_1(TABLE_ID,FIELD)", Collections.emptyList());
}
// Table create lowercase, index create lowercase
{
execute(manager, "CREATE TABLE tbl1.table_2 (TABLE_ID BIGINT NOT NULL, FIELD INT NOT NULL, PRIMARY KEY (TABLE_ID))", Collections.emptyList());
execute(manager, "CREATE INDEX table_2_index ON tbl1.table_2(TABLE_ID,FIELD)", Collections.emptyList());
}
// Table create uppercase, index create lowercase
{
execute(manager, "CREATE TABLE tbl1.TABLE_3 (TABLE_ID BIGINT NOT NULL, FIELD INT NOT NULL, PRIMARY KEY (TABLE_ID))", Collections.emptyList());
execute(manager, "CREATE INDEX TABLE_3_INDEX ON tbl1.table_3(TABLE_ID,FIELD)", Collections.emptyList());
}
// Table create lowercase, index create uppercase
{
execute(manager, "CREATE TABLE tbl1.table_4 (TABLE_ID BIGINT NOT NULL, FIELD INT NOT NULL, PRIMARY KEY (TABLE_ID))", Collections.emptyList());
execute(manager, "CREATE INDEX table_4_index ON tbl1.TABLE_4(TABLE_ID,FIELD)", Collections.emptyList());
}
// create index and table with one single statement
execute(manager, "CREATE TABLE tbl1.table_5 (TABLE_ID BIGINT NOT NULL," + "FIELD STRING NOT NULL," + // single unique column
"SECONDFIELD TIMESTAMP UNIQUE," + "PRIMARY KEY (TABLE_ID)," + "INDEX t5index(field)," + "KEY t5index2(field,secondfield)," + // we don't support "UNIQUE INDEX", but "UNIQUE KEY"
"UNIQUE KEY index5u(field,table_id)" + ")", Collections.emptyList());
Map<String, AbstractIndexManager> indexesOnTable = manager.getTableSpaceManager("tbl1").getIndexesOnTable("table_5");
assertFalse(indexesOnTable.get("t5index").isUnique());
assertEquals(ColumnTypes.NOTNULL_STRING, indexesOnTable.get("t5index").getIndex().columnByName.get("field").type);
assertFalse(indexesOnTable.get("t5index2").isUnique());
assertEquals(ColumnTypes.NOTNULL_STRING, indexesOnTable.get("t5index2").getIndex().columnByName.get("field").type);
assertEquals(ColumnTypes.TIMESTAMP, indexesOnTable.get("t5index2").getIndex().columnByName.get("secondfield").type);
assertTrue(indexesOnTable.get("index5u").isUnique());
assertEquals(ColumnTypes.NOTNULL_STRING, indexesOnTable.get("index5u").getIndex().columnByName.get("field").type);
assertEquals(ColumnTypes.NOTNULL_LONG, indexesOnTable.get("index5u").getIndex().columnByName.get("table_id").type);
assertTrue(indexesOnTable.get("table_5_unique_secondfield").isUnique());
assertEquals(ColumnTypes.TIMESTAMP, indexesOnTable.get("table_5_unique_secondfield").getIndex().columnByName.get("secondfield").type);
assertEquals("Missing some index " + indexesOnTable.keySet(), 4, indexesOnTable.size());
// create index and table with one single statement, in transaction
StatementExecutionResult execute = execute(manager, "CREATE TABLE tbl1.table_6 (TABLE_ID BIGINT NOT NULL," + "FIELD STRING NOT NULL," + // single unique column
"SECONDFIELD TIMESTAMP UNIQUE," + "PRIMARY KEY (TABLE_ID)," + "INDEX t6index(field)," + "KEY t6index2(field,secondfield)," + // we don't support "UNIQUE INDEX", but "UNIQUE KEY"
"UNIQUE KEY index6u(field,table_id)" + ")", Collections.emptyList(), TransactionContext.AUTOTRANSACTION_TRANSACTION);
long tx = execute.transactionId;
indexesOnTable = manager.getTableSpaceManager("tbl1").getIndexesOnTable("table_6");
assertFalse(indexesOnTable.get("t6index").isUnique());
assertEquals(ColumnTypes.NOTNULL_STRING, indexesOnTable.get("t6index").getIndex().columnByName.get("field").type);
assertFalse(indexesOnTable.get("t6index2").isUnique());
assertEquals(ColumnTypes.NOTNULL_STRING, indexesOnTable.get("t6index2").getIndex().columnByName.get("field").type);
assertEquals(ColumnTypes.TIMESTAMP, indexesOnTable.get("t6index2").getIndex().columnByName.get("secondfield").type);
assertTrue(indexesOnTable.get("index6u").isUnique());
assertEquals(ColumnTypes.NOTNULL_STRING, indexesOnTable.get("index6u").getIndex().columnByName.get("field").type);
assertEquals(ColumnTypes.NOTNULL_LONG, indexesOnTable.get("index6u").getIndex().columnByName.get("table_id").type);
assertTrue(indexesOnTable.get("table_6_unique_secondfield").isUnique());
assertEquals(ColumnTypes.TIMESTAMP, indexesOnTable.get("table_6_unique_secondfield").getIndex().columnByName.get("secondfield").type);
assertEquals("Missing some index " + indexesOnTable.keySet(), 4, indexesOnTable.size());
TestUtils.commitTransaction(manager, "tbl1", tx);
}
}
Aggregations