Search in sources :

Example 26 with InsertStatement

use of herddb.model.commands.InsertStatement in project herddb by diennea.

the class RestartTestBase method recoverTableCreatedInTransaction2.

@Test
public void recoverTableCreatedInTransaction2() throws Exception {
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    Bytes key = Bytes.from_string("k1");
    String nodeId = "localhost";
    try (DBManager manager = buildDBManager(nodeId, metadataPath, dataPath, logsPath, tmoDir)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertTrue(manager.waitForTablespace("tblspace1", 10000));
        Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.STRING).column("name", ColumnTypes.STRING).primaryKey("id").build();
        long tx = ((TransactionResult) manager.executeStatement(new BeginTransactionStatement("tblspace1"), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION)).getTransactionId();
        manager.executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx));
        manager.executeStatement(new InsertStatement("tblspace1", table.name, new Record(key, key)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx));
        manager.checkpoint();
        DMLStatementExecutionResult executeStatement = (DMLStatementExecutionResult) manager.executeStatement(new UpdateStatement("tblspace1", "t1", new Record(key, key), null), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx));
        assertEquals(1, executeStatement.getUpdateCount());
        manager.executeStatement(new CommitTransactionStatement("tblspace1", tx), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
    }
    try (DBManager manager = buildDBManager(nodeId, metadataPath, dataPath, logsPath, tmoDir)) {
        manager.start();
        assertTrue(manager.waitForBootOfLocalTablespaces(10000));
        DMLStatementExecutionResult executeStatement = (DMLStatementExecutionResult) manager.executeStatement(new UpdateStatement("tblspace1", "t1", new Record(key, key), null), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertEquals(1, executeStatement.getUpdateCount());
    // on the log there is an UPDATE for a record which is not on the log
    }
    try (DBManager manager = buildDBManager(nodeId, metadataPath, dataPath, logsPath, tmoDir)) {
        manager.start();
        assertTrue(manager.waitForBootOfLocalTablespaces(10000));
        GetResult result = manager.get(new GetStatement("tblspace1", "t1", key, null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertTrue(result.found());
    }
}
Also used : Path(java.nio.file.Path) TransactionResult(herddb.model.TransactionResult) UpdateStatement(herddb.model.commands.UpdateStatement) Table(herddb.model.Table) GetResult(herddb.model.GetResult) CommitTransactionStatement(herddb.model.commands.CommitTransactionStatement) CreateTableStatement(herddb.model.commands.CreateTableStatement) InsertStatement(herddb.model.commands.InsertStatement) Bytes(herddb.utils.Bytes) CreateTableSpaceStatement(herddb.model.commands.CreateTableSpaceStatement) DMLStatementExecutionResult(herddb.model.DMLStatementExecutionResult) TransactionContext(herddb.model.TransactionContext) GetStatement(herddb.model.commands.GetStatement) BeginTransactionStatement(herddb.model.commands.BeginTransactionStatement) Record(herddb.model.Record) Test(org.junit.Test)

Example 27 with InsertStatement

use of herddb.model.commands.InsertStatement in project herddb by diennea.

the class RestartTestBase method recoverTableCreatedInTransaction3.

@Test
public void recoverTableCreatedInTransaction3() throws Exception {
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    Bytes key = Bytes.from_string("k1");
    String nodeId = "localhost";
    try (DBManager manager = buildDBManager(nodeId, metadataPath, dataPath, logsPath, tmoDir)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertTrue(manager.waitForTablespace("tblspace1", 10000));
        Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.STRING).column("name", ColumnTypes.STRING).primaryKey("id").build();
        long tx = ((TransactionResult) manager.executeStatement(new BeginTransactionStatement("tblspace1"), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION)).getTransactionId();
        manager.executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx));
        manager.executeStatement(new CommitTransactionStatement("tblspace1", tx), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.executeStatement(new InsertStatement("tblspace1", table.name, new Record(key, key)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.checkpoint();
        DMLStatementExecutionResult executeStatement = (DMLStatementExecutionResult) manager.executeStatement(new UpdateStatement("tblspace1", "t1", new Record(key, key), null), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertEquals(1, executeStatement.getUpdateCount());
    }
    try (DBManager manager = buildDBManager(nodeId, metadataPath, dataPath, logsPath, tmoDir)) {
        manager.start();
        assertTrue(manager.waitForBootOfLocalTablespaces(10000));
        GetResult result = manager.get(new GetStatement("tblspace1", "t1", key, null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertTrue(result.found());
    }
}
Also used : Path(java.nio.file.Path) TransactionResult(herddb.model.TransactionResult) UpdateStatement(herddb.model.commands.UpdateStatement) Table(herddb.model.Table) GetResult(herddb.model.GetResult) CommitTransactionStatement(herddb.model.commands.CommitTransactionStatement) CreateTableStatement(herddb.model.commands.CreateTableStatement) InsertStatement(herddb.model.commands.InsertStatement) Bytes(herddb.utils.Bytes) CreateTableSpaceStatement(herddb.model.commands.CreateTableSpaceStatement) DMLStatementExecutionResult(herddb.model.DMLStatementExecutionResult) TransactionContext(herddb.model.TransactionContext) GetStatement(herddb.model.commands.GetStatement) BeginTransactionStatement(herddb.model.commands.BeginTransactionStatement) Record(herddb.model.Record) Test(org.junit.Test)

Example 28 with InsertStatement

use of herddb.model.commands.InsertStatement in project herddb by diennea.

the class RestartTestBase method recoverTableCreatedInTransaction.

@Test
public void recoverTableCreatedInTransaction() throws Exception {
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    Bytes key = Bytes.from_string("k1");
    String nodeId = "localhost";
    try (DBManager manager = buildDBManager(nodeId, metadataPath, dataPath, logsPath, tmoDir)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertTrue(manager.waitForTablespace("tblspace1", 10000));
        Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.STRING).column("name", ColumnTypes.STRING).primaryKey("id").build();
        long tx = ((TransactionResult) manager.executeStatement(new BeginTransactionStatement("tblspace1"), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION)).getTransactionId();
        manager.executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx));
        manager.executeStatement(new InsertStatement("tblspace1", table.name, new Record(key, key)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx));
        manager.checkpoint();
        manager.executeStatement(new CommitTransactionStatement("tblspace1", tx), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
    }
    try (DBManager manager = buildDBManager(nodeId, metadataPath, dataPath, logsPath, tmoDir)) {
        manager.start();
        assertTrue(manager.waitForBootOfLocalTablespaces(10000));
        DMLStatementExecutionResult executeStatement = (DMLStatementExecutionResult) manager.executeStatement(new UpdateStatement("tblspace1", "t1", new Record(key, key), null), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertEquals(1, executeStatement.getUpdateCount());
    // on the log there is an UPDATE for a record which is not on the log
    }
    try (DBManager manager = buildDBManager(nodeId, metadataPath, dataPath, logsPath, tmoDir)) {
        manager.start();
        assertTrue(manager.waitForBootOfLocalTablespaces(10000));
        GetResult result = manager.get(new GetStatement("tblspace1", "t1", key, null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertTrue(result.found());
    }
}
Also used : Path(java.nio.file.Path) TransactionResult(herddb.model.TransactionResult) UpdateStatement(herddb.model.commands.UpdateStatement) Table(herddb.model.Table) GetResult(herddb.model.GetResult) CommitTransactionStatement(herddb.model.commands.CommitTransactionStatement) CreateTableStatement(herddb.model.commands.CreateTableStatement) InsertStatement(herddb.model.commands.InsertStatement) Bytes(herddb.utils.Bytes) CreateTableSpaceStatement(herddb.model.commands.CreateTableSpaceStatement) DMLStatementExecutionResult(herddb.model.DMLStatementExecutionResult) TransactionContext(herddb.model.TransactionContext) GetStatement(herddb.model.commands.GetStatement) BeginTransactionStatement(herddb.model.commands.BeginTransactionStatement) Record(herddb.model.Record) Test(org.junit.Test)

Example 29 with InsertStatement

use of herddb.model.commands.InsertStatement in project herddb by diennea.

the class ScanDuringCheckPointTest method bigTableScan.

@Test
public // @Ignore
void bigTableScan() throws Exception {
    int testSize = 10000;
    String nodeId = "localhost";
    try (DBManager manager = new DBManager("localhost", new MemoryMetadataStorageManager(), new MemoryDataStorageManager(), new MemoryCommitLogManager(), null, null)) {
        // manager.setMaxLogicalPageSize(10);
        // manager.setMaxPagesUsedMemory(Long.MAX_VALUE);
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        String tableSpaceUUID = manager.getTableSpaceManager("tblspace1").getTableSpaceUUID();
        Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.STRING).column("name", ColumnTypes.STRING).primaryKey("id").build();
        CreateTableStatement st2 = new CreateTableStatement(table);
        manager.executeStatement(st2, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        for (int i = 0; i < testSize / 2; i++) {
            InsertStatement insert = new InsertStatement(table.tablespace, table.name, RecordSerializer.makeRecord(table, "id", "k" + i, "name", RandomString.getInstance().nextString(50, new StringBuilder().append("testname").append(i).append("_")).toString()));
            assertEquals(1, manager.executeUpdate(insert, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
        }
        manager.checkpoint();
        for (int i = testSize / 2; i < testSize; i++) {
            InsertStatement insert = new InsertStatement(table.tablespace, table.name, RecordSerializer.makeRecord(table, "id", "k" + i, "name", RandomString.getInstance().nextString(50, new StringBuilder().append("testname").append(i).append("_")).toString()));
            assertEquals(1, manager.executeUpdate(insert, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
        }
        manager.checkpoint();
        assertTrue(manager.getDataStorageManager().getActualNumberOfPages(tableSpaceUUID, table.uuid) > 1);
        TableManagerStats stats = manager.getTableSpaceManager(table.tablespace).getTableManager(table.name).getStats();
        assertEquals(testSize, stats.getTablesize());
        assertTrue(testSize > 100);
        ExecutorService service = Executors.newFixedThreadPool(1);
        Runnable checkPointPerformer = new Runnable() {

            @Override
            public void run() {
                CountDownLatch checkpointDone = new CountDownLatch(1);
                Thread fakeCheckpointThread = new Thread(new Runnable() {

                    @Override
                    public void run() {
                        try {
                            manager.checkpoint();
                        } catch (Throwable t) {
                            t.printStackTrace();
                            fail();
                        }
                        checkpointDone.countDown();
                    }
                });
                fakeCheckpointThread.setDaemon(true);
                try {
                    fakeCheckpointThread.start();
                    checkpointDone.await();
                    fakeCheckpointThread.join();
                } catch (InterruptedException err) {
                    throw new RuntimeException(err);
                }
            }
        };
        AtomicInteger done = new AtomicInteger();
        Predicate slowPredicate = new Predicate() {

            int count = 0;

            @Override
            public boolean evaluate(Record record, StatementEvaluationContext context) throws StatementExecutionException {
                if (count++ % 10 == 0) {
                    // checkpoint will flush buffers, in the middle of the scan
                    try {
                        System.out.println("GO checkpoint !");
                        service.submit(checkPointPerformer).get();
                        done.incrementAndGet();
                    } catch (ExecutionException | InterruptedException err) {
                        throw new StatementExecutionException(err);
                    }
                }
                return true;
            }
        };
        try (DataScanner scan = manager.scan(new ScanStatement(table.tablespace, table, slowPredicate), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION)) {
            AtomicInteger count = new AtomicInteger();
            scan.forEach(tuple -> {
                count.incrementAndGet();
            });
            assertEquals(testSize, count.get());
        }
        assertEquals(testSize, stats.getTablesize());
        assertEquals(testSize / 10, done.get());
    }
}
Also used : RandomString(herddb.utils.RandomString) InsertStatement(herddb.model.commands.InsertStatement) StatementExecutionException(herddb.model.StatementExecutionException) Predicate(herddb.model.Predicate) CreateTableSpaceStatement(herddb.model.commands.CreateTableSpaceStatement) DataScanner(herddb.model.DataScanner) TableManagerStats(herddb.core.stats.TableManagerStats) MemoryCommitLogManager(herddb.mem.MemoryCommitLogManager) Record(herddb.model.Record) StatementExecutionException(herddb.model.StatementExecutionException) ExecutionException(java.util.concurrent.ExecutionException) MemoryMetadataStorageManager(herddb.mem.MemoryMetadataStorageManager) ScanStatement(herddb.model.commands.ScanStatement) Table(herddb.model.Table) MemoryDataStorageManager(herddb.mem.MemoryDataStorageManager) CreateTableStatement(herddb.model.commands.CreateTableStatement) CountDownLatch(java.util.concurrent.CountDownLatch) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ExecutorService(java.util.concurrent.ExecutorService) StatementEvaluationContext(herddb.model.StatementEvaluationContext) Test(org.junit.Test)

Example 30 with InsertStatement

use of herddb.model.commands.InsertStatement in project herddb by diennea.

the class ScanTest method testTransaction.

@Test
public void testTransaction() throws Exception {
    for (int i = 0; i < 5; i++) {
        Map<String, Object> data = new HashMap<>();
        data.put("id", "key_" + i);
        data.put("number", i);
        Record record = RecordSerializer.toRecord(data, table);
        InsertStatement st = new InsertStatement(tableSpace, tableName, record);
        assertEquals(1, manager.executeUpdate(st, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
    }
    BeginTransactionStatement begin = new BeginTransactionStatement(tableSpace);
    long tx = ((TransactionResult) manager.executeStatement(begin, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION)).getTransactionId();
    for (int i = 5; i < 10; i++) {
        Map<String, Object> data = new HashMap<>();
        data.put("id", "key_" + i);
        data.put("number", i);
        Record record = RecordSerializer.toRecord(data, table);
        InsertStatement st = new InsertStatement(tableSpace, tableName, record);
        assertEquals(1, manager.executeUpdate(st, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx)).getUpdateCount());
    }
    {
        ScanStatement scan = new ScanStatement(tableSpace, table, new Predicate() {

            @Override
            public boolean evaluate(Record record, StatementEvaluationContext context) throws StatementExecutionException {
                return true;
            }
        });
        DataScanner scanner = manager.scan(scan, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx));
        List<?> result = scanner.consumeAndClose();
        assertEquals(10, result.size());
    }
    for (int i = 0; i < 10; i++) {
        DeleteStatement st = new DeleteStatement(tableSpace, tableName, Bytes.from_string("key_" + i), null);
        assertEquals(1, manager.executeUpdate(st, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx)).getUpdateCount());
    }
    {
        ScanStatement scan = new ScanStatement(tableSpace, table, new Predicate() {

            @Override
            public boolean evaluate(Record record, StatementEvaluationContext context) throws StatementExecutionException {
                return true;
            }
        });
        DataScanner scanner = manager.scan(scan, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx));
        List<?> result = scanner.consumeAndClose();
        assertEquals(0, result.size());
    }
}
Also used : TransactionResult(herddb.model.TransactionResult) HashMap(java.util.HashMap) DeleteStatement(herddb.model.commands.DeleteStatement) InsertStatement(herddb.model.commands.InsertStatement) Predicate(herddb.model.Predicate) DataScanner(herddb.model.DataScanner) TransactionContext(herddb.model.TransactionContext) BeginTransactionStatement(herddb.model.commands.BeginTransactionStatement) Record(herddb.model.Record) List(java.util.List) StatementEvaluationContext(herddb.model.StatementEvaluationContext) ScanStatement(herddb.model.commands.ScanStatement) Test(org.junit.Test)

Aggregations

InsertStatement (herddb.model.commands.InsertStatement)140 Test (org.junit.Test)126 Table (herddb.model.Table)111 CreateTableStatement (herddb.model.commands.CreateTableStatement)102 GetStatement (herddb.model.commands.GetStatement)88 GetResult (herddb.model.GetResult)86 Record (herddb.model.Record)75 TransactionContext (herddb.model.TransactionContext)59 Bytes (herddb.utils.Bytes)59 CreateTableSpaceStatement (herddb.model.commands.CreateTableSpaceStatement)48 Path (java.nio.file.Path)42 DataScanner (herddb.model.DataScanner)37 ServerConfiguration (herddb.server.ServerConfiguration)36 CommitTransactionStatement (herddb.model.commands.CommitTransactionStatement)34 DeleteStatement (herddb.model.commands.DeleteStatement)33 Server (herddb.server.Server)32 UpdateStatement (herddb.model.commands.UpdateStatement)31 TransactionResult (herddb.model.TransactionResult)29 BeginTransactionStatement (herddb.model.commands.BeginTransactionStatement)29 HashSet (java.util.HashSet)29