Examples with FileCommitLogManager herddb.file.FileCommitLogManager used on opensource projects

Example 21 with FileCommitLogManager

use of herddb.file.FileCommitLogManager in project herddb by diennea.

the class SimpleRecoveryTest method ignoreInsertWithMissingTransactionOnRecovery.

@Test
public void ignoreInsertWithMissingTransactionOnRecovery() throws Exception {
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Bytes key = Bytes.from_int(1234);
    Bytes value = Bytes.from_long(8888);
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    String nodeId = "localhost";
    String tablespaceUUID = null;
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.STRING).column("name", ColumnTypes.STRING).primaryKey("id").build();
        CreateTableStatement st2 = new CreateTableStatement(table);
        manager.executeStatement(st2, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.checkpoint();
        tablespaceUUID = manager.getTableSpaceManager("tblspace1").getTableSpaceUUID();
    }
    try (FileCommitLogManager commitLogManager = new FileCommitLogManager(logsPath)) {
        commitLogManager.start();
        try (FileCommitLog log = commitLogManager.createCommitLog(tablespaceUUID, "tblspace1", nodeId)) {
            log.recovery(LogSequenceNumber.START_OF_TIME, (n, e) -> {
            }, false);
            log.startWriting(1);
            /* Insert an entry for a unknown transaction id */
            LogEntry entry = new LogEntry(System.currentTimeMillis(), LogEntryType.INSERT, 1024, "t1", key, value);
            log.log(entry, true).getLogSequenceNumber();
        }
    }
    final boolean original = TableManager.ignoreMissingTransactionsOnRecovery;
    TableManager.ignoreMissingTransactionsOnRecovery = true;
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        manager.waitForTablespace("tblspace1", 10000);
        {
            GetStatement get = new GetStatement("tblspace1", "t1", key, null, false);
            GetResult result = manager.get(get, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
            assertFalse(result.found());
        }
    } finally {
        TableManager.ignoreMissingTransactionsOnRecovery = original;
    }
}

Example 22 with FileCommitLogManager

use of herddb.file.FileCommitLogManager in project herddb by diennea.

the class SimpleRecoveryTest method autoIncrementAfterRestart_from_snapshot.

@Test
public void autoIncrementAfterRestart_from_snapshot() throws Exception {
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    String nodeId = "localhost";
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.INTEGER).primaryKey("id", true).build();
        CreateTableStatement st2 = new CreateTableStatement(table);
        manager.executeStatement(st2, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.checkpoint();
        InsertStatement insert = new InsertStatement("tblspace1", "t1", new AutoIncrementPrimaryKeyRecordFunction(), new ConstValueRecordFunction(new byte[0]), false);
        DMLStatementExecutionResult insertResult = manager.executeUpdate(insert, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertEquals(1, insertResult.getUpdateCount());
        int newValue = insertResult.getKey().to_int();
        assertEquals(1, newValue);
        GetResult get = manager.get(new GetStatement("tblspace1", "t1", insertResult.getKey(), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertTrue(get.found());
        long next_value = manager.getTableSpaceManager("tblspace1").getTableManager("t1").getNextPrimaryKeyValue();
        assertEquals(2, next_value);
        manager.checkpoint();
    }
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        manager.waitForTablespace("tblspace1", 10000);
        InsertStatement insert = new InsertStatement("tblspace1", "t1", new AutoIncrementPrimaryKeyRecordFunction(), new ConstValueRecordFunction(new byte[0]), false);
        DMLStatementExecutionResult insertResult = manager.executeUpdate(insert, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertEquals(1, insertResult.getUpdateCount());
        int newValue = insertResult.getKey().to_int();
        assertEquals(2, newValue);
        GetResult get = manager.get(new GetStatement("tblspace1", "t1", insertResult.getKey(), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertTrue(get.found());
        long next_value = manager.getTableSpaceManager("tblspace1").getTableManager("t1").getNextPrimaryKeyValue();
        assertEquals(3, next_value);
    }
}

Example 23 with FileCommitLogManager

use of herddb.file.FileCommitLogManager in project herddb by diennea.

the class SimpleRecoveryTest method autoIncrementAfterRestart_from_log.

@Test
public void autoIncrementAfterRestart_from_log() throws Exception {
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    String nodeId = "localhost";
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.INTEGER).primaryKey("id", true).build();
        CreateTableStatement st2 = new CreateTableStatement(table);
        manager.executeStatement(st2, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.checkpoint();
        InsertStatement insert = new InsertStatement("tblspace1", "t1", new AutoIncrementPrimaryKeyRecordFunction(), new ConstValueRecordFunction(new byte[0]), false);
        DMLStatementExecutionResult insertResult = manager.executeUpdate(insert, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertEquals(1, insertResult.getUpdateCount());
        int newValue = insertResult.getKey().to_int();
        assertEquals(1, newValue);
        GetResult get = manager.get(new GetStatement("tblspace1", "t1", insertResult.getKey(), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertTrue(get.found());
        long next_value = manager.getTableSpaceManager("tblspace1").getTableManager("t1").getNextPrimaryKeyValue();
        assertEquals(2, next_value);
    }
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        manager.waitForTablespace("tblspace1", 10000);
        InsertStatement insert = new InsertStatement("tblspace1", "t1", new AutoIncrementPrimaryKeyRecordFunction(), new ConstValueRecordFunction(new byte[0]), false);
        DMLStatementExecutionResult insertResult = manager.executeUpdate(insert, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertEquals(1, insertResult.getUpdateCount());
        int newValue = insertResult.getKey().to_int();
        assertEquals(2, newValue);
        GetResult get = manager.get(new GetStatement("tblspace1", "t1", insertResult.getKey(), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertTrue(get.found());
        long next_value = manager.getTableSpaceManager("tblspace1").getTableManager("t1").getNextPrimaryKeyValue();
        assertEquals(3, next_value);
    }
}

Example 24 with FileCommitLogManager

use of herddb.file.FileCommitLogManager in project herddb by diennea.

the class SimpleRecoveryTest method rollbackUpdateTransactionOnRestart.

@Test
public void rollbackUpdateTransactionOnRestart() throws Exception {
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Bytes key = Bytes.from_int(1234);
    Bytes value = Bytes.from_long(8888);
    Bytes value2 = Bytes.from_long(8889);
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    String nodeId = "localhost";
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.STRING).column("name", ColumnTypes.STRING).primaryKey("id").build();
        CreateTableStatement st2 = new CreateTableStatement(table);
        manager.executeStatement(st2, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.checkpoint();
        InsertStatement insert = new InsertStatement("tblspace1", "t1", new Record(key, value));
        assertEquals(1, manager.executeUpdate(insert, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
        // transaction is not committed, and the leader dies, key appear again with the old value
        long tx = ((TransactionResult) manager.executeStatement(new BeginTransactionStatement("tblspace1"), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION)).getTransactionId();
        UpdateStatement update = new UpdateStatement("tblspace1", "t1", new Record(key, value2), null);
        assertEquals(1, manager.executeUpdate(update, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx)).getUpdateCount());
    }
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        manager.waitForTablespace("tblspace1", 10000);
        {
            GetStatement get = new GetStatement("tblspace1", "t1", key, null, false);
            GetResult result = manager.get(get, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
            assertTrue(result.found());
            assertEquals(key, result.getRecord().key);
            assertEquals(value, result.getRecord().value);
        }
    }
}

Example 25 with FileCommitLogManager

use of herddb.file.FileCommitLogManager in project herddb by diennea.

the class CheckpointTest method rebuildSmallPages.

/**
 * Rebuild all small pages
 */
@Test
public void rebuildSmallPages() throws Exception {
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmpDir = folder.newFolder("tmpDir").toPath();
    String nodeId = "localhost";
    ServerConfiguration config1 = newServerConfigurationWithAutoPort();
    config1.set(ServerConfiguration.PROPERTY_MAX_LOGICAL_PAGE_SIZE, 1024L);
    int records = 1000;
    int keylen = 25;
    int strlen = 50;
    int originalPages;
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmpDir, null, config1, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        execute(manager, "CREATE TABLE tblspace1.tsql (k1 string, s1 string, n1 int, primary key(k1))", Collections.emptyList());
        for (int i = 0; i < records; ++i) {
            executeUpdate(manager, "INSERT INTO tblspace1.tsql(k1,s1,n1) values(?,?,?)", Arrays.asList(RandomString.getInstance().nextString(keylen), RandomString.getInstance().nextString(strlen), Integer.valueOf(i)));
        }
        manager.checkpoint();
        String uuid = manager.getMetadataStorageManager().describeTableSpace("tblspace1").uuid;
        String tableUuid = manager.getTableSpaceManager("tblspace1").getTableManager("tsql").getTable().uuid;
        originalPages = manager.getDataStorageManager().getActualNumberOfPages(uuid, tableUuid);
        assertTrue(originalPages > 10);
    }
    ServerConfiguration config2 = newServerConfigurationWithAutoPort();
    config2.set(ServerConfiguration.PROPERTY_MAX_LOGICAL_PAGE_SIZE, 2048L);
    config2.set(ServerConfiguration.PROPERTY_COMPACTION_DURATION, -1L);
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmpDir, null, config2, null)) {
        manager.start();
        assertTrue(manager.waitForTablespace("tblspace1", 20000));
        manager.checkpoint();
        String tableUuid = manager.getTableSpaceManager("tblspace1").getTableManager("tsql").getTable().uuid;
        String uuid = manager.getMetadataStorageManager().describeTableSpace("tblspace1").uuid;
        int pages = manager.getDataStorageManager().getActualNumberOfPages(uuid, tableUuid);
        /* There are at least half pages! */
        assertTrue(pages <= (originalPages / 2) + (originalPages % 2));
    }
}