Examples with FileCommitLogManager herddb.file.FileCommitLogManager used on opensource projects

Example 36 with FileCommitLogManager

use of herddb.file.FileCommitLogManager in project herddb by diennea.

the class AutocheckPointTest method autoCheckPointDuringActivityTest.

@Test
public // @Ignore
void autoCheckPointDuringActivityTest() throws Exception {
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    String nodeId = "localhost";
    try (DBManager manager = new DBManager(nodeId, new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        execute(manager, "CREATE TABLE tblspace1.tsql (K1 int ,s1 string,n1 int, primary key(k1))", Collections.emptyList());
        for (int i = 0; i < 100; i++) {
            assertEquals(1, executeUpdate(manager, "INSERT INTO tblspace1.tsql(k1,s1,n1) values(?,?,?)", Arrays.asList(i, "a", Integer.valueOf(1234))).getUpdateCount());
        }
        manager.checkpoint();
        long lastCheckpont = manager.getLastCheckPointTs();
        // we want to checkpoint very ofter
        manager.setCheckpointPeriod(100);
        Random random = new Random();
        for (int trial = 0; trial < 1000; trial++) {
            int i = random.nextInt(100);
            execute(manager, "UPDATE tblspace1.tsql set s1='b" + trial + "' where k1=?", Arrays.asList(i), TransactionContext.NO_TRANSACTION);
            Thread.sleep(10);
        }
        assertNotEquals(lastCheckpont, manager.getLastCheckPointTs());
    }
}

Example 37 with FileCommitLogManager

use of herddb.file.FileCommitLogManager in project herddb by diennea.

the class Server method buildCommitLogManager.

protected CommitLogManager buildCommitLogManager() {
    switch(mode) {
        case ServerConfiguration.PROPERTY_MODE_LOCAL:
            return new MemoryCommitLogManager(false);
        case ServerConfiguration.PROPERTY_MODE_STANDALONE:
            Path logDirectory = this.baseDirectory.resolve(configuration.getString(ServerConfiguration.PROPERTY_LOGDIR, ServerConfiguration.PROPERTY_LOGDIR_DEFAULT));
            return new FileCommitLogManager(logDirectory, configuration.getLong(ServerConfiguration.PROPERTY_MAX_LOG_FILE_SIZE, ServerConfiguration.PROPERTY_MAX_LOG_FILE_SIZE_DEFAULT), configuration.getInt(ServerConfiguration.PROPERTY_MAX_UNSYNCHED_BATCH, ServerConfiguration.PROPERTY_MAX_UNSYNCHED_BATCH_DEFAULT), configuration.getInt(ServerConfiguration.PROPERTY_MAX_UNSYNCHED_BATCH_BYTES, ServerConfiguration.PROPERTY_MAX_UNSYNCHED_BATCH_BYTES_DEFAULT), configuration.getInt(ServerConfiguration.PROPERTY_MAX_SYNC_TIME, ServerConfiguration.PROPERTY_MAX_SYNC_TIME_DEFAULT), configuration.getBoolean(ServerConfiguration.PROPERTY_REQUIRE_FSYNC, ServerConfiguration.PROPERTY_REQUIRE_FSYNC_DEFAULT), configuration.getBoolean(ServerConfiguration.PROPERTY_TXLOG_USE_ODIRECT, ServerConfiguration.PROPERTY_TXLOG_USE_ODIRECT_DEFAULT), configuration.getInt(ServerConfiguration.PROPERTY_DEFERRED_SYNC_PERIOD, ServerConfiguration.PROPERTY_DEFERRED_SYNC_PERIOD_DEFAULT), statsLogger.scope("txlog"));
        case ServerConfiguration.PROPERTY_MODE_CLUSTER:
        case ServerConfiguration.PROPERTY_MODE_DISKLESSCLUSTER:
            BookkeeperCommitLogManager bkmanager = new BookkeeperCommitLogManager((ZookeeperMetadataStorageManager) this.metadataStorageManager, configuration, statsLogger);
            bkmanager.setAckQuorumSize(configuration.getInt(ServerConfiguration.PROPERTY_BOOKKEEPER_ACKQUORUMSIZE, ServerConfiguration.PROPERTY_BOOKKEEPER_ACKQUORUMSIZE_DEFAULT));
            bkmanager.setEnsemble(configuration.getInt(ServerConfiguration.PROPERTY_BOOKKEEPER_ENSEMBLE, ServerConfiguration.PROPERTY_BOOKKEEPER_ENSEMBLE_DEFAULT));
            bkmanager.setWriteQuorumSize(configuration.getInt(ServerConfiguration.PROPERTY_BOOKKEEPER_WRITEQUORUMSIZE, ServerConfiguration.PROPERTY_BOOKKEEPER_WRITEQUORUMSIZE_DEFAULT));
            long ledgersRetentionPeriod = configuration.getLong(ServerConfiguration.PROPERTY_BOOKKEEPER_LEDGERS_RETENTION_PERIOD, ServerConfiguration.PROPERTY_BOOKKEEPER_LEDGERS_RETENTION_PERIOD_DEFAULT);
            bkmanager.setLedgersRetentionPeriod(ledgersRetentionPeriod);
            long maxLedgerSizeBytes = configuration.getLong(ServerConfiguration.PROPERTY_BOOKKEEPER_LEDGERS_MAX_SIZE, ServerConfiguration.PROPERTY_BOOKKEEPER_LEDGERS_MAX_SIZE_DEFAULT);
            bkmanager.setMaxLedgerSizeBytes(maxLedgerSizeBytes);
            long maxIdleTime = configuration.getLong(ServerConfiguration.PROPERTY_BOOKKEEPER_MAX_IDLE_TIME, ServerConfiguration.PROPERTY_BOOKKEEPER_MAX_IDLE_TIME_DEFAULT);
            bkmanager.setMaxIdleTime(maxIdleTime);
            long checkPointperiod = configuration.getLong(ServerConfiguration.PROPERTY_CHECKPOINT_PERIOD, ServerConfiguration.PROPERTY_CHECKPOINT_PERIOD_DEFAULT);
            if (checkPointperiod > 0 && ledgersRetentionPeriod > 0) {
                long limit = ledgersRetentionPeriod / 2;
                if (checkPointperiod > limit) {
                    throw new RuntimeException(ServerConfiguration.PROPERTY_CHECKPOINT_PERIOD + "=" + checkPointperiod + " must be less then " + ServerConfiguration.PROPERTY_BOOKKEEPER_LEDGERS_RETENTION_PERIOD + "/2=" + limit);
                }
            }
            return bkmanager;
        default:
            throw new RuntimeException();
    }
}

Example 38 with FileCommitLogManager

use of herddb.file.FileCommitLogManager in project herddb by diennea.

the class SimpleIHashIndexRecoveryTest method createRecoveryIndex_withduoblecheckpoint.

@Test
public void createRecoveryIndex_withduoblecheckpoint() throws Exception {
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    String nodeId = "localhost";
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.STRING).column("name", ColumnTypes.STRING).primaryKey("id").build();
        CreateTableStatement st2 = new CreateTableStatement(table);
        manager.executeStatement(st2, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        Index index = Index.builder().onTable(table).type(Index.TYPE_HASH).column("name", ColumnTypes.STRING).build();
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('a','n1')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('b','n1')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('c','n1')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('d','n2')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('e','n2')", Collections.emptyList());
        CreateIndexStatement st3 = new CreateIndexStatement(index);
        manager.executeStatement(st3, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE name='n1'", Collections.emptyList(), true, true, false, -1);
        ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
        assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexSeek);
        try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
            assertEquals(3, scan1.consume().size());
        }
        manager.checkpoint();
        manager.checkpoint();
    }
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        assertTrue(manager.waitForTablespace("tblspace1", 10000));
        TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE name='n1'", Collections.emptyList(), true, true, false, -1);
        ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
        assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexSeek);
        try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
            assertEquals(3, scan1.consume().size());
        }
    }
}

Example 39 with FileCommitLogManager

use of herddb.file.FileCommitLogManager in project herddb by diennea.

the class SimpleRecoveryTest method ignoreUpdateWithMissingTransactionOnRecovery.

@Test
public void ignoreUpdateWithMissingTransactionOnRecovery() throws Exception {
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Bytes key = Bytes.from_int(1234);
    Bytes value = Bytes.from_long(8888);
    Bytes value2 = Bytes.from_long(9999);
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    String nodeId = "localhost";
    String tablespaceUUID = null;
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.STRING).column("name", ColumnTypes.STRING).primaryKey("id").build();
        CreateTableStatement st2 = new CreateTableStatement(table);
        manager.executeStatement(st2, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.checkpoint();
        InsertStatement insert = new InsertStatement("tblspace1", "t1", new Record(key, value));
        assertEquals(1, manager.executeUpdate(insert, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION).getUpdateCount());
        tablespaceUUID = manager.getTableSpaceManager("tblspace1").getTableSpaceUUID();
    }
    try (FileCommitLogManager commitLogManager = new FileCommitLogManager(logsPath)) {
        commitLogManager.start();
        try (FileCommitLog log = commitLogManager.createCommitLog(tablespaceUUID, "tblspace1", nodeId)) {
            log.recovery(LogSequenceNumber.START_OF_TIME, (n, e) -> {
            }, false);
            log.startWriting(1);
            /* Insert an entry for a unknown transaction id */
            LogEntry entry = new LogEntry(System.currentTimeMillis(), LogEntryType.UPDATE, 1024, "t1", key, value2);
            log.log(entry, true).getLogSequenceNumber();
        }
    }
    final boolean original = TableManager.ignoreMissingTransactionsOnRecovery;
    TableManager.ignoreMissingTransactionsOnRecovery = true;
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        manager.waitForTablespace("tblspace1", 10000);
        {
            GetStatement get = new GetStatement("tblspace1", "t1", key, null, false);
            GetResult result = manager.get(get, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
            assertTrue(result.found());
            assertEquals(key, result.getRecord().key);
            assertEquals(value, result.getRecord().value);
        }
    } finally {
        TableManager.ignoreMissingTransactionsOnRecovery = original;
    }
}

Example 40 with FileCommitLogManager

use of herddb.file.FileCommitLogManager in project herddb by diennea.

the class SimpleRecoveryTest method createTable1_nocheckpoint.

@Test
public void createTable1_nocheckpoint() throws Exception {
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    String nodeId = "localhost";
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.STRING).column("name", ColumnTypes.STRING).primaryKey("id").build();
        CreateTableStatement st2 = new CreateTableStatement(table);
        manager.executeStatement(st2, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
    }
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        manager.waitForTablespace("tblspace1", 10000);
        Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.STRING).column("name", ColumnTypes.STRING).primaryKey("id").build();
        CreateTableStatement st2 = new CreateTableStatement(table);
        try {
            manager.executeStatement(st2, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
            fail();
        } catch (TableAlreadyExistsException alreadyExists) {
        }
    }
}