Examples with FileCommitLogManager herddb.file.FileCommitLogManager used on opensource projects

Example 26 with FileCommitLogManager

use of herddb.file.FileCommitLogManager in project herddb by diennea.

the class CheckpointTest method restartTest.

/**
 * Reload dirty pages on restart
 */
@Test
public void restartTest() throws Exception {
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmpDir = folder.newFolder("tmpDir").toPath();
    String nodeId = "localhost";
    ServerConfiguration config1 = newServerConfigurationWithAutoPort();
    /* Disable page compaction (avoid compaction of dirty page) */
    config1.set(ServerConfiguration.PROPERTY_FILL_PAGE_THRESHOLD, 0.0D);
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmpDir, null, config1, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        execute(manager, "CREATE TABLE tblspace1.tsql (K1 string ,s1 string,n1 int, primary key(k1))", Collections.emptyList());
        assertEquals(1, executeUpdate(manager, "INSERT INTO tblspace1.tsql(k1,s1,n1) values(?,?,?)", Arrays.asList("mykey", "a", Integer.valueOf(1234))).getUpdateCount());
        assertEquals(1, executeUpdate(manager, "INSERT INTO tblspace1.tsql(k1,s1,n1) values(?,?,?)", Arrays.asList("mykey2", "a", Integer.valueOf(1234))).getUpdateCount());
        assertEquals(1, executeUpdate(manager, "INSERT INTO tblspace1.tsql(k1,s1,n1) values(?,?,?)", Arrays.asList("mykey3", "a", Integer.valueOf(1234))).getUpdateCount());
        assertEquals(1, executeUpdate(manager, "INSERT INTO tblspace1.tsql(k1,s1,n1) values(?,?,?)", Arrays.asList("mykey4", "a", Integer.valueOf(1234))).getUpdateCount());
        manager.checkpoint();
        assertEquals(1, executeUpdate(manager, "UPDATE tblspace1.tsql set s1=? where k1=?", Arrays.asList("b", "mykey4")).getUpdateCount());
        manager.checkpoint();
        assertEquals(3, manager.getTableSpaceManager("tblspace1").getTableManager("tsql").getStats().getLoadedpages());
        assertEquals(1, manager.getTableSpaceManager("tblspace1").getTableManager("tsql").getStats().getDirtypages());
    }
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmpDir, null, config1, null)) {
        manager.start();
        manager.waitForTablespace("tblspace1", 10000);
        assertEquals(1, manager.getTableSpaceManager("tblspace1").getTableManager("tsql").getStats().getLoadedpages());
        assertEquals(1, manager.getTableSpaceManager("tblspace1").getTableManager("tsql").getStats().getDirtypages());
    }
}

Example 27 with FileCommitLogManager

use of herddb.file.FileCommitLogManager in project herddb by diennea.

the class SimpleJoinTest method testExists.

@Test
public void testExists() throws Exception {
    String nodeId = "localhost";
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    // reproduces an error due to the order or elements returned during the scan
    try (DBManager manager = new DBManager(nodeId, new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        assumeThat(manager.getPlanner(), instanceOf(CalcitePlanner.class));
        manager.start();
        manager.waitForTablespace(TableSpace.DEFAULT, 10000);
        execute(manager, "CREATE TABLE a (k1 int primary key, v1 int)", Collections.emptyList());
        execute(manager, "CREATE TABLE b (k2 int primary key, v2 int)", Collections.emptyList());
        execute(manager, "INSERT INTO a (k1,v1) values(1,1)", Collections.emptyList());
        execute(manager, "INSERT INTO b (k2,v2) values(1,1)", Collections.emptyList());
        {
            try (DataScanner scan1 = scan(manager, " select k1 as kk, v1 as vv from a where 1=1 and ( NOT EXISTS (SELECT NULL FROM b WHERE b.v2=a.k1 AND  1=1  AND b.k2 = 2 ))", Collections.emptyList())) {
                List<DataAccessor> consume = scan1.consume();
                System.out.println("NUM " + consume.size());
                for (DataAccessor r : consume) {
                    System.out.println("RECORD " + r.toMap());
                }
                assertEquals(1, consume.size());
            }
        }
        {
            try (DataScanner scan1 = scan(manager, " select k1 as kk, v1 as vv from a where 1=1 and ( NOT EXISTS (SELECT NULL FROM b WHERE b.v2=a.k1 AND  1=1  AND b.k2 = 1 ))", Collections.emptyList())) {
                List<DataAccessor> consume = scan1.consume();
                System.out.println("NUM " + consume.size());
                for (DataAccessor r : consume) {
                    System.out.println("RECORD " + r.toMap());
                }
                assertEquals(0, consume.size());
            }
        }
        // joins during transactions must release transaction resources properly
        {
            long tx = TestUtils.beginTransaction(manager, TableSpace.DEFAULT);
            try (DataScanner scan1 = scan(manager, " select k1 as kk, v1 as vv from a where 1=1 and ( NOT EXISTS (SELECT NULL FROM b WHERE b.v2=a.k1 AND  1=1  AND b.k2 = 2 ))", Collections.emptyList(), new TransactionContext(tx))) {
                List<DataAccessor> consume = scan1.consume();
                System.out.println("NUM " + consume.size());
                for (DataAccessor r : consume) {
                    System.out.println("RECORD " + r.toMap());
                }
                assertEquals(1, consume.size());
            }
            TestUtils.commitTransaction(manager, TableSpace.DEFAULT, tx);
        }
        {
            long tx = TestUtils.beginTransaction(manager, TableSpace.DEFAULT);
            try (DataScanner scan1 = scan(manager, " select k1 as kk, v1 as vv from a where 1=1 and ( EXISTS (SELECT 1 FROM b WHERE b.v2=a.k1 AND b.k2 = 1 ))", Collections.emptyList(), new TransactionContext(tx))) {
                List<DataAccessor> consume = scan1.consume();
                System.out.println("NUM " + consume.size());
                for (DataAccessor r : consume) {
                    System.out.println("RECORD " + r.toMap());
                }
                assertEquals(1, consume.size());
            }
            TestUtils.commitTransaction(manager, TableSpace.DEFAULT, tx);
        }
    }
}

Example 28 with FileCommitLogManager

use of herddb.file.FileCommitLogManager in project herddb by diennea.

the class SimpleJoinTest method testJoinUnsortedKey.

@Test
public void testJoinUnsortedKey() throws Exception {
    String nodeId = "localhost";
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    // reproduces an error due to the order or elements returned during the scan
    try (DBManager manager = new DBManager(nodeId, new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        manager.waitForTablespace(TableSpace.DEFAULT, 10000);
        execute(manager, "CREATE TABLE customer (customer_id long primary key,contact_email string,contact_person string, creation long, deleted boolean, modification long, name string, vetting boolean)", Collections.emptyList());
        execute(manager, "CREATE TABLE license (license_id long primary key,application string, creation long,data string, deleted boolean, modification long,signature string, customer_id long , license_key_id string)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(1,1)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(2,1)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(3,3)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(4,4)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(5,3)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(6,3)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(7,3)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(8,3)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(9,3)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(10,3)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(11,3)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(12,5)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(13,6)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(14,7)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(15,3)", Collections.emptyList());
        execute(manager, "INSERT INTO customer (customer_id) values(1)", Collections.emptyList());
        execute(manager, "INSERT INTO customer (customer_id) values(2)", Collections.emptyList());
        execute(manager, "INSERT INTO customer (customer_id) values(3)", Collections.emptyList());
        execute(manager, "INSERT INTO customer (customer_id) values(4)", Collections.emptyList());
        execute(manager, "INSERT INTO customer (customer_id) values(5)", Collections.emptyList());
        execute(manager, "INSERT INTO customer (customer_id) values(6)", Collections.emptyList());
        execute(manager, "INSERT INTO customer (customer_id) values(7)", Collections.emptyList());
        {
            try (DataScanner scan1 = scan(manager, "SELECT t0.license_id,c.customer_id FROM license t0, customer c WHERE c.customer_id = 3 AND t0.customer_id = 3 AND c.customer_id = t0.customer_id", Collections.emptyList())) {
                List<DataAccessor> consume = scan1.consumeAndClose();
                System.out.println("NUM " + consume.size());
                assertEquals(9, consume.size());
                for (DataAccessor r : consume) {
                    System.out.println("RECORD " + r.toMap());
                }
            }
        }
        // joins during transactions must release transaction resources properly
        {
            long tx = TestUtils.beginTransaction(manager, TableSpace.DEFAULT);
            try (DataScanner scan1 = scan(manager, "SELECT t0.license_id,c.customer_id FROM license t0, customer c WHERE c.customer_id = 3 AND t0.customer_id = 3 AND c.customer_id = t0.customer_id", Collections.emptyList(), new TransactionContext(tx))) {
                List<DataAccessor> consume = scan1.consume();
                System.out.println("NUM " + consume.size());
                assertEquals(9, consume.size());
                for (DataAccessor r : consume) {
                    System.out.println("RECORD " + r.toMap());
                }
            }
            TestUtils.commitTransaction(manager, TableSpace.DEFAULT, tx);
        }
    }
}

Example 29 with FileCommitLogManager

use of herddb.file.FileCommitLogManager in project herddb by diennea.

the class SimpleBrinIndexRecoveryTest method createRecoveryIndex_withrepeatedkey.

@Test
public void createRecoveryIndex_withrepeatedkey() throws Exception {
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    int id = 0;
    String nodeId = "localhost";
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertTrue(manager.waitForTablespace("tblspace1", 10000));
        Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.STRING).column("name", ColumnTypes.STRING).primaryKey("id").build();
        CreateTableStatement st2 = new CreateTableStatement(table);
        manager.executeStatement(st2, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        Index index = Index.builder().onTable(table).type(Index.TYPE_BRIN).column("name", ColumnTypes.STRING).build();
        CreateIndexStatement st3 = new CreateIndexStatement(index);
        manager.executeStatement(st3, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        BRINIndexManager brin = (BRINIndexManager) manager.getTableSpaceManager(table.tablespace).getIndexesOnTable(table.name).get(index.name);
        while (brin.getNumBlocks() < 2) {
            id++;
            TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values(?,?)", Arrays.asList(id, "my_repeatad_key"), TransactionContext.NO_TRANSACTION);
        }
        // some data on disk
        manager.checkpoint();
        // some data to be recovered from log
        while (brin.getNumBlocks() < 3) {
            id++;
            TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values(?,?)", Arrays.asList(id, "my_repeatad_key"), TransactionContext.NO_TRANSACTION);
        }
        try (DataScanner scan1 = scan(manager, "SELECT * FROM tblspace1.t1", Collections.emptyList())) {
            assertEquals(id, scan1.consume().size());
        }
    }
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        assertTrue(manager.waitForBootOfLocalTablespaces(10000));
        TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE name='my_repeatad_key'", Collections.emptyList(), true, true, false, -1);
        ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
        assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexSeek);
        try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
            assertEquals(id, scan1.consume().size());
        }
    }
}

Example 30 with FileCommitLogManager

use of herddb.file.FileCommitLogManager in project herddb by diennea.

the class SimpleBrinIndexRecoveryTest method createRecoveryIndex_withcheckpoint.

@Test
public void createRecoveryIndex_withcheckpoint() throws Exception {
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    String nodeId = "localhost";
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertTrue(manager.waitForTablespace("tblspace1", 10000));
        Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.STRING).column("name", ColumnTypes.STRING).primaryKey("id").build();
        CreateTableStatement st2 = new CreateTableStatement(table);
        manager.executeStatement(st2, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        Index index = Index.builder().onTable(table).type(Index.TYPE_BRIN).column("name", ColumnTypes.STRING).build();
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('a','n1')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('b','n1')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('c','n1')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('d','n2')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('e','n2')", Collections.emptyList());
        CreateIndexStatement st3 = new CreateIndexStatement(index);
        manager.executeStatement(st3, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE name='n1'", Collections.emptyList(), true, true, false, -1);
        ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
        assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexSeek);
        try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
            assertEquals(3, scan1.consume().size());
        }
        manager.checkpoint();
    }
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        assertTrue(manager.waitForBootOfLocalTablespaces(10000));
        TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE name='n1'", Collections.emptyList(), true, true, false, -1);
        ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
        assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexSeek);
        try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
            assertEquals(3, scan1.consume().size());
        }
    }
}