Examples with FileCommitLogManager herddb.file.FileCommitLogManager used on opensource projects

Example 11 with FileCommitLogManager

use of herddb.file.FileCommitLogManager in project herddb by diennea.

the class IndexCreationTest method recoverTableAndIndexWithCheckpoint.

private void recoverTableAndIndexWithCheckpoint(String indexType) throws Exception {
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    String nodeId = "localhost";
    Table table;
    Index index;
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath, 64 * 1024 * 1024), tmoDir, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.INTEGER).column("name", ColumnTypes.STRING).primaryKey("id").build();
        manager.executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.executeStatement(new InsertStatement("tblspace1", table.name, RecordSerializer.makeRecord(table, "id", 1, "name", "uno")), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        GetResult result = manager.get(new GetStatement("tblspace1", table.name, Bytes.from_int(1), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertTrue(result.found());
        manager.checkpoint();
    }
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath, 64 * 1024 * 1024), tmoDir, null)) {
        manager.start();
        manager.waitForTablespace("tblspace1", 10000);
        index = Index.builder().onTable(table).column("name", ColumnTypes.STRING).type(indexType).build();
        manager.executeStatement(new CreateIndexStatement(index), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        /* Access through index  */
        TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE name=\'uno\'", Collections.emptyList(), true, true, false, -1);
        ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
        assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexSeek);
        try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
            assertEquals(1, scan1.consume().size());
        }
    }
}

Example 12 with FileCommitLogManager

use of herddb.file.FileCommitLogManager in project herddb by diennea.

the class RunHerdDB070Test method test.

@Test
public void test() throws Exception {
    String file = "herddb.070.joinerror.zip";
    File dbdatadir = folder.newFolder("dbdata070_" + file);
    try (InputStream in = RunHerdDB070Test.class.getResourceAsStream(file)) {
        ZIPUtils.unZip(in, dbdatadir);
    }
    System.out.println("UNZIPPED TO " + dbdatadir);
    final Path dbdata = dbdatadir.toPath().resolve("herddb.070.joinerror").resolve("dbdata");
    Path metadataPath = dbdata.resolve("metadata");
    Path dataPath = dbdata.resolve("data");
    Path logsPath = dbdata.resolve("txlog");
    Path tmoDir = dbdata.resolve("tmp");
    assertTrue(Files.isDirectory(metadataPath));
    assertTrue(Files.isDirectory(dataPath));
    assertTrue(Files.isDirectory(logsPath));
    Path nodeid = dataPath.resolve("nodeid");
    assertTrue(Files.isRegularFile(nodeid));
    String id = new String(Files.readAllBytes(nodeid), StandardCharsets.UTF_8);
    System.out.println("id:" + id);
    String expectedNodeId = "asino";
    assertTrue(id.endsWith("\n" + expectedNodeId));
    try (DBManager manager = new DBManager(expectedNodeId, new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        final String tableSpace = "herd";
        final String tableName = "testtable";
        assertEquals(expectedNodeId, manager.getNodeId());
        assertTrue(manager.waitForTablespace(tableSpace, 10000));
        AbstractTableManager tableManagerLicense = manager.getTableSpaceManager("herd").getTableManager("license");
        Table tableLicense = tableManagerLicense.getTable();
        System.out.println("TABLE PK: " + Arrays.toString(tableLicense.primaryKey));
        for (Column c : tableLicense.columns) {
            System.out.println("COL: " + c.name + " serialPos: " + c.serialPosition);
        }
        AbstractTableManager tableManagerCustomer = manager.getTableSpaceManager("herd").getTableManager("customer");
        Table tableCustomer = tableManagerCustomer.getTable();
        System.out.println("TABLE PK: " + Arrays.toString(tableCustomer.primaryKey));
        for (Column c : tableCustomer.columns) {
            System.out.println("COL: " + c.name + " serialPos: " + c.serialPosition);
        }
        {
            TranslatedQuery translated = manager.getPlanner().translate(tableSpace, "SELECT * FROM license", Collections.emptyList(), true, true, false, -1);
            System.out.println("TABLE CONTENTS");
            try (DataScanner scan1 = ((ScanResult) manager.executePlan(translated.plan, translated.context, TransactionContext.NO_TRANSACTION)).dataScanner) {
                List<DataAccessor> consume = scan1.consume();
                System.out.println("NUM " + consume.size());
                assertEquals(15, consume.size());
                for (DataAccessor r : consume) {
                    System.out.println("RECORD " + r.toMap());
                }
            }
        }
        {
            TranslatedQuery translated = manager.getPlanner().translate(tableSpace, "SELECT * FROM customer", Collections.emptyList(), true, true, false, -1);
            System.out.println("TABLE CONTENTS");
            try (DataScanner scan1 = ((ScanResult) manager.executePlan(translated.plan, translated.context, TransactionContext.NO_TRANSACTION)).dataScanner) {
                List<DataAccessor> consume = scan1.consume();
                System.out.println("NUM " + consume.size());
                assertEquals(7, consume.size());
                for (DataAccessor r : consume) {
                    System.out.println("RECORD " + r.toMap());
                }
            }
        }
        {
            TranslatedQuery translated = manager.getPlanner().translate(tableSpace, "SELECT t0.license_id,c.customer_id FROM license t0, customer c WHERE c.customer_id = 3 AND t0.customer_id = 3 AND c.customer_id = t0.customer_id\n" + "            ", Collections.emptyList(), true, true, false, -1);
            System.out.println("TABLE CONTENTS");
            try (DataScanner scan1 = ((ScanResult) manager.executePlan(translated.plan, translated.context, TransactionContext.NO_TRANSACTION)).dataScanner) {
                List<DataAccessor> consume = scan1.consume();
                System.out.println("NUM " + consume.size());
                assertEquals(9, consume.size());
                for (DataAccessor r : consume) {
                    System.out.println("RECORD " + r.toMap());
                }
            }
        }
        {
            TranslatedQuery translated = manager.getPlanner().translate(tableSpace, "SELECT * FROM license t0, customer c WHERE c.customer_id = t0.customer_id", Collections.emptyList(), true, true, false, -1);
            System.out.println("TABLE CONTENTS");
            try (DataScanner scan1 = ((ScanResult) manager.executePlan(translated.plan, translated.context, TransactionContext.NO_TRANSACTION)).dataScanner) {
                List<DataAccessor> consume = scan1.consume();
                System.out.println("NUM " + consume.size());
                assertEquals(15, consume.size());
                for (DataAccessor r : consume) {
                    System.out.println("RECORD " + r.toMap());
                }
            }
        }
    }
}

Example 13 with FileCommitLogManager

use of herddb.file.FileCommitLogManager in project herddb by diennea.

the class UpgradeFrom050WithBrinIndexesTest method test.

private void test(String file) throws Exception {
    File dbdatadir = folder.newFolder("dbdata050_" + file);
    try (InputStream in = UpgradeFrom050WithBrinIndexesTest.class.getResourceAsStream(file)) {
        ZIPUtils.unZip(in, dbdatadir);
    }
    final Path dbdata = dbdatadir.toPath().resolve("dbdata");
    Path metadataPath = dbdata.resolve("metadata");
    Path dataPath = dbdata.resolve("data");
    Path logsPath = dbdata.resolve("txlog");
    Path tmoDir = dbdata.resolve("tmp");
    assertTrue(Files.isDirectory(metadataPath));
    assertTrue(Files.isDirectory(dataPath));
    assertTrue(Files.isDirectory(logsPath));
    Path nodeid = dataPath.resolve("nodeid");
    assertTrue(Files.isRegularFile(nodeid));
    String id = new String(Files.readAllBytes(nodeid), StandardCharsets.UTF_8);
    System.out.println("id:" + id);
    String expectedNodeId = "capra";
    assertTrue(id.endsWith("\n" + expectedNodeId));
    try (DBManager manager = new DBManager(expectedNodeId, new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        final String tableSpace = "herd";
        final String tableName = "testtable";
        assertEquals(expectedNodeId, manager.getNodeId());
        assertTrue(manager.waitForTablespace(tableSpace, 10000));
        TableSpaceManager tableSpaceManager = manager.getTableSpaceManager(tableSpace);
        AbstractTableManager tableManager = tableSpaceManager.getTableManager(tableName);
        List<Index> indexes = tableManager.getAvailableIndexes();
        for (Index e : indexes) {
            System.out.println("INDEX: " + e);
            assertEquals(e.type, Index.TYPE_BRIN);
        }
        assertEquals(4, indexes.size());
        for (Column c : tableManager.getTable().getColumns()) {
            System.out.println("COLUMN :" + c);
        }
        {
            TranslatedQuery translated = manager.getPlanner().translate(tableSpace, "SELECT * FROM " + tableName + " ORDER BY pk,n1,n2", Collections.emptyList(), true, true, false, -1);
            ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
            System.out.println("TABLE CONTENTS");
            try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
                for (DataAccessor r : scan1.consume()) {
                    System.out.println("RECORD " + r.toMap());
                }
            }
        }
        test(new TestCase("SELECT * FROM " + tableSpace + "." + tableName + " WHERE n1=1", SecondaryIndexSeek.class, 4), manager, tableSpace);
        // this could be SecondaryIndexSeek but we have more indexes and the planner is not so smart
        test(new TestCase("SELECT * FROM " + tableSpace + "." + tableName + " WHERE n2=3", SecondaryIndexPrefixScan.class, 2), manager, tableSpace);
        test(new TestCase("SELECT * FROM " + tableSpace + "." + tableName + " WHERE n2>=3", SecondaryIndexRangeScan.class, 3), manager, tableSpace);
        test(new TestCase("SELECT * FROM " + tableSpace + "." + tableName + " WHERE n1=1 and n2=3", SecondaryIndexPrefixScan.class, 1), manager, tableSpace);
    }
}

Example 14 with FileCommitLogManager

use of herddb.file.FileCommitLogManager in project herddb by diennea.

the class LocalTableSnapshotRecoveryTest method test_1.

@Test
public void test_1() throws Exception {
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    String nodeId = "localhost";
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        manager.waitForTablespace(TableSpace.DEFAULT, 10000);
        Table table = Table.builder().tablespace(TableSpace.DEFAULT).name("t1").column("id", ColumnTypes.STRING).column("name", ColumnTypes.STRING).primaryKey("id").build();
        CreateTableStatement st2 = new CreateTableStatement(table);
        manager.executeStatement(st2, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        TestUtils.execute(manager, "INSERT INTO t1(id,name) values(?,?)", Arrays.asList("a", "b"));
        manager.getTableSpaceManager(TableSpace.DEFAULT).checkpoint(false, false, false);
        TestUtils.execute(manager, "INSERT INTO t1(id,name) values(?,?)", Arrays.asList("c", "d"));
        manager.getTableSpaceManager(TableSpace.DEFAULT).getTableManager("t1").flush();
        TestUtils.execute(manager, "INSERT INTO t1(id,name) values(?,?)", Arrays.asList("e", "f"));
        try (DataScanner scan = TestUtils.scan(manager, "SELECT * FROM t1", Collections.emptyList())) {
            assertEquals(3, scan.consume().size());
        }
    }
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        manager.waitForTablespace(TableSpace.DEFAULT, 10000);
        try (DataScanner scan = TestUtils.scan(manager, "SELECT * FROM t1", Collections.emptyList())) {
            assertEquals(3, scan.consume().size());
        }
    }
}

Example 15 with FileCommitLogManager

use of herddb.file.FileCommitLogManager in project herddb by diennea.

the class AutocheckPointTest method autoCheckPointTest.

@Test
public void autoCheckPointTest() throws Exception {
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    String nodeId = "localhost";
    try (DBManager manager = new DBManager(nodeId, new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        execute(manager, "CREATE TABLE tblspace1.tsql (K1 int ,s1 string,n1 int, primary key(k1))", Collections.emptyList());
        assertEquals(1, executeUpdate(manager, "INSERT INTO tblspace1.tsql(k1,s1,n1) values(?,?,?)", Arrays.asList(1, "a", Integer.valueOf(1234))).getUpdateCount());
        assertEquals(1, executeUpdate(manager, "INSERT INTO tblspace1.tsql(k1,s1,n1) values(?,?,?)", Arrays.asList(2, "a", Integer.valueOf(1234))).getUpdateCount());
        assertEquals(1, executeUpdate(manager, "INSERT INTO tblspace1.tsql(k1,s1,n1) values(?,?,?)", Arrays.asList(3, "a", Integer.valueOf(1234))).getUpdateCount());
        assertEquals(1, executeUpdate(manager, "INSERT INTO tblspace1.tsql(k1,s1,n1) values(?,?,?)", Arrays.asList(4, "a", Integer.valueOf(1234))).getUpdateCount());
        manager.checkpoint();
        long tx = ((TransactionResult) execute(manager, "EXECUTE begintransaction 'tblspace1'", Collections.emptyList())).getTransactionId();
        execute(manager, "UPDATE tblspace1.tsql set s1='b' where k1=1", Collections.emptyList(), new TransactionContext(tx));
        long lastCheckpont = manager.getLastCheckPointTs();
        try (DataScanner scan = scan(manager, "SELECT * FROM tblspace1.tsql WHERE N1=1234", Collections.emptyList(), new TransactionContext(tx))) {
            List<DataAccessor> data = scan.consume();
            assertEquals(4, data.size());
        }
        manager.setCheckpointPeriod(1000);
        for (int i = 0; i < 100; i++) {
            if (lastCheckpont != manager.getLastCheckPointTs()) {
                break;
            }
            Thread.sleep(100);
        }
        assertNotEquals(lastCheckpont, manager.getLastCheckPointTs());
        execute(manager, "EXECUTE committransaction 'tblspace1'," + tx, Collections.emptyList());
        assertEquals(1, scan(manager, "SELECT * FROM tblspace1.tsql WHERE N1=1234 and s1='b'", Collections.emptyList()).consumeAndClose().size());
    }
}