Examples with FileDataStorageManager herddb.file.FileDataStorageManager used on opensource projects

Example 11 with FileDataStorageManager

use of herddb.file.FileDataStorageManager in project herddb by diennea.

the class IndexCreationTest method recoverTableAndIndexWithCheckpoint.

private void recoverTableAndIndexWithCheckpoint(String indexType) throws Exception {
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    String nodeId = "localhost";
    Table table;
    Index index;
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath, 64 * 1024 * 1024), tmoDir, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.INTEGER).column("name", ColumnTypes.STRING).primaryKey("id").build();
        manager.executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.executeStatement(new InsertStatement("tblspace1", table.name, RecordSerializer.makeRecord(table, "id", 1, "name", "uno")), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        GetResult result = manager.get(new GetStatement("tblspace1", table.name, Bytes.from_int(1), null, false), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertTrue(result.found());
        manager.checkpoint();
    }
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath, 64 * 1024 * 1024), tmoDir, null)) {
        manager.start();
        manager.waitForTablespace("tblspace1", 10000);
        index = Index.builder().onTable(table).column("name", ColumnTypes.STRING).type(indexType).build();
        manager.executeStatement(new CreateIndexStatement(index), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        /* Access through index  */
        TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE name=\'uno\'", Collections.emptyList(), true, true, false, -1);
        ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
        assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexSeek);
        try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
            assertEquals(1, scan1.consume().size());
        }
    }
}

Example 12 with FileDataStorageManager

use of herddb.file.FileDataStorageManager in project herddb by diennea.

the class RunHerdDB070Test method test.

@Test
public void test() throws Exception {
    String file = "herddb.070.joinerror.zip";
    File dbdatadir = folder.newFolder("dbdata070_" + file);
    try (InputStream in = RunHerdDB070Test.class.getResourceAsStream(file)) {
        ZIPUtils.unZip(in, dbdatadir);
    }
    System.out.println("UNZIPPED TO " + dbdatadir);
    final Path dbdata = dbdatadir.toPath().resolve("herddb.070.joinerror").resolve("dbdata");
    Path metadataPath = dbdata.resolve("metadata");
    Path dataPath = dbdata.resolve("data");
    Path logsPath = dbdata.resolve("txlog");
    Path tmoDir = dbdata.resolve("tmp");
    assertTrue(Files.isDirectory(metadataPath));
    assertTrue(Files.isDirectory(dataPath));
    assertTrue(Files.isDirectory(logsPath));
    Path nodeid = dataPath.resolve("nodeid");
    assertTrue(Files.isRegularFile(nodeid));
    String id = new String(Files.readAllBytes(nodeid), StandardCharsets.UTF_8);
    System.out.println("id:" + id);
    String expectedNodeId = "asino";
    assertTrue(id.endsWith("\n" + expectedNodeId));
    try (DBManager manager = new DBManager(expectedNodeId, new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        final String tableSpace = "herd";
        final String tableName = "testtable";
        assertEquals(expectedNodeId, manager.getNodeId());
        assertTrue(manager.waitForTablespace(tableSpace, 10000));
        AbstractTableManager tableManagerLicense = manager.getTableSpaceManager("herd").getTableManager("license");
        Table tableLicense = tableManagerLicense.getTable();
        System.out.println("TABLE PK: " + Arrays.toString(tableLicense.primaryKey));
        for (Column c : tableLicense.columns) {
            System.out.println("COL: " + c.name + " serialPos: " + c.serialPosition);
        }
        AbstractTableManager tableManagerCustomer = manager.getTableSpaceManager("herd").getTableManager("customer");
        Table tableCustomer = tableManagerCustomer.getTable();
        System.out.println("TABLE PK: " + Arrays.toString(tableCustomer.primaryKey));
        for (Column c : tableCustomer.columns) {
            System.out.println("COL: " + c.name + " serialPos: " + c.serialPosition);
        }
        {
            TranslatedQuery translated = manager.getPlanner().translate(tableSpace, "SELECT * FROM license", Collections.emptyList(), true, true, false, -1);
            System.out.println("TABLE CONTENTS");
            try (DataScanner scan1 = ((ScanResult) manager.executePlan(translated.plan, translated.context, TransactionContext.NO_TRANSACTION)).dataScanner) {
                List<DataAccessor> consume = scan1.consume();
                System.out.println("NUM " + consume.size());
                assertEquals(15, consume.size());
                for (DataAccessor r : consume) {
                    System.out.println("RECORD " + r.toMap());
                }
            }
        }
        {
            TranslatedQuery translated = manager.getPlanner().translate(tableSpace, "SELECT * FROM customer", Collections.emptyList(), true, true, false, -1);
            System.out.println("TABLE CONTENTS");
            try (DataScanner scan1 = ((ScanResult) manager.executePlan(translated.plan, translated.context, TransactionContext.NO_TRANSACTION)).dataScanner) {
                List<DataAccessor> consume = scan1.consume();
                System.out.println("NUM " + consume.size());
                assertEquals(7, consume.size());
                for (DataAccessor r : consume) {
                    System.out.println("RECORD " + r.toMap());
                }
            }
        }
        {
            TranslatedQuery translated = manager.getPlanner().translate(tableSpace, "SELECT t0.license_id,c.customer_id FROM license t0, customer c WHERE c.customer_id = 3 AND t0.customer_id = 3 AND c.customer_id = t0.customer_id\n" + "            ", Collections.emptyList(), true, true, false, -1);
            System.out.println("TABLE CONTENTS");
            try (DataScanner scan1 = ((ScanResult) manager.executePlan(translated.plan, translated.context, TransactionContext.NO_TRANSACTION)).dataScanner) {
                List<DataAccessor> consume = scan1.consume();
                System.out.println("NUM " + consume.size());
                assertEquals(9, consume.size());
                for (DataAccessor r : consume) {
                    System.out.println("RECORD " + r.toMap());
                }
            }
        }
        {
            TranslatedQuery translated = manager.getPlanner().translate(tableSpace, "SELECT * FROM license t0, customer c WHERE c.customer_id = t0.customer_id", Collections.emptyList(), true, true, false, -1);
            System.out.println("TABLE CONTENTS");
            try (DataScanner scan1 = ((ScanResult) manager.executePlan(translated.plan, translated.context, TransactionContext.NO_TRANSACTION)).dataScanner) {
                List<DataAccessor> consume = scan1.consume();
                System.out.println("NUM " + consume.size());
                assertEquals(15, consume.size());
                for (DataAccessor r : consume) {
                    System.out.println("RECORD " + r.toMap());
                }
            }
        }
    }
}

Example 13 with FileDataStorageManager

use of herddb.file.FileDataStorageManager in project herddb by diennea.

the class UpgradeFrom050WithBrinIndexesTest method test.

private void test(String file) throws Exception {
    File dbdatadir = folder.newFolder("dbdata050_" + file);
    try (InputStream in = UpgradeFrom050WithBrinIndexesTest.class.getResourceAsStream(file)) {
        ZIPUtils.unZip(in, dbdatadir);
    }
    final Path dbdata = dbdatadir.toPath().resolve("dbdata");
    Path metadataPath = dbdata.resolve("metadata");
    Path dataPath = dbdata.resolve("data");
    Path logsPath = dbdata.resolve("txlog");
    Path tmoDir = dbdata.resolve("tmp");
    assertTrue(Files.isDirectory(metadataPath));
    assertTrue(Files.isDirectory(dataPath));
    assertTrue(Files.isDirectory(logsPath));
    Path nodeid = dataPath.resolve("nodeid");
    assertTrue(Files.isRegularFile(nodeid));
    String id = new String(Files.readAllBytes(nodeid), StandardCharsets.UTF_8);
    System.out.println("id:" + id);
    String expectedNodeId = "capra";
    assertTrue(id.endsWith("\n" + expectedNodeId));
    try (DBManager manager = new DBManager(expectedNodeId, new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        final String tableSpace = "herd";
        final String tableName = "testtable";
        assertEquals(expectedNodeId, manager.getNodeId());
        assertTrue(manager.waitForTablespace(tableSpace, 10000));
        TableSpaceManager tableSpaceManager = manager.getTableSpaceManager(tableSpace);
        AbstractTableManager tableManager = tableSpaceManager.getTableManager(tableName);
        List<Index> indexes = tableManager.getAvailableIndexes();
        for (Index e : indexes) {
            System.out.println("INDEX: " + e);
            assertEquals(e.type, Index.TYPE_BRIN);
        }
        assertEquals(4, indexes.size());
        for (Column c : tableManager.getTable().getColumns()) {
            System.out.println("COLUMN :" + c);
        }
        {
            TranslatedQuery translated = manager.getPlanner().translate(tableSpace, "SELECT * FROM " + tableName + " ORDER BY pk,n1,n2", Collections.emptyList(), true, true, false, -1);
            ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
            System.out.println("TABLE CONTENTS");
            try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
                for (DataAccessor r : scan1.consume()) {
                    System.out.println("RECORD " + r.toMap());
                }
            }
        }
        test(new TestCase("SELECT * FROM " + tableSpace + "." + tableName + " WHERE n1=1", SecondaryIndexSeek.class, 4), manager, tableSpace);
        // this could be SecondaryIndexSeek but we have more indexes and the planner is not so smart
        test(new TestCase("SELECT * FROM " + tableSpace + "." + tableName + " WHERE n2=3", SecondaryIndexPrefixScan.class, 2), manager, tableSpace);
        test(new TestCase("SELECT * FROM " + tableSpace + "." + tableName + " WHERE n2>=3", SecondaryIndexRangeScan.class, 3), manager, tableSpace);
        test(new TestCase("SELECT * FROM " + tableSpace + "." + tableName + " WHERE n1=1 and n2=3", SecondaryIndexPrefixScan.class, 1), manager, tableSpace);
    }
}

Example 14 with FileDataStorageManager

use of herddb.file.FileDataStorageManager in project herddb by diennea.

the class RestartPendingTransactionBookKeeperTest method buildDBManager.

@Override
protected DBManager buildDBManager(String nodeId, Path metadataPath, Path dataPath, Path logsPath, Path tmoDir) throws Exception {
    ServerConfiguration serverconfig_1 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
    serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    ZookeeperMetadataStorageManager zk = new ZookeeperMetadataStorageManager(testEnv.getAddress(), testEnv.getTimeout(), testEnv.getPath());
    DBManager manager = new DBManager(nodeId, zk, new FileDataStorageManager(dataPath), new BookkeeperCommitLogManager(zk, serverconfig_1, NullStatsLogger.INSTANCE), tmoDir, null);
    return manager;
}

Example 15 with FileDataStorageManager

use of herddb.file.FileDataStorageManager in project herddb by diennea.

the class LocalTableSnapshotRecoveryTest method test_1.

@Test
public void test_1() throws Exception {
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    String nodeId = "localhost";
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        manager.waitForTablespace(TableSpace.DEFAULT, 10000);
        Table table = Table.builder().tablespace(TableSpace.DEFAULT).name("t1").column("id", ColumnTypes.STRING).column("name", ColumnTypes.STRING).primaryKey("id").build();
        CreateTableStatement st2 = new CreateTableStatement(table);
        manager.executeStatement(st2, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        TestUtils.execute(manager, "INSERT INTO t1(id,name) values(?,?)", Arrays.asList("a", "b"));
        manager.getTableSpaceManager(TableSpace.DEFAULT).checkpoint(false, false, false);
        TestUtils.execute(manager, "INSERT INTO t1(id,name) values(?,?)", Arrays.asList("c", "d"));
        manager.getTableSpaceManager(TableSpace.DEFAULT).getTableManager("t1").flush();
        TestUtils.execute(manager, "INSERT INTO t1(id,name) values(?,?)", Arrays.asList("e", "f"));
        try (DataScanner scan = TestUtils.scan(manager, "SELECT * FROM t1", Collections.emptyList())) {
            assertEquals(3, scan.consume().size());
        }
    }
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        manager.waitForTablespace(TableSpace.DEFAULT, 10000);
        try (DataScanner scan = TestUtils.scan(manager, "SELECT * FROM t1", Collections.emptyList())) {
            assertEquals(3, scan.consume().size());
        }
    }
}