Examples with FileDataStorageManager herddb.file.FileDataStorageManager used on opensource projects

Example 31 with FileDataStorageManager

use of herddb.file.FileDataStorageManager in project herddb by diennea.

the class ReplicatedLogtestcase method startDBManager.

protected DBManager startDBManager(String nodeId) throws Exception {
    File nodeDirectory = new File(folder.getRoot(), nodeId + "");
    nodeDirectory.mkdirs();
    Path path = nodeDirectory.toPath();
    ZookeeperMetadataStorageManager metadataStorageManager = new ZookeeperMetadataStorageManager(testEnv.getAddress(), testEnv.getTimeout(), testEnv.getPath());
    BookkeeperCommitLogManager commitLogManager = new BookkeeperCommitLogManager(metadataStorageManager, newServerConfigurationWithAutoPort(), new NullStatsLogger());
    FileDataStorageManager dataStorageManager = new FileDataStorageManager(path);
    System.setErr(System.out);
    DBManager manager = new DBManager(nodeId, metadataStorageManager, dataStorageManager, commitLogManager, folder.newFolder().toPath(), null);
    manager.start();
    return manager;
}

Example 32 with FileDataStorageManager

use of herddb.file.FileDataStorageManager in project herddb by diennea.

the class SimpleJoinTest method testExists.

@Test
public void testExists() throws Exception {
    String nodeId = "localhost";
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    // reproduces an error due to the order or elements returned during the scan
    try (DBManager manager = new DBManager(nodeId, new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        assumeThat(manager.getPlanner(), instanceOf(CalcitePlanner.class));
        manager.start();
        manager.waitForTablespace(TableSpace.DEFAULT, 10000);
        execute(manager, "CREATE TABLE a (k1 int primary key, v1 int)", Collections.emptyList());
        execute(manager, "CREATE TABLE b (k2 int primary key, v2 int)", Collections.emptyList());
        execute(manager, "INSERT INTO a (k1,v1) values(1,1)", Collections.emptyList());
        execute(manager, "INSERT INTO b (k2,v2) values(1,1)", Collections.emptyList());
        {
            try (DataScanner scan1 = scan(manager, " select k1 as kk, v1 as vv from a where 1=1 and ( NOT EXISTS (SELECT NULL FROM b WHERE b.v2=a.k1 AND  1=1  AND b.k2 = 2 ))", Collections.emptyList())) {
                List<DataAccessor> consume = scan1.consume();
                System.out.println("NUM " + consume.size());
                for (DataAccessor r : consume) {
                    System.out.println("RECORD " + r.toMap());
                }
                assertEquals(1, consume.size());
            }
        }
        {
            try (DataScanner scan1 = scan(manager, " select k1 as kk, v1 as vv from a where 1=1 and ( NOT EXISTS (SELECT NULL FROM b WHERE b.v2=a.k1 AND  1=1  AND b.k2 = 1 ))", Collections.emptyList())) {
                List<DataAccessor> consume = scan1.consume();
                System.out.println("NUM " + consume.size());
                for (DataAccessor r : consume) {
                    System.out.println("RECORD " + r.toMap());
                }
                assertEquals(0, consume.size());
            }
        }
        // joins during transactions must release transaction resources properly
        {
            long tx = TestUtils.beginTransaction(manager, TableSpace.DEFAULT);
            try (DataScanner scan1 = scan(manager, " select k1 as kk, v1 as vv from a where 1=1 and ( NOT EXISTS (SELECT NULL FROM b WHERE b.v2=a.k1 AND  1=1  AND b.k2 = 2 ))", Collections.emptyList(), new TransactionContext(tx))) {
                List<DataAccessor> consume = scan1.consume();
                System.out.println("NUM " + consume.size());
                for (DataAccessor r : consume) {
                    System.out.println("RECORD " + r.toMap());
                }
                assertEquals(1, consume.size());
            }
            TestUtils.commitTransaction(manager, TableSpace.DEFAULT, tx);
        }
        {
            long tx = TestUtils.beginTransaction(manager, TableSpace.DEFAULT);
            try (DataScanner scan1 = scan(manager, " select k1 as kk, v1 as vv from a where 1=1 and ( EXISTS (SELECT 1 FROM b WHERE b.v2=a.k1 AND b.k2 = 1 ))", Collections.emptyList(), new TransactionContext(tx))) {
                List<DataAccessor> consume = scan1.consume();
                System.out.println("NUM " + consume.size());
                for (DataAccessor r : consume) {
                    System.out.println("RECORD " + r.toMap());
                }
                assertEquals(1, consume.size());
            }
            TestUtils.commitTransaction(manager, TableSpace.DEFAULT, tx);
        }
    }
}

Example 33 with FileDataStorageManager

use of herddb.file.FileDataStorageManager in project herddb by diennea.

the class SimpleJoinTest method testJoinUnsortedKey.

@Test
public void testJoinUnsortedKey() throws Exception {
    String nodeId = "localhost";
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    // reproduces an error due to the order or elements returned during the scan
    try (DBManager manager = new DBManager(nodeId, new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        manager.waitForTablespace(TableSpace.DEFAULT, 10000);
        execute(manager, "CREATE TABLE customer (customer_id long primary key,contact_email string,contact_person string, creation long, deleted boolean, modification long, name string, vetting boolean)", Collections.emptyList());
        execute(manager, "CREATE TABLE license (license_id long primary key,application string, creation long,data string, deleted boolean, modification long,signature string, customer_id long , license_key_id string)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(1,1)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(2,1)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(3,3)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(4,4)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(5,3)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(6,3)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(7,3)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(8,3)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(9,3)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(10,3)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(11,3)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(12,5)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(13,6)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(14,7)", Collections.emptyList());
        execute(manager, "INSERT INTO license (license_id,customer_id) values(15,3)", Collections.emptyList());
        execute(manager, "INSERT INTO customer (customer_id) values(1)", Collections.emptyList());
        execute(manager, "INSERT INTO customer (customer_id) values(2)", Collections.emptyList());
        execute(manager, "INSERT INTO customer (customer_id) values(3)", Collections.emptyList());
        execute(manager, "INSERT INTO customer (customer_id) values(4)", Collections.emptyList());
        execute(manager, "INSERT INTO customer (customer_id) values(5)", Collections.emptyList());
        execute(manager, "INSERT INTO customer (customer_id) values(6)", Collections.emptyList());
        execute(manager, "INSERT INTO customer (customer_id) values(7)", Collections.emptyList());
        {
            try (DataScanner scan1 = scan(manager, "SELECT t0.license_id,c.customer_id FROM license t0, customer c WHERE c.customer_id = 3 AND t0.customer_id = 3 AND c.customer_id = t0.customer_id", Collections.emptyList())) {
                List<DataAccessor> consume = scan1.consumeAndClose();
                System.out.println("NUM " + consume.size());
                assertEquals(9, consume.size());
                for (DataAccessor r : consume) {
                    System.out.println("RECORD " + r.toMap());
                }
            }
        }
        // joins during transactions must release transaction resources properly
        {
            long tx = TestUtils.beginTransaction(manager, TableSpace.DEFAULT);
            try (DataScanner scan1 = scan(manager, "SELECT t0.license_id,c.customer_id FROM license t0, customer c WHERE c.customer_id = 3 AND t0.customer_id = 3 AND c.customer_id = t0.customer_id", Collections.emptyList(), new TransactionContext(tx))) {
                List<DataAccessor> consume = scan1.consume();
                System.out.println("NUM " + consume.size());
                assertEquals(9, consume.size());
                for (DataAccessor r : consume) {
                    System.out.println("RECORD " + r.toMap());
                }
            }
            TestUtils.commitTransaction(manager, TableSpace.DEFAULT, tx);
        }
    }
}

Example 34 with FileDataStorageManager

use of herddb.file.FileDataStorageManager in project herddb by diennea.

the class SimpleBrinIndexRecoveryTest method createRecoveryIndex_withrepeatedkey.

@Test
public void createRecoveryIndex_withrepeatedkey() throws Exception {
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    int id = 0;
    String nodeId = "localhost";
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertTrue(manager.waitForTablespace("tblspace1", 10000));
        Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.STRING).column("name", ColumnTypes.STRING).primaryKey("id").build();
        CreateTableStatement st2 = new CreateTableStatement(table);
        manager.executeStatement(st2, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        Index index = Index.builder().onTable(table).type(Index.TYPE_BRIN).column("name", ColumnTypes.STRING).build();
        CreateIndexStatement st3 = new CreateIndexStatement(index);
        manager.executeStatement(st3, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        BRINIndexManager brin = (BRINIndexManager) manager.getTableSpaceManager(table.tablespace).getIndexesOnTable(table.name).get(index.name);
        while (brin.getNumBlocks() < 2) {
            id++;
            TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values(?,?)", Arrays.asList(id, "my_repeatad_key"), TransactionContext.NO_TRANSACTION);
        }
        // some data on disk
        manager.checkpoint();
        // some data to be recovered from log
        while (brin.getNumBlocks() < 3) {
            id++;
            TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values(?,?)", Arrays.asList(id, "my_repeatad_key"), TransactionContext.NO_TRANSACTION);
        }
        try (DataScanner scan1 = scan(manager, "SELECT * FROM tblspace1.t1", Collections.emptyList())) {
            assertEquals(id, scan1.consume().size());
        }
    }
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        assertTrue(manager.waitForBootOfLocalTablespaces(10000));
        TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE name='my_repeatad_key'", Collections.emptyList(), true, true, false, -1);
        ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
        assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexSeek);
        try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
            assertEquals(id, scan1.consume().size());
        }
    }
}

Example 35 with FileDataStorageManager

use of herddb.file.FileDataStorageManager in project herddb by diennea.

the class SimpleBrinIndexRecoveryTest method createRecoveryIndex_withcheckpoint.

@Test
public void createRecoveryIndex_withcheckpoint() throws Exception {
    Path dataPath = folder.newFolder("data").toPath();
    Path logsPath = folder.newFolder("logs").toPath();
    Path metadataPath = folder.newFolder("metadata").toPath();
    Path tmoDir = folder.newFolder("tmoDir").toPath();
    String nodeId = "localhost";
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        assertTrue(manager.waitForTablespace("tblspace1", 10000));
        Table table = Table.builder().tablespace("tblspace1").name("t1").column("id", ColumnTypes.STRING).column("name", ColumnTypes.STRING).primaryKey("id").build();
        CreateTableStatement st2 = new CreateTableStatement(table);
        manager.executeStatement(st2, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        Index index = Index.builder().onTable(table).type(Index.TYPE_BRIN).column("name", ColumnTypes.STRING).build();
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('a','n1')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('b','n1')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('c','n1')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('d','n2')", Collections.emptyList());
        TestUtils.executeUpdate(manager, "INSERT INTO tblspace1.t1(id,name) values('e','n2')", Collections.emptyList());
        CreateIndexStatement st3 = new CreateIndexStatement(index);
        manager.executeStatement(st3, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE name='n1'", Collections.emptyList(), true, true, false, -1);
        ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
        assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexSeek);
        try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
            assertEquals(3, scan1.consume().size());
        }
        manager.checkpoint();
    }
    try (DBManager manager = new DBManager("localhost", new FileMetadataStorageManager(metadataPath), new FileDataStorageManager(dataPath), new FileCommitLogManager(logsPath), tmoDir, null)) {
        manager.start();
        assertTrue(manager.waitForBootOfLocalTablespaces(10000));
        TranslatedQuery translated = manager.getPlanner().translate(TableSpace.DEFAULT, "SELECT * FROM tblspace1.t1 WHERE name='n1'", Collections.emptyList(), true, true, false, -1);
        ScanStatement scan = translated.plan.mainStatement.unwrap(ScanStatement.class);
        assertTrue(scan.getPredicate().getIndexOperation() instanceof SecondaryIndexSeek);
        try (DataScanner scan1 = manager.scan(scan, translated.context, TransactionContext.NO_TRANSACTION)) {
            assertEquals(3, scan1.consume().size());
        }
    }
}