Examples with ConnectorVacuumTableInfo io.prestosql.spi.connector.ConnectorVacuumTableInfo used on opensource projects

Example 1 with ConnectorVacuumTableInfo

use of io.prestosql.spi.connector.ConnectorVacuumTableInfo in project hetu-core by openlookeng.

the class TestCarbonAutoVacuum method testAutoVacuum.

@Test
public void testAutoVacuum() throws SQLException {
    hetuServer.execute("drop table if exists carbontestdb1.autovacuumtable1");
    hetuServer.execute("drop table if exists carbontestdb1.autovacuumtable2");
    hetuServer.execute("drop table if exists carbontestdb1.autovacuumtable3");
    hetuServer.execute("drop table if exists carbontestdb1.autovacuumtable4");
    hetuServer.execute("CREATE TABLE carbontestdb1.autovacuumtable1(a int, b int)");
    hetuServer.execute("INSERT INTO carbontestdb1.autovacuumtable1 VALUES (10, 11)");
    hetuServer.execute("CREATE TABLE carbontestdb1.autovacuumtable2(a int, b int)");
    hetuServer.execute("INSERT INTO carbontestdb1.autovacuumtable2 VALUES (10, 11)");
    hetuServer.execute("INSERT INTO carbontestdb1.autovacuumtable2 VALUES (20, 11)");
    hetuServer.execute("INSERT INTO carbontestdb1.autovacuumtable2 VALUES (30, 11)");
    hetuServer.execute("INSERT INTO carbontestdb1.autovacuumtable2 VALUES (30, 11)");
    hetuServer.execute("CREATE TABLE carbontestdb1.autovacuumtable3 as select a, b from  carbontestdb1.autovacuumtable2 ");
    hetuServer.execute("CREATE TABLE carbontestdb1.autovacuumtable4(a int, b int)");
    hetuServer.execute("INSERT INTO carbontestdb1.autovacuumtable4 VALUES (10, 11)");
    hetuServer.execute("INSERT INTO carbontestdb1.autovacuumtable4 VALUES (20, 11)");
    hetuServer.execute("INSERT INTO carbontestdb1.autovacuumtable4 VALUES (30, 12)");
    hetuServer.execute("INSERT INTO carbontestdb1.autovacuumtable4 VALUES (40, 13)");
    CatalogManager catalogManager = hetuServer.getCatalog();
    List<Catalog> catalogs = catalogManager.getCatalogs();
    Catalog catalog = null;
    for (Catalog catalog1 : catalogs) {
        if (catalog1.getConnectorCatalogName().getCatalogName().equals("carbondata")) {
            catalog = catalog1;
            break;
        }
    }
    Connector connector;
    ConnectorMetadata connectorMetadata = null;
    List<ConnectorVacuumTableInfo> tables;
    List<String> tablesNames = new ArrayList<>();
    try {
        CarbondataAutoVacuumThread.enableTracingVacuumTask(true);
        assertNotNull(catalog);
        connector = catalog.getConnector(catalog.getConnectorCatalogName());
        connectorMetadata = connector.getConnectorMetadata();
        connectorMetadata.getTablesForVacuum();
    } catch (Exception e) {
        logger.debug(e.getMessage());
    }
    CarbondataAutoVacuumThread.waitForSubmittedVacuumTasksFinish();
    CarbondataAutoVacuumThread.enableTracingVacuumTask(false);
    tables = connectorMetadata.getTablesForVacuum();
    if (tables != null && 0 != tables.size()) {
        for (ConnectorVacuumTableInfo vacuumTable : tables) {
            tablesNames.add(vacuumTable.getSchemaTableName());
        }
    }
    assertTrue(tablesNames.contains("carbontestdb1.autovacuumtable4"));
    assertTrue(tablesNames.contains("carbontestdb1.autovacuumtable2"));
    hetuServer.execute("drop table if exists carbontestdb1.autovacuumtable1");
    hetuServer.execute("drop table if exists carbontestdb1.autovacuumtable2");
    hetuServer.execute("drop table if exists carbontestdb1.autovacuumtable3");
    hetuServer.execute("drop table if exists carbontestdb1.autovacuumtable4");
}

Example 2 with ConnectorVacuumTableInfo

use of io.prestosql.spi.connector.ConnectorVacuumTableInfo in project hetu-core by openlookeng.

the class TestCarbonAutoVacuum method testAutoVacuumDiffDB.

@Test
public void testAutoVacuumDiffDB() throws SQLException, InterruptedException {
    hetuServer.execute("drop table if exists carbontestdb1.autovacuumtable1");
    hetuServer.execute("drop table if exists carbontestdb2.autovacuumtable2");
    hetuServer.execute("drop table if exists carbontestdb3.autovacuumtable3");
    hetuServer.execute("drop table if exists carbontestdb4.autovacuumtable4");
    hetuServer.execute("CREATE TABLE carbontestdb1.autovacuumtable1(a int, b int)");
    hetuServer.execute("INSERT INTO carbontestdb1.autovacuumtable1 VALUES (10, 11)");
    hetuServer.execute("CREATE TABLE carbontestdb2.autovacuumtable2(a int, b int)");
    hetuServer.execute("INSERT INTO carbontestdb2.autovacuumtable2 VALUES (10, 11)");
    hetuServer.execute("INSERT INTO carbontestdb2.autovacuumtable2 VALUES (20, 11)");
    hetuServer.execute("INSERT INTO carbontestdb2.autovacuumtable2 VALUES (30, 11)");
    hetuServer.execute("INSERT INTO carbontestdb2.autovacuumtable2 VALUES (30, 11)");
    hetuServer.execute("CREATE TABLE carbontestdb3.autovacuumtable3 as select a, b from  carbontestdb2.autovacuumtable2 ");
    hetuServer.execute("CREATE TABLE carbontestdb4.autovacuumtable4(a int, b int)");
    hetuServer.execute("INSERT INTO carbontestdb4.autovacuumtable4 VALUES (10, 11)");
    hetuServer.execute("INSERT INTO carbontestdb4.autovacuumtable4 VALUES (20, 11)");
    hetuServer.execute("INSERT INTO carbontestdb4.autovacuumtable4 VALUES (30, 12)");
    hetuServer.execute("INSERT INTO carbontestdb4.autovacuumtable4 VALUES (40, 13)");
    CatalogManager catalogManager = hetuServer.getCatalog();
    List<Catalog> catalogs = catalogManager.getCatalogs();
    Catalog catalog = null;
    for (Catalog catalog1 : catalogs) {
        if (catalog1.getConnectorCatalogName().getCatalogName().equals("carbondata")) {
            catalog = catalog1;
            break;
        }
    }
    Connector connector;
    ConnectorMetadata connectorMetadata = null;
    List<ConnectorVacuumTableInfo> tables = null;
    List<String> tablesNames = new ArrayList<>();
    try {
        CarbondataAutoVacuumThread.enableTracingVacuumTask(true);
        connector = catalog.getConnector(catalog.getConnectorCatalogName());
        connectorMetadata = connector.getConnectorMetadata();
        connectorMetadata.getTablesForVacuum();
    } catch (Exception e) {
        logger.debug(e.getMessage());
    }
    CarbondataAutoVacuumThread.waitForSubmittedVacuumTasksFinish();
    CarbondataAutoVacuumThread.enableTracingVacuumTask(false);
    tables = connectorMetadata.getTablesForVacuum();
    if (tables != null && 0 != tables.size()) {
        for (ConnectorVacuumTableInfo vacuumTable : tables) {
            tablesNames.add(vacuumTable.getSchemaTableName());
        }
    }
    assertTrue(tablesNames.contains("carbontestdb4.autovacuumtable4"));
    assertTrue(tablesNames.contains("carbontestdb2.autovacuumtable2"));
    hetuServer.execute("drop table if exists carbontestdb1.autovacuumtable1");
    hetuServer.execute("drop table if exists carbontestdb2.autovacuumtable2");
    hetuServer.execute("drop table if exists carbontestdb3.autovacuumtable3");
    hetuServer.execute("drop table if exists carbontestdb4.autovacuumtable4");
}

Example 3 with ConnectorVacuumTableInfo

use of io.prestosql.spi.connector.ConnectorVacuumTableInfo in project hetu-core by openlookeng.

the class CarbondataAutoVacuumThread method identifyVacuumTables.

private void identifyVacuumTables(Table target, String schemaName, String tableName, String dbNameTableName) {
    Configuration initialConfiguration = ConfigurationUtils.toJobConf(hdfsEnvironment.getConfiguration(hdfsContext, new Path(target.getStorage().getLocation())));
    Properties schema = MetastoreUtil.getHiveSchema(target);
    schema.setProperty("tablePath", target.getStorage().getLocation());
    CarbonTable carbonTable;
    try {
        carbonTable = CarbondataMetadata.getCarbonTable(schemaName, tableName, schema, initialConfiguration, carbondataTableReader);
    } catch (RuntimeException e) {
        return;
    }
    try {
        if (null != carbonTable) {
            CarbonLoadModel carbonLoadModel = HiveCarbonUtil.getCarbonLoadModel(schema, initialConfiguration);
            // in carbondata we support only miner vacuum
            List<List<LoadMetadataDetails>> allGroupedSegList = CarbondataHetuCompactorUtil.identifyAndGroupSegmentsToBeMerged(carbonLoadModel, initialConfiguration, CompactionType.MINOR, majorVacuumSegSize, minorVacuumSegCount);
            if (!allGroupedSegList.isEmpty()) {
                ConnectorVacuumTableInfo connectorVacuumTableInfo = new ConnectorVacuumTableInfo(dbNameTableName, false);
                synchronized (connectorVacuumTableInfoList) {
                    // Selected Minor Vacuum tables are inserted in to list
                    connectorVacuumTableInfoList.add(connectorVacuumTableInfo);
                }
            }
        }
    } catch (RuntimeException e) {
        LOG.error("error when identify Vacuum Table " + schemaName + "." + tableName, e);
    // Ignore error cases and continue
    }
}

Example 4 with ConnectorVacuumTableInfo

use of io.prestosql.spi.connector.ConnectorVacuumTableInfo in project hetu-core by openlookeng.

the class VacuumEligibleTableCollector method addToVacuumTableList.

private synchronized void addToVacuumTableList(List<ConnectorVacuumTableInfo> tablesForVacuum) {
    for (ConnectorVacuumTableInfo tableInfo : tablesForVacuum) {
        if (!inProgressVacuums.containsKey(tableInfo.getSchemaTableName()) && !vacuumTableList.contains(tableInfo)) {
            inProgressVacuums.put(tableInfo.getSchemaTableName(), tableInfo);
            vacuumTableList.add(tableInfo);
        }
    }
}