Examples with Catalog io.prestosql.metadata.Catalog used on opensource projects

Example 1 with Catalog

use of io.prestosql.metadata.Catalog in project hetu-core by openlookeng.

the class TestTransactionManager method registerConnector.

private static void registerConnector(CatalogManager catalogManager, TransactionManager transactionManager, String catalogName, CatalogName catalog, Connector connector) {
    CatalogName systemId = createSystemTablesCatalogName(catalog);
    InternalNodeManager nodeManager = new InMemoryNodeManager();
    Metadata metadata = createTestMetadataManager(catalogManager);
    catalogManager.registerCatalog(new Catalog(catalogName, catalog, connector, createInformationSchemaCatalogName(catalog), new InformationSchemaConnector(catalogName, nodeManager, metadata, new AllowAllAccessControl()), systemId, new SystemConnector(nodeManager, connector.getSystemTables(), transactionId -> transactionManager.getConnectorTransaction(transactionId, catalog))));
}

Example 2 with Catalog

use of io.prestosql.metadata.Catalog in project hetu-core by openlookeng.

the class TestCarbonAutoVacuum method testAutoVacuum.

@Test
public void testAutoVacuum() throws SQLException {
    hetuServer.execute("drop table if exists carbontestdb1.autovacuumtable1");
    hetuServer.execute("drop table if exists carbontestdb1.autovacuumtable2");
    hetuServer.execute("drop table if exists carbontestdb1.autovacuumtable3");
    hetuServer.execute("drop table if exists carbontestdb1.autovacuumtable4");
    hetuServer.execute("CREATE TABLE carbontestdb1.autovacuumtable1(a int, b int)");
    hetuServer.execute("INSERT INTO carbontestdb1.autovacuumtable1 VALUES (10, 11)");
    hetuServer.execute("CREATE TABLE carbontestdb1.autovacuumtable2(a int, b int)");
    hetuServer.execute("INSERT INTO carbontestdb1.autovacuumtable2 VALUES (10, 11)");
    hetuServer.execute("INSERT INTO carbontestdb1.autovacuumtable2 VALUES (20, 11)");
    hetuServer.execute("INSERT INTO carbontestdb1.autovacuumtable2 VALUES (30, 11)");
    hetuServer.execute("INSERT INTO carbontestdb1.autovacuumtable2 VALUES (30, 11)");
    hetuServer.execute("CREATE TABLE carbontestdb1.autovacuumtable3 as select a, b from  carbontestdb1.autovacuumtable2 ");
    hetuServer.execute("CREATE TABLE carbontestdb1.autovacuumtable4(a int, b int)");
    hetuServer.execute("INSERT INTO carbontestdb1.autovacuumtable4 VALUES (10, 11)");
    hetuServer.execute("INSERT INTO carbontestdb1.autovacuumtable4 VALUES (20, 11)");
    hetuServer.execute("INSERT INTO carbontestdb1.autovacuumtable4 VALUES (30, 12)");
    hetuServer.execute("INSERT INTO carbontestdb1.autovacuumtable4 VALUES (40, 13)");
    CatalogManager catalogManager = hetuServer.getCatalog();
    List<Catalog> catalogs = catalogManager.getCatalogs();
    Catalog catalog = null;
    for (Catalog catalog1 : catalogs) {
        if (catalog1.getConnectorCatalogName().getCatalogName().equals("carbondata")) {
            catalog = catalog1;
            break;
        }
    }
    Connector connector;
    ConnectorMetadata connectorMetadata = null;
    List<ConnectorVacuumTableInfo> tables;
    List<String> tablesNames = new ArrayList<>();
    try {
        CarbondataAutoVacuumThread.enableTracingVacuumTask(true);
        assertNotNull(catalog);
        connector = catalog.getConnector(catalog.getConnectorCatalogName());
        connectorMetadata = connector.getConnectorMetadata();
        connectorMetadata.getTablesForVacuum();
    } catch (Exception e) {
        logger.debug(e.getMessage());
    }
    CarbondataAutoVacuumThread.waitForSubmittedVacuumTasksFinish();
    CarbondataAutoVacuumThread.enableTracingVacuumTask(false);
    tables = connectorMetadata.getTablesForVacuum();
    if (tables != null && 0 != tables.size()) {
        for (ConnectorVacuumTableInfo vacuumTable : tables) {
            tablesNames.add(vacuumTable.getSchemaTableName());
        }
    }
    assertTrue(tablesNames.contains("carbontestdb1.autovacuumtable4"));
    assertTrue(tablesNames.contains("carbontestdb1.autovacuumtable2"));
    hetuServer.execute("drop table if exists carbontestdb1.autovacuumtable1");
    hetuServer.execute("drop table if exists carbontestdb1.autovacuumtable2");
    hetuServer.execute("drop table if exists carbontestdb1.autovacuumtable3");
    hetuServer.execute("drop table if exists carbontestdb1.autovacuumtable4");
}

Example 3 with Catalog

use of io.prestosql.metadata.Catalog in project hetu-core by openlookeng.

the class AutoVacuumScanner method startVacuum.

private void startVacuum(Catalog catalog, String vacuumTable, boolean isFull) {
    String catalogNameVacuumTable = catalog.getCatalogName() + "." + vacuumTable;
    if (vacuumInProgressMap.containsKey(catalogNameVacuumTable)) {
        log.debug("return Present in vacuumInProgressMap %s ", catalogNameVacuumTable);
        return;
    }
    long attempts = 0;
    QueryId queryId = dispatchManager.createQueryId();
    String slug = "x" + randomUUID().toString().toLowerCase(ENGLISH).replace("-", "");
    String vacuumQuery;
    if (isFull) {
        vacuumQuery = "vacuum table " + catalogNameVacuumTable + "  full";
    } else {
        vacuumQuery = "vacuum table " + catalogNameVacuumTable;
    }
    Session.SessionBuilder sessionBuilder = Session.builder(sessionPropertyManager).setQueryId(queryId).setIdentity(new Identity("openLooKeng", Optional.empty())).setSource("auto-vacuum");
    Session session = sessionBuilder.build();
    AutoVacuumSessionContext sessionContext = new AutoVacuumSessionContext(session);
    vacuumInProgressMap.put(catalogNameVacuumTable, System.currentTimeMillis());
    log.debug("Query.create queryId %s  catalogNameVacuumTable: %s ", queryId.toString(), catalogNameVacuumTable);
    ListenableFuture<?> lf = waitForDispatched(queryId, slug, sessionContext, vacuumQuery);
    Futures.addCallback(lf, new FutureCallback<Object>() {

        @Override
        public void onSuccess(@Nullable Object result) {
            try {
                DispatchQuery dispatchQuery = dispatchManager.getQuery(queryId);
                dispatchQuery.addStateChangeListener((state) -> {
                    Query query = getQuery(queryId, slug);
                    if ((null != query) && (!dispatchManager.getQueryInfo(queryId).getState().isDone())) {
                        query.waitForResults(attempts, Duration.valueOf("1s"), DataSize.valueOf("1MB"));
                    }
                    if (state.isDone()) {
                        log.debug("STATUS  %s QueryID %s Query %s", state.name(), queryId.toString(), vacuumQuery);
                        vacuumInProgressMap.remove(catalogNameVacuumTable);
                    }
                });
            } catch (Throwable e) {
                vacuumInProgressMap.remove(catalogNameVacuumTable);
                log.error("Filed to execute vacuum for table %s QueryID %s", catalogNameVacuumTable, queryId.toString(), e.getMessage());
            }
        }

        @Override
        public void onFailure(Throwable t) {
            vacuumInProgressMap.remove(catalogNameVacuumTable);
            log.error("Query %s request to start vacuum scan failed at queryId[%s]: %s ", vacuumQuery, queryId, t.getMessage());
        }
    }, directExecutor());
}

Example 4 with Catalog

use of io.prestosql.metadata.Catalog in project hetu-core by openlookeng.

the class TestAccessControlManager method registerBogusConnector.

private static CatalogName registerBogusConnector(CatalogManager catalogManager, TransactionManager transactionManager, AccessControl accessControl, String catalogName) {
    CatalogName catalog = new CatalogName(catalogName);
    Connector connector = new TpchConnectorFactory().create(catalogName, ImmutableMap.of(), new TestingConnectorContext());
    InMemoryNodeManager nodeManager = new InMemoryNodeManager();
    Metadata metadata = createTestMetadataManager(catalogManager);
    CatalogName systemId = createSystemTablesCatalogName(catalog);
    catalogManager.registerCatalog(new Catalog(catalogName, catalog, connector, createInformationSchemaCatalogName(catalog), new InformationSchemaConnector(catalogName, nodeManager, metadata, accessControl), systemId, new SystemConnector(nodeManager, connector.getSystemTables(), transactionId -> transactionManager.getConnectorTransaction(transactionId, catalog))));
    return catalog;
}

Example 5 with Catalog

use of io.prestosql.metadata.Catalog in project hetu-core by openlookeng.

the class ConnectorManager method updateConnectorIds.

/**
 * update the catalogs this node own in the service announcer.
 */
public synchronized void updateConnectorIds() {
    // get existing announcement
    ServiceAnnouncement announcement = getPrestoAnnouncement(announcer.getServiceAnnouncements());
    Set<String> connectorIds = new LinkedHashSet<>();
    Set<String> allConnectorIds = new LinkedHashSet<>();
    // automatically build connectorIds if not configured
    List<Catalog> catalogs = catalogManager.getCatalogs();
    // add data center names to connectorIds, then NodeScheduler can schedule task to this node.
    List<String> dataCenterNames = catalogConnectorStore.getDataCenterNames();
    // if this is a dedicated coordinator, only add jmx
    if (serverConfig.isCoordinator() && !schedulerConfig.isIncludeCoordinator()) {
        catalogs.stream().map(Catalog::getConnectorCatalogName).filter(connectorId -> connectorId.getCatalogName().equals("jmx")).map(Object::toString).forEach(connectorIds::add);
    } else {
        catalogs.stream().map(Catalog::getConnectorCatalogName).map(Object::toString).forEach(connectorIds::add);
        dataCenterNames.stream().forEach(connectorIds::add);
    }
    catalogs.stream().map(Catalog::getConnectorCatalogName).map(Object::toString).forEach(allConnectorIds::add);
    dataCenterNames.stream().forEach(allConnectorIds::add);
    // build announcement with updated sources
    Map<String, String> properties = new HashMap<>();
    properties.putAll(announcement.getProperties());
    properties.put("connectorIds", Joiner.on(",").join(connectorIds));
    properties.put("allConnectorIds", Joiner.on(",").join(allConnectorIds));
    // update announcement
    try {
        Field propertiesField = announcement.getClass().getDeclaredField("properties");
        propertiesField.setAccessible(true);
        propertiesField.set(announcement, properties);
    } catch (NoSuchFieldException | IllegalAccessException ex) {
        log.error(ex, "Set announcement properties failed");
        throw new RuntimeException("Set announcement properties failed", ex);
    }
    announcer.addServiceAnnouncement(announcement);
    announcer.forceAnnounce();
    nodeManager.refreshNodes();
    log.info("Announce connector ids, ACTIVE %s, All %s", connectorIds, allConnectorIds);
}