Examples with ConnectorId com.facebook.presto.spi.ConnectorId used on opensource projects

Example 1 with ConnectorId

use of com.facebook.presto.spi.ConnectorId in project presto by prestodb.

the class RaptorQueryRunner method createSession.

public static Session createSession(String schema) {
    SessionPropertyManager sessionPropertyManager = new SessionPropertyManager();
    sessionPropertyManager.addConnectorSessionProperties(new ConnectorId("raptor"), new RaptorSessionProperties(new StorageManagerConfig()).getSessionProperties());
    return testSessionBuilder(sessionPropertyManager).setCatalog("raptor").setSchema(schema).setSystemProperty("enable_intermediate_aggregations", "true").build();
}

Example 2 with ConnectorId

use of com.facebook.presto.spi.ConnectorId in project presto by prestodb.

the class TestLocalQueries method createLocalQueryRunner.

public static LocalQueryRunner createLocalQueryRunner() {
    Session defaultSession = testSessionBuilder().setCatalog("local").setSchema(TINY_SCHEMA_NAME).setSystemProperty(PUSH_PARTIAL_AGGREGATION_THROUGH_JOIN, "true").build();
    LocalQueryRunner localQueryRunner = new LocalQueryRunner(defaultSession);
    // add the tpch catalog
    // local queries run directly against the generator
    localQueryRunner.createCatalog(defaultSession.getCatalog().get(), new TpchConnectorFactory(1), ImmutableMap.of());
    localQueryRunner.getMetadata().registerBuiltInFunctions(CUSTOM_FUNCTIONS);
    SessionPropertyManager sessionPropertyManager = localQueryRunner.getMetadata().getSessionPropertyManager();
    sessionPropertyManager.addSystemSessionProperties(TEST_SYSTEM_PROPERTIES);
    sessionPropertyManager.addConnectorSessionProperties(new ConnectorId(TESTING_CATALOG), TEST_CATALOG_PROPERTIES);
    return localQueryRunner;
}

Example 3 with ConnectorId

use of com.facebook.presto.spi.ConnectorId in project presto by prestodb.

the class TestMetadataManager method testGetTableStatisticsThrows.

@Test(expectedExceptions = UnsupportedOperationException.class)
public void testGetTableStatisticsThrows() {
    Session session = testSessionBuilder().setSystemProperty(IGNORE_STATS_CALCULATOR_FAILURES, "false").build();
    TableHandle tableHandle = new TableHandle(new ConnectorId("upper_case_schema_catalog"), new ConnectorTableHandle() {
    }, TestingTransactionHandle.create(), Optional.empty());
    TransactionBuilder.transaction(queryRunner.getTransactionManager(), queryRunner.getAccessControl()).execute(session, transactionSession -> {
        queryRunner.getMetadata().getCatalogHandle(transactionSession, "upper_case_schema_catalog");
        assertEquals(queryRunner.getMetadata().getTableStatistics(transactionSession, tableHandle, ImmutableList.of(), alwaysTrue()), TableStatistics.empty());
    });
}

Example 4 with ConnectorId

use of com.facebook.presto.spi.ConnectorId in project presto by prestodb.

the class PrestoSparkQueryExecutionFactory method checkPageSinkCommitIsSupported.

private void checkPageSinkCommitIsSupported(Session session, ExecutionWriterTarget writerTarget) {
    ConnectorId connectorId;
    if (writerTarget instanceof ExecutionWriterTarget.DeleteHandle) {
        throw new PrestoException(NOT_SUPPORTED, "delete queries are not supported by presto on spark");
    } else if (writerTarget instanceof ExecutionWriterTarget.CreateHandle) {
        connectorId = ((ExecutionWriterTarget.CreateHandle) writerTarget).getHandle().getConnectorId();
    } else if (writerTarget instanceof ExecutionWriterTarget.InsertHandle) {
        connectorId = ((ExecutionWriterTarget.InsertHandle) writerTarget).getHandle().getConnectorId();
    } else if (writerTarget instanceof ExecutionWriterTarget.RefreshMaterializedViewHandle) {
        connectorId = ((ExecutionWriterTarget.RefreshMaterializedViewHandle) writerTarget).getHandle().getConnectorId();
    } else {
        throw new IllegalArgumentException("unexpected writer target type: " + writerTarget.getClass());
    }
    verify(connectorId != null, "connectorId is null");
    Set<ConnectorCapabilities> connectorCapabilities = metadata.getConnectorCapabilities(session, connectorId);
    if (!connectorCapabilities.contains(SUPPORTS_PAGE_SINK_COMMIT)) {
        throw new PrestoException(NOT_SUPPORTED, "catalog does not support page sink commit: " + connectorId);
    }
}

Example 5 with ConnectorId

use of com.facebook.presto.spi.ConnectorId in project presto by prestodb.

the class DiscoveryNodeManager method refreshNodesInternal.

private synchronized void refreshNodesInternal() {
    // This is currently a blacklist.
    // TODO: make it a whitelist (a failure-detecting service selector) and maybe build in support for injecting this in airlift
    Set<ServiceDescriptor> services = serviceSelector.selectAllServices().stream().filter(service -> !failureDetector.getFailed().contains(service)).filter(service -> !nodeStatusService.isPresent() || nodeStatusService.get().isAllowed(service.getLocation()) || isCoordinator(service) || isResourceManager(service)).collect(toImmutableSet());
    ImmutableSet.Builder<InternalNode> activeNodesBuilder = ImmutableSortedSet.orderedBy(comparing(InternalNode::getNodeIdentifier));
    ImmutableSet.Builder<InternalNode> inactiveNodesBuilder = ImmutableSet.builder();
    ImmutableSet.Builder<InternalNode> shuttingDownNodesBuilder = ImmutableSet.builder();
    ImmutableSet.Builder<InternalNode> coordinatorsBuilder = ImmutableSet.builder();
    ImmutableSet.Builder<InternalNode> resourceManagersBuilder = ImmutableSet.builder();
    ImmutableSetMultimap.Builder<ConnectorId, InternalNode> byConnectorIdBuilder = ImmutableSetMultimap.builder();
    Map<String, InternalNode> nodes = new HashMap<>();
    SetMultimap<String, ConnectorId> connectorIdsByNodeId = HashMultimap.create();
    // For a given connectorId, sort the nodes based on their nodeIdentifier
    byConnectorIdBuilder.orderValuesBy(comparing(InternalNode::getNodeIdentifier));
    if (isMemoizeDeadNodesEnabled && this.nodes != null) {
        nodes.putAll(this.nodes);
    }
    if (isMemoizeDeadNodesEnabled && this.connectorIdsByNodeId != null) {
        connectorIdsByNodeId.putAll(this.connectorIdsByNodeId);
    }
    for (ServiceDescriptor service : services) {
        URI uri = getHttpUri(service, httpsRequired);
        OptionalInt thriftPort = getThriftServerPort(service);
        NodeVersion nodeVersion = getNodeVersion(service);
        // Currently, a node may have the roles of both a coordinator and a worker.  In the future, a resource manager may also
        // take the form of a coordinator, hence these flags are not exclusive.
        boolean coordinator = isCoordinator(service);
        boolean resourceManager = isResourceManager(service);
        if (uri != null && nodeVersion != null) {
            InternalNode node = new InternalNode(service.getNodeId(), uri, thriftPort, nodeVersion, coordinator, resourceManager, ALIVE);
            NodeState nodeState = getNodeState(node);
            switch(nodeState) {
                case ACTIVE:
                    activeNodesBuilder.add(node);
                    if (coordinator) {
                        coordinatorsBuilder.add(node);
                    }
                    if (resourceManager) {
                        resourceManagersBuilder.add(node);
                    }
                    nodes.put(node.getNodeIdentifier(), node);
                    // record available active nodes organized by connector id
                    String connectorIds = service.getProperties().get("connectorIds");
                    if (connectorIds != null) {
                        connectorIds = connectorIds.toLowerCase(ENGLISH);
                        for (String id : CONNECTOR_ID_SPLITTER.split(connectorIds)) {
                            ConnectorId connectorId = new ConnectorId(id);
                            byConnectorIdBuilder.put(connectorId, node);
                            connectorIdsByNodeId.put(node.getNodeIdentifier(), connectorId);
                        }
                    }
                    // always add system connector
                    byConnectorIdBuilder.put(new ConnectorId(GlobalSystemConnector.NAME), node);
                    break;
                case INACTIVE:
                    inactiveNodesBuilder.add(node);
                    break;
                case SHUTTING_DOWN:
                    shuttingDownNodesBuilder.add(node);
                    break;
                default:
                    log.error("Unknown state %s for node %s", nodeState, node);
            }
        }
    }
    if (allNodes != null) {
        // log node that are no longer active (but not shutting down)
        SetView<InternalNode> missingNodes = difference(allNodes.getActiveNodes(), Sets.union(activeNodesBuilder.build(), shuttingDownNodesBuilder.build()));
        for (InternalNode missingNode : missingNodes) {
            log.info("Previously active node is missing: %s (last seen at %s)", missingNode.getNodeIdentifier(), missingNode.getHost());
        }
    }
    // nodes by connector id changes anytime a node adds or removes a connector (note: this is not part of the listener system)
    activeNodesByConnectorId = byConnectorIdBuilder.build();
    if (isMemoizeDeadNodesEnabled) {
        SetView<String> deadNodeIds = difference(nodes.keySet(), activeNodesBuilder.build().stream().map(InternalNode::getNodeIdentifier).collect(toImmutableSet()));
        for (String nodeId : deadNodeIds) {
            InternalNode deadNode = nodes.get(nodeId);
            Set<ConnectorId> deadNodeConnectorIds = connectorIdsByNodeId.get(nodeId);
            for (ConnectorId id : deadNodeConnectorIds) {
                byConnectorIdBuilder.put(id, new InternalNode(deadNode.getNodeIdentifier(), deadNode.getInternalUri(), deadNode.getThriftPort(), deadNode.getNodeVersion(), deadNode.isCoordinator(), deadNode.isResourceManager(), DEAD));
            }
        }
    }
    this.nodes = ImmutableMap.copyOf(nodes);
    this.nodesByConnectorId = byConnectorIdBuilder.build();
    this.connectorIdsByNodeId = ImmutableSetMultimap.copyOf(connectorIdsByNodeId);
    AllNodes allNodes = new AllNodes(activeNodesBuilder.build(), inactiveNodesBuilder.build(), shuttingDownNodesBuilder.build(), coordinatorsBuilder.build(), resourceManagersBuilder.build());
    // only update if all nodes actually changed (note: this does not include the connectors registered with the nodes)
    if (!allNodes.equals(this.allNodes)) {
        // assign allNodes to a local variable for use in the callback below
        this.allNodes = allNodes;
        coordinators = coordinatorsBuilder.build();
        resourceManagers = resourceManagersBuilder.build();
        // notify listeners
        List<Consumer<AllNodes>> listeners = ImmutableList.copyOf(this.listeners);
        nodeStateEventExecutor.submit(() -> listeners.forEach(listener -> listener.accept(allNodes)));
    }
}