Examples with ConnectorNodePartitioningProvider com.facebook.presto.spi.connector.ConnectorNodePartitioningProvider used on opensource projects

Example 1 with ConnectorNodePartitioningProvider

use of com.facebook.presto.spi.connector.ConnectorNodePartitioningProvider in project presto by prestodb.

the class NodePartitioningManager method getSplitToBucket.

private ToIntFunction<Split> getSplitToBucket(Session session, PartitioningHandle partitioningHandle) {
    ConnectorNodePartitioningProvider partitioningProvider = partitioningProviderManager.getPartitioningProvider(partitioningHandle.getConnectorId().get());
    ToIntFunction<ConnectorSplit> splitBucketFunction = partitioningProvider.getSplitBucketFunction(partitioningHandle.getTransactionHandle().orElse(null), session.toConnectorSession(), partitioningHandle.getConnectorHandle());
    checkArgument(splitBucketFunction != null, "No partitioning %s", partitioningHandle);
    return split -> {
        int bucket;
        if (split.getConnectorSplit() instanceof EmptySplit) {
            bucket = split.getLifespan().isTaskWide() ? 0 : split.getLifespan().getId();
        } else {
            bucket = splitBucketFunction.applyAsInt(split.getConnectorSplit());
        }
        if (!split.getLifespan().isTaskWide()) {
            checkArgument(split.getLifespan().getId() == bucket);
        }
        return bucket;
    };
}

Example 2 with ConnectorNodePartitioningProvider

use of com.facebook.presto.spi.connector.ConnectorNodePartitioningProvider in project presto by prestodb.

the class NodePartitioningManager method getConnectorBucketNodeMap.

private ConnectorBucketNodeMap getConnectorBucketNodeMap(Session session, PartitioningHandle partitioningHandle) {
    checkArgument(!(partitioningHandle.getConnectorHandle() instanceof SystemPartitioningHandle));
    ConnectorId connectorId = partitioningHandle.getConnectorId().orElseThrow(() -> new IllegalArgumentException("No connector ID for partitioning handle: " + partitioningHandle));
    List<Node> nodes = getNodes(session, connectorId);
    ConnectorNodePartitioningProvider partitioningProvider = partitioningProviderManager.getPartitioningProvider(partitioningHandle.getConnectorId().get());
    ConnectorBucketNodeMap connectorBucketNodeMap = partitioningProvider.getBucketNodeMap(partitioningHandle.getTransactionHandle().orElse(null), session.toConnectorSession(partitioningHandle.getConnectorId().get()), partitioningHandle.getConnectorHandle(), nodes);
    checkArgument(connectorBucketNodeMap != null, "No partition map %s", partitioningHandle);
    return connectorBucketNodeMap;
}

Example 3 with ConnectorNodePartitioningProvider

use of com.facebook.presto.spi.connector.ConnectorNodePartitioningProvider in project presto by prestodb.

the class NodePartitioningManager method getPartitionFunction.

public PartitionFunction getPartitionFunction(Session session, PartitioningScheme partitioningScheme, List<Type> partitionChannelTypes) {
    Optional<int[]> bucketToPartition = partitioningScheme.getBucketToPartition();
    checkArgument(bucketToPartition.isPresent(), "Bucket to partition must be set before a partition function can be created");
    PartitioningHandle partitioningHandle = partitioningScheme.getPartitioning().getHandle();
    BucketFunction bucketFunction;
    if (partitioningHandle.getConnectorHandle() instanceof SystemPartitioningHandle) {
        checkArgument(partitioningScheme.getBucketToPartition().isPresent(), "Bucket to partition must be set before a partition function can be created");
        return ((SystemPartitioningHandle) partitioningHandle.getConnectorHandle()).getPartitionFunction(partitionChannelTypes, partitioningScheme.getHashColumn().isPresent(), partitioningScheme.getBucketToPartition().get());
    } else {
        ConnectorNodePartitioningProvider partitioningProvider = partitioningProviderManager.getPartitioningProvider(partitioningHandle.getConnectorId().get());
        bucketFunction = partitioningProvider.getBucketFunction(partitioningHandle.getTransactionHandle().orElse(null), session.toConnectorSession(), partitioningHandle.getConnectorHandle(), partitionChannelTypes, bucketToPartition.get().length);
        checkArgument(bucketFunction != null, "No function %s", partitioningHandle);
    }
    return new BucketPartitionFunction(bucketFunction, partitioningScheme.getBucketToPartition().get());
}

Example 4 with ConnectorNodePartitioningProvider

use of com.facebook.presto.spi.connector.ConnectorNodePartitioningProvider in project presto by prestodb.

the class PartitioningProviderManager method getPartitioningProvider.

public ConnectorNodePartitioningProvider getPartitioningProvider(ConnectorId connectorId) {
    ConnectorNodePartitioningProvider partitioningProvider = partitioningProviders.get(connectorId);
    checkArgument(partitioningProvider != null, "No partitioning provider for connector %s", connectorId);
    return partitioningProvider;
}

Example 5 with ConnectorNodePartitioningProvider

use of com.facebook.presto.spi.connector.ConnectorNodePartitioningProvider in project presto by prestodb.

the class HiveConnectorFactory method create.

@Override
public Connector create(String catalogName, Map<String, String> config, ConnectorContext context) {
    requireNonNull(config, "config is null");
    try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(classLoader)) {
        Bootstrap app = new Bootstrap(new EventModule(), new MBeanModule(), new JsonModule(), new SmileModule(), new HiveClientModule(catalogName), new HiveS3Module(catalogName), new HiveGcsModule(), new HiveMetastoreModule(catalogName, metastore), new HiveSecurityModule(), new HiveAuthenticationModule(), new HiveProcedureModule(), new CachingModule(), binder -> {
            MBeanServer platformMBeanServer = ManagementFactory.getPlatformMBeanServer();
            binder.bind(MBeanServer.class).toInstance(new RebindSafeMBeanServer(platformMBeanServer));
            binder.bind(NodeVersion.class).toInstance(new NodeVersion(context.getNodeManager().getCurrentNode().getVersion()));
            binder.bind(NodeManager.class).toInstance(context.getNodeManager());
            binder.bind(TypeManager.class).toInstance(context.getTypeManager());
            binder.bind(PageIndexerFactory.class).toInstance(context.getPageIndexerFactory());
            binder.bind(PageSorter.class).toInstance(context.getPageSorter());
            binder.bind(StandardFunctionResolution.class).toInstance(context.getStandardFunctionResolution());
            binder.bind(FunctionMetadataManager.class).toInstance(context.getFunctionMetadataManager());
            binder.bind(RowExpressionService.class).toInstance(context.getRowExpressionService());
            binder.bind(FilterStatsCalculatorService.class).toInstance(context.getFilterStatsCalculatorService());
            binder.bind(BlockEncodingSerde.class).toInstance(context.getBlockEncodingSerde());
        });
        Injector injector = app.doNotInitializeLogging().setRequiredConfigurationProperties(config).quiet().initialize();
        LifeCycleManager lifeCycleManager = injector.getInstance(LifeCycleManager.class);
        HiveMetadataFactory metadataFactory = injector.getInstance(HiveMetadataFactory.class);
        HiveTransactionManager transactionManager = injector.getInstance(HiveTransactionManager.class);
        ConnectorSplitManager splitManager = injector.getInstance(ConnectorSplitManager.class);
        ConnectorPageSourceProvider connectorPageSource = injector.getInstance(ConnectorPageSourceProvider.class);
        ConnectorPageSinkProvider pageSinkProvider = injector.getInstance(ConnectorPageSinkProvider.class);
        ConnectorNodePartitioningProvider connectorDistributionProvider = injector.getInstance(ConnectorNodePartitioningProvider.class);
        HiveSessionProperties hiveSessionProperties = injector.getInstance(HiveSessionProperties.class);
        HiveTableProperties hiveTableProperties = injector.getInstance(HiveTableProperties.class);
        HiveAnalyzeProperties hiveAnalyzeProperties = injector.getInstance(HiveAnalyzeProperties.class);
        ConnectorAccessControl accessControl = new SystemTableAwareAccessControl(injector.getInstance(ConnectorAccessControl.class));
        Set<Procedure> procedures = injector.getInstance(Key.get(new TypeLiteral<Set<Procedure>>() {
        }));
        ConnectorPlanOptimizerProvider planOptimizerProvider = injector.getInstance(ConnectorPlanOptimizerProvider.class);
        ConnectorMetadataUpdaterProvider metadataUpdaterProvider = injector.getInstance(ConnectorMetadataUpdaterProvider.class);
        return new HiveConnector(lifeCycleManager, metadataFactory, transactionManager, new ClassLoaderSafeConnectorSplitManager(splitManager, classLoader), new ClassLoaderSafeConnectorPageSourceProvider(connectorPageSource, classLoader), new ClassLoaderSafeConnectorPageSinkProvider(pageSinkProvider, classLoader), new ClassLoaderSafeNodePartitioningProvider(connectorDistributionProvider, classLoader), ImmutableSet.of(), procedures, hiveSessionProperties.getSessionProperties(), HiveSchemaProperties.SCHEMA_PROPERTIES, hiveTableProperties.getTableProperties(), hiveAnalyzeProperties.getAnalyzeProperties(), accessControl, planOptimizerProvider, metadataUpdaterProvider, classLoader);
    } catch (Exception e) {
        throwIfUnchecked(e);
        throw new RuntimeException(e);
    }
}