Examples with FixedSplitSource com.facebook.presto.spi.FixedSplitSource used on opensource projects

Example 1 with FixedSplitSource

use of com.facebook.presto.spi.FixedSplitSource in project presto by prestodb.

the class AccumuloSplitManager method getSplits.

@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorTableLayoutHandle layout) {
    AccumuloTableLayoutHandle layoutHandle = (AccumuloTableLayoutHandle) layout;
    AccumuloTableHandle tableHandle = layoutHandle.getTable();
    String schemaName = tableHandle.getSchema();
    String tableName = tableHandle.getTable();
    String rowIdName = tableHandle.getRowId();
    // Get non-row ID column constraints
    List<AccumuloColumnConstraint> constraints = getColumnConstraints(rowIdName, layoutHandle.getConstraint());
    // Get the row domain column range
    Optional<Domain> rDom = getRangeDomain(rowIdName, layoutHandle.getConstraint());
    // Call out to our client to retrieve all tablet split metadata using the row ID domain and the secondary index
    List<TabletSplitMetadata> tabletSplits = client.getTabletSplits(session, schemaName, tableName, rDom, constraints, tableHandle.getSerializerInstance());
    // Pack the tablet split metadata into a connector split
    ImmutableList.Builder<ConnectorSplit> cSplits = ImmutableList.builder();
    for (TabletSplitMetadata splitMetadata : tabletSplits) {
        AccumuloSplit split = new AccumuloSplit(connectorId, schemaName, tableName, rowIdName, tableHandle.getSerializerClassName(), splitMetadata.getRanges().stream().map(WrappedRange::new).collect(Collectors.toList()), constraints, tableHandle.getScanAuthorizations(), splitMetadata.getHostPort());
        cSplits.add(split);
    }
    return new FixedSplitSource(cSplits.build());
}

Example 2 with FixedSplitSource

use of com.facebook.presto.spi.FixedSplitSource in project presto by prestodb.

the class AtopSplitManager method getSplits.

@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorTableLayoutHandle layoutHandle) {
    AtopTableLayoutHandle handle = (AtopTableLayoutHandle) layoutHandle;
    AtopTableHandle table = handle.getTableHandle();
    List<ConnectorSplit> splits = new ArrayList<>();
    ZonedDateTime end = ZonedDateTime.now(timeZone);
    for (Node node : nodeManager.getWorkerNodes()) {
        ZonedDateTime start = end.minusDays(maxHistoryDays - 1).withHour(0).withMinute(0).withSecond(0).withNano(0);
        while (start.isBefore(end)) {
            ZonedDateTime splitEnd = start.withHour(23).withMinute(59).withSecond(59).withNano(0);
            Domain splitDomain = Domain.create(ValueSet.ofRanges(Range.range(TIMESTAMP_WITH_TIME_ZONE, 1000 * start.toEpochSecond(), true, 1000 * splitEnd.toEpochSecond(), true)), false);
            if (handle.getStartTimeConstraint().overlaps(splitDomain) && handle.getEndTimeConstraint().overlaps(splitDomain)) {
                splits.add(new AtopSplit(table.getTable(), node.getHostAndPort(), start.toEpochSecond(), start.getZone()));
            }
            start = start.plusDays(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
        }
    }
    return new FixedSplitSource(splits);
}

Example 3 with FixedSplitSource

use of com.facebook.presto.spi.FixedSplitSource in project presto by prestodb.

the class HiveSplitManager method getSplits.

@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorTableLayoutHandle layoutHandle) {
    HiveTableLayoutHandle layout = (HiveTableLayoutHandle) layoutHandle;
    List<HivePartition> partitions = layout.getPartitions().get();
    HivePartition partition = Iterables.getFirst(partitions, null);
    if (partition == null) {
        return new FixedSplitSource(ImmutableList.of());
    }
    SchemaTableName tableName = partition.getTableName();
    List<HiveBucketing.HiveBucket> buckets = partition.getBuckets();
    Optional<HiveBucketHandle> bucketHandle = layout.getBucketHandle();
    // sort partitions
    partitions = Ordering.natural().onResultOf(HivePartition::getPartitionId).reverse().sortedCopy(partitions);
    SemiTransactionalHiveMetastore metastore = metastoreProvider.apply((HiveTransactionHandle) transaction);
    Optional<Table> table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName());
    if (!table.isPresent()) {
        throw new TableNotFoundException(tableName);
    }
    Iterable<HivePartitionMetadata> hivePartitions = getPartitionMetadata(metastore, table.get(), tableName, partitions, bucketHandle.map(HiveBucketHandle::toBucketProperty));
    HiveSplitLoader hiveSplitLoader = new BackgroundHiveSplitLoader(connectorId, table.get(), hivePartitions, bucketHandle, buckets, session, hdfsEnvironment, namenodeStats, directoryLister, executor, maxPartitionBatchSize, maxInitialSplits, recursiveDfsWalkerEnabled);
    HiveSplitSource splitSource = new HiveSplitSource(maxOutstandingSplits, hiveSplitLoader, executor);
    hiveSplitLoader.start(splitSource);
    return splitSource;
}

Example 4 with FixedSplitSource

use of com.facebook.presto.spi.FixedSplitSource in project presto by prestodb.

the class ExampleSplitManager method getSplits.

@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle handle, ConnectorSession session, ConnectorTableLayoutHandle layout) {
    ExampleTableLayoutHandle layoutHandle = (ExampleTableLayoutHandle) layout;
    ExampleTableHandle tableHandle = layoutHandle.getTable();
    ExampleTable table = exampleClient.getTable(tableHandle.getSchemaName(), tableHandle.getTableName());
    // this can happen if table is removed during a query
    checkState(table != null, "Table %s.%s no longer exists", tableHandle.getSchemaName(), tableHandle.getTableName());
    List<ConnectorSplit> splits = new ArrayList<>();
    for (URI uri : table.getSources()) {
        splits.add(new ExampleSplit(connectorId, tableHandle.getSchemaName(), tableHandle.getTableName(), uri));
    }
    Collections.shuffle(splits);
    return new FixedSplitSource(splits);
}

Example 5 with FixedSplitSource

use of com.facebook.presto.spi.FixedSplitSource in project presto by prestodb.

the class InformationSchemaSplitManager method getSplits.

@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorTableLayoutHandle layout) {
    InformationSchemaTableLayoutHandle handle = (InformationSchemaTableLayoutHandle) layout;
    Map<ColumnHandle, NullableValue> bindings = extractFixedValues(handle.getConstraint()).orElse(ImmutableMap.of());
    List<HostAddress> localAddress = ImmutableList.of(nodeManager.getCurrentNode().getHostAndPort());
    Map<String, NullableValue> filters = bindings.entrySet().stream().collect(toMap(entry -> ((InformationSchemaColumnHandle) entry.getKey()).getColumnName(), Entry::getValue));
    ConnectorSplit split = new InformationSchemaSplit(handle.getTable(), filters, localAddress);
    return new FixedSplitSource(ImmutableList.of(split));
}