Examples with ConnectorSplit com.facebook.presto.spi.ConnectorSplit used on opensource projects

Example 1 with ConnectorSplit

use of com.facebook.presto.spi.ConnectorSplit in project presto by prestodb.

the class AccumuloSplitManager method getSplits.

@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorTableLayoutHandle layout) {
    AccumuloTableLayoutHandle layoutHandle = (AccumuloTableLayoutHandle) layout;
    AccumuloTableHandle tableHandle = layoutHandle.getTable();
    String schemaName = tableHandle.getSchema();
    String tableName = tableHandle.getTable();
    String rowIdName = tableHandle.getRowId();
    // Get non-row ID column constraints
    List<AccumuloColumnConstraint> constraints = getColumnConstraints(rowIdName, layoutHandle.getConstraint());
    // Get the row domain column range
    Optional<Domain> rDom = getRangeDomain(rowIdName, layoutHandle.getConstraint());
    // Call out to our client to retrieve all tablet split metadata using the row ID domain and the secondary index
    List<TabletSplitMetadata> tabletSplits = client.getTabletSplits(session, schemaName, tableName, rDom, constraints, tableHandle.getSerializerInstance());
    // Pack the tablet split metadata into a connector split
    ImmutableList.Builder<ConnectorSplit> cSplits = ImmutableList.builder();
    for (TabletSplitMetadata splitMetadata : tabletSplits) {
        AccumuloSplit split = new AccumuloSplit(connectorId, schemaName, tableName, rowIdName, tableHandle.getSerializerClassName(), splitMetadata.getRanges().stream().map(WrappedRange::new).collect(Collectors.toList()), constraints, tableHandle.getScanAuthorizations(), splitMetadata.getHostPort());
        cSplits.add(split);
    }
    return new FixedSplitSource(cSplits.build());
}

Example 2 with ConnectorSplit

use of com.facebook.presto.spi.ConnectorSplit in project presto by prestodb.

the class AtopSplitManager method getSplits.

@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorTableLayoutHandle layoutHandle) {
    AtopTableLayoutHandle handle = (AtopTableLayoutHandle) layoutHandle;
    AtopTableHandle table = handle.getTableHandle();
    List<ConnectorSplit> splits = new ArrayList<>();
    ZonedDateTime end = ZonedDateTime.now(timeZone);
    for (Node node : nodeManager.getWorkerNodes()) {
        ZonedDateTime start = end.minusDays(maxHistoryDays - 1).withHour(0).withMinute(0).withSecond(0).withNano(0);
        while (start.isBefore(end)) {
            ZonedDateTime splitEnd = start.withHour(23).withMinute(59).withSecond(59).withNano(0);
            Domain splitDomain = Domain.create(ValueSet.ofRanges(Range.range(TIMESTAMP_WITH_TIME_ZONE, 1000 * start.toEpochSecond(), true, 1000 * splitEnd.toEpochSecond(), true)), false);
            if (handle.getStartTimeConstraint().overlaps(splitDomain) && handle.getEndTimeConstraint().overlaps(splitDomain)) {
                splits.add(new AtopSplit(table.getTable(), node.getHostAndPort(), start.toEpochSecond(), start.getZone()));
            }
            start = start.plusDays(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
        }
    }
    return new FixedSplitSource(splits);
}

Example 3 with ConnectorSplit

use of com.facebook.presto.spi.ConnectorSplit in project presto by prestodb.

the class HiveSplitSource method addToQueue.

CompletableFuture<?> addToQueue(Iterator<? extends ConnectorSplit> splits) {
    CompletableFuture<?> lastResult = CompletableFuture.completedFuture(null);
    while (splits.hasNext()) {
        ConnectorSplit split = splits.next();
        lastResult = addToQueue(split);
    }
    return lastResult;
}

Example 4 with ConnectorSplit

use of com.facebook.presto.spi.ConnectorSplit in project presto by prestodb.

the class ExampleSplitManager method getSplits.

@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle handle, ConnectorSession session, ConnectorTableLayoutHandle layout) {
    ExampleTableLayoutHandle layoutHandle = (ExampleTableLayoutHandle) layout;
    ExampleTableHandle tableHandle = layoutHandle.getTable();
    ExampleTable table = exampleClient.getTable(tableHandle.getSchemaName(), tableHandle.getTableName());
    // this can happen if table is removed during a query
    checkState(table != null, "Table %s.%s no longer exists", tableHandle.getSchemaName(), tableHandle.getTableName());
    List<ConnectorSplit> splits = new ArrayList<>();
    for (URI uri : table.getSources()) {
        splits.add(new ExampleSplit(connectorId, tableHandle.getSchemaName(), tableHandle.getTableName(), uri));
    }
    Collections.shuffle(splits);
    return new FixedSplitSource(splits);
}

Example 5 with ConnectorSplit

use of com.facebook.presto.spi.ConnectorSplit in project presto by prestodb.

the class TestHiveSplitSource method testReaderWaitsForSplits.

@Test
public void testReaderWaitsForSplits() throws Exception {
    final HiveSplitSource hiveSplitSource = new HiveSplitSource(10, new TestingHiveSplitLoader(), Executors.newFixedThreadPool(5));
    final SettableFuture<ConnectorSplit> splits = SettableFuture.create();
    // create a thread that will get a split
    final CountDownLatch started = new CountDownLatch(1);
    Thread getterThread = new Thread(new Runnable() {

        @Override
        public void run() {
            try {
                started.countDown();
                List<ConnectorSplit> batch = getFutureValue(hiveSplitSource.getNextBatch(1));
                assertEquals(batch.size(), 1);
                splits.set(batch.get(0));
            } catch (Throwable e) {
                splits.setException(e);
            }
        }
    });
    getterThread.start();
    try {
        // wait for the thread to be started
        assertTrue(started.await(1, TimeUnit.SECONDS));
        // sleep for a bit, and assure the thread is blocked
        TimeUnit.MILLISECONDS.sleep(200);
        assertTrue(!splits.isDone());
        // add a split
        hiveSplitSource.addToQueue(new TestSplit(33));
        // wait for thread to get the split
        ConnectorSplit split = splits.get(200, TimeUnit.MILLISECONDS);
        assertSame(split.getInfo(), 33);
    } finally {
        // make sure the thread exits
        getterThread.interrupt();
    }
}