Examples with ConnectorSplitSource io.prestosql.spi.connector.ConnectorSplitSource used on opensource projects

Example 21 with ConnectorSplitSource

use of io.prestosql.spi.connector.ConnectorSplitSource in project incubator-pulsar by apache.

the class TestPulsarSplitManager method testGetSplitNonSchema.

@Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true)
public void testGetSplitNonSchema(String delimiter) throws Exception {
    updateRewriteNamespaceDelimiterIfNeeded(delimiter);
    TopicName topicName = NON_SCHEMA_TOPIC;
    setup();
    log.info("!----- topic: %s -----!", topicName);
    PulsarTableHandle pulsarTableHandle = new PulsarTableHandle(pulsarConnectorId.toString(), topicName.getNamespace(), topicName.getLocalName(), topicName.getLocalName());
    Map<ColumnHandle, Domain> domainMap = new HashMap<>();
    TupleDomain<ColumnHandle> tupleDomain = TupleDomain.withColumnDomains(domainMap);
    PulsarTableLayoutHandle pulsarTableLayoutHandle = new PulsarTableLayoutHandle(pulsarTableHandle, tupleDomain);
    ConnectorSplitSource connectorSplitSource = this.pulsarSplitManager.getSplits(mock(ConnectorTransactionHandle.class), mock(ConnectorSession.class), pulsarTableLayoutHandle, null);
    assertNotNull(connectorSplitSource);
}

Example 22 with ConnectorSplitSource

use of io.prestosql.spi.connector.ConnectorSplitSource in project incubator-pulsar by apache.

the class TestPulsarSplitManager method testTopic.

@Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true)
public void testTopic(String delimiter) throws Exception {
    updateRewriteNamespaceDelimiterIfNeeded(delimiter);
    List<TopicName> topics = new LinkedList<>();
    topics.addAll(topicNames.stream().filter(topicName -> !topicName.equals(NON_SCHEMA_TOPIC)).collect(Collectors.toList()));
    for (TopicName topicName : topics) {
        setup();
        log.info("!----- topic: %s -----!", topicName);
        PulsarTableHandle pulsarTableHandle = new PulsarTableHandle(pulsarConnectorId.toString(), topicName.getNamespace(), topicName.getLocalName(), topicName.getLocalName());
        PulsarTableLayoutHandle pulsarTableLayoutHandle = new PulsarTableLayoutHandle(pulsarTableHandle, TupleDomain.all());
        final ResultCaptor<Collection<PulsarSplit>> resultCaptor = new ResultCaptor<>();
        doAnswer(resultCaptor).when(this.pulsarSplitManager).getSplitsNonPartitionedTopic(anyInt(), any(), any(), any(), any(), any());
        ConnectorSplitSource connectorSplitSource = this.pulsarSplitManager.getSplits(mock(ConnectorTransactionHandle.class), mock(ConnectorSession.class), pulsarTableLayoutHandle, null);
        verify(this.pulsarSplitManager, times(1)).getSplitsNonPartitionedTopic(anyInt(), any(), any(), any(), any(), any());
        int totalSize = 0;
        for (PulsarSplit pulsarSplit : resultCaptor.getResult()) {
            assertEquals(pulsarSplit.getConnectorId(), pulsarConnectorId.toString());
            assertEquals(pulsarSplit.getSchemaName(), topicName.getNamespace());
            assertEquals(pulsarSplit.getTableName(), topicName.getLocalName());
            assertEquals(pulsarSplit.getSchema(), new String(topicsToSchemas.get(topicName.getSchemaName()).getSchema()));
            assertEquals(pulsarSplit.getSchemaType(), topicsToSchemas.get(topicName.getSchemaName()).getType());
            assertEquals(pulsarSplit.getStartPositionEntryId(), totalSize);
            assertEquals(pulsarSplit.getStartPositionLedgerId(), 0);
            assertEquals(pulsarSplit.getStartPosition(), PositionImpl.get(0, totalSize));
            assertEquals(pulsarSplit.getEndPositionLedgerId(), 0);
            assertEquals(pulsarSplit.getEndPositionEntryId(), totalSize + pulsarSplit.getSplitSize());
            assertEquals(pulsarSplit.getEndPosition(), PositionImpl.get(0, totalSize + pulsarSplit.getSplitSize()));
            totalSize += pulsarSplit.getSplitSize();
        }
        assertEquals(totalSize, topicsToNumEntries.get(topicName.getSchemaName()).intValue());
        cleanup();
    }
}

Example 23 with ConnectorSplitSource

use of io.prestosql.spi.connector.ConnectorSplitSource in project pulsar by yahoo.

the class TestPulsarSplitManager method testTopic.

@Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true)
public void testTopic(String delimiter) throws Exception {
    updateRewriteNamespaceDelimiterIfNeeded(delimiter);
    List<TopicName> topics = new LinkedList<>();
    topics.addAll(topicNames.stream().filter(topicName -> !topicName.equals(NON_SCHEMA_TOPIC)).collect(Collectors.toList()));
    for (TopicName topicName : topics) {
        setup();
        log.info("!----- topic: %s -----!", topicName);
        PulsarTableHandle pulsarTableHandle = new PulsarTableHandle(pulsarConnectorId.toString(), topicName.getNamespace(), topicName.getLocalName(), topicName.getLocalName());
        PulsarTableLayoutHandle pulsarTableLayoutHandle = new PulsarTableLayoutHandle(pulsarTableHandle, TupleDomain.all());
        final ResultCaptor<Collection<PulsarSplit>> resultCaptor = new ResultCaptor<>();
        doAnswer(resultCaptor).when(this.pulsarSplitManager).getSplitsNonPartitionedTopic(anyInt(), any(), any(), any(), any(), any());
        ConnectorSplitSource connectorSplitSource = this.pulsarSplitManager.getSplits(mock(ConnectorTransactionHandle.class), mock(ConnectorSession.class), pulsarTableLayoutHandle, null);
        verify(this.pulsarSplitManager, times(1)).getSplitsNonPartitionedTopic(anyInt(), any(), any(), any(), any(), any());
        int totalSize = 0;
        for (PulsarSplit pulsarSplit : resultCaptor.getResult()) {
            assertEquals(pulsarSplit.getConnectorId(), pulsarConnectorId.toString());
            assertEquals(pulsarSplit.getSchemaName(), topicName.getNamespace());
            assertEquals(pulsarSplit.getTableName(), topicName.getLocalName());
            assertEquals(pulsarSplit.getSchema(), new String(topicsToSchemas.get(topicName.getSchemaName()).getSchema()));
            assertEquals(pulsarSplit.getSchemaType(), topicsToSchemas.get(topicName.getSchemaName()).getType());
            assertEquals(pulsarSplit.getStartPositionEntryId(), totalSize);
            assertEquals(pulsarSplit.getStartPositionLedgerId(), 0);
            assertEquals(pulsarSplit.getStartPosition(), PositionImpl.get(0, totalSize));
            assertEquals(pulsarSplit.getEndPositionLedgerId(), 0);
            assertEquals(pulsarSplit.getEndPositionEntryId(), totalSize + pulsarSplit.getSplitSize());
            assertEquals(pulsarSplit.getEndPosition(), PositionImpl.get(0, totalSize + pulsarSplit.getSplitSize()));
            totalSize += pulsarSplit.getSplitSize();
        }
        assertEquals(totalSize, topicsToNumEntries.get(topicName.getSchemaName()).intValue());
        cleanup();
    }
}

Example 24 with ConnectorSplitSource

use of io.prestosql.spi.connector.ConnectorSplitSource in project pulsar by yahoo.

the class TestPulsarSplitManager method testPublishTimePredicatePushdown.

@Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true)
public void testPublishTimePredicatePushdown(String delimiter) throws Exception {
    updateRewriteNamespaceDelimiterIfNeeded(delimiter);
    TopicName topicName = TOPIC_1;
    setup();
    log.info("!----- topic: %s -----!", topicName);
    PulsarTableHandle pulsarTableHandle = new PulsarTableHandle(pulsarConnectorId.toString(), topicName.getNamespace(), topicName.getLocalName(), topicName.getLocalName());
    Map<ColumnHandle, Domain> domainMap = new HashMap<>();
    Domain domain = Domain.create(ValueSet.ofRanges(Range.range(TIMESTAMP, currentTimeMs + 1L, true, currentTimeMs + 50L, true)), false);
    domainMap.put(PulsarInternalColumn.PUBLISH_TIME.getColumnHandle(pulsarConnectorId.toString(), false), domain);
    TupleDomain<ColumnHandle> tupleDomain = TupleDomain.withColumnDomains(domainMap);
    PulsarTableLayoutHandle pulsarTableLayoutHandle = new PulsarTableLayoutHandle(pulsarTableHandle, tupleDomain);
    final ResultCaptor<Collection<PulsarSplit>> resultCaptor = new ResultCaptor<>();
    doAnswer(resultCaptor).when(this.pulsarSplitManager).getSplitsNonPartitionedTopic(anyInt(), any(), any(), any(), any(), any());
    ConnectorSplitSource connectorSplitSource = this.pulsarSplitManager.getSplits(mock(ConnectorTransactionHandle.class), mock(ConnectorSession.class), pulsarTableLayoutHandle, null);
    verify(this.pulsarSplitManager, times(1)).getSplitsNonPartitionedTopic(anyInt(), any(), any(), any(), any(), any());
    int totalSize = 0;
    int initalStart = 1;
    for (PulsarSplit pulsarSplit : resultCaptor.getResult()) {
        assertEquals(pulsarSplit.getConnectorId(), pulsarConnectorId.toString());
        assertEquals(pulsarSplit.getSchemaName(), topicName.getNamespace());
        assertEquals(pulsarSplit.getTableName(), topicName.getLocalName());
        assertEquals(pulsarSplit.getSchema(), new String(topicsToSchemas.get(topicName.getSchemaName()).getSchema()));
        assertEquals(pulsarSplit.getSchemaType(), topicsToSchemas.get(topicName.getSchemaName()).getType());
        assertEquals(pulsarSplit.getStartPositionEntryId(), initalStart);
        assertEquals(pulsarSplit.getStartPositionLedgerId(), 0);
        assertEquals(pulsarSplit.getStartPosition(), PositionImpl.get(0, initalStart));
        assertEquals(pulsarSplit.getEndPositionLedgerId(), 0);
        assertEquals(pulsarSplit.getEndPositionEntryId(), initalStart + pulsarSplit.getSplitSize());
        assertEquals(pulsarSplit.getEndPosition(), PositionImpl.get(0, initalStart + pulsarSplit.getSplitSize()));
        initalStart += pulsarSplit.getSplitSize();
        totalSize += pulsarSplit.getSplitSize();
    }
    assertEquals(totalSize, 49);
}

Example 25 with ConnectorSplitSource

use of io.prestosql.spi.connector.ConnectorSplitSource in project pulsar by apache.

the class TestPulsarSplitManager method testGetSplitNonSchema.

@Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true)
public void testGetSplitNonSchema(String delimiter) throws Exception {
    updateRewriteNamespaceDelimiterIfNeeded(delimiter);
    TopicName topicName = NON_SCHEMA_TOPIC;
    setup();
    log.info("!----- topic: %s -----!", topicName);
    PulsarTableHandle pulsarTableHandle = new PulsarTableHandle(pulsarConnectorId.toString(), topicName.getNamespace(), topicName.getLocalName(), topicName.getLocalName());
    Map<ColumnHandle, Domain> domainMap = new HashMap<>();
    TupleDomain<ColumnHandle> tupleDomain = TupleDomain.withColumnDomains(domainMap);
    PulsarTableLayoutHandle pulsarTableLayoutHandle = new PulsarTableLayoutHandle(pulsarTableHandle, tupleDomain);
    ConnectorSplitSource connectorSplitSource = this.pulsarSplitManager.getSplits(mock(ConnectorTransactionHandle.class), mock(ConnectorSession.class), pulsarTableLayoutHandle, null);
    assertNotNull(connectorSplitSource);
}