Examples with TestingConnectorContext io.prestosql.testing.TestingConnectorContext used on opensource projects

Example 16 with TestingConnectorContext

use of io.prestosql.testing.TestingConnectorContext in project incubator-pulsar by apache.

the class TestCacheSizeAllocator method cacheSizeAllocatorTest.

@Test(dataProvider = "cacheSizeProvider", timeOut = 1000 * 20)
public void cacheSizeAllocatorTest(long entryQueueSizeBytes) throws Exception {
    TopicName topicName = TopicName.get("public/default/cache-size-" + entryQueueSizeBytes + "test_" + +RandomUtils.nextInt());
    int totalMsgCnt = 1000;
    MessageIdImpl firstMessageId = prepareData(topicName, totalMsgCnt);
    ReadOnlyCursor readOnlyCursor = pulsar.getManagedLedgerFactory().openReadOnlyCursor(topicName.getPersistenceNamingEncoding(), PositionImpl.get(firstMessageId.getLedgerId(), firstMessageId.getEntryId()), new ManagedLedgerConfig());
    readOnlyCursor.skipEntries(totalMsgCnt);
    PositionImpl lastPosition = (PositionImpl) readOnlyCursor.getReadPosition();
    ObjectMapper objectMapper = new ObjectMapper();
    PulsarSplit pulsarSplit = new PulsarSplit(0, "connector-id", topicName.getNamespace(), topicName.getLocalName(), topicName.getLocalName(), totalMsgCnt, new String(Schema.BYTES.getSchemaInfo().getSchema()), Schema.BYTES.getSchemaInfo().getType(), firstMessageId.getEntryId(), lastPosition.getEntryId(), firstMessageId.getLedgerId(), lastPosition.getLedgerId(), TupleDomain.all(), objectMapper.writeValueAsString(new HashMap<>()), null);
    List<PulsarColumnHandle> pulsarColumnHandles = TestPulsarConnector.getColumnColumnHandles(topicName, Schema.BYTES.getSchemaInfo(), PulsarColumnHandle.HandleKeyValueType.NONE, true);
    PulsarConnectorConfig connectorConfig = new PulsarConnectorConfig();
    connectorConfig.setMaxSplitQueueSizeBytes(entryQueueSizeBytes);
    ConnectorContext prestoConnectorContext = new TestingConnectorContext();
    PulsarRecordCursor pulsarRecordCursor = new PulsarRecordCursor(pulsarColumnHandles, pulsarSplit, connectorConfig, pulsar.getManagedLedgerFactory(), new ManagedLedgerConfig(), new PulsarConnectorMetricsTracker(new NullStatsProvider()), new PulsarDispatchingRowDecoderFactory(prestoConnectorContext.getTypeManager()));
    Class<PulsarRecordCursor> recordCursorClass = PulsarRecordCursor.class;
    Field entryQueueField = recordCursorClass.getDeclaredField("entryQueue");
    entryQueueField.setAccessible(true);
    SpscArrayQueue<Entry> entryQueue = (SpscArrayQueue<Entry>) entryQueueField.get(pulsarRecordCursor);
    Field messageQueueField = recordCursorClass.getDeclaredField("messageQueue");
    messageQueueField.setAccessible(true);
    SpscArrayQueue<RawMessageImpl> messageQueue = (SpscArrayQueue<RawMessageImpl>) messageQueueField.get(pulsarRecordCursor);
    long maxQueueSize = 0;
    if (entryQueueSizeBytes == -1) {
        maxQueueSize = Long.MAX_VALUE;
    } else if (entryQueueSizeBytes == 0) {
        maxQueueSize = 1;
    } else if (entryQueueSizeBytes > 0) {
        maxQueueSize = entryQueueSizeBytes / 2 / singleEntrySize + 1;
    }
    int receiveCnt = 0;
    while (receiveCnt != totalMsgCnt) {
        if (pulsarRecordCursor.advanceNextPosition()) {
            receiveCnt++;
        }
        Assert.assertTrue(entryQueue.size() <= maxQueueSize);
        Assert.assertTrue(messageQueue.size() <= maxQueueSize);
    }
}

Example 17 with TestingConnectorContext

use of io.prestosql.testing.TestingConnectorContext in project incubator-pulsar by apache.

the class TestPulsarConnector method mockColumnMetadata.

public static PulsarMetadata mockColumnMetadata() {
    ConnectorContext prestoConnectorContext = new TestingConnectorContext();
    PulsarConnectorConfig pulsarConnectorConfig = spy(PulsarConnectorConfig.class);
    pulsarConnectorConfig.setMaxEntryReadBatchSize(1);
    pulsarConnectorConfig.setMaxSplitEntryQueueSize(10);
    pulsarConnectorConfig.setMaxSplitMessageQueueSize(100);
    PulsarDispatchingRowDecoderFactory dispatchingRowDecoderFactory = new PulsarDispatchingRowDecoderFactory(prestoConnectorContext.getTypeManager());
    PulsarAuth pulsarAuth = new PulsarAuth(pulsarConnectorConfig);
    PulsarMetadata pulsarMetadata = new PulsarMetadata(pulsarConnectorId, pulsarConnectorConfig, dispatchingRowDecoderFactory, pulsarAuth);
    return pulsarMetadata;
}

Example 18 with TestingConnectorContext

use of io.prestosql.testing.TestingConnectorContext in project incubator-pulsar by apache.

the class TestReadChunkedMessages method queryTest.

@Test
public void queryTest() throws Exception {
    String topic = "chunk-topic";
    TopicName topicName = TopicName.get(topic);
    int messageCnt = 20;
    Set<MovieMessage> messageSet = prepareChunkedData(topic, messageCnt);
    SchemaInfo schemaInfo = Schema.AVRO(Movie.class).getSchemaInfo();
    PulsarConnectorConfig connectorConfig = new PulsarConnectorConfig();
    connectorConfig.setWebServiceUrl(pulsar.getWebServiceAddress());
    PulsarSplitManager pulsarSplitManager = new PulsarSplitManager(new PulsarConnectorId("1"), connectorConfig);
    Collection<PulsarSplit> splits = pulsarSplitManager.getSplitsForTopic(topicName.getPersistenceNamingEncoding(), pulsar.getManagedLedgerFactory(), new ManagedLedgerConfig(), 3, new PulsarTableHandle("1", topicName.getNamespace(), topic, topic), schemaInfo, topic, TupleDomain.all(), null);
    List<PulsarColumnHandle> columnHandleList = TestPulsarConnector.getColumnColumnHandles(topicName, schemaInfo, PulsarColumnHandle.HandleKeyValueType.NONE, true);
    ConnectorContext prestoConnectorContext = new TestingConnectorContext();
    for (PulsarSplit split : splits) {
        queryAndCheck(columnHandleList, split, connectorConfig, prestoConnectorContext, messageSet);
    }
    Assert.assertTrue(messageSet.isEmpty());
}

Example 19 with TestingConnectorContext

use of io.prestosql.testing.TestingConnectorContext in project incubator-pulsar by apache.

the class AbstractDecoderTester method init.

protected void init() {
    ConnectorContext prestoConnectorContext = new TestingConnectorContext();
    this.decoderFactory = new PulsarDispatchingRowDecoderFactory(prestoConnectorContext.getTypeManager());
    this.pulsarConnectorConfig = spy(PulsarConnectorConfig.class);
    this.pulsarConnectorConfig.setMaxEntryReadBatchSize(1);
    this.pulsarConnectorConfig.setMaxSplitEntryQueueSize(10);
    this.pulsarConnectorConfig.setMaxSplitMessageQueueSize(100);
    this.pulsarMetadata = new PulsarMetadata(pulsarConnectorId, this.pulsarConnectorConfig, decoderFactory, new PulsarAuth(this.pulsarConnectorConfig));
    this.topicName = TopicName.get("persistent", NamespaceName.get("tenant-1", "ns-1"), "topic-1");
}

Example 20 with TestingConnectorContext

use of io.prestosql.testing.TestingConnectorContext in project pulsar by yahoo.

the class TestCacheSizeAllocator method cacheSizeAllocatorTest.

@Test(dataProvider = "cacheSizeProvider", timeOut = 1000 * 20)
public void cacheSizeAllocatorTest(long entryQueueSizeBytes) throws Exception {
    TopicName topicName = TopicName.get("public/default/cache-size-" + entryQueueSizeBytes + "test_" + +RandomUtils.nextInt());
    int totalMsgCnt = 1000;
    MessageIdImpl firstMessageId = prepareData(topicName, totalMsgCnt);
    ReadOnlyCursor readOnlyCursor = pulsar.getManagedLedgerFactory().openReadOnlyCursor(topicName.getPersistenceNamingEncoding(), PositionImpl.get(firstMessageId.getLedgerId(), firstMessageId.getEntryId()), new ManagedLedgerConfig());
    readOnlyCursor.skipEntries(totalMsgCnt);
    PositionImpl lastPosition = (PositionImpl) readOnlyCursor.getReadPosition();
    ObjectMapper objectMapper = new ObjectMapper();
    PulsarSplit pulsarSplit = new PulsarSplit(0, "connector-id", topicName.getNamespace(), topicName.getLocalName(), topicName.getLocalName(), totalMsgCnt, new String(Schema.BYTES.getSchemaInfo().getSchema()), Schema.BYTES.getSchemaInfo().getType(), firstMessageId.getEntryId(), lastPosition.getEntryId(), firstMessageId.getLedgerId(), lastPosition.getLedgerId(), TupleDomain.all(), objectMapper.writeValueAsString(new HashMap<>()), null);
    List<PulsarColumnHandle> pulsarColumnHandles = TestPulsarConnector.getColumnColumnHandles(topicName, Schema.BYTES.getSchemaInfo(), PulsarColumnHandle.HandleKeyValueType.NONE, true);
    PulsarConnectorConfig connectorConfig = new PulsarConnectorConfig();
    connectorConfig.setMaxSplitQueueSizeBytes(entryQueueSizeBytes);
    ConnectorContext prestoConnectorContext = new TestingConnectorContext();
    PulsarRecordCursor pulsarRecordCursor = new PulsarRecordCursor(pulsarColumnHandles, pulsarSplit, connectorConfig, pulsar.getManagedLedgerFactory(), new ManagedLedgerConfig(), new PulsarConnectorMetricsTracker(new NullStatsProvider()), new PulsarDispatchingRowDecoderFactory(prestoConnectorContext.getTypeManager()));
    Class<PulsarRecordCursor> recordCursorClass = PulsarRecordCursor.class;
    Field entryQueueField = recordCursorClass.getDeclaredField("entryQueue");
    entryQueueField.setAccessible(true);
    SpscArrayQueue<Entry> entryQueue = (SpscArrayQueue<Entry>) entryQueueField.get(pulsarRecordCursor);
    Field messageQueueField = recordCursorClass.getDeclaredField("messageQueue");
    messageQueueField.setAccessible(true);
    SpscArrayQueue<RawMessageImpl> messageQueue = (SpscArrayQueue<RawMessageImpl>) messageQueueField.get(pulsarRecordCursor);
    long maxQueueSize = 0;
    if (entryQueueSizeBytes == -1) {
        maxQueueSize = Long.MAX_VALUE;
    } else if (entryQueueSizeBytes == 0) {
        maxQueueSize = 1;
    } else if (entryQueueSizeBytes > 0) {
        maxQueueSize = entryQueueSizeBytes / 2 / singleEntrySize + 1;
    }
    int receiveCnt = 0;
    while (receiveCnt != totalMsgCnt) {
        if (pulsarRecordCursor.advanceNextPosition()) {
            receiveCnt++;
        }
        Assert.assertTrue(entryQueue.size() <= maxQueueSize);
        Assert.assertTrue(messageQueue.size() <= maxQueueSize);
    }
}