Examples with ReadOnlyCursorImpl org.apache.bookkeeper.mledger.impl.ReadOnlyCursorImpl used on opensource projects

Example 1 with ReadOnlyCursorImpl

use of org.apache.bookkeeper.mledger.impl.ReadOnlyCursorImpl in project pulsar by apache.

the class TestPulsarConnector method setup.

@BeforeMethod
public void setup() throws Exception {
    this.pulsarConnectorConfig = spy(PulsarConnectorConfig.class);
    this.pulsarConnectorConfig.setMaxEntryReadBatchSize(1);
    this.pulsarConnectorConfig.setMaxSplitEntryQueueSize(10);
    this.pulsarConnectorConfig.setMaxSplitMessageQueueSize(100);
    Tenants tenants = mock(Tenants.class);
    doReturn(new LinkedList<>(topicNames.stream().map(TopicName::getTenant).collect(Collectors.toSet()))).when(tenants).getTenants();
    Namespaces namespaces = mock(Namespaces.class);
    when(namespaces.getNamespaces(anyString())).thenAnswer(new Answer<List<String>>() {

        @Override
        public List<String> answer(InvocationOnMock invocation) throws Throwable {
            Object[] args = invocation.getArguments();
            String tenant = (String) args[0];
            List<String> ns = getNamespace(tenant);
            if (ns.isEmpty()) {
                ClientErrorException cee = new ClientErrorException(Response.status(404).build());
                throw new PulsarAdminException(cee, cee.getMessage(), cee.getResponse().getStatus());
            }
            return ns;
        }
    });
    Topics topics = mock(Topics.class);
    when(topics.getList(anyString(), any())).thenAnswer(new Answer<List<String>>() {

        @Override
        public List<String> answer(InvocationOnMock invocationOnMock) throws Throwable {
            Object[] args = invocationOnMock.getArguments();
            String ns = (String) args[0];
            List<String> topics = getTopics(ns);
            if (topics.isEmpty()) {
                ClientErrorException cee = new ClientErrorException(Response.status(404).build());
                throw new PulsarAdminException(cee, cee.getMessage(), cee.getResponse().getStatus());
            }
            return topics;
        }
    });
    when(topics.getPartitionedTopicList(anyString())).thenAnswer(new Answer<List<String>>() {

        @Override
        public List<String> answer(InvocationOnMock invocationOnMock) throws Throwable {
            Object[] args = invocationOnMock.getArguments();
            String ns = (String) args[0];
            List<String> topics = getPartitionedTopics(ns);
            if (topics.isEmpty()) {
                ClientErrorException cee = new ClientErrorException(Response.status(404).build());
                throw new PulsarAdminException(cee, cee.getMessage(), cee.getResponse().getStatus());
            }
            return topics;
        }
    });
    when(topics.getPartitionedTopicMetadata(anyString())).thenAnswer(new Answer<PartitionedTopicMetadata>() {

        @Override
        public PartitionedTopicMetadata answer(InvocationOnMock invocationOnMock) throws Throwable {
            Object[] args = invocationOnMock.getArguments();
            String topic = (String) args[0];
            int partitions = partitionedTopicsToPartitions.get(topic) == null ? 0 : partitionedTopicsToPartitions.get(topic);
            return new PartitionedTopicMetadata(partitions);
        }
    });
    schemas = mock(Schemas.class);
    when(schemas.getSchemaInfo(anyString())).thenAnswer(new Answer<SchemaInfo>() {

        @Override
        public SchemaInfo answer(InvocationOnMock invocationOnMock) throws Throwable {
            Object[] args = invocationOnMock.getArguments();
            String topic = (String) args[0];
            if (topicsToSchemas.get(topic) != null) {
                return topicsToSchemas.get(topic);
            } else {
                ClientErrorException cee = new ClientErrorException(Response.status(404).build());
                throw new PulsarAdminException(cee, cee.getMessage(), cee.getResponse().getStatus());
            }
        }
    });
    pulsarAdmin = mock(PulsarAdmin.class);
    doReturn(tenants).when(pulsarAdmin).tenants();
    doReturn(namespaces).when(pulsarAdmin).namespaces();
    doReturn(topics).when(pulsarAdmin).topics();
    doReturn(schemas).when(pulsarAdmin).schemas();
    doReturn(pulsarAdmin).when(this.pulsarConnectorConfig).getPulsarAdmin();
    this.pulsarMetadata = new PulsarMetadata(pulsarConnectorId, this.pulsarConnectorConfig, dispatchingRowDecoderFactory);
    this.pulsarSplitManager = Mockito.spy(new PulsarSplitManager(pulsarConnectorId, this.pulsarConnectorConfig));
    ManagedLedgerFactory managedLedgerFactory = mock(ManagedLedgerFactory.class);
    when(managedLedgerFactory.openReadOnlyCursor(any(), any(), any())).then(new Answer<ReadOnlyCursor>() {

        private Map<String, Integer> positions = new HashMap<>();

        private int count = 0;

        @Override
        public ReadOnlyCursor answer(InvocationOnMock invocationOnMock) throws Throwable {
            Object[] args = invocationOnMock.getArguments();
            String topic = (String) args[0];
            PositionImpl positionImpl = (PositionImpl) args[1];
            int position = positionImpl.getEntryId() == -1 ? 0 : (int) positionImpl.getEntryId();
            positions.put(topic, position);
            String schemaName = TopicName.get(TopicName.get(topic.replaceAll("/persistent", "")).getPartitionedTopicName()).getSchemaName();
            long entries = topicsToNumEntries.get(schemaName);
            ReadOnlyCursorImpl readOnlyCursor = mock(ReadOnlyCursorImpl.class);
            doReturn(entries).when(readOnlyCursor).getNumberOfEntries();
            doAnswer(new Answer<Void>() {

                @Override
                public Void answer(InvocationOnMock invocation) throws Throwable {
                    Object[] args = invocation.getArguments();
                    Integer skipEntries = (Integer) args[0];
                    positions.put(topic, positions.get(topic) + skipEntries);
                    return null;
                }
            }).when(readOnlyCursor).skipEntries(anyInt());
            when(readOnlyCursor.getReadPosition()).thenAnswer(new Answer<PositionImpl>() {

                @Override
                public PositionImpl answer(InvocationOnMock invocationOnMock) throws Throwable {
                    return PositionImpl.get(0, positions.get(topic));
                }
            });
            doAnswer(new Answer() {

                @Override
                public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
                    Object[] args = invocationOnMock.getArguments();
                    Integer readEntries = (Integer) args[0];
                    AsyncCallbacks.ReadEntriesCallback callback = (AsyncCallbacks.ReadEntriesCallback) args[2];
                    Object ctx = args[3];
                    new Thread(new Runnable() {

                        @Override
                        public void run() {
                            List<Entry> entries = new LinkedList<>();
                            for (int i = 0; i < readEntries; i++) {
                                TestPulsarConnector.Bar bar = new TestPulsarConnector.Bar();
                                bar.field1 = fooFunctions.get("bar.field1").apply(count) == null ? null : (int) fooFunctions.get("bar.field1").apply(count);
                                bar.field2 = fooFunctions.get("bar.field2").apply(count) == null ? null : (String) fooFunctions.get("bar.field2").apply(count);
                                bar.field3 = (float) fooFunctions.get("bar.field3").apply(count);
                                Foo foo = new Foo();
                                foo.field1 = (int) fooFunctions.get("field1").apply(count);
                                foo.field2 = (String) fooFunctions.get("field2").apply(count);
                                foo.field3 = (float) fooFunctions.get("field3").apply(count);
                                foo.field4 = (double) fooFunctions.get("field4").apply(count);
                                foo.field5 = (boolean) fooFunctions.get("field5").apply(count);
                                foo.field6 = (long) fooFunctions.get("field6").apply(count);
                                foo.timestamp = (long) fooFunctions.get("timestamp").apply(count);
                                foo.time = (int) fooFunctions.get("time").apply(count);
                                foo.date = (int) fooFunctions.get("date").apply(count);
                                foo.bar = bar;
                                foo.field7 = (Foo.TestEnum) fooFunctions.get("field7").apply(count);
                                MessageMetadata messageMetadata = new MessageMetadata().setProducerName("test-producer").setSequenceId(positions.get(topic)).setPublishTime(System.currentTimeMillis());
                                Schema schema = topicsToSchemas.get(schemaName).getType() == SchemaType.AVRO ? AvroSchema.of(Foo.class) : JSONSchema.of(Foo.class);
                                ByteBuf payload = io.netty.buffer.Unpooled.copiedBuffer(schema.encode(foo));
                                ByteBuf byteBuf = serializeMetadataAndPayload(Commands.ChecksumType.Crc32c, messageMetadata, payload);
                                completedBytes += byteBuf.readableBytes();
                                entries.add(EntryImpl.create(0, positions.get(topic), byteBuf));
                                positions.put(topic, positions.get(topic) + 1);
                                count++;
                            }
                            callback.readEntriesComplete(entries, ctx);
                        }
                    }).start();
                    return null;
                }
            }).when(readOnlyCursor).asyncReadEntries(anyInt(), anyLong(), any(), any(), any());
            when(readOnlyCursor.hasMoreEntries()).thenAnswer(new Answer<Boolean>() {

                @Override
                public Boolean answer(InvocationOnMock invocationOnMock) throws Throwable {
                    return positions.get(topic) < entries;
                }
            });
            when(readOnlyCursor.findNewestMatching(any(), any())).then(new Answer<Position>() {

                @Override
                public Position answer(InvocationOnMock invocationOnMock) throws Throwable {
                    Object[] args = invocationOnMock.getArguments();
                    com.google.common.base.Predicate<Entry> predicate = (com.google.common.base.Predicate<Entry>) args[1];
                    String schemaName = TopicName.get(TopicName.get(topic.replaceAll("/persistent", "")).getPartitionedTopicName()).getSchemaName();
                    List<Entry> entries = getTopicEntries(schemaName);
                    Integer target = null;
                    for (int i = entries.size() - 1; i >= 0; i--) {
                        Entry entry = entries.get(i);
                        if (predicate.apply(entry)) {
                            target = i;
                            break;
                        }
                    }
                    return target == null ? null : new PositionImpl(0, target);
                }
            });
            when(readOnlyCursor.getNumberOfEntries(any())).then(new Answer<Long>() {

                @Override
                public Long answer(InvocationOnMock invocationOnMock) throws Throwable {
                    Object[] args = invocationOnMock.getArguments();
                    com.google.common.collect.Range<PositionImpl> range = (com.google.common.collect.Range<PositionImpl>) args[0];
                    return (range.upperEndpoint().getEntryId() + 1) - range.lowerEndpoint().getEntryId();
                }
            });
            when(readOnlyCursor.getCurrentLedgerInfo()).thenReturn(MLDataFormats.ManagedLedgerInfo.LedgerInfo.newBuilder().setLedgerId(0).build());
            return readOnlyCursor;
        }
    });
    PulsarConnectorCache.instance = mock(PulsarConnectorCache.class);
    when(PulsarConnectorCache.instance.getManagedLedgerFactory()).thenReturn(managedLedgerFactory);
    for (Map.Entry<TopicName, PulsarSplit> split : splits.entrySet()) {
        PulsarRecordCursor pulsarRecordCursor = spy(new PulsarRecordCursor(topicsToColumnHandles.get(split.getKey()), split.getValue(), pulsarConnectorConfig, managedLedgerFactory, new ManagedLedgerConfig(), new PulsarConnectorMetricsTracker(new NullStatsProvider()), dispatchingRowDecoderFactory));
        this.pulsarRecordCursors.put(split.getKey(), pulsarRecordCursor);
    }
}

Example 2 with ReadOnlyCursorImpl

use of org.apache.bookkeeper.mledger.impl.ReadOnlyCursorImpl in project pulsar by yahoo.

the class TestPulsarConnector method setup.

@BeforeMethod
public void setup() throws Exception {
    this.pulsarConnectorConfig = spy(PulsarConnectorConfig.class);
    this.pulsarConnectorConfig.setMaxEntryReadBatchSize(1);
    this.pulsarConnectorConfig.setMaxSplitEntryQueueSize(10);
    this.pulsarConnectorConfig.setMaxSplitMessageQueueSize(100);
    Tenants tenants = mock(Tenants.class);
    doReturn(new LinkedList<>(topicNames.stream().map(TopicName::getTenant).collect(Collectors.toSet()))).when(tenants).getTenants();
    Namespaces namespaces = mock(Namespaces.class);
    when(namespaces.getNamespaces(anyString())).thenAnswer(new Answer<List<String>>() {

        @Override
        public List<String> answer(InvocationOnMock invocation) throws Throwable {
            Object[] args = invocation.getArguments();
            String tenant = (String) args[0];
            List<String> ns = getNamespace(tenant);
            if (ns.isEmpty()) {
                ClientErrorException cee = new ClientErrorException(Response.status(404).build());
                throw new PulsarAdminException(cee, cee.getMessage(), cee.getResponse().getStatus());
            }
            return ns;
        }
    });
    Topics topics = mock(Topics.class);
    when(topics.getList(anyString(), any())).thenAnswer(new Answer<List<String>>() {

        @Override
        public List<String> answer(InvocationOnMock invocationOnMock) throws Throwable {
            Object[] args = invocationOnMock.getArguments();
            String ns = (String) args[0];
            List<String> topics = getTopics(ns);
            if (topics.isEmpty()) {
                ClientErrorException cee = new ClientErrorException(Response.status(404).build());
                throw new PulsarAdminException(cee, cee.getMessage(), cee.getResponse().getStatus());
            }
            return topics;
        }
    });
    when(topics.getPartitionedTopicList(anyString())).thenAnswer(new Answer<List<String>>() {

        @Override
        public List<String> answer(InvocationOnMock invocationOnMock) throws Throwable {
            Object[] args = invocationOnMock.getArguments();
            String ns = (String) args[0];
            List<String> topics = getPartitionedTopics(ns);
            if (topics.isEmpty()) {
                ClientErrorException cee = new ClientErrorException(Response.status(404).build());
                throw new PulsarAdminException(cee, cee.getMessage(), cee.getResponse().getStatus());
            }
            return topics;
        }
    });
    when(topics.getPartitionedTopicMetadata(anyString())).thenAnswer(new Answer<PartitionedTopicMetadata>() {

        @Override
        public PartitionedTopicMetadata answer(InvocationOnMock invocationOnMock) throws Throwable {
            Object[] args = invocationOnMock.getArguments();
            String topic = (String) args[0];
            int partitions = partitionedTopicsToPartitions.get(topic) == null ? 0 : partitionedTopicsToPartitions.get(topic);
            return new PartitionedTopicMetadata(partitions);
        }
    });
    schemas = mock(Schemas.class);
    when(schemas.getSchemaInfo(anyString())).thenAnswer(new Answer<SchemaInfo>() {

        @Override
        public SchemaInfo answer(InvocationOnMock invocationOnMock) throws Throwable {
            Object[] args = invocationOnMock.getArguments();
            String topic = (String) args[0];
            if (topicsToSchemas.get(topic) != null) {
                return topicsToSchemas.get(topic);
            } else {
                ClientErrorException cee = new ClientErrorException(Response.status(404).build());
                throw new PulsarAdminException(cee, cee.getMessage(), cee.getResponse().getStatus());
            }
        }
    });
    pulsarAdmin = mock(PulsarAdmin.class);
    doReturn(tenants).when(pulsarAdmin).tenants();
    doReturn(namespaces).when(pulsarAdmin).namespaces();
    doReturn(topics).when(pulsarAdmin).topics();
    doReturn(schemas).when(pulsarAdmin).schemas();
    doReturn(pulsarAdmin).when(this.pulsarConnectorConfig).getPulsarAdmin();
    this.pulsarMetadata = new PulsarMetadata(pulsarConnectorId, this.pulsarConnectorConfig, dispatchingRowDecoderFactory);
    this.pulsarSplitManager = Mockito.spy(new PulsarSplitManager(pulsarConnectorId, this.pulsarConnectorConfig));
    ManagedLedgerFactory managedLedgerFactory = mock(ManagedLedgerFactory.class);
    when(managedLedgerFactory.openReadOnlyCursor(any(), any(), any())).then(new Answer<ReadOnlyCursor>() {

        private Map<String, Integer> positions = new HashMap<>();

        private int count = 0;

        @Override
        public ReadOnlyCursor answer(InvocationOnMock invocationOnMock) throws Throwable {
            Object[] args = invocationOnMock.getArguments();
            String topic = (String) args[0];
            PositionImpl positionImpl = (PositionImpl) args[1];
            int position = positionImpl.getEntryId() == -1 ? 0 : (int) positionImpl.getEntryId();
            positions.put(topic, position);
            String schemaName = TopicName.get(TopicName.get(topic.replaceAll("/persistent", "")).getPartitionedTopicName()).getSchemaName();
            long entries = topicsToNumEntries.get(schemaName);
            ReadOnlyCursorImpl readOnlyCursor = mock(ReadOnlyCursorImpl.class);
            doReturn(entries).when(readOnlyCursor).getNumberOfEntries();
            doAnswer(new Answer<Void>() {

                @Override
                public Void answer(InvocationOnMock invocation) throws Throwable {
                    Object[] args = invocation.getArguments();
                    Integer skipEntries = (Integer) args[0];
                    positions.put(topic, positions.get(topic) + skipEntries);
                    return null;
                }
            }).when(readOnlyCursor).skipEntries(anyInt());
            when(readOnlyCursor.getReadPosition()).thenAnswer(new Answer<PositionImpl>() {

                @Override
                public PositionImpl answer(InvocationOnMock invocationOnMock) throws Throwable {
                    return PositionImpl.get(0, positions.get(topic));
                }
            });
            doAnswer(new Answer() {

                @Override
                public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
                    Object[] args = invocationOnMock.getArguments();
                    Integer readEntries = (Integer) args[0];
                    AsyncCallbacks.ReadEntriesCallback callback = (AsyncCallbacks.ReadEntriesCallback) args[2];
                    Object ctx = args[3];
                    new Thread(new Runnable() {

                        @Override
                        public void run() {
                            List<Entry> entries = new LinkedList<>();
                            for (int i = 0; i < readEntries; i++) {
                                TestPulsarConnector.Bar bar = new TestPulsarConnector.Bar();
                                bar.field1 = fooFunctions.get("bar.field1").apply(count) == null ? null : (int) fooFunctions.get("bar.field1").apply(count);
                                bar.field2 = fooFunctions.get("bar.field2").apply(count) == null ? null : (String) fooFunctions.get("bar.field2").apply(count);
                                bar.field3 = (float) fooFunctions.get("bar.field3").apply(count);
                                Foo foo = new Foo();
                                foo.field1 = (int) fooFunctions.get("field1").apply(count);
                                foo.field2 = (String) fooFunctions.get("field2").apply(count);
                                foo.field3 = (float) fooFunctions.get("field3").apply(count);
                                foo.field4 = (double) fooFunctions.get("field4").apply(count);
                                foo.field5 = (boolean) fooFunctions.get("field5").apply(count);
                                foo.field6 = (long) fooFunctions.get("field6").apply(count);
                                foo.timestamp = (long) fooFunctions.get("timestamp").apply(count);
                                foo.time = (int) fooFunctions.get("time").apply(count);
                                foo.date = (int) fooFunctions.get("date").apply(count);
                                foo.decimal = (BigDecimal) fooFunctions.get("decimal").apply(count);
                                foo.bar = bar;
                                foo.field7 = (Foo.TestEnum) fooFunctions.get("field7").apply(count);
                                MessageMetadata messageMetadata = new MessageMetadata().setProducerName("test-producer").setSequenceId(positions.get(topic)).setPublishTime(System.currentTimeMillis());
                                Schema schema = topicsToSchemas.get(schemaName).getType() == SchemaType.AVRO ? AvroSchema.of(Foo.class) : JSONSchema.of(Foo.class);
                                ByteBuf payload = io.netty.buffer.Unpooled.copiedBuffer(schema.encode(foo));
                                ByteBuf byteBuf = serializeMetadataAndPayload(Commands.ChecksumType.Crc32c, messageMetadata, payload);
                                completedBytes += byteBuf.readableBytes();
                                entries.add(EntryImpl.create(0, positions.get(topic), byteBuf));
                                positions.put(topic, positions.get(topic) + 1);
                                count++;
                            }
                            callback.readEntriesComplete(entries, ctx);
                        }
                    }).start();
                    return null;
                }
            }).when(readOnlyCursor).asyncReadEntries(anyInt(), anyLong(), any(), any(), any());
            when(readOnlyCursor.hasMoreEntries()).thenAnswer(new Answer<Boolean>() {

                @Override
                public Boolean answer(InvocationOnMock invocationOnMock) throws Throwable {
                    return positions.get(topic) < entries;
                }
            });
            when(readOnlyCursor.findNewestMatching(any(), any())).then(new Answer<Position>() {

                @Override
                public Position answer(InvocationOnMock invocationOnMock) throws Throwable {
                    Object[] args = invocationOnMock.getArguments();
                    com.google.common.base.Predicate<Entry> predicate = (com.google.common.base.Predicate<Entry>) args[1];
                    String schemaName = TopicName.get(TopicName.get(topic.replaceAll("/persistent", "")).getPartitionedTopicName()).getSchemaName();
                    List<Entry> entries = getTopicEntries(schemaName);
                    Integer target = null;
                    for (int i = entries.size() - 1; i >= 0; i--) {
                        Entry entry = entries.get(i);
                        if (predicate.apply(entry)) {
                            target = i;
                            break;
                        }
                    }
                    return target == null ? null : new PositionImpl(0, target);
                }
            });
            when(readOnlyCursor.getNumberOfEntries(any())).then(new Answer<Long>() {

                @Override
                public Long answer(InvocationOnMock invocationOnMock) throws Throwable {
                    Object[] args = invocationOnMock.getArguments();
                    com.google.common.collect.Range<PositionImpl> range = (com.google.common.collect.Range<PositionImpl>) args[0];
                    return (range.upperEndpoint().getEntryId() + 1) - range.lowerEndpoint().getEntryId();
                }
            });
            when(readOnlyCursor.getCurrentLedgerInfo()).thenReturn(MLDataFormats.ManagedLedgerInfo.LedgerInfo.newBuilder().setLedgerId(0).build());
            return readOnlyCursor;
        }
    });
    PulsarConnectorCache.instance = mock(PulsarConnectorCache.class);
    when(PulsarConnectorCache.instance.getManagedLedgerFactory()).thenReturn(managedLedgerFactory);
    for (Map.Entry<TopicName, PulsarSplit> split : splits.entrySet()) {
        PulsarRecordCursor pulsarRecordCursor = spy(new PulsarRecordCursor(topicsToColumnHandles.get(split.getKey()), split.getValue(), pulsarConnectorConfig, managedLedgerFactory, new ManagedLedgerConfig(), new PulsarConnectorMetricsTracker(new NullStatsProvider()), dispatchingRowDecoderFactory));
        this.pulsarRecordCursors.put(split.getKey(), pulsarRecordCursor);
    }
}

Example 3 with ReadOnlyCursorImpl

use of org.apache.bookkeeper.mledger.impl.ReadOnlyCursorImpl in project incubator-pulsar by apache.

the class TestPulsarConnector method setup.

@BeforeMethod
public void setup() throws Exception {
    this.pulsarConnectorConfig = spy(PulsarConnectorConfig.class);
    this.pulsarConnectorConfig.setMaxEntryReadBatchSize(1);
    this.pulsarConnectorConfig.setMaxSplitEntryQueueSize(10);
    this.pulsarConnectorConfig.setMaxSplitMessageQueueSize(100);
    Tenants tenants = mock(Tenants.class);
    doReturn(new LinkedList<>(topicNames.stream().map(TopicName::getTenant).collect(Collectors.toSet()))).when(tenants).getTenants();
    Namespaces namespaces = mock(Namespaces.class);
    when(namespaces.getNamespaces(anyString())).thenAnswer(new Answer<List<String>>() {

        @Override
        public List<String> answer(InvocationOnMock invocation) throws Throwable {
            Object[] args = invocation.getArguments();
            String tenant = (String) args[0];
            List<String> ns = getNamespace(tenant);
            if (ns.isEmpty()) {
                ClientErrorException cee = new ClientErrorException(Response.status(404).build());
                throw new PulsarAdminException(cee, cee.getMessage(), cee.getResponse().getStatus());
            }
            return ns;
        }
    });
    Topics topics = mock(Topics.class);
    when(topics.getList(anyString(), any())).thenAnswer(new Answer<List<String>>() {

        @Override
        public List<String> answer(InvocationOnMock invocationOnMock) throws Throwable {
            Object[] args = invocationOnMock.getArguments();
            String ns = (String) args[0];
            List<String> topics = getTopics(ns);
            if (topics.isEmpty()) {
                ClientErrorException cee = new ClientErrorException(Response.status(404).build());
                throw new PulsarAdminException(cee, cee.getMessage(), cee.getResponse().getStatus());
            }
            return topics;
        }
    });
    when(topics.getPartitionedTopicList(anyString())).thenAnswer(new Answer<List<String>>() {

        @Override
        public List<String> answer(InvocationOnMock invocationOnMock) throws Throwable {
            Object[] args = invocationOnMock.getArguments();
            String ns = (String) args[0];
            List<String> topics = getPartitionedTopics(ns);
            if (topics.isEmpty()) {
                ClientErrorException cee = new ClientErrorException(Response.status(404).build());
                throw new PulsarAdminException(cee, cee.getMessage(), cee.getResponse().getStatus());
            }
            return topics;
        }
    });
    when(topics.getPartitionedTopicMetadata(anyString())).thenAnswer(new Answer<PartitionedTopicMetadata>() {

        @Override
        public PartitionedTopicMetadata answer(InvocationOnMock invocationOnMock) throws Throwable {
            Object[] args = invocationOnMock.getArguments();
            String topic = (String) args[0];
            int partitions = partitionedTopicsToPartitions.get(topic) == null ? 0 : partitionedTopicsToPartitions.get(topic);
            return new PartitionedTopicMetadata(partitions);
        }
    });
    schemas = mock(Schemas.class);
    when(schemas.getSchemaInfo(anyString())).thenAnswer(new Answer<SchemaInfo>() {

        @Override
        public SchemaInfo answer(InvocationOnMock invocationOnMock) throws Throwable {
            Object[] args = invocationOnMock.getArguments();
            String topic = (String) args[0];
            if (topicsToSchemas.get(topic) != null) {
                return topicsToSchemas.get(topic);
            } else {
                ClientErrorException cee = new ClientErrorException(Response.status(404).build());
                throw new PulsarAdminException(cee, cee.getMessage(), cee.getResponse().getStatus());
            }
        }
    });
    pulsarAdmin = mock(PulsarAdmin.class);
    doReturn(tenants).when(pulsarAdmin).tenants();
    doReturn(namespaces).when(pulsarAdmin).namespaces();
    doReturn(topics).when(pulsarAdmin).topics();
    doReturn(schemas).when(pulsarAdmin).schemas();
    doReturn(pulsarAdmin).when(this.pulsarConnectorConfig).getPulsarAdmin();
    this.pulsarAuth = mock(PulsarAuth.class);
    this.pulsarMetadata = new PulsarMetadata(pulsarConnectorId, this.pulsarConnectorConfig, dispatchingRowDecoderFactory, this.pulsarAuth);
    this.pulsarSplitManager = Mockito.spy(new PulsarSplitManager(pulsarConnectorId, this.pulsarConnectorConfig));
    ManagedLedgerFactory managedLedgerFactory = mock(ManagedLedgerFactory.class);
    when(managedLedgerFactory.openReadOnlyCursor(any(), any(), any())).then(new Answer<ReadOnlyCursor>() {

        private Map<String, Integer> positions = new HashMap<>();

        private int count = 0;

        @Override
        public ReadOnlyCursor answer(InvocationOnMock invocationOnMock) throws Throwable {
            Object[] args = invocationOnMock.getArguments();
            String topic = (String) args[0];
            PositionImpl positionImpl = (PositionImpl) args[1];
            int position = positionImpl.getEntryId() == -1 ? 0 : (int) positionImpl.getEntryId();
            positions.put(topic, position);
            String schemaName = TopicName.get(TopicName.get(topic.replaceAll("/persistent", "")).getPartitionedTopicName()).getSchemaName();
            long entries = topicsToNumEntries.get(schemaName);
            ReadOnlyCursorImpl readOnlyCursor = mock(ReadOnlyCursorImpl.class);
            doReturn(entries).when(readOnlyCursor).getNumberOfEntries();
            doAnswer(new Answer<Void>() {

                @Override
                public Void answer(InvocationOnMock invocation) throws Throwable {
                    Object[] args = invocation.getArguments();
                    Integer skipEntries = (Integer) args[0];
                    positions.put(topic, positions.get(topic) + skipEntries);
                    return null;
                }
            }).when(readOnlyCursor).skipEntries(anyInt());
            when(readOnlyCursor.getReadPosition()).thenAnswer(new Answer<PositionImpl>() {

                @Override
                public PositionImpl answer(InvocationOnMock invocationOnMock) throws Throwable {
                    return PositionImpl.get(0, positions.get(topic));
                }
            });
            doAnswer(new Answer() {

                @Override
                public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
                    Object[] args = invocationOnMock.getArguments();
                    Integer readEntries = (Integer) args[0];
                    AsyncCallbacks.ReadEntriesCallback callback = (AsyncCallbacks.ReadEntriesCallback) args[2];
                    Object ctx = args[3];
                    new Thread(new Runnable() {

                        @Override
                        public void run() {
                            List<Entry> entries = new LinkedList<>();
                            for (int i = 0; i < readEntries; i++) {
                                TestPulsarConnector.Bar bar = new TestPulsarConnector.Bar();
                                bar.field1 = fooFunctions.get("bar.field1").apply(count) == null ? null : (int) fooFunctions.get("bar.field1").apply(count);
                                bar.field2 = fooFunctions.get("bar.field2").apply(count) == null ? null : (String) fooFunctions.get("bar.field2").apply(count);
                                bar.field3 = (float) fooFunctions.get("bar.field3").apply(count);
                                Foo foo = new Foo();
                                foo.field1 = (int) fooFunctions.get("field1").apply(count);
                                foo.field2 = (String) fooFunctions.get("field2").apply(count);
                                foo.field3 = (float) fooFunctions.get("field3").apply(count);
                                foo.field4 = (double) fooFunctions.get("field4").apply(count);
                                foo.field5 = (boolean) fooFunctions.get("field5").apply(count);
                                foo.field6 = (long) fooFunctions.get("field6").apply(count);
                                foo.timestamp = (long) fooFunctions.get("timestamp").apply(count);
                                foo.time = (int) fooFunctions.get("time").apply(count);
                                foo.date = (int) fooFunctions.get("date").apply(count);
                                foo.decimal = (BigDecimal) fooFunctions.get("decimal").apply(count);
                                foo.bar = bar;
                                foo.field7 = (Foo.TestEnum) fooFunctions.get("field7").apply(count);
                                MessageMetadata messageMetadata = new MessageMetadata().setProducerName("test-producer").setSequenceId(positions.get(topic)).setPublishTime(System.currentTimeMillis());
                                Schema schema = topicsToSchemas.get(schemaName).getType() == SchemaType.AVRO ? AvroSchema.of(Foo.class) : JSONSchema.of(Foo.class);
                                ByteBuf payload = io.netty.buffer.Unpooled.copiedBuffer(schema.encode(foo));
                                ByteBuf byteBuf = serializeMetadataAndPayload(Commands.ChecksumType.Crc32c, messageMetadata, payload);
                                completedBytes += byteBuf.readableBytes();
                                entries.add(EntryImpl.create(0, positions.get(topic), byteBuf));
                                positions.put(topic, positions.get(topic) + 1);
                                count++;
                            }
                            callback.readEntriesComplete(entries, ctx);
                        }
                    }).start();
                    return null;
                }
            }).when(readOnlyCursor).asyncReadEntries(anyInt(), anyLong(), any(), any(), any());
            when(readOnlyCursor.hasMoreEntries()).thenAnswer(new Answer<Boolean>() {

                @Override
                public Boolean answer(InvocationOnMock invocationOnMock) throws Throwable {
                    return positions.get(topic) < entries;
                }
            });
            when(readOnlyCursor.findNewestMatching(any(), any())).then(new Answer<Position>() {

                @Override
                public Position answer(InvocationOnMock invocationOnMock) throws Throwable {
                    Object[] args = invocationOnMock.getArguments();
                    com.google.common.base.Predicate<Entry> predicate = (com.google.common.base.Predicate<Entry>) args[1];
                    String schemaName = TopicName.get(TopicName.get(topic.replaceAll("/persistent", "")).getPartitionedTopicName()).getSchemaName();
                    List<Entry> entries = getTopicEntries(schemaName);
                    Integer target = null;
                    for (int i = entries.size() - 1; i >= 0; i--) {
                        Entry entry = entries.get(i);
                        if (predicate.apply(entry)) {
                            target = i;
                            break;
                        }
                    }
                    return target == null ? null : new PositionImpl(0, target);
                }
            });
            when(readOnlyCursor.getNumberOfEntries(any())).then(new Answer<Long>() {

                @Override
                public Long answer(InvocationOnMock invocationOnMock) throws Throwable {
                    Object[] args = invocationOnMock.getArguments();
                    com.google.common.collect.Range<PositionImpl> range = (com.google.common.collect.Range<PositionImpl>) args[0];
                    return (range.upperEndpoint().getEntryId() + 1) - range.lowerEndpoint().getEntryId();
                }
            });
            when(readOnlyCursor.getCurrentLedgerInfo()).thenReturn(MLDataFormats.ManagedLedgerInfo.LedgerInfo.newBuilder().setLedgerId(0).build());
            return readOnlyCursor;
        }
    });
    PulsarConnectorCache.instance = mock(PulsarConnectorCache.class);
    when(PulsarConnectorCache.instance.getManagedLedgerFactory()).thenReturn(managedLedgerFactory);
    for (Map.Entry<TopicName, PulsarSplit> split : splits.entrySet()) {
        PulsarRecordCursor pulsarRecordCursor = spy(new PulsarRecordCursor(topicsToColumnHandles.get(split.getKey()), split.getValue(), pulsarConnectorConfig, managedLedgerFactory, new ManagedLedgerConfig(), new PulsarConnectorMetricsTracker(new NullStatsProvider()), dispatchingRowDecoderFactory));
        this.pulsarRecordCursors.put(split.getKey(), pulsarRecordCursor);
    }
}

Example 4 with ReadOnlyCursorImpl

use of org.apache.bookkeeper.mledger.impl.ReadOnlyCursorImpl in project incubator-pulsar by apache.

the class TestPulsarRecordCursor method mockKeyValueSchemaPulsarRecordCursor.

/**
 * mock a simple PulsarRecordCursor for KeyValueSchema test.
 * @param entriesNum
 * @param topicName
 * @param schema
 * @param message
 * @param ColumnHandles
 * @return
 * @throws Exception
 */
private PulsarRecordCursor mockKeyValueSchemaPulsarRecordCursor(final Long entriesNum, final TopicName topicName, final KeyValueSchemaImpl schema, KeyValue message, List<PulsarColumnHandle> ColumnHandles) throws Exception {
    ManagedLedgerFactory managedLedgerFactory = mock(ManagedLedgerFactory.class);
    when(managedLedgerFactory.openReadOnlyCursor(any(), any(), any())).then(new Answer<ReadOnlyCursor>() {

        private Map<String, Integer> positions = new HashMap<>();

        @Override
        public ReadOnlyCursor answer(InvocationOnMock invocationOnMock) throws Throwable {
            Object[] args = invocationOnMock.getArguments();
            String topic = (String) args[0];
            PositionImpl positionImpl = (PositionImpl) args[1];
            int position = positionImpl.getEntryId() == -1 ? 0 : (int) positionImpl.getEntryId();
            positions.put(topic, position);
            ReadOnlyCursorImpl readOnlyCursor = mock(ReadOnlyCursorImpl.class);
            doReturn(entriesNum).when(readOnlyCursor).getNumberOfEntries();
            doAnswer(new Answer<Void>() {

                @Override
                public Void answer(InvocationOnMock invocation) throws Throwable {
                    Object[] args = invocation.getArguments();
                    Integer skipEntries = (Integer) args[0];
                    positions.put(topic, positions.get(topic) + skipEntries);
                    return null;
                }
            }).when(readOnlyCursor).skipEntries(anyInt());
            when(readOnlyCursor.getReadPosition()).thenAnswer(new Answer<PositionImpl>() {

                @Override
                public PositionImpl answer(InvocationOnMock invocationOnMock) throws Throwable {
                    return PositionImpl.get(0, positions.get(topic));
                }
            });
            doAnswer(new Answer() {

                @Override
                public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
                    Object[] args = invocationOnMock.getArguments();
                    Integer readEntries = (Integer) args[0];
                    AsyncCallbacks.ReadEntriesCallback callback = (AsyncCallbacks.ReadEntriesCallback) args[2];
                    Object ctx = args[3];
                    new Thread(new Runnable() {

                        @Override
                        public void run() {
                            List<Entry> entries = new LinkedList<>();
                            for (int i = 0; i < readEntries; i++) {
                                MessageMetadata messageMetadata = new MessageMetadata().setProducerName("test-producer").setSequenceId(positions.get(topic)).setPublishTime(System.currentTimeMillis());
                                if (i % 2 == 0) {
                                    messageMetadata.setSchemaVersion(new LongSchemaVersion(1L).bytes());
                                }
                                if (KeyValueEncodingType.SEPARATED.equals(schema.getKeyValueEncodingType())) {
                                    messageMetadata.setPartitionKey(new String(schema.getKeySchema().encode(message.getKey()), Charset.forName("UTF-8"))).setPartitionKeyB64Encoded(false);
                                }
                                ByteBuf dataPayload = io.netty.buffer.Unpooled.copiedBuffer(schema.encode(message));
                                ByteBuf byteBuf = serializeMetadataAndPayload(Commands.ChecksumType.Crc32c, messageMetadata, dataPayload);
                                entries.add(EntryImpl.create(0, positions.get(topic), byteBuf));
                                positions.put(topic, positions.get(topic) + 1);
                            }
                            callback.readEntriesComplete(entries, ctx);
                        }
                    }).start();
                    return null;
                }
            }).when(readOnlyCursor).asyncReadEntries(anyInt(), anyLong(), any(), any(), any());
            when(readOnlyCursor.hasMoreEntries()).thenAnswer(new Answer<Boolean>() {

                @Override
                public Boolean answer(InvocationOnMock invocationOnMock) throws Throwable {
                    return positions.get(topic) < entriesNum;
                }
            });
            when(readOnlyCursor.getNumberOfEntries(any())).then(new Answer<Long>() {

                @Override
                public Long answer(InvocationOnMock invocationOnMock) throws Throwable {
                    Object[] args = invocationOnMock.getArguments();
                    com.google.common.collect.Range<PositionImpl> range = (com.google.common.collect.Range<PositionImpl>) args[0];
                    return (range.upperEndpoint().getEntryId() + 1) - range.lowerEndpoint().getEntryId();
                }
            });
            when(readOnlyCursor.getCurrentLedgerInfo()).thenReturn(MLDataFormats.ManagedLedgerInfo.LedgerInfo.newBuilder().setLedgerId(0).build());
            return readOnlyCursor;
        }
    });
    ObjectMapper objectMapper = new ObjectMapper();
    PulsarSplit split = new PulsarSplit(0, pulsarConnectorId.toString(), topicName.getNamespace(), topicName.getLocalName(), topicName.getLocalName(), entriesNum, new String(schema.getSchemaInfo().getSchema(), "ISO8859-1"), schema.getSchemaInfo().getType(), 0, entriesNum, 0, 0, TupleDomain.all(), objectMapper.writeValueAsString(schema.getSchemaInfo().getProperties()), null);
    PulsarRecordCursor pulsarRecordCursor = spy(new PulsarRecordCursor(ColumnHandles, split, pulsarConnectorConfig, managedLedgerFactory, new ManagedLedgerConfig(), new PulsarConnectorMetricsTracker(new NullStatsProvider()), dispatchingRowDecoderFactory));
    PulsarSqlSchemaInfoProvider pulsarSqlSchemaInfoProvider = mock(PulsarSqlSchemaInfoProvider.class);
    when(pulsarSqlSchemaInfoProvider.getSchemaByVersion(any())).thenReturn(completedFuture(schema.getSchemaInfo()));
    pulsarRecordCursor.setPulsarSqlSchemaInfoProvider(pulsarSqlSchemaInfoProvider);
    return pulsarRecordCursor;
}

Example 5 with ReadOnlyCursorImpl

use of org.apache.bookkeeper.mledger.impl.ReadOnlyCursorImpl in project pulsar by yahoo.

the class TestPulsarRecordCursor method mockKeyValueSchemaPulsarRecordCursor.

/**
 * mock a simple PulsarRecordCursor for KeyValueSchema test.
 * @param entriesNum
 * @param topicName
 * @param schema
 * @param message
 * @param ColumnHandles
 * @return
 * @throws Exception
 */
private PulsarRecordCursor mockKeyValueSchemaPulsarRecordCursor(final Long entriesNum, final TopicName topicName, final KeyValueSchemaImpl schema, KeyValue message, List<PulsarColumnHandle> ColumnHandles) throws Exception {
    ManagedLedgerFactory managedLedgerFactory = mock(ManagedLedgerFactory.class);
    when(managedLedgerFactory.openReadOnlyCursor(any(), any(), any())).then(new Answer<ReadOnlyCursor>() {

        private Map<String, Integer> positions = new HashMap<>();

        @Override
        public ReadOnlyCursor answer(InvocationOnMock invocationOnMock) throws Throwable {
            Object[] args = invocationOnMock.getArguments();
            String topic = (String) args[0];
            PositionImpl positionImpl = (PositionImpl) args[1];
            int position = positionImpl.getEntryId() == -1 ? 0 : (int) positionImpl.getEntryId();
            positions.put(topic, position);
            ReadOnlyCursorImpl readOnlyCursor = mock(ReadOnlyCursorImpl.class);
            doReturn(entriesNum).when(readOnlyCursor).getNumberOfEntries();
            doAnswer(new Answer<Void>() {

                @Override
                public Void answer(InvocationOnMock invocation) throws Throwable {
                    Object[] args = invocation.getArguments();
                    Integer skipEntries = (Integer) args[0];
                    positions.put(topic, positions.get(topic) + skipEntries);
                    return null;
                }
            }).when(readOnlyCursor).skipEntries(anyInt());
            when(readOnlyCursor.getReadPosition()).thenAnswer(new Answer<PositionImpl>() {

                @Override
                public PositionImpl answer(InvocationOnMock invocationOnMock) throws Throwable {
                    return PositionImpl.get(0, positions.get(topic));
                }
            });
            doAnswer(new Answer() {

                @Override
                public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
                    Object[] args = invocationOnMock.getArguments();
                    Integer readEntries = (Integer) args[0];
                    AsyncCallbacks.ReadEntriesCallback callback = (AsyncCallbacks.ReadEntriesCallback) args[2];
                    Object ctx = args[3];
                    new Thread(new Runnable() {

                        @Override
                        public void run() {
                            List<Entry> entries = new LinkedList<>();
                            for (int i = 0; i < readEntries; i++) {
                                MessageMetadata messageMetadata = new MessageMetadata().setProducerName("test-producer").setSequenceId(positions.get(topic)).setPublishTime(System.currentTimeMillis());
                                if (i % 2 == 0) {
                                    messageMetadata.setSchemaVersion(new LongSchemaVersion(1L).bytes());
                                }
                                if (KeyValueEncodingType.SEPARATED.equals(schema.getKeyValueEncodingType())) {
                                    messageMetadata.setPartitionKey(new String(schema.getKeySchema().encode(message.getKey()), Charset.forName("UTF-8"))).setPartitionKeyB64Encoded(false);
                                }
                                ByteBuf dataPayload = io.netty.buffer.Unpooled.copiedBuffer(schema.encode(message));
                                ByteBuf byteBuf = serializeMetadataAndPayload(Commands.ChecksumType.Crc32c, messageMetadata, dataPayload);
                                entries.add(EntryImpl.create(0, positions.get(topic), byteBuf));
                                positions.put(topic, positions.get(topic) + 1);
                            }
                            callback.readEntriesComplete(entries, ctx);
                        }
                    }).start();
                    return null;
                }
            }).when(readOnlyCursor).asyncReadEntries(anyInt(), anyLong(), any(), any(), any());
            when(readOnlyCursor.hasMoreEntries()).thenAnswer(new Answer<Boolean>() {

                @Override
                public Boolean answer(InvocationOnMock invocationOnMock) throws Throwable {
                    return positions.get(topic) < entriesNum;
                }
            });
            when(readOnlyCursor.getNumberOfEntries(any())).then(new Answer<Long>() {

                @Override
                public Long answer(InvocationOnMock invocationOnMock) throws Throwable {
                    Object[] args = invocationOnMock.getArguments();
                    com.google.common.collect.Range<PositionImpl> range = (com.google.common.collect.Range<PositionImpl>) args[0];
                    return (range.upperEndpoint().getEntryId() + 1) - range.lowerEndpoint().getEntryId();
                }
            });
            when(readOnlyCursor.getCurrentLedgerInfo()).thenReturn(MLDataFormats.ManagedLedgerInfo.LedgerInfo.newBuilder().setLedgerId(0).build());
            return readOnlyCursor;
        }
    });
    ObjectMapper objectMapper = new ObjectMapper();
    PulsarSplit split = new PulsarSplit(0, pulsarConnectorId.toString(), topicName.getNamespace(), topicName.getLocalName(), topicName.getLocalName(), entriesNum, new String(schema.getSchemaInfo().getSchema(), "ISO8859-1"), schema.getSchemaInfo().getType(), 0, entriesNum, 0, 0, TupleDomain.all(), objectMapper.writeValueAsString(schema.getSchemaInfo().getProperties()), null);
    PulsarRecordCursor pulsarRecordCursor = spy(new PulsarRecordCursor(ColumnHandles, split, pulsarConnectorConfig, managedLedgerFactory, new ManagedLedgerConfig(), new PulsarConnectorMetricsTracker(new NullStatsProvider()), dispatchingRowDecoderFactory));
    PulsarSqlSchemaInfoProvider pulsarSqlSchemaInfoProvider = mock(PulsarSqlSchemaInfoProvider.class);
    when(pulsarSqlSchemaInfoProvider.getSchemaByVersion(any())).thenReturn(completedFuture(schema.getSchemaInfo()));
    pulsarRecordCursor.setPulsarSqlSchemaInfoProvider(pulsarSqlSchemaInfoProvider);
    return pulsarRecordCursor;
}