use of org.apache.bookkeeper.mledger.ReadOnlyCursor in project incubator-pulsar by apache.
the class ManagedLedgerFactoryImpl method openReadOnlyCursor.
@Override
public ReadOnlyCursor openReadOnlyCursor(String managedLedgerName, Position startPosition, ManagedLedgerConfig config) throws InterruptedException, ManagedLedgerException {
class Result {
ReadOnlyCursor c = null;
ManagedLedgerException e = null;
}
final Result r = new Result();
final CountDownLatch latch = new CountDownLatch(1);
asyncOpenReadOnlyCursor(managedLedgerName, startPosition, config, new OpenReadOnlyCursorCallback() {
@Override
public void openReadOnlyCursorComplete(ReadOnlyCursor cursor, Object ctx) {
r.c = cursor;
latch.countDown();
}
@Override
public void openReadOnlyCursorFailed(ManagedLedgerException exception, Object ctx) {
r.e = exception;
latch.countDown();
}
}, null);
latch.await();
if (r.e != null) {
throw r.e;
}
return r.c;
}
use of org.apache.bookkeeper.mledger.ReadOnlyCursor in project incubator-pulsar by apache.
the class ReadOnlyManagedLedgerImpl method initializeAndCreateCursor.
CompletableFuture<ReadOnlyCursor> initializeAndCreateCursor(PositionImpl startPosition) {
CompletableFuture<ReadOnlyCursor> future = new CompletableFuture<>();
// Fetch the list of existing ledgers in the managed ledger
store.getManagedLedgerInfo(name, false, new MetaStoreCallback<ManagedLedgerInfo>() {
@Override
public void operationComplete(ManagedLedgerInfo mlInfo, Stat stat) {
state = State.LedgerOpened;
for (LedgerInfo ls : mlInfo.getLedgerInfoList()) {
ledgers.put(ls.getLedgerId(), ls);
}
// Last ledger stat may be zeroed, we must update it
if (ledgers.size() > 0 && ledgers.lastEntry().getValue().getEntries() == 0) {
long lastLedgerId = ledgers.lastKey();
// Fetch last add confirmed for last ledger
bookKeeper.newOpenLedgerOp().withRecovery(false).withLedgerId(lastLedgerId).withDigestType(config.getDigestType()).withPassword(config.getPassword()).execute().thenAccept(readHandle -> {
readHandle.readLastAddConfirmedAsync().thenAccept(lastAddConfirmed -> {
LedgerInfo info = LedgerInfo.newBuilder().setLedgerId(lastLedgerId).setEntries(lastAddConfirmed + 1).setSize(readHandle.getLength()).setTimestamp(clock.millis()).build();
ledgers.put(lastLedgerId, info);
future.complete(createReadOnlyCursor(startPosition));
}).exceptionally(ex -> {
if (ex instanceof CompletionException && ex.getCause() instanceof IllegalArgumentException) {
// The last ledger was empty, so we cannot read the last add confirmed.
LedgerInfo info = LedgerInfo.newBuilder().setLedgerId(lastLedgerId).setEntries(0).setSize(0).setTimestamp(clock.millis()).build();
ledgers.put(lastLedgerId, info);
future.complete(createReadOnlyCursor(startPosition));
} else {
future.completeExceptionally(new ManagedLedgerException(ex));
}
return null;
});
}).exceptionally(ex -> {
if (ex instanceof CompletionException && ex.getCause() instanceof ArrayIndexOutOfBoundsException) {
// The last ledger was empty, so we cannot read the last add confirmed.
LedgerInfo info = LedgerInfo.newBuilder().setLedgerId(lastLedgerId).setEntries(0).setSize(0).setTimestamp(clock.millis()).build();
ledgers.put(lastLedgerId, info);
future.complete(createReadOnlyCursor(startPosition));
} else {
future.completeExceptionally(new ManagedLedgerException(ex));
}
return null;
});
} else {
// The read-only managed ledger is ready to use
future.complete(createReadOnlyCursor(startPosition));
}
}
@Override
public void operationFailed(MetaStoreException e) {
if (e instanceof MetadataNotFoundException) {
future.completeExceptionally(new ManagedLedgerNotFoundException(e));
} else {
future.completeExceptionally(new ManagedLedgerException(e));
}
}
});
return future;
}
use of org.apache.bookkeeper.mledger.ReadOnlyCursor in project incubator-pulsar by apache.
the class TestPulsarConnector method setup.
@BeforeMethod
public void setup() throws Exception {
this.pulsarConnectorConfig = spy(PulsarConnectorConfig.class);
this.pulsarConnectorConfig.setMaxEntryReadBatchSize(1);
this.pulsarConnectorConfig.setMaxSplitEntryQueueSize(10);
this.pulsarConnectorConfig.setMaxSplitMessageQueueSize(100);
Tenants tenants = mock(Tenants.class);
doReturn(new LinkedList<>(topicNames.stream().map(TopicName::getTenant).collect(Collectors.toSet()))).when(tenants).getTenants();
Namespaces namespaces = mock(Namespaces.class);
when(namespaces.getNamespaces(anyString())).thenAnswer(new Answer<List<String>>() {
@Override
public List<String> answer(InvocationOnMock invocation) throws Throwable {
Object[] args = invocation.getArguments();
String tenant = (String) args[0];
List<String> ns = getNamespace(tenant);
if (ns.isEmpty()) {
ClientErrorException cee = new ClientErrorException(Response.status(404).build());
throw new PulsarAdminException(cee, cee.getMessage(), cee.getResponse().getStatus());
}
return ns;
}
});
Topics topics = mock(Topics.class);
when(topics.getList(anyString(), any())).thenAnswer(new Answer<List<String>>() {
@Override
public List<String> answer(InvocationOnMock invocationOnMock) throws Throwable {
Object[] args = invocationOnMock.getArguments();
String ns = (String) args[0];
List<String> topics = getTopics(ns);
if (topics.isEmpty()) {
ClientErrorException cee = new ClientErrorException(Response.status(404).build());
throw new PulsarAdminException(cee, cee.getMessage(), cee.getResponse().getStatus());
}
return topics;
}
});
when(topics.getPartitionedTopicList(anyString())).thenAnswer(new Answer<List<String>>() {
@Override
public List<String> answer(InvocationOnMock invocationOnMock) throws Throwable {
Object[] args = invocationOnMock.getArguments();
String ns = (String) args[0];
List<String> topics = getPartitionedTopics(ns);
if (topics.isEmpty()) {
ClientErrorException cee = new ClientErrorException(Response.status(404).build());
throw new PulsarAdminException(cee, cee.getMessage(), cee.getResponse().getStatus());
}
return topics;
}
});
when(topics.getPartitionedTopicMetadata(anyString())).thenAnswer(new Answer<PartitionedTopicMetadata>() {
@Override
public PartitionedTopicMetadata answer(InvocationOnMock invocationOnMock) throws Throwable {
Object[] args = invocationOnMock.getArguments();
String topic = (String) args[0];
int partitions = partitionedTopicsToPartitions.get(topic) == null ? 0 : partitionedTopicsToPartitions.get(topic);
return new PartitionedTopicMetadata(partitions);
}
});
schemas = mock(Schemas.class);
when(schemas.getSchemaInfo(anyString())).thenAnswer(new Answer<SchemaInfo>() {
@Override
public SchemaInfo answer(InvocationOnMock invocationOnMock) throws Throwable {
Object[] args = invocationOnMock.getArguments();
String topic = (String) args[0];
if (topicsToSchemas.get(topic) != null) {
return topicsToSchemas.get(topic);
} else {
ClientErrorException cee = new ClientErrorException(Response.status(404).build());
throw new PulsarAdminException(cee, cee.getMessage(), cee.getResponse().getStatus());
}
}
});
pulsarAdmin = mock(PulsarAdmin.class);
doReturn(tenants).when(pulsarAdmin).tenants();
doReturn(namespaces).when(pulsarAdmin).namespaces();
doReturn(topics).when(pulsarAdmin).topics();
doReturn(schemas).when(pulsarAdmin).schemas();
doReturn(pulsarAdmin).when(this.pulsarConnectorConfig).getPulsarAdmin();
this.pulsarAuth = mock(PulsarAuth.class);
this.pulsarMetadata = new PulsarMetadata(pulsarConnectorId, this.pulsarConnectorConfig, dispatchingRowDecoderFactory, this.pulsarAuth);
this.pulsarSplitManager = Mockito.spy(new PulsarSplitManager(pulsarConnectorId, this.pulsarConnectorConfig));
ManagedLedgerFactory managedLedgerFactory = mock(ManagedLedgerFactory.class);
when(managedLedgerFactory.openReadOnlyCursor(any(), any(), any())).then(new Answer<ReadOnlyCursor>() {
private Map<String, Integer> positions = new HashMap<>();
private int count = 0;
@Override
public ReadOnlyCursor answer(InvocationOnMock invocationOnMock) throws Throwable {
Object[] args = invocationOnMock.getArguments();
String topic = (String) args[0];
PositionImpl positionImpl = (PositionImpl) args[1];
int position = positionImpl.getEntryId() == -1 ? 0 : (int) positionImpl.getEntryId();
positions.put(topic, position);
String schemaName = TopicName.get(TopicName.get(topic.replaceAll("/persistent", "")).getPartitionedTopicName()).getSchemaName();
long entries = topicsToNumEntries.get(schemaName);
ReadOnlyCursorImpl readOnlyCursor = mock(ReadOnlyCursorImpl.class);
doReturn(entries).when(readOnlyCursor).getNumberOfEntries();
doAnswer(new Answer<Void>() {
@Override
public Void answer(InvocationOnMock invocation) throws Throwable {
Object[] args = invocation.getArguments();
Integer skipEntries = (Integer) args[0];
positions.put(topic, positions.get(topic) + skipEntries);
return null;
}
}).when(readOnlyCursor).skipEntries(anyInt());
when(readOnlyCursor.getReadPosition()).thenAnswer(new Answer<PositionImpl>() {
@Override
public PositionImpl answer(InvocationOnMock invocationOnMock) throws Throwable {
return PositionImpl.get(0, positions.get(topic));
}
});
doAnswer(new Answer() {
@Override
public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
Object[] args = invocationOnMock.getArguments();
Integer readEntries = (Integer) args[0];
AsyncCallbacks.ReadEntriesCallback callback = (AsyncCallbacks.ReadEntriesCallback) args[2];
Object ctx = args[3];
new Thread(new Runnable() {
@Override
public void run() {
List<Entry> entries = new LinkedList<>();
for (int i = 0; i < readEntries; i++) {
TestPulsarConnector.Bar bar = new TestPulsarConnector.Bar();
bar.field1 = fooFunctions.get("bar.field1").apply(count) == null ? null : (int) fooFunctions.get("bar.field1").apply(count);
bar.field2 = fooFunctions.get("bar.field2").apply(count) == null ? null : (String) fooFunctions.get("bar.field2").apply(count);
bar.field3 = (float) fooFunctions.get("bar.field3").apply(count);
Foo foo = new Foo();
foo.field1 = (int) fooFunctions.get("field1").apply(count);
foo.field2 = (String) fooFunctions.get("field2").apply(count);
foo.field3 = (float) fooFunctions.get("field3").apply(count);
foo.field4 = (double) fooFunctions.get("field4").apply(count);
foo.field5 = (boolean) fooFunctions.get("field5").apply(count);
foo.field6 = (long) fooFunctions.get("field6").apply(count);
foo.timestamp = (long) fooFunctions.get("timestamp").apply(count);
foo.time = (int) fooFunctions.get("time").apply(count);
foo.date = (int) fooFunctions.get("date").apply(count);
foo.decimal = (BigDecimal) fooFunctions.get("decimal").apply(count);
foo.bar = bar;
foo.field7 = (Foo.TestEnum) fooFunctions.get("field7").apply(count);
MessageMetadata messageMetadata = new MessageMetadata().setProducerName("test-producer").setSequenceId(positions.get(topic)).setPublishTime(System.currentTimeMillis());
Schema schema = topicsToSchemas.get(schemaName).getType() == SchemaType.AVRO ? AvroSchema.of(Foo.class) : JSONSchema.of(Foo.class);
ByteBuf payload = io.netty.buffer.Unpooled.copiedBuffer(schema.encode(foo));
ByteBuf byteBuf = serializeMetadataAndPayload(Commands.ChecksumType.Crc32c, messageMetadata, payload);
completedBytes += byteBuf.readableBytes();
entries.add(EntryImpl.create(0, positions.get(topic), byteBuf));
positions.put(topic, positions.get(topic) + 1);
count++;
}
callback.readEntriesComplete(entries, ctx);
}
}).start();
return null;
}
}).when(readOnlyCursor).asyncReadEntries(anyInt(), anyLong(), any(), any(), any());
when(readOnlyCursor.hasMoreEntries()).thenAnswer(new Answer<Boolean>() {
@Override
public Boolean answer(InvocationOnMock invocationOnMock) throws Throwable {
return positions.get(topic) < entries;
}
});
when(readOnlyCursor.findNewestMatching(any(), any())).then(new Answer<Position>() {
@Override
public Position answer(InvocationOnMock invocationOnMock) throws Throwable {
Object[] args = invocationOnMock.getArguments();
com.google.common.base.Predicate<Entry> predicate = (com.google.common.base.Predicate<Entry>) args[1];
String schemaName = TopicName.get(TopicName.get(topic.replaceAll("/persistent", "")).getPartitionedTopicName()).getSchemaName();
List<Entry> entries = getTopicEntries(schemaName);
Integer target = null;
for (int i = entries.size() - 1; i >= 0; i--) {
Entry entry = entries.get(i);
if (predicate.apply(entry)) {
target = i;
break;
}
}
return target == null ? null : new PositionImpl(0, target);
}
});
when(readOnlyCursor.getNumberOfEntries(any())).then(new Answer<Long>() {
@Override
public Long answer(InvocationOnMock invocationOnMock) throws Throwable {
Object[] args = invocationOnMock.getArguments();
com.google.common.collect.Range<PositionImpl> range = (com.google.common.collect.Range<PositionImpl>) args[0];
return (range.upperEndpoint().getEntryId() + 1) - range.lowerEndpoint().getEntryId();
}
});
when(readOnlyCursor.getCurrentLedgerInfo()).thenReturn(MLDataFormats.ManagedLedgerInfo.LedgerInfo.newBuilder().setLedgerId(0).build());
return readOnlyCursor;
}
});
PulsarConnectorCache.instance = mock(PulsarConnectorCache.class);
when(PulsarConnectorCache.instance.getManagedLedgerFactory()).thenReturn(managedLedgerFactory);
for (Map.Entry<TopicName, PulsarSplit> split : splits.entrySet()) {
PulsarRecordCursor pulsarRecordCursor = spy(new PulsarRecordCursor(topicsToColumnHandles.get(split.getKey()), split.getValue(), pulsarConnectorConfig, managedLedgerFactory, new ManagedLedgerConfig(), new PulsarConnectorMetricsTracker(new NullStatsProvider()), dispatchingRowDecoderFactory));
this.pulsarRecordCursors.put(split.getKey(), pulsarRecordCursor);
}
}
use of org.apache.bookkeeper.mledger.ReadOnlyCursor in project incubator-pulsar by apache.
the class TestPulsarRecordCursor method mockKeyValueSchemaPulsarRecordCursor.
/**
* mock a simple PulsarRecordCursor for KeyValueSchema test.
* @param entriesNum
* @param topicName
* @param schema
* @param message
* @param ColumnHandles
* @return
* @throws Exception
*/
private PulsarRecordCursor mockKeyValueSchemaPulsarRecordCursor(final Long entriesNum, final TopicName topicName, final KeyValueSchemaImpl schema, KeyValue message, List<PulsarColumnHandle> ColumnHandles) throws Exception {
ManagedLedgerFactory managedLedgerFactory = mock(ManagedLedgerFactory.class);
when(managedLedgerFactory.openReadOnlyCursor(any(), any(), any())).then(new Answer<ReadOnlyCursor>() {
private Map<String, Integer> positions = new HashMap<>();
@Override
public ReadOnlyCursor answer(InvocationOnMock invocationOnMock) throws Throwable {
Object[] args = invocationOnMock.getArguments();
String topic = (String) args[0];
PositionImpl positionImpl = (PositionImpl) args[1];
int position = positionImpl.getEntryId() == -1 ? 0 : (int) positionImpl.getEntryId();
positions.put(topic, position);
ReadOnlyCursorImpl readOnlyCursor = mock(ReadOnlyCursorImpl.class);
doReturn(entriesNum).when(readOnlyCursor).getNumberOfEntries();
doAnswer(new Answer<Void>() {
@Override
public Void answer(InvocationOnMock invocation) throws Throwable {
Object[] args = invocation.getArguments();
Integer skipEntries = (Integer) args[0];
positions.put(topic, positions.get(topic) + skipEntries);
return null;
}
}).when(readOnlyCursor).skipEntries(anyInt());
when(readOnlyCursor.getReadPosition()).thenAnswer(new Answer<PositionImpl>() {
@Override
public PositionImpl answer(InvocationOnMock invocationOnMock) throws Throwable {
return PositionImpl.get(0, positions.get(topic));
}
});
doAnswer(new Answer() {
@Override
public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
Object[] args = invocationOnMock.getArguments();
Integer readEntries = (Integer) args[0];
AsyncCallbacks.ReadEntriesCallback callback = (AsyncCallbacks.ReadEntriesCallback) args[2];
Object ctx = args[3];
new Thread(new Runnable() {
@Override
public void run() {
List<Entry> entries = new LinkedList<>();
for (int i = 0; i < readEntries; i++) {
MessageMetadata messageMetadata = new MessageMetadata().setProducerName("test-producer").setSequenceId(positions.get(topic)).setPublishTime(System.currentTimeMillis());
if (i % 2 == 0) {
messageMetadata.setSchemaVersion(new LongSchemaVersion(1L).bytes());
}
if (KeyValueEncodingType.SEPARATED.equals(schema.getKeyValueEncodingType())) {
messageMetadata.setPartitionKey(new String(schema.getKeySchema().encode(message.getKey()), Charset.forName("UTF-8"))).setPartitionKeyB64Encoded(false);
}
ByteBuf dataPayload = io.netty.buffer.Unpooled.copiedBuffer(schema.encode(message));
ByteBuf byteBuf = serializeMetadataAndPayload(Commands.ChecksumType.Crc32c, messageMetadata, dataPayload);
entries.add(EntryImpl.create(0, positions.get(topic), byteBuf));
positions.put(topic, positions.get(topic) + 1);
}
callback.readEntriesComplete(entries, ctx);
}
}).start();
return null;
}
}).when(readOnlyCursor).asyncReadEntries(anyInt(), anyLong(), any(), any(), any());
when(readOnlyCursor.hasMoreEntries()).thenAnswer(new Answer<Boolean>() {
@Override
public Boolean answer(InvocationOnMock invocationOnMock) throws Throwable {
return positions.get(topic) < entriesNum;
}
});
when(readOnlyCursor.getNumberOfEntries(any())).then(new Answer<Long>() {
@Override
public Long answer(InvocationOnMock invocationOnMock) throws Throwable {
Object[] args = invocationOnMock.getArguments();
com.google.common.collect.Range<PositionImpl> range = (com.google.common.collect.Range<PositionImpl>) args[0];
return (range.upperEndpoint().getEntryId() + 1) - range.lowerEndpoint().getEntryId();
}
});
when(readOnlyCursor.getCurrentLedgerInfo()).thenReturn(MLDataFormats.ManagedLedgerInfo.LedgerInfo.newBuilder().setLedgerId(0).build());
return readOnlyCursor;
}
});
ObjectMapper objectMapper = new ObjectMapper();
PulsarSplit split = new PulsarSplit(0, pulsarConnectorId.toString(), topicName.getNamespace(), topicName.getLocalName(), topicName.getLocalName(), entriesNum, new String(schema.getSchemaInfo().getSchema(), "ISO8859-1"), schema.getSchemaInfo().getType(), 0, entriesNum, 0, 0, TupleDomain.all(), objectMapper.writeValueAsString(schema.getSchemaInfo().getProperties()), null);
PulsarRecordCursor pulsarRecordCursor = spy(new PulsarRecordCursor(ColumnHandles, split, pulsarConnectorConfig, managedLedgerFactory, new ManagedLedgerConfig(), new PulsarConnectorMetricsTracker(new NullStatsProvider()), dispatchingRowDecoderFactory));
PulsarSqlSchemaInfoProvider pulsarSqlSchemaInfoProvider = mock(PulsarSqlSchemaInfoProvider.class);
when(pulsarSqlSchemaInfoProvider.getSchemaByVersion(any())).thenReturn(completedFuture(schema.getSchemaInfo()));
pulsarRecordCursor.setPulsarSqlSchemaInfoProvider(pulsarSqlSchemaInfoProvider);
return pulsarRecordCursor;
}
use of org.apache.bookkeeper.mledger.ReadOnlyCursor in project incubator-pulsar by apache.
the class PulsarSplitManager method getSplitsForTopic.
@VisibleForTesting
Collection<PulsarSplit> getSplitsForTopic(String topicNamePersistenceEncoding, ManagedLedgerFactory managedLedgerFactory, ManagedLedgerConfig managedLedgerConfig, int numSplits, PulsarTableHandle tableHandle, SchemaInfo schemaInfo, String tableName, TupleDomain<ColumnHandle> tupleDomain, OffloadPoliciesImpl offloadPolicies) throws ManagedLedgerException, InterruptedException, IOException {
ReadOnlyCursor readOnlyCursor = null;
try {
readOnlyCursor = managedLedgerFactory.openReadOnlyCursor(topicNamePersistenceEncoding, PositionImpl.EARLIEST, managedLedgerConfig);
long numEntries = readOnlyCursor.getNumberOfEntries();
if (numEntries <= 0) {
return Collections.EMPTY_LIST;
}
PredicatePushdownInfo predicatePushdownInfo = PredicatePushdownInfo.getPredicatePushdownInfo(this.connectorId, tupleDomain, managedLedgerFactory, managedLedgerConfig, topicNamePersistenceEncoding, numEntries);
PositionImpl initialStartPosition;
if (predicatePushdownInfo != null) {
numEntries = predicatePushdownInfo.getNumOfEntries();
initialStartPosition = predicatePushdownInfo.getStartPosition();
} else {
initialStartPosition = (PositionImpl) readOnlyCursor.getReadPosition();
}
readOnlyCursor.close();
readOnlyCursor = managedLedgerFactory.openReadOnlyCursor(topicNamePersistenceEncoding, initialStartPosition, new ManagedLedgerConfig());
long remainder = numEntries % numSplits;
long avgEntriesPerSplit = numEntries / numSplits;
List<PulsarSplit> splits = new LinkedList<>();
for (int i = 0; i < numSplits; i++) {
long entriesForSplit = (remainder > i) ? avgEntriesPerSplit + 1 : avgEntriesPerSplit;
PositionImpl startPosition = (PositionImpl) readOnlyCursor.getReadPosition();
readOnlyCursor.skipEntries(Math.toIntExact(entriesForSplit));
PositionImpl endPosition = (PositionImpl) readOnlyCursor.getReadPosition();
PulsarSplit pulsarSplit = new PulsarSplit(i, this.connectorId, restoreNamespaceDelimiterIfNeeded(tableHandle.getSchemaName(), pulsarConnectorConfig), schemaInfo.getName(), tableName, entriesForSplit, new String(schemaInfo.getSchema(), "ISO8859-1"), schemaInfo.getType(), startPosition.getEntryId(), endPosition.getEntryId(), startPosition.getLedgerId(), endPosition.getLedgerId(), tupleDomain, objectMapper.writeValueAsString(schemaInfo.getProperties()), offloadPolicies);
splits.add(pulsarSplit);
}
return splits;
} finally {
if (readOnlyCursor != null) {
try {
readOnlyCursor.close();
} catch (Exception e) {
log.error(e);
}
}
}
}
Aggregations