use of io.pravega.shared.protocol.netty.WireCommands.ConditionalBlockEnd in project pravega by pravega.
the class LargeEventWriter method write.
private void write(Segment parentSegment, List<ByteBuf> payloads, RawClient client, DelegationTokenProvider tokenProvider) throws TokenExpiredException, NoSuchSegmentException, AuthenticationException, SegmentSealedException, ConnectionFailedException {
long requestId = client.getFlow().getNextSequenceNumber();
log.debug("Writing large event to segment {} with writer id {}", parentSegment, writerId);
String token = getThrowingException(tokenProvider.retrieveToken());
CreateTransientSegment createSegment = new CreateTransientSegment(requestId, writerId, parentSegment.getScopedName(), token);
SegmentCreated created = transformSegmentCreated(getThrowingException(client.sendRequest(requestId, createSegment)), parentSegment.getScopedName());
requestId = client.getFlow().getNextSequenceNumber();
SetupAppend setup = new SetupAppend(requestId, writerId, created.getSegment(), token);
AppendSetup appendSetup = transformAppendSetup(getThrowingException(client.sendRequest(requestId, setup)), created.getSegment());
if (appendSetup.getLastEventNumber() != WireCommands.NULL_ATTRIBUTE_VALUE) {
throw new IllegalStateException("Server indicates that transient segment was already written to: " + created.getSegment());
}
long expectedOffset = 0;
val futures = new ArrayList<CompletableFuture<Reply>>();
for (int i = 0; i < payloads.size(); i++) {
requestId = client.getFlow().getNextSequenceNumber();
ByteBuf payload = payloads.get(i);
val request = new ConditionalBlockEnd(writerId, i, expectedOffset, Unpooled.wrappedBuffer(payload), requestId);
expectedOffset += payload.readableBytes();
val reply = client.sendRequest(requestId, request);
failFast(futures, created.getSegment());
futures.add(reply);
}
for (int i = 0; i < futures.size(); i++) {
transformDataAppended(getThrowingException(futures.get(i)), created.getSegment());
}
requestId = client.getFlow().getNextSequenceNumber();
MergeSegments merge = new MergeSegments(requestId, parentSegment.getScopedName(), created.getSegment(), token);
transformSegmentMerged(getThrowingException(client.sendRequest(requestId, merge)), created.getSegment());
}
use of io.pravega.shared.protocol.netty.WireCommands.ConditionalBlockEnd in project pravega by pravega.
the class LargeEventWriterTest method testPipelining.
@Test(timeout = 5000)
public void testPipelining() throws NoSuchSegmentException, AuthenticationException, SegmentSealedException, ConnectionFailedException {
Segment segment = Segment.fromScopedName("foo/bar/1");
MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
MockController controller = new MockController("localhost", 0, connectionFactory, false);
ClientConnection connection = Mockito.mock(ClientConnection.class);
PravegaNodeUri location = new PravegaNodeUri("localhost", 0);
connectionFactory.provideConnection(location, connection);
ArrayList<ByteBuffer> buffers = new ArrayList<>();
buffers.add(ByteBuffer.allocate(Serializer.MAX_EVENT_SIZE - WireCommands.TYPE_PLUS_LENGTH_SIZE));
buffers.add(ByteBuffer.allocate(Serializer.MAX_EVENT_SIZE - WireCommands.TYPE_PLUS_LENGTH_SIZE));
buffers.add(ByteBuffer.allocate(Serializer.MAX_EVENT_SIZE - WireCommands.TYPE_PLUS_LENGTH_SIZE));
ArrayList<ConditionalBlockEnd> written = new ArrayList<>();
answerRequest(connectionFactory, connection, location, CreateTransientSegment.class, r -> new SegmentCreated(r.getRequestId(), "transient-segment"));
answerRequest(connectionFactory, connection, location, SetupAppend.class, r -> new AppendSetup(r.getRequestId(), segment.getScopedName(), r.getWriterId(), WireCommands.NULL_ATTRIBUTE_VALUE));
// If appends are not pipelined, the call to writeLargeEvents will stall waiting for the first reply.
Mockito.doAnswer(new Answer<Void>() {
@Override
public Void answer(InvocationOnMock invocation) throws Throwable {
ConditionalBlockEnd argument = (ConditionalBlockEnd) invocation.getArgument(0);
written.add(argument);
if (written.size() == buffers.size()) {
for (ConditionalBlockEnd append : written) {
connectionFactory.getProcessor(location).process(new DataAppended(append.getRequestId(), writerId, append.getEventNumber(), append.getEventNumber() - 1, append.getExpectedOffset() + append.getData().readableBytes()));
}
}
return null;
}
}).when(connection).send(any(ConditionalBlockEnd.class));
answerRequest(connectionFactory, connection, location, MergeSegments.class, r -> {
return new SegmentsMerged(r.getRequestId(), r.getSource(), r.getTarget(), -1);
});
LargeEventWriter writer = new LargeEventWriter(writerId, controller, connectionFactory);
EmptyTokenProviderImpl tokenProvider = new EmptyTokenProviderImpl();
writer.writeLargeEvent(segment, buffers, tokenProvider, EventWriterConfig.builder().build());
}
use of io.pravega.shared.protocol.netty.WireCommands.ConditionalBlockEnd in project pravega by pravega.
the class LargeEventWriterTest method testUnexpectedErrors.
@Test(timeout = 5000)
public void testUnexpectedErrors() throws ConnectionFailedException, NoSuchSegmentException, AuthenticationException, SegmentSealedException {
Segment segment = Segment.fromScopedName("foo/bar/1");
MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
MockController controller = new MockController("localhost", 0, connectionFactory, false);
ClientConnection connection = Mockito.mock(ClientConnection.class);
PravegaNodeUri location = new PravegaNodeUri("localhost", 0);
connectionFactory.provideConnection(location, connection);
EmptyTokenProviderImpl tokenProvider = new EmptyTokenProviderImpl();
EventWriterConfig config = EventWriterConfig.builder().initialBackoffMillis(0).build();
ArrayList<ByteBuffer> events = new ArrayList<>();
events.add(ByteBuffer.allocate(1));
AtomicBoolean failed = new AtomicBoolean(false);
AtomicBoolean succeeded = new AtomicBoolean(false);
answerRequest(connectionFactory, connection, location, CreateTransientSegment.class, r -> new SegmentCreated(r.getRequestId(), "transient-segment"));
answerRequest(connectionFactory, connection, location, SetupAppend.class, r -> new AppendSetup(r.getRequestId(), segment.getScopedName(), r.getWriterId(), WireCommands.NULL_ATTRIBUTE_VALUE));
Mockito.doAnswer(new Answer<Void>() {
@Override
public Void answer(InvocationOnMock invocation) throws Throwable {
ConditionalBlockEnd argument = (ConditionalBlockEnd) invocation.getArgument(0);
failed.set(true);
connectionFactory.getProcessor(location).process(new InvalidEventNumber(argument.getWriterId(), argument.getEventNumber(), "stacktrace"));
return null;
}
}).doAnswer(new Answer<Void>() {
@Override
public Void answer(InvocationOnMock invocation) throws Throwable {
ConditionalBlockEnd argument = (ConditionalBlockEnd) invocation.getArgument(0);
ByteBuf data = argument.getData();
succeeded.set(true);
connectionFactory.getProcessor(location).process(new DataAppended(argument.getRequestId(), argument.getWriterId(), argument.getEventNumber(), argument.getEventNumber() - 1, argument.getExpectedOffset() + data.readableBytes()));
return null;
}
}).when(connection).send(any(ConditionalBlockEnd.class));
answerRequest(connectionFactory, connection, location, MergeSegments.class, r -> {
return new SegmentsMerged(r.getRequestId(), r.getSource(), r.getTarget(), -1);
});
LargeEventWriter writer = new LargeEventWriter(writerId, controller, connectionFactory);
writer.writeLargeEvent(segment, events, tokenProvider, EventWriterConfig.builder().build());
assertTrue(failed.getAndSet(false));
assertTrue(succeeded.getAndSet(false));
answerRequest(connectionFactory, connection, location, CreateTransientSegment.class, r -> new SegmentCreated(r.getRequestId(), "transient-segment"));
answerRequest(connectionFactory, connection, location, SetupAppend.class, r -> new AppendSetup(r.getRequestId(), segment.getScopedName(), r.getWriterId(), WireCommands.NULL_ATTRIBUTE_VALUE));
Mockito.doAnswer(new Answer<Void>() {
@Override
public Void answer(InvocationOnMock invocation) throws Throwable {
ConditionalBlockEnd argument = (ConditionalBlockEnd) invocation.getArgument(0);
failed.set(true);
connectionFactory.getProcessor(location).process(new ConditionalCheckFailed(argument.getWriterId(), argument.getEventNumber(), argument.getRequestId()));
return null;
}
}).doAnswer(new Answer<Void>() {
@Override
public Void answer(InvocationOnMock invocation) throws Throwable {
ConditionalBlockEnd argument = (ConditionalBlockEnd) invocation.getArgument(0);
ByteBuf data = argument.getData();
succeeded.set(true);
connectionFactory.getProcessor(location).process(new DataAppended(argument.getRequestId(), argument.getWriterId(), argument.getEventNumber(), argument.getEventNumber() - 1, argument.getExpectedOffset() + data.readableBytes()));
return null;
}
}).when(connection).send(any(ConditionalBlockEnd.class));
answerRequest(connectionFactory, connection, location, MergeSegments.class, r -> {
return new SegmentsMerged(r.getRequestId(), r.getSource(), r.getTarget(), -1);
});
writer = new LargeEventWriter(writerId, controller, connectionFactory);
writer.writeLargeEvent(segment, events, tokenProvider, EventWriterConfig.builder().build());
assertTrue(failed.getAndSet(false));
assertTrue(succeeded.getAndSet(false));
}
Aggregations