Examples with CreateTransientSegment io.pravega.shared.protocol.netty.WireCommands.CreateTransientSegment used on opensource projects

Example 1 with CreateTransientSegment

use of io.pravega.shared.protocol.netty.WireCommands.CreateTransientSegment in project pravega by pravega.

the class LargeEventWriterTest method testRetriedErrors.

@Test(timeout = 5000)
public void testRetriedErrors() throws ConnectionFailedException, NoSuchSegmentException, AuthenticationException, SegmentSealedException {
    Segment segment = Segment.fromScopedName("foo/bar/1");
    MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
    MockController controller = new MockController("localhost", 0, connectionFactory, false);
    ClientConnection connection = Mockito.mock(ClientConnection.class);
    PravegaNodeUri location = new PravegaNodeUri("localhost", 0);
    connectionFactory.provideConnection(location, connection);
    EmptyTokenProviderImpl tokenProvider = new EmptyTokenProviderImpl();
    ArrayList<ByteBuffer> events = new ArrayList<>();
    events.add(ByteBuffer.allocate(1));
    AtomicBoolean failed = new AtomicBoolean(false);
    AtomicBoolean succeeded = new AtomicBoolean(false);
    answerRequest(connectionFactory, connection, location, SetupAppend.class, r -> new AppendSetup(r.getRequestId(), segment.getScopedName(), r.getWriterId(), WireCommands.NULL_ATTRIBUTE_VALUE));
    answerRequest(connectionFactory, connection, location, ConditionalBlockEnd.class, r -> {
        ByteBuf data = r.getData();
        return new DataAppended(r.getRequestId(), r.getWriterId(), r.getEventNumber(), r.getEventNumber() - 1, r.getExpectedOffset() + data.readableBytes());
    });
    answerRequest(connectionFactory, connection, location, MergeSegments.class, r -> {
        return new SegmentsMerged(r.getRequestId(), r.getSource(), r.getTarget(), -1);
    });
    Mockito.doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            CreateTransientSegment argument = (CreateTransientSegment) invocation.getArgument(0);
            failed.set(true);
            connectionFactory.getProcessor(location).process(new AuthTokenCheckFailed(argument.getRequestId(), "stacktrace", ErrorCode.TOKEN_EXPIRED));
            return null;
        }
    }).doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            CreateTransientSegment argument = (CreateTransientSegment) invocation.getArgument(0);
            succeeded.set(true);
            connectionFactory.getProcessor(location).process(new SegmentCreated(argument.getRequestId(), "transient-segment"));
            return null;
        }
    }).when(connection).send(any(CreateTransientSegment.class));
    LargeEventWriter writer = new LargeEventWriter(writerId, controller, connectionFactory);
    writer.writeLargeEvent(segment, events, tokenProvider, EventWriterConfig.builder().build());
    assertTrue(failed.getAndSet(false));
    assertTrue(succeeded.getAndSet(false));
    Mockito.doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            CreateTransientSegment argument = (CreateTransientSegment) invocation.getArgument(0);
            failed.set(true);
            connectionFactory.getProcessor(location).process(new WrongHost(argument.getRequestId(), "foo/bar/1", null, "stacktrace"));
            return null;
        }
    }).doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            CreateTransientSegment argument = (CreateTransientSegment) invocation.getArgument(0);
            succeeded.set(true);
            connectionFactory.getProcessor(location).process(new SegmentCreated(argument.getRequestId(), "transient-segment"));
            return null;
        }
    }).when(connection).send(any(CreateTransientSegment.class));
    writer = new LargeEventWriter(writerId, controller, connectionFactory);
    writer.writeLargeEvent(segment, events, tokenProvider, EventWriterConfig.builder().build());
    assertTrue(failed.getAndSet(false));
    assertTrue(succeeded.getAndSet(false));
    Mockito.doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            CreateTransientSegment argument = (CreateTransientSegment) invocation.getArgument(0);
            failed.set(true);
            connectionFactory.getProcessor(location).processingFailure(new ConnectionFailedException());
            return null;
        }
    }).doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            CreateTransientSegment argument = (CreateTransientSegment) invocation.getArgument(0);
            succeeded.set(true);
            connectionFactory.getProcessor(location).process(new SegmentCreated(argument.getRequestId(), "transient-segment"));
            return null;
        }
    }).when(connection).send(any(CreateTransientSegment.class));
    writer = new LargeEventWriter(writerId, controller, connectionFactory);
    writer.writeLargeEvent(segment, events, tokenProvider, EventWriterConfig.builder().build());
    assertTrue(failed.getAndSet(false));
    assertTrue(succeeded.getAndSet(false));
}

Example 2 with CreateTransientSegment

use of io.pravega.shared.protocol.netty.WireCommands.CreateTransientSegment in project pravega by pravega.

the class LargeEventWriterTest method testSegmentSealed.

@Test(timeout = 5000)
public void testSegmentSealed() throws ConnectionFailedException, SegmentSealedException {
    String scope = "scope";
    String streamName = "stream";
    StreamImpl stream = new StreamImpl(scope, streamName);
    Segment segment = new Segment(scope, streamName, 0);
    @Cleanup InlineExecutor executor = new InlineExecutor();
    EventWriterConfig config = EventWriterConfig.builder().enableLargeEvents(true).build();
    SegmentOutputStreamFactory streamFactory = Mockito.mock(SegmentOutputStreamFactory.class);
    MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
    MockController controller = spy(new MockController("localhost", 0, connectionFactory, false));
    controller.createScope(scope).join();
    controller.createStream(scope, streamName, StreamConfiguration.builder().build());
    PravegaNodeUri location = new PravegaNodeUri("localhost", 0);
    ClientConnection connection = Mockito.mock(ClientConnection.class);
    connectionFactory.provideConnection(location, connection);
    SegmentOutputStream outputStream = Mockito.mock(SegmentOutputStream.class);
    Mockito.when(streamFactory.createOutputStreamForSegment(any(), any(), any(), any())).thenReturn(outputStream);
    AtomicBoolean failed = new AtomicBoolean(false);
    AtomicBoolean succeeded = new AtomicBoolean(false);
    Mockito.doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            CreateTransientSegment argument = (CreateTransientSegment) invocation.getArgument(0);
            failed.set(true);
            connectionFactory.getProcessor(location).process(new SegmentIsSealed(argument.getRequestId(), segment.getScopedName(), "stacktrace", 0));
            return null;
        }
    }).doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            CreateTransientSegment argument = (CreateTransientSegment) invocation.getArgument(0);
            succeeded.set(true);
            connectionFactory.getProcessor(location).process(new SegmentCreated(argument.getRequestId(), "transient-segment"));
            return null;
        }
    }).when(connection).send(any(CreateTransientSegment.class));
    Mockito.doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            List<Long> predecessors = Arrays.asList(0L);
            return CompletableFuture.completedFuture(new StreamSegmentsWithPredecessors(ImmutableMap.of(new SegmentWithRange(new Segment(scope, streamName, NameUtils.computeSegmentId(1, 1)), 0.0, 0.5), predecessors, new SegmentWithRange(new Segment(scope, streamName, NameUtils.computeSegmentId(2, 1)), 0.5, 1.0), predecessors), ""));
        }
    }).when(controller).getSuccessors(segment);
    answerRequest(connectionFactory, connection, location, SetupAppend.class, r -> new AppendSetup(r.getRequestId(), segment.getScopedName(), r.getWriterId(), WireCommands.NULL_ATTRIBUTE_VALUE));
    answerRequest(connectionFactory, connection, location, ConditionalBlockEnd.class, r -> {
        ByteBuf data = r.getData();
        return new DataAppended(r.getRequestId(), r.getWriterId(), r.getEventNumber(), r.getEventNumber() - 1, r.getExpectedOffset() + data.readableBytes());
    });
    answerRequest(connectionFactory, connection, location, MergeSegments.class, r -> {
        return new SegmentsMerged(r.getRequestId(), r.getSource(), r.getTarget(), -1);
    });
    @Cleanup EventStreamWriter<byte[]> writer = new EventStreamWriterImpl<>(stream, "id", controller, streamFactory, new ByteArraySerializer(), config, executor, executor, connectionFactory);
    writer.writeEvent(new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8 });
    writer.writeEvent(new byte[Serializer.MAX_EVENT_SIZE * 2]);
    assertTrue(failed.get());
    assertTrue(succeeded.get());
    InOrder order = Mockito.inOrder(connection, outputStream);
    order.verify(outputStream).write(any(PendingEvent.class));
    order.verify(outputStream).flush();
    order.verify(connection, times(2)).send(any(CreateTransientSegment.class));
    order.verify(connection).send(any(SetupAppend.class));
    order.verify(connection, times(3)).send(any(ConditionalBlockEnd.class));
    order.verify(connection).send(any(MergeSegments.class));
    order.verify(connection).close();
    order.verifyNoMoreInteractions();
}

Example 3 with CreateTransientSegment

use of io.pravega.shared.protocol.netty.WireCommands.CreateTransientSegment in project pravega by pravega.

the class AppendProcessor method createTransientSegment.

@Override
public void createTransientSegment(CreateTransientSegment createTransientSegment) {
    String operation = "createTransientSegment";
    long traceId = LoggerHelpers.traceEnter(log, operation, createTransientSegment);
    Collection<AttributeUpdate> attributes = Arrays.asList(new AttributeUpdate(CREATION_TIME, AttributeUpdateType.None, System.currentTimeMillis()), new AttributeUpdate(ATTRIBUTE_SEGMENT_TYPE, AttributeUpdateType.None, SegmentType.TRANSIENT_SEGMENT.getValue()));
    String transientSegmentName = NameUtils.getTransientNameFromId(createTransientSegment.getParentSegment(), createTransientSegment.getWriterId());
    store.createStreamSegment(transientSegmentName, SegmentType.TRANSIENT_SEGMENT, attributes, TIMEOUT).thenAccept(v -> {
        transientSegmentNames.add(transientSegmentName);
        connection.send(new SegmentCreated(createTransientSegment.getRequestId(), transientSegmentName));
    }).exceptionally(e -> handleException(createTransientSegment.getWriterId(), createTransientSegment.getRequestId(), transientSegmentName, operation, e));
    LoggerHelpers.traceLeave(log, operation, traceId);
}

Example 4 with CreateTransientSegment

use of io.pravega.shared.protocol.netty.WireCommands.CreateTransientSegment in project pravega by pravega.

the class AppendProcessorTest method testCreateTransientSegment.

@Test
public void testCreateTransientSegment() {
    String parentSegmentName = "scope/stream/parentSegment";
    UUID writerId = UUID.randomUUID();
    StreamSegmentStore store = mock(StreamSegmentStore.class);
    ServerConnection connection = mock(ServerConnection.class);
    ConnectionTracker tracker = mock(ConnectionTracker.class);
    val mockedRecorder = Mockito.mock(SegmentStatsRecorder.class);
    @Cleanup AppendProcessor processor = AppendProcessor.defaultBuilder().store(store).connection(new TrackedConnection(connection, tracker)).statsRecorder(mockedRecorder).build();
    interceptCreateTransient(store, parentSegmentName);
    CreateTransientSegment createTransientSegment = new CreateTransientSegment(1, writerId, parentSegmentName, "");
    processor.createTransientSegment(createTransientSegment);
    verify(connection).send(new WireCommands.SegmentCreated(1, any()));
    verifyNoMoreInteractions(connection);
}

Example 5 with CreateTransientSegment

use of io.pravega.shared.protocol.netty.WireCommands.CreateTransientSegment in project pravega by pravega.

the class LargeEventWriter method write.

private void write(Segment parentSegment, List<ByteBuf> payloads, RawClient client, DelegationTokenProvider tokenProvider) throws TokenExpiredException, NoSuchSegmentException, AuthenticationException, SegmentSealedException, ConnectionFailedException {
    long requestId = client.getFlow().getNextSequenceNumber();
    log.debug("Writing large event to segment {} with writer id {}", parentSegment, writerId);
    String token = getThrowingException(tokenProvider.retrieveToken());
    CreateTransientSegment createSegment = new CreateTransientSegment(requestId, writerId, parentSegment.getScopedName(), token);
    SegmentCreated created = transformSegmentCreated(getThrowingException(client.sendRequest(requestId, createSegment)), parentSegment.getScopedName());
    requestId = client.getFlow().getNextSequenceNumber();
    SetupAppend setup = new SetupAppend(requestId, writerId, created.getSegment(), token);
    AppendSetup appendSetup = transformAppendSetup(getThrowingException(client.sendRequest(requestId, setup)), created.getSegment());
    if (appendSetup.getLastEventNumber() != WireCommands.NULL_ATTRIBUTE_VALUE) {
        throw new IllegalStateException("Server indicates that transient segment was already written to: " + created.getSegment());
    }
    long expectedOffset = 0;
    val futures = new ArrayList<CompletableFuture<Reply>>();
    for (int i = 0; i < payloads.size(); i++) {
        requestId = client.getFlow().getNextSequenceNumber();
        ByteBuf payload = payloads.get(i);
        val request = new ConditionalBlockEnd(writerId, i, expectedOffset, Unpooled.wrappedBuffer(payload), requestId);
        expectedOffset += payload.readableBytes();
        val reply = client.sendRequest(requestId, request);
        failFast(futures, created.getSegment());
        futures.add(reply);
    }
    for (int i = 0; i < futures.size(); i++) {
        transformDataAppended(getThrowingException(futures.get(i)), created.getSegment());
    }
    requestId = client.getFlow().getNextSequenceNumber();
    MergeSegments merge = new MergeSegments(requestId, parentSegment.getScopedName(), created.getSegment(), token);
    transformSegmentMerged(getThrowingException(client.sendRequest(requestId, merge)), created.getSegment());
}