Search in sources :

Example 61 with ClientConnection

use of io.pravega.client.connection.impl.ClientConnection in project pravega by pravega.

the class SegmentOutputStreamTest method testFlushWithMultipleConnectFailures.

@Test(timeout = 10000)
public void testFlushWithMultipleConnectFailures() throws Exception {
    UUID cid = UUID.randomUUID();
    PravegaNodeUri uri = new PravegaNodeUri("endpoint", SERVICE_PORT);
    RetryWithBackoff retryConfig = Retry.withExpBackoff(1, 1, 1);
    MockConnectionFactoryImpl cf = new MockConnectionFactoryImpl();
    ScheduledExecutorService executor = mock(ScheduledExecutorService.class);
    // Ensure task submitted to executor is run inline.
    implementAsDirectExecutor(executor);
    cf.setExecutor(executor);
    MockController controller = new MockController(uri.getEndpoint(), uri.getPort(), cf, true);
    ClientConnection connection = mock(ClientConnection.class);
    cf.provideConnection(uri, connection);
    SegmentOutputStreamImpl output = new SegmentOutputStreamImpl(SEGMENT, true, controller, cf, cid, segmentSealedCallback, retryConfig, DelegationTokenProviderFactory.createWithEmptyToken());
    try {
        output.reconnect();
        verify(connection).send(new SetupAppend(output.getRequestId(), cid, SEGMENT, ""));
        // Simulate a successful connection setup.
        cf.getProcessor(uri).appendSetup(new AppendSetup(output.getRequestId(), SEGMENT, cid, 0));
        // try sending an event.
        byte[] eventData = "test data".getBytes();
        CompletableFuture<Void> acked = new CompletableFuture<>();
        // this is an inflight event and the client will track it until there is a response from SSS.
        output.write(PendingEvent.withoutHeader(null, ByteBuffer.wrap(eventData), acked));
        verify(connection).send(new Append(SEGMENT, cid, 1, 1, Unpooled.wrappedBuffer(eventData), null, output.getRequestId()));
        reset(connection);
        // simulate a connection drop and verify if the writer tries to establish a new connection.
        cf.getProcessor(uri).connectionDropped();
        verify(connection).send(new SetupAppend(output.getRequestId(), cid, SEGMENT, ""));
        reset(connection);
        // Verify flush blocks until there is a response from SSS. Incase of connection error the client retries. If the
        // retry count more than the configuration ensure flush returns exceptionally.
        AssertExtensions.assertBlocks(() -> AssertExtensions.assertThrows(RetriesExhaustedException.class, output::flush), () -> cf.getProcessor(uri).connectionDropped());
        assertTrue("Connection is  exceptionally closed with RetriesExhaustedException", output.getConnection().isCompletedExceptionally());
        AssertExtensions.assertThrows(RetriesExhaustedException.class, () -> Futures.getThrowingException(output.getConnection()));
        // Verify that the inflight event future is completed exceptionally.
        AssertExtensions.assertThrows(RetriesExhaustedException.class, () -> Futures.getThrowingException(acked));
    } finally {
        // Verify that a close on the SegmentOutputStream does throw a RetriesExhaustedException.
        AssertExtensions.assertThrows(RetriesExhaustedException.class, output::close);
    }
}
Also used : ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) RetryWithBackoff(io.pravega.common.util.Retry.RetryWithBackoff) AppendSetup(io.pravega.shared.protocol.netty.WireCommands.AppendSetup) CompletableFuture(java.util.concurrent.CompletableFuture) Append(io.pravega.shared.protocol.netty.Append) SetupAppend(io.pravega.shared.protocol.netty.WireCommands.SetupAppend) PravegaNodeUri(io.pravega.shared.protocol.netty.PravegaNodeUri) RetriesExhaustedException(io.pravega.common.util.RetriesExhaustedException) MockConnectionFactoryImpl(io.pravega.client.stream.mock.MockConnectionFactoryImpl) SetupAppend(io.pravega.shared.protocol.netty.WireCommands.SetupAppend) MockController(io.pravega.client.stream.mock.MockController) ClientConnection(io.pravega.client.connection.impl.ClientConnection) UUID(java.util.UUID) Test(org.junit.Test)

Example 62 with ClientConnection

use of io.pravega.client.connection.impl.ClientConnection in project pravega by pravega.

the class SegmentOutputStreamTest method testAlreadySealedSegment.

@Test(timeout = 10000)
public void testAlreadySealedSegment() throws Exception {
    UUID cid = UUID.randomUUID();
    PravegaNodeUri uri = new PravegaNodeUri("endpoint", SERVICE_PORT);
    MockConnectionFactoryImpl cf = new MockConnectionFactoryImpl();
    cf.setExecutor(executorService());
    MockController controller = new MockController(uri.getEndpoint(), uri.getPort(), cf, true);
    ClientConnection connection = mock(ClientConnection.class);
    cf.provideConnection(uri, connection);
    InOrder order = Mockito.inOrder(connection);
    @SuppressWarnings("resource") SegmentOutputStreamImpl output = new SegmentOutputStreamImpl(SEGMENT, true, controller, cf, cid, segmentSealedCallback, RETRY_SCHEDULE, DelegationTokenProviderFactory.createWithEmptyToken());
    output.reconnect();
    order.verify(connection).send(new SetupAppend(output.getRequestId(), cid, SEGMENT, ""));
    cf.getProcessor(uri).segmentIsSealed(new WireCommands.SegmentIsSealed(output.getRequestId(), SEGMENT, "SomeException", 1));
    cf.getProcessor(uri).appendSetup(new AppendSetup(output.getRequestId(), SEGMENT, cid, 0));
    order.verifyNoMoreInteractions();
}
Also used : InOrder(org.mockito.InOrder) PravegaNodeUri(io.pravega.shared.protocol.netty.PravegaNodeUri) MockConnectionFactoryImpl(io.pravega.client.stream.mock.MockConnectionFactoryImpl) SetupAppend(io.pravega.shared.protocol.netty.WireCommands.SetupAppend) MockController(io.pravega.client.stream.mock.MockController) ClientConnection(io.pravega.client.connection.impl.ClientConnection) UUID(java.util.UUID) WireCommands(io.pravega.shared.protocol.netty.WireCommands) AppendSetup(io.pravega.shared.protocol.netty.WireCommands.AppendSetup) Test(org.junit.Test)

Example 63 with ClientConnection

use of io.pravega.client.connection.impl.ClientConnection in project pravega by pravega.

the class SegmentOutputStreamTest method testConnectionFailureWithSegmentSealed.

@Test(timeout = 10000)
public void testConnectionFailureWithSegmentSealed() throws Exception {
    UUID cid = UUID.randomUUID();
    PravegaNodeUri uri = new PravegaNodeUri("endpoint", SERVICE_PORT);
    MockConnectionFactoryImpl cf = new MockConnectionFactoryImpl();
    cf.setExecutor(executorService());
    MockController controller = new MockController(uri.getEndpoint(), uri.getPort(), cf, true);
    ClientConnection connection = mock(ClientConnection.class);
    cf.provideConnection(uri, connection);
    @SuppressWarnings("resource") SegmentOutputStreamImpl output = new SegmentOutputStreamImpl(SEGMENT, true, controller, cf, cid, segmentSealedCallback, RETRY_SCHEDULE, DelegationTokenProviderFactory.createWithEmptyToken());
    output.reconnect();
    InOrder inOrder = Mockito.inOrder(connection);
    inOrder.verify(connection).send(new SetupAppend(output.getRequestId(), cid, SEGMENT, ""));
    cf.getProcessor(uri).appendSetup(new AppendSetup(output.getRequestId(), SEGMENT, cid, 0));
    // connection is setup .
    ByteBuffer data = getBuffer("test");
    CompletableFuture<Void> acked = new CompletableFuture<>();
    Append append = new Append(SEGMENT, cid, 1, 1, Unpooled.wrappedBuffer(data), null, output.getRequestId());
    CompletableFuture<Void> acked2 = new CompletableFuture<>();
    Append append2 = new Append(SEGMENT, cid, 2, 1, Unpooled.wrappedBuffer(data), null, output.getRequestId());
    doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            // simulate in a race with segment is sealed callback and a connection drop.
            cf.getProcessor(uri).connectionDropped();
            // wait until the writer reattempts to establish a connection to simulate the race.
            verify(connection, times(2)).send(new SetupAppend(output.getRequestId(), cid, SEGMENT, ""));
            // the connection object throws a throws a
            throw new ConnectionFailedException();
        }
    }).when(connection).send(append);
    // trigger the first write
    output.write(PendingEvent.withoutHeader(null, data, acked));
    AssertExtensions.assertBlocks(() -> {
        // the below write will be blocked since the connection setup is not complete.
        output.write(PendingEvent.withoutHeader(null, data, acked2));
    }, () -> {
        // simulate a race between segmentIsSealed response and appendSetup.
        cf.getProcessor(uri).segmentIsSealed(new WireCommands.SegmentIsSealed(output.getRequestId(), SEGMENT, "Segment is sealed", 1));
        // invoke the segment is sealed to ensure
        cf.getProcessor(uri).appendSetup(new AppendSetup(output.getRequestId(), SEGMENT, cid, 0));
        // ensure the reconnect is invoked inline.
        output.reconnect();
    });
    CompletableFuture<Void> acked3 = new CompletableFuture<>();
    output.write(PendingEvent.withoutHeader(null, data, acked3));
    inOrder.verify(connection).send(append);
    inOrder.verify(connection).send(new SetupAppend(output.getRequestId(), cid, SEGMENT, ""));
    inOrder.verify(connection).send(eq(append2));
    // the setup append should not transmit the inflight events given that the segment is sealed.
    inOrder.verifyNoMoreInteractions();
    assertFalse(acked.isDone());
    assertFalse(acked2.isDone());
    assertFalse(acked3.isDone());
}
Also used : InOrder(org.mockito.InOrder) ByteBuffer(java.nio.ByteBuffer) AppendSetup(io.pravega.shared.protocol.netty.WireCommands.AppendSetup) CompletableFuture(java.util.concurrent.CompletableFuture) Append(io.pravega.shared.protocol.netty.Append) SetupAppend(io.pravega.shared.protocol.netty.WireCommands.SetupAppend) PravegaNodeUri(io.pravega.shared.protocol.netty.PravegaNodeUri) InvocationOnMock(org.mockito.invocation.InvocationOnMock) MockConnectionFactoryImpl(io.pravega.client.stream.mock.MockConnectionFactoryImpl) SetupAppend(io.pravega.shared.protocol.netty.WireCommands.SetupAppend) MockController(io.pravega.client.stream.mock.MockController) ClientConnection(io.pravega.client.connection.impl.ClientConnection) UUID(java.util.UUID) WireCommands(io.pravega.shared.protocol.netty.WireCommands) ConnectionFailedException(io.pravega.shared.protocol.netty.ConnectionFailedException) Test(org.junit.Test)

Example 64 with ClientConnection

use of io.pravega.client.connection.impl.ClientConnection in project pravega by pravega.

the class SegmentOutputStreamTest method testSegmentSealedFollowedbyConnectionDrop.

@Test(timeout = 10000)
public void testSegmentSealedFollowedbyConnectionDrop() throws Exception {
    @Cleanup("shutdownNow") ScheduledExecutorService executor = ExecutorServiceHelpers.newScheduledThreadPool(2, "netty-callback");
    // Segment sealed callback will finish execution only when the releaseCallbackLatch is released;
    ReusableLatch releaseCallbackLatch = new ReusableLatch(false);
    ReusableLatch callBackInvokedLatch = new ReusableLatch(false);
    final Consumer<Segment> segmentSealedCallback = segment -> Exceptions.handleInterrupted(() -> {
        callBackInvokedLatch.release();
        releaseCallbackLatch.await();
    });
    // Setup mocks.
    UUID cid = UUID.randomUUID();
    PravegaNodeUri uri = new PravegaNodeUri("endpoint", SERVICE_PORT);
    MockConnectionFactoryImpl cf = new MockConnectionFactoryImpl();
    cf.setExecutor(executorService());
    MockController controller = new MockController(uri.getEndpoint(), uri.getPort(), cf, true);
    // Mock client connection that is returned for every invocation of ConnectionFactory#establishConnection.
    ClientConnection connection = mock(ClientConnection.class);
    cf.provideConnection(uri, connection);
    InOrder order = Mockito.inOrder(connection);
    // Create a Segment writer.
    @SuppressWarnings("resource") SegmentOutputStreamImpl output = new SegmentOutputStreamImpl(SEGMENT, true, controller, cf, cid, segmentSealedCallback, RETRY_SCHEDULE, DelegationTokenProviderFactory.createWithEmptyToken());
    // trigger establishment of connection.
    output.reconnect();
    // Verify if SetupAppend is sent over the connection.
    order.verify(connection).send(new SetupAppend(output.getRequestId(), cid, SEGMENT, ""));
    cf.getProcessor(uri).appendSetup(new AppendSetup(output.getRequestId(), SEGMENT, cid, 0));
    // Write an event and ensure inflight has an event.
    ByteBuffer data = getBuffer("test");
    CompletableFuture<Void> ack = new CompletableFuture<>();
    output.write(PendingEvent.withoutHeader(null, data, ack));
    order.verify(connection).send(new Append(SEGMENT, cid, 1, 1, Unpooled.wrappedBuffer(data), null, output.getRequestId()));
    assertFalse(ack.isDone());
    // Simulate a SegmentIsSealed WireCommand from SegmentStore.
    executor.submit(() -> cf.getProcessor(uri).segmentIsSealed(new WireCommands.SegmentIsSealed(output.getRequestId(), SEGMENT, "SomeException", 1)));
    // Wait until callback invocation has been triggered, but has not completed.
    // If the callback is not invoked the test will fail due to a timeout.
    callBackInvokedLatch.await();
    // Now trigger a connection drop netty callback and wait until it is executed.
    executor.submit(() -> cf.getProcessor(uri).connectionDropped()).get();
    // close is invoked on the connection.
    order.verify(connection).close();
    // Verify no further reconnection attempts which involves sending of SetupAppend wire command.
    order.verifyNoMoreInteractions();
    // Release latch to ensure the callback is completed.
    releaseCallbackLatch.release();
    // Verify no further reconnection attempts which involves sending of SetupAppend wire command.
    order.verifyNoMoreInteractions();
    // Trigger a reconnect again and verify if any new connections are initiated.
    output.reconnect();
    // Reconnect operation will be executed on the executor service.
    ScheduledExecutorService service = executorService();
    service.shutdown();
    // Wait until all the tasks for reconnect have been completed.
    service.awaitTermination(10, TimeUnit.SECONDS);
    // Verify no further reconnection attempts which involves sending of SetupAppend wire command.
    order.verifyNoMoreInteractions();
}
Also used : ArgumentMatchers(org.mockito.ArgumentMatchers) Retry(io.pravega.common.util.Retry) AssertExtensions(io.pravega.test.common.AssertExtensions) ArgumentMatchers.eq(org.mockito.ArgumentMatchers.eq) Cleanup(lombok.Cleanup) ByteBuffer(java.nio.ByteBuffer) Unpooled(io.netty.buffer.Unpooled) Mockito.doThrow(org.mockito.Mockito.doThrow) MockController(io.pravega.client.stream.mock.MockController) ClientConnection(io.pravega.client.connection.impl.ClientConnection) Mockito.verifyNoMoreInteractions(org.mockito.Mockito.verifyNoMoreInteractions) AccessOperation(io.pravega.shared.security.auth.AccessOperation) Mockito.doAnswer(org.mockito.Mockito.doAnswer) PravegaNodeUri(io.pravega.shared.protocol.netty.PravegaNodeUri) Assert.fail(org.junit.Assert.fail) LeakDetectorTestSuite(io.pravega.test.common.LeakDetectorTestSuite) AppendSetup(io.pravega.shared.protocol.netty.WireCommands.AppendSetup) AssertExtensions.assertThrows(io.pravega.test.common.AssertExtensions.assertThrows) DelegationTokenProviderFactory(io.pravega.client.security.auth.DelegationTokenProviderFactory) UUID(java.util.UUID) RetriesExhaustedException(io.pravega.common.util.RetriesExhaustedException) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) Assert.assertFalse(org.junit.Assert.assertFalse) Mockito.inOrder(org.mockito.Mockito.inOrder) Futures(io.pravega.common.concurrent.Futures) ReplyProcessor(io.pravega.shared.protocol.netty.ReplyProcessor) Mockito.mock(org.mockito.Mockito.mock) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) ConnectionFailedException(io.pravega.shared.protocol.netty.ConnectionFailedException) Exceptions(io.pravega.common.Exceptions) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Callable(java.util.concurrent.Callable) CompletableFuture(java.util.concurrent.CompletableFuture) Mockito.spy(org.mockito.Mockito.spy) PendingEvent(io.pravega.client.stream.impl.PendingEvent) Append(io.pravega.shared.protocol.netty.Append) Answer(org.mockito.stubbing.Answer) SetupAppend(io.pravega.shared.protocol.netty.WireCommands.SetupAppend) InvocationOnMock(org.mockito.invocation.InvocationOnMock) ImmutableList(com.google.common.collect.ImmutableList) CompletedCallback(io.pravega.client.connection.impl.ClientConnection.CompletedCallback) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) ReusableLatch(io.pravega.common.util.ReusableLatch) RetryWithBackoff(io.pravega.common.util.Retry.RetryWithBackoff) InOrder(org.mockito.InOrder) ConnectionPool(io.pravega.client.connection.impl.ConnectionPool) MockConnectionFactoryImpl(io.pravega.client.stream.mock.MockConnectionFactoryImpl) lombok.val(lombok.val) Assert.assertTrue(org.junit.Assert.assertTrue) IOException(java.io.IOException) Test(org.junit.Test) Mockito.times(org.mockito.Mockito.times) WireCommands(io.pravega.shared.protocol.netty.WireCommands) Mockito.verify(org.mockito.Mockito.verify) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) Mockito(org.mockito.Mockito) Mockito.never(org.mockito.Mockito.never) InlineExecutor(io.pravega.test.common.InlineExecutor) ExecutorServiceHelpers(io.pravega.common.concurrent.ExecutorServiceHelpers) Collections(java.util.Collections) Mockito.reset(org.mockito.Mockito.reset) Assert.assertEquals(org.junit.Assert.assertEquals) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) InOrder(org.mockito.InOrder) Cleanup(lombok.Cleanup) ByteBuffer(java.nio.ByteBuffer) AppendSetup(io.pravega.shared.protocol.netty.WireCommands.AppendSetup) CompletableFuture(java.util.concurrent.CompletableFuture) ReusableLatch(io.pravega.common.util.ReusableLatch) Append(io.pravega.shared.protocol.netty.Append) SetupAppend(io.pravega.shared.protocol.netty.WireCommands.SetupAppend) PravegaNodeUri(io.pravega.shared.protocol.netty.PravegaNodeUri) MockConnectionFactoryImpl(io.pravega.client.stream.mock.MockConnectionFactoryImpl) SetupAppend(io.pravega.shared.protocol.netty.WireCommands.SetupAppend) MockController(io.pravega.client.stream.mock.MockController) ClientConnection(io.pravega.client.connection.impl.ClientConnection) UUID(java.util.UUID) Test(org.junit.Test)

Example 65 with ClientConnection

use of io.pravega.client.connection.impl.ClientConnection in project pravega by pravega.

the class SegmentOutputStreamTest method testReconnectOnBadAcks.

@Test(timeout = 10000)
public void testReconnectOnBadAcks() throws ConnectionFailedException {
    UUID cid = UUID.randomUUID();
    PravegaNodeUri uri = new PravegaNodeUri("endpoint", SERVICE_PORT);
    MockConnectionFactoryImpl cf = spy(new MockConnectionFactoryImpl());
    ScheduledExecutorService executor = mock(ScheduledExecutorService.class);
    // Ensure task submitted to executor is run inline.
    implementAsDirectExecutor(executor);
    cf.setExecutor(executor);
    MockController controller = new MockController(uri.getEndpoint(), uri.getPort(), cf, true);
    ClientConnection connection = mock(ClientConnection.class);
    cf.provideConnection(uri, connection);
    InOrder order = Mockito.inOrder(connection);
    @SuppressWarnings("resource") SegmentOutputStreamImpl output = new SegmentOutputStreamImpl(SEGMENT, true, controller, cf, cid, segmentSealedCallback, RETRY_SCHEDULE, DelegationTokenProviderFactory.createWithEmptyToken());
    output.reconnect();
    order.verify(connection).send(new SetupAppend(output.getRequestId(), cid, SEGMENT, ""));
    cf.getProcessor(uri).appendSetup(new AppendSetup(1, SEGMENT, cid, 0));
    ByteBuffer data = getBuffer("test");
    CompletableFuture<Void> acked1 = new CompletableFuture<>();
    output.write(PendingEvent.withoutHeader(null, data, acked1));
    order.verify(connection).send(new Append(SEGMENT, cid, 1, 1, Unpooled.wrappedBuffer(data), null, output.getRequestId()));
    assertEquals(false, acked1.isDone());
    AssertExtensions.assertBlocks(() -> output.flush(), () -> cf.getProcessor(uri).dataAppended(new WireCommands.DataAppended(output.getRequestId(), cid, 1, 0, -1)));
    assertEquals(false, acked1.isCompletedExceptionally());
    assertEquals(true, acked1.isDone());
    order.verify(connection).send(new WireCommands.KeepAlive());
    // simulate bad ack
    CompletableFuture<Void> acked2 = new CompletableFuture<>();
    output.write(PendingEvent.withoutHeader(null, data, acked2));
    order.verify(connection).send(new Append(SEGMENT, cid, 2, 1, Unpooled.wrappedBuffer(data), null, output.getRequestId()));
    assertEquals(false, acked2.isDone());
    cf.getProcessor(uri).dataAppended(new WireCommands.DataAppended(output.getRequestId(), cid, 2, 3, -1));
    // check that client reconnected
    verify(cf, times(2)).establishConnection(any(), any());
}
Also used : ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) InOrder(org.mockito.InOrder) ByteBuffer(java.nio.ByteBuffer) AppendSetup(io.pravega.shared.protocol.netty.WireCommands.AppendSetup) CompletableFuture(java.util.concurrent.CompletableFuture) Append(io.pravega.shared.protocol.netty.Append) SetupAppend(io.pravega.shared.protocol.netty.WireCommands.SetupAppend) PravegaNodeUri(io.pravega.shared.protocol.netty.PravegaNodeUri) MockConnectionFactoryImpl(io.pravega.client.stream.mock.MockConnectionFactoryImpl) SetupAppend(io.pravega.shared.protocol.netty.WireCommands.SetupAppend) MockController(io.pravega.client.stream.mock.MockController) ClientConnection(io.pravega.client.connection.impl.ClientConnection) UUID(java.util.UUID) WireCommands(io.pravega.shared.protocol.netty.WireCommands) Test(org.junit.Test)

Aggregations

ClientConnection (io.pravega.client.connection.impl.ClientConnection)87 PravegaNodeUri (io.pravega.shared.protocol.netty.PravegaNodeUri)81 MockController (io.pravega.client.stream.mock.MockController)79 MockConnectionFactoryImpl (io.pravega.client.stream.mock.MockConnectionFactoryImpl)78 Test (org.junit.Test)77 Cleanup (lombok.Cleanup)51 InvocationOnMock (org.mockito.invocation.InvocationOnMock)43 WireCommands (io.pravega.shared.protocol.netty.WireCommands)40 SetupAppend (io.pravega.shared.protocol.netty.WireCommands.SetupAppend)35 AppendSetup (io.pravega.shared.protocol.netty.WireCommands.AppendSetup)34 UUID (java.util.UUID)34 ReplyProcessor (io.pravega.shared.protocol.netty.ReplyProcessor)33 ByteBuffer (java.nio.ByteBuffer)31 CompletableFuture (java.util.concurrent.CompletableFuture)30 InOrder (org.mockito.InOrder)27 Append (io.pravega.shared.protocol.netty.Append)23 ConnectionFailedException (io.pravega.shared.protocol.netty.ConnectionFailedException)23 Answer (org.mockito.stubbing.Answer)17 ScheduledExecutorService (java.util.concurrent.ScheduledExecutorService)16 ArrayList (java.util.ArrayList)14