Examples with MockTime com.github.ambry.utils.MockTime used on opensource projects

Example 21 with MockTime

use of com.github.ambry.utils.MockTime in project ambry by linkedin.

the class ReplicationTestHelper method createStorageManagerAndReplicationManager.

/**
 * Helper method to create storage manager and replication manager
 * @param clusterMap {@link ClusterMap} to use
 * @param clusterMapConfig {@link ClusterMapConfig} to use
 * @param clusterParticipant {@link com.github.ambry.clustermap.ClusterParticipant} for listener registration.
 * @return a pair of storage manager and replication manager
 * @throws Exception
 */
protected Pair<StorageManager, ReplicationManager> createStorageManagerAndReplicationManager(ClusterMap clusterMap, ClusterMapConfig clusterMapConfig, MockHelixParticipant clusterParticipant, ConnectionPool mockConnectionPool) throws Exception {
    StoreConfig storeConfig = new StoreConfig(verifiableProperties);
    DataNodeId dataNodeId = clusterMap.getDataNodeIds().get(0);
    MockStoreKeyConverterFactory storeKeyConverterFactory = new MockStoreKeyConverterFactory(null, null);
    storeKeyConverterFactory.setConversionMap(new HashMap<>());
    storeKeyConverterFactory.setReturnInputIfAbsent(true);
    StoreKeyFactory storeKeyFactory = new BlobIdFactory(clusterMap);
    StorageManager storageManager = new StorageManager(storeConfig, new DiskManagerConfig(verifiableProperties), Utils.newScheduler(1, true), new MetricRegistry(), null, clusterMap, dataNodeId, null, clusterParticipant == null ? null : Collections.singletonList(clusterParticipant), new MockTime(), null, new InMemAccountService(false, false));
    storageManager.start();
    MockReplicationManager replicationManager = new MockReplicationManager(replicationConfig, clusterMapConfig, storeConfig, storageManager, clusterMap, dataNodeId, storeKeyConverterFactory, clusterParticipant, mockConnectionPool, new MockFindTokenHelper(storeKeyFactory, replicationConfig), BlobIdTransformer.class.getName(), storeKeyFactory, time);
    return new Pair<>(storageManager, replicationManager);
}

Example 22 with MockTime

use of com.github.ambry.utils.MockTime in project ambry by linkedin.

the class NonBlockingRouterTest method testSuccessfulPutDataChunkDelete.

/**
 * Test that even when a composite blob put succeeds, the slipped put data chunks are deleted.
 */
@Test
public void testSuccessfulPutDataChunkDelete() throws Exception {
    try {
        // This test is somehow probabilistic. Since it is not possible to devise a mocking to enforce the occurrence of
        // slipped puts given we cannot control the order of the hosts requests are sent and not all requests are sent when
        // put requests are guaranteed to fail/succeed. So, we are setting the number of chunks and max attempts high enough
        // to guarantee that slipped puts would eventually happen and operation would succeed.
        maxPutChunkSize = PUT_CONTENT_SIZE / 8;
        final int NUM_MAX_ATTEMPTS = 100;
        Properties props = getNonBlockingRouterProperties("DC1");
        props.setProperty("router.max.slipped.put.attempts", Integer.toString(NUM_MAX_ATTEMPTS));
        VerifiableProperties verifiableProperties = new VerifiableProperties((props));
        RouterConfig routerConfig = new RouterConfig(verifiableProperties);
        MockClusterMap mockClusterMap = new MockClusterMap();
        MockTime mockTime = new MockTime();
        MockServerLayout mockServerLayout = new MockServerLayout(mockClusterMap);
        // Since this test wants to ensure that successfully put data chunks are deleted when the overall put operation
        // succeeds but some chunks succeed only after a retry, it uses a notification system to track the deletions.
        final CountDownLatch deletesDoneLatch = new CountDownLatch(1);
        final Map<String, String> blobsThatAreDeleted = new HashMap<>();
        LoggingNotificationSystem deleteTrackingNotificationSystem = new LoggingNotificationSystem() {

            @Override
            public void onBlobDeleted(String blobId, String serviceId, Account account, Container container) {
                blobsThatAreDeleted.put(blobId, serviceId);
                deletesDoneLatch.countDown();
            }
        };
        router = new NonBlockingRouter(routerConfig, new NonBlockingRouterMetrics(mockClusterMap, routerConfig), new MockNetworkClientFactory(verifiableProperties, mockSelectorState, MAX_PORTS_PLAIN_TEXT, MAX_PORTS_SSL, CHECKOUT_TIMEOUT_MS, mockServerLayout, mockTime), deleteTrackingNotificationSystem, mockClusterMap, kms, cryptoService, cryptoJobHandler, accountService, mockTime, MockClusterMap.DEFAULT_PARTITION_CLASS);
        setOperationParams();
        // In each DC, set up the servers such that one node always succeeds and the other nodes return an unknown_error and
        // no_error alternately. This will make it with a very high probability that there will at least be a time that a
        // put will succeed on a node but will fail on the other two.
        List<DataNodeId> dataNodeIds = mockClusterMap.getDataNodeIds();
        List<ServerErrorCode> serverErrorList = new ArrayList<>();
        for (int i = 0; i < NUM_MAX_ATTEMPTS; i++) {
            serverErrorList.add(ServerErrorCode.Unknown_Error);
            serverErrorList.add(ServerErrorCode.No_Error);
        }
        Set<String> healthyNodeDC = new HashSet<>();
        for (DataNodeId dataNodeId : dataNodeIds) {
            MockServer server = mockServerLayout.getMockServer(dataNodeId.getHostname(), dataNodeId.getPort());
            if (healthyNodeDC.contains(dataNodeId.getDatacenterName())) {
                server.setServerErrors(serverErrorList);
            } else {
                server.resetServerErrors();
            }
            healthyNodeDC.add(dataNodeId.getDatacenterName());
        }
        // Submit the put operation and wait for it to succeed.
        String blobId = router.putBlob(putBlobProperties, putUserMetadata, putChannel, new PutBlobOptionsBuilder().build()).get();
        // Now, wait until at least one delete happens within AWAIT_TIMEOUT_MS.
        Assert.assertTrue("Some blobs should have been deleted within " + AWAIT_TIMEOUT_MS, deletesDoneLatch.await(AWAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS));
        // Wait for the rest of the deletes to finish.
        long waitStart = SystemTime.getInstance().milliseconds();
        while (router.getBackgroundOperationsCount() != 0 && SystemTime.getInstance().milliseconds() < waitStart + AWAIT_TIMEOUT_MS) {
            Thread.sleep(1000);
        }
        for (Map.Entry<String, String> blobIdAndServiceId : blobsThatAreDeleted.entrySet()) {
            Assert.assertNotSame("We should not be deleting the valid blob by mistake", blobId, blobIdAndServiceId.getKey());
            Assert.assertEquals("Unexpected service ID for deleted blob", BackgroundDeleteRequest.SERVICE_ID_PREFIX + putBlobProperties.getServiceId(), blobIdAndServiceId.getValue());
        }
    } finally {
        if (router != null) {
            router.close();
            assertClosed();
        }
    }
}

Example 23 with MockTime

use of com.github.ambry.utils.MockTime in project ambry by linkedin.

the class NonBlockingRouterTest method testRequestResponseHandlerThreadExitFlow.

/**
 * Test RequestResponseHandler thread exit flow. If the RequestResponseHandlerThread exits on its own (due to a
 * Throwable), then the router gets closed immediately along with the completion of all the operations.
 */
@Test
public void testRequestResponseHandlerThreadExitFlow() throws Exception {
    nettyByteBufLeakHelper.setDisabled(true);
    Properties props = getNonBlockingRouterProperties("DC1");
    VerifiableProperties verifiableProperties = new VerifiableProperties((props));
    RouterConfig routerConfig = new RouterConfig(verifiableProperties);
    MockClusterMap mockClusterMap = new MockClusterMap();
    MockTime mockTime = new MockTime();
    router = new NonBlockingRouter(routerConfig, new NonBlockingRouterMetrics(mockClusterMap, routerConfig), new MockNetworkClientFactory(verifiableProperties, mockSelectorState, MAX_PORTS_PLAIN_TEXT, MAX_PORTS_SSL, CHECKOUT_TIMEOUT_MS, new MockServerLayout(mockClusterMap), mockTime), new LoggingNotificationSystem(), mockClusterMap, kms, cryptoService, cryptoJobHandler, accountService, mockTime, MockClusterMap.DEFAULT_PARTITION_CLASS);
    assertExpectedThreadCounts(2, 1);
    setOperationParams();
    mockSelectorState.set(MockSelectorState.ThrowExceptionOnAllPoll);
    Future future = router.putBlob(putBlobProperties, putUserMetadata, putChannel, new PutBlobOptionsBuilder().build());
    try {
        while (!future.isDone()) {
            mockTime.sleep(1000);
            Thread.yield();
        }
        future.get();
        Assert.fail("The operation should have failed");
    } catch (ExecutionException e) {
        Assert.assertEquals(RouterErrorCode.OperationTimedOut, ((RouterException) e.getCause()).getErrorCode());
    }
    setOperationParams();
    mockSelectorState.set(MockSelectorState.ThrowThrowableOnSend);
    future = router.putBlob(putBlobProperties, putUserMetadata, putChannel, new PutBlobOptionsBuilder().build());
    Thread requestResponseHandlerThreadRegular = TestUtils.getThreadByThisName("RequestResponseHandlerThread-0");
    Thread requestResponseHandlerThreadBackground = TestUtils.getThreadByThisName("RequestResponseHandlerThread-backgroundDeleter");
    if (requestResponseHandlerThreadRegular != null) {
        requestResponseHandlerThreadRegular.join(NonBlockingRouter.SHUTDOWN_WAIT_MS);
    }
    if (requestResponseHandlerThreadBackground != null) {
        requestResponseHandlerThreadBackground.join(NonBlockingRouter.SHUTDOWN_WAIT_MS);
    }
    try {
        future.get();
        Assert.fail("The operation should have failed");
    } catch (ExecutionException e) {
        Assert.assertEquals(RouterErrorCode.RouterClosed, ((RouterException) e.getCause()).getErrorCode());
    }
    assertClosed();
    // Ensure that both operations failed and with the right exceptions.
    Assert.assertEquals("No ChunkFiller Thread should be running after the router is closed", 0, TestUtils.numThreadsByThisName("ChunkFillerThread"));
    Assert.assertEquals("No RequestResponseHandler should be running after the router is closed", 0, TestUtils.numThreadsByThisName("RequestResponseHandlerThread"));
    Assert.assertEquals("All operations should have completed", 0, router.getOperationsCount());
}

Example 24 with MockTime

use of com.github.ambry.utils.MockTime in project ambry by linkedin.

the class ChunkFillTest method testChunkNumAndSizeCalculations.

/**
 * Test the calculation of number of chunks and the size of each chunk, using a very large blob size. No content
 * comparison is done. This test does not consume memory more than chunkSize.
 */
@Test
public void testChunkNumAndSizeCalculations() throws Exception {
    chunkSize = 4 * 1024 * 1024;
    // a large blob greater than Integer.MAX_VALUE and not at chunk size boundary.
    final long blobSize = ((long) Integer.MAX_VALUE / chunkSize + 1) * chunkSize + random.nextInt(chunkSize - 1) + 1;
    VerifiableProperties vProps = getNonBlockingRouterProperties();
    MockClusterMap mockClusterMap = new MockClusterMap();
    RouterConfig routerConfig = new RouterConfig(vProps);
    NonBlockingRouterMetrics routerMetrics = new NonBlockingRouterMetrics(mockClusterMap, routerConfig);
    short accountId = Utils.getRandomShort(random);
    short containerId = Utils.getRandomShort(random);
    BlobProperties putBlobProperties = new BlobProperties(blobSize, "serviceId", "memberId", "contentType", false, Utils.Infinite_Time, accountId, containerId, false, null, null, null);
    Random random = new Random();
    byte[] putUserMetadata = new byte[10];
    random.nextBytes(putUserMetadata);
    final MockReadableStreamChannel putChannel = new MockReadableStreamChannel(blobSize, false);
    FutureResult<String> futureResult = new FutureResult<String>();
    MockTime time = new MockTime();
    MockNetworkClientFactory networkClientFactory = new MockNetworkClientFactory(vProps, null, 0, 0, 0, null, time);
    PutOperation op = PutOperation.forUpload(routerConfig, routerMetrics, mockClusterMap, new LoggingNotificationSystem(), new InMemAccountService(true, false), putUserMetadata, putChannel, PutBlobOptions.DEFAULT, futureResult, null, new RouterCallback(networkClientFactory.getNetworkClient(), new ArrayList<>()), null, null, null, null, new MockTime(), putBlobProperties, MockClusterMap.DEFAULT_PARTITION_CLASS, quotaChargeCallback);
    op.startOperation();
    numChunks = RouterUtils.getNumChunksForBlobAndChunkSize(blobSize, chunkSize);
    // largeBlobSize is not a multiple of chunkSize
    int expectedNumChunks = (int) (blobSize / chunkSize + 1);
    Assert.assertEquals("numChunks should be as expected", expectedNumChunks, numChunks);
    int lastChunkSize = (int) (blobSize % chunkSize);
    final AtomicReference<Exception> channelException = new AtomicReference<Exception>(null);
    int chunkIndex = 0;
    // The write to the MockReadableStreamChannel blocks until the data is read as part fo the chunk filling,
    // so create a thread that fills the MockReadableStreamChannel.
    Utils.newThread(new Runnable() {

        @Override
        public void run() {
            try {
                byte[] writeBuf = new byte[chunkSize];
                long written = 0;
                while (written < blobSize) {
                    int toWrite = (int) Math.min(chunkSize, blobSize - written);
                    putChannel.write(ByteBuffer.wrap(writeBuf, 0, toWrite));
                    written += toWrite;
                }
            } catch (Exception e) {
                channelException.set(e);
            }
        }
    }, false).start();
    // Do the chunk filling.
    boolean fillingComplete = false;
    do {
        op.fillChunks();
        // since the channel is ByteBuffer based.
        for (PutOperation.PutChunk putChunk : op.putChunks) {
            Assert.assertNull("Mock channel write should not have caused an exception", channelException.get());
            if (putChunk.isFree()) {
                continue;
            }
            if (chunkIndex == numChunks - 1) {
                // last chunk may not be Ready as it is dependent on the completion callback to be called.
                Assert.assertTrue("Chunk should be Building or Ready.", putChunk.getState() == PutOperation.ChunkState.Ready || putChunk.getState() == PutOperation.ChunkState.Building);
                if (putChunk.getState() == PutOperation.ChunkState.Ready) {
                    Assert.assertEquals("Chunk size should be the last chunk size", lastChunkSize, putChunk.buf.readableBytes());
                    Assert.assertTrue("Chunk Filling should be complete at this time", op.isChunkFillingDone());
                    fillingComplete = true;
                }
            } else {
                // if not last chunk, then the chunk should be full and Ready.
                Assert.assertEquals("Chunk should be ready.", PutOperation.ChunkState.Ready, putChunk.getState());
                Assert.assertEquals("Chunk size should be maxChunkSize", chunkSize, putChunk.buf.readableBytes());
                chunkIndex++;
                putChunk.clear();
            }
        }
    } while (!fillingComplete);
}

Example 25 with MockTime

use of com.github.ambry.utils.MockTime in project ambry by linkedin.

the class ServerHttp2Test method initializeTests.

@BeforeClass
public static void initializeTests() throws Exception {
    File trustStoreFile = File.createTempFile("truststore", ".jks");
    Properties clientSSLProps = new Properties();
    TestSSLUtils.addSSLProperties(clientSSLProps, "DC1,DC2,DC3", SSLFactory.Mode.CLIENT, trustStoreFile, "http2-blocking-channel-client");
    TestSSLUtils.addHttp2Properties(clientSSLProps, SSLFactory.Mode.CLIENT, false);
    clientSSLConfig1 = new SSLConfig(new VerifiableProperties(clientSSLProps));
    clientSSLConfig2 = new SSLConfig(new VerifiableProperties(clientSSLProps));
    clientSSLConfig3 = new SSLConfig(new VerifiableProperties(clientSSLProps));
    // Router
    routerProps = new Properties();
    routerProps.setProperty("kms.default.container.key", TestUtils.getRandomKey(32));
    routerProps.setProperty("clustermap.default.partition.class", MockClusterMap.DEFAULT_PARTITION_CLASS);
    routerProps.setProperty(RouterConfig.ROUTER_ENABLE_HTTP2_NETWORK_CLIENT, "true");
    TestSSLUtils.addHttp2Properties(routerProps, SSLFactory.Mode.CLIENT, false);
    TestSSLUtils.addSSLProperties(routerProps, "DC1,DC2,DC3", SSLFactory.Mode.CLIENT, trustStoreFile, "router-client");
    // Server
    Properties serverSSLProps;
    serverSSLProps = new Properties();
    TestSSLUtils.addSSLProperties(serverSSLProps, "DC1,DC2,DC3", SSLFactory.Mode.SERVER, trustStoreFile, "server");
    TestSSLUtils.addHttp2Properties(serverSSLProps, SSLFactory.Mode.SERVER, false);
    serverSSLProps.setProperty("clustermap.enable.http2.replication", "true");
    http2Cluster = new MockCluster(serverSSLProps, false, new MockTime(SystemTime.getInstance().milliseconds()), 9, 3, 3);
    notificationSystem = new MockNotificationSystem(http2Cluster.getClusterMap());
    http2Cluster.initializeServers(notificationSystem);
    http2Cluster.startServers();
}