Examples with Releasable org.opensearch.common.lease.Releasable used on opensource projects

Example 1 with Releasable

use of org.opensearch.common.lease.Releasable in project OpenSearch by opensearch-project.

the class TransportAction method execute.

/**
 * Use this method when the transport action call should result in creation of a new task associated with the call.
 *
 * This is a typical behavior.
 */
public final Task execute(Request request, ActionListener<Response> listener) {
    /*
         * While this version of execute could delegate to the TaskListener
         * version of execute that'd add yet another layer of wrapping on the
         * listener and prevent us from using the listener bare if there isn't a
         * task. That just seems like too many objects. Thus the two versions of
         * this method.
         */
    final Releasable unregisterChildNode = registerChildNode(request.getParentTask());
    final Task task;
    try {
        task = taskManager.register("transport", actionName, request);
    } catch (TaskCancelledException e) {
        unregisterChildNode.close();
        throw e;
    }
    execute(task, request, new ActionListener<Response>() {

        @Override
        public void onResponse(Response response) {
            try {
                Releasables.close(unregisterChildNode, () -> taskManager.unregister(task));
            } finally {
                listener.onResponse(response);
            }
        }

        @Override
        public void onFailure(Exception e) {
            try {
                Releasables.close(unregisterChildNode, () -> taskManager.unregister(task));
            } finally {
                listener.onFailure(e);
            }
        }
    });
    return task;
}

Example 2 with Releasable

use of org.opensearch.common.lease.Releasable in project OpenSearch by opensearch-project.

the class ClusterApplierService method runTask.

private void runTask(UpdateTask task) {
    if (!lifecycle.started()) {
        logger.debug("processing [{}]: ignoring, cluster applier service not started", task.source);
        return;
    }
    logger.debug("processing [{}]: execute", task.source);
    final ClusterState previousClusterState = state.get();
    long startTimeMS = currentTimeInMillis();
    final StopWatch stopWatch = new StopWatch();
    final ClusterState newClusterState;
    try {
        try (Releasable ignored = stopWatch.timing("running task [" + task.source + ']')) {
            newClusterState = task.apply(previousClusterState);
        }
    } catch (Exception e) {
        TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, currentTimeInMillis() - startTimeMS));
        logger.trace(() -> new ParameterizedMessage("failed to execute cluster state applier in [{}], state:\nversion [{}], source [{}]\n{}", executionTime, previousClusterState.version(), task.source, previousClusterState), e);
        warnAboutSlowTaskIfNeeded(executionTime, task.source, stopWatch);
        task.listener.onFailure(task.source, e);
        return;
    }
    if (previousClusterState == newClusterState) {
        TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, currentTimeInMillis() - startTimeMS));
        logger.debug("processing [{}]: took [{}] no change in cluster state", task.source, executionTime);
        warnAboutSlowTaskIfNeeded(executionTime, task.source, stopWatch);
        task.listener.onSuccess(task.source);
    } else {
        if (logger.isTraceEnabled()) {
            logger.debug("cluster state updated, version [{}], source [{}]\n{}", newClusterState.version(), task.source, newClusterState);
        } else {
            logger.debug("cluster state updated, version [{}], source [{}]", newClusterState.version(), task.source);
        }
        try {
            applyChanges(task, previousClusterState, newClusterState, stopWatch);
            TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, currentTimeInMillis() - startTimeMS));
            logger.debug("processing [{}]: took [{}] done applying updated cluster state (version: {}, uuid: {})", task.source, executionTime, newClusterState.version(), newClusterState.stateUUID());
            warnAboutSlowTaskIfNeeded(executionTime, task.source, stopWatch);
            task.listener.onSuccess(task.source);
        } catch (Exception e) {
            TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, currentTimeInMillis() - startTimeMS));
            if (logger.isTraceEnabled()) {
                logger.warn(new ParameterizedMessage("failed to apply updated cluster state in [{}]:\nversion [{}], uuid [{}], source [{}]\n{}", executionTime, newClusterState.version(), newClusterState.stateUUID(), task.source, newClusterState), e);
            } else {
                logger.warn(new ParameterizedMessage("failed to apply updated cluster state in [{}]:\nversion [{}], uuid [{}], source [{}]", executionTime, newClusterState.version(), newClusterState.stateUUID(), task.source), e);
            }
            // continue we will retry with the same cluster state but that might not help.
            assert applicationMayFail();
            task.listener.onFailure(task.source, e);
        }
    }
}

Example 3 with Releasable

use of org.opensearch.common.lease.Releasable in project OpenSearch by opensearch-project.

the class DeflateCompressor method inputStream.

/**
 * Creates a new input stream that decompresses the contents read from the provided input stream.
 * Closing the returned stream will close the provided input stream.
 * Optionally uses thread-local, pooled resources to save off-heap allocations if the stream is guaranteed to not escape the current
 * thread.
 *
 * @param in           input stream to wrap
 * @param threadLocal  whether this stream will only be used on the current thread or not
 * @return             decompressing stream
 */
public static InputStream inputStream(InputStream in, boolean threadLocal) throws IOException {
    final byte[] headerBytes = new byte[HEADER.length];
    int len = 0;
    while (len < headerBytes.length) {
        final int read = in.read(headerBytes, len, headerBytes.length - len);
        if (read == -1) {
            break;
        }
        len += read;
    }
    if (len != HEADER.length || Arrays.equals(headerBytes, HEADER) == false) {
        throw new IllegalArgumentException("Input stream is not compressed with DEFLATE!");
    }
    final Releasable releasable;
    final Inflater inflater;
    if (threadLocal) {
        final ReleasableReference<Inflater> current = inflaterForStreamRef.get();
        if (current.inUse) {
            // Nested de-compression streams should not happen but we still handle them safely by using a fresh Inflater
            inflater = new Inflater(true);
            releasable = inflater::end;
        } else {
            inflater = current.get();
            releasable = current;
        }
    } else {
        inflater = new Inflater(true);
        releasable = inflater::end;
    }
    return new BufferedInputStream(new InflaterInputStream(in, inflater, BUFFER_SIZE) {

        @Override
        public void close() throws IOException {
            try {
                super.close();
            } finally {
                // We are ensured to only call this once since we wrap this stream in a BufferedInputStream that will only close
                // its delegate once
                releasable.close();
            }
        }
    }, BUFFER_SIZE);
}

Example 4 with Releasable

use of org.opensearch.common.lease.Releasable in project OpenSearch by opensearch-project.

the class IndexingPressureIT method testWriteBytesAreIncremented.

public void testWriteBytesAreIncremented() throws Exception {
    assertAcked(prepareCreate(INDEX_NAME, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)));
    ensureGreen(INDEX_NAME);
    Tuple<String, String> primaryReplicaNodeNames = getPrimaryReplicaNodeNames();
    String primaryName = primaryReplicaNodeNames.v1();
    String replicaName = primaryReplicaNodeNames.v2();
    String coordinatingOnlyNode = getCoordinatingOnlyNode();
    final CountDownLatch replicationSendPointReached = new CountDownLatch(1);
    final CountDownLatch latchBlockingReplicationSend = new CountDownLatch(1);
    TransportService primaryService = internalCluster().getInstance(TransportService.class, primaryName);
    final MockTransportService primaryTransportService = (MockTransportService) primaryService;
    TransportService replicaService = internalCluster().getInstance(TransportService.class, replicaName);
    final MockTransportService replicaTransportService = (MockTransportService) replicaService;
    primaryTransportService.addSendBehavior((connection, requestId, action, request, options) -> {
        if (action.equals(TransportShardBulkAction.ACTION_NAME + "[r]")) {
            try {
                replicationSendPointReached.countDown();
                latchBlockingReplicationSend.await();
            } catch (InterruptedException e) {
                throw new IllegalStateException(e);
            }
        }
        connection.sendRequest(requestId, action, request, options);
    });
    final ThreadPool replicaThreadPool = replicaTransportService.getThreadPool();
    final Releasable replicaRelease = blockReplicas(replicaThreadPool);
    final BulkRequest bulkRequest = new BulkRequest();
    int totalRequestSize = 0;
    for (int i = 0; i < 80; ++i) {
        IndexRequest request = new IndexRequest(INDEX_NAME).id(UUIDs.base64UUID()).source(Collections.singletonMap("key", randomAlphaOfLength(50)));
        totalRequestSize += request.ramBytesUsed();
        assertTrue(request.ramBytesUsed() > request.source().length());
        bulkRequest.add(request);
    }
    final long bulkRequestSize = bulkRequest.ramBytesUsed();
    final long bulkShardRequestSize = totalRequestSize;
    try {
        final ActionFuture<BulkResponse> successFuture = client(coordinatingOnlyNode).bulk(bulkRequest);
        replicationSendPointReached.await();
        IndexingPressure primaryWriteLimits = internalCluster().getInstance(IndexingPressureService.class, primaryName).getShardIndexingPressure();
        IndexingPressure replicaWriteLimits = internalCluster().getInstance(IndexingPressureService.class, replicaName).getShardIndexingPressure();
        ;
        IndexingPressure coordinatingWriteLimits = internalCluster().getInstance(IndexingPressureService.class, coordinatingOnlyNode).getShardIndexingPressure();
        ;
        assertThat(primaryWriteLimits.getCurrentCombinedCoordinatingAndPrimaryBytes(), greaterThan(bulkShardRequestSize));
        assertThat(primaryWriteLimits.getCurrentPrimaryBytes(), greaterThan(bulkShardRequestSize));
        assertEquals(0, primaryWriteLimits.getCurrentCoordinatingBytes());
        assertEquals(0, primaryWriteLimits.getCurrentReplicaBytes());
        assertEquals(0, replicaWriteLimits.getCurrentCombinedCoordinatingAndPrimaryBytes());
        assertEquals(0, replicaWriteLimits.getCurrentCoordinatingBytes());
        assertEquals(0, replicaWriteLimits.getCurrentPrimaryBytes());
        assertEquals(0, replicaWriteLimits.getCurrentReplicaBytes());
        assertEquals(bulkRequestSize, coordinatingWriteLimits.getCurrentCombinedCoordinatingAndPrimaryBytes());
        assertEquals(bulkRequestSize, coordinatingWriteLimits.getCurrentCoordinatingBytes());
        assertEquals(0, coordinatingWriteLimits.getCurrentPrimaryBytes());
        assertEquals(0, coordinatingWriteLimits.getCurrentReplicaBytes());
        latchBlockingReplicationSend.countDown();
        IndexRequest request = new IndexRequest(INDEX_NAME).id(UUIDs.base64UUID()).source(Collections.singletonMap("key", randomAlphaOfLength(50)));
        final BulkRequest secondBulkRequest = new BulkRequest();
        secondBulkRequest.add(request);
        // Use the primary or the replica data node as the coordinating node this time
        boolean usePrimaryAsCoordinatingNode = randomBoolean();
        final ActionFuture<BulkResponse> secondFuture;
        if (usePrimaryAsCoordinatingNode) {
            secondFuture = client(primaryName).bulk(secondBulkRequest);
        } else {
            secondFuture = client(replicaName).bulk(secondBulkRequest);
        }
        final long secondBulkRequestSize = secondBulkRequest.ramBytesUsed();
        final long secondBulkShardRequestSize = request.ramBytesUsed();
        if (usePrimaryAsCoordinatingNode) {
            assertBusy(() -> {
                assertThat(primaryWriteLimits.getCurrentCombinedCoordinatingAndPrimaryBytes(), greaterThan(bulkShardRequestSize + secondBulkRequestSize));
                assertEquals(secondBulkRequestSize, primaryWriteLimits.getCurrentCoordinatingBytes());
                assertThat(primaryWriteLimits.getCurrentPrimaryBytes(), greaterThan(bulkShardRequestSize + secondBulkRequestSize));
                assertEquals(0, replicaWriteLimits.getCurrentCombinedCoordinatingAndPrimaryBytes());
                assertEquals(0, replicaWriteLimits.getCurrentCoordinatingBytes());
                assertEquals(0, replicaWriteLimits.getCurrentPrimaryBytes());
            });
        } else {
            assertThat(primaryWriteLimits.getCurrentCombinedCoordinatingAndPrimaryBytes(), greaterThan(bulkShardRequestSize));
            assertEquals(secondBulkRequestSize, replicaWriteLimits.getCurrentCombinedCoordinatingAndPrimaryBytes());
            assertEquals(secondBulkRequestSize, replicaWriteLimits.getCurrentCoordinatingBytes());
            assertEquals(0, replicaWriteLimits.getCurrentPrimaryBytes());
        }
        assertEquals(bulkRequestSize, coordinatingWriteLimits.getCurrentCombinedCoordinatingAndPrimaryBytes());
        assertBusy(() -> assertThat(replicaWriteLimits.getCurrentReplicaBytes(), greaterThan(bulkShardRequestSize + secondBulkShardRequestSize)));
        replicaRelease.close();
        successFuture.actionGet();
        secondFuture.actionGet();
        assertEquals(0, primaryWriteLimits.getCurrentCombinedCoordinatingAndPrimaryBytes());
        assertEquals(0, primaryWriteLimits.getCurrentCoordinatingBytes());
        assertEquals(0, primaryWriteLimits.getCurrentPrimaryBytes());
        assertEquals(0, primaryWriteLimits.getCurrentReplicaBytes());
        assertEquals(0, replicaWriteLimits.getCurrentCombinedCoordinatingAndPrimaryBytes());
        assertEquals(0, replicaWriteLimits.getCurrentCoordinatingBytes());
        assertEquals(0, replicaWriteLimits.getCurrentPrimaryBytes());
        assertEquals(0, replicaWriteLimits.getCurrentReplicaBytes());
        assertEquals(0, coordinatingWriteLimits.getCurrentCombinedCoordinatingAndPrimaryBytes());
        assertEquals(0, coordinatingWriteLimits.getCurrentCoordinatingBytes());
        assertEquals(0, coordinatingWriteLimits.getCurrentPrimaryBytes());
        assertEquals(0, coordinatingWriteLimits.getCurrentReplicaBytes());
    } finally {
        if (replicationSendPointReached.getCount() > 0) {
            replicationSendPointReached.countDown();
        }
        replicaRelease.close();
        if (latchBlockingReplicationSend.getCount() > 0) {
            latchBlockingReplicationSend.countDown();
        }
        replicaRelease.close();
        primaryTransportService.clearAllRules();
    }
}

Example 5 with Releasable

use of org.opensearch.common.lease.Releasable in project OpenSearch by opensearch-project.

the class ShardIndexingPressureIT method testShardIndexingPressureTrackingDuringBulkWrites.

public void testShardIndexingPressureTrackingDuringBulkWrites() throws Exception {
    assertAcked(prepareCreate(INDEX_NAME, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)));
    ensureGreen(INDEX_NAME);
    Tuple<String, String> primaryReplicaNodeNames = getPrimaryReplicaNodeNames(INDEX_NAME);
    String primaryName = primaryReplicaNodeNames.v1();
    String replicaName = primaryReplicaNodeNames.v2();
    String coordinatingOnlyNode = getCoordinatingOnlyNode();
    final CountDownLatch replicationSendPointReached = new CountDownLatch(1);
    final CountDownLatch latchBlockingReplicationSend = new CountDownLatch(1);
    TransportService primaryService = internalCluster().getInstance(TransportService.class, primaryName);
    final MockTransportService primaryTransportService = (MockTransportService) primaryService;
    TransportService replicaService = internalCluster().getInstance(TransportService.class, replicaName);
    final MockTransportService replicaTransportService = (MockTransportService) replicaService;
    primaryTransportService.addSendBehavior((connection, requestId, action, request, options) -> {
        if (action.equals(TransportShardBulkAction.ACTION_NAME + "[r]")) {
            try {
                replicationSendPointReached.countDown();
                latchBlockingReplicationSend.await();
            } catch (InterruptedException e) {
                throw new IllegalStateException(e);
            }
        }
        connection.sendRequest(requestId, action, request, options);
    });
    final ThreadPool replicaThreadPool = replicaTransportService.getThreadPool();
    final Releasable replicaRelease = blockReplicas(replicaThreadPool);
    final BulkRequest bulkRequest = new BulkRequest();
    int totalRequestSize = 0;
    for (int i = 0; i < 80; ++i) {
        IndexRequest request = new IndexRequest(INDEX_NAME).id(UUIDs.base64UUID()).source(Collections.singletonMap("key", randomAlphaOfLength(50)));
        totalRequestSize += request.ramBytesUsed();
        assertTrue(request.ramBytesUsed() > request.source().length());
        bulkRequest.add(request);
    }
    final long bulkShardRequestSize = totalRequestSize + (RamUsageEstimator.shallowSizeOfInstance(BulkItemRequest.class) * 80) + RamUsageEstimator.shallowSizeOfInstance(BulkShardRequest.class);
    try {
        final ActionFuture<BulkResponse> successFuture = client(coordinatingOnlyNode).bulk(bulkRequest);
        replicationSendPointReached.await();
        IndexService indexService = internalCluster().getInstance(IndicesService.class, primaryName).iterator().next();
        Index index = indexService.getIndexSettings().getIndex();
        ShardId shardId = new ShardId(index, 0);
        ShardIndexingPressureTracker primaryShardTracker = internalCluster().getInstance(IndexingPressureService.class, primaryName).getShardIndexingPressure().getShardIndexingPressureTracker(shardId);
        ShardIndexingPressureTracker replicaShardTracker = internalCluster().getInstance(IndexingPressureService.class, replicaName).getShardIndexingPressure().getShardIndexingPressureTracker(shardId);
        ShardIndexingPressureTracker coordinatingShardTracker = internalCluster().getInstance(IndexingPressureService.class, coordinatingOnlyNode).getShardIndexingPressure().getShardIndexingPressureTracker(shardId);
        assertThat(primaryShardTracker.getCommonOperationTracker().getCurrentCombinedCoordinatingAndPrimaryBytes(), equalTo(bulkShardRequestSize));
        assertThat(primaryShardTracker.getPrimaryOperationTracker().getStatsTracker().getCurrentBytes(), equalTo(bulkShardRequestSize));
        assertEquals(0, primaryShardTracker.getCoordinatingOperationTracker().getStatsTracker().getCurrentBytes());
        assertEquals(0, primaryShardTracker.getReplicaOperationTracker().getStatsTracker().getCurrentBytes());
        assertEquals(0, replicaShardTracker.getCommonOperationTracker().getCurrentCombinedCoordinatingAndPrimaryBytes());
        assertEquals(0, replicaShardTracker.getCoordinatingOperationTracker().getStatsTracker().getCurrentBytes());
        assertEquals(0, replicaShardTracker.getPrimaryOperationTracker().getStatsTracker().getCurrentBytes());
        assertEquals(0, replicaShardTracker.getReplicaOperationTracker().getStatsTracker().getCurrentBytes());
        assertEquals(bulkShardRequestSize, coordinatingShardTracker.getCommonOperationTracker().getCurrentCombinedCoordinatingAndPrimaryBytes());
        assertEquals(bulkShardRequestSize, coordinatingShardTracker.getCoordinatingOperationTracker().getStatsTracker().getCurrentBytes());
        assertEquals(0, coordinatingShardTracker.getPrimaryOperationTracker().getStatsTracker().getCurrentBytes());
        assertEquals(0, coordinatingShardTracker.getReplicaOperationTracker().getStatsTracker().getCurrentBytes());
        latchBlockingReplicationSend.countDown();
        IndexRequest request = new IndexRequest(INDEX_NAME).id(UUIDs.base64UUID()).source(Collections.singletonMap("key", randomAlphaOfLength(50)));
        final BulkRequest secondBulkRequest = new BulkRequest();
        secondBulkRequest.add(request);
        // Use the primary or the replica data node as the coordinating node this time
        boolean usePrimaryAsCoordinatingNode = randomBoolean();
        final ActionFuture<BulkResponse> secondFuture;
        if (usePrimaryAsCoordinatingNode) {
            secondFuture = client(primaryName).bulk(secondBulkRequest);
        } else {
            secondFuture = client(replicaName).bulk(secondBulkRequest);
        }
        final long secondBulkShardRequestSize = request.ramBytesUsed() + RamUsageEstimator.shallowSizeOfInstance(BulkItemRequest.class) + RamUsageEstimator.shallowSizeOfInstance(BulkShardRequest.class);
        if (usePrimaryAsCoordinatingNode) {
            assertBusy(() -> {
                assertThat(primaryShardTracker.getCommonOperationTracker().getCurrentCombinedCoordinatingAndPrimaryBytes(), equalTo(bulkShardRequestSize + secondBulkShardRequestSize));
                assertEquals(secondBulkShardRequestSize, primaryShardTracker.getCoordinatingOperationTracker().getStatsTracker().getCurrentBytes());
                assertThat(primaryShardTracker.getPrimaryOperationTracker().getStatsTracker().getCurrentBytes(), equalTo(bulkShardRequestSize + secondBulkShardRequestSize));
                assertEquals(0, replicaShardTracker.getCommonOperationTracker().getCurrentCombinedCoordinatingAndPrimaryBytes());
                assertEquals(0, replicaShardTracker.getCoordinatingOperationTracker().getStatsTracker().getCurrentBytes());
                assertEquals(0, replicaShardTracker.getPrimaryOperationTracker().getStatsTracker().getCurrentBytes());
            });
        } else {
            assertThat(primaryShardTracker.getCommonOperationTracker().getCurrentCombinedCoordinatingAndPrimaryBytes(), equalTo(bulkShardRequestSize));
            assertEquals(secondBulkShardRequestSize, replicaShardTracker.getCommonOperationTracker().getCurrentCombinedCoordinatingAndPrimaryBytes());
            assertEquals(secondBulkShardRequestSize, replicaShardTracker.getCoordinatingOperationTracker().getStatsTracker().getCurrentBytes());
            assertEquals(0, replicaShardTracker.getPrimaryOperationTracker().getStatsTracker().getCurrentBytes());
        }
        assertEquals(bulkShardRequestSize, coordinatingShardTracker.getCommonOperationTracker().getCurrentCombinedCoordinatingAndPrimaryBytes());
        assertBusy(() -> assertThat(replicaShardTracker.getReplicaOperationTracker().getStatsTracker().getCurrentBytes(), equalTo(bulkShardRequestSize + secondBulkShardRequestSize)));
        replicaRelease.close();
        successFuture.actionGet();
        secondFuture.actionGet();
        assertEquals(0, primaryShardTracker.getCommonOperationTracker().getCurrentCombinedCoordinatingAndPrimaryBytes());
        assertEquals(0, primaryShardTracker.getCoordinatingOperationTracker().getStatsTracker().getCurrentBytes());
        assertEquals(0, primaryShardTracker.getPrimaryOperationTracker().getStatsTracker().getCurrentBytes());
        assertEquals(0, primaryShardTracker.getReplicaOperationTracker().getStatsTracker().getCurrentBytes());
        assertEquals(0, replicaShardTracker.getCommonOperationTracker().getCurrentCombinedCoordinatingAndPrimaryBytes());
        assertEquals(0, replicaShardTracker.getCoordinatingOperationTracker().getStatsTracker().getCurrentBytes());
        assertEquals(0, replicaShardTracker.getReplicaOperationTracker().getStatsTracker().getCurrentBytes());
        assertEquals(0, replicaShardTracker.getReplicaOperationTracker().getStatsTracker().getCurrentBytes());
        assertEquals(0, coordinatingShardTracker.getCommonOperationTracker().getCurrentCombinedCoordinatingAndPrimaryBytes());
        assertEquals(0, coordinatingShardTracker.getCoordinatingOperationTracker().getStatsTracker().getCurrentBytes());
        assertEquals(0, coordinatingShardTracker.getPrimaryOperationTracker().getStatsTracker().getCurrentBytes());
        assertEquals(0, coordinatingShardTracker.getReplicaOperationTracker().getStatsTracker().getCurrentBytes());
    } finally {
        if (replicationSendPointReached.getCount() > 0) {
            replicationSendPointReached.countDown();
        }
        replicaRelease.close();
        if (latchBlockingReplicationSend.getCount() > 0) {
            latchBlockingReplicationSend.countDown();
        }
        replicaRelease.close();
        primaryTransportService.clearAllRules();
    }
}