Examples with Releasable org.opensearch.common.lease.Releasable used on opensource projects

Example 41 with Releasable

use of org.opensearch.common.lease.Releasable in project OpenSearch by opensearch-project.

the class SoftDeletesPolicyTests method testSoftDeletesRetentionLock.

/**
 * Makes sure we won't advance the retained seq# if the retention lock is held
 */
public void testSoftDeletesRetentionLock() {
    long retainedOps = between(0, 10000);
    AtomicLong globalCheckpoint = new AtomicLong(NO_OPS_PERFORMED);
    final AtomicLong[] retainingSequenceNumbers = new AtomicLong[randomIntBetween(0, 8)];
    for (int i = 0; i < retainingSequenceNumbers.length; i++) {
        retainingSequenceNumbers[i] = new AtomicLong();
    }
    final Supplier<RetentionLeases> retentionLeasesSupplier = () -> {
        final List<RetentionLease> leases = new ArrayList<>(retainingSequenceNumbers.length);
        for (int i = 0; i < retainingSequenceNumbers.length; i++) {
            leases.add(new RetentionLease(Integer.toString(i), retainingSequenceNumbers[i].get(), 0L, "test"));
        }
        return new RetentionLeases(1, 1, leases);
    };
    long safeCommitCheckpoint = globalCheckpoint.get();
    SoftDeletesPolicy policy = new SoftDeletesPolicy(globalCheckpoint::get, between(1, 10000), retainedOps, retentionLeasesSupplier);
    long minRetainedSeqNo = policy.getMinRetainedSeqNo();
    List<Releasable> locks = new ArrayList<>();
    int iters = scaledRandomIntBetween(10, 1000);
    for (int i = 0; i < iters; i++) {
        if (randomBoolean()) {
            locks.add(policy.acquireRetentionLock());
        }
        // Advances the global checkpoint and the local checkpoint of a safe commit
        globalCheckpoint.addAndGet(between(0, 1000));
        for (final AtomicLong retainingSequenceNumber : retainingSequenceNumbers) {
            retainingSequenceNumber.set(randomLongBetween(retainingSequenceNumber.get(), Math.max(globalCheckpoint.get(), 0L)));
        }
        safeCommitCheckpoint = randomLongBetween(safeCommitCheckpoint, globalCheckpoint.get());
        policy.setLocalCheckpointOfSafeCommit(safeCommitCheckpoint);
        if (rarely()) {
            retainedOps = between(0, 10000);
            policy.setRetentionOperations(retainedOps);
        }
        // Release some locks
        List<Releasable> releasingLocks = randomSubsetOf(locks);
        locks.removeAll(releasingLocks);
        releasingLocks.forEach(Releasable::close);
        // getting the query has side effects, updating the internal state of the policy
        final Query query = policy.getRetentionQuery();
        assertThat(query, instanceOf(PointRangeQuery.class));
        final PointRangeQuery retentionQuery = (PointRangeQuery) query;
        // we only expose the minimum sequence number to the merge policy if the retention lock is not held
        if (locks.isEmpty()) {
            final long minimumRetainingSequenceNumber = Arrays.stream(retainingSequenceNumbers).mapToLong(AtomicLong::get).min().orElse(Long.MAX_VALUE);
            long retainedSeqNo = Math.min(1 + safeCommitCheckpoint, Math.min(minimumRetainingSequenceNumber, 1 + globalCheckpoint.get() - retainedOps));
            minRetainedSeqNo = Math.max(minRetainedSeqNo, retainedSeqNo);
        }
        assertThat(retentionQuery.getNumDims(), equalTo(1));
        assertThat(LongPoint.decodeDimension(retentionQuery.getLowerPoint(), 0), equalTo(minRetainedSeqNo));
        assertThat(LongPoint.decodeDimension(retentionQuery.getUpperPoint(), 0), equalTo(Long.MAX_VALUE));
        assertThat(policy.getMinRetainedSeqNo(), equalTo(minRetainedSeqNo));
    }
    locks.forEach(Releasable::close);
    final long minimumRetainingSequenceNumber = Arrays.stream(retainingSequenceNumbers).mapToLong(AtomicLong::get).min().orElse(Long.MAX_VALUE);
    long retainedSeqNo = Math.min(1 + safeCommitCheckpoint, Math.min(minimumRetainingSequenceNumber, 1 + globalCheckpoint.get() - retainedOps));
    minRetainedSeqNo = Math.max(minRetainedSeqNo, retainedSeqNo);
    assertThat(policy.getMinRetainedSeqNo(), equalTo(minRetainedSeqNo));
}

Example 42 with Releasable

use of org.opensearch.common.lease.Releasable in project OpenSearch by opensearch-project.

the class LiveVersionMapTests method testPruneTombstonesWhileLocked.

public void testPruneTombstonesWhileLocked() throws InterruptedException, IOException {
    LiveVersionMap map = new LiveVersionMap();
    BytesRef uid = uid("1");
    try (Releasable ignore = map.acquireLock(uid)) {
        map.putDeleteUnderLock(uid, new DeleteVersionValue(0, 0, 0, 0));
        // refresh otherwise we won't prune since it's tracked by the current map
        map.beforeRefresh();
        map.afterRefresh(false);
        Thread thread = new Thread(() -> {
            map.pruneTombstones(Long.MAX_VALUE, 0);
        });
        thread.start();
        thread.join();
        assertEquals(1, map.getAllTombstones().size());
    }
    Thread thread = new Thread(() -> {
        map.pruneTombstones(Long.MAX_VALUE, 0);
    });
    thread.start();
    thread.join();
    assertEquals(0, map.getAllTombstones().size());
}

Example 43 with Releasable

use of org.opensearch.common.lease.Releasable in project OpenSearch by opensearch-project.

the class LiveVersionMapTests method testRefreshTransition.

public void testRefreshTransition() throws IOException {
    LiveVersionMap map = new LiveVersionMap();
    try (Releasable r = map.acquireLock(uid("1"))) {
        map.maybePutIndexUnderLock(uid("1"), randomIndexVersionValue());
        assertTrue(map.isUnsafe());
        assertNull(map.getUnderLock(uid("1")));
        map.beforeRefresh();
        assertTrue(map.isUnsafe());
        assertNull(map.getUnderLock(uid("1")));
        map.afterRefresh(randomBoolean());
        assertNull(map.getUnderLock(uid("1")));
        assertFalse(map.isUnsafe());
        map.enforceSafeAccess();
        map.maybePutIndexUnderLock(uid("1"), randomIndexVersionValue());
        assertFalse(map.isUnsafe());
        assertNotNull(map.getUnderLock(uid("1")));
        map.beforeRefresh();
        assertFalse(map.isUnsafe());
        assertTrue(map.isSafeAccessRequired());
        assertNotNull(map.getUnderLock(uid("1")));
        map.afterRefresh(randomBoolean());
        assertNull(map.getUnderLock(uid("1")));
        assertFalse(map.isUnsafe());
        assertTrue(map.isSafeAccessRequired());
    }
}

Example 44 with Releasable

use of org.opensearch.common.lease.Releasable in project OpenSearch by opensearch-project.

the class LiveVersionMapTests method testConcurrently.

public void testConcurrently() throws IOException, InterruptedException {
    HashSet<BytesRef> keySet = new HashSet<>();
    int numKeys = randomIntBetween(50, 200);
    for (int i = 0; i < numKeys; i++) {
        keySet.add(uid(TestUtil.randomSimpleString(random(), 10, 20)));
    }
    List<BytesRef> keyList = new ArrayList<>(keySet);
    ConcurrentHashMap<BytesRef, VersionValue> values = new ConcurrentHashMap<>();
    ConcurrentHashMap<BytesRef, DeleteVersionValue> deletes = new ConcurrentHashMap<>();
    LiveVersionMap map = new LiveVersionMap();
    int numThreads = randomIntBetween(2, 5);
    Thread[] threads = new Thread[numThreads];
    CountDownLatch startGun = new CountDownLatch(numThreads);
    CountDownLatch done = new CountDownLatch(numThreads);
    int randomValuesPerThread = randomIntBetween(5000, 20000);
    final AtomicLong clock = new AtomicLong(0);
    final AtomicLong lastPrunedTimestamp = new AtomicLong(-1);
    final AtomicLong maxSeqNo = new AtomicLong();
    final AtomicLong lastPrunedSeqNo = new AtomicLong();
    for (int j = 0; j < threads.length; j++) {
        threads[j] = new Thread(() -> {
            startGun.countDown();
            try {
                startGun.await();
            } catch (InterruptedException e) {
                done.countDown();
                throw new AssertionError(e);
            }
            try {
                for (int i = 0; i < randomValuesPerThread; ++i) {
                    BytesRef bytesRef = randomFrom(random(), keyList);
                    try (Releasable r = map.acquireLock(bytesRef)) {
                        VersionValue versionValue = values.computeIfAbsent(bytesRef, v -> new IndexVersionValue(randomTranslogLocation(), randomLong(), maxSeqNo.incrementAndGet(), randomLong()));
                        boolean isDelete = versionValue instanceof DeleteVersionValue;
                        if (isDelete) {
                            map.removeTombstoneUnderLock(bytesRef);
                            deletes.remove(bytesRef);
                        }
                        if (isDelete == false && rarely()) {
                            versionValue = new DeleteVersionValue(versionValue.version + 1, maxSeqNo.incrementAndGet(), versionValue.term, clock.getAndIncrement());
                            deletes.put(bytesRef, (DeleteVersionValue) versionValue);
                            map.putDeleteUnderLock(bytesRef, (DeleteVersionValue) versionValue);
                        } else {
                            versionValue = new IndexVersionValue(randomTranslogLocation(), versionValue.version + 1, maxSeqNo.incrementAndGet(), versionValue.term);
                            map.putIndexUnderLock(bytesRef, (IndexVersionValue) versionValue);
                        }
                        values.put(bytesRef, versionValue);
                    }
                    if (rarely()) {
                        final long pruneSeqNo = randomLongBetween(0, maxSeqNo.get());
                        final long clockTick = randomLongBetween(0, clock.get());
                        map.pruneTombstones(clockTick, pruneSeqNo);
                        // make sure we track the latest timestamp and seqno we pruned the deletes
                        lastPrunedTimestamp.updateAndGet(prev -> Math.max(clockTick, prev));
                        lastPrunedSeqNo.updateAndGet(prev -> Math.max(pruneSeqNo, prev));
                    }
                }
            } finally {
                done.countDown();
            }
        });
        threads[j].start();
    }
    do {
        final Map<BytesRef, VersionValue> valueMap = new HashMap<>(map.getAllCurrent());
        map.beforeRefresh();
        valueMap.forEach((k, v) -> {
            try (Releasable r = map.acquireLock(k)) {
                VersionValue actualValue = map.getUnderLock(k);
                assertNotNull(actualValue);
                assertTrue(v.version <= actualValue.version);
            }
        });
        map.afterRefresh(randomBoolean());
        valueMap.forEach((k, v) -> {
            try (Releasable r = map.acquireLock(k)) {
                VersionValue actualValue = map.getUnderLock(k);
                if (actualValue != null) {
                    if (actualValue instanceof DeleteVersionValue) {
                        // deletes can be the same version
                        assertTrue(v.version <= actualValue.version);
                    } else {
                        assertTrue(v.version < actualValue.version);
                    }
                }
            }
        });
        if (randomBoolean()) {
            Thread.yield();
        }
    } while (done.getCount() != 0);
    for (int j = 0; j < threads.length; j++) {
        threads[j].join();
    }
    map.getAllCurrent().forEach((k, v) -> {
        VersionValue versionValue = values.get(k);
        assertNotNull(versionValue);
        assertEquals(v, versionValue);
    });
    Runnable assertTombstones = () -> map.getAllTombstones().entrySet().forEach(e -> {
        VersionValue versionValue = values.get(e.getKey());
        assertNotNull(versionValue);
        assertEquals(e.getValue(), versionValue);
        assertTrue(versionValue instanceof DeleteVersionValue);
    });
    assertTombstones.run();
    map.beforeRefresh();
    assertTombstones.run();
    map.afterRefresh(false);
    assertTombstones.run();
    deletes.entrySet().forEach(e -> {
        try (Releasable r = map.acquireLock(e.getKey())) {
            VersionValue value = map.getUnderLock(e.getKey());
            // here we keep track of the deletes and ensure that all deletes that are not visible anymore ie. not in the map
            // have a timestamp that is smaller or equal to the maximum timestamp that we pruned on
            final DeleteVersionValue delete = e.getValue();
            if (value == null) {
                assertTrue(delete.time + " > " + lastPrunedTimestamp.get() + "," + delete.seqNo + " > " + lastPrunedSeqNo.get(), delete.time <= lastPrunedTimestamp.get() && delete.seqNo <= lastPrunedSeqNo.get());
            } else {
                assertEquals(value, delete);
            }
        }
    });
    map.pruneTombstones(clock.incrementAndGet(), maxSeqNo.get());
    assertThat(map.getAllTombstones().entrySet(), empty());
}

Example 45 with Releasable

use of org.opensearch.common.lease.Releasable in project OpenSearch by opensearch-project.

the class LiveVersionMapTests method testAddAndDeleteRefreshConcurrently.

public void testAddAndDeleteRefreshConcurrently() throws IOException, InterruptedException {
    LiveVersionMap map = new LiveVersionMap();
    int numIters = randomIntBetween(1000, 5000);
    AtomicBoolean done = new AtomicBoolean(false);
    AtomicLong version = new AtomicLong();
    CountDownLatch start = new CountDownLatch(2);
    BytesRef uid = uid("1");
    VersionValue initialVersion;
    try (Releasable ignore = map.acquireLock(uid)) {
        initialVersion = new IndexVersionValue(randomTranslogLocation(), version.incrementAndGet(), 1, 1);
        map.putIndexUnderLock(uid, (IndexVersionValue) initialVersion);
    }
    Thread t = new Thread(() -> {
        start.countDown();
        try {
            start.await();
            VersionValue nextVersionValue = initialVersion;
            for (int i = 0; i < numIters; i++) {
                try (Releasable ignore = map.acquireLock(uid)) {
                    VersionValue underLock = map.getUnderLock(uid);
                    if (underLock != null) {
                        assertEquals(underLock, nextVersionValue);
                    } else {
                        underLock = nextVersionValue;
                    }
                    if (underLock.isDelete() || randomBoolean()) {
                        nextVersionValue = new IndexVersionValue(randomTranslogLocation(), version.incrementAndGet(), 1, 1);
                        map.putIndexUnderLock(uid, (IndexVersionValue) nextVersionValue);
                    } else {
                        nextVersionValue = new DeleteVersionValue(version.incrementAndGet(), 1, 1, 0);
                        map.putDeleteUnderLock(uid, (DeleteVersionValue) nextVersionValue);
                    }
                }
            }
        } catch (Exception e) {
            throw new AssertionError(e);
        } finally {
            done.set(true);
        }
    });
    t.start();
    start.countDown();
    while (done.get() == false) {
        map.beforeRefresh();
        Thread.yield();
        map.afterRefresh(false);
    }
    t.join();
    try (Releasable ignore = map.acquireLock(uid)) {
        VersionValue underLock = map.getUnderLock(uid);
        if (underLock != null) {
            assertEquals(version.get(), underLock.version);
        }
    }
}