Examples with KeySliceQuery org.janusgraph.diskstorage.keycolumnvalue.KeySliceQuery used on opensource projects

Example 11 with KeySliceQuery

use of org.janusgraph.diskstorage.keycolumnvalue.KeySliceQuery in project janusgraph by JanusGraph.

the class ConsistentKeyLocker method checkSingleLock.

@Override
protected void checkSingleLock(final KeyColumn kc, final ConsistentKeyLockStatus ls, final StoreTransaction tx) throws BackendException, InterruptedException {
    if (ls.isChecked())
        return;
    // Sleep, if necessary
    // We could be smarter about sleeping by iterating oldest -> latest...
    final Instant now = times.sleepPast(ls.getWriteTimestamp().plus(lockWait));
    // Slice the store
    KeySliceQuery ksq = new KeySliceQuery(serializer.toLockKey(kc.getKey(), kc.getColumn()), LOCK_COL_START, LOCK_COL_END);
    List<Entry> claimEntries = getSliceWithRetries(ksq, tx);
    // Extract timestamp and rid from the column in each returned Entry...
    final Iterable<TimestampRid> iterable = Iterables.transform(claimEntries, e -> serializer.fromLockColumn(e.getColumnAs(StaticBuffer.STATIC_FACTORY), times));
    // ...and then filter out the TimestampRid objects with expired timestamps
    // (This doesn't use Iterables.filter and Predicate so that we can throw a checked exception if necessary)
    final List<TimestampRid> unexpiredTRs = new ArrayList<>(Iterables.size(iterable));
    final Instant cutoffTime = now.minus(lockExpire);
    for (TimestampRid tr : iterable) {
        if (tr.getTimestamp().isBefore(cutoffTime)) {
            log.warn("Discarded expired claim on {} with timestamp {}", kc, tr.getTimestamp());
            if (null != cleanerService)
                cleanerService.clean(kc, cutoffTime, tx);
            // but also throw a descriptive exception
            if (rid.equals(tr.getRid()) && ls.getWriteTimestamp().equals(tr.getTimestamp())) {
                throw new ExpiredLockException("Expired lock on " + kc + ": lock timestamp " + tr.getTimestamp() + " " + times.getUnit() + " is older than " + ConfigElement.getPath(GraphDatabaseConfiguration.LOCK_EXPIRE) + "=" + lockExpire);
            // Really shouldn't refer to GDC.LOCK_EXPIRE here,
            // but this will typically be accurate in a real use case
            }
            continue;
        }
        unexpiredTRs.add(tr);
    }
    checkSeniority(kc, ls, unexpiredTRs);
    ls.setChecked();
}

Example 12 with KeySliceQuery

use of org.janusgraph.diskstorage.keycolumnvalue.KeySliceQuery in project janusgraph by JanusGraph.

the class ExpectedValueCheckingTransaction method checkSingleExpectedValueUnsafe.

private void checkSingleExpectedValueUnsafe(final KeyColumn kc, final StaticBuffer ev, final ExpectedValueCheckingStore store) throws BackendException {
    final StaticBuffer nextBuf = BufferUtil.nextBiggerBuffer(kc.getColumn());
    KeySliceQuery ksq = new KeySliceQuery(kc.getKey(), kc.getColumn(), nextBuf);
    // Call getSlice on the wrapped store using the quorum+ consistency tx
    Iterable<Entry> actualEntries = store.getBackingStore().getSlice(ksq, strongConsistentTx);
    if (null == actualEntries)
        actualEntries = Collections.emptyList();
    /*
         * Discard any columns which do not exactly match kc.getColumn().
         *
         * For example, it's possible that the slice returned columns which for
         * which kc.getColumn() is a prefix.
         */
    actualEntries = Iterables.filter(actualEntries, input -> {
        if (!input.getColumn().equals(kc.getColumn())) {
            log.debug("Dropping entry {} (only accepting column {})", input, kc.getColumn());
            return false;
        }
        log.debug("Accepting entry {}", input);
        return true;
    });
    // Extract values from remaining Entry instances
    final Iterable<StaticBuffer> actualValues = Iterables.transform(actualEntries, e -> {
        final StaticBuffer actualCol = e.getColumnAs(StaticBuffer.STATIC_FACTORY);
        assert null != actualCol;
        assert null != kc.getColumn();
        assert 0 >= kc.getColumn().compareTo(actualCol);
        assert 0 > actualCol.compareTo(nextBuf);
        return e.getValueAs(StaticBuffer.STATIC_FACTORY);
    });
    final Iterable<StaticBuffer> expectedValues;
    if (null == ev) {
        expectedValues = Collections.emptyList();
    } else {
        expectedValues = Collections.singletonList(ev);
    }
    if (!Iterables.elementsEqual(expectedValues, actualValues)) {
        throw new PermanentLockingException("Expected value mismatch for " + kc + ": expected=" + expectedValues + " vs actual=" + actualValues + " (store=" + store.getName() + ")");
    }
}

Example 13 with KeySliceQuery

use of org.janusgraph.diskstorage.keycolumnvalue.KeySliceQuery in project janusgraph by JanusGraph.

the class StandardLockCleanerRunnable method runWithExceptions.

private void runWithExceptions() throws BackendException {
    StaticBuffer lockKey = serializer.toLockKey(target.getKey(), target.getColumn());
    // TODO reduce LOCK_COL_END based on cutoff
    List<Entry> locks = store.getSlice(new KeySliceQuery(lockKey, LOCK_COL_START, LOCK_COL_END), tx);
    List<StaticBuffer> deletions = new LinkedList<>();
    for (Entry lc : locks) {
        TimestampRid tr = serializer.fromLockColumn(lc.getColumn(), times);
        if (tr.getTimestamp().isBefore(cutoff)) {
            log.info("Deleting expired lock on {} by rid {} with timestamp {} (before or at cutoff {})", target, tr.getRid(), tr.getTimestamp(), cutoff);
            deletions.add(lc.getColumn());
        } else {
            log.debug("Ignoring lock on {} by rid {} with timestamp {} (timestamp is after cutoff {})", target, tr.getRid(), tr.getTimestamp(), cutoff);
        }
    }
    deletions = Collections.unmodifiableList(deletions);
    if (!deletions.isEmpty()) {
        store.mutate(lockKey, Collections.emptyList(), deletions, tx);
        log.info("Deleted {} expired locks (before or at cutoff {})", deletions.size(), cutoff);
    }
}

Example 14 with KeySliceQuery

use of org.janusgraph.diskstorage.keycolumnvalue.KeySliceQuery in project janusgraph by JanusGraph.

the class ExpectedValueCheckingTest method testMutateManyWithLockUsesConsistentTx.

@Test
public void testMutateManyWithLockUsesConsistentTx() throws BackendException {
    final ImmutableList<Entry> adds = ImmutableList.of(StaticArrayEntry.of(DATA_COL, DATA_VAL));
    final ImmutableList<StaticBuffer> deletions = ImmutableList.of();
    Map<String, Map<StaticBuffer, KCVMutation>> mutations = ImmutableMap.of(STORE_NAME, ImmutableMap.of(DATA_KEY, new KCVMutation(adds, deletions)));
    final KeyColumn kc = new KeyColumn(LOCK_KEY, LOCK_COL);
    // Acquire a lock
    backingLocker.writeLock(kc, consistentTx);
    // 2. Run mutateMany
    // 2.1. Check locks & expected values before mutating data
    backingLocker.checkLocks(consistentTx);
    StaticBuffer nextBuf = BufferUtil.nextBiggerBuffer(kc.getColumn());
    KeySliceQuery expectedValueQuery = new KeySliceQuery(kc.getKey(), kc.getColumn(), nextBuf);
    // expected value read must use strong consistency
    expect(backingStore.getSlice(expectedValueQuery, consistentTx)).andReturn(StaticArrayEntryList.of(StaticArrayEntry.of(LOCK_COL, LOCK_VAL)));
    // 2.2. Run mutateMany on backing manager to modify data
    // writes by txs with locks must use strong consistency
    backingManager.mutateMany(mutations, consistentTx);
    ctrl.replay();
    // Lock acquisition
    expectStore.acquireLock(LOCK_KEY, LOCK_COL, LOCK_VAL, expectTx);
    // Mutate
    expectManager.mutateMany(mutations, expectTx);
}

Example 15 with KeySliceQuery

use of org.janusgraph.diskstorage.keycolumnvalue.KeySliceQuery in project janusgraph by JanusGraph.

the class KCVSCacheTest method testSmallCache.

@Test
public void testSmallCache() throws Exception {
    // numCols must be greater than or equal to 10 as it is assumed below
    final int numKeys = 100, numCols = 10;
    final int repeats = 100, clearEvery = 20, numMulti = 10;
    loadStore(numKeys, numCols);
    // Repeatedly read from cache and clear in between
    int calls = 0;
    assertEquals(calls, store.getSliceCalls());
    for (int t = 0; t < repeats; t++) {
        if (t % clearEvery == 0) {
            cache.clearCache();
            calls += numKeys * 2 + 1;
        }
        CacheTransaction tx = getCacheTx();
        for (int i = 1; i <= numKeys; i++) {
            assertEquals(10, cache.getSlice(getQuery(i, 0, numCols + 1).setLimit(10), tx).size());
            assertEquals(3, cache.getSlice(getQuery(i, 2, 5), tx).size());
        }
        // Multi-query
        final List<StaticBuffer> keys = new ArrayList<>();
        for (int i = 10; i < 10 + numMulti; i++) keys.add(BufferUtil.getIntBuffer(i));
        Map<StaticBuffer, EntryList> result = cache.getSlice(keys, getQuery(4, 9), tx);
        assertEquals(keys.size(), result.size());
        for (StaticBuffer key : keys) assertTrue(result.containsKey(key));
        for (EntryList r : result.values()) {
            assertEquals(5, r.size());
        }
        tx.commit();
        assertEquals(calls, store.getSliceCalls());
    }
    store.resetCounter();
    // Check invalidation
    StaticBuffer key = BufferUtil.getIntBuffer(23);
    final List<StaticBuffer> keys = new ArrayList<>();
    keys.add(key);
    keys.add(BufferUtil.getIntBuffer(12));
    keys.add(BufferUtil.getIntBuffer(5));
    // Read
    CacheTransaction tx = getCacheTx();
    assertEquals(numCols, cache.getSlice(new KeySliceQuery(key, getQuery(0, numCols + 1)), tx).size());
    Map<StaticBuffer, EntryList> result = cache.getSlice(keys, getQuery(2, 8), tx);
    assertEquals(keys.size(), result.size());
    assertEquals(6, result.get(key).size());
    // Update
    final List<Entry> deletions = new ArrayList<>(numCols / 2);
    for (int j = 1; j <= numCols; j = j + 2) deletions.add(getEntry(j, j));
    cache.mutateEntries(key, KeyColumnValueStore.NO_ADDITIONS, deletions, tx);
    tx.commit();
    assertEquals(2, store.getSliceCalls());
    // Ensure updates are correctly read
    tx = getCacheTx();
    assertEquals(numCols / 2, cache.getSlice(new KeySliceQuery(key, getQuery(0, numCols + 1)), tx).size());
    result = cache.getSlice(keys, getQuery(2, 8), tx);
    assertEquals(keys.size(), result.size());
    assertEquals(3, result.get(key).size());
    tx.commit();
    assertEquals(4, store.getSliceCalls());
}