Examples with Entry org.janusgraph.diskstorage.Entry used on opensource projects

Example 31 with Entry

use of org.janusgraph.diskstorage.Entry in project janusgraph by JanusGraph.

the class LockCleanerRunnableTest method testDeletionWithExpiredAndValidLocks.

/**
 * Test the cleaner against a set of locks where some locks have timestamps
 * before the cutoff and others have timestamps after the cutoff. One lock
 * has a timestamp equal to the cutoff.
 */
@Test
public void testDeletionWithExpiredAndValidLocks() throws BackendException {
    final int lockCount = 10;
    final int expiredCount = 3;
    assertTrue(expiredCount + 2 <= lockCount);
    final long timeIncrement = 1L;
    final Instant timeStart = Instant.EPOCH;
    final Instant timeCutoff = timeStart.plusMillis(expiredCount * timeIncrement);
    ImmutableList.Builder<Entry> locksBuilder = ImmutableList.builder();
    ImmutableList.Builder<Entry> deletionBuilder = ImmutableList.builder();
    for (int i = 0; i < lockCount; i++) {
        final Instant ts = timeStart.plusMillis(timeIncrement * i);
        Entry lock = StaticArrayEntry.of(codec.toLockCol(ts, defaultLockRid, TimestampProviders.MILLI), BufferUtil.getIntBuffer(0));
        if (ts.isBefore(timeCutoff)) {
            deletionBuilder.add(lock);
        }
        locksBuilder.add(lock);
    }
    EntryList locks = StaticArrayEntryList.of(locksBuilder.build());
    EntryList deletions = StaticArrayEntryList.of(deletionBuilder.build());
    assertEquals(expiredCount, deletions.size());
    del = new StandardLockCleanerRunnable(store, kc, tx, codec, timeCutoff, TimestampProviders.MILLI);
    expect(store.getSlice(eq(ksq), eq(tx))).andReturn(locks);
    store.mutate(eq(key), eq(ImmutableList.of()), eq(columnsOf(deletions)), anyObject(StoreTransaction.class));
    ctrl.replay();
    del.run();
}

Example 32 with Entry

use of org.janusgraph.diskstorage.Entry in project janusgraph by JanusGraph.

the class StaticArrayEntryTest method testToStringWithNoValue.

private void testToStringWithNoValue(long l) {
    StaticBuffer b = BufferUtil.getLongBuffer(l);
    long column = b.getLong(0);
    Entry entry = new StaticArrayEntry(b, 8);
    final String[] split = entry.toString().split("->");
    assertEquals(String.format("0x%016X", column), split[0]);
    assertEquals("[no value]", split[1]);
}

Example 33 with Entry

use of org.janusgraph.diskstorage.Entry in project janusgraph by JanusGraph.

the class CassandraBinaryRecordReader method completeNextKV.

private KV completeNextKV() throws IOException {
    KV completedKV = null;
    boolean hasNext;
    do {
        hasNext = reader.nextKeyValue();
        if (!hasNext) {
            completedKV = incompleteKV;
            incompleteKV = null;
        } else {
            StaticArrayBuffer key = StaticArrayBuffer.of(reader.getCurrentKey());
            SortedMap<ByteBuffer, Cell> valueSortedMap = reader.getCurrentValue();
            List<Entry> entries = new ArrayList<>(valueSortedMap.size());
            for (Map.Entry<ByteBuffer, Cell> ent : valueSortedMap.entrySet()) {
                ByteBuffer col = ent.getKey();
                ByteBuffer val = ent.getValue().value();
                entries.add(StaticArrayEntry.of(StaticArrayBuffer.of(col), StaticArrayBuffer.of(val)));
            }
            if (null == incompleteKV) {
                // Initialization; this should happen just once in an instance's lifetime
                incompleteKV = new KV(key);
            } else if (!incompleteKV.key.equals(key)) {
                // The underlying Cassandra reader has just changed to a key we haven't seen yet
                // This implies that there will be no more entries for the prior key
                completedKV = incompleteKV;
                incompleteKV = new KV(key);
            }
            incompleteKV.addEntries(entries);
        }
    /* Loop ends when either
             * A) the cassandra reader ran out of data
             * or
             * B) the cassandra reader switched keys, thereby completing a KV */
    } while (hasNext && null == completedKV);
    return completedKV;
}

Example 34 with Entry

use of org.janusgraph.diskstorage.Entry in project janusgraph by JanusGraph.

the class ColumnValueStore method mutate.

synchronized void mutate(List<Entry> additions, List<StaticBuffer> deletions, StoreTransaction txh) {
    // Prepare data
    Entry[] add;
    if (!additions.isEmpty()) {
        add = new Entry[additions.size()];
        int pos = 0;
        for (Entry e : additions) {
            add[pos] = e;
            pos++;
        }
        Arrays.sort(add);
    } else
        add = new Entry[0];
    // Filter out deletions that are also added
    Entry[] del;
    if (!deletions.isEmpty()) {
        del = new Entry[deletions.size()];
        int pos = 0;
        for (StaticBuffer deletion : deletions) {
            Entry delEntry = StaticArrayEntry.of(deletion);
            if (Arrays.binarySearch(add, delEntry) >= 0)
                continue;
            del[pos++] = delEntry;
        }
        if (pos < deletions.size())
            del = Arrays.copyOf(del, pos);
        Arrays.sort(del);
    } else
        del = new Entry[0];
    Lock lock = getLock(txh);
    lock.lock();
    try {
        Entry[] oldData = data.array;
        int oldSize = data.size;
        Entry[] newData = new Entry[oldSize + add.length];
        // Merge sort
        int i = 0, indexOld = 0, indexAdd = 0, indexDelete = 0;
        while (indexOld < oldSize) {
            Entry e = oldData[indexOld];
            indexOld++;
            // Compare with additions
            if (indexAdd < add.length) {
                int compare = e.compareTo(add[indexAdd]);
                if (compare >= 0) {
                    e = add[indexAdd];
                    indexAdd++;
                    // Skip duplicates
                    while (indexAdd < add.length && e.equals(add[indexAdd])) indexAdd++;
                }
                if (compare > 0)
                    indexOld--;
            }
            // Compare with deletions
            if (indexDelete < del.length) {
                int compare = e.compareTo(del[indexDelete]);
                if (compare == 0)
                    e = null;
                if (compare >= 0)
                    indexDelete++;
            }
            if (e != null) {
                newData[i] = e;
                i++;
            }
        }
        while (indexAdd < add.length) {
            newData[i] = add[indexAdd];
            i++;
            indexAdd++;
        }
        if (i * 1.0 / newData.length < SIZE_THRESHOLD) {
            // shrink array to free space
            Entry[] tempData = newData;
            newData = new Entry[i];
            System.arraycopy(tempData, 0, newData, 0, i);
        }
        data = new Data(newData, i);
    } finally {
        lock.unlock();
    }
}

Example 35 with Entry

use of org.janusgraph.diskstorage.Entry in project janusgraph by JanusGraph.

the class HBaseStoreManager method convertToCommands.

/**
 * Convert JanusGraph internal Mutation representation into HBase native commands.
 *
 * @param mutations    Mutations to convert into HBase commands.
 * @param putTimestamp The timestamp to use for Put commands.
 * @param delTimestamp The timestamp to use for Delete commands.
 * @return Commands sorted by key converted from JanusGraph internal representation.
 * @throws org.janusgraph.diskstorage.PermanentBackendException
 */
@VisibleForTesting
Map<StaticBuffer, Pair<List<Put>, Delete>> convertToCommands(Map<String, Map<StaticBuffer, KCVMutation>> mutations, final Long putTimestamp, final Long delTimestamp) throws PermanentBackendException {
    // A map of rowkey to commands (list of Puts, Delete)
    final Map<StaticBuffer, Pair<List<Put>, Delete>> commandsPerKey = new HashMap<>();
    for (Map.Entry<String, Map<StaticBuffer, KCVMutation>> entry : mutations.entrySet()) {
        String cfString = getCfNameForStoreName(entry.getKey());
        byte[] cfName = Bytes.toBytes(cfString);
        for (Map.Entry<StaticBuffer, KCVMutation> m : entry.getValue().entrySet()) {
            final byte[] key = m.getKey().as(StaticBuffer.ARRAY_FACTORY);
            KCVMutation mutation = m.getValue();
            Pair<List<Put>, Delete> commands = commandsPerKey.get(m.getKey());
            // create the holder for a particular rowkey
            if (commands == null) {
                commands = new Pair<>();
                // List of all the Puts for this rowkey, including the ones without TTL and with TTL.
                final List<Put> putList = new ArrayList<>();
                commands.setFirst(putList);
                commandsPerKey.put(m.getKey(), commands);
            }
            if (mutation.hasDeletions()) {
                if (commands.getSecond() == null) {
                    Delete d = new Delete(key);
                    if (delTimestamp != null) {
                        d.setTimestamp(delTimestamp);
                    }
                    commands.setSecond(d);
                }
                for (StaticBuffer b : mutation.getDeletions()) {
                    // commands.getSecond() is a Delete for this rowkey.
                    addColumnToDelete(commands.getSecond(), cfName, b.as(StaticBuffer.ARRAY_FACTORY), delTimestamp);
                }
            }
            if (mutation.hasAdditions()) {
                // All the entries (column cells) with the rowkey use this one Put, except the ones with TTL.
                final Put putColumnsWithoutTtl = putTimestamp != null ? new Put(key, putTimestamp) : new Put(key);
                // that have TTL set.
                for (Entry e : mutation.getAdditions()) {
                    // Deal with TTL within the entry (column cell) first
                    // HBase cell level TTL is actually set at the Mutation/Put level.
                    // Therefore we need to construct a new Put for each entry (column cell) with TTL.
                    // We can not combine them because column cells within the same rowkey may:
                    // 1. have no TTL
                    // 2. have TTL
                    // 3. have different TTL
                    final Integer ttl = (Integer) e.getMetaData().get(EntryMetaData.TTL);
                    if (null != ttl && ttl > 0) {
                        // Create a new Put
                        Put putColumnWithTtl = putTimestamp != null ? new Put(key, putTimestamp) : new Put(key);
                        addColumnToPut(putColumnWithTtl, cfName, putTimestamp, e);
                        // Convert ttl from second (JanusGraph TTL) to milliseconds (HBase TTL)
                        // @see JanusGraphManagement#setTTL(JanusGraphSchemaType, Duration)
                        // HBase supports cell-level TTL for versions 0.98.6 and above.
                        (putColumnWithTtl).setTTL(TimeUnit.SECONDS.toMillis((long) ttl));
                        // commands.getFirst() is the list of Puts for this rowkey. Add this
                        // Put column with TTL to the list.
                        commands.getFirst().add(putColumnWithTtl);
                    } else {
                        addColumnToPut(putColumnsWithoutTtl, cfName, putTimestamp, e);
                    }
                }
                // If there were any mutations without TTL set, add them to commands.getFirst()
                if (!putColumnsWithoutTtl.isEmpty()) {
                    commands.getFirst().add(putColumnsWithoutTtl);
                }
            }
        }
    }
    return commandsPerKey;
}