Examples with StaticArrayBuffer org.janusgraph.diskstorage.util.StaticArrayBuffer used on opensource projects

Example 1 with StaticArrayBuffer

use of org.janusgraph.diskstorage.util.StaticArrayBuffer in project janusgraph by JanusGraph.

the class CassandraBinaryRecordReader method completeNextKV.

private KV completeNextKV() throws IOException {
    KV completedKV = null;
    boolean hasNext;
    do {
        hasNext = reader.nextKeyValue();
        if (!hasNext) {
            completedKV = incompleteKV;
            incompleteKV = null;
        } else {
            StaticArrayBuffer key = StaticArrayBuffer.of(reader.getCurrentKey());
            SortedMap<ByteBuffer, Cell> valueSortedMap = reader.getCurrentValue();
            List<Entry> entries = new ArrayList<>(valueSortedMap.size());
            for (Map.Entry<ByteBuffer, Cell> ent : valueSortedMap.entrySet()) {
                ByteBuffer col = ent.getKey();
                ByteBuffer val = ent.getValue().value();
                entries.add(StaticArrayEntry.of(StaticArrayBuffer.of(col), StaticArrayBuffer.of(val)));
            }
            if (null == incompleteKV) {
                // Initialization; this should happen just once in an instance's lifetime
                incompleteKV = new KV(key);
            } else if (!incompleteKV.key.equals(key)) {
                // The underlying Cassandra reader has just changed to a key we haven't seen yet
                // This implies that there will be no more entries for the prior key
                completedKV = incompleteKV;
                incompleteKV = new KV(key);
            }
            incompleteKV.addEntries(entries);
        }
    /* Loop ends when either
             * A) the cassandra reader ran out of data
             * or
             * B) the cassandra reader switched keys, thereby completing a KV */
    } while (hasNext && null == completedKV);
    return completedKV;
}

Example 2 with StaticArrayBuffer

use of org.janusgraph.diskstorage.util.StaticArrayBuffer in project janusgraph by JanusGraph.

the class HBaseStoreManagerMutationTest method testKCVMutationToPuts.

@Test
public void testKCVMutationToPuts() throws Exception {
    final Map<String, Map<StaticBuffer, KCVMutation>> storeMutationMap = new HashMap<>();
    final Map<StaticBuffer, KCVMutation> rowkeyMutationMap = new HashMap<>();
    final List<Long> expectedColumnsWithTTL = new ArrayList<>();
    final List<Long> expectedColumnsWithoutTTL = new ArrayList<>();
    final List<Long> expectedColumnDelete = new ArrayList<>();
    StaticArrayEntry e = null;
    StaticBuffer rowkey, col, val;
    // 2 rows
    for (int row = 0; row < 2; row++) {
        rowkey = KeyColumnValueStoreUtil.longToByteBuffer(row);
        List<Entry> additions = new ArrayList<>();
        List<StaticBuffer> deletions = new ArrayList<>();
        // 100 columns each row
        int i;
        for (i = 0; i < 100; i++) {
            col = KeyColumnValueStoreUtil.longToByteBuffer(i);
            val = KeyColumnValueStoreUtil.longToByteBuffer(i + 100);
            e = (StaticArrayEntry) StaticArrayEntry.of(col, val);
            // Set half of the columns with TTL, also vary the TTL values
            if (i % 2 == 0) {
                e.setMetaData(EntryMetaData.TTL, i % 10 + 1);
                // Collect the columns with TTL. Only do this for one row
                if (row == 1) {
                    expectedColumnsWithTTL.add((long) i);
                }
                additions.add(e);
            } else {
                // Collect the columns without TTL. Only do this for one row
                if (row == 1) {
                    expectedColumnsWithoutTTL.add((long) i);
                }
                additions.add(e);
            }
        }
        // Add one deletion to the row
        if (row == 1) {
            expectedColumnDelete.add((long) (i - 1));
        }
        deletions.add(e);
        rowkeyMutationMap.put(rowkey, new KCVMutation(additions, deletions));
    }
    storeMutationMap.put("store1", rowkeyMutationMap);
    HBaseStoreManager manager = new HBaseStoreManager(HBaseStorageSetup.getHBaseConfiguration());
    final Map<StaticBuffer, Pair<List<Put>, Delete>> commandsPerRowKey = manager.convertToCommands(storeMutationMap, 0, 0);
    // 2 rows
    Assert.assertEquals(commandsPerRowKey.size(), 2);
    // Verify puts
    final List<Long> putColumnsWithTTL = new ArrayList<>();
    final List<Long> putColumnsWithoutTTL = new ArrayList<>();
    Pair<List<Put>, Delete> commands = commandsPerRowKey.values().iterator().next();
    long colName;
    for (Put p : commands.getFirst()) {
        // In Put, Long.MAX_VALUE means no TTL
        for (Map.Entry<byte[], List<Cell>> me : p.getFamilyCellMap().entrySet()) {
            for (Cell c : me.getValue()) {
                colName = KeyColumnValueStoreUtil.bufferToLong(new StaticArrayBuffer(CellUtil.cloneQualifier(c)));
                if (p.getTTL() < Long.MAX_VALUE) {
                    putColumnsWithTTL.add(colName);
                } else {
                    putColumnsWithoutTTL.add(colName);
                }
            }
        }
    }
    Collections.sort(putColumnsWithoutTTL);
    Collections.sort(putColumnsWithTTL);
    Assert.assertArrayEquals(expectedColumnsWithoutTTL.toArray(), putColumnsWithoutTTL.toArray());
    Assert.assertArrayEquals(expectedColumnsWithTTL.toArray(), putColumnsWithTTL.toArray());
    // Verify deletes
    final List<Long> deleteColumns = new ArrayList<>();
    Delete d = commands.getSecond();
    for (Map.Entry<byte[], List<Cell>> me : d.getFamilyCellMap().entrySet()) {
        for (Cell c : me.getValue()) {
            colName = KeyColumnValueStoreUtil.bufferToLong(new StaticArrayBuffer(CellUtil.cloneQualifier(c)));
            deleteColumns.add(colName);
        }
    }
    Collections.sort(deleteColumns);
    Assert.assertArrayEquals(expectedColumnDelete.toArray(), deleteColumns.toArray());
}

Example 3 with StaticArrayBuffer

use of org.janusgraph.diskstorage.util.StaticArrayBuffer in project janusgraph by JanusGraph.

the class MultiWriteKeyColumnValueStoreTest method mutateState.

/**
 * Change the supplied {@code state} with a pseudorandom number generator.
 * <p/>
 * This method removes {@code min(maxDeletionCount, S)} entries from the
 * maps in {@code state.values()}, where {@code S} is the sum of the sizes
 * of the maps in {@code state.values()}; this method then adds
 * {@code additionCount} pseudorandom entries spread across
 * {@code state.values()}, potentially adding new keys to {@code state}
 * since they are randomly generated. This method then returns a map of keys
 * to Mutations representing the changes it has made to {@code state}.
 *
 * @param state            Maps keys -> columns -> values
 * @param maxDeletionCount Remove at most this many entries from state
 * @param additionCount    Add exactly this many entries to state
 * @return A KCVMutation map
 */
public Map<StaticBuffer, KCVEntryMutation> mutateState(Map<StaticBuffer, Map<StaticBuffer, StaticBuffer>> state, int maxDeletionCount, int additionCount) {
    final int keyLength = 8;
    final int colLength = 16;
    final Map<StaticBuffer, KCVEntryMutation> result = new HashMap<>();
    // deletion pass
    int deletions = 0;
    StaticBuffer key = null, col = null;
    Entry entry = null;
    Iterator<StaticBuffer> iterator = state.keySet().iterator();
    while (iterator.hasNext() && deletions < maxDeletionCount) {
        key = iterator.next();
        Iterator<Map.Entry<StaticBuffer, StaticBuffer>> columnIterator = state.get(key).entrySet().iterator();
        while (columnIterator.hasNext() && deletions < maxDeletionCount) {
            Map.Entry<StaticBuffer, StaticBuffer> colEntry = columnIterator.next();
            entry = StaticArrayEntry.of(colEntry.getKey(), colEntry.getValue());
            if (!result.containsKey(key)) {
                KCVEntryMutation m = new KCVEntryMutation(new LinkedList<>(), new LinkedList<>());
                result.put(key, m);
            }
            result.get(key).deletion(entry);
            deletions++;
            columnIterator.remove();
            if (state.get(key).isEmpty()) {
                assert !columnIterator.hasNext();
                iterator.remove();
            }
        }
    }
    // addition pass
    for (int i = 0; i < additionCount; i++) {
        while (true) {
            byte[] keyBuf = new byte[keyLength];
            rand.nextBytes(keyBuf);
            key = new StaticArrayBuffer(keyBuf);
            byte[] colBuf = new byte[colLength];
            rand.nextBytes(colBuf);
            col = new StaticArrayBuffer(colBuf);
            if (!state.containsKey(key) || !state.get(key).containsKey(col)) {
                break;
            }
        }
        if (!state.containsKey(key)) {
            Map<StaticBuffer, StaticBuffer> m = new HashMap<>();
            state.put(key, m);
        }
        state.get(key).put(col, col);
        if (!result.containsKey(key)) {
            KCVEntryMutation m = new KCVEntryMutation(new LinkedList<>(), new LinkedList<>());
            result.put(key, m);
        }
        result.get(key).addition(StaticArrayEntry.of(col, col));
    }
    return result;
}

Example 4 with StaticArrayBuffer

use of org.janusgraph.diskstorage.util.StaticArrayBuffer in project janusgraph by JanusGraph.

the class ConsistentKeyLockerSerializer method fromLockColumn.

public TimestampRid fromLockColumn(StaticBuffer lockKey, TimestampProvider provider) {
    ReadBuffer r = lockKey.asReadBuffer();
    int len = r.length();
    long tsNS = r.getLong();
    len -= 8;
    byte[] curRid = new byte[len];
    for (int i = 0; r.hasRemaining(); i++) {
        curRid[i] = r.getByte();
    }
    StaticBuffer rid = new StaticArrayBuffer(curRid);
    Instant time = provider.getTime(tsNS);
    return new TimestampRid(time, rid);
}