Examples with StaticBuffer org.janusgraph.diskstorage.StaticBuffer used on opensource projects

Example 51 with StaticBuffer

use of org.janusgraph.diskstorage.StaticBuffer in project janusgraph by JanusGraph.

the class LogTest method testFuzzMessagesSerial.

@Test
@Tag(LogTest.requiresOrderPreserving)
public void testFuzzMessagesSerial() throws Exception {
    final int maxLen = 1024 * 4;
    final int rounds = 32;
    StoringReader reader = new StoringReader(rounds);
    final List<StaticBuffer> expected = new ArrayList<>(rounds);
    Log l = manager.openLog("fuzz");
    l.registerReader(ReadMarker.fromNow(), reader);
    Random rand = new Random();
    for (int i = 0; i < rounds; i++) {
        // int len = rand.nextInt(maxLen + 1);
        int len = maxLen;
        if (0 == len)
            // 0 would throw IllegalArgumentException
            len = 1;
        byte[] raw = new byte[len];
        rand.nextBytes(raw);
        StaticBuffer sb = StaticArrayBuffer.of(raw);
        l.add(sb);
        expected.add(sb);
        Thread.sleep(100L);
    }
    reader.await(TIMEOUT_MS);
    assertEquals(rounds, reader.msgCount);
    assertEquals(expected, reader.messages);
}

Example 52 with StaticBuffer

use of org.janusgraph.diskstorage.StaticBuffer in project janusgraph by JanusGraph.

the class HBaseStoreManager method convertToCommands.

/**
 * Convert JanusGraph internal Mutation representation into HBase native commands.
 *
 * @param mutations    Mutations to convert into HBase commands.
 * @param putTimestamp The timestamp to use for Put commands.
 * @param delTimestamp The timestamp to use for Delete commands.
 * @return Commands sorted by key converted from JanusGraph internal representation.
 * @throws org.janusgraph.diskstorage.PermanentBackendException
 */
@VisibleForTesting
Map<StaticBuffer, Pair<List<Put>, Delete>> convertToCommands(Map<String, Map<StaticBuffer, KCVMutation>> mutations, final Long putTimestamp, final Long delTimestamp) throws PermanentBackendException {
    // A map of rowkey to commands (list of Puts, Delete)
    final Map<StaticBuffer, Pair<List<Put>, Delete>> commandsPerKey = new HashMap<>();
    for (Map.Entry<String, Map<StaticBuffer, KCVMutation>> entry : mutations.entrySet()) {
        String cfString = getCfNameForStoreName(entry.getKey());
        byte[] cfName = Bytes.toBytes(cfString);
        for (Map.Entry<StaticBuffer, KCVMutation> m : entry.getValue().entrySet()) {
            final byte[] key = m.getKey().as(StaticBuffer.ARRAY_FACTORY);
            KCVMutation mutation = m.getValue();
            Pair<List<Put>, Delete> commands = commandsPerKey.get(m.getKey());
            // create the holder for a particular rowkey
            if (commands == null) {
                commands = new Pair<>();
                // List of all the Puts for this rowkey, including the ones without TTL and with TTL.
                final List<Put> putList = new ArrayList<>();
                commands.setFirst(putList);
                commandsPerKey.put(m.getKey(), commands);
            }
            if (mutation.hasDeletions()) {
                if (commands.getSecond() == null) {
                    Delete d = new Delete(key);
                    if (delTimestamp != null) {
                        d.setTimestamp(delTimestamp);
                    }
                    commands.setSecond(d);
                }
                for (StaticBuffer b : mutation.getDeletions()) {
                    // commands.getSecond() is a Delete for this rowkey.
                    addColumnToDelete(commands.getSecond(), cfName, b.as(StaticBuffer.ARRAY_FACTORY), delTimestamp);
                }
            }
            if (mutation.hasAdditions()) {
                // All the entries (column cells) with the rowkey use this one Put, except the ones with TTL.
                final Put putColumnsWithoutTtl = putTimestamp != null ? new Put(key, putTimestamp) : new Put(key);
                // that have TTL set.
                for (Entry e : mutation.getAdditions()) {
                    // Deal with TTL within the entry (column cell) first
                    // HBase cell level TTL is actually set at the Mutation/Put level.
                    // Therefore we need to construct a new Put for each entry (column cell) with TTL.
                    // We can not combine them because column cells within the same rowkey may:
                    // 1. have no TTL
                    // 2. have TTL
                    // 3. have different TTL
                    final Integer ttl = (Integer) e.getMetaData().get(EntryMetaData.TTL);
                    if (null != ttl && ttl > 0) {
                        // Create a new Put
                        Put putColumnWithTtl = putTimestamp != null ? new Put(key, putTimestamp) : new Put(key);
                        addColumnToPut(putColumnWithTtl, cfName, putTimestamp, e);
                        // Convert ttl from second (JanusGraph TTL) to milliseconds (HBase TTL)
                        // @see JanusGraphManagement#setTTL(JanusGraphSchemaType, Duration)
                        // HBase supports cell-level TTL for versions 0.98.6 and above.
                        (putColumnWithTtl).setTTL(TimeUnit.SECONDS.toMillis((long) ttl));
                        // commands.getFirst() is the list of Puts for this rowkey. Add this
                        // Put column with TTL to the list.
                        commands.getFirst().add(putColumnWithTtl);
                    } else {
                        addColumnToPut(putColumnsWithoutTtl, cfName, putTimestamp, e);
                    }
                }
                // If there were any mutations without TTL set, add them to commands.getFirst()
                if (!putColumnsWithoutTtl.isEmpty()) {
                    commands.getFirst().add(putColumnsWithoutTtl);
                }
            }
        }
    }
    return commandsPerKey;
}

Example 53 with StaticBuffer

use of org.janusgraph.diskstorage.StaticBuffer in project janusgraph by JanusGraph.

the class HBaseStoreManager method mutateMany.

@Override
public void mutateMany(Map<String, Map<StaticBuffer, KCVMutation>> mutations, StoreTransaction txh) throws BackendException {
    Long putTimestamp = null;
    Long delTimestamp = null;
    MaskedTimestamp commitTime = null;
    if (assignTimestamp) {
        commitTime = new MaskedTimestamp(txh);
        putTimestamp = commitTime.getAdditionTime(times);
        delTimestamp = commitTime.getDeletionTime(times);
    }
    // In case of an addition and deletion with identical timestamps, the
    // deletion tombstone wins.
    // https://hbase.apache.org/book/versions.html#d244e4250
    final Map<StaticBuffer, Pair<List<Put>, Delete>> commandsPerKey = convertToCommands(mutations, putTimestamp, delTimestamp);
    // actual batch operation
    final List<Row> batch = new ArrayList<>(commandsPerKey.size());
    // convert sorted commands into representation required for 'batch' operation
    for (Pair<List<Put>, Delete> commands : commandsPerKey.values()) {
        if (commands.getFirst() != null && !commands.getFirst().isEmpty())
            batch.addAll(commands.getFirst());
        if (commands.getSecond() != null)
            batch.add(commands.getSecond());
    }
    try {
        Table table = null;
        try {
            table = cnx.getTable(tableName);
            table.batch(batch, new Object[batch.size()]);
        } finally {
            IOUtils.closeQuietly(table);
        }
    } catch (IOException | InterruptedException e) {
        throw new TemporaryBackendException(e);
    }
    if (commitTime != null) {
        sleepAfterWrite(commitTime);
    }
}

Example 54 with StaticBuffer

use of org.janusgraph.diskstorage.StaticBuffer in project janusgraph by JanusGraph.

the class MultiPageEntryBuffer method mutate.

@Override
public void mutate(Entry[] add, Entry[] del, int maxPageSize) {
    int pageHits = 0;
    int oldPageCount = pages.size();
    // if new page is going to hit max size - insert new one
    if (pages.size() == 0) {
        pages.add(buildFromEntryArray(new Entry[] {}, 0));
    }
    int iadd = 0;
    int idel = 0;
    // NOTE: if it finds min of first add/first delete via getPageIndex (binary search), jumps straight to that page
    // - could be better for big stores updated sparsely. However in practice it doesn't seem to be any noticeable bottleneck
    int currPageNo = 0;
    while (currPageNo < pages.size() && (iadd < add.length || idel < del.length)) {
        BufferPage currPage = pages.get(currPageNo);
        BufferPage nextPage = (currPageNo + 1 < pages.size()) ? pages.get(currPageNo + 1) : null;
        // assumes there will be no pages with zero entries - i.e. we will delete a page if it contains no data
        Preconditions.checkArgument(nextPage == null || nextPage.numEntries() > 0);
        StaticBuffer nextPageStart = nextPage == null ? null : nextPage.getNoCopy(0);
        boolean pageNeedsMerging = false;
        // Compare with additions
        if (// still have things to add
        iadd < add.length) {
            // if there's no next page then we definitely need to merge into this page
            pageNeedsMerging = nextPageStart == null;
            if (!pageNeedsMerging) {
                // if next page start is bigger than the key we need to add, this means we need to merge this page
                // if next page start is smaller, then we can skip this page - we will merge one of the next pages we see
                int compare = nextPageStart.compareTo(add[iadd]);
                pageNeedsMerging = compare >= 0;
            }
        }
        // Compare with deletions
        if (// still have things to delete, and still not sure if we need to merge this page
        !pageNeedsMerging && idel < del.length) {
            // if this page end is bigger than the key we need to delete, this means we need to merge this page
            // if it is smaller, then we won't find anything to delete in this page anyway
            StaticBuffer thisPageEnd = currPage.getNoCopy(currPage.numEntries() - 1);
            int compare = thisPageEnd.compareTo(del[idel]);
            pageNeedsMerging = compare >= 0;
        }
        if (pageNeedsMerging) {
            int addLimit;
            int delLimit;
            if (// this is the last page, everything we still need to add/delete applies to it
            nextPageStart == null) {
                addLimit = add.length;
                delLimit = del.length;
            } else // this is not the last page, we need to determine which adds/deletes go to this page, and which go to next page(s)
            {
                // NOTE: for long mutation lists, it could be better to do binary search here,
                // otherwise it could be up to maxPageSize linear comparisons.
                // However it was not seen as a bottleneck in practice so far
                addLimit = iadd;
                while (addLimit < add.length && nextPageStart.compareTo(add[addLimit]) > 0) {
                    addLimit++;
                }
                delLimit = idel;
                while (delLimit < del.length && nextPageStart.compareTo(del[delLimit]) > 0) {
                    delLimit++;
                }
            }
            List<BufferPage> mergedPages = currPage.merge(add, iadd, addLimit, del, idel, delLimit, maxPageSize);
            if (// there was no data left in the page as a result of merge - remove old page
            mergedPages.size() == 0) {
                pages.remove(currPageNo);
            // do NOT increase currPageNo here as the next page moved in to this place
            } else // there is at least one page as a result of merge - replace the current one and insert any additional overflow pages
            {
                // replace the currPage with the newly merged version
                pages.set(currPageNo, mergedPages.get(0));
                // move to next page
                currPageNo++;
                if (// more than one page as a result of merge - insert all additional ones
                mergedPages.size() > 1) {
                    mergedPages.remove(0);
                    pages.addAll(currPageNo, mergedPages);
                    // skip over the pages we just added as they cannot contain any work we might still need to do
                    currPageNo += mergedPages.size();
                    pageHits += mergedPages.size();
                }
            }
            iadd = addLimit;
            idel = delLimit;
            pageHits++;
        } else {
            currPageNo++;
        }
    }
    if (oldPageCount >= pages.size()) {
        // it grew before but not this time, assume it stopped growing for now and trim to size to save memory
        pages.trimToSize();
    }
}

Example 55 with StaticBuffer

use of org.janusgraph.diskstorage.StaticBuffer in project janusgraph by JanusGraph.

the class InMemoryStoreManager method mutateMany.

@Override
public void mutateMany(Map<String, Map<StaticBuffer, KCVMutation>> mutations, StoreTransaction txh) throws BackendException {
    for (Map.Entry<String, Map<StaticBuffer, KCVMutation>> storeMut : mutations.entrySet()) {
        KeyColumnValueStore store = stores.get(storeMut.getKey());
        Preconditions.checkNotNull(store);
        for (Map.Entry<StaticBuffer, KCVMutation> keyMut : storeMut.getValue().entrySet()) {
            store.mutate(keyMut.getKey(), keyMut.getValue().getAdditions(), keyMut.getValue().getDeletions(), txh);
        }
    }
}