Examples with Entry org.janusgraph.diskstorage.Entry used on opensource projects

Example 36 with Entry

use of org.janusgraph.diskstorage.Entry in project janusgraph by JanusGraph.

the class BufferPage method merge.

public List<BufferPage> merge(Entry[] add, int iaddp, int addLimit, Entry[] del, int idelp, int delLimit, int maxPageSize) {
    int iadd = iaddp;
    int idel = idelp;
    Entry[] newdata = new Entry[numEntries() + addLimit - iadd];
    // Merge sort
    int i = 0;
    int iold = 0;
    while (iold < numEntries()) {
        Entry e = getNoCopy(iold);
        iold++;
        // Compare with additions
        if (iadd < addLimit) {
            int compare = e.compareTo(add[iadd]);
            if (compare >= 0) {
                e = add[iadd];
                iadd++;
                // Skip duplicates
                while (iadd < addLimit && e.equals(add[iadd])) {
                    iadd++;
                }
            }
            if (compare > 0) {
                iold--;
            }
        }
        // Compare with deletions
        if (idel < delLimit) {
            int compare = e.compareTo(del[idel]);
            if (compare == 0) {
                e = null;
            }
            if (compare >= 0) {
                idel++;
            }
        }
        if (e != null) {
            newdata[i] = e;
            i++;
        }
    }
    while (iadd < addLimit) {
        newdata[i] = add[iadd];
        i++;
        iadd++;
    }
    int newDataEnd = i;
    int numNewPages = newDataEnd / maxPageSize;
    if (newDataEnd % maxPageSize > 0) {
        numNewPages++;
    }
    List<BufferPage> newPages = new ArrayList<>(numNewPages);
    for (i = 0; i < numNewPages; i++) {
        newPages.add(BufferPageUtils.buildFromEntryArray(newdata, i * maxPageSize, Math.min(newDataEnd, (i + 1) * maxPageSize)));
    }
    return newPages;
}

Example 37 with Entry

use of org.janusgraph.diskstorage.Entry in project janusgraph by JanusGraph.

the class MultiPageEntryBuffer method mutate.

@Override
public void mutate(Entry[] add, Entry[] del, int maxPageSize) {
    int pageHits = 0;
    int oldPageCount = pages.size();
    // if new page is going to hit max size - insert new one
    if (pages.size() == 0) {
        pages.add(buildFromEntryArray(new Entry[] {}, 0));
    }
    int iadd = 0;
    int idel = 0;
    // NOTE: if it finds min of first add/first delete via getPageIndex (binary search), jumps straight to that page
    // - could be better for big stores updated sparsely. However in practice it doesn't seem to be any noticeable bottleneck
    int currPageNo = 0;
    while (currPageNo < pages.size() && (iadd < add.length || idel < del.length)) {
        BufferPage currPage = pages.get(currPageNo);
        BufferPage nextPage = (currPageNo + 1 < pages.size()) ? pages.get(currPageNo + 1) : null;
        // assumes there will be no pages with zero entries - i.e. we will delete a page if it contains no data
        Preconditions.checkArgument(nextPage == null || nextPage.numEntries() > 0);
        StaticBuffer nextPageStart = nextPage == null ? null : nextPage.getNoCopy(0);
        boolean pageNeedsMerging = false;
        // Compare with additions
        if (// still have things to add
        iadd < add.length) {
            // if there's no next page then we definitely need to merge into this page
            pageNeedsMerging = nextPageStart == null;
            if (!pageNeedsMerging) {
                // if next page start is bigger than the key we need to add, this means we need to merge this page
                // if next page start is smaller, then we can skip this page - we will merge one of the next pages we see
                int compare = nextPageStart.compareTo(add[iadd]);
                pageNeedsMerging = compare >= 0;
            }
        }
        // Compare with deletions
        if (// still have things to delete, and still not sure if we need to merge this page
        !pageNeedsMerging && idel < del.length) {
            // if this page end is bigger than the key we need to delete, this means we need to merge this page
            // if it is smaller, then we won't find anything to delete in this page anyway
            StaticBuffer thisPageEnd = currPage.getNoCopy(currPage.numEntries() - 1);
            int compare = thisPageEnd.compareTo(del[idel]);
            pageNeedsMerging = compare >= 0;
        }
        if (pageNeedsMerging) {
            int addLimit;
            int delLimit;
            if (// this is the last page, everything we still need to add/delete applies to it
            nextPageStart == null) {
                addLimit = add.length;
                delLimit = del.length;
            } else // this is not the last page, we need to determine which adds/deletes go to this page, and which go to next page(s)
            {
                // NOTE: for long mutation lists, it could be better to do binary search here,
                // otherwise it could be up to maxPageSize linear comparisons.
                // However it was not seen as a bottleneck in practice so far
                addLimit = iadd;
                while (addLimit < add.length && nextPageStart.compareTo(add[addLimit]) > 0) {
                    addLimit++;
                }
                delLimit = idel;
                while (delLimit < del.length && nextPageStart.compareTo(del[delLimit]) > 0) {
                    delLimit++;
                }
            }
            List<BufferPage> mergedPages = currPage.merge(add, iadd, addLimit, del, idel, delLimit, maxPageSize);
            if (// there was no data left in the page as a result of merge - remove old page
            mergedPages.size() == 0) {
                pages.remove(currPageNo);
            // do NOT increase currPageNo here as the next page moved in to this place
            } else // there is at least one page as a result of merge - replace the current one and insert any additional overflow pages
            {
                // replace the currPage with the newly merged version
                pages.set(currPageNo, mergedPages.get(0));
                // move to next page
                currPageNo++;
                if (// more than one page as a result of merge - insert all additional ones
                mergedPages.size() > 1) {
                    mergedPages.remove(0);
                    pages.addAll(currPageNo, mergedPages);
                    // skip over the pages we just added as they cannot contain any work we might still need to do
                    currPageNo += mergedPages.size();
                    pageHits += mergedPages.size();
                }
            }
            iadd = addLimit;
            idel = delLimit;
            pageHits++;
        } else {
            currPageNo++;
        }
    }
    if (oldPageCount >= pages.size()) {
        // it grew before but not this time, assume it stopped growing for now and trim to size to save memory
        pages.trimToSize();
    }
}

Example 38 with Entry

use of org.janusgraph.diskstorage.Entry in project janusgraph by JanusGraph.

the class HBaseStoreManagerMutationTest method testMutationToPutsTTL.

@Test
public void testMutationToPutsTTL() throws Exception {
    final Map<String, Map<StaticBuffer, KCVMutation>> storeMutationMap = new HashMap<>();
    final Map<StaticBuffer, KCVMutation> rowkeyMutationMap = new HashMap<>();
    final List<Long> expectedColumnsWithTTL = new ArrayList<>();
    final List<Long> putColumnsWithTTL = new ArrayList<>();
    List<Entry> additions = new ArrayList<>();
    List<StaticBuffer> deletions = new ArrayList<>();
    StaticBuffer rowkey = KeyColumnValueStoreUtil.longToByteBuffer(0);
    StaticBuffer col = KeyColumnValueStoreUtil.longToByteBuffer(1);
    StaticBuffer val = KeyColumnValueStoreUtil.longToByteBuffer(2);
    StaticArrayEntry e = (StaticArrayEntry) StaticArrayEntry.of(col, val);
    // Test TTL with int max value / 1000 + 1
    // When convert this value from second to millisec will over Integer limit
    e.setMetaData(EntryMetaData.TTL, Integer.MAX_VALUE / 1000 + 1);
    Integer ttl = (Integer) e.getMetaData().get(EntryMetaData.TTL);
    // convert second to millisec with long format
    expectedColumnsWithTTL.add(TimeUnit.SECONDS.toMillis((long) ttl));
    additions.add(e);
    deletions.add(e);
    rowkeyMutationMap.put(rowkey, new KCVMutation(additions, deletions));
    storeMutationMap.put("store1", rowkeyMutationMap);
    HBaseStoreManager manager = new HBaseStoreManager(hBaseContainer.getModifiableConfiguration());
    final Map<StaticBuffer, Pair<List<Put>, Delete>> commandsPerRowKey = manager.convertToCommands(storeMutationMap, 0L, 0L);
    Pair<List<Put>, Delete> commands = commandsPerRowKey.values().iterator().next();
    // Verify Put TTL
    Put put = commands.getFirst().get(0);
    putColumnsWithTTL.add(put.getTTL());
    assertArrayEquals(expectedColumnsWithTTL.toArray(), putColumnsWithTTL.toArray());
}

Example 39 with Entry

use of org.janusgraph.diskstorage.Entry in project janusgraph by JanusGraph.

the class HBaseStoreManagerMutationTest method testKCVMutationToPuts.

@Test
public void testKCVMutationToPuts() throws Exception {
    final Map<String, Map<StaticBuffer, KCVMutation>> storeMutationMap = new HashMap<>();
    final Map<StaticBuffer, KCVMutation> rowkeyMutationMap = new HashMap<>();
    final List<Long> expectedColumnsWithTTL = new ArrayList<>();
    final List<Long> expectedColumnsWithoutTTL = new ArrayList<>();
    final List<Long> expectedColumnDelete = new ArrayList<>();
    StaticArrayEntry e = null;
    StaticBuffer rowkey, col, val;
    // 2 rows
    for (int row = 0; row < 2; row++) {
        rowkey = KeyColumnValueStoreUtil.longToByteBuffer(row);
        List<Entry> additions = new ArrayList<>();
        List<StaticBuffer> deletions = new ArrayList<>();
        // 100 columns each row
        int i;
        for (i = 0; i < 100; i++) {
            col = KeyColumnValueStoreUtil.longToByteBuffer(i);
            val = KeyColumnValueStoreUtil.longToByteBuffer(i + 100);
            e = (StaticArrayEntry) StaticArrayEntry.of(col, val);
            // Set half of the columns with TTL, also vary the TTL values
            if (i % 2 == 0) {
                e.setMetaData(EntryMetaData.TTL, i % 10 + 1);
                // Collect the columns with TTL. Only do this for one row
                if (row == 1) {
                    expectedColumnsWithTTL.add((long) i);
                }
            } else {
                // Collect the columns without TTL. Only do this for one row
                if (row == 1) {
                    expectedColumnsWithoutTTL.add((long) i);
                }
            }
            additions.add(e);
        }
        // Add one deletion to the row
        if (row == 1) {
            expectedColumnDelete.add((long) (i - 1));
        }
        deletions.add(e);
        rowkeyMutationMap.put(rowkey, new KCVMutation(additions, deletions));
    }
    storeMutationMap.put("store1", rowkeyMutationMap);
    HBaseStoreManager manager = new HBaseStoreManager(hBaseContainer.getModifiableConfiguration());
    final Map<StaticBuffer, Pair<List<Put>, Delete>> commandsPerRowKey = manager.convertToCommands(storeMutationMap, 0L, 0L);
    // 2 rows
    assertEquals(commandsPerRowKey.size(), 2);
    // Verify puts
    final List<Long> putColumnsWithTTL = new ArrayList<>();
    final List<Long> putColumnsWithoutTTL = new ArrayList<>();
    Pair<List<Put>, Delete> commands = commandsPerRowKey.values().iterator().next();
    long colName;
    for (Put p : commands.getFirst()) {
        // In Put, Long.MAX_VALUE means no TTL
        for (Map.Entry<byte[], List<Cell>> me : p.getFamilyCellMap().entrySet()) {
            for (Cell c : me.getValue()) {
                colName = KeyColumnValueStoreUtil.bufferToLong(new StaticArrayBuffer(CellUtil.cloneQualifier(c)));
                if (p.getTTL() < Long.MAX_VALUE) {
                    putColumnsWithTTL.add(colName);
                } else {
                    putColumnsWithoutTTL.add(colName);
                }
            }
        }
    }
    Collections.sort(putColumnsWithoutTTL);
    Collections.sort(putColumnsWithTTL);
    assertArrayEquals(expectedColumnsWithoutTTL.toArray(), putColumnsWithoutTTL.toArray());
    assertArrayEquals(expectedColumnsWithTTL.toArray(), putColumnsWithTTL.toArray());
    // Verify deletes
    final List<Long> deleteColumns = new ArrayList<>();
    Delete d = commands.getSecond();
    for (Map.Entry<byte[], List<Cell>> me : d.getFamilyCellMap().entrySet()) {
        for (Cell c : me.getValue()) {
            colName = KeyColumnValueStoreUtil.bufferToLong(new StaticArrayBuffer(CellUtil.cloneQualifier(c)));
            deleteColumns.add(colName);
        }
    }
    Collections.sort(deleteColumns);
    assertArrayEquals(expectedColumnDelete.toArray(), deleteColumns.toArray());
}

Example 40 with Entry

use of org.janusgraph.diskstorage.Entry in project janusgraph by JanusGraph.

the class InMemoryColumnValueStoreTest method testMultipageDelete.

@Test
public void testMultipageDelete() throws TemporaryLockingException {
    int numEntries = 1001;
    StoreTransaction txh = mock(StoreTransaction.class);
    BaseTransactionConfig mockConfig = mock(BaseTransactionConfig.class);
    when(txh.getConfiguration()).thenReturn(mockConfig);
    when(mockConfig.getCustomOption(eq(STORAGE_TRANSACTIONAL))).thenReturn(true);
    InMemoryColumnValueStore cvs = new InMemoryColumnValueStore();
    // ColumnValueStore cvs = new DeflatedEntryColumnValueStore(false);
    List<Entry> additions = generateEntries(0, numEntries, "orig");
    cvs.mutate(additions, Collections.emptyList(), txh);
    EntryList result = cvs.getSlice(new KeySliceQuery(makeStaticBuffer("someRow"), makeStaticBuffer(VERY_START), // if we pass COL_END, it doesn't get included
    makeStaticBuffer(VERY_END)), txh);
    assertEquals(additions.size(), result.size());
    int windowStart = 494;
    int windowEnd = 501;
    List<StaticBuffer> deletions = new ArrayList<>(windowEnd - windowStart);
    deletions.addAll(additions.subList(windowStart, windowEnd).stream().map(Entry::getColumn).collect(Collectors.toList()));
    cvs.mutate(Collections.emptyList(), deletions, txh);
    result = cvs.getSlice(new KeySliceQuery(makeStaticBuffer("someRow"), makeStaticBuffer(VERY_START), // if we pass COL_END, it doesn't get included
    makeStaticBuffer(VERY_END)), txh);
    assertEquals(additions.size() - deletions.size(), result.size());
}