Examples with KeySliceQuery org.janusgraph.diskstorage.keycolumnvalue.KeySliceQuery used on opensource projects

Example 1 with KeySliceQuery

use of org.janusgraph.diskstorage.keycolumnvalue.KeySliceQuery in project janusgraph by JanusGraph.

the class InMemoryColumnValueStoreTest method testMultipageSlicing.

@Test
public void testMultipageSlicing() throws TemporaryLockingException {
    int numEntries = 502;
    int windowStart = 494;
    int windowEnd = 501;
    StoreTransaction txh = mock(StoreTransaction.class);
    BaseTransactionConfig mockConfig = mock(BaseTransactionConfig.class);
    when(txh.getConfiguration()).thenReturn(mockConfig);
    when(mockConfig.getCustomOption(eq(STORAGE_TRANSACTIONAL))).thenReturn(true);
    InMemoryColumnValueStore cvs = new InMemoryColumnValueStore();
    // ColumnValueStore cvs = new DeflatedEntryColumnValueStore(false);
    List<Entry> additions = generateEntries(0, numEntries, "orig");
    cvs.mutate(additions, Collections.emptyList(), txh);
    EntryList result = cvs.getSlice(new KeySliceQuery(makeStaticBuffer("someRow"), makeStaticBuffer(VERY_START), // if we pass COL_END, it doesn't get included
    makeStaticBuffer(VERY_END)), // if we pass COL_END, it doesn't get included
    txh);
    assertEquals(additions.size(), result.size());
    // this getSlice spans two pages and doesn't retrieve either page in full
    result = cvs.getSlice(new KeySliceQuery(makeStaticBuffer("someRow"), additions.get(windowStart).getColumn(), additions.get(windowEnd).getColumn()), txh);
    assertEquals(windowEnd - windowStart, result.size());
}

Example 2 with KeySliceQuery

use of org.janusgraph.diskstorage.keycolumnvalue.KeySliceQuery in project janusgraph by JanusGraph.

the class InMemoryColumnValueStoreTest method testAddInterleaved.

@Test
public void testAddInterleaved() throws TemporaryLockingException {
    int pageSize = InMemoryColumnValueStore.DEF_PAGE_SIZE;
    StoreTransaction txh = mock(StoreTransaction.class);
    BaseTransactionConfig mockConfig = mock(BaseTransactionConfig.class);
    when(txh.getConfiguration()).thenReturn(mockConfig);
    when(mockConfig.getCustomOption(eq(STORAGE_TRANSACTIONAL))).thenReturn(true);
    InMemoryColumnValueStore cvs = new InMemoryColumnValueStore();
    List<Entry> additions1 = generateEntries(0, pageSize * 2 + pageSize / 2, "orig");
    cvs.mutate(additions1, Collections.emptyList(), txh);
    List<Entry> additions2 = generateEntries(pageSize * 10, pageSize * 10 + pageSize * 2 + pageSize / 2, "orig");
    cvs.mutate(additions2, Collections.emptyList(), txh);
    List<Entry> additions3 = generateEntries(pageSize * 5, pageSize * 5 + pageSize * 2 + pageSize / 2, "orig");
    cvs.mutate(additions3, Collections.emptyList(), txh);
    EntryList result = cvs.getSlice(new KeySliceQuery(makeStaticBuffer("someRow"), makeStaticBuffer(VERY_START), // if we pass COL_END, it doesn't get included
    makeStaticBuffer(VERY_END)), txh);
    assertEquals(additions1.size() + additions2.size() + additions3.size(), result.size());
    for (int i = 0; i < result.size() - 1; i++) {
        assertTrue(result.get(i).compareTo(result.get(i + 1)) < 0);
    }
}

Example 3 with KeySliceQuery

use of org.janusgraph.diskstorage.keycolumnvalue.KeySliceQuery in project janusgraph by JanusGraph.

the class InMemoryColumnValueStoreTest method testRoundtrip.

@Test
public void testRoundtrip() {
    InMemoryColumnValueStore[] cvStores = new InMemoryColumnValueStore[] { new InMemoryColumnValueStore(), // go untested with default page size of 500 due to small sample data set
    new TestPagedBufferColumnValueStore(3) };
    // first, add some columns
    List<Entry> additions1 = Arrays.asList(makeEntry("02col2", "val2"), makeEntry("01col1", "val1"), makeEntry("03col3", "val3"), makeEntry(COL_END, "valEnd"), makeEntry(COL_START, "valStart"), makeEntry("04emptycol", ""));
    List<StaticBuffer> deletions1 = Collections.emptyList();
    StoreTransaction txh = mock(StoreTransaction.class);
    BaseTransactionConfig mockConfig = mock(BaseTransactionConfig.class);
    when(txh.getConfiguration()).thenReturn(mockConfig);
    when(mockConfig.getCustomOption(eq(STORAGE_TRANSACTIONAL))).thenReturn(true);
    for (InMemoryColumnValueStore cvs : cvStores) {
        cvs.mutate(additions1, deletions1, txh);
    }
    EntryList[] results = new EntryList[cvStores.length];
    // check the added/retrieved columns against originals
    for (int i = 0; i < cvStores.length; i++) {
        results[i] = cvStores[i].getSlice(new KeySliceQuery(makeStaticBuffer("someRow"), makeStaticBuffer(COL_START), makeStaticBuffer(VERY_END)), // if we pass COL_END, it doesn't get included
        txh);
        assertEquals(additions1.size(), results[i].size());
    }
    // we'll get the results back sorted, so we sort the initial entries as well, for ease of comparison
    additions1.sort(Comparator.naturalOrder());
    for (int j = 0; j < cvStores.length; j++) {
        for (int i = 0; i < results[j].size(); i++) {
            assertEquals(additions1.get(i), results[j].get(i));
            assertEquals(additions1.get(i), results[j].get(i));
            assertEquals(additions1.get(i).getColumn(), results[j].get(i).getColumn());
            assertEquals(additions1.get(i).getValue(), results[j].get(i).getValue());
        }
    }
    // now delete and update some columns
    List<Entry> additions2 = Arrays.asList(makeEntry("02col2", "val2_updated"), makeEntry("03col3", "val3_updated"));
    List<StaticBuffer> deletions2 = Arrays.asList(makeStaticBuffer("01col1"), makeStaticBuffer("02col2"));
    for (InMemoryColumnValueStore cvs : cvStores) {
        cvs.mutate(additions2, deletions2, txh);
    }
    for (int i = 0; i < cvStores.length; i++) {
        results[i] = cvStores[i].getSlice(new KeySliceQuery(makeStaticBuffer("someRow"), makeStaticBuffer(COL_START), makeStaticBuffer(VERY_END)), // if we pass COL_END, it doesn't get included
        txh);
        // col2 was in both deletes AND updates, JanusGraph logic is to honour update over delete, so we expect only col1 to be deleted
        assertEquals(additions1.size() - 1, results[i].size());
    }
    for (int i = 0; i < cvStores.length; i++) {
        final Map<StaticBuffer, StaticBuffer> seenValues = new HashMap<>(additions1.size() - 1);
        StaticArrayBuffer expectDeleted = makeStaticBuffer("01col1");
        results[i].forEach(entry -> seenValues.put(entry.getColumn(), entry.getValue()));
        assertFalse(seenValues.containsKey(expectDeleted));
        additions2.forEach(entry -> {
            assertTrue(seenValues.containsKey(entry.getColumn()));
            assertEquals(seenValues.get(entry.getColumn()), entry.getValue());
        });
    }
    for (int i = 0; i < cvStores.length; i++) {
        results[i] = cvStores[i].getSlice(new KeySliceQuery(makeStaticBuffer("someRow"), additions1.get(1).getColumn(), additions1.get(3).getColumn()), // if we pass COL_END, it doesn't get included
        txh);
        assertEquals(1, results[i].size());
    }
}

Example 4 with KeySliceQuery

use of org.janusgraph.diskstorage.keycolumnvalue.KeySliceQuery in project janusgraph by JanusGraph.

the class InMemoryStoreManagerTest method testStoreCycle.

@Test
public void testStoreCycle() throws Exception {
    InMemoryStoreManager imsm = new InMemoryStoreManager();
    KeyColumnValueStore kcvs1 = imsm.openDatabase("testStore1");
    KeyColumnValueStore kcvs2 = imsm.openDatabase("testStore2");
    assertEquals("testStore1", kcvs1.getName());
    assertEquals("testStore2", kcvs2.getName());
    StoreTransaction txh = imsm.beginTransaction(StandardBaseTransactionConfig.of(TimestampProviders.MICRO, imsm.getFeatures().getKeyConsistentTxConfig()));
    Map<String, Map<StaticBuffer, KCVMutation>> allMut = new HashMap<>();
    Map<StaticBuffer, KCVMutation> store1Mut = new HashMap<>();
    Map<StaticBuffer, KCVMutation> store2Mut = new HashMap<>();
    allMut.put("testStore1", store1Mut);
    allMut.put("testStore2", store2Mut);
    // first, add some columns
    List<Entry> additions1 = Arrays.asList(makeEntry("02col2", "val2"), makeEntry("01col1", "val1"), makeEntry("03col3", "val3"), makeEntry(InMemoryColumnValueStoreTest.COL_END, "valEnd"), makeEntry(InMemoryColumnValueStoreTest.COL_START, "valStart"), makeEntry("04emptycol", ""));
    KCVMutation kcvMut1 = new KCVMutation(additions1, Collections.EMPTY_LIST);
    store1Mut.put(makeStaticBuffer("row1"), kcvMut1);
    List<Entry> additions2 = Arrays.asList(makeEntry("01col1", "val1"), makeEntry("03col3", "val3"));
    KCVMutation kcvMut2 = new KCVMutation(additions2, Collections.EMPTY_LIST);
    store2Mut.put(makeStaticBuffer("row1"), kcvMut2);
    imsm.mutateMany(allMut, txh);
    EntryList result1 = kcvs1.getSlice(new KeySliceQuery(makeStaticBuffer("row1"), makeStaticBuffer(InMemoryColumnValueStoreTest.COL_START), makeStaticBuffer(InMemoryColumnValueStoreTest.VERY_END)), txh);
    Map<StaticBuffer, EntryList> result2 = kcvs2.getSlice(Arrays.asList(makeStaticBuffer("row1"), makeStaticBuffer("row2")), new KeySliceQuery(makeStaticBuffer("row1"), makeStaticBuffer(InMemoryColumnValueStoreTest.COL_START), makeStaticBuffer(InMemoryColumnValueStoreTest.VERY_END)), txh);
    assertEquals(2, result2.size());
    assertEquals(additions1.size(), result1.size());
    assertEquals(additions2.size(), result2.get(makeStaticBuffer("row1")).size());
    assertEquals(0, result2.get(makeStaticBuffer("row2")).size());
    for (boolean rollbackIfFailed : new boolean[] { true, false }) {
        File testSnapshotDir = new File(SystemUtils.JAVA_IO_TMPDIR, Long.toString(System.currentTimeMillis()));
        testSnapshotDir.deleteOnExit();
        try {
            imsm.makeSnapshot(testSnapshotDir, ForkJoinPool.commonPool());
            // to make the fact that the previous contents were cleared visible
            imsm.clearStorage();
            imsm.restoreFromSnapshot(testSnapshotDir, rollbackIfFailed, ForkJoinPool.commonPool());
            kcvs1 = imsm.openDatabase("testStore1");
            kcvs2 = imsm.openDatabase("testStore2");
            EntryList result1AfterRestore = kcvs1.getSlice(new KeySliceQuery(makeStaticBuffer("row1"), makeStaticBuffer(InMemoryColumnValueStoreTest.COL_START), makeStaticBuffer(InMemoryColumnValueStoreTest.VERY_END)), txh);
            Map<StaticBuffer, EntryList> result2AfterRestore = kcvs2.getSlice(Arrays.asList(makeStaticBuffer("row1"), makeStaticBuffer("row2")), new KeySliceQuery(makeStaticBuffer("row1"), makeStaticBuffer(InMemoryColumnValueStoreTest.COL_START), makeStaticBuffer(InMemoryColumnValueStoreTest.VERY_END)), txh);
            assertEquals(result1, result1AfterRestore);
            assertEquals(result2, result2AfterRestore);
        } finally {
            FileUtils.cleanDirectory(testSnapshotDir);
            testSnapshotDir.delete();
        }
    }
    imsm.close();
}

Example 5 with KeySliceQuery

use of org.janusgraph.diskstorage.keycolumnvalue.KeySliceQuery in project janusgraph by JanusGraph.

the class MultiPageEntryBufferTest method testBuffer.

@Test
public void testBuffer() throws Exception {
    SinglePageEntryBuffer singlePage = new SinglePageEntryBuffer();
    MultiPageEntryBuffer buffer = new MultiPageEntryBuffer(singlePage);
    int maxPageSize = 50;
    Entry[] empty = new Entry[0];
    Entry[] add1 = InMemoryColumnValueStoreTest.generateEntries(0, 123, "qq").toArray(empty);
    Entry[] add2 = InMemoryColumnValueStoreTest.generateEntries(113, 347, "qq").toArray(empty);
    Entry[] add3 = InMemoryColumnValueStoreTest.generateEntries(340, 497, "qq").toArray(empty);
    Entry[] del1 = new Entry[] {};
    Entry[] del2 = InMemoryColumnValueStoreTest.generateEntries(3, 43, "qq").toArray(empty);
    Entry[] del3 = InMemoryColumnValueStoreTest.generateEntries(55, 99, "qq").toArray(empty);
    Entry[] del4 = InMemoryColumnValueStoreTest.generateEntries(415, 446, "qq").toArray(empty);
    Entry[] del5 = InMemoryColumnValueStoreTest.generateEntries(453, 495, "qq").toArray(empty);
    buffer.mutate(add1, del1, maxPageSize);
    assertEquals(3, buffer.numPages());
    buffer.getSlice(new KeySliceQuery(makeStaticBuffer("someRow"), makeStaticBuffer(InMemoryColumnValueStoreTest.VERY_START), makeStaticBuffer(InMemoryColumnValueStoreTest.VERY_END)));
    buffer.mutate(add2, del1, maxPageSize);
    assertEquals(7, buffer.numPages());
    buffer.mutate(add3, del1, maxPageSize);
    assertEquals(10, buffer.numPages());
    buffer.mutate(new Entry[] {}, del2, maxPageSize);
    assertEquals(10, buffer.numPages());
    buffer.mutate(new Entry[] {}, del3, maxPageSize);
    assertEquals(10, buffer.numPages());
    buffer.mutate(new Entry[] {}, del4, maxPageSize);
    assertEquals(10, buffer.numPages());
    buffer.mutate(new Entry[] {}, del5, maxPageSize);
    assertEquals(10, buffer.numPages());
    buffer.isEmpty();
    buffer.isPaged();
    buffer.numEntries();
    buffer.createFragmentationReport(maxPageSize);
    buffer.quickDefragment(maxPageSize);
    assertEquals(8, buffer.numPages());
    buffer.createFragmentationReport(maxPageSize);
}