Search in sources :

Example 1 with RecordIterator

use of org.janusgraph.diskstorage.util.RecordIterator in project janusgraph by JanusGraph.

the class CQLResultSetKeyIteratorTest method testPartialIterateColumns.

@Test
public void testPartialIterateColumns() throws IOException {
    final Random random = new Random();
    final Function1<Integer, ByteBuffer> randomLong = idx -> {
        final ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES).putLong(random.nextLong());
        buffer.flip();
        return buffer;
    };
    final Array<Tuple2<ByteBuffer, Array<Tuple2<ByteBuffer, ByteBuffer>>>> keysMap = Array.range(0, random.nextInt(100) + 100).map(randomLong).map(key -> Tuple.of(key, Array.rangeClosed(0, random.nextInt(100) + 1).map(idx -> Tuple.of(randomLong.apply(idx), randomLong.apply(idx)))));
    final Seq<Row> rows = keysMap.flatMap(tuple -> tuple._2.map(columnAndValue -> {
        final Row row = mock(Row.class);
        when(row.getBytes("key")).thenReturn(tuple._1);
        when(row.getBytes("column1")).thenReturn(columnAndValue._1);
        when(row.getBytes("value")).thenReturn(columnAndValue._2);
        return row;
    }));
    final ResultSet resultSet = mock(ResultSet.class);
    when(resultSet.iterator()).thenReturn(rows.iterator());
    final CQLColValGetter getter = new CQLColValGetter(new EntryMetaData[0]);
    try (final CQLResultSetKeyIterator resultSetKeyIterator = new CQLResultSetKeyIterator(ALL_COLUMNS, getter, resultSet)) {
        final Iterator<Tuple2<ByteBuffer, Array<Tuple2<ByteBuffer, ByteBuffer>>>> iterator = keysMap.iterator();
        while (resultSetKeyIterator.hasNext()) {
            final StaticBuffer next = resultSetKeyIterator.next();
            try (final RecordIterator<Entry> entries = resultSetKeyIterator.getEntries()) {
                final Tuple2<ByteBuffer, Array<Tuple2<ByteBuffer, ByteBuffer>>> current = iterator.next();
                final ByteBuffer currentKey = current._1;
                final Array<Tuple2<ByteBuffer, ByteBuffer>> columnValues = current._2;
                final Iterator<Tuple2<ByteBuffer, ByteBuffer>> columnIterator = columnValues.iterator();
                while (entries.hasNext()) {
                    final Entry entry = entries.next();
                    final Tuple2<ByteBuffer, ByteBuffer> columnAndValue = columnIterator.next();
                    assertEquals(currentKey, next.asByteBuffer());
                    assertEquals(columnAndValue._1, entry.getColumn().asByteBuffer());
                    assertEquals(columnAndValue._2, entry.getValue().asByteBuffer());
                    assertEquals(columnIterator.hasNext(), entries.hasNext());
                    // 10% of the time, don't complete the iteration
                    if (random.nextInt(10) == 0) {
                        break;
                    }
                }
            }
        }
    }
}
Also used : Tuple(io.vavr.Tuple) EntryMetaData(org.janusgraph.diskstorage.EntryMetaData) Row(com.datastax.driver.core.Row) SliceQuery(org.janusgraph.diskstorage.keycolumnvalue.SliceQuery) Array(io.vavr.collection.Array) IOException(java.io.IOException) Random(java.util.Random) Test(org.junit.Test) Mockito.when(org.mockito.Mockito.when) Function1(io.vavr.Function1) ByteBuffer(java.nio.ByteBuffer) RecordIterator(org.janusgraph.diskstorage.util.RecordIterator) ResultSet(com.datastax.driver.core.ResultSet) Tuple2(io.vavr.Tuple2) Assert.assertFalse(org.junit.Assert.assertFalse) Entry(org.janusgraph.diskstorage.Entry) StaticBuffer(org.janusgraph.diskstorage.StaticBuffer) BufferUtil(org.janusgraph.diskstorage.util.BufferUtil) Iterator(io.vavr.collection.Iterator) Seq(io.vavr.collection.Seq) Assert.assertEquals(org.junit.Assert.assertEquals) Mockito.mock(org.mockito.Mockito.mock) ByteBuffer(java.nio.ByteBuffer) Array(io.vavr.collection.Array) Entry(org.janusgraph.diskstorage.Entry) Random(java.util.Random) Tuple2(io.vavr.Tuple2) ResultSet(com.datastax.driver.core.ResultSet) StaticBuffer(org.janusgraph.diskstorage.StaticBuffer) Row(com.datastax.driver.core.Row) Test(org.junit.Test)

Example 2 with RecordIterator

use of org.janusgraph.diskstorage.util.RecordIterator in project janusgraph by JanusGraph.

the class BerkeleyJEKeyValueStore method getSlice.

@Override
public RecordIterator<KeyValueEntry> getSlice(KVQuery query, StoreTransaction txh) throws BackendException {
    log.trace("beginning db={}, op=getSlice, tx={}", name, txh);
    final Transaction tx = getTransaction(txh);
    final StaticBuffer keyStart = query.getStart();
    final StaticBuffer keyEnd = query.getEnd();
    final KeySelector selector = query.getKeySelector();
    final List<KeyValueEntry> result = new ArrayList<>();
    final DatabaseEntry foundKey = keyStart.as(ENTRY_FACTORY);
    final DatabaseEntry foundData = new DatabaseEntry();
    try (final Cursor cursor = db.openCursor(tx, null)) {
        OperationStatus status = cursor.getSearchKeyRange(foundKey, foundData, getLockMode(txh));
        // Iterate until given condition is satisfied or end of records
        while (status == OperationStatus.SUCCESS) {
            StaticBuffer key = getBuffer(foundKey);
            if (key.compareTo(keyEnd) >= 0)
                break;
            if (selector.include(key)) {
                result.add(new KeyValueEntry(key, getBuffer(foundData)));
            }
            if (selector.reachedLimit())
                break;
            status = cursor.getNext(foundKey, foundData, getLockMode(txh));
        }
    } catch (Exception e) {
        throw new PermanentBackendException(e);
    }
    log.trace("db={}, op=getSlice, tx={}, resultcount={}", name, txh, result.size());
    return new RecordIterator<KeyValueEntry>() {

        private final Iterator<KeyValueEntry> entries = result.iterator();

        @Override
        public boolean hasNext() {
            return entries.hasNext();
        }

        @Override
        public KeyValueEntry next() {
            return entries.next();
        }

        @Override
        public void close() {
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }
    };
}
Also used : RecordIterator(org.janusgraph.diskstorage.util.RecordIterator) PermanentBackendException(org.janusgraph.diskstorage.PermanentBackendException) ArrayList(java.util.ArrayList) KeySelector(org.janusgraph.diskstorage.keycolumnvalue.keyvalue.KeySelector) BackendException(org.janusgraph.diskstorage.BackendException) PermanentBackendException(org.janusgraph.diskstorage.PermanentBackendException) StoreTransaction(org.janusgraph.diskstorage.keycolumnvalue.StoreTransaction) Iterator(java.util.Iterator) RecordIterator(org.janusgraph.diskstorage.util.RecordIterator) StaticBuffer(org.janusgraph.diskstorage.StaticBuffer) KeyValueEntry(org.janusgraph.diskstorage.keycolumnvalue.keyvalue.KeyValueEntry)

Example 3 with RecordIterator

use of org.janusgraph.diskstorage.util.RecordIterator in project janusgraph by JanusGraph.

the class VertexIterable method iterator.

@Override
public Iterator<InternalVertex> iterator() {
    return new Iterator<InternalVertex>() {

        final RecordIterator<Long> iterator = graph.getVertexIDs(tx.getTxHandle());

        InternalVertex nextVertex = nextVertex();

        private InternalVertex nextVertex() {
            InternalVertex v = null;
            while (v == null && iterator.hasNext()) {
                final long nextId = iterator.next();
                // Filter out invisible vertices
                if (IDManager.VertexIDType.Invisible.is(nextId))
                    continue;
                v = tx.getInternalVertex(nextId);
                // Filter out deleted vertices and types
                if (v.isRemoved())
                    v = null;
            }
            return v;
        }

        @Override
        public boolean hasNext() {
            return nextVertex != null;
        }

        @Override
        public InternalVertex next() {
            if (!hasNext())
                throw new NoSuchElementException();
            final InternalVertex returnVertex = nextVertex;
            nextVertex = nextVertex();
            return returnVertex;
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }
    };
}
Also used : RecordIterator(org.janusgraph.diskstorage.util.RecordIterator) Iterator(java.util.Iterator) RecordIterator(org.janusgraph.diskstorage.util.RecordIterator) InternalVertex(org.janusgraph.graphdb.internal.InternalVertex) NoSuchElementException(java.util.NoSuchElementException)

Example 4 with RecordIterator

use of org.janusgraph.diskstorage.util.RecordIterator in project janusgraph by JanusGraph.

the class CQLResultSetKeyIteratorTest method testUneven.

@Test
public void testUneven() throws IOException {
    final Random random = new Random();
    final Function1<Integer, ByteBuffer> randomLong = idx -> {
        final ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES).putLong(random.nextLong());
        buffer.flip();
        return buffer;
    };
    final Array<Tuple2<ByteBuffer, Array<Tuple2<ByteBuffer, ByteBuffer>>>> keysMap = Array.range(0, random.nextInt(100) + 100).map(randomLong).map(key -> Tuple.of(key, Array.rangeClosed(0, random.nextInt(100) + 1).map(idx -> Tuple.of(randomLong.apply(idx), randomLong.apply(idx)))));
    final Seq<Row> rows = keysMap.flatMap(tuple -> tuple._2.map(columnAndValue -> {
        final Row row = mock(Row.class);
        when(row.getBytes("key")).thenReturn(tuple._1);
        when(row.getBytes("column1")).thenReturn(columnAndValue._1);
        when(row.getBytes("value")).thenReturn(columnAndValue._2);
        return row;
    }));
    final ResultSet resultSet = mock(ResultSet.class);
    when(resultSet.iterator()).thenReturn(rows.iterator());
    final CQLColValGetter getter = new CQLColValGetter(new EntryMetaData[0]);
    try (final CQLResultSetKeyIterator resultSetKeyIterator = new CQLResultSetKeyIterator(ALL_COLUMNS, getter, resultSet)) {
        final Iterator<Tuple2<ByteBuffer, Array<Tuple2<ByteBuffer, ByteBuffer>>>> iterator = keysMap.iterator();
        while (resultSetKeyIterator.hasNext()) {
            final StaticBuffer next = resultSetKeyIterator.next();
            try (final RecordIterator<Entry> entries = resultSetKeyIterator.getEntries()) {
                final Tuple2<ByteBuffer, Array<Tuple2<ByteBuffer, ByteBuffer>>> current = iterator.next();
                final ByteBuffer currentKey = current._1;
                final Array<Tuple2<ByteBuffer, ByteBuffer>> columnValues = current._2;
                final Iterator<Tuple2<ByteBuffer, ByteBuffer>> columnIterator = columnValues.iterator();
                while (entries.hasNext()) {
                    final Entry entry = entries.next();
                    final Tuple2<ByteBuffer, ByteBuffer> columnAndValue = columnIterator.next();
                    assertEquals(currentKey, next.asByteBuffer());
                    assertEquals(columnAndValue._1, entry.getColumn().asByteBuffer());
                    assertEquals(columnAndValue._2, entry.getValue().asByteBuffer());
                    assertEquals(columnIterator.hasNext(), entries.hasNext());
                }
            }
        }
    }
}
Also used : Tuple(io.vavr.Tuple) EntryMetaData(org.janusgraph.diskstorage.EntryMetaData) Row(com.datastax.driver.core.Row) SliceQuery(org.janusgraph.diskstorage.keycolumnvalue.SliceQuery) Array(io.vavr.collection.Array) IOException(java.io.IOException) Random(java.util.Random) Test(org.junit.Test) Mockito.when(org.mockito.Mockito.when) Function1(io.vavr.Function1) ByteBuffer(java.nio.ByteBuffer) RecordIterator(org.janusgraph.diskstorage.util.RecordIterator) ResultSet(com.datastax.driver.core.ResultSet) Tuple2(io.vavr.Tuple2) Assert.assertFalse(org.junit.Assert.assertFalse) Entry(org.janusgraph.diskstorage.Entry) StaticBuffer(org.janusgraph.diskstorage.StaticBuffer) BufferUtil(org.janusgraph.diskstorage.util.BufferUtil) Iterator(io.vavr.collection.Iterator) Seq(io.vavr.collection.Seq) Assert.assertEquals(org.junit.Assert.assertEquals) Mockito.mock(org.mockito.Mockito.mock) ByteBuffer(java.nio.ByteBuffer) Array(io.vavr.collection.Array) Entry(org.janusgraph.diskstorage.Entry) Random(java.util.Random) Tuple2(io.vavr.Tuple2) ResultSet(com.datastax.driver.core.ResultSet) StaticBuffer(org.janusgraph.diskstorage.StaticBuffer) Row(com.datastax.driver.core.Row) Test(org.junit.Test)

Aggregations

RecordIterator (org.janusgraph.diskstorage.util.RecordIterator)4 StaticBuffer (org.janusgraph.diskstorage.StaticBuffer)3 ResultSet (com.datastax.driver.core.ResultSet)2 Row (com.datastax.driver.core.Row)2 Function1 (io.vavr.Function1)2 Tuple (io.vavr.Tuple)2 Tuple2 (io.vavr.Tuple2)2 Array (io.vavr.collection.Array)2 Iterator (io.vavr.collection.Iterator)2 Seq (io.vavr.collection.Seq)2 IOException (java.io.IOException)2 ByteBuffer (java.nio.ByteBuffer)2 Iterator (java.util.Iterator)2 Random (java.util.Random)2 Entry (org.janusgraph.diskstorage.Entry)2 EntryMetaData (org.janusgraph.diskstorage.EntryMetaData)2 SliceQuery (org.janusgraph.diskstorage.keycolumnvalue.SliceQuery)2 BufferUtil (org.janusgraph.diskstorage.util.BufferUtil)2 Assert.assertEquals (org.junit.Assert.assertEquals)2 Assert.assertFalse (org.junit.Assert.assertFalse)2