Examples with ReadBuffer org.janusgraph.diskstorage.ReadBuffer used on opensource projects

Example 11 with ReadBuffer

use of org.janusgraph.diskstorage.ReadBuffer in project janusgraph by JanusGraph.

the class TransactionLogHeader method parse.

public static Entry parse(StaticBuffer buffer, Serializer serializer, TimestampProvider times) {
    ReadBuffer read = buffer.asReadBuffer();
    Instant txTimestamp = times.getTime(read.getLong());
    TransactionLogHeader header = new TransactionLogHeader(VariableLong.readPositive(read), txTimestamp, times);
    LogTxStatus status = serializer.readObjectNotNull(read, LogTxStatus.class);
    final EnumMap<LogTxMeta, Object> metadata = new EnumMap<>(LogTxMeta.class);
    int metaSize = VariableLong.unsignedByte(read.getByte());
    for (int i = 0; i < metaSize; i++) {
        LogTxMeta meta = LogTxMeta.values()[VariableLong.unsignedByte(read.getByte())];
        metadata.put(meta, serializer.readObjectNotNull(read, meta.dataType()));
    }
    if (read.hasRemaining()) {
        StaticBuffer content = read.subrange(read.getPosition(), read.length() - read.getPosition());
        return new Entry(header, content, status, metadata);
    } else {
        return new Entry(header, null, status, metadata);
    }
}

Example 12 with ReadBuffer

use of org.janusgraph.diskstorage.ReadBuffer in project janusgraph by JanusGraph.

the class IDManagementTest method edgeTypeIDTest.

@Test
public void edgeTypeIDTest() {
    int partitionBits = 16;
    IDManager eid = new IDManager(partitionBits);
    int trails = 1000000;
    assertEquals(eid.getPartitionBound(), (1L << partitionBits));
    Serializer serializer = new StandardSerializer();
    for (int t = 0; t < trails; t++) {
        long count = RandomGenerator.randomLong(1, IDManager.getSchemaCountBound());
        long id;
        IDHandler.DirectionID dirID;
        RelationCategory type;
        if (Math.random() < 0.5) {
            id = IDManager.getSchemaId(IDManager.VertexIDType.UserEdgeLabel, count);
            assertTrue(eid.isEdgeLabelId(id));
            assertFalse(IDManager.isSystemRelationTypeId(id));
            type = RelationCategory.EDGE;
            if (Math.random() < 0.5)
                dirID = IDHandler.DirectionID.EDGE_IN_DIR;
            else
                dirID = IDHandler.DirectionID.EDGE_OUT_DIR;
        } else {
            type = RelationCategory.PROPERTY;
            id = IDManager.getSchemaId(IDManager.VertexIDType.UserPropertyKey, count);
            assertTrue(eid.isPropertyKeyId(id));
            assertFalse(IDManager.isSystemRelationTypeId(id));
            dirID = IDHandler.DirectionID.PROPERTY_DIR;
        }
        assertTrue(eid.isRelationTypeId(id));
        StaticBuffer b = IDHandler.getRelationType(id, dirID, false);
        // System.out.println(dirID);
        // System.out.println(getBinary(id));
        // System.out.println(getBuffer(b.asReadBuffer()));
        ReadBuffer rb = b.asReadBuffer();
        IDHandler.RelationTypeParse parse = IDHandler.readRelationType(rb);
        assertEquals(id, parse.typeId);
        assertEquals(dirID, parse.dirID);
        assertFalse(rb.hasRemaining());
        // Inline edge type
        WriteBuffer wb = new WriteByteBuffer(9);
        IDHandler.writeInlineRelationType(wb, id);
        long newId = IDHandler.readInlineRelationType(wb.getStaticBuffer().asReadBuffer());
        assertEquals(id, newId);
        // Compare to Kryo
        DataOutput out = serializer.getDataOutput(10);
        IDHandler.writeRelationType(out, id, dirID, false);
        assertEquals(b, out.getStaticBuffer());
        // Make sure the bounds are right
        StaticBuffer[] bounds = IDHandler.getBounds(type, false);
        assertTrue(bounds[0].compareTo(b) < 0);
        assertTrue(bounds[1].compareTo(b) > 0);
        bounds = IDHandler.getBounds(RelationCategory.RELATION, false);
        assertTrue(bounds[0].compareTo(b) < 0);
        assertTrue(bounds[1].compareTo(b) > 0);
    }
}

Example 13 with ReadBuffer

use of org.janusgraph.diskstorage.ReadBuffer in project janusgraph by JanusGraph.

the class IDManagementTest method writingInlineEdgeTypes.

@Test
public void writingInlineEdgeTypes() {
    int numTries = 100;
    WriteBuffer out = new WriteByteBuffer(8 * numTries);
    for (SystemRelationType t : SYSTEM_TYPES) {
        IDHandler.writeInlineRelationType(out, t.longId());
    }
    for (long i = 1; i <= numTries; i++) {
        IDHandler.writeInlineRelationType(out, IDManager.getSchemaId(IDManager.VertexIDType.UserEdgeLabel, i * 1000));
    }
    ReadBuffer in = out.getStaticBuffer().asReadBuffer();
    for (SystemRelationType t : SYSTEM_TYPES) {
        assertEquals(t, SystemTypeManager.getSystemType(IDHandler.readInlineRelationType(in)));
    }
    for (long i = 1; i <= numTries; i++) {
        assertEquals(i * 1000, IDManager.stripEntireRelationTypePadding(IDHandler.readInlineRelationType(in)));
    }
}

Example 14 with ReadBuffer

use of org.janusgraph.diskstorage.ReadBuffer in project janusgraph by JanusGraph.

the class VariableLongTest method readWriteTest.

private void readWriteTest(final ReadWriteLong impl, long maxValue, long jump, boolean negative, boolean backward) {
    Preconditions.checkArgument(maxValue % jump == 0);
    long allocate = maxValue / jump * 8 * (negative ? 2 : 1);
    Preconditions.checkArgument(allocate < (1 << 28));
    WriteBuffer wb = new WriteByteBuffer((int) allocate);
    int num = 0;
    StopWatch w = new StopWatch();
    w.start();
    for (long i = (negative ? -maxValue : 0); i <= maxValue; i += jump) {
        impl.write(wb, i);
        num++;
    }
    // for (int i=0;i<b.remaining();i++) System.out.print(b.get(i)+"|");
    w.stop();
    ReadBuffer rb = wb.getStaticBuffer().asReadBuffer();
    log.info("Writing " + num + " longs in " + rb.length() + " bytes. in time: " + w.getTime());
    final ReadVerify read = (rb1, expected) -> {
        int beforePos = rb1.getPosition();
        long value = impl.read(rb1);
        assertEquals(expected, value);
        int length = Math.abs(rb1.getPosition() - beforePos);
        assertEquals("On: " + expected, length, impl.length(expected));
    };
    if (backward) {
        rb.movePositionTo(rb.length());
        for (long i = maxValue; i != (negative ? -maxValue : 0); i -= jump) {
            read.next(rb, i);
        }
    } else {
        for (long i = (negative ? -maxValue : 0); i <= maxValue; i += jump) {
            read.next(rb, i);
        }
    }
    // Test boundaries
    wb = new WriteByteBuffer(512);
    impl.write(wb, 0);
    impl.write(wb, Long.MAX_VALUE);
    if (negative)
        impl.write(wb, -Long.MAX_VALUE);
    rb = wb.getStaticBuffer().asReadBuffer();
    if (backward) {
        rb.movePositionTo(rb.length());
        if (negative)
            assertEquals(-Long.MAX_VALUE, impl.read(rb));
        assertEquals(Long.MAX_VALUE, impl.read(rb));
        assertEquals(0, impl.read(rb));
    } else {
        assertEquals(0, impl.read(rb));
        assertEquals(Long.MAX_VALUE, impl.read(rb));
        if (negative)
            assertEquals(-Long.MAX_VALUE, impl.read(rb));
    }
}

Example 15 with ReadBuffer

use of org.janusgraph.diskstorage.ReadBuffer in project janusgraph by JanusGraph.

the class SerializerTest method testStringCompression.

@Test
public void testStringCompression() {
    // ASCII encoding
    for (int t = 0; t < 100; t++) {
        String x = getRandomString(StringSerializer.TEXT_COMPRESSION_THRESHOLD - 1, ASCII_VALUE);
        assertEquals(x.length() + 1, getStringBuffer(x).length());
    }
    // SMAZ Encoding
    // String[] texts = {
    // "To Sherlock Holmes she is always the woman. I have seldom heard him mention her under any other name. In his eyes she eclipses and predominates the whole of her sex.",
    // "His manner was not effusive. It seldom was; but he was glad, I think, to see me. With hardly a word spoken, but with a kindly eye, he waved me to an armchair",
    // "I could not help laughing at the ease with which he explained his process of deduction.",
    // "A man entered who could hardly have been less than six feet six inches in height, with the chest and limbs of a Hercules. His dress was rich with a richness which would, in England"
    // };
    // for (String text : texts) {
    // assertTrue(text.length()> StringSerializer.TEXT_COMPRESSION_THRESHOLD);
    // StaticBuffer s = getStringBuffer(text);
    // //            System.out.println(String.format("String length [%s] -> byte size [%s]",text.length(),s.length()));
    // assertTrue(text.length()>s.length()); //Test that actual compression is happening
    // }
    // Gzip Encoding
    String[] patterns = { "aQd>@!as/df5h", "sdfodoiwk", "sdf", "ab", "asdfwewefefwdfkajhqwkdhj" };
    int targetLength = StringSerializer.LONG_COMPRESSION_THRESHOLD * 5;
    for (String pattern : patterns) {
        StringBuilder sb = new StringBuilder(targetLength);
        for (int i = 0; i < targetLength / pattern.length(); i++) sb.append(pattern);
        String text = sb.toString();
        assertTrue(text.length() > StringSerializer.LONG_COMPRESSION_THRESHOLD);
        StaticBuffer s = getStringBuffer(text);
        // System.out.println(String.format("String length [%s] -> byte size [%s]",text.length(),s.length()));
        // Test that radical compression is happening
        assertTrue(text.length() > s.length() * 10);
    }
    for (int t = 0; t < 10000; t++) {
        String x = STRING_FACTORY.newInstance();
        DataOutput o = serialize.getDataOutput(64);
        o.writeObject(x, String.class);
        ReadBuffer r = o.getStaticBuffer().asReadBuffer();
        String y = serialize.readObject(r, String.class);
        assertEquals(x, y);
    }
}