Examples with StaticBuffer com.thinkaurelius.titan.diskstorage.StaticBuffer used on opensource projects

Example 1 with StaticBuffer

use of com.thinkaurelius.titan.diskstorage.StaticBuffer in project titan by thinkaurelius.

the class TitanGraphTest method simpleLogTest.

public void simpleLogTest(final boolean withLogFailure) throws InterruptedException {
    final String userlogName = "test";
    final Serializer serializer = graph.getDataSerializer();
    final EdgeSerializer edgeSerializer = graph.getEdgeSerializer();
    final TimestampProvider times = graph.getConfiguration().getTimestampProvider();
    final Instant startTime = times.getTime();
    clopen(option(SYSTEM_LOG_TRANSACTIONS), true, option(LOG_BACKEND, USER_LOG), (withLogFailure ? TestMockLog.class.getName() : LOG_BACKEND.getDefaultValue()), option(TestMockLog.LOG_MOCK_FAILADD, USER_LOG), withLogFailure, option(KCVSLog.LOG_READ_LAG_TIME, USER_LOG), Duration.ofMillis(50), option(LOG_READ_INTERVAL, USER_LOG), Duration.ofMillis(250), option(LOG_SEND_DELAY, USER_LOG), Duration.ofMillis(100), option(KCVSLog.LOG_READ_LAG_TIME, TRANSACTION_LOG), Duration.ofMillis(50), option(LOG_READ_INTERVAL, TRANSACTION_LOG), Duration.ofMillis(250), option(MAX_COMMIT_TIME), Duration.ofSeconds(1));
    final String instanceid = graph.getConfiguration().getUniqueGraphId();
    PropertyKey weight = tx.makePropertyKey("weight").dataType(Float.class).cardinality(Cardinality.SINGLE).make();
    EdgeLabel knows = tx.makeEdgeLabel("knows").make();
    TitanVertex n1 = tx.addVertex("weight", 10.5);
    newTx();
    final Instant[] txTimes = new Instant[4];
    //Transaction with custom userlog name
    txTimes[0] = times.getTime();
    TitanTransaction tx2 = graph.buildTransaction().logIdentifier(userlogName).start();
    TitanVertex v1 = tx2.addVertex("weight", 111.1);
    v1.addEdge("knows", v1);
    tx2.commit();
    final long v1id = getId(v1);
    txTimes[1] = times.getTime();
    tx2 = graph.buildTransaction().logIdentifier(userlogName).start();
    TitanVertex v2 = tx2.addVertex("weight", 222.2);
    v2.addEdge("knows", getV(tx2, v1id));
    tx2.commit();
    final long v2id = getId(v2);
    //Only read tx
    tx2 = graph.buildTransaction().logIdentifier(userlogName).start();
    v1 = getV(tx2, v1id);
    assertEquals(111.1, v1.<Float>value("weight").doubleValue(), 0.01);
    assertEquals(222.2, getV(tx2, v2).<Float>value("weight").doubleValue(), 0.01);
    tx2.commit();
    //Deleting transaction
    txTimes[2] = times.getTime();
    tx2 = graph.buildTransaction().logIdentifier(userlogName).start();
    v2 = getV(tx2, v2id);
    assertEquals(222.2, v2.<Float>value("weight").doubleValue(), 0.01);
    v2.remove();
    tx2.commit();
    //Edge modifying transaction
    txTimes[3] = times.getTime();
    tx2 = graph.buildTransaction().logIdentifier(userlogName).start();
    v1 = getV(tx2, v1id);
    assertEquals(111.1, v1.<Float>value("weight").doubleValue(), 0.01);
    Edge e = getOnlyElement(v1.query().direction(Direction.OUT).labels("knows").edges());
    assertFalse(e.property("weight").isPresent());
    e.property("weight", 44.4);
    tx2.commit();
    close();
    final Instant endTime = times.getTime();
    final ReadMarker startMarker = ReadMarker.fromTime(startTime);
    Log txlog = openTxLog();
    Log userLog = openUserLog(userlogName);
    final EnumMap<LogTxStatus, AtomicInteger> txMsgCounter = new EnumMap<LogTxStatus, AtomicInteger>(LogTxStatus.class);
    for (LogTxStatus status : LogTxStatus.values()) txMsgCounter.put(status, new AtomicInteger(0));
    final AtomicInteger userlogMeta = new AtomicInteger(0);
    txlog.registerReader(startMarker, new MessageReader() {

        @Override
        public void read(Message message) {
            Instant msgTime = message.getTimestamp();
            assertTrue(msgTime.isAfter(startTime) || msgTime.equals(startTime));
            assertNotNull(message.getSenderId());
            TransactionLogHeader.Entry txEntry = TransactionLogHeader.parse(message.getContent(), serializer, times);
            TransactionLogHeader header = txEntry.getHeader();
            //                System.out.println(header.getTimestamp(TimeUnit.MILLISECONDS));
            assertTrue(header.getTimestamp().isAfter(startTime) || header.getTimestamp().equals(startTime));
            assertTrue(header.getTimestamp().isBefore(msgTime) || header.getTimestamp().equals(msgTime));
            assertNotNull(txEntry.getMetadata());
            assertNull(txEntry.getMetadata().get(LogTxMeta.GROUPNAME));
            LogTxStatus status = txEntry.getStatus();
            if (status == LogTxStatus.PRECOMMIT) {
                assertTrue(txEntry.hasContent());
                Object logid = txEntry.getMetadata().get(LogTxMeta.LOG_ID);
                if (logid != null) {
                    assertTrue(logid instanceof String);
                    assertEquals(userlogName, logid);
                    userlogMeta.incrementAndGet();
                }
            } else if (withLogFailure) {
                assertTrue(status.isPrimarySuccess() || status == LogTxStatus.SECONDARY_FAILURE);
                if (status == LogTxStatus.SECONDARY_FAILURE) {
                    TransactionLogHeader.SecondaryFailures secFail = txEntry.getContentAsSecondaryFailures(serializer);
                    assertTrue(secFail.failedIndexes.isEmpty());
                    assertTrue(secFail.userLogFailure);
                }
            } else {
                assertFalse(txEntry.hasContent());
                assertTrue(status.isSuccess());
            }
            txMsgCounter.get(txEntry.getStatus()).incrementAndGet();
        }
    });
    final EnumMap<Change, AtomicInteger> userChangeCounter = new EnumMap<Change, AtomicInteger>(Change.class);
    for (Change change : Change.values()) userChangeCounter.put(change, new AtomicInteger(0));
    final AtomicInteger userLogMsgCounter = new AtomicInteger(0);
    userLog.registerReader(startMarker, new MessageReader() {

        @Override
        public void read(Message message) {
            Instant msgTime = message.getTimestamp();
            assertTrue(msgTime.isAfter(startTime) || msgTime.equals(startTime));
            assertNotNull(message.getSenderId());
            StaticBuffer content = message.getContent();
            assertTrue(content != null && content.length() > 0);
            TransactionLogHeader.Entry txentry = TransactionLogHeader.parse(content, serializer, times);
            Instant txTime = txentry.getHeader().getTimestamp();
            assertTrue(txTime.isBefore(msgTime) || txTime.equals(msgTime));
            assertTrue(txTime.isAfter(startTime) || txTime.equals(msgTime));
            long txid = txentry.getHeader().getId();
            assertTrue(txid > 0);
            for (TransactionLogHeader.Modification modification : txentry.getContentAsModifications(serializer)) {
                assertTrue(modification.state == Change.ADDED || modification.state == Change.REMOVED);
                userChangeCounter.get(modification.state).incrementAndGet();
            }
            userLogMsgCounter.incrementAndGet();
        }
    });
    Thread.sleep(4000);
    assertEquals(5, txMsgCounter.get(LogTxStatus.PRECOMMIT).get());
    assertEquals(4, txMsgCounter.get(LogTxStatus.PRIMARY_SUCCESS).get());
    assertEquals(1, txMsgCounter.get(LogTxStatus.COMPLETE_SUCCESS).get());
    assertEquals(4, userlogMeta.get());
    if (withLogFailure)
        assertEquals(4, txMsgCounter.get(LogTxStatus.SECONDARY_FAILURE).get());
    else
        assertEquals(4, txMsgCounter.get(LogTxStatus.SECONDARY_SUCCESS).get());
    //User-Log
    if (withLogFailure) {
        assertEquals(0, userLogMsgCounter.get());
    } else {
        assertEquals(4, userLogMsgCounter.get());
        assertEquals(7, userChangeCounter.get(Change.ADDED).get());
        assertEquals(4, userChangeCounter.get(Change.REMOVED).get());
    }
    clopen(option(VERBOSE_TX_RECOVERY), true);
    /*
        Transaction Recovery
         */
    TransactionRecovery recovery = TitanFactory.startTransactionRecovery(graph, startTime);
    /*
        Use user log processing framework
         */
    final AtomicInteger userLogCount = new AtomicInteger(0);
    LogProcessorFramework userlogs = TitanFactory.openTransactionLog(graph);
    userlogs.addLogProcessor(userlogName).setStartTime(startTime).setRetryAttempts(1).addProcessor(new ChangeProcessor() {

        @Override
        public void process(TitanTransaction tx, TransactionId txId, ChangeState changes) {
            assertEquals(instanceid, txId.getInstanceId());
            //Just some reasonable upper bound
            assertTrue(txId.getTransactionId() > 0 && txId.getTransactionId() < 100);
            final Instant txTime = txId.getTransactionTime();
            assertTrue(String.format("tx timestamp %s not between start %s and end time %s", txTime, startTime, endTime), //Times should be within a second
            (txTime.isAfter(startTime) || txTime.equals(startTime)) && (txTime.isBefore(endTime) || txTime.equals(endTime)));
            assertTrue(tx.containsRelationType("knows"));
            assertTrue(tx.containsRelationType("weight"));
            EdgeLabel knows = tx.getEdgeLabel("knows");
            PropertyKey weight = tx.getPropertyKey("weight");
            Instant txTimeMicro = txId.getTransactionTime();
            int txNo;
            if (txTimeMicro.isBefore(txTimes[1])) {
                txNo = 1;
                //v1 addition transaction
                assertEquals(1, Iterables.size(changes.getVertices(Change.ADDED)));
                assertEquals(0, Iterables.size(changes.getVertices(Change.REMOVED)));
                assertEquals(1, Iterables.size(changes.getVertices(Change.ANY)));
                assertEquals(2, Iterables.size(changes.getRelations(Change.ADDED)));
                assertEquals(1, Iterables.size(changes.getRelations(Change.ADDED, knows)));
                assertEquals(1, Iterables.size(changes.getRelations(Change.ADDED, weight)));
                assertEquals(2, Iterables.size(changes.getRelations(Change.ANY)));
                assertEquals(0, Iterables.size(changes.getRelations(Change.REMOVED)));
                TitanVertex v = Iterables.getOnlyElement(changes.getVertices(Change.ADDED));
                assertEquals(v1id, getId(v));
                VertexProperty<Float> p = Iterables.getOnlyElement(changes.getProperties(v, Change.ADDED, "weight"));
                assertEquals(111.1, p.value().doubleValue(), 0.01);
                assertEquals(1, Iterables.size(changes.getEdges(v, Change.ADDED, OUT)));
                assertEquals(1, Iterables.size(changes.getEdges(v, Change.ADDED, BOTH)));
            } else if (txTimeMicro.isBefore(txTimes[2])) {
                txNo = 2;
                //v2 addition transaction
                assertEquals(1, Iterables.size(changes.getVertices(Change.ADDED)));
                assertEquals(0, Iterables.size(changes.getVertices(Change.REMOVED)));
                assertEquals(2, Iterables.size(changes.getVertices(Change.ANY)));
                assertEquals(2, Iterables.size(changes.getRelations(Change.ADDED)));
                assertEquals(1, Iterables.size(changes.getRelations(Change.ADDED, knows)));
                assertEquals(1, Iterables.size(changes.getRelations(Change.ADDED, weight)));
                assertEquals(2, Iterables.size(changes.getRelations(Change.ANY)));
                assertEquals(0, Iterables.size(changes.getRelations(Change.REMOVED)));
                TitanVertex v = Iterables.getOnlyElement(changes.getVertices(Change.ADDED));
                assertEquals(v2id, getId(v));
                VertexProperty<Float> p = Iterables.getOnlyElement(changes.getProperties(v, Change.ADDED, "weight"));
                assertEquals(222.2, p.value().doubleValue(), 0.01);
                assertEquals(1, Iterables.size(changes.getEdges(v, Change.ADDED, OUT)));
                assertEquals(1, Iterables.size(changes.getEdges(v, Change.ADDED, BOTH)));
            } else if (txTimeMicro.isBefore(txTimes[3])) {
                txNo = 3;
                //v2 deletion transaction
                assertEquals(0, Iterables.size(changes.getVertices(Change.ADDED)));
                assertEquals(1, Iterables.size(changes.getVertices(Change.REMOVED)));
                assertEquals(2, Iterables.size(changes.getVertices(Change.ANY)));
                assertEquals(0, Iterables.size(changes.getRelations(Change.ADDED)));
                assertEquals(2, Iterables.size(changes.getRelations(Change.REMOVED)));
                assertEquals(1, Iterables.size(changes.getRelations(Change.REMOVED, knows)));
                assertEquals(1, Iterables.size(changes.getRelations(Change.REMOVED, weight)));
                assertEquals(2, Iterables.size(changes.getRelations(Change.ANY)));
                TitanVertex v = Iterables.getOnlyElement(changes.getVertices(Change.REMOVED));
                assertEquals(v2id, getId(v));
                VertexProperty<Float> p = Iterables.getOnlyElement(changes.getProperties(v, Change.REMOVED, "weight"));
                assertEquals(222.2, p.value().doubleValue(), 0.01);
                assertEquals(1, Iterables.size(changes.getEdges(v, Change.REMOVED, OUT)));
                assertEquals(0, Iterables.size(changes.getEdges(v, Change.ADDED, BOTH)));
            } else {
                txNo = 4;
                //v1 edge modification
                assertEquals(0, Iterables.size(changes.getVertices(Change.ADDED)));
                assertEquals(0, Iterables.size(changes.getVertices(Change.REMOVED)));
                assertEquals(1, Iterables.size(changes.getVertices(Change.ANY)));
                assertEquals(1, Iterables.size(changes.getRelations(Change.ADDED)));
                assertEquals(1, Iterables.size(changes.getRelations(Change.REMOVED)));
                assertEquals(1, Iterables.size(changes.getRelations(Change.REMOVED, knows)));
                assertEquals(2, Iterables.size(changes.getRelations(Change.ANY)));
                TitanVertex v = Iterables.getOnlyElement(changes.getVertices(Change.ANY));
                assertEquals(v1id, getId(v));
                TitanEdge e = Iterables.getOnlyElement(changes.getEdges(v, Change.REMOVED, Direction.OUT, "knows"));
                assertFalse(e.property("weight").isPresent());
                assertEquals(v, e.vertex(Direction.IN));
                e = Iterables.getOnlyElement(changes.getEdges(v, Change.ADDED, Direction.OUT, "knows"));
                assertEquals(44.4, e.<Float>value("weight").doubleValue(), 0.01);
                assertEquals(v, e.vertex(Direction.IN));
            }
            //See only current state of graph in transaction
            TitanVertex v1 = getV(tx, v1id);
            assertNotNull(v1);
            assertTrue(v1.isLoaded());
            if (txNo != 2) {
                //In the transaction that adds v2, v2 will be considered "loaded"
                assertMissing(tx, v2id);
            //                    assertTrue(txNo + " - " + v2, v2 == null || v2.isRemoved());
            }
            assertEquals(111.1, v1.<Float>value("weight").doubleValue(), 0.01);
            assertCount(1, v1.query().direction(Direction.OUT).edges());
            userLogCount.incrementAndGet();
        }
    }).build();
    //wait
    Thread.sleep(22000L);
    recovery.shutdown();
    long[] recoveryStats = ((StandardTransactionLogProcessor) recovery).getStatistics();
    if (withLogFailure) {
        assertEquals(1, recoveryStats[0]);
        assertEquals(4, recoveryStats[1]);
    } else {
        assertEquals(5, recoveryStats[0]);
        assertEquals(0, recoveryStats[1]);
    }
    userlogs.removeLogProcessor(userlogName);
    userlogs.shutdown();
    assertEquals(4, userLogCount.get());
}

Example 2 with StaticBuffer

use of com.thinkaurelius.titan.diskstorage.StaticBuffer in project titan by thinkaurelius.

the class KCVSCacheTest method testSmallCache.

@Test
public void testSmallCache() throws Exception {
    final int numKeys = 100, numCols = 10;
    final int repeats = 100, clearEvery = 20, numMulti = 10;
    //Assumed below
    assertTrue(numCols >= 10);
    loadStore(numKeys, numCols);
    //Repeatedly read from cache and clear in between
    int calls = 0;
    assertEquals(calls, store.getSliceCalls());
    for (int t = 0; t < repeats; t++) {
        if (t % clearEvery == 0) {
            cache.clearCache();
            calls += numKeys * 2 + 1;
        }
        CacheTransaction tx = getCacheTx();
        for (int i = 1; i <= numKeys; i++) {
            assertEquals(10, cache.getSlice(getQuery(i, 0, numCols + 1).setLimit(10), tx).size());
            assertEquals(3, cache.getSlice(getQuery(i, 2, 5), tx).size());
        }
        //Multi-query
        List<StaticBuffer> keys = new ArrayList<StaticBuffer>();
        for (int i = 10; i < 10 + numMulti; i++) keys.add(BufferUtil.getIntBuffer(i));
        Map<StaticBuffer, EntryList> result = cache.getSlice(keys, getQuery(4, 9), tx);
        assertEquals(keys.size(), result.size());
        for (StaticBuffer key : keys) assertTrue(result.containsKey(key));
        for (EntryList r : result.values()) {
            assertEquals(5, r.size());
        }
        tx.commit();
        assertEquals(calls, store.getSliceCalls());
    }
    store.resetCounter();
    //Check invalidation
    StaticBuffer key = BufferUtil.getIntBuffer(23);
    List<StaticBuffer> keys = new ArrayList<StaticBuffer>();
    keys.add(key);
    keys.add(BufferUtil.getIntBuffer(12));
    keys.add(BufferUtil.getIntBuffer(5));
    //Read
    CacheTransaction tx = getCacheTx();
    assertEquals(numCols, cache.getSlice(new KeySliceQuery(key, getQuery(0, numCols + 1)), tx).size());
    Map<StaticBuffer, EntryList> result = cache.getSlice(keys, getQuery(2, 8), tx);
    assertEquals(keys.size(), result.size());
    assertEquals(6, result.get(key).size());
    //Update
    List<Entry> dels = new ArrayList<Entry>(numCols / 2);
    for (int j = 1; j <= numCols; j = j + 2) dels.add(getEntry(j, j));
    cache.mutateEntries(key, KeyColumnValueStore.NO_ADDITIONS, dels, tx);
    tx.commit();
    assertEquals(2, store.getSliceCalls());
    //Ensure updates are correctly read
    tx = getCacheTx();
    assertEquals(numCols / 2, cache.getSlice(new KeySliceQuery(key, getQuery(0, numCols + 1)), tx).size());
    result = cache.getSlice(keys, getQuery(2, 8), tx);
    assertEquals(keys.size(), result.size());
    assertEquals(3, result.get(key).size());
    tx.commit();
    assertEquals(4, store.getSliceCalls());
}

Example 3 with StaticBuffer

use of com.thinkaurelius.titan.diskstorage.StaticBuffer in project titan by thinkaurelius.

the class BerkeleyJEKeyValueStore method getSlice.

@Override
public RecordIterator<KeyValueEntry> getSlice(KVQuery query, StoreTransaction txh) throws BackendException {
    log.trace("beginning db={}, op=getSlice, tx={}", name, txh);
    Transaction tx = getTransaction(txh);
    Cursor cursor = null;
    final StaticBuffer keyStart = query.getStart();
    final StaticBuffer keyEnd = query.getEnd();
    final KeySelector selector = query.getKeySelector();
    final List<KeyValueEntry> result = new ArrayList<KeyValueEntry>();
    try {
        DatabaseEntry foundKey = keyStart.as(ENTRY_FACTORY);
        DatabaseEntry foundData = new DatabaseEntry();
        cursor = db.openCursor(tx, null);
        OperationStatus status = cursor.getSearchKeyRange(foundKey, foundData, getLockMode(txh));
        //Iterate until given condition is satisfied or end of records
        while (status == OperationStatus.SUCCESS) {
            StaticBuffer key = getBuffer(foundKey);
            if (key.compareTo(keyEnd) >= 0)
                break;
            if (selector.include(key)) {
                result.add(new KeyValueEntry(key, getBuffer(foundData)));
            }
            if (selector.reachedLimit())
                break;
            status = cursor.getNext(foundKey, foundData, getLockMode(txh));
        }
        log.trace("db={}, op=getSlice, tx={}, resultcount={}", name, txh, result.size());
        return new RecordIterator<KeyValueEntry>() {

            private final Iterator<KeyValueEntry> entries = result.iterator();

            @Override
            public boolean hasNext() {
                return entries.hasNext();
            }

            @Override
            public KeyValueEntry next() {
                return entries.next();
            }

            @Override
            public void close() {
            }

            @Override
            public void remove() {
                throw new UnsupportedOperationException();
            }
        };
    } catch (Exception e) {
        throw new PermanentBackendException(e);
    } finally {
        try {
            if (cursor != null)
                cursor.close();
        } catch (Exception e) {
            throw new PermanentBackendException(e);
        }
    }
}

Example 4 with StaticBuffer

use of com.thinkaurelius.titan.diskstorage.StaticBuffer in project titan by thinkaurelius.

the class ExpirationCacheTest method verifyResults.

private void verifyResults(StaticBuffer key, List<StaticBuffer> keys, SliceQuery query, int expectedResults) throws Exception {
    CacheTransaction tx = getCacheTx();
    assertEquals(expectedResults, cache.getSlice(new KeySliceQuery(key, query), tx).size());
    Map<StaticBuffer, EntryList> results = cache.getSlice(keys, query, tx);
    assertEquals(keys.size(), results.size());
    assertEquals(expectedResults, results.get(key).size());
    tx.commit();
}

Example 5 with StaticBuffer

use of com.thinkaurelius.titan.diskstorage.StaticBuffer in project titan by thinkaurelius.

the class ExpirationCacheTest method testExpiration.

private void testExpiration(Duration expirationTime) throws Exception {
    final int numKeys = 100, numCols = 10;
    loadStore(numKeys, numCols);
    //Replace cache with proper times
    cache = getCache(store, expirationTime, Duration.ZERO);
    StaticBuffer key = BufferUtil.getIntBuffer(81);
    List<StaticBuffer> keys = new ArrayList<StaticBuffer>();
    keys.add(key);
    keys.add(BufferUtil.getIntBuffer(37));
    keys.add(BufferUtil.getIntBuffer(2));
    SliceQuery query = getQuery(2, 8);
    verifyResults(key, keys, query, 6);
    //Modify store directly
    StoreTransaction txs = getStoreTx();
    store.mutate(key, KeyColumnValueStore.NO_ADDITIONS, Lists.newArrayList(BufferUtil.getIntBuffer(5)), txs);
    txs.commit();
    Instant utime = times.getTime();
    //Should still see cached results
    verifyResults(key, keys, query, 6);
    //Sleep half way through expiration time
    times.sleepPast(utime.plus(expirationTime.dividedBy(2)));
    verifyResults(key, keys, query, 6);
    //Sleep past expiration time...
    times.sleepPast(utime.plus(expirationTime));
    //...and just a little bit longer
    times.sleepFor(Duration.ofMillis(5));
    //Now the results should be different
    verifyResults(key, keys, query, 5);
    //If we modify through cache store...
    CacheTransaction tx = getCacheTx();
    cache.mutateEntries(key, KeyColumnValueStore.NO_ADDITIONS, Lists.newArrayList(getEntry(4, 4)), tx);
    tx.commit();
    store.resetCounter();
    //...invalidation should happen and the result set is updated immediately
    verifyResults(key, keys, query, 4);
}