Examples with StaticBuffer com.thinkaurelius.titan.diskstorage.StaticBuffer used on opensource projects

Example 56 with StaticBuffer

use of com.thinkaurelius.titan.diskstorage.StaticBuffer in project titan by thinkaurelius.

the class HadoopScanMapper method finishSetup.

protected void finishSetup(ModifiableHadoopConfiguration scanConf, Configuration graphConf) {
    jobConf = getJobConfiguration(scanConf);
    Preconditions.checkNotNull(metrics);
    // Allowed to be null for jobs that specify no configuration and no configuration root
    //Preconditions.checkNotNull(jobConf);
    Preconditions.checkNotNull(job);
    job.workerIterationStart(jobConf, graphConf, metrics);
    keyFilter = job.getKeyFilter();
    List<SliceQuery> sliceQueries = job.getQueries();
    Preconditions.checkArgument(null != sliceQueries, "Job cannot specify null query list");
    Preconditions.checkArgument(0 < sliceQueries.size(), "Job must specify at least one query");
    // Assign head of getQueries() to "initialQuery"
    initialQuery = sliceQueries.get(0);
    // Assign tail of getQueries() to "subsequentQueries"
    subsequentQueries = new ArrayList<>(sliceQueries.subList(1, sliceQueries.size()));
    Preconditions.checkState(sliceQueries.size() == subsequentQueries.size() + 1);
    Preconditions.checkNotNull(initialQuery);
    if (0 < subsequentQueries.size()) {
        //It is assumed that the first query is the grounding query if multiple queries exist
        StaticBuffer start = initialQuery.getSliceStart();
        Preconditions.checkArgument(start.equals(BufferUtil.zeroBuffer(1)), "Expected start of first query to be all 0s: %s", start);
        StaticBuffer end = initialQuery.getSliceEnd();
        Preconditions.checkArgument(end.equals(BufferUtil.oneBuffer(end.length())), "Expected end of first query to be all 1s: %s", end);
    }
}

Example 57 with StaticBuffer

use of com.thinkaurelius.titan.diskstorage.StaticBuffer in project titan by thinkaurelius.

the class HadoopScanMapper method findEntriesMatchingQuery.

private EntryList findEntriesMatchingQuery(SliceQuery query, EntryList sortedEntries) {
    // Inclusive
    int lowestStartMatch = sortedEntries.size();
    // Inclusive
    int highestEndMatch = -1;
    final StaticBuffer queryStart = query.getSliceStart();
    final StaticBuffer queryEnd = query.getSliceEnd();
    // Find the lowest matchStart s.t. query.getSliceStart <= sortedEntries.get(matchStart)
    int low = 0;
    int high = sortedEntries.size() - 1;
    while (low <= high) {
        int mid = (low + high) >>> 1;
        Entry midVal = sortedEntries.get(mid);
        int cmpStart = queryStart.compareTo(midVal.getColumn());
        if (0 < cmpStart) {
            // query lower bound exceeds entry (no match)
            if (lowestStartMatch == mid + 1) {
                // lowestStartMatch located
                break;
            }
            // Move to higher list index
            low = mid + 1;
        } else /* (0 >= cmpStart) */
        {
            // entry equals or exceeds query lower bound (match, but not necessarily lowest match)
            if (mid < lowestStartMatch) {
                lowestStartMatch = mid;
            }
            // Move to a lower list index
            high = mid - 1;
        }
    }
    // so we can bypass the highestEndMatch search and just return an empty result.
    if (sortedEntries.size() == lowestStartMatch) {
        return EntryList.EMPTY_LIST;
    }
    // Find the highest matchEnd s.t. sortedEntries.get(matchEnd) < query.getSliceEnd
    low = 0;
    high = sortedEntries.size() - 1;
    while (low <= high) {
        int mid = (low + high) >>> 1;
        Entry midVal = sortedEntries.get(mid);
        int cmpEnd = queryEnd.compareTo(midVal.getColumn());
        if (0 < cmpEnd) {
            // query upper bound exceeds entry (match, not necessarily highest)
            if (mid > highestEndMatch) {
                highestEndMatch = mid;
            }
            // Move to higher list index
            low = mid + 1;
        } else /* (0 >= cmpEnd) */
        {
            // entry equals or exceeds query upper bound (no match)
            if (highestEndMatch == mid - 1) {
                // highestEndMatch located
                break;
            }
            // Move to a lower list index
            high = mid - 1;
        }
    }
    if (0 <= highestEndMatch - lowestStartMatch) {
        // Return sublist between indices (inclusive at both indices)
        // This will be passed into subList, which interprets it exclusively
        int endIndex = highestEndMatch + 1;
        if (query.hasLimit()) {
            endIndex = Math.min(endIndex, query.getLimit() + lowestStartMatch);
        }
        // TODO avoid unnecessary copy here
        return EntryArrayList.of(sortedEntries.subList(lowestStartMatch, endIndex));
    } else {
        return EntryList.EMPTY_LIST;
    }
}

Example 58 with StaticBuffer

use of com.thinkaurelius.titan.diskstorage.StaticBuffer in project titan by thinkaurelius.

the class LogTest method testFuzzMessagesSerial.

@Test
public void testFuzzMessagesSerial() throws Exception {
    final int maxLen = 1024 * 4;
    final int rounds = 32;
    StoringReader reader = new StoringReader(rounds);
    List<StaticBuffer> expected = new ArrayList<StaticBuffer>(rounds);
    Log l = manager.openLog("fuzz");
    l.registerReader(ReadMarker.fromNow(), reader);
    Random rand = new Random();
    for (int i = 0; i < rounds; i++) {
        //int len = rand.nextInt(maxLen + 1);
        int len = maxLen;
        if (0 == len)
            // 0 would throw IllegalArgumentException
            len = 1;
        byte[] raw = new byte[len];
        rand.nextBytes(raw);
        StaticBuffer sb = StaticArrayBuffer.of(raw);
        l.add(sb);
        expected.add(sb);
        Thread.sleep(50L);
    }
    reader.await(TIMEOUT_MS);
    assertEquals(rounds, reader.msgCount);
    assertEquals(expected, reader.msgs);
}

Example 59 with StaticBuffer

use of com.thinkaurelius.titan.diskstorage.StaticBuffer in project titan by thinkaurelius.

the class EdgeSerializer method getQuery.

public SliceQuery getQuery(InternalRelationType type, Direction dir, TypedInterval[] sortKey) {
    Preconditions.checkNotNull(type);
    Preconditions.checkNotNull(dir);
    Preconditions.checkArgument(type.isUnidirected(Direction.BOTH) || type.isUnidirected(dir));
    StaticBuffer sliceStart = null, sliceEnd = null;
    RelationCategory rt = type.isPropertyKey() ? RelationCategory.PROPERTY : RelationCategory.EDGE;
    if (dir == Direction.BOTH) {
        assert type.isEdgeLabel();
        sliceStart = IDHandler.getRelationType(type.longId(), getDirID(Direction.OUT, rt), type.isInvisibleType());
        sliceEnd = IDHandler.getRelationType(type.longId(), getDirID(Direction.IN, rt), type.isInvisibleType());
        assert sliceStart.compareTo(sliceEnd) < 0;
        sliceEnd = BufferUtil.nextBiggerBuffer(sliceEnd);
    } else {
        DirectionID dirID = getDirID(dir, rt);
        DataOutput colStart = serializer.getDataOutput(DEFAULT_COLUMN_CAPACITY);
        DataOutput colEnd = serializer.getDataOutput(DEFAULT_COLUMN_CAPACITY);
        IDHandler.writeRelationType(colStart, type.longId(), dirID, type.isInvisibleType());
        IDHandler.writeRelationType(colEnd, type.longId(), dirID, type.isInvisibleType());
        long[] sortKeyIDs = type.getSortKey();
        Preconditions.checkArgument(sortKey.length >= sortKeyIDs.length);
        assert colStart.getPosition() == colEnd.getPosition();
        int keyStartPos = colStart.getPosition();
        int keyEndPos = -1;
        for (int i = 0; i < sortKey.length && sortKey[i] != null; i++) {
            PropertyKey skey = sortKey[i].key;
            Interval interval = sortKey[i].interval;
            if (i >= sortKeyIDs.length) {
                assert !type.multiplicity().isUnique(dir);
                assert (skey instanceof ImplicitKey) && (skey == ImplicitKey.TITANID || skey == ImplicitKey.ADJACENT_ID);
                assert skey != ImplicitKey.ADJACENT_ID || (i == sortKeyIDs.length);
                assert skey != ImplicitKey.TITANID || (!type.multiplicity().isConstrained() && (i == sortKeyIDs.length && skey.isPropertyKey() || i == sortKeyIDs.length + 1 && skey.isEdgeLabel()));
                assert colStart.getPosition() == colEnd.getPosition();
                assert interval == null || interval.isPoints();
                keyEndPos = colStart.getPosition();
            } else {
                assert !type.multiplicity().isConstrained();
                assert skey.longId() == sortKeyIDs[i];
            }
            if (interval == null || interval.isEmpty()) {
                break;
            }
            if (interval.isPoints()) {
                if (skey == ImplicitKey.TITANID || skey == ImplicitKey.ADJACENT_ID) {
                    assert !type.multiplicity().isUnique(dir);
                    VariableLong.writePositiveBackward(colStart, (Long) interval.getStart());
                    VariableLong.writePositiveBackward(colEnd, (Long) interval.getEnd());
                } else {
                    writeInline(colStart, skey, interval.getStart(), InlineType.KEY);
                    writeInline(colEnd, skey, interval.getEnd(), InlineType.KEY);
                }
            } else {
                if (interval.getStart() != null)
                    writeInline(colStart, skey, interval.getStart(), InlineType.KEY);
                if (interval.getEnd() != null)
                    writeInline(colEnd, skey, interval.getEnd(), InlineType.KEY);
                switch(type.getSortOrder()) {
                    case ASC:
                        sliceStart = colStart.getStaticBuffer();
                        sliceEnd = colEnd.getStaticBuffer();
                        if (!interval.startInclusive())
                            sliceStart = BufferUtil.nextBiggerBuffer(sliceStart);
                        if (interval.endInclusive())
                            sliceEnd = BufferUtil.nextBiggerBuffer(sliceEnd);
                        break;
                    case DESC:
                        sliceEnd = colStart.getStaticBufferFlipBytes(keyStartPos, colStart.getPosition());
                        sliceStart = colEnd.getStaticBufferFlipBytes(keyStartPos, colEnd.getPosition());
                        if (interval.startInclusive())
                            sliceEnd = BufferUtil.nextBiggerBuffer(sliceEnd);
                        if (!interval.endInclusive())
                            sliceStart = BufferUtil.nextBiggerBuffer(sliceStart);
                        break;
                    default:
                        throw new AssertionError(type.getSortOrder().toString());
                }
                assert sliceStart.compareTo(sliceEnd) <= 0;
                break;
            }
        }
        if (sliceStart == null) {
            assert sliceEnd == null && colStart.getPosition() == colEnd.getPosition();
            if (keyEndPos < 0)
                keyEndPos = colStart.getPosition();
            switch(type.getSortOrder()) {
                case ASC:
                    sliceStart = colStart.getStaticBuffer();
                    break;
                case DESC:
                    sliceStart = colStart.getStaticBufferFlipBytes(keyStartPos, keyEndPos);
                    break;
                default:
                    throw new AssertionError(type.getSortOrder().toString());
            }
            sliceEnd = BufferUtil.nextBiggerBuffer(sliceStart);
        }
    }
    return new SliceQuery(sliceStart, sliceEnd);
}

Example 60 with StaticBuffer

use of com.thinkaurelius.titan.diskstorage.StaticBuffer in project titan by thinkaurelius.

the class IndexRemoveJob method process.

@Override
public void process(StaticBuffer key, Map<SliceQuery, EntryList> entries, ScanMetrics metrics) {
    //The queries are already tailored enough => everything should be removed
    try {
        BackendTransaction mutator = writeTx.getTxHandle();
        final List<Entry> deletions;
        if (entries.size() == 1)
            deletions = Iterables.getOnlyElement(entries.values());
        else {
            int size = IteratorUtils.stream(entries.values().iterator()).map(e -> e.size()).reduce(0, (x, y) -> x + y);
            deletions = new ArrayList<>(size);
            entries.values().forEach(e -> deletions.addAll(e));
        }
        metrics.incrementCustom(DELETED_RECORDS_COUNT, deletions.size());
        if (isRelationTypeIndex()) {
            mutator.mutateEdges(key, KCVSCache.NO_ADDITIONS, deletions);
        } else {
            mutator.mutateIndex(key, KCVSCache.NO_ADDITIONS, deletions);
        }
    } catch (final Exception e) {
        mgmt.rollback();
        writeTx.rollback();
        metrics.incrementCustom(FAILED_TX);
        throw new TitanException(e.getMessage(), e);
    }
}