Examples with StaticBuffer org.janusgraph.diskstorage.StaticBuffer used on opensource projects

Example 61 with StaticBuffer

use of org.janusgraph.diskstorage.StaticBuffer in project janusgraph by JanusGraph.

the class StandardJanusGraph method prepareCommit.

public ModificationSummary prepareCommit(final Collection<InternalRelation> addedRelations, final Collection<InternalRelation> deletedRelations, final Predicate<InternalRelation> filter, final BackendTransaction mutator, final StandardJanusGraphTx tx, final boolean acquireLocks) throws BackendException {
    ListMultimap<Long, InternalRelation> mutations = ArrayListMultimap.create();
    ListMultimap<InternalVertex, InternalRelation> mutatedProperties = ArrayListMultimap.create();
    List<IndexSerializer.IndexUpdate> indexUpdates = Lists.newArrayList();
    // 1) Collect deleted edges and their index updates and acquire edge locks
    for (InternalRelation del : Iterables.filter(deletedRelations, filter)) {
        Preconditions.checkArgument(del.isRemoved());
        for (int pos = 0; pos < del.getLen(); pos++) {
            InternalVertex vertex = del.getVertex(pos);
            if (pos == 0 || !del.isLoop()) {
                if (del.isProperty())
                    mutatedProperties.put(vertex, del);
                mutations.put(vertex.longId(), del);
            }
            if (acquireLock(del, pos, acquireLocks)) {
                Entry entry = edgeSerializer.writeRelation(del, pos, tx);
                mutator.acquireEdgeLock(idManager.getKey(vertex.longId()), entry);
            }
        }
        indexUpdates.addAll(indexSerializer.getIndexUpdates(del));
    }
    // 2) Collect added edges and their index updates and acquire edge locks
    for (InternalRelation add : Iterables.filter(addedRelations, filter)) {
        Preconditions.checkArgument(add.isNew());
        for (int pos = 0; pos < add.getLen(); pos++) {
            InternalVertex vertex = add.getVertex(pos);
            if (pos == 0 || !add.isLoop()) {
                if (add.isProperty())
                    mutatedProperties.put(vertex, add);
                mutations.put(vertex.longId(), add);
            }
            if (!vertex.isNew() && acquireLock(add, pos, acquireLocks)) {
                Entry entry = edgeSerializer.writeRelation(add, pos, tx);
                mutator.acquireEdgeLock(idManager.getKey(vertex.longId()), entry.getColumn());
            }
        }
        indexUpdates.addAll(indexSerializer.getIndexUpdates(add));
    }
    // 3) Collect all index update for vertices
    for (InternalVertex v : mutatedProperties.keySet()) {
        indexUpdates.addAll(indexSerializer.getIndexUpdates(v, mutatedProperties.get(v)));
    }
    // 4) Acquire index locks (deletions first)
    for (IndexSerializer.IndexUpdate update : indexUpdates) {
        if (!update.isCompositeIndex() || !update.isDeletion())
            continue;
        CompositeIndexType iIndex = (CompositeIndexType) update.getIndex();
        if (acquireLock(iIndex, acquireLocks)) {
            mutator.acquireIndexLock((StaticBuffer) update.getKey(), (Entry) update.getEntry());
        }
    }
    for (IndexSerializer.IndexUpdate update : indexUpdates) {
        if (!update.isCompositeIndex() || !update.isAddition())
            continue;
        CompositeIndexType iIndex = (CompositeIndexType) update.getIndex();
        if (acquireLock(iIndex, acquireLocks)) {
            mutator.acquireIndexLock((StaticBuffer) update.getKey(), ((Entry) update.getEntry()).getColumn());
        }
    }
    // 5) Add relation mutations
    for (Long vertexId : mutations.keySet()) {
        Preconditions.checkArgument(vertexId > 0, "Vertex has no id: %s", vertexId);
        final List<InternalRelation> edges = mutations.get(vertexId);
        final List<Entry> additions = new ArrayList<>(edges.size());
        final List<Entry> deletions = new ArrayList<>(Math.max(10, edges.size() / 10));
        for (final InternalRelation edge : edges) {
            final InternalRelationType baseType = (InternalRelationType) edge.getType();
            assert baseType.getBaseType() == null;
            for (InternalRelationType type : baseType.getRelationIndexes()) {
                if (type.getStatus() == SchemaStatus.DISABLED)
                    continue;
                for (int pos = 0; pos < edge.getArity(); pos++) {
                    if (!type.isUnidirected(Direction.BOTH) && !type.isUnidirected(EdgeDirection.fromPosition(pos)))
                        // Directionality is not covered
                        continue;
                    if (edge.getVertex(pos).longId() == vertexId) {
                        StaticArrayEntry entry = edgeSerializer.writeRelation(edge, type, pos, tx);
                        if (edge.isRemoved()) {
                            deletions.add(entry);
                        } else {
                            Preconditions.checkArgument(edge.isNew());
                            int ttl = getTTL(edge);
                            if (ttl > 0) {
                                entry.setMetaData(EntryMetaData.TTL, ttl);
                            }
                            additions.add(entry);
                        }
                    }
                }
            }
        }
        StaticBuffer vertexKey = idManager.getKey(vertexId);
        mutator.mutateEdges(vertexKey, additions, deletions);
    }
    // 6) Add index updates
    boolean has2iMods = false;
    for (IndexSerializer.IndexUpdate indexUpdate : indexUpdates) {
        assert indexUpdate.isAddition() || indexUpdate.isDeletion();
        if (indexUpdate.isCompositeIndex()) {
            final IndexSerializer.IndexUpdate<StaticBuffer, Entry> update = indexUpdate;
            if (update.isAddition())
                mutator.mutateIndex(update.getKey(), Lists.newArrayList(update.getEntry()), KCVSCache.NO_DELETIONS);
            else
                mutator.mutateIndex(update.getKey(), KeyColumnValueStore.NO_ADDITIONS, Lists.newArrayList(update.getEntry()));
        } else {
            final IndexSerializer.IndexUpdate<String, IndexEntry> update = indexUpdate;
            has2iMods = true;
            IndexTransaction itx = mutator.getIndexTransaction(update.getIndex().getBackingIndexName());
            String indexStore = ((MixedIndexType) update.getIndex()).getStoreName();
            if (update.isAddition())
                itx.add(indexStore, update.getKey(), update.getEntry(), update.getElement().isNew());
            else
                itx.delete(indexStore, update.getKey(), update.getEntry().field, update.getEntry().value, update.getElement().isRemoved());
        }
    }
    return new ModificationSummary(!mutations.isEmpty(), has2iMods);
}

Example 62 with StaticBuffer

use of org.janusgraph.diskstorage.StaticBuffer in project janusgraph by JanusGraph.

the class StandardJanusGraph method edgeMultiQuery.

public List<EntryList> edgeMultiQuery(LongArrayList vertexIdsAsLongs, SliceQuery query, BackendTransaction tx) {
    Preconditions.checkArgument(vertexIdsAsLongs != null && !vertexIdsAsLongs.isEmpty());
    final List<StaticBuffer> vertexIds = new ArrayList<>(vertexIdsAsLongs.size());
    for (int i = 0; i < vertexIdsAsLongs.size(); i++) {
        Preconditions.checkArgument(vertexIdsAsLongs.get(i) > 0);
        vertexIds.add(idManager.getKey(vertexIdsAsLongs.get(i)));
    }
    final Map<StaticBuffer, EntryList> result = tx.edgeStoreMultiQuery(vertexIds, query);
    final List<EntryList> resultList = new ArrayList<>(result.size());
    for (StaticBuffer v : vertexIds) resultList.add(result.get(v));
    return resultList;
}

Example 63 with StaticBuffer

use of org.janusgraph.diskstorage.StaticBuffer in project janusgraph by JanusGraph.

the class StandardTransactionLogProcessor method fixSecondaryFailure.

private void fixSecondaryFailure(final StandardTransactionId txId, final TxEntry entry) {
    logRecoveryMsg("Attempting to repair partially failed transaction [%s]", txId);
    if (entry.entry == null) {
        logRecoveryMsg("Trying to repair expired or unpersisted transaction [%s] (Ignore in startup)", txId);
        return;
    }
    boolean userLogFailure = true;
    boolean secIndexFailure = true;
    final Predicate<String> isFailedIndex;
    final TransactionLogHeader.Entry commitEntry = entry.entry;
    final TransactionLogHeader.SecondaryFailures secFail = entry.failures;
    if (secFail != null) {
        userLogFailure = secFail.userLogFailure;
        secIndexFailure = !secFail.failedIndexes.isEmpty();
        isFailedIndex = secFail.failedIndexes::contains;
    } else {
        isFailedIndex = Predicates.alwaysTrue();
    }
    // I) Restore external indexes
    if (secIndexFailure) {
        restoreExternalIndexes(isFailedIndex, commitEntry);
    }
    // II) Restore log messages
    final String logTxIdentifier = (String) commitEntry.getMetadata().get(LogTxMeta.LOG_ID);
    if (userLogFailure && logTxIdentifier != null) {
        TransactionLogHeader txHeader = new TransactionLogHeader(txCounter.incrementAndGet(), times.getTime(), times);
        final StaticBuffer userLogContent = txHeader.serializeUserLog(serializer, commitEntry, txId);
        BackendOperation.execute(() -> {
            final Log userLog = graph.getBackend().getUserLog(logTxIdentifier);
            final Future<Message> env = userLog.add(userLogContent);
            if (env.isDone()) {
                env.get();
            }
            return true;
        }, persistenceTime);
    }
}

Example 64 with StaticBuffer

use of org.janusgraph.diskstorage.StaticBuffer in project janusgraph by JanusGraph.

the class ConsistentKeyIDAuthority method getIDBlock.

@Override
public synchronized IDBlock getIDBlock(final int partition, final int idNamespace, Duration timeout) throws BackendException {
    Preconditions.checkArgument(partition >= 0 && partition < (1 << partitionBitWidth), "Invalid partition id [%s] for bit width [%s]", partition, partitionBitWidth);
    // can be any non-negative value
    Preconditions.checkArgument(idNamespace >= 0);
    final Timer methodTime = times.getTimer().start();
    final long blockSize = getBlockSize(idNamespace);
    final long idUpperBound = getIdUpperBound(idNamespace);
    final int maxAvailableBits = (VariableLong.unsignedBitLength(idUpperBound) - 1) - uniqueIdBitWidth;
    Preconditions.checkArgument(maxAvailableBits > 0, "Unique id bit width [%s] is too wide for id-namespace [%s] id bound [%s]", uniqueIdBitWidth, idNamespace, idUpperBound);
    final long idBlockUpperBound = (1L << maxAvailableBits);
    final List<Integer> exhaustedUniquePIDs = new ArrayList<>(randomUniqueIDLimit);
    Duration backoffMS = idApplicationWaitMS;
    Preconditions.checkArgument(idBlockUpperBound > blockSize, "Block size [%s] is larger than upper bound [%s] for bit width [%s]", blockSize, idBlockUpperBound, uniqueIdBitWidth);
    while (methodTime.elapsed().compareTo(timeout) < 0) {
        final int uniquePID = getUniquePartitionID();
        final StaticBuffer partitionKey = getPartitionKey(partition, idNamespace, uniquePID);
        try {
            long nextStart = getCurrentID(partitionKey);
            if (idBlockUpperBound - blockSize <= nextStart) {
                log.info("ID overflow detected on partition({})-namespace({}) with uniqueid {}. Current id {}, block size {}, and upper bound {} for bit width {}.", partition, idNamespace, uniquePID, nextStart, blockSize, idBlockUpperBound, uniqueIdBitWidth);
                if (randomizeUniqueId) {
                    exhaustedUniquePIDs.add(uniquePID);
                    if (exhaustedUniquePIDs.size() == randomUniqueIDLimit)
                        throw new IDPoolExhaustedException(String.format("Exhausted %d uniqueid(s) on partition(%d)-namespace(%d): %s", exhaustedUniquePIDs.size(), partition, idNamespace, StringUtils.join(exhaustedUniquePIDs, ",")));
                    else
                        throw new UniqueIDExhaustedException(String.format("Exhausted ID partition(%d)-namespace(%d) with uniqueid %d (uniqueid attempt %d/%d)", partition, idNamespace, uniquePID, exhaustedUniquePIDs.size(), randomUniqueIDLimit));
                }
                throw new IDPoolExhaustedException("Exhausted id block for partition(" + partition + ")-namespace(" + idNamespace + ") with upper bound: " + idBlockUpperBound);
            }
            // calculate the start (inclusive) and end (exclusive) of the allocation we're about to attempt
            assert idBlockUpperBound - blockSize > nextStart;
            long nextEnd = nextStart + blockSize;
            StaticBuffer target = null;
            // attempt to write our claim on the next id block
            boolean success = false;
            try {
                Timer writeTimer = times.getTimer().start();
                target = getBlockApplication(nextEnd, writeTimer.getStartTime());
                // copy for the inner class
                final StaticBuffer finalTarget = target;
                BackendOperation.execute(txh -> {
                    idStore.mutate(partitionKey, Collections.singletonList(StaticArrayEntry.of(finalTarget)), KeyColumnValueStore.NO_DELETIONS, txh);
                    return true;
                }, this, times);
                writeTimer.stop();
                final boolean distributed = manager.getFeatures().isDistributed();
                Duration writeElapsed = writeTimer.elapsed();
                if (idApplicationWaitMS.compareTo(writeElapsed) < 0 && distributed) {
                    throw new TemporaryBackendException("Wrote claim for id block [" + nextStart + ", " + nextEnd + ") in " + (writeElapsed) + " => too slow, threshold is: " + idApplicationWaitMS);
                } else {
                    assert 0 != target.length();
                    final StaticBuffer[] slice = getBlockSlice(nextEnd);
                    if (distributed) {
                        sleepAndConvertInterrupts(idApplicationWaitMS.plus(waitGracePeriod));
                    }
                    // Read all id allocation claims on this partition, for the counter value we're claiming
                    final List<Entry> blocks = BackendOperation.execute((BackendOperation.Transactional<List<Entry>>) txh -> idStore.getSlice(new KeySliceQuery(partitionKey, slice[0], slice[1]), txh), this, times);
                    if (blocks == null)
                        throw new TemporaryBackendException("Could not read from storage");
                    if (blocks.isEmpty())
                        throw new PermanentBackendException("It seems there is a race-condition in the block application. " + "If you have multiple JanusGraph instances running on one physical machine, ensure that they have unique machine idAuthorities");
                    /* If our claim is the lexicographically first one, then our claim
                         * is the most senior one and we own this id block
                         */
                    if (target.equals(blocks.get(0).getColumnAs(StaticBuffer.STATIC_FACTORY))) {
                        ConsistentKeyIDBlock idBlock = new ConsistentKeyIDBlock(nextStart, blockSize, uniqueIdBitWidth, uniquePID);
                        if (log.isDebugEnabled()) {
                            log.debug("Acquired ID block [{}] on partition({})-namespace({}) (my rid is {})", idBlock, partition, idNamespace, uid);
                        }
                        success = true;
                        return idBlock;
                    } else {
                        // Another claimant beat us to this id block -- try again.
                        log.debug("Failed to acquire ID block [{},{}) (another host claimed it first)", nextStart, nextEnd);
                    }
                }
            } finally {
                if (!success && null != target) {
                    // Delete claim to not pollute id space
                    for (int attempt = 0; attempt < ROLLBACK_ATTEMPTS; attempt++) {
                        try {
                            // copy for the inner class
                            final StaticBuffer finalTarget = target;
                            BackendOperation.execute(txh -> {
                                idStore.mutate(partitionKey, KeyColumnValueStore.NO_ADDITIONS, Collections.singletonList(finalTarget), txh);
                                return true;
                            }, new // Use normal consistency level for these non-critical delete operations
                            BackendOperation.TransactionalProvider() {

                                @Override
                                public StoreTransaction openTx() throws BackendException {
                                    return manager.beginTransaction(storeTxConfigBuilder.build());
                                }

                                @Override
                                public void close() {
                                }
                            }, times);
                            break;
                        } catch (BackendException e) {
                            log.warn("Storage exception while deleting old block application - retrying in {}", rollbackWaitTime, e);
                            if (!rollbackWaitTime.isZero())
                                sleepAndConvertInterrupts(rollbackWaitTime);
                        }
                    }
                }
            }
        } catch (UniqueIDExhaustedException e) {
            // No need to increment the backoff wait time or to sleep
            log.warn(e.getMessage());
        } catch (TemporaryBackendException e) {
            backoffMS = Durations.min(backoffMS.multipliedBy(2), idApplicationWaitMS.multipliedBy(32));
            log.warn("Temporary storage exception while acquiring id block - retrying in {}: {}", backoffMS, e);
            sleepAndConvertInterrupts(backoffMS);
        }
    }
    throw new TemporaryLockingException(String.format("Reached timeout %d (%s elapsed) when attempting to allocate id block on partition(%d)-namespace(%d)", timeout.getNano(), methodTime, partition, idNamespace));
}

Example 65 with StaticBuffer

use of org.janusgraph.diskstorage.StaticBuffer in project janusgraph by JanusGraph.

the class TransactionLogHeader method parse.

public static Entry parse(StaticBuffer buffer, Serializer serializer, TimestampProvider times) {
    ReadBuffer read = buffer.asReadBuffer();
    Instant txTimestamp = times.getTime(read.getLong());
    TransactionLogHeader header = new TransactionLogHeader(VariableLong.readPositive(read), txTimestamp, times);
    LogTxStatus status = serializer.readObjectNotNull(read, LogTxStatus.class);
    final EnumMap<LogTxMeta, Object> metadata = new EnumMap<>(LogTxMeta.class);
    int metaSize = VariableLong.unsignedByte(read.getByte());
    for (int i = 0; i < metaSize; i++) {
        LogTxMeta meta = LogTxMeta.values()[VariableLong.unsignedByte(read.getByte())];
        metadata.put(meta, serializer.readObjectNotNull(read, meta.dataType()));
    }
    if (read.hasRemaining()) {
        StaticBuffer content = read.subrange(read.getPosition(), read.length() - read.getPosition());
        return new Entry(header, content, status, metadata);
    } else {
        return new Entry(header, null, status, metadata);
    }
}