use of org.janusgraph.diskstorage.Entry in project janusgraph by JanusGraph.
the class ConsistentKeyIDAuthority method getIDBlock.
@Override
public synchronized IDBlock getIDBlock(final int partition, final int idNamespace, Duration timeout) throws BackendException {
Preconditions.checkArgument(partition >= 0 && partition < (1 << partitionBitWidth), "Invalid partition id [%s] for bit width [%s]", partition, partitionBitWidth);
// can be any non-negative value
Preconditions.checkArgument(idNamespace >= 0);
final Timer methodTime = times.getTimer().start();
final long blockSize = getBlockSize(idNamespace);
final long idUpperBound = getIdUpperBound(idNamespace);
final int maxAvailableBits = (VariableLong.unsignedBitLength(idUpperBound) - 1) - uniqueIdBitWidth;
Preconditions.checkArgument(maxAvailableBits > 0, "Unique id bit width [%s] is too wide for id-namespace [%s] id bound [%s]", uniqueIdBitWidth, idNamespace, idUpperBound);
final long idBlockUpperBound = (1L << maxAvailableBits);
final List<Integer> exhaustedUniquePIDs = new ArrayList<>(randomUniqueIDLimit);
Duration backoffMS = idApplicationWaitMS;
Preconditions.checkArgument(idBlockUpperBound > blockSize, "Block size [%s] is larger than upper bound [%s] for bit width [%s]", blockSize, idBlockUpperBound, uniqueIdBitWidth);
while (methodTime.elapsed().compareTo(timeout) < 0) {
final int uniquePID = getUniquePartitionID();
final StaticBuffer partitionKey = getPartitionKey(partition, idNamespace, uniquePID);
try {
long nextStart = getCurrentID(partitionKey);
if (idBlockUpperBound - blockSize <= nextStart) {
log.info("ID overflow detected on partition({})-namespace({}) with uniqueid {}. Current id {}, block size {}, and upper bound {} for bit width {}.", partition, idNamespace, uniquePID, nextStart, blockSize, idBlockUpperBound, uniqueIdBitWidth);
if (randomizeUniqueId) {
exhaustedUniquePIDs.add(uniquePID);
if (exhaustedUniquePIDs.size() == randomUniqueIDLimit)
throw new IDPoolExhaustedException(String.format("Exhausted %d uniqueid(s) on partition(%d)-namespace(%d): %s", exhaustedUniquePIDs.size(), partition, idNamespace, StringUtils.join(exhaustedUniquePIDs, ",")));
else
throw new UniqueIDExhaustedException(String.format("Exhausted ID partition(%d)-namespace(%d) with uniqueid %d (uniqueid attempt %d/%d)", partition, idNamespace, uniquePID, exhaustedUniquePIDs.size(), randomUniqueIDLimit));
}
throw new IDPoolExhaustedException("Exhausted id block for partition(" + partition + ")-namespace(" + idNamespace + ") with upper bound: " + idBlockUpperBound);
}
// calculate the start (inclusive) and end (exclusive) of the allocation we're about to attempt
assert idBlockUpperBound - blockSize > nextStart;
long nextEnd = nextStart + blockSize;
StaticBuffer target = null;
// attempt to write our claim on the next id block
boolean success = false;
try {
Timer writeTimer = times.getTimer().start();
target = getBlockApplication(nextEnd, writeTimer.getStartTime());
// copy for the inner class
final StaticBuffer finalTarget = target;
BackendOperation.execute(txh -> {
idStore.mutate(partitionKey, Collections.singletonList(StaticArrayEntry.of(finalTarget)), KeyColumnValueStore.NO_DELETIONS, txh);
return true;
}, this, times);
writeTimer.stop();
final boolean distributed = manager.getFeatures().isDistributed();
Duration writeElapsed = writeTimer.elapsed();
if (idApplicationWaitMS.compareTo(writeElapsed) < 0 && distributed) {
throw new TemporaryBackendException("Wrote claim for id block [" + nextStart + ", " + nextEnd + ") in " + (writeElapsed) + " => too slow, threshold is: " + idApplicationWaitMS);
} else {
assert 0 != target.length();
final StaticBuffer[] slice = getBlockSlice(nextEnd);
if (distributed) {
sleepAndConvertInterrupts(idApplicationWaitMS.plus(waitGracePeriod));
}
// Read all id allocation claims on this partition, for the counter value we're claiming
final List<Entry> blocks = BackendOperation.execute((BackendOperation.Transactional<List<Entry>>) txh -> idStore.getSlice(new KeySliceQuery(partitionKey, slice[0], slice[1]), txh), this, times);
if (blocks == null)
throw new TemporaryBackendException("Could not read from storage");
if (blocks.isEmpty())
throw new PermanentBackendException("It seems there is a race-condition in the block application. " + "If you have multiple JanusGraph instances running on one physical machine, ensure that they have unique machine idAuthorities");
/* If our claim is the lexicographically first one, then our claim
* is the most senior one and we own this id block
*/
if (target.equals(blocks.get(0).getColumnAs(StaticBuffer.STATIC_FACTORY))) {
ConsistentKeyIDBlock idBlock = new ConsistentKeyIDBlock(nextStart, blockSize, uniqueIdBitWidth, uniquePID);
if (log.isDebugEnabled()) {
log.debug("Acquired ID block [{}] on partition({})-namespace({}) (my rid is {})", idBlock, partition, idNamespace, uid);
}
success = true;
return idBlock;
} else {
// Another claimant beat us to this id block -- try again.
log.debug("Failed to acquire ID block [{},{}) (another host claimed it first)", nextStart, nextEnd);
}
}
} finally {
if (!success && null != target) {
// Delete claim to not pollute id space
for (int attempt = 0; attempt < ROLLBACK_ATTEMPTS; attempt++) {
try {
// copy for the inner class
final StaticBuffer finalTarget = target;
BackendOperation.execute(txh -> {
idStore.mutate(partitionKey, KeyColumnValueStore.NO_ADDITIONS, Collections.singletonList(finalTarget), txh);
return true;
}, new // Use normal consistency level for these non-critical delete operations
BackendOperation.TransactionalProvider() {
@Override
public StoreTransaction openTx() throws BackendException {
return manager.beginTransaction(storeTxConfigBuilder.build());
}
@Override
public void close() {
}
}, times);
break;
} catch (BackendException e) {
log.warn("Storage exception while deleting old block application - retrying in {}", rollbackWaitTime, e);
if (!rollbackWaitTime.isZero())
sleepAndConvertInterrupts(rollbackWaitTime);
}
}
}
}
} catch (UniqueIDExhaustedException e) {
// No need to increment the backoff wait time or to sleep
log.warn(e.getMessage());
} catch (TemporaryBackendException e) {
backoffMS = Durations.min(backoffMS.multipliedBy(2), idApplicationWaitMS.multipliedBy(32));
log.warn("Temporary storage exception while acquiring id block - retrying in {}: {}", backoffMS, e);
sleepAndConvertInterrupts(backoffMS);
}
}
throw new TemporaryLockingException(String.format("Reached timeout %d (%s elapsed) when attempting to allocate id block on partition(%d)-namespace(%d)", timeout.getNano(), methodTime, partition, idNamespace));
}
use of org.janusgraph.diskstorage.Entry in project janusgraph by JanusGraph.
the class IndexSerializer method query.
/* ################################################
Querying
################################################### */
public Stream<Object> query(final JointIndexQuery.Subquery query, final BackendTransaction tx) {
final IndexType index = query.getIndex();
if (index.isCompositeIndex()) {
final MultiKeySliceQuery sq = query.getCompositeQuery();
final List<EntryList> rs = sq.execute(tx);
final List<Object> results = new ArrayList<>(rs.get(0).size());
for (final EntryList r : rs) {
for (final java.util.Iterator<Entry> iterator = r.reuseIterator(); iterator.hasNext(); ) {
final Entry entry = iterator.next();
final ReadBuffer entryValue = entry.asReadBuffer();
entryValue.movePositionTo(entry.getValuePosition());
switch(index.getElement()) {
case VERTEX:
results.add(VariableLong.readPositive(entryValue));
break;
default:
results.add(bytebuffer2RelationId(entryValue));
}
}
}
return results.stream();
} else {
return tx.indexQuery(index.getBackingIndexName(), query.getMixedQuery()).map(IndexSerializer::string2ElementId);
}
}
use of org.janusgraph.diskstorage.Entry in project janusgraph by JanusGraph.
the class JanusGraphVertexDeserializer method readHadoopVertex.
// Read a single row from the edgestore and create a TinkerVertex corresponding to the row
// The neighboring vertices are represented by DetachedVertex instances
public TinkerVertex readHadoopVertex(final StaticBuffer key, Iterable<Entry> entries) {
// Convert key to a vertex ID
final long vertexId = idManager.getKeyID(key);
Preconditions.checkArgument(vertexId > 0);
// Partitioned vertex handling
if (idManager.isPartitionedVertex(vertexId)) {
Preconditions.checkState(setup.getFilterPartitionedVertices(), "Read partitioned vertex (ID=%s), but partitioned vertex filtering is disabled.", vertexId);
log.debug("Skipping partitioned vertex with ID {}", vertexId);
return null;
}
// Create TinkerVertex
TinkerGraph tg = TinkerGraph.open();
TinkerVertex tv = null;
// Iterate over edgestore columns to find the vertex's label relation
for (final Entry data : entries) {
RelationReader relationReader = setup.getRelationReader();
final RelationCache relation = relationReader.parseRelation(data, false, typeManager);
if (systemTypes.isVertexLabelSystemType(relation.typeId)) {
// Found vertex Label
long vertexLabelId = relation.getOtherVertexId();
VertexLabel vl = typeManager.getExistingVertexLabel(vertexLabelId);
// Create TinkerVertex with this label
tv = getOrCreateVertex(vertexId, vl.name(), tg);
} else if (systemTypes.isTypeSystemType(relation.typeId)) {
log.trace("Vertex {} is a system vertex", vertexId);
return null;
}
}
// Added this following testing
if (null == tv) {
tv = getOrCreateVertex(vertexId, null, tg);
}
Preconditions.checkNotNull(tv, "Unable to determine vertex label for vertex with ID %s", vertexId);
// Iterate over and decode edgestore columns (relations) on this vertex
for (final Entry data : entries) {
try {
RelationReader relationReader = setup.getRelationReader();
final RelationCache relation = relationReader.parseRelation(data, false, typeManager);
// Ignore system types
if (systemTypes.isSystemType(relation.typeId))
continue;
final RelationType type = typeManager.getExistingRelationType(relation.typeId);
// Ignore hidden types
if (((InternalRelationType) type).isInvisibleType())
continue;
// Decode and create the relation (edge or property)
if (type.isPropertyKey()) {
// Decode property
Object value = relation.getValue();
Preconditions.checkNotNull(value);
VertexProperty.Cardinality card = getPropertyKeyCardinality(type.name());
VertexProperty<Object> vp = tv.property(card, type.name(), value, T.id, relation.relationId);
// Decode meta properties
decodeProperties(relation, vp);
} else {
assert type.isEdgeLabel();
// Partitioned vertex handling
if (idManager.isPartitionedVertex(relation.getOtherVertexId())) {
Preconditions.checkState(setup.getFilterPartitionedVertices(), "Read edge incident on a partitioned vertex, but partitioned vertex filtering is disabled. " + "Relation ID: %s. This vertex ID: %s. Other vertex ID: %s. Edge label: %s.", relation.relationId, vertexId, relation.getOtherVertexId(), type.name());
log.debug("Skipping edge with ID {} incident on partitioned vertex with ID {} (and nonpartitioned vertex with ID {})", relation.relationId, relation.getOtherVertexId(), vertexId);
continue;
}
// Decode edge
TinkerEdge te;
// We don't know the label of the other vertex, but one must be provided
TinkerVertex adjacentVertex = getOrCreateVertex(relation.getOtherVertexId(), null, tg);
// skip self-loop edges that were already processed, but from a different direction
if (tv.equals(adjacentVertex) && edgeExists(tv, type, relation)) {
continue;
}
if (relation.direction.equals(Direction.IN)) {
te = (TinkerEdge) adjacentVertex.addEdge(type.name(), tv, T.id, relation.relationId);
} else if (relation.direction.equals(Direction.OUT)) {
te = (TinkerEdge) tv.addEdge(type.name(), adjacentVertex, T.id, relation.relationId);
} else {
throw new RuntimeException("Direction.BOTH is not supported");
}
decodeProperties(relation, te);
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}
return tv;
}
use of org.janusgraph.diskstorage.Entry in project janusgraph by JanusGraph.
the class OrderedKeyValueStoreManagerAdapter method mutateMany.
@Override
public void mutateMany(Map<String, Map<StaticBuffer, KCVMutation>> mutations, StoreTransaction txh) throws BackendException {
final Map<String, KVMutation> converted = new HashMap<>(mutations.size());
for (Map.Entry<String, Map<StaticBuffer, KCVMutation>> storeEntry : mutations.entrySet()) {
OrderedKeyValueStoreAdapter store = openDatabase(storeEntry.getKey());
Preconditions.checkNotNull(store);
KVMutation mut = new KVMutation();
for (Map.Entry<StaticBuffer, KCVMutation> entry : storeEntry.getValue().entrySet()) {
StaticBuffer key = entry.getKey();
KCVMutation mutation = entry.getValue();
if (mutation.hasAdditions()) {
for (Entry addition : mutation.getAdditions()) {
KeyValueEntry concatenate = store.concatenate(key, addition);
concatenate.setTTL((Integer) addition.getMetaData().get(EntryMetaData.TTL));
mut.addition(concatenate);
}
}
if (mutation.hasDeletions()) {
for (StaticBuffer del : mutation.getDeletions()) {
mut.deletion(store.concatenate(key, del));
}
}
}
converted.put(storeEntry.getKey(), mut);
}
manager.mutateMany(converted, txh);
}
use of org.janusgraph.diskstorage.Entry in project janusgraph by JanusGraph.
the class KCVSConfiguration method toMap.
private Map<String, Object> toMap() {
List<Entry> result = BackendOperation.execute(new BackendOperation.Transactional<List<Entry>>() {
@Override
public List<Entry> call(StoreTransaction txh) throws BackendException {
return store.getSlice(new KeySliceQuery(rowKey, BufferUtil.zeroBuffer(1), BufferUtil.oneBuffer(128)), txh);
}
@Override
public String toString() {
return "setConfiguration";
}
}, txProvider, times, maxOperationWaitTime);
Map<String, Object> entries = new HashMap<>(result.size());
for (Entry entry : result) {
String key = staticBuffer2String(entry.getColumnAs(StaticBuffer.STATIC_FACTORY));
Object value = staticBuffer2Object(entry.getValueAs(StaticBuffer.STATIC_FACTORY), Object.class);
entries.put(key, value);
}
return entries;
}
Aggregations