use of org.janusgraph.graphdb.database.idassigner.IDPoolExhaustedException in project janusgraph by JanusGraph.
the class IDPoolTest method testPoolExhaustion2.
@Test
public void testPoolExhaustion2() {
int idUpper = 10000;
MockIDAuthority idAuthority = new MockIDAuthority(200, idUpper);
StandardIDPool pool = new StandardIDPool(idAuthority, 0, 1, Integer.MAX_VALUE, Duration.ofMillis(2000), 0.2);
for (int i = 1; i < idUpper * 2; i++) {
try {
long id = pool.nextID();
assertTrue(id < idUpper);
} catch (IDPoolExhaustedException e) {
assertEquals(idUpper, i);
break;
}
}
}
use of org.janusgraph.graphdb.database.idassigner.IDPoolExhaustedException in project janusgraph by JanusGraph.
the class PropertyPlacementStrategy method getPartitionIDbyKey.
private int getPartitionIDbyKey(JanusGraphVertex vertex) {
Preconditions.checkState(idManager != null && key != null, "PropertyPlacementStrategy has not been initialized correctly");
assert idManager.getPartitionBound() <= Integer.MAX_VALUE;
int partitionBound = (int) idManager.getPartitionBound();
final JanusGraphVertexProperty p = Iterables.getFirst(vertex.query().keys(key).properties(), null);
if (p == null)
return -1;
int hashPid = Math.abs(p.value().hashCode()) % partitionBound;
assert hashPid >= 0 && hashPid < partitionBound;
if (isExhaustedPartition(hashPid)) {
// We keep trying consecutive partition ids until we find a non-exhausted one
int newPid = hashPid;
do {
newPid = (newPid + 1) % partitionBound;
if (// We have gone full circle - no more ids to try
newPid == hashPid)
throw new IDPoolExhaustedException("Could not find non-exhausted partition");
} while (isExhaustedPartition(newPid));
return newPid;
} else
return hashPid;
}
use of org.janusgraph.graphdb.database.idassigner.IDPoolExhaustedException in project janusgraph by JanusGraph.
the class SimpleBulkPlacementStrategy method updateElement.
private void updateElement(int index) {
Preconditions.checkArgument(localPartitionIdRanges != null && !localPartitionIdRanges.isEmpty(), "Local partition id ranges have not been initialized");
int newPartition;
int attempts = 0;
do {
attempts++;
newPartition = localPartitionIdRanges.get(random.nextInt(localPartitionIdRanges.size())).getRandomID();
if (attempts > PARTITION_FINDING_ATTEMPTS)
throw new IDPoolExhaustedException("Could not find non-exhausted partition");
} while (exhaustedPartitions.contains(newPartition));
currentPartitions[index] = newPartition;
log.debug("Setting partition at index [{}] to: {}", index, newPartition);
}
use of org.janusgraph.graphdb.database.idassigner.IDPoolExhaustedException in project janusgraph by JanusGraph.
the class ConsistentKeyIDAuthority method getIDBlock.
@Override
public synchronized IDBlock getIDBlock(final int partition, final int idNamespace, Duration timeout) throws BackendException {
Preconditions.checkArgument(partition >= 0 && partition < (1 << partitionBitWidth), "Invalid partition id [%s] for bit width [%s]", partition, partitionBitWidth);
// can be any non-negative value
Preconditions.checkArgument(idNamespace >= 0);
final Timer methodTime = times.getTimer().start();
final long blockSize = getBlockSize(idNamespace);
final long idUpperBound = getIdUpperBound(idNamespace);
final int maxAvailableBits = (VariableLong.unsignedBitLength(idUpperBound) - 1) - uniqueIdBitWidth;
Preconditions.checkArgument(maxAvailableBits > 0, "Unique id bit width [%s] is too wide for id-namespace [%s] id bound [%s]", uniqueIdBitWidth, idNamespace, idUpperBound);
final long idBlockUpperBound = (1L << maxAvailableBits);
final List<Integer> exhaustedUniquePIDs = new ArrayList<>(randomUniqueIDLimit);
Duration backoffMS = idApplicationWaitMS;
Preconditions.checkArgument(idBlockUpperBound > blockSize, "Block size [%s] is larger than upper bound [%s] for bit width [%s]", blockSize, idBlockUpperBound, uniqueIdBitWidth);
while (methodTime.elapsed().compareTo(timeout) < 0) {
final int uniquePID = getUniquePartitionID();
final StaticBuffer partitionKey = getPartitionKey(partition, idNamespace, uniquePID);
try {
long nextStart = getCurrentID(partitionKey);
if (idBlockUpperBound - blockSize <= nextStart) {
log.info("ID overflow detected on partition({})-namespace({}) with uniqueid {}. Current id {}, block size {}, and upper bound {} for bit width {}.", partition, idNamespace, uniquePID, nextStart, blockSize, idBlockUpperBound, uniqueIdBitWidth);
if (randomizeUniqueId) {
exhaustedUniquePIDs.add(uniquePID);
if (exhaustedUniquePIDs.size() == randomUniqueIDLimit)
throw new IDPoolExhaustedException(String.format("Exhausted %d uniqueid(s) on partition(%d)-namespace(%d): %s", exhaustedUniquePIDs.size(), partition, idNamespace, Joiner.on(",").join(exhaustedUniquePIDs)));
else
throw new UniqueIDExhaustedException(String.format("Exhausted ID partition(%d)-namespace(%d) with uniqueid %d (uniqueid attempt %d/%d)", partition, idNamespace, uniquePID, exhaustedUniquePIDs.size(), randomUniqueIDLimit));
}
throw new IDPoolExhaustedException("Exhausted id block for partition(" + partition + ")-namespace(" + idNamespace + ") with upper bound: " + idBlockUpperBound);
}
// calculate the start (inclusive) and end (exclusive) of the allocation we're about to attempt
assert idBlockUpperBound - blockSize > nextStart;
long nextEnd = nextStart + blockSize;
StaticBuffer target = null;
// attempt to write our claim on the next id block
boolean success = false;
try {
Timer writeTimer = times.getTimer().start();
target = getBlockApplication(nextEnd, writeTimer.getStartTime());
// copy for the inner class
final StaticBuffer finalTarget = target;
BackendOperation.execute(txh -> {
idStore.mutate(partitionKey, Collections.singletonList(StaticArrayEntry.of(finalTarget)), KeyColumnValueStore.NO_DELETIONS, txh);
return true;
}, this, times);
writeTimer.stop();
Duration writeElapsed = writeTimer.elapsed();
if (idApplicationWaitMS.compareTo(writeElapsed) < 0) {
throw new TemporaryBackendException("Wrote claim for id block [" + nextStart + ", " + nextEnd + ") in " + (writeElapsed) + " => too slow, threshold is: " + idApplicationWaitMS);
} else {
assert 0 != target.length();
final StaticBuffer[] slice = getBlockSlice(nextEnd);
/* At this point we've written our claim on [nextStart, nextEnd),
* but we haven't yet guaranteed the absence of a contending claim on
* the same id block from another machine
*/
sleepAndConvertInterrupts(idApplicationWaitMS.plus(waitGracePeriod));
// Read all id allocation claims on this partition, for the counter value we're claiming
final List<Entry> blocks = BackendOperation.execute((BackendOperation.Transactional<List<Entry>>) txh -> idStore.getSlice(new KeySliceQuery(partitionKey, slice[0], slice[1]), txh), this, times);
if (blocks == null)
throw new TemporaryBackendException("Could not read from storage");
if (blocks.isEmpty())
throw new PermanentBackendException("It seems there is a race-condition in the block application. " + "If you have multiple JanusGraph instances running on one physical machine, ensure that they have unique machine idAuthorities");
/* If our claim is the lexicographically first one, then our claim
* is the most senior one and we own this id block
*/
if (target.equals(blocks.get(0).getColumnAs(StaticBuffer.STATIC_FACTORY))) {
ConsistentKeyIDBlock idBlock = new ConsistentKeyIDBlock(nextStart, blockSize, uniqueIdBitWidth, uniquePID);
if (log.isDebugEnabled()) {
log.debug("Acquired ID block [{}] on partition({})-namespace({}) (my rid is {})", idBlock, partition, idNamespace, uid);
}
success = true;
return idBlock;
} else {
// Another claimant beat us to this id block -- try again.
log.debug("Failed to acquire ID block [{},{}) (another host claimed it first)", nextStart, nextEnd);
}
}
} finally {
if (!success && null != target) {
// Delete claim to not pollute id space
for (int attempt = 0; attempt < ROLLBACK_ATTEMPTS; attempt++) {
try {
// copy for the inner class
final StaticBuffer finalTarget = target;
BackendOperation.execute(txh -> {
idStore.mutate(partitionKey, KeyColumnValueStore.NO_ADDITIONS, Collections.singletonList(finalTarget), txh);
return true;
}, new // Use normal consistency level for these non-critical delete operations
BackendOperation.TransactionalProvider() {
@Override
public StoreTransaction openTx() throws BackendException {
return manager.beginTransaction(storeTxConfigBuilder.build());
}
@Override
public void close() {
}
}, times);
break;
} catch (BackendException e) {
log.warn("Storage exception while deleting old block application - retrying in {}", rollbackWaitTime, e);
if (!rollbackWaitTime.isZero())
sleepAndConvertInterrupts(rollbackWaitTime);
}
}
}
}
} catch (UniqueIDExhaustedException e) {
// No need to increment the backoff wait time or to sleep
log.warn(e.getMessage());
} catch (TemporaryBackendException e) {
backoffMS = Durations.min(backoffMS.multipliedBy(2), idApplicationWaitMS.multipliedBy(32));
log.warn("Temporary storage exception while acquiring id block - retrying in {}: {}", backoffMS, e);
sleepAndConvertInterrupts(backoffMS);
}
}
throw new TemporaryLockingException(String.format("Reached timeout %d (%s elapsed) when attempting to allocate id block on partition(%d)-namespace(%d)", timeout.getNano(), methodTime.toString(), partition, idNamespace));
}
use of org.janusgraph.graphdb.database.idassigner.IDPoolExhaustedException in project janusgraph by JanusGraph.
the class IDAuthorityTest method testIDExhaustion.
@Test
public void testIDExhaustion() throws BackendException {
final int chunks = 30;
final IDBlockSizer blockSizer = new IDBlockSizer() {
@Override
public long getBlockSize(int idNamespace) {
return ((1L << (idUpperBoundBitWidth - uidBitWidth)) - 1) / chunks;
}
@Override
public long getIdUpperBound(int idNamespace) {
return idUpperBound;
}
};
idAuthorities[0].setIDBlockSizer(blockSizer);
if (hasFixedUid) {
for (int i = 0; i < chunks; i++) {
idAuthorities[0].getIDBlock(0, 0, GET_ID_BLOCK_TIMEOUT);
}
try {
idAuthorities[0].getIDBlock(0, 0, GET_ID_BLOCK_TIMEOUT);
Assert.fail();
} catch (IDPoolExhaustedException ignored) {
}
} else {
for (int i = 0; i < (chunks * Math.max(1, (1 << uidBitWidth) / 10)); i++) {
idAuthorities[0].getIDBlock(0, 0, GET_ID_BLOCK_TIMEOUT);
}
try {
for (int i = 0; i < (chunks * Math.max(1, (1 << uidBitWidth) * 9 / 10)); i++) {
idAuthorities[0].getIDBlock(0, 0, GET_ID_BLOCK_TIMEOUT);
}
Assert.fail();
} catch (IDPoolExhaustedException ignored) {
}
}
}
Aggregations