use of java.util.concurrent.atomic.AtomicLongArray in project flink by apache.
the class JobManagerHACheckpointRecoveryITCase method testCheckpointedStreamingSumProgram.
/**
* Simple checkpointed streaming sum.
*
* <p>The sources (Parallelism) count until sequenceEnd. The sink (1) sums up all counts and
* returns it to the main thread via a static variable. We wait until some checkpoints are
* completed and sanity check that the sources recover with an updated state to make sure that
* this test actually tests something.
*/
@Test
@RetryOnFailure(times = 1)
public void testCheckpointedStreamingSumProgram() throws Exception {
// Config
final int checkpointingInterval = 200;
final int sequenceEnd = 5000;
final long expectedSum = Parallelism * sequenceEnd * (sequenceEnd + 1) / 2;
final StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
env.setParallelism(Parallelism);
env.enableCheckpointing(checkpointingInterval);
env.addSource(new CheckpointedSequenceSource(sequenceEnd)).addSink(new CountingSink()).setParallelism(1);
JobGraph jobGraph = env.getStreamGraph().getJobGraph();
Configuration config = ZooKeeperTestUtils.createZooKeeperHAConfig(ZooKeeper.getConnectString(), FileStateBackendBasePath.getAbsoluteFile().toURI().toString());
config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, Parallelism);
ActorSystem testSystem = null;
final JobManagerProcess[] jobManagerProcess = new JobManagerProcess[2];
LeaderRetrievalService leaderRetrievalService = null;
ActorSystem taskManagerSystem = null;
try {
final Deadline deadline = TestTimeOut.fromNow();
// Test actor system
testSystem = AkkaUtils.createActorSystem(new Configuration(), new Some<>(new Tuple2<String, Object>("localhost", 0)));
// The job managers
jobManagerProcess[0] = new JobManagerProcess(0, config);
jobManagerProcess[1] = new JobManagerProcess(1, config);
jobManagerProcess[0].startProcess();
jobManagerProcess[1].startProcess();
// Leader listener
TestingListener leaderListener = new TestingListener();
leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(config);
leaderRetrievalService.start(leaderListener);
// The task manager
taskManagerSystem = AkkaUtils.createActorSystem(config, Option.apply(new Tuple2<String, Object>("localhost", 0)));
TaskManager.startTaskManagerComponentsAndActor(config, ResourceID.generate(), taskManagerSystem, "localhost", Option.<String>empty(), Option.<LeaderRetrievalService>empty(), false, TaskManager.class);
{
// Initial submission
leaderListener.waitForNewLeader(deadline.timeLeft().toMillis());
String leaderAddress = leaderListener.getAddress();
UUID leaderId = leaderListener.getLeaderSessionID();
// Get the leader ref
ActorRef leaderRef = AkkaUtils.getActorRef(leaderAddress, testSystem, deadline.timeLeft());
ActorGateway leader = new AkkaActorGateway(leaderRef, leaderId);
// Submit the job in detached mode
leader.tell(new SubmitJob(jobGraph, ListeningBehaviour.DETACHED));
JobManagerActorTestUtils.waitForJobStatus(jobGraph.getJobID(), JobStatus.RUNNING, leader, deadline.timeLeft());
}
// Who's the boss?
JobManagerProcess leadingJobManagerProcess;
if (jobManagerProcess[0].getJobManagerAkkaURL(deadline.timeLeft()).equals(leaderListener.getAddress())) {
leadingJobManagerProcess = jobManagerProcess[0];
} else {
leadingJobManagerProcess = jobManagerProcess[1];
}
CompletedCheckpointsLatch.await();
// Kill the leading job manager process
leadingJobManagerProcess.destroy();
{
// Recovery by the standby JobManager
leaderListener.waitForNewLeader(deadline.timeLeft().toMillis());
String leaderAddress = leaderListener.getAddress();
UUID leaderId = leaderListener.getLeaderSessionID();
ActorRef leaderRef = AkkaUtils.getActorRef(leaderAddress, testSystem, deadline.timeLeft());
ActorGateway leader = new AkkaActorGateway(leaderRef, leaderId);
JobManagerActorTestUtils.waitForJobStatus(jobGraph.getJobID(), JobStatus.RUNNING, leader, deadline.timeLeft());
}
// Wait to finish
FinalCountLatch.await();
assertEquals(expectedSum, (long) FinalCount.get());
for (int i = 0; i < Parallelism; i++) {
assertNotEquals(0, RecoveredStates.get(i));
}
} catch (Throwable t) {
// Reset all static state for test retries
CompletedCheckpointsLatch = new CountDownLatch(2);
RecoveredStates = new AtomicLongArray(Parallelism);
FinalCountLatch = new CountDownLatch(1);
FinalCount = new AtomicReference<>();
LastElement = -1;
// Print early (in some situations the process logs get too big
// for Travis and the root problem is not shown)
t.printStackTrace();
// In case of an error, print the job manager process logs.
if (jobManagerProcess[0] != null) {
jobManagerProcess[0].printProcessLog();
}
if (jobManagerProcess[1] != null) {
jobManagerProcess[1].printProcessLog();
}
throw t;
} finally {
if (jobManagerProcess[0] != null) {
jobManagerProcess[0].destroy();
}
if (jobManagerProcess[1] != null) {
jobManagerProcess[1].destroy();
}
if (leaderRetrievalService != null) {
leaderRetrievalService.stop();
}
if (taskManagerSystem != null) {
taskManagerSystem.shutdown();
}
if (testSystem != null) {
testSystem.shutdown();
}
}
}
use of java.util.concurrent.atomic.AtomicLongArray in project fastjson by alibaba.
the class AtomicCodec method deserialze.
@SuppressWarnings("unchecked")
public <T> T deserialze(DefaultJSONParser parser, Type clazz, Object fieldName) {
if (parser.lexer.token() == JSONToken.NULL) {
parser.lexer.nextToken(JSONToken.COMMA);
return null;
}
JSONArray array = new JSONArray();
parser.parseArray(array);
if (clazz == AtomicIntegerArray.class) {
AtomicIntegerArray atomicArray = new AtomicIntegerArray(array.size());
for (int i = 0; i < array.size(); ++i) {
atomicArray.set(i, array.getInteger(i));
}
return (T) atomicArray;
}
AtomicLongArray atomicArray = new AtomicLongArray(array.size());
for (int i = 0; i < array.size(); ++i) {
atomicArray.set(i, array.getLong(i));
}
return (T) atomicArray;
}
use of java.util.concurrent.atomic.AtomicLongArray in project voltdb by VoltDB.
the class AtomicDoubleArray method readObject.
/**
* Reconstitutes the instance from a stream (that is, deserializes it).
*/
private void readObject(java.io.ObjectInputStream s) throws java.io.IOException, ClassNotFoundException {
s.defaultReadObject();
// Read in array length and allocate array
int length = s.readInt();
this.longs = new AtomicLongArray(length);
// Read in all elements in the proper order.
for (int i = 0; i < length; i++) {
set(i, s.readDouble());
}
}
use of java.util.concurrent.atomic.AtomicLongArray in project voltdb by VoltDB.
the class ConcurrentHistogram method resize.
@Override
void resize(long newHighestTrackableValue) {
try {
wrp.readerLock();
assert (countsArrayLength == activeCounts.length());
assert (countsArrayLength == inactiveCounts.length());
int newArrayLength = determineArrayLengthNeeded(newHighestTrackableValue);
int countsDelta = newArrayLength - countsArrayLength;
if (countsDelta <= 0) {
// This resize need was already covered by a concurrent resize op.
return;
}
int oldNormalizedZeroIndex = normalizeIndex(0, inactiveCounts.getNormalizingIndexOffset(), inactiveCounts.length());
// Resize the current inactiveCounts:
AtomicLongArray oldInactiveCounts = inactiveCounts;
inactiveCounts = new AtomicLongArrayWithNormalizingOffset(newArrayLength, inactiveCounts.getNormalizingIndexOffset());
// Copy inactive contents to newly sized inactiveCounts:
for (int i = 0; i < oldInactiveCounts.length(); i++) {
inactiveCounts.lazySet(i, oldInactiveCounts.get(i));
}
if (oldNormalizedZeroIndex != 0) {
// We need to shift the stuff from the zero index and up to the end of the array:
int newNormalizedZeroIndex = oldNormalizedZeroIndex + countsDelta;
int lengthToCopy = (newArrayLength - countsDelta) - oldNormalizedZeroIndex;
int src, dst;
for (src = oldNormalizedZeroIndex, dst = newNormalizedZeroIndex; src < oldNormalizedZeroIndex + lengthToCopy; src++, dst++) {
inactiveCounts.lazySet(dst, oldInactiveCounts.get(src));
}
for (dst = oldNormalizedZeroIndex; dst < newNormalizedZeroIndex; dst++) {
inactiveCounts.lazySet(dst, 0);
}
}
// switch active and inactive:
AtomicLongArrayWithNormalizingOffset tmp = activeCounts;
activeCounts = inactiveCounts;
inactiveCounts = tmp;
wrp.flipPhase();
// Resize the newly inactiveCounts:
oldInactiveCounts = inactiveCounts;
inactiveCounts = new AtomicLongArrayWithNormalizingOffset(newArrayLength, inactiveCounts.getNormalizingIndexOffset());
// Copy inactive contents to newly sized inactiveCounts:
for (int i = 0; i < oldInactiveCounts.length(); i++) {
inactiveCounts.lazySet(i, oldInactiveCounts.get(i));
}
if (oldNormalizedZeroIndex != 0) {
// We need to shift the stuff from the zero index and up to the end of the array:
int newNormalizedZeroIndex = oldNormalizedZeroIndex + countsDelta;
int lengthToCopy = (newArrayLength - countsDelta) - oldNormalizedZeroIndex;
int src, dst;
for (src = oldNormalizedZeroIndex, dst = newNormalizedZeroIndex; src < oldNormalizedZeroIndex + lengthToCopy; src++, dst++) {
inactiveCounts.lazySet(dst, oldInactiveCounts.get(src));
}
for (dst = oldNormalizedZeroIndex; dst < newNormalizedZeroIndex; dst++) {
inactiveCounts.lazySet(dst, 0);
}
}
// switch active and inactive again:
tmp = activeCounts;
activeCounts = inactiveCounts;
inactiveCounts = tmp;
wrp.flipPhase();
// At this point, both active and inactive have been safely resized,
// and the switch in each was done without any writers modifying it in flight.
// We resized things. We can now make the historam establish size accordingly for future recordings:
establishSize(newHighestTrackableValue);
assert (countsArrayLength == activeCounts.length());
assert (countsArrayLength == inactiveCounts.length());
} finally {
wrp.readerUnlock();
}
}
use of java.util.concurrent.atomic.AtomicLongArray in project ignite by apache.
the class HadoopConcurrentHashMultimap method rehashIfNeeded.
/**
* @param fromTbl Table.
*/
private void rehashIfNeeded(AtomicLongArray fromTbl) {
if (fromTbl.length() == Integer.MAX_VALUE)
return;
long keys0 = keys();
if (// New size has to be >= than 3/4 of capacity to rehash.
keys0 < 3 * (fromTbl.length() >>> 2))
return;
if (// Check if someone else have done the job.
fromTbl != newTbl)
return;
if (!state.compareAndSet(State.READING_WRITING, State.REHASHING)) {
// Visiting is allowed, but we will not rehash.
assert state.get() != State.CLOSING;
return;
}
if (fromTbl != newTbl) {
// Double check.
// Switch back.
state(State.REHASHING, State.READING_WRITING);
return;
}
// Calculate new table capacity.
int newLen = fromTbl.length();
do {
newLen <<= 1;
} while (newLen < keys0);
if (// Still more than 3/4.
keys0 >= 3 * (newLen >>> 2))
newLen <<= 1;
// This is our target table for rehashing.
AtomicLongArray toTbl = new AtomicLongArray(newLen);
// Make the new table visible before rehashing.
newTbl = toTbl;
// Rehash.
int newMask = newLen - 1;
long failedMeta = 0;
GridLongList collisions = new GridLongList(16);
for (int i = 0; i < fromTbl.length(); i++) {
// Scan source table.
long meta = fromTbl.get(i);
assert meta != -1;
if (meta == 0) {
// No entry.
failedMeta = 0;
if (// Mark as moved.
!fromTbl.compareAndSet(i, 0, -1))
// Retry.
i--;
continue;
}
do {
// Collect all the collisions before the last one failed to nullify or 0.
collisions.add(meta);
meta = collision(meta);
} while (meta != failedMeta);
do {
// Go from the last to the first to avoid 'in-flight' state for meta entries.
meta = collisions.remove();
int addr = keyHash(meta) & newMask;
for (; ; ) {
// Move meta entry to the new table.
long toCollision = toTbl.get(addr);
collision(meta, toCollision);
if (toTbl.compareAndSet(addr, toCollision, meta))
break;
}
} while (!collisions.isEmpty());
// Here 'meta' will be a root pointer in old table.
if (!fromTbl.compareAndSet(i, meta, -1)) {
// Try to mark as moved.
failedMeta = meta;
// Retry the same address in table because new keys were added.
i--;
} else
failedMeta = 0;
}
// Now old and new tables will be the same again.
oldTbl = toTbl;
state(State.REHASHING, State.READING_WRITING);
}
Aggregations