Examples with AtomicLongArray java.util.concurrent.atomic.AtomicLongArray used on opensource projects

Example 1 with AtomicLongArray

use of java.util.concurrent.atomic.AtomicLongArray in project flink by apache.

the class JobManagerHACheckpointRecoveryITCase method testCheckpointedStreamingSumProgram.

/**
	 * Simple checkpointed streaming sum.
	 *
	 * <p>The sources (Parallelism) count until sequenceEnd. The sink (1) sums up all counts and
	 * returns it to the main thread via a static variable. We wait until some checkpoints are
	 * completed and sanity check that the sources recover with an updated state to make sure that
	 * this test actually tests something.
	 */
@Test
@RetryOnFailure(times = 1)
public void testCheckpointedStreamingSumProgram() throws Exception {
    // Config
    final int checkpointingInterval = 200;
    final int sequenceEnd = 5000;
    final long expectedSum = Parallelism * sequenceEnd * (sequenceEnd + 1) / 2;
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
    env.setParallelism(Parallelism);
    env.enableCheckpointing(checkpointingInterval);
    env.addSource(new CheckpointedSequenceSource(sequenceEnd)).addSink(new CountingSink()).setParallelism(1);
    JobGraph jobGraph = env.getStreamGraph().getJobGraph();
    Configuration config = ZooKeeperTestUtils.createZooKeeperHAConfig(ZooKeeper.getConnectString(), FileStateBackendBasePath.getAbsoluteFile().toURI().toString());
    config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, Parallelism);
    ActorSystem testSystem = null;
    final JobManagerProcess[] jobManagerProcess = new JobManagerProcess[2];
    LeaderRetrievalService leaderRetrievalService = null;
    ActorSystem taskManagerSystem = null;
    try {
        final Deadline deadline = TestTimeOut.fromNow();
        // Test actor system
        testSystem = AkkaUtils.createActorSystem(new Configuration(), new Some<>(new Tuple2<String, Object>("localhost", 0)));
        // The job managers
        jobManagerProcess[0] = new JobManagerProcess(0, config);
        jobManagerProcess[1] = new JobManagerProcess(1, config);
        jobManagerProcess[0].startProcess();
        jobManagerProcess[1].startProcess();
        // Leader listener
        TestingListener leaderListener = new TestingListener();
        leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(config);
        leaderRetrievalService.start(leaderListener);
        // The task manager
        taskManagerSystem = AkkaUtils.createActorSystem(config, Option.apply(new Tuple2<String, Object>("localhost", 0)));
        TaskManager.startTaskManagerComponentsAndActor(config, ResourceID.generate(), taskManagerSystem, "localhost", Option.<String>empty(), Option.<LeaderRetrievalService>empty(), false, TaskManager.class);
        {
            // Initial submission
            leaderListener.waitForNewLeader(deadline.timeLeft().toMillis());
            String leaderAddress = leaderListener.getAddress();
            UUID leaderId = leaderListener.getLeaderSessionID();
            // Get the leader ref
            ActorRef leaderRef = AkkaUtils.getActorRef(leaderAddress, testSystem, deadline.timeLeft());
            ActorGateway leader = new AkkaActorGateway(leaderRef, leaderId);
            // Submit the job in detached mode
            leader.tell(new SubmitJob(jobGraph, ListeningBehaviour.DETACHED));
            JobManagerActorTestUtils.waitForJobStatus(jobGraph.getJobID(), JobStatus.RUNNING, leader, deadline.timeLeft());
        }
        // Who's the boss?
        JobManagerProcess leadingJobManagerProcess;
        if (jobManagerProcess[0].getJobManagerAkkaURL(deadline.timeLeft()).equals(leaderListener.getAddress())) {
            leadingJobManagerProcess = jobManagerProcess[0];
        } else {
            leadingJobManagerProcess = jobManagerProcess[1];
        }
        CompletedCheckpointsLatch.await();
        // Kill the leading job manager process
        leadingJobManagerProcess.destroy();
        {
            // Recovery by the standby JobManager
            leaderListener.waitForNewLeader(deadline.timeLeft().toMillis());
            String leaderAddress = leaderListener.getAddress();
            UUID leaderId = leaderListener.getLeaderSessionID();
            ActorRef leaderRef = AkkaUtils.getActorRef(leaderAddress, testSystem, deadline.timeLeft());
            ActorGateway leader = new AkkaActorGateway(leaderRef, leaderId);
            JobManagerActorTestUtils.waitForJobStatus(jobGraph.getJobID(), JobStatus.RUNNING, leader, deadline.timeLeft());
        }
        // Wait to finish
        FinalCountLatch.await();
        assertEquals(expectedSum, (long) FinalCount.get());
        for (int i = 0; i < Parallelism; i++) {
            assertNotEquals(0, RecoveredStates.get(i));
        }
    } catch (Throwable t) {
        // Reset all static state for test retries
        CompletedCheckpointsLatch = new CountDownLatch(2);
        RecoveredStates = new AtomicLongArray(Parallelism);
        FinalCountLatch = new CountDownLatch(1);
        FinalCount = new AtomicReference<>();
        LastElement = -1;
        // Print early (in some situations the process logs get too big
        // for Travis and the root problem is not shown)
        t.printStackTrace();
        // In case of an error, print the job manager process logs.
        if (jobManagerProcess[0] != null) {
            jobManagerProcess[0].printProcessLog();
        }
        if (jobManagerProcess[1] != null) {
            jobManagerProcess[1].printProcessLog();
        }
        throw t;
    } finally {
        if (jobManagerProcess[0] != null) {
            jobManagerProcess[0].destroy();
        }
        if (jobManagerProcess[1] != null) {
            jobManagerProcess[1].destroy();
        }
        if (leaderRetrievalService != null) {
            leaderRetrievalService.stop();
        }
        if (taskManagerSystem != null) {
            taskManagerSystem.shutdown();
        }
        if (testSystem != null) {
            testSystem.shutdown();
        }
    }
}

Example 2 with AtomicLongArray

use of java.util.concurrent.atomic.AtomicLongArray in project fastjson by alibaba.

the class AtomicCodec method deserialze.

@SuppressWarnings("unchecked")
public <T> T deserialze(DefaultJSONParser parser, Type clazz, Object fieldName) {
    if (parser.lexer.token() == JSONToken.NULL) {
        parser.lexer.nextToken(JSONToken.COMMA);
        return null;
    }
    JSONArray array = new JSONArray();
    parser.parseArray(array);
    if (clazz == AtomicIntegerArray.class) {
        AtomicIntegerArray atomicArray = new AtomicIntegerArray(array.size());
        for (int i = 0; i < array.size(); ++i) {
            atomicArray.set(i, array.getInteger(i));
        }
        return (T) atomicArray;
    }
    AtomicLongArray atomicArray = new AtomicLongArray(array.size());
    for (int i = 0; i < array.size(); ++i) {
        atomicArray.set(i, array.getLong(i));
    }
    return (T) atomicArray;
}

Example 3 with AtomicLongArray

use of java.util.concurrent.atomic.AtomicLongArray in project voltdb by VoltDB.

the class AtomicDoubleArray method readObject.

/**
   * Reconstitutes the instance from a stream (that is, deserializes it).
   */
private void readObject(java.io.ObjectInputStream s) throws java.io.IOException, ClassNotFoundException {
    s.defaultReadObject();
    // Read in array length and allocate array
    int length = s.readInt();
    this.longs = new AtomicLongArray(length);
    // Read in all elements in the proper order.
    for (int i = 0; i < length; i++) {
        set(i, s.readDouble());
    }
}

Example 4 with AtomicLongArray

use of java.util.concurrent.atomic.AtomicLongArray in project voltdb by VoltDB.

the class ConcurrentHistogram method resize.

@Override
void resize(long newHighestTrackableValue) {
    try {
        wrp.readerLock();
        assert (countsArrayLength == activeCounts.length());
        assert (countsArrayLength == inactiveCounts.length());
        int newArrayLength = determineArrayLengthNeeded(newHighestTrackableValue);
        int countsDelta = newArrayLength - countsArrayLength;
        if (countsDelta <= 0) {
            // This resize need was already covered by a concurrent resize op.
            return;
        }
        int oldNormalizedZeroIndex = normalizeIndex(0, inactiveCounts.getNormalizingIndexOffset(), inactiveCounts.length());
        // Resize the current inactiveCounts:
        AtomicLongArray oldInactiveCounts = inactiveCounts;
        inactiveCounts = new AtomicLongArrayWithNormalizingOffset(newArrayLength, inactiveCounts.getNormalizingIndexOffset());
        // Copy inactive contents to newly sized inactiveCounts:
        for (int i = 0; i < oldInactiveCounts.length(); i++) {
            inactiveCounts.lazySet(i, oldInactiveCounts.get(i));
        }
        if (oldNormalizedZeroIndex != 0) {
            // We need to shift the stuff from the zero index and up to the end of the array:
            int newNormalizedZeroIndex = oldNormalizedZeroIndex + countsDelta;
            int lengthToCopy = (newArrayLength - countsDelta) - oldNormalizedZeroIndex;
            int src, dst;
            for (src = oldNormalizedZeroIndex, dst = newNormalizedZeroIndex; src < oldNormalizedZeroIndex + lengthToCopy; src++, dst++) {
                inactiveCounts.lazySet(dst, oldInactiveCounts.get(src));
            }
            for (dst = oldNormalizedZeroIndex; dst < newNormalizedZeroIndex; dst++) {
                inactiveCounts.lazySet(dst, 0);
            }
        }
        // switch active and inactive:
        AtomicLongArrayWithNormalizingOffset tmp = activeCounts;
        activeCounts = inactiveCounts;
        inactiveCounts = tmp;
        wrp.flipPhase();
        // Resize the newly inactiveCounts:
        oldInactiveCounts = inactiveCounts;
        inactiveCounts = new AtomicLongArrayWithNormalizingOffset(newArrayLength, inactiveCounts.getNormalizingIndexOffset());
        // Copy inactive contents to newly sized inactiveCounts:
        for (int i = 0; i < oldInactiveCounts.length(); i++) {
            inactiveCounts.lazySet(i, oldInactiveCounts.get(i));
        }
        if (oldNormalizedZeroIndex != 0) {
            // We need to shift the stuff from the zero index and up to the end of the array:
            int newNormalizedZeroIndex = oldNormalizedZeroIndex + countsDelta;
            int lengthToCopy = (newArrayLength - countsDelta) - oldNormalizedZeroIndex;
            int src, dst;
            for (src = oldNormalizedZeroIndex, dst = newNormalizedZeroIndex; src < oldNormalizedZeroIndex + lengthToCopy; src++, dst++) {
                inactiveCounts.lazySet(dst, oldInactiveCounts.get(src));
            }
            for (dst = oldNormalizedZeroIndex; dst < newNormalizedZeroIndex; dst++) {
                inactiveCounts.lazySet(dst, 0);
            }
        }
        // switch active and inactive again:
        tmp = activeCounts;
        activeCounts = inactiveCounts;
        inactiveCounts = tmp;
        wrp.flipPhase();
        // At this point, both active and inactive have been safely resized,
        // and the switch in each was done without any writers modifying it in flight.
        // We resized things. We can now make the historam establish size accordingly for future recordings:
        establishSize(newHighestTrackableValue);
        assert (countsArrayLength == activeCounts.length());
        assert (countsArrayLength == inactiveCounts.length());
    } finally {
        wrp.readerUnlock();
    }
}

Example 5 with AtomicLongArray

use of java.util.concurrent.atomic.AtomicLongArray in project ignite by apache.

the class HadoopConcurrentHashMultimap method rehashIfNeeded.

/**
     * @param fromTbl Table.
     */
private void rehashIfNeeded(AtomicLongArray fromTbl) {
    if (fromTbl.length() == Integer.MAX_VALUE)
        return;
    long keys0 = keys();
    if (// New size has to be >= than 3/4 of capacity to rehash.
    keys0 < 3 * (fromTbl.length() >>> 2))
        return;
    if (// Check if someone else have done the job.
    fromTbl != newTbl)
        return;
    if (!state.compareAndSet(State.READING_WRITING, State.REHASHING)) {
        // Visiting is allowed, but we will not rehash.
        assert state.get() != State.CLOSING;
        return;
    }
    if (fromTbl != newTbl) {
        // Double check.
        // Switch back.
        state(State.REHASHING, State.READING_WRITING);
        return;
    }
    // Calculate new table capacity.
    int newLen = fromTbl.length();
    do {
        newLen <<= 1;
    } while (newLen < keys0);
    if (// Still more than 3/4.
    keys0 >= 3 * (newLen >>> 2))
        newLen <<= 1;
    // This is our target table for rehashing.
    AtomicLongArray toTbl = new AtomicLongArray(newLen);
    // Make the new table visible before rehashing.
    newTbl = toTbl;
    // Rehash.
    int newMask = newLen - 1;
    long failedMeta = 0;
    GridLongList collisions = new GridLongList(16);
    for (int i = 0; i < fromTbl.length(); i++) {
        // Scan source table.
        long meta = fromTbl.get(i);
        assert meta != -1;
        if (meta == 0) {
            // No entry.
            failedMeta = 0;
            if (// Mark as moved.
            !fromTbl.compareAndSet(i, 0, -1))
                // Retry.
                i--;
            continue;
        }
        do {
            // Collect all the collisions before the last one failed to nullify or 0.
            collisions.add(meta);
            meta = collision(meta);
        } while (meta != failedMeta);
        do {
            // Go from the last to the first to avoid 'in-flight' state for meta entries.
            meta = collisions.remove();
            int addr = keyHash(meta) & newMask;
            for (; ; ) {
                // Move meta entry to the new table.
                long toCollision = toTbl.get(addr);
                collision(meta, toCollision);
                if (toTbl.compareAndSet(addr, toCollision, meta))
                    break;
            }
        } while (!collisions.isEmpty());
        // Here 'meta' will be a root pointer in old table.
        if (!fromTbl.compareAndSet(i, meta, -1)) {
            // Try to mark as moved.
            failedMeta = meta;
            // Retry the same address in table because new keys were added.
            i--;
        } else
            failedMeta = 0;
    }
    // Now old and new tables will be the same again.
    oldTbl = toTbl;
    state(State.REHASHING, State.READING_WRITING);
}