Examples with AtomicLong java.util.concurrent.atomic.AtomicLong used on opensource projects

Example 11 with AtomicLong

use of java.util.concurrent.atomic.AtomicLong in project hadoop by apache.

the class LegacyTaskRuntimeEstimator method updateAttempt.

@Override
public void updateAttempt(TaskAttemptStatus status, long timestamp) {
    super.updateAttempt(status, timestamp);
    TaskAttemptId attemptID = status.id;
    TaskId taskID = attemptID.getTaskId();
    JobId jobID = taskID.getJobId();
    Job job = context.getJob(jobID);
    if (job == null) {
        return;
    }
    Task task = job.getTask(taskID);
    if (task == null) {
        return;
    }
    TaskAttempt taskAttempt = task.getAttempt(attemptID);
    if (taskAttempt == null) {
        return;
    }
    Long boxedStart = startTimes.get(attemptID);
    long start = boxedStart == null ? Long.MIN_VALUE : boxedStart;
    //
    if (taskAttempt.getState() == TaskAttemptState.RUNNING) {
        // See if this task is already in the registry
        AtomicLong estimateContainer = attemptRuntimeEstimates.get(taskAttempt);
        AtomicLong estimateVarianceContainer = attemptRuntimeEstimateVariances.get(taskAttempt);
        if (estimateContainer == null) {
            if (attemptRuntimeEstimates.get(taskAttempt) == null) {
                attemptRuntimeEstimates.put(taskAttempt, new AtomicLong());
                estimateContainer = attemptRuntimeEstimates.get(taskAttempt);
            }
        }
        if (estimateVarianceContainer == null) {
            attemptRuntimeEstimateVariances.putIfAbsent(taskAttempt, new AtomicLong());
            estimateVarianceContainer = attemptRuntimeEstimateVariances.get(taskAttempt);
        }
        long estimate = -1;
        long varianceEstimate = -1;
        //  speculative task attempt if two are already running for this task
        if (start > 0 && timestamp > start) {
            estimate = (long) ((timestamp - start) / Math.max(0.0001, status.progress));
            varianceEstimate = (long) (estimate * status.progress / 10);
        }
        if (estimateContainer != null) {
            estimateContainer.set(estimate);
        }
        if (estimateVarianceContainer != null) {
            estimateVarianceContainer.set(varianceEstimate);
        }
    }
}

Example 12 with AtomicLong

use of java.util.concurrent.atomic.AtomicLong in project hadoop by apache.

the class StepTracking method clone.

@Override
public StepTracking clone() {
    StepTracking clone = new StepTracking();
    super.copy(clone);
    clone.count = new AtomicLong(count.get());
    clone.total = total;
    return clone;
}

Example 13 with AtomicLong

use of java.util.concurrent.atomic.AtomicLong in project hadoop by apache.

the class TestDirectoryScanner method testThrottling.

/**
   * Test that the timeslice throttle limits the report compiler thread's
   * execution time correctly.  We test by scanning a large block pool and
   * comparing the time spent waiting to the time spent running.
   *
   * The block pool has to be large, or the ratio will be off.  The throttle
   * allows the report compiler thread to finish its current cycle when
   * blocking it, so the ratio will always be a little lower than expected.
   * The smaller the block pool, the further off the ratio will be.
   *
   * @throws Exception thrown on unexpected failure
   */
@Test(timeout = 600000)
public void testThrottling() throws Exception {
    Configuration conf = new Configuration(CONF);
    // We need lots of blocks so the report compiler threads have enough to
    // keep them busy while we watch them.
    int blocks = 20000;
    int maxRetries = 3;
    cluster = new MiniDFSCluster.Builder(conf).build();
    try {
        cluster.waitActive();
        bpid = cluster.getNamesystem().getBlockPoolId();
        fds = DataNodeTestUtils.getFSDataset(cluster.getDataNodes().get(0));
        client = cluster.getFileSystem().getClient();
        conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_THROTTLE_LIMIT_MS_PER_SEC_KEY, 100);
        DataNode dataNode = cluster.getDataNodes().get(0);
        final int maxBlocksPerFile = (int) DFSConfigKeys.DFS_NAMENODE_MAX_BLOCKS_PER_FILE_DEFAULT;
        int numBlocksToCreate = blocks;
        while (numBlocksToCreate > 0) {
            final int toCreate = Math.min(maxBlocksPerFile, numBlocksToCreate);
            createFile(GenericTestUtils.getMethodName() + numBlocksToCreate, BLOCK_LENGTH * toCreate, false);
            numBlocksToCreate -= toCreate;
        }
        float ratio = 0.0f;
        int retries = maxRetries;
        while ((retries > 0) && ((ratio < 7f) || (ratio > 10f))) {
            scanner = new DirectoryScanner(dataNode, fds, conf);
            ratio = runThrottleTest(blocks);
            retries -= 1;
        }
        // Waiting should be about 9x running.
        LOG.info("RATIO: " + ratio);
        assertTrue("Throttle is too restrictive", ratio <= 10f);
        assertTrue("Throttle is too permissive", ratio >= 7f);
        // Test with a different limit
        conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_THROTTLE_LIMIT_MS_PER_SEC_KEY, 200);
        ratio = 0.0f;
        retries = maxRetries;
        while ((retries > 0) && ((ratio < 2.75f) || (ratio > 4.5f))) {
            scanner = new DirectoryScanner(dataNode, fds, conf);
            ratio = runThrottleTest(blocks);
            retries -= 1;
        }
        // Waiting should be about 4x running.
        LOG.info("RATIO: " + ratio);
        assertTrue("Throttle is too restrictive", ratio <= 4.5f);
        assertTrue("Throttle is too permissive", ratio >= 2.75f);
        // Test with more than 1 thread
        conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_THREADS_KEY, 3);
        conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_THROTTLE_LIMIT_MS_PER_SEC_KEY, 100);
        ratio = 0.0f;
        retries = maxRetries;
        while ((retries > 0) && ((ratio < 7f) || (ratio > 10f))) {
            scanner = new DirectoryScanner(dataNode, fds, conf);
            ratio = runThrottleTest(blocks);
            retries -= 1;
        }
        // Waiting should be about 9x running.
        LOG.info("RATIO: " + ratio);
        assertTrue("Throttle is too restrictive", ratio <= 10f);
        assertTrue("Throttle is too permissive", ratio >= 7f);
        // Test with no limit
        scanner = new DirectoryScanner(dataNode, fds, CONF);
        scanner.setRetainDiffs(true);
        scan(blocks, 0, 0, 0, 0, 0);
        scanner.shutdown();
        assertFalse(scanner.getRunStatus());
        assertTrue("Throttle appears to be engaged", scanner.timeWaitingMs.get() < 10L);
        assertTrue("Report complier threads logged no execution time", scanner.timeRunningMs.get() > 0L);
        // Test with a 1ms limit.  This also tests whether the scanner can be
        // shutdown cleanly in mid stride.
        conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_THROTTLE_LIMIT_MS_PER_SEC_KEY, 1);
        ratio = 0.0f;
        retries = maxRetries;
        ScheduledExecutorService interruptor = Executors.newScheduledThreadPool(maxRetries);
        try {
            while ((retries > 0) && (ratio < 10)) {
                scanner = new DirectoryScanner(dataNode, fds, conf);
                scanner.setRetainDiffs(true);
                final AtomicLong nowMs = new AtomicLong();
                // Stop the scanner after 2 seconds because otherwise it will take an
                // eternity to complete it's run
                interruptor.schedule(new Runnable() {

                    @Override
                    public void run() {
                        nowMs.set(Time.monotonicNow());
                        scanner.shutdown();
                    }
                }, 2L, TimeUnit.SECONDS);
                scanner.reconcile();
                assertFalse(scanner.getRunStatus());
                long finalMs = nowMs.get();
                // that the shutdown was timely
                if (finalMs > 0) {
                    LOG.info("Scanner took " + (Time.monotonicNow() - finalMs) + "ms to shutdown");
                    assertTrue("Scanner took too long to shutdown", Time.monotonicNow() - finalMs < 1000L);
                }
                ratio = (float) scanner.timeWaitingMs.get() / scanner.timeRunningMs.get();
                retries -= 1;
            }
        } finally {
            interruptor.shutdown();
        }
        // We just want to test that it waits a lot, but it also runs some
        LOG.info("RATIO: " + ratio);
        assertTrue("Throttle is too permissive", ratio > 10);
        assertTrue("Report complier threads logged no execution time", scanner.timeRunningMs.get() > 0L);
        // Test with a 0 limit, i.e. disabled
        conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_THROTTLE_LIMIT_MS_PER_SEC_KEY, 0);
        scanner = new DirectoryScanner(dataNode, fds, conf);
        scanner.setRetainDiffs(true);
        scan(blocks, 0, 0, 0, 0, 0);
        scanner.shutdown();
        assertFalse(scanner.getRunStatus());
        assertTrue("Throttle appears to be engaged", scanner.timeWaitingMs.get() < 10L);
        assertTrue("Report complier threads logged no execution time", scanner.timeRunningMs.get() > 0L);
        // Test with a 1000 limit, i.e. disabled
        conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_THROTTLE_LIMIT_MS_PER_SEC_KEY, 1000);
        scanner = new DirectoryScanner(dataNode, fds, conf);
        scanner.setRetainDiffs(true);
        scan(blocks, 0, 0, 0, 0, 0);
        scanner.shutdown();
        assertFalse(scanner.getRunStatus());
        assertTrue("Throttle appears to be engaged", scanner.timeWaitingMs.get() < 10L);
        assertTrue("Report complier threads logged no execution time", scanner.timeRunningMs.get() > 0L);
        // Test that throttle works from regular start
        conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_THREADS_KEY, 1);
        conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_THROTTLE_LIMIT_MS_PER_SEC_KEY, 10);
        conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 1);
        scanner = new DirectoryScanner(dataNode, fds, conf);
        scanner.setRetainDiffs(true);
        scanner.start();
        int count = 50;
        while ((count > 0) && (scanner.timeWaitingMs.get() < 500L)) {
            Thread.sleep(100L);
            count -= 1;
        }
        scanner.shutdown();
        assertFalse(scanner.getRunStatus());
        assertTrue("Throttle does not appear to be engaged", count > 0);
    } finally {
        cluster.shutdown();
    }
}

Example 14 with AtomicLong

use of java.util.concurrent.atomic.AtomicLong in project hbase by apache.

the class SnapshotInfo method getSnapshotFilesMap.

/**
   * Gets the store files map for snapshot
   * @param conf the {@link Configuration} to use
   * @param snapshot {@link SnapshotDescription} to get stats from
   * @param exec the {@link ExecutorService} to use
   * @param filesMap {@link Map} the map to put the mapping entries
   * @param uniqueHFilesArchiveSize {@link AtomicLong} the accumulated store file size in archive
   * @param uniqueHFilesSize {@link AtomicLong} the accumulated store file size shared
   * @param uniqueHFilesMobSize {@link AtomicLong} the accumulated mob store file size shared
   * @return the snapshot stats
   */
private static void getSnapshotFilesMap(final Configuration conf, final SnapshotDescription snapshot, final ExecutorService exec, final ConcurrentHashMap<Path, Integer> filesMap, final AtomicLong uniqueHFilesArchiveSize, final AtomicLong uniqueHFilesSize, final AtomicLong uniqueHFilesMobSize) throws IOException {
    HBaseProtos.SnapshotDescription snapshotDesc = ProtobufUtil.createHBaseProtosSnapshotDesc(snapshot);
    Path rootDir = FSUtils.getRootDir(conf);
    final FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotDesc, rootDir);
    SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
    SnapshotReferenceUtil.concurrentVisitReferencedFiles(conf, fs, manifest, exec, new SnapshotReferenceUtil.SnapshotVisitor() {

        @Override
        public void storeFile(final HRegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
            if (!storeFile.hasReference()) {
                HFileLink link = HFileLink.build(conf, snapshot.getTableName(), regionInfo.getEncodedName(), family, storeFile.getName());
                long size;
                Integer count;
                Path p;
                AtomicLong al;
                int c = 0;
                if (fs.exists(link.getArchivePath())) {
                    p = link.getArchivePath();
                    al = uniqueHFilesArchiveSize;
                    size = fs.getFileStatus(p).getLen();
                } else if (fs.exists(link.getMobPath())) {
                    p = link.getMobPath();
                    al = uniqueHFilesMobSize;
                    size = fs.getFileStatus(p).getLen();
                } else {
                    p = link.getOriginPath();
                    al = uniqueHFilesSize;
                    size = link.getFileStatus(fs).getLen();
                }
                // If it has been counted, do not double count
                count = filesMap.get(p);
                if (count != null) {
                    c = count.intValue();
                } else {
                    al.addAndGet(size);
                }
                filesMap.put(p, ++c);
            }
        }
    });
}

Example 15 with AtomicLong

use of java.util.concurrent.atomic.AtomicLong in project hbase by apache.

the class AggregationClient method rowCount.

/**
   * It gives the row count, by summing up the individual results obtained from
   * regions. In case the qualifier is null, FirstKeyValueFilter is used to
   * optimised the operation. In case qualifier is provided, I can't use the
   * filter as it may set the flag to skip to next row, but the value read is
   * not of the given filter: in this case, this particular row will not be
   * counted ==> an error.
   * @param table
   * @param ci
   * @param scan
   * @return <R, S>
   * @throws Throwable
   */
public <R, S, P extends Message, Q extends Message, T extends Message> long rowCount(final Table table, final ColumnInterpreter<R, S, P, Q, T> ci, final Scan scan) throws Throwable {
    final AggregateRequest requestArg = validateArgAndGetPB(scan, ci, true);
    class RowNumCallback implements Batch.Callback<Long> {

        private final AtomicLong rowCountL = new AtomicLong(0);

        public long getRowNumCount() {
            return rowCountL.get();
        }

        @Override
        public void update(byte[] region, byte[] row, Long result) {
            rowCountL.addAndGet(result.longValue());
        }
    }
    RowNumCallback rowNum = new RowNumCallback();
    table.coprocessorService(AggregateService.class, scan.getStartRow(), scan.getStopRow(), new Batch.Call<AggregateService, Long>() {

        @Override
        public Long call(AggregateService instance) throws IOException {
            RpcController controller = new AggregationClientRpcController();
            CoprocessorRpcUtils.BlockingRpcCallback<AggregateResponse> rpcCallback = new CoprocessorRpcUtils.BlockingRpcCallback<>();
            instance.getRowNum(controller, requestArg, rpcCallback);
            AggregateResponse response = rpcCallback.get();
            if (controller.failed()) {
                throw new IOException(controller.errorText());
            }
            byte[] bytes = getBytesFromResponse(response.getFirstPart(0));
            ByteBuffer bb = ByteBuffer.allocate(8).put(bytes);
            bb.rewind();
            return bb.getLong();
        }
    }, rowNum);
    return rowNum.getRowNumCount();
}