Examples with MutableInt org.apache.commons.lang.mutable.MutableInt used on opensource projects

Example 1 with MutableInt

use of org.apache.commons.lang.mutable.MutableInt in project hive by apache.

the class LlapTaskSchedulerService method schedulePendingTasks.

@VisibleForTesting
protected void schedulePendingTasks() {
    writeLock.lock();
    try {
        if (LOG.isDebugEnabled()) {
            LOG.debug("ScheduleRun: {}", constructPendingTaskCountsLogMessage());
        }
        Iterator<Entry<Priority, List<TaskInfo>>> pendingIterator = pendingTasks.entrySet().iterator();
        Resource totalResource = getTotalResources();
        while (pendingIterator.hasNext()) {
            Entry<Priority, List<TaskInfo>> entry = pendingIterator.next();
            List<TaskInfo> taskListAtPriority = entry.getValue();
            Iterator<TaskInfo> taskIter = taskListAtPriority.iterator();
            boolean scheduledAllAtPriority = true;
            while (taskIter.hasNext()) {
                // TODO Optimization: Add a check to see if there's any capacity available. No point in
                // walking through all active nodes, if they don't have potential capacity.
                TaskInfo taskInfo = taskIter.next();
                if (taskInfo.getNumPreviousAssignAttempts() == 1) {
                    dagStats.registerDelayedAllocation();
                }
                taskInfo.triedAssigningTask();
                ScheduleResult scheduleResult = scheduleTask(taskInfo, totalResource);
                if (LOG.isDebugEnabled()) {
                    LOG.debug("ScheduleResult for Task: {} = {}", taskInfo, scheduleResult);
                }
                if (scheduleResult == ScheduleResult.SCHEDULED) {
                    taskIter.remove();
                } else {
                    if (scheduleResult == ScheduleResult.INADEQUATE_TOTAL_RESOURCES) {
                        LOG.info("Inadequate total resources before scheduling pending tasks." + " Signalling scheduler timeout monitor thread to start timer.");
                        startTimeoutMonitor();
                    // TODO Nothing else should be done for this task. Move on.
                    }
                    // Try pre-empting a task so that a higher priority task can take it's place.
                    // Preempt only if there's no pending preemptions to avoid preempting twice for a task.
                    String[] potentialHosts;
                    if (scheduleResult == ScheduleResult.DELAYED_LOCALITY) {
                        // Add the task to the delayed task queue if it does not already exist.
                        maybeAddToDelayedTaskQueue(taskInfo);
                        // Try preempting a lower priority task in any case.
                        // preempt only on specific hosts, if no preemptions already exist on those.
                        potentialHosts = taskInfo.requestedHosts;
                        //Protect against a bad location being requested.
                        if (potentialHosts == null || potentialHosts.length == 0) {
                            potentialHosts = null;
                        }
                    } else {
                        // preempt on any host.
                        potentialHosts = null;
                    }
                    // At this point we're dealing with all return types, except ScheduleResult.SCHEDULED.
                    if (potentialHosts != null) {
                        if (LOG.isDebugEnabled()) {
                            LOG.debug("Attempting to preempt on requested host for task={}, potentialHosts={}", taskInfo, Arrays.toString(potentialHosts));
                        }
                        // Preempt on specific host
                        boolean shouldPreempt = true;
                        for (String host : potentialHosts) {
                            // Preempt only if there are no pending preemptions on the same host
                            // When the premption registers, the request at the highest priority will be given the slot,
                            // even if the initial preemption was caused by some other task.
                            // TODO Maybe register which task the preemption was for, to avoid a bad non-local allocation.
                            MutableInt pendingHostPreemptions = pendingPreemptionsPerHost.get(host);
                            if (pendingHostPreemptions != null && pendingHostPreemptions.intValue() > 0) {
                                shouldPreempt = false;
                                LOG.debug("Not preempting for task={}. Found an existing preemption request on host={}, pendingPreemptionCount={}", taskInfo.task, host, pendingHostPreemptions.intValue());
                                break;
                            }
                        }
                        if (shouldPreempt) {
                            if (LOG.isDebugEnabled()) {
                                LOG.debug("Attempting to preempt for {} on potential hosts={}. TotalPendingPreemptions={}", taskInfo.task, Arrays.toString(potentialHosts), pendingPreemptions.get());
                            }
                            preemptTasks(entry.getKey().getPriority(), 1, potentialHosts);
                        } else {
                            if (LOG.isDebugEnabled()) {
                                LOG.debug("Not preempting for {} on potential hosts={}. An existing preemption request exists", taskInfo.task, Arrays.toString(potentialHosts));
                            }
                        }
                    } else {
                        // Either DELAYED_RESOURCES or DELAYED_LOCALITY with an unknown requested host.
                        // Request for a preemption if there's none pending. If a single preemption is pending,
                        // and this is the next task to be assigned, it will be assigned once that slot becomes available.
                        LOG.debug("Attempting to preempt on any host for task={}, pendingPreemptions={}", taskInfo.task, pendingPreemptions.get());
                        if (pendingPreemptions.get() == 0) {
                            if (LOG.isDebugEnabled()) {
                                LOG.debug("Attempting to preempt for task={}, priority={} on any available host", taskInfo.task, taskInfo.priority);
                            }
                            preemptTasks(entry.getKey().getPriority(), 1, null);
                        } else {
                            if (LOG.isDebugEnabled()) {
                                LOG.debug("Skipping preemption since there are {} pending preemption request. For task={}", pendingPreemptions.get(), taskInfo);
                            }
                        }
                    }
                    // Since there was an allocation failure - don't try assigning tasks at the next priority.
                    scheduledAllAtPriority = false;
                    // Don't break if this allocation failure was a result of a LOCALITY_DELAY. Others could still be allocated.
                    if (scheduleResult != ScheduleResult.DELAYED_LOCALITY) {
                        break;
                    }
                }
            // end of else - i.e. could not allocate
            }
            // end of loop over pending tasks
            if (taskListAtPriority.isEmpty()) {
                // Remove the entry, if there's nothing left at the specific priority level
                pendingIterator.remove();
            }
            if (!scheduledAllAtPriority) {
                LOG.debug("Unable to schedule all requests at priority={}. Skipping subsequent priority levels", entry.getKey());
                // Don't attempt scheduling for additional priorities
                break;
            }
        }
    } finally {
        writeLock.unlock();
    }
}

Example 2 with MutableInt

use of org.apache.commons.lang.mutable.MutableInt in project hive by apache.

the class LlapTaskSchedulerService method registerPendingPreemption.

private void registerPendingPreemption(String host) {
    writeLock.lock();
    try {
        pendingPreemptions.incrementAndGet();
        if (metrics != null) {
            metrics.incrPendingPreemptionTasksCount();
        }
        MutableInt val = pendingPreemptionsPerHost.get(host);
        if (val == null) {
            val = new MutableInt(0);
            pendingPreemptionsPerHost.put(host, val);
        }
        val.increment();
    } finally {
        writeLock.unlock();
    }
}

Example 3 with MutableInt

use of org.apache.commons.lang.mutable.MutableInt in project druid by druid-io.

the class RemoteTaskRunner method start.

@Override
@LifecycleStart
public void start() {
    try {
        if (started) {
            return;
        }
        final MutableInt waitingFor = new MutableInt(1);
        final Object waitingForMonitor = new Object();
        // Add listener for creation/deletion of workers
        workerPathCache.getListenable().addListener(new PathChildrenCacheListener() {

            @Override
            public void childEvent(CuratorFramework client, final PathChildrenCacheEvent event) throws Exception {
                final Worker worker;
                switch(event.getType()) {
                    case CHILD_ADDED:
                        worker = jsonMapper.readValue(event.getData().getData(), Worker.class);
                        synchronized (waitingForMonitor) {
                            waitingFor.increment();
                        }
                        Futures.addCallback(addWorker(worker), new FutureCallback<ZkWorker>() {

                            @Override
                            public void onSuccess(ZkWorker zkWorker) {
                                synchronized (waitingForMonitor) {
                                    waitingFor.decrement();
                                    waitingForMonitor.notifyAll();
                                }
                            }

                            @Override
                            public void onFailure(Throwable throwable) {
                                synchronized (waitingForMonitor) {
                                    waitingFor.decrement();
                                    waitingForMonitor.notifyAll();
                                }
                            }
                        });
                        break;
                    case CHILD_UPDATED:
                        worker = jsonMapper.readValue(event.getData().getData(), Worker.class);
                        updateWorker(worker);
                        break;
                    case CHILD_REMOVED:
                        worker = jsonMapper.readValue(event.getData().getData(), Worker.class);
                        removeWorker(worker);
                        break;
                    case INITIALIZED:
                        // Schedule cleanup for task status of the workers that might have disconnected while overlord was not running
                        List<String> workers;
                        try {
                            workers = cf.getChildren().forPath(indexerZkConfig.getStatusPath());
                        } catch (KeeperException.NoNodeException e) {
                            // statusPath doesn't exist yet; can occur if no middleManagers have started.
                            workers = ImmutableList.of();
                        }
                        for (String workerId : workers) {
                            final String workerAnnouncePath = JOINER.join(indexerZkConfig.getAnnouncementsPath(), workerId);
                            final String workerStatusPath = JOINER.join(indexerZkConfig.getStatusPath(), workerId);
                            if (!zkWorkers.containsKey(workerId) && cf.checkExists().forPath(workerAnnouncePath) == null) {
                                try {
                                    scheduleTasksCleanupForWorker(workerId, cf.getChildren().forPath(workerStatusPath));
                                } catch (Exception e) {
                                    log.warn(e, "Could not schedule cleanup for worker[%s] during startup (maybe someone removed the status znode[%s]?). Skipping.", workerId, workerStatusPath);
                                }
                            }
                        }
                        synchronized (waitingForMonitor) {
                            waitingFor.decrement();
                            waitingForMonitor.notifyAll();
                        }
                    default:
                        break;
                }
            }
        });
        workerPathCache.start(PathChildrenCache.StartMode.POST_INITIALIZED_EVENT);
        synchronized (waitingForMonitor) {
            while (waitingFor.intValue() > 0) {
                waitingForMonitor.wait();
            }
        }
        scheduleBlackListedNodesCleanUp();
        resourceManagement.startManagement(this);
        started = true;
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}

Example 4 with MutableInt

use of org.apache.commons.lang.mutable.MutableInt in project apex-core by apache.

the class StramRecoveryTest method testWriteAheadLog.

@Test
public void testWriteAheadLog() throws Exception {
    final MutableInt flushCount = new MutableInt();
    final MutableBoolean isClosed = new MutableBoolean(false);
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new FSStorageAgent(testMeta.getPath(), null));
    TestGeneratorInputOperator o1 = dag.addOperator("o1", TestGeneratorInputOperator.class);
    StreamingContainerManager scm = new StreamingContainerManager(dag);
    PhysicalPlan plan = scm.getPhysicalPlan();
    Journal j = scm.getJournal();
    ByteArrayOutputStream bos = new ByteArrayOutputStream() {

        @Override
        public void flush() throws IOException {
            super.flush();
            flushCount.increment();
        }

        @Override
        public void close() throws IOException {
            super.close();
            isClosed.setValue(true);
        }
    };
    j.setOutputStream(new DataOutputStream(bos));
    PTOperator o1p1 = plan.getOperators(dag.getMeta(o1)).get(0);
    assertEquals(PTOperator.State.PENDING_DEPLOY, o1p1.getState());
    String externalId = new MockContainer(scm, o1p1.getContainer()).container.getExternalId();
    assertEquals("flush count", 1, flushCount.intValue());
    o1p1.setState(PTOperator.State.ACTIVE);
    assertEquals(PTOperator.State.ACTIVE, o1p1.getState());
    assertEquals("flush count", 2, flushCount.intValue());
    assertEquals("is closed", false, isClosed.booleanValue());
    // this will close the stream. There are 2 calls to flush() during the close() - one in Kryo Output and one
    // in FilterOutputStream
    j.setOutputStream(null);
    assertEquals("flush count", 4, flushCount.intValue());
    assertEquals("is closed", true, isClosed.booleanValue());
    // output stream is closed, so state will be changed without recording it in the journal
    o1p1.setState(PTOperator.State.INACTIVE);
    assertEquals(PTOperator.State.INACTIVE, o1p1.getState());
    assertEquals("flush count", 4, flushCount.intValue());
    ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
    j.replay(new DataInputStream(bis));
    assertEquals(PTOperator.State.ACTIVE, o1p1.getState());
    InetSocketAddress addr1 = InetSocketAddress.createUnresolved("host1", 1);
    PTContainer c1 = plan.getContainers().get(0);
    c1.setState(PTContainer.State.ALLOCATED);
    c1.host = "host1";
    c1.bufferServerAddress = addr1;
    c1.setAllocatedMemoryMB(2);
    c1.setRequiredMemoryMB(1);
    c1.setAllocatedVCores(3);
    c1.setRequiredVCores(4);
    j.setOutputStream(new DataOutputStream(bos));
    j.write(c1.getSetContainerState());
    c1.setExternalId(null);
    c1.setState(PTContainer.State.NEW);
    c1.setExternalId(null);
    c1.host = null;
    c1.bufferServerAddress = null;
    bis = new ByteArrayInputStream(bos.toByteArray());
    j.replay(new DataInputStream(bis));
    assertEquals(externalId, c1.getExternalId());
    assertEquals(PTContainer.State.ALLOCATED, c1.getState());
    assertEquals("host1", c1.host);
    assertEquals(addr1, c1.bufferServerAddress);
    assertEquals(1, c1.getRequiredMemoryMB());
    assertEquals(2, c1.getAllocatedMemoryMB());
    assertEquals(3, c1.getAllocatedVCores());
    assertEquals(4, c1.getRequiredVCores());
    j.write(scm.getSetOperatorProperty("o1", "maxTuples", "100"));
    o1.setMaxTuples(10);
    j.setOutputStream(null);
    bis = new ByteArrayInputStream(bos.toByteArray());
    j.replay(new DataInputStream(bis));
    assertEquals(100, o1.getMaxTuples());
    j.setOutputStream(new DataOutputStream(bos));
    scm.setOperatorProperty("o1", "maxTuples", "10");
    assertEquals(10, o1.getMaxTuples());
    o1.setMaxTuples(100);
    assertEquals(100, o1.getMaxTuples());
    j.setOutputStream(null);
    bis = new ByteArrayInputStream(bos.toByteArray());
    j.replay(new DataInputStream(bis));
    assertEquals(10, o1.getMaxTuples());
    j.setOutputStream(new DataOutputStream(bos));
    scm.setPhysicalOperatorProperty(o1p1.getId(), "maxTuples", "50");
}

Example 5 with MutableInt

use of org.apache.commons.lang.mutable.MutableInt in project voldemort by voldemort.

the class ThreadPoolRoutedStore method getAll.

@Override
public Map<ByteArray, List<Versioned<byte[]>>> getAll(Iterable<ByteArray> keys, Map<ByteArray, byte[]> transforms) throws VoldemortException {
    StoreUtils.assertValidKeys(keys);
    Map<ByteArray, List<Versioned<byte[]>>> result = StoreUtils.newEmptyHashMap(keys);
    // Keys for each node needed to satisfy storeDef.getPreferredReads() if
    // no failures.
    Map<Node, List<ByteArray>> nodeToKeysMap = Maps.newHashMap();
    // Keep track of nodes per key that might be needed if there are
    // failures during getAll
    Map<ByteArray, List<Node>> keyToExtraNodesMap = Maps.newHashMap();
    for (ByteArray key : keys) {
        List<Node> availableNodes = availableNodes(routingStrategy.routeRequest(key.get()));
        // quickly fail if there aren't enough nodes to meet the requirement
        checkRequiredReads(availableNodes);
        int preferredReads = storeDef.getPreferredReads();
        List<Node> preferredNodes = Lists.newArrayListWithCapacity(preferredReads);
        List<Node> extraNodes = Lists.newArrayListWithCapacity(3);
        for (Node node : availableNodes) {
            if (preferredNodes.size() < preferredReads)
                preferredNodes.add(node);
            else
                extraNodes.add(node);
        }
        for (Node node : preferredNodes) {
            List<ByteArray> nodeKeys = nodeToKeysMap.get(node);
            if (nodeKeys == null) {
                nodeKeys = Lists.newArrayList();
                nodeToKeysMap.put(node, nodeKeys);
            }
            nodeKeys.add(key);
        }
        if (!extraNodes.isEmpty()) {
            List<Node> nodes = keyToExtraNodesMap.get(key);
            if (nodes == null)
                keyToExtraNodesMap.put(key, extraNodes);
            else
                nodes.addAll(extraNodes);
        }
    }
    List<Callable<GetAllResult>> callables = Lists.newArrayList();
    for (Map.Entry<Node, List<ByteArray>> entry : nodeToKeysMap.entrySet()) {
        final Node node = entry.getKey();
        final Collection<ByteArray> nodeKeys = entry.getValue();
        if (failureDetector.isAvailable(node))
            callables.add(new GetAllCallable(node, nodeKeys, transforms));
    }
    // A list of thrown exceptions, indicating the number of failures
    List<Throwable> failures = new CopyOnWriteArrayList<Throwable>();
    List<NodeValue<ByteArray, byte[]>> nodeValues = Lists.newArrayList();
    Map<ByteArray, MutableInt> keyToSuccessCount = Maps.newHashMap();
    for (ByteArray key : keys) keyToSuccessCount.put(key, new MutableInt(0));
    List<Future<GetAllResult>> futures;
    long timeoutMs = timeoutConfig.getOperationTimeout(VoldemortOpCode.GET_ALL_OP_CODE);
    try {
        // TODO What to do about timeouts? They should be longer as getAll
        // is likely to
        // take longer. At the moment, it's just timeoutMs * 3, but should
        // this be based on the number of the keys?
        futures = executor.invokeAll(callables, timeoutMs * 3, TimeUnit.MILLISECONDS);
    } catch (InterruptedException e) {
        throw new InsufficientOperationalNodesException("getAll operation interrupted.", e);
    }
    for (Future<GetAllResult> f : futures) {
        if (f.isCancelled()) {
            logger.warn("Get operation timed out after " + timeoutMs + " ms.");
            continue;
        }
        try {
            GetAllResult getResult = f.get();
            if (getResult.exception != null) {
                if (getResult.exception instanceof VoldemortApplicationException) {
                    throw (VoldemortException) getResult.exception;
                }
                failures.add(getResult.exception);
                continue;
            }
            for (ByteArray key : getResult.callable.nodeKeys) {
                List<Versioned<byte[]>> retrieved = getResult.retrieved.get(key);
                MutableInt successCount = keyToSuccessCount.get(key);
                successCount.increment();
                /*
                     * retrieved can be null if there are no values for the key
                     * provided
                     */
                if (retrieved != null) {
                    List<Versioned<byte[]>> existing = result.get(key);
                    if (existing == null)
                        result.put(key, Lists.newArrayList(retrieved));
                    else
                        existing.addAll(retrieved);
                }
            }
            nodeValues.addAll(getResult.nodeValues);
        } catch (InterruptedException e) {
            throw new InsufficientOperationalNodesException("getAll operation interrupted.", e);
        } catch (ExecutionException e) {
            // should never happen
            if (e.getCause() instanceof Error)
                throw (Error) e.getCause();
            else
                logger.error(e.getMessage(), e);
        }
    }
    for (ByteArray key : keys) {
        MutableInt successCountWrapper = keyToSuccessCount.get(key);
        int successCount = successCountWrapper.intValue();
        if (successCount < storeDef.getPreferredReads()) {
            List<Node> extraNodes = keyToExtraNodesMap.get(key);
            if (extraNodes != null) {
                for (Node node : extraNodes) {
                    long startNs = System.nanoTime();
                    try {
                        List<Versioned<byte[]>> values = innerStores.get(node.getId()).get(key, transforms == null ? null : transforms.get(key));
                        fillRepairReadsValues(nodeValues, key, node, values);
                        List<Versioned<byte[]>> versioneds = result.get(key);
                        if (versioneds == null)
                            result.put(key, Lists.newArrayList(values));
                        else
                            versioneds.addAll(values);
                        recordSuccess(node, startNs);
                        if (++successCount >= storeDef.getPreferredReads())
                            break;
                    } catch (UnreachableStoreException e) {
                        failures.add(e);
                        recordException(node, startNs, e);
                    } catch (VoldemortApplicationException e) {
                        throw e;
                    } catch (Exception e) {
                        logger.warn("Error in GET_ALL on node " + node.getId() + "(" + node.getHost() + ")", e);
                        failures.add(e);
                    }
                }
            }
        }
        successCountWrapper.setValue(successCount);
    }
    repairReads(nodeValues, repairReads && (transforms == null || transforms.size() == 0));
    for (Map.Entry<ByteArray, MutableInt> mapEntry : keyToSuccessCount.entrySet()) {
        int successCount = mapEntry.getValue().intValue();
        if (successCount < storeDef.getRequiredReads())
            throw new InsufficientOperationalNodesException(this.storeDef.getRequiredReads() + " reads required, but " + successCount + " succeeded.", failures);
    }
    return result;
}