Examples with Pair org.apache.commons.lang3.tuple.Pair used on opensource projects

Example 91 with Pair

use of org.apache.commons.lang3.tuple.Pair in project incubator-systemml by apache.

the class PlanSelectionFuseCostBased method rGetPlanCosts.

private static double rGetPlanCosts(CPlanMemoTable memo, Hop current, HashSet<Pair<Long, Long>> visited, HashSet<Long> partition, ArrayList<Long> M, boolean[] plan, HashMap<Long, Double> computeCosts, CostVector costsCurrent, TemplateType currentType) {
    //memoization per hop id and cost vector to account for redundant
    //computation without double counting materialized results or compute
    //costs of complex operation DAGs within a single fused operator
    Pair<Long, Long> tag = Pair.of(current.getHopID(), (costsCurrent == null) ? 0 : costsCurrent.ID);
    if (visited.contains(tag))
        return 0;
    visited.add(tag);
    //open template if necessary, including memoization
    //under awareness of current plan choice
    MemoTableEntry best = null;
    boolean opened = false;
    if (memo.contains(current.getHopID())) {
        if (currentType == null) {
            best = memo.get(current.getHopID()).stream().filter(p -> isValid(p, current)).filter(p -> hasNoRefToMaterialization(p, M, plan)).min(new BasicPlanComparator()).orElse(null);
            opened = true;
        } else {
            best = memo.get(current.getHopID()).stream().filter(p -> p.type == currentType || p.type == TemplateType.CellTpl).filter(p -> hasNoRefToMaterialization(p, M, plan)).min(Comparator.comparing(p -> 7 - ((p.type == currentType) ? 4 : 0) - p.countPlanRefs())).orElse(null);
        }
    }
    //create new cost vector if opened, initialized with write costs
    CostVector costVect = !opened ? costsCurrent : new CostVector(Math.max(current.getDim1(), 1) * Math.max(current.getDim2(), 1));
    //add compute costs of current operator to costs vector 
    if (partition.contains(current.getHopID()))
        costVect.computeCosts += computeCosts.get(current.getHopID());
    //process children recursively
    double costs = 0;
    for (int i = 0; i < current.getInput().size(); i++) {
        Hop c = current.getInput().get(i);
        if (best != null && best.isPlanRef(i))
            costs += rGetPlanCosts(memo, c, visited, partition, M, plan, computeCosts, costVect, best.type);
        else if (best != null && isImplicitlyFused(current, i, best.type))
            costVect.addInputSize(c.getInput().get(0).getHopID(), Math.max(c.getDim1(), 1) * Math.max(c.getDim2(), 1));
        else {
            //include children and I/O costs
            costs += rGetPlanCosts(memo, c, visited, partition, M, plan, computeCosts, null, null);
            if (costVect != null && c.getDataType().isMatrix())
                costVect.addInputSize(c.getHopID(), Math.max(c.getDim1(), 1) * Math.max(c.getDim2(), 1));
        }
    }
    //add costs for opened fused operator
    if (partition.contains(current.getHopID())) {
        if (opened) {
            if (LOG.isTraceEnabled())
                LOG.trace("Cost vector for fused operator (hop " + current.getHopID() + "): " + costVect);
            //time for output write
            costs += costVect.outSize * 8 / WRITE_BANDWIDTH;
            costs += Math.max(costVect.computeCosts * costVect.getMaxInputSize() / COMPUTE_BANDWIDTH, costVect.getSumInputSizes() * 8 / READ_BANDWIDTH);
        } else //add costs for non-partition read in the middle of fused operator
        if (hasNonPartitionConsumer(current, partition)) {
            costs += rGetPlanCosts(memo, current, visited, partition, M, plan, computeCosts, null, null);
        }
    }
    //sanity check non-negative costs
    if (costs < 0 || Double.isNaN(costs) || Double.isInfinite(costs))
        throw new RuntimeException("Wrong cost estimate: " + costs);
    return costs;
}

Example 92 with Pair

use of org.apache.commons.lang3.tuple.Pair in project distributedlog by twitter.

the class BKLogHandler method asyncGetLedgerListInternal.

private void asyncGetLedgerListInternal(final Comparator<LogSegmentMetadata> comparator, final LogSegmentFilter segmentFilter, final Watcher watcher, final GenericCallback<List<LogSegmentMetadata>> finalCallback, final AtomicInteger numAttemptsLeft, final AtomicLong backoffMillis) {
    final Stopwatch stopwatch = Stopwatch.createStarted();
    try {
        if (LOG.isTraceEnabled()) {
            LOG.trace("Async getting ledger list for {}.", getFullyQualifiedName());
        }
        final GenericCallback<List<LogSegmentMetadata>> callback = new GenericCallback<List<LogSegmentMetadata>>() {

            @Override
            public void operationComplete(int rc, List<LogSegmentMetadata> result) {
                long elapsedMicros = stopwatch.stop().elapsed(TimeUnit.MICROSECONDS);
                if (KeeperException.Code.OK.intValue() != rc) {
                    getListStat.registerFailedEvent(elapsedMicros);
                } else {
                    if (LogSegmentFilter.DEFAULT_FILTER == segmentFilter) {
                        isFullListFetched.set(true);
                    }
                    getListStat.registerSuccessfulEvent(elapsedMicros);
                }
                finalCallback.operationComplete(rc, result);
            }
        };
        zooKeeperClient.get().getChildren(logMetadata.getLogSegmentsPath(), watcher, new AsyncCallback.Children2Callback() {

            @Override
            public void processResult(final int rc, final String path, final Object ctx, final List<String> children, final Stat stat) {
                if (KeeperException.Code.OK.intValue() != rc) {
                    if ((KeeperException.Code.CONNECTIONLOSS.intValue() == rc || KeeperException.Code.SESSIONEXPIRED.intValue() == rc || KeeperException.Code.SESSIONMOVED.intValue() == rc) && numAttemptsLeft.decrementAndGet() > 0) {
                        long backoffMs = backoffMillis.get();
                        backoffMillis.set(Math.min(conf.getZKRetryBackoffMaxMillis(), 2 * backoffMs));
                        scheduler.schedule(new Runnable() {

                            @Override
                            public void run() {
                                asyncGetLedgerListInternal(comparator, segmentFilter, watcher, finalCallback, numAttemptsLeft, backoffMillis);
                            }
                        }, backoffMs, TimeUnit.MILLISECONDS);
                        return;
                    }
                    callback.operationComplete(rc, null);
                    return;
                }
                if (LOG.isTraceEnabled()) {
                    LOG.trace("Got ledger list from {} : {}", logMetadata.getLogSegmentsPath(), children);
                }
                ledgerListWatchSet.set(true);
                Set<String> segmentsReceived = new HashSet<String>();
                segmentsReceived.addAll(segmentFilter.filter(children));
                Set<String> segmentsAdded;
                final Set<String> removedSegments = Collections.synchronizedSet(new HashSet<String>());
                final Map<String, LogSegmentMetadata> addedSegments = Collections.synchronizedMap(new HashMap<String, LogSegmentMetadata>());
                Pair<Set<String>, Set<String>> segmentChanges = logSegmentCache.diff(segmentsReceived);
                segmentsAdded = segmentChanges.getLeft();
                removedSegments.addAll(segmentChanges.getRight());
                if (segmentsAdded.isEmpty()) {
                    if (LOG.isTraceEnabled()) {
                        LOG.trace("No segments added for {}.", getFullyQualifiedName());
                    }
                    // update the cache before fetch
                    logSegmentCache.update(removedSegments, addedSegments);
                    List<LogSegmentMetadata> segmentList;
                    try {
                        segmentList = getCachedLogSegments(comparator);
                    } catch (UnexpectedException e) {
                        callback.operationComplete(KeeperException.Code.DATAINCONSISTENCY.intValue(), null);
                        return;
                    }
                    callback.operationComplete(KeeperException.Code.OK.intValue(), segmentList);
                    notifyUpdatedLogSegments(segmentList);
                    if (!removedSegments.isEmpty()) {
                        notifyOnOperationComplete();
                    }
                    return;
                }
                final AtomicInteger numChildren = new AtomicInteger(segmentsAdded.size());
                final AtomicInteger numFailures = new AtomicInteger(0);
                for (final String segment : segmentsAdded) {
                    metadataStore.getLogSegment(logMetadata.getLogSegmentPath(segment)).addEventListener(new FutureEventListener<LogSegmentMetadata>() {

                        @Override
                        public void onSuccess(LogSegmentMetadata result) {
                            addedSegments.put(segment, result);
                            complete();
                        }

                        @Override
                        public void onFailure(Throwable cause) {
                            // 2. In progress segment has been completed => inprogress ZNode does not exist
                            if (cause instanceof KeeperException && KeeperException.Code.NONODE == ((KeeperException) cause).code()) {
                                removedSegments.add(segment);
                                complete();
                            } else {
                                // fail fast
                                if (1 == numFailures.incrementAndGet()) {
                                    int rcToReturn = KeeperException.Code.SYSTEMERROR.intValue();
                                    if (cause instanceof KeeperException) {
                                        rcToReturn = ((KeeperException) cause).code().intValue();
                                    } else if (cause instanceof ZKException) {
                                        rcToReturn = ((ZKException) cause).getKeeperExceptionCode().intValue();
                                    }
                                    // :( properly we need dlog related response code.
                                    callback.operationComplete(rcToReturn, null);
                                    return;
                                }
                            }
                        }

                        private void complete() {
                            if (0 == numChildren.decrementAndGet() && numFailures.get() == 0) {
                                // update the cache only when fetch completed
                                logSegmentCache.update(removedSegments, addedSegments);
                                List<LogSegmentMetadata> segmentList;
                                try {
                                    segmentList = getCachedLogSegments(comparator);
                                } catch (UnexpectedException e) {
                                    callback.operationComplete(KeeperException.Code.DATAINCONSISTENCY.intValue(), null);
                                    return;
                                }
                                callback.operationComplete(KeeperException.Code.OK.intValue(), segmentList);
                                notifyUpdatedLogSegments(segmentList);
                                notifyOnOperationComplete();
                            }
                        }
                    });
                }
            }
        }, null);
    } catch (ZooKeeperClient.ZooKeeperConnectionException e) {
        getListStat.registerFailedEvent(stopwatch.stop().elapsed(TimeUnit.MICROSECONDS));
        finalCallback.operationComplete(KeeperException.Code.CONNECTIONLOSS.intValue(), null);
    } catch (InterruptedException e) {
        getListStat.registerFailedEvent(stopwatch.stop().elapsed(TimeUnit.MICROSECONDS));
        finalCallback.operationComplete(KeeperException.Code.CONNECTIONLOSS.intValue(), null);
    }
}

Example 93 with Pair

use of org.apache.commons.lang3.tuple.Pair in project distributedlog by twitter.

the class ZKSessionLock method asyncTryLock.

@Override
public Future<LockWaiter> asyncTryLock(final long timeout, final TimeUnit unit) {
    final Promise<String> result = new Promise<String>();
    final boolean wait = DistributedLogConstants.LOCK_IMMEDIATE != timeout;
    if (wait) {
        asyncTryLock(wait, result);
    } else {
        // try to check locks first
        zk.getChildren(lockPath, null, new AsyncCallback.Children2Callback() {

            @Override
            public void processResult(final int rc, String path, Object ctx, final List<String> children, Stat stat) {
                lockStateExecutor.submit(lockPath, new SafeRunnable() {

                    @Override
                    public void safeRun() {
                        if (!lockState.inState(State.INIT)) {
                            result.setException(new LockStateChangedException(lockPath, lockId, State.INIT, lockState.getState()));
                            return;
                        }
                        if (KeeperException.Code.OK.intValue() != rc) {
                            result.setException(KeeperException.create(KeeperException.Code.get(rc)));
                            return;
                        }
                        FailpointUtils.checkFailPointNoThrow(FailpointUtils.FailPointName.FP_LockTryAcquire);
                        Collections.sort(children, MEMBER_COMPARATOR);
                        if (children.size() > 0) {
                            asyncParseClientID(zk, lockPath, children.get(0)).addEventListener(new FutureEventListener<Pair<String, Long>>() {

                                @Override
                                public void onSuccess(Pair<String, Long> owner) {
                                    if (!checkOrClaimLockOwner(owner, result)) {
                                        acquireFuture.updateIfEmpty(new Return<Boolean>(false));
                                    }
                                }

                                @Override
                                public void onFailure(final Throwable cause) {
                                    lockStateExecutor.submit(lockPath, new SafeRunnable() {

                                        @Override
                                        public void safeRun() {
                                            result.setException(cause);
                                        }
                                    });
                                }
                            });
                        } else {
                            asyncTryLock(wait, result);
                        }
                    }
                });
            }
        }, null);
    }
    final Promise<Boolean> waiterAcquireFuture = new Promise<Boolean>(new com.twitter.util.Function<Throwable, BoxedUnit>() {

        @Override
        public BoxedUnit apply(Throwable t) {
            acquireFuture.raise(t);
            return BoxedUnit.UNIT;
        }
    });
    return result.map(new AbstractFunction1<String, LockWaiter>() {

        @Override
        public LockWaiter apply(final String currentOwner) {
            final Exception acquireException = new OwnershipAcquireFailedException(lockPath, currentOwner);
            FutureUtils.within(acquireFuture, timeout, unit, acquireException, lockStateExecutor, lockPath).addEventListener(new FutureEventListener<Boolean>() {

                @Override
                public void onSuccess(Boolean acquired) {
                    completeOrFail(acquireException);
                }

                @Override
                public void onFailure(final Throwable acquireCause) {
                    completeOrFail(acquireException);
                }

                private void completeOrFail(final Throwable acquireCause) {
                    if (isLockHeld()) {
                        waiterAcquireFuture.setValue(true);
                    } else {
                        asyncUnlock().addEventListener(new FutureEventListener<BoxedUnit>() {

                            @Override
                            public void onSuccess(BoxedUnit value) {
                                waiterAcquireFuture.setException(acquireCause);
                            }

                            @Override
                            public void onFailure(Throwable cause) {
                                waiterAcquireFuture.setException(acquireCause);
                            }
                        });
                    }
                }
            });
            ;
            return new LockWaiter(lockId.getLeft(), currentOwner, waiterAcquireFuture);
        }
    });
}

Example 94 with Pair

use of org.apache.commons.lang3.tuple.Pair in project RecurrentComplex by Ivorforce.

the class FactorMatch method consider.

@Override
public List<Pair<LineSelection, Float>> consider(WorldCache cache, LineSelection considerable, @Nullable IvBlockCollection blockCollection, StructurePlaceContext context) {
    if (blockCollection == null)
        throw new IllegalArgumentException("Missing a block collection!");
    List<Pair<LineSelection, Float>> consideration = new ArrayList<>();
    int[] size = StructureBoundingBoxes.size(context.boundingBox);
    BlockPos lowerCoord = StructureBoundingBoxes.min(context.boundingBox);
    Set<BlockPos.MutableBlockPos> sources = BlockAreas.streamMutablePositions(blockCollection.area()).filter(p -> sourceMatcher.evaluate(() -> blockCollection.getBlockState(p))).map(p -> new BlockPos.MutableBlockPos(context.transform.apply(p, size).add(lowerCoord.getX(), 0, lowerCoord.getZ()))).collect(Collectors.toSet());
    for (IntegerRange range : (Iterable<IntegerRange>) considerable.streamSections(null, true)::iterator) {
        Float curConformity = null;
        int lastY = range.getMax();
        int end = range.getMin();
        for (int y = lastY; y >= end; y--) {
            int finalY = y;
            sources.forEach(p -> p.move(EnumFacing.UP, finalY));
            float conformity = weight(cache, sources, requiredConformity);
            sources.forEach(p -> p.move(EnumFacing.DOWN, finalY));
            if (curConformity == null) {
                curConformity = conformity;
                lastY = y;
            } else if (!DoubleMath.fuzzyEquals(conformity, curConformity, 0.01)) {
                consideration.add(Pair.of(LineSelection.fromRange(IntegerRanges.from(lastY, y + 1), true), weight(curConformity)));
                curConformity = conformity;
                lastY = y;
            }
        }
        if (curConformity != null)
            consideration.add(Pair.of(LineSelection.fromRange(IntegerRanges.from(lastY, end), true), weight(curConformity)));
    }
    return consideration;
}

Example 95 with Pair

use of org.apache.commons.lang3.tuple.Pair in project RecurrentComplex by Ivorforce.

the class CommandRetrogen method existingRegions.

public static Stream<Pair<Integer, Integer>> existingRegions(File worldDir) {
    File regionsDirectory = RCFiles.getValidatedFolder(new File(worldDir, "region"), false);
    if (regionsDirectory == null)
        return Stream.empty();
    String[] mcas = regionsDirectory.list(new FileSuffixFilter("mca"));
    if (mcas == null)
        throw new IllegalStateException();
    return Arrays.stream(mcas).map(s -> s.split("\\.")).filter(// Is region file
    p -> p.length == 4 && p[0].equals("r")).map(p -> Pair.of(Integer.parseInt(p[1]), Integer.parseInt(p[2]))).filter(// Has coords
    rfc -> rfc.getLeft() != null && rfc.getRight() != null);
}