Search in sources :

Example 46 with Range

use of org.apache.commons.lang3.Range in project asterixdb by apache.

the class BTreeAccessMethod method createSecondaryToPrimaryPlan.

@Override
public ILogicalOperator createSecondaryToPrimaryPlan(Mutable<ILogicalExpression> conditionRef, OptimizableOperatorSubTree indexSubTree, OptimizableOperatorSubTree probeSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx, boolean retainInput, boolean retainNull, boolean requiresBroadcast, IOptimizationContext context) throws AlgebricksException {
    Dataset dataset = indexSubTree.getDataset();
    ARecordType recordType = indexSubTree.getRecordType();
    ARecordType metaRecordType = indexSubTree.getMetaRecordType();
    // we made sure indexSubTree has datasource scan
    AbstractDataSourceOperator dataSourceOp = (AbstractDataSourceOperator) indexSubTree.getDataSourceRef().getValue();
    List<Pair<Integer, Integer>> exprAndVarList = analysisCtx.getIndexExprsFromIndexExprsAndVars(chosenIndex);
    int numSecondaryKeys = analysisCtx.getNumberOfMatchedKeys(chosenIndex);
    // List of function expressions that will be replaced by the secondary-index search.
    // These func exprs will be removed from the select condition at the very end of this method.
    Set<ILogicalExpression> replacedFuncExprs = new HashSet<>();
    // Info on high and low keys for the BTree search predicate.
    ILogicalExpression[] lowKeyExprs = new ILogicalExpression[numSecondaryKeys];
    ILogicalExpression[] highKeyExprs = new ILogicalExpression[numSecondaryKeys];
    LimitType[] lowKeyLimits = new LimitType[numSecondaryKeys];
    LimitType[] highKeyLimits = new LimitType[numSecondaryKeys];
    boolean[] lowKeyInclusive = new boolean[numSecondaryKeys];
    boolean[] highKeyInclusive = new boolean[numSecondaryKeys];
    ILogicalExpression[] constantAtRuntimeExpressions = new ILogicalExpression[numSecondaryKeys];
    LogicalVariable[] constAtRuntimeExprVars = new LogicalVariable[numSecondaryKeys];
    /* TODO: For now we don't do any sophisticated analysis of the func exprs to come up with "the best" range
         * predicate. If we can't figure out how to integrate a certain funcExpr into the current predicate,
         * we just bail by setting this flag.*/
    boolean couldntFigureOut = false;
    boolean doneWithExprs = false;
    boolean isEqCondition = false;
    BitSet setLowKeys = new BitSet(numSecondaryKeys);
    BitSet setHighKeys = new BitSet(numSecondaryKeys);
    // Go through the func exprs listed as optimizable by the chosen index,
    // and formulate a range predicate on the secondary-index keys.
    // checks whether a type casting happened from a real (FLOAT, DOUBLE) value to an INT value
    // since we have a round issues when dealing with LT(<) OR GT(>) operator.
    boolean realTypeConvertedToIntegerType;
    for (Pair<Integer, Integer> exprIndex : exprAndVarList) {
        // Position of the field of matchedFuncExprs.get(exprIndex) in the chosen index's indexed exprs.
        IOptimizableFuncExpr optFuncExpr = analysisCtx.getMatchedFuncExpr(exprIndex.first);
        int keyPos = indexOf(optFuncExpr.getFieldName(0), chosenIndex.getKeyFieldNames());
        if (keyPos < 0 && optFuncExpr.getNumLogicalVars() > 1) {
            // If we are optimizing a join, the matching field may be the second field name.
            keyPos = indexOf(optFuncExpr.getFieldName(1), chosenIndex.getKeyFieldNames());
        }
        if (keyPos < 0) {
            throw CompilationException.create(ErrorCode.NO_INDEX_FIELD_NAME_FOR_GIVEN_FUNC_EXPR);
        }
        Pair<ILogicalExpression, Boolean> returnedSearchKeyExpr = AccessMethodUtils.createSearchKeyExpr(optFuncExpr, indexSubTree, probeSubTree);
        ILogicalExpression searchKeyExpr = returnedSearchKeyExpr.first;
        if (searchKeyExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
            constantAtRuntimeExpressions[keyPos] = searchKeyExpr;
            constAtRuntimeExprVars[keyPos] = context.newVar();
            searchKeyExpr = new VariableReferenceExpression(constAtRuntimeExprVars[keyPos]);
        }
        realTypeConvertedToIntegerType = returnedSearchKeyExpr.second;
        LimitType limit = getLimitType(optFuncExpr, probeSubTree);
        //
        if (realTypeConvertedToIntegerType) {
            if (limit == LimitType.HIGH_EXCLUSIVE) {
                limit = LimitType.HIGH_INCLUSIVE;
            } else if (limit == LimitType.LOW_EXCLUSIVE) {
                limit = LimitType.LOW_INCLUSIVE;
            }
        }
        switch(limit) {
            case EQUAL:
                {
                    if (lowKeyLimits[keyPos] == null && highKeyLimits[keyPos] == null) {
                        lowKeyLimits[keyPos] = highKeyLimits[keyPos] = limit;
                        lowKeyInclusive[keyPos] = highKeyInclusive[keyPos] = true;
                        lowKeyExprs[keyPos] = highKeyExprs[keyPos] = searchKeyExpr;
                        setLowKeys.set(keyPos);
                        setHighKeys.set(keyPos);
                        isEqCondition = true;
                    } else {
                        // (once from analyzing each side of the join)
                        if (lowKeyLimits[keyPos] == limit && lowKeyInclusive[keyPos] == true && lowKeyExprs[keyPos].equals(searchKeyExpr) && highKeyLimits[keyPos] == limit && highKeyInclusive[keyPos] == true && highKeyExprs[keyPos].equals(searchKeyExpr)) {
                            isEqCondition = true;
                            break;
                        }
                        couldntFigureOut = true;
                    }
                    // If high and low keys are set, we exit for now.
                    if (setLowKeys.cardinality() == numSecondaryKeys && setHighKeys.cardinality() == numSecondaryKeys) {
                        doneWithExprs = true;
                    }
                    break;
                }
            case HIGH_EXCLUSIVE:
                {
                    if (highKeyLimits[keyPos] == null || (highKeyLimits[keyPos] != null && highKeyInclusive[keyPos])) {
                        highKeyLimits[keyPos] = limit;
                        highKeyExprs[keyPos] = searchKeyExpr;
                        highKeyInclusive[keyPos] = false;
                    } else {
                        // (once from analyzing each side of the join)
                        if (highKeyLimits[keyPos] == limit && highKeyInclusive[keyPos] == false && highKeyExprs[keyPos].equals(searchKeyExpr)) {
                            break;
                        }
                        couldntFigureOut = true;
                        doneWithExprs = true;
                    }
                    break;
                }
            case HIGH_INCLUSIVE:
                {
                    if (highKeyLimits[keyPos] == null) {
                        highKeyLimits[keyPos] = limit;
                        highKeyExprs[keyPos] = searchKeyExpr;
                        highKeyInclusive[keyPos] = true;
                    } else {
                        // (once from analyzing each side of the join)
                        if (highKeyLimits[keyPos] == limit && highKeyInclusive[keyPos] == true && highKeyExprs[keyPos].equals(searchKeyExpr)) {
                            break;
                        }
                        couldntFigureOut = true;
                        doneWithExprs = true;
                    }
                    break;
                }
            case LOW_EXCLUSIVE:
                {
                    if (lowKeyLimits[keyPos] == null || (lowKeyLimits[keyPos] != null && lowKeyInclusive[keyPos])) {
                        lowKeyLimits[keyPos] = limit;
                        lowKeyExprs[keyPos] = searchKeyExpr;
                        lowKeyInclusive[keyPos] = false;
                    } else {
                        // (once from analyzing each side of the join)
                        if (lowKeyLimits[keyPos] == limit && lowKeyInclusive[keyPos] == false && lowKeyExprs[keyPos].equals(searchKeyExpr)) {
                            break;
                        }
                        couldntFigureOut = true;
                        doneWithExprs = true;
                    }
                    break;
                }
            case LOW_INCLUSIVE:
                {
                    if (lowKeyLimits[keyPos] == null) {
                        lowKeyLimits[keyPos] = limit;
                        lowKeyExprs[keyPos] = searchKeyExpr;
                        lowKeyInclusive[keyPos] = true;
                    } else {
                        // (once from analyzing each side of the join)
                        if (lowKeyLimits[keyPos] == limit && lowKeyInclusive[keyPos] == true && lowKeyExprs[keyPos].equals(searchKeyExpr)) {
                            break;
                        }
                        couldntFigureOut = true;
                        doneWithExprs = true;
                    }
                    break;
                }
            default:
                {
                    throw new IllegalStateException();
                }
        }
        if (!couldntFigureOut) {
            // Remember to remove this funcExpr later.
            replacedFuncExprs.add(analysisCtx.getMatchedFuncExpr(exprIndex.first).getFuncExpr());
        }
        if (doneWithExprs) {
            break;
        }
    }
    if (couldntFigureOut) {
        return null;
    }
    // If the select condition contains mixed open/closed intervals on multiple keys, then we make all intervals
    // closed to obtain a superset of answers and leave the original selection in place.
    boolean primaryIndexPostProccessingIsNeeded = false;
    for (int i = 1; i < numSecondaryKeys; ++i) {
        if (lowKeyInclusive[i] != lowKeyInclusive[0]) {
            Arrays.fill(lowKeyInclusive, true);
            primaryIndexPostProccessingIsNeeded = true;
            break;
        }
    }
    for (int i = 1; i < numSecondaryKeys; ++i) {
        if (highKeyInclusive[i] != highKeyInclusive[0]) {
            Arrays.fill(highKeyInclusive, true);
            primaryIndexPostProccessingIsNeeded = true;
            break;
        }
    }
    // determine cases when prefix search could be applied
    for (int i = 1; i < lowKeyExprs.length; i++) {
        if (lowKeyLimits[0] == null && lowKeyLimits[i] != null || lowKeyLimits[0] != null && lowKeyLimits[i] == null || highKeyLimits[0] == null && highKeyLimits[i] != null || highKeyLimits[0] != null && highKeyLimits[i] == null) {
            numSecondaryKeys--;
            primaryIndexPostProccessingIsNeeded = true;
        }
    }
    if (lowKeyLimits[0] == null) {
        lowKeyInclusive[0] = true;
    }
    if (highKeyLimits[0] == null) {
        highKeyInclusive[0] = true;
    }
    // Here we generate vars and funcs for assigning the secondary-index keys to be fed into the secondary-index
    // search.
    // List of variables for the assign.
    ArrayList<LogicalVariable> keyVarList = new ArrayList<>();
    // List of variables and expressions for the assign.
    ArrayList<LogicalVariable> assignKeyVarList = new ArrayList<>();
    ArrayList<Mutable<ILogicalExpression>> assignKeyExprList = new ArrayList<>();
    int numLowKeys = createKeyVarsAndExprs(numSecondaryKeys, lowKeyLimits, lowKeyExprs, assignKeyVarList, assignKeyExprList, keyVarList, context, constantAtRuntimeExpressions, constAtRuntimeExprVars);
    int numHighKeys = createKeyVarsAndExprs(numSecondaryKeys, highKeyLimits, highKeyExprs, assignKeyVarList, assignKeyExprList, keyVarList, context, constantAtRuntimeExpressions, constAtRuntimeExprVars);
    BTreeJobGenParams jobGenParams = new BTreeJobGenParams(chosenIndex.getIndexName(), IndexType.BTREE, dataset.getDataverseName(), dataset.getDatasetName(), retainInput, requiresBroadcast);
    jobGenParams.setLowKeyInclusive(lowKeyInclusive[0]);
    jobGenParams.setHighKeyInclusive(highKeyInclusive[0]);
    jobGenParams.setIsEqCondition(isEqCondition);
    jobGenParams.setLowKeyVarList(keyVarList, 0, numLowKeys);
    jobGenParams.setHighKeyVarList(keyVarList, numLowKeys, numHighKeys);
    ILogicalOperator inputOp = null;
    if (!assignKeyVarList.isEmpty()) {
        // Assign operator that sets the constant secondary-index search-key fields if necessary.
        AssignOperator assignConstantSearchKeys = new AssignOperator(assignKeyVarList, assignKeyExprList);
        // Input to this assign is the EmptyTupleSource (which the dataSourceScan also must have had as input).
        assignConstantSearchKeys.getInputs().add(new MutableObject<>(OperatorManipulationUtil.deepCopy(dataSourceOp.getInputs().get(0).getValue())));
        assignConstantSearchKeys.setExecutionMode(dataSourceOp.getExecutionMode());
        inputOp = assignConstantSearchKeys;
    } else if (probeSubTree == null) {
        //nonpure case
        //Make sure that the nonpure function is unpartitioned
        ILogicalOperator checkOp = dataSourceOp.getInputs().get(0).getValue();
        while (checkOp.getExecutionMode() != ExecutionMode.UNPARTITIONED) {
            if (checkOp.getInputs().size() == 1) {
                checkOp = checkOp.getInputs().get(0).getValue();
            } else {
                return null;
            }
        }
        inputOp = dataSourceOp.getInputs().get(0).getValue();
    } else {
        // All index search keys are variables.
        inputOp = probeSubTree.getRoot();
    }
    ILogicalOperator secondaryIndexUnnestOp = AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType, metaRecordType, chosenIndex, inputOp, jobGenParams, context, false, retainInput, retainNull);
    // Generate the rest of the upstream plan which feeds the search results into the primary index.
    AbstractUnnestMapOperator primaryIndexUnnestOp = null;
    boolean isPrimaryIndex = chosenIndex.isPrimaryIndex();
    if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
        // External dataset
        UnnestMapOperator externalDataAccessOp = AccessMethodUtils.createExternalDataLookupUnnestMap(dataSourceOp, dataset, recordType, secondaryIndexUnnestOp, context, retainInput, retainNull);
        indexSubTree.getDataSourceRef().setValue(externalDataAccessOp);
        return externalDataAccessOp;
    } else if (!isPrimaryIndex) {
        primaryIndexUnnestOp = AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceOp, dataset, recordType, metaRecordType, secondaryIndexUnnestOp, context, true, retainInput, retainNull, false);
        // Adds equivalence classes --- one equivalent class between a primary key
        // variable and a record field-access expression.
        EquivalenceClassUtils.addEquivalenceClassesForPrimaryIndexAccess(primaryIndexUnnestOp, dataSourceOp.getVariables(), recordType, metaRecordType, dataset, context);
    } else {
        List<Object> primaryIndexOutputTypes = new ArrayList<>();
        AccessMethodUtils.appendPrimaryIndexTypes(dataset, recordType, metaRecordType, primaryIndexOutputTypes);
        List<LogicalVariable> scanVariables = dataSourceOp.getVariables();
        // If not, we create a new condition based on remaining ones.
        if (!primaryIndexPostProccessingIsNeeded) {
            List<Mutable<ILogicalExpression>> remainingFuncExprs = new ArrayList<>();
            try {
                getNewConditionExprs(conditionRef, replacedFuncExprs, remainingFuncExprs);
            } catch (CompilationException e) {
                return null;
            }
            // Generate new condition.
            if (!remainingFuncExprs.isEmpty()) {
                ILogicalExpression pulledCond = createSelectCondition(remainingFuncExprs);
                conditionRef.setValue(pulledCond);
            } else {
                conditionRef.setValue(null);
            }
        }
        // Checks whether LEFT_OUTER_UNNESTMAP operator is required.
        boolean leftOuterUnnestMapRequired = false;
        if (retainNull && retainInput) {
            leftOuterUnnestMapRequired = true;
        } else {
            leftOuterUnnestMapRequired = false;
        }
        if (conditionRef.getValue() != null) {
            // The job gen parameters are transferred to the actual job gen
            // via the UnnestMapOperator's function arguments.
            List<Mutable<ILogicalExpression>> primaryIndexFuncArgs = new ArrayList<>();
            jobGenParams.writeToFuncArgs(primaryIndexFuncArgs);
            // An index search is expressed as an unnest-map over an
            // index-search function.
            IFunctionInfo primaryIndexSearch = FunctionUtil.getFunctionInfo(BuiltinFunctions.INDEX_SEARCH);
            UnnestingFunctionCallExpression primaryIndexSearchFunc = new UnnestingFunctionCallExpression(primaryIndexSearch, primaryIndexFuncArgs);
            primaryIndexSearchFunc.setReturnsUniqueValues(true);
            if (!leftOuterUnnestMapRequired) {
                primaryIndexUnnestOp = new UnnestMapOperator(scanVariables, new MutableObject<ILogicalExpression>(primaryIndexSearchFunc), primaryIndexOutputTypes, retainInput);
            } else {
                primaryIndexUnnestOp = new LeftOuterUnnestMapOperator(scanVariables, new MutableObject<ILogicalExpression>(primaryIndexSearchFunc), primaryIndexOutputTypes, true);
            }
        } else {
            if (!leftOuterUnnestMapRequired) {
                primaryIndexUnnestOp = new UnnestMapOperator(scanVariables, ((UnnestMapOperator) secondaryIndexUnnestOp).getExpressionRef(), primaryIndexOutputTypes, retainInput);
            } else {
                primaryIndexUnnestOp = new LeftOuterUnnestMapOperator(scanVariables, ((LeftOuterUnnestMapOperator) secondaryIndexUnnestOp).getExpressionRef(), primaryIndexOutputTypes, true);
            }
        }
        primaryIndexUnnestOp.getInputs().add(new MutableObject<>(inputOp));
        // Adds equivalence classes --- one equivalent class between a primary key
        // variable and a record field-access expression.
        EquivalenceClassUtils.addEquivalenceClassesForPrimaryIndexAccess(primaryIndexUnnestOp, scanVariables, recordType, metaRecordType, dataset, context);
    }
    return primaryIndexUnnestOp;
}
Also used : IFunctionInfo(org.apache.hyracks.algebricks.core.algebra.functions.IFunctionInfo) LeftOuterUnnestMapOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterUnnestMapOperator) UnnestMapOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator) AbstractUnnestMapOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestMapOperator) ArrayList(java.util.ArrayList) ArrayList(java.util.ArrayList) List(java.util.List) LeftOuterUnnestMapOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterUnnestMapOperator) Pair(org.apache.hyracks.algebricks.common.utils.Pair) HashSet(java.util.HashSet) MutableObject(org.apache.commons.lang3.mutable.MutableObject) LogicalVariable(org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable) CompilationException(org.apache.asterix.common.exceptions.CompilationException) AbstractDataSourceOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractDataSourceOperator) UnnestingFunctionCallExpression(org.apache.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression) Dataset(org.apache.asterix.metadata.entities.Dataset) ILogicalOperator(org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator) BitSet(java.util.BitSet) AssignOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator) Mutable(org.apache.commons.lang3.mutable.Mutable) ILogicalExpression(org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression) VariableReferenceExpression(org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression) AbstractUnnestMapOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestMapOperator) ARecordType(org.apache.asterix.om.types.ARecordType)

Example 47 with Range

use of org.apache.commons.lang3.Range in project gatk-protected by broadinstitute.

the class CoverageModelEMWorkspace method getViterbiAsNDArray.

/**
     * Fetches the Viterbi copy ratio (or copy number) states as a target-sample matrix
     *
     * @return an {@link INDArray}
     */
protected INDArray getViterbiAsNDArray(final List<List<HiddenStateSegmentRecord<STATE, Target>>> segments) {
    final INDArray res = Nd4j.create(numSamples, numTargets);
    final TargetCollection<Target> targetCollection = new HashedListTargetCollection<>(processedTargetList);
    for (int si = 0; si < numSamples; si++) {
        final SexGenotypeData sampleSexGenotype = processedSampleSexGenotypeData.get(si);
        /* start with all ref */
        final double[] calls = IntStream.range(0, numTargets).mapToDouble(ti -> referenceStateFactory.apply(sampleSexGenotype, processedTargetList.get(ti)).getScalar()).toArray();
        /* go through segments and mutate ref calls as necessary */
        segments.get(si).forEach(seg -> {
            final IndexRange range = targetCollection.indexRange(seg.getSegment());
            final double copyRatio = seg.getSegment().getCall().getScalar();
            for (int ti = range.from; ti < range.to; ti++) {
                calls[ti] = copyRatio;
            }
        });
        res.get(NDArrayIndex.point(si), NDArrayIndex.all()).assign(Nd4j.create(calls, new int[] { 1, numTargets }));
    }
    return res.transpose();
}
Also used : ScalarProducer(org.broadinstitute.hellbender.utils.hmm.interfaces.ScalarProducer) Function2(org.apache.spark.api.java.function.Function2) HMMSegmentProcessor(org.broadinstitute.hellbender.utils.hmm.segmentation.HMMSegmentProcessor) GermlinePloidyAnnotatedTargetCollection(org.broadinstitute.hellbender.tools.exome.sexgenotyper.GermlinePloidyAnnotatedTargetCollection) HiddenStateSegmentRecordWriter(org.broadinstitute.hellbender.utils.hmm.segmentation.HiddenStateSegmentRecordWriter) BiFunction(java.util.function.BiFunction) GATKException(org.broadinstitute.hellbender.exceptions.GATKException) SexGenotypeData(org.broadinstitute.hellbender.tools.exome.sexgenotyper.SexGenotypeData) ParamUtils(org.broadinstitute.hellbender.utils.param.ParamUtils) CallStringProducer(org.broadinstitute.hellbender.utils.hmm.interfaces.CallStringProducer) StorageLevel(org.apache.spark.storage.StorageLevel) SynchronizedUnivariateSolver(org.broadinstitute.hellbender.tools.coveragemodel.math.SynchronizedUnivariateSolver) CopyRatioExpectationsCalculator(org.broadinstitute.hellbender.tools.coveragemodel.interfaces.CopyRatioExpectationsCalculator) UnivariateSolverSpecifications(org.broadinstitute.hellbender.tools.coveragemodel.math.UnivariateSolverSpecifications) IndexRange(org.broadinstitute.hellbender.utils.IndexRange) Broadcast(org.apache.spark.broadcast.Broadcast) ExitStatus(org.broadinstitute.hellbender.tools.coveragemodel.linalg.IterativeLinearSolverNDArray.ExitStatus) SexGenotypeDataCollection(org.broadinstitute.hellbender.tools.exome.sexgenotyper.SexGenotypeDataCollection) HashPartitioner(org.apache.spark.HashPartitioner) Predicate(java.util.function.Predicate) GeneralLinearOperator(org.broadinstitute.hellbender.tools.coveragemodel.linalg.GeneralLinearOperator) Nd4j(org.nd4j.linalg.factory.Nd4j) INDArrayIndex(org.nd4j.linalg.indexing.INDArrayIndex) FastMath(org.apache.commons.math3.util.FastMath) org.broadinstitute.hellbender.tools.exome(org.broadinstitute.hellbender.tools.exome) Tuple2(scala.Tuple2) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) AbstractUnivariateSolver(org.apache.commons.math3.analysis.solvers.AbstractUnivariateSolver) FourierLinearOperatorNDArray(org.broadinstitute.hellbender.tools.coveragemodel.linalg.FourierLinearOperatorNDArray) Logger(org.apache.logging.log4j.Logger) Stream(java.util.stream.Stream) UserException(org.broadinstitute.hellbender.exceptions.UserException) UnivariateFunction(org.apache.commons.math3.analysis.UnivariateFunction) TooManyEvaluationsException(org.apache.commons.math3.exception.TooManyEvaluationsException) Utils(org.broadinstitute.hellbender.utils.Utils) Function(org.apache.spark.api.java.function.Function) DataBuffer(org.nd4j.linalg.api.buffer.DataBuffer) IntStream(java.util.stream.IntStream) java.util(java.util) NDArrayIndex(org.nd4j.linalg.indexing.NDArrayIndex) JavaSparkContext(org.apache.spark.api.java.JavaSparkContext) AlleleMetadataProducer(org.broadinstitute.hellbender.utils.hmm.interfaces.AlleleMetadataProducer) EmissionCalculationStrategy(org.broadinstitute.hellbender.tools.coveragemodel.CoverageModelCopyRatioEmissionProbabilityCalculator.EmissionCalculationStrategy) RobustBrentSolver(org.broadinstitute.hellbender.tools.coveragemodel.math.RobustBrentSolver) IntervalUtils(org.broadinstitute.hellbender.utils.IntervalUtils) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) HiddenStateSegmentRecord(org.broadinstitute.hellbender.utils.hmm.segmentation.HiddenStateSegmentRecord) ImmutableTriple(org.apache.commons.lang3.tuple.ImmutableTriple) IterativeLinearSolverNDArray(org.broadinstitute.hellbender.tools.coveragemodel.linalg.IterativeLinearSolverNDArray) GATKProtectedMathUtils(org.broadinstitute.hellbender.utils.GATKProtectedMathUtils) Nd4jIOUtils(org.broadinstitute.hellbender.tools.coveragemodel.nd4jutils.Nd4jIOUtils) IOException(java.io.IOException) JavaPairRDD(org.apache.spark.api.java.JavaPairRDD) ImmutablePair(org.apache.commons.lang3.tuple.ImmutablePair) File(java.io.File) INDArray(org.nd4j.linalg.api.ndarray.INDArray) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Transforms(org.nd4j.linalg.ops.transforms.Transforms) LogManager(org.apache.logging.log4j.LogManager) NoBracketingException(org.apache.commons.math3.exception.NoBracketingException) IndexRange(org.broadinstitute.hellbender.utils.IndexRange) INDArray(org.nd4j.linalg.api.ndarray.INDArray) SexGenotypeData(org.broadinstitute.hellbender.tools.exome.sexgenotyper.SexGenotypeData)

Example 48 with Range

use of org.apache.commons.lang3.Range in project alf.io by alfio-event.

the class CheckInManager method extractStatus.

private TicketAndCheckInResult extractStatus(Optional<Event> maybeEvent, Optional<Ticket> maybeTicket, String ticketIdentifier, Optional<String> ticketCode) {
    if (!maybeEvent.isPresent()) {
        return new TicketAndCheckInResult(null, new DefaultCheckInResult(EVENT_NOT_FOUND, "Event not found"));
    }
    if (!maybeTicket.isPresent()) {
        return new TicketAndCheckInResult(null, new DefaultCheckInResult(TICKET_NOT_FOUND, "Ticket with uuid " + ticketIdentifier + " not found"));
    }
    if (!ticketCode.filter(StringUtils::isNotEmpty).isPresent()) {
        return new TicketAndCheckInResult(null, new DefaultCheckInResult(EMPTY_TICKET_CODE, "Missing ticket code"));
    }
    Ticket ticket = maybeTicket.get();
    Event event = maybeEvent.get();
    String code = ticketCode.get();
    TicketCategory tc = ticketCategoryRepository.getById(ticket.getCategoryId());
    ZonedDateTime now = ZonedDateTime.now(event.getZoneId());
    if (!tc.hasValidCheckIn(now, event.getZoneId())) {
        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd/MM/yyyy - hh:mm");
        String from = tc.getValidCheckInFrom() == null ? ".." : formatter.format(tc.getValidCheckInFrom(event.getZoneId()));
        String to = tc.getValidCheckInTo() == null ? ".." : formatter.format(tc.getValidCheckInTo(event.getZoneId()));
        String formattedNow = formatter.format(now);
        return new TicketAndCheckInResult(ticket, new DefaultCheckInResult(INVALID_TICKET_CATEGORY_CHECK_IN_DATE, String.format("Invalid check-in date: valid range for category %s is from %s to %s, current time is: %s", tc.getName(), from, to, formattedNow)));
    }
    log.trace("scanned code is {}", code);
    log.trace("true code    is {}", ticket.ticketCode(event.getPrivateKey()));
    if (!code.equals(ticket.ticketCode(event.getPrivateKey()))) {
        return new TicketAndCheckInResult(null, new DefaultCheckInResult(INVALID_TICKET_CODE, "Ticket qr code does not match"));
    }
    final TicketStatus ticketStatus = ticket.getStatus();
    if (ticketStatus == TicketStatus.TO_BE_PAID) {
        return new TicketAndCheckInResult(ticket, new OnSitePaymentResult(MUST_PAY, "Must pay for ticket", MonetaryUtil.centsToUnit(ticket.getFinalPriceCts()), event.getCurrency()));
    }
    if (ticketStatus == TicketStatus.CHECKED_IN) {
        return new TicketAndCheckInResult(ticket, new DefaultCheckInResult(ALREADY_CHECK_IN, "Error: already checked in"));
    }
    if (ticket.getStatus() != TicketStatus.ACQUIRED) {
        return new TicketAndCheckInResult(ticket, new DefaultCheckInResult(INVALID_TICKET_STATE, "Invalid ticket state, expected ACQUIRED state, received " + ticket.getStatus()));
    }
    return new TicketAndCheckInResult(ticket, new DefaultCheckInResult(OK_READY_TO_BE_CHECKED_IN, "Ready to be checked in"));
}
Also used : OnSitePaymentResult(alfio.manager.support.OnSitePaymentResult) TicketAndCheckInResult(alfio.manager.support.TicketAndCheckInResult) ZonedDateTime(java.time.ZonedDateTime) StringUtils(org.apache.commons.lang3.StringUtils) TicketStatus(alfio.model.Ticket.TicketStatus) DefaultCheckInResult(alfio.manager.support.DefaultCheckInResult) DateTimeFormatter(java.time.format.DateTimeFormatter)

Example 49 with Range

use of org.apache.commons.lang3.Range in project metron by apache.

the class WindowProcessorTest method testRepeatWithConflictingExclusionInclusion.

@Test
public void testRepeatWithConflictingExclusionInclusion() throws ParseException {
    Window w = WindowProcessor.process("30 minute window every 24 hours from 7 days ago including saturdays excluding weekends");
    Date now = new Date();
    // avoid DST impacts if near Midnight
    now.setHours(6);
    List<Range<Long>> intervals = w.toIntervals(now.getTime());
    Assert.assertEquals(0, intervals.size());
}
Also used : Range(org.apache.commons.lang3.Range) Date(java.util.Date) Test(org.junit.Test)

Example 50 with Range

use of org.apache.commons.lang3.Range in project metron by apache.

the class WindowProcessorTest method testDateDaySpecifier.

@Test
public void testDateDaySpecifier() throws ParseException {
    for (String text : new String[] { "30 minute window every 24 hours from 14 days ago including date:20171225:yyyyMMdd", "30 minute window every 24 hours from 14 days ago including date:2017-12-25:yyyy-MM-dd", "30 minute window every 24 hours from 14 days ago including date:2017/12/25" }) {
        Window w = WindowProcessor.process(text);
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm");
        Date now = sdf.parse("2017/12/26 12:00");
        List<Range<Long>> intervals = w.toIntervals(now.getTime());
        Assert.assertEquals(1, intervals.size());
        Date includedDate = new Date(intervals.get(0).getMinimum());
        SimpleDateFormat equalityFormat = new SimpleDateFormat("yyyyMMdd");
        Assert.assertEquals("20171225", equalityFormat.format(includedDate));
    }
    {
        Window w = WindowProcessor.process("30 minute window every 24 hours from 14 days ago excluding date:2017/12/25");
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm");
        Date now = sdf.parse("2017/12/26 12:00");
        List<Range<Long>> intervals = w.toIntervals(now.getTime());
        Assert.assertEquals(13, intervals.size());
    }
    {
        Window w = WindowProcessor.process("30 minute window every 24 hours from 14 days ago including date:2017/12/25, date:2017/12/24");
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm");
        Date now = sdf.parse("2017/12/26 12:00");
        List<Range<Long>> intervals = w.toIntervals(now.getTime());
        Assert.assertEquals(2, intervals.size());
        {
            Date includedDate = new Date(intervals.get(0).getMinimum());
            SimpleDateFormat equalityFormat = new SimpleDateFormat("yyyyMMdd");
            Assert.assertEquals("20171224", equalityFormat.format(includedDate));
        }
        {
            Date includedDate = new Date(intervals.get(1).getMinimum());
            SimpleDateFormat equalityFormat = new SimpleDateFormat("yyyyMMdd");
            Assert.assertEquals("20171225", equalityFormat.format(includedDate));
        }
    }
}
Also used : List(java.util.List) Range(org.apache.commons.lang3.Range) SimpleDateFormat(java.text.SimpleDateFormat) Date(java.util.Date) Test(org.junit.Test)

Aggregations

List (java.util.List)26 Map (java.util.Map)21 HashMap (java.util.HashMap)20 ArrayList (java.util.ArrayList)19 Collectors (java.util.stream.Collectors)19 StringUtils (org.apache.commons.lang3.StringUtils)16 Logger (org.slf4j.Logger)15 LoggerFactory (org.slf4j.LoggerFactory)15 Set (java.util.Set)14 Pair (org.apache.commons.lang3.tuple.Pair)14 Test (org.junit.Test)14 IOException (java.io.IOException)12 Optional (java.util.Optional)11 Range (org.apache.commons.lang3.Range)11 Date (java.util.Date)10 HashSet (java.util.HashSet)9 ExecutorService (java.util.concurrent.ExecutorService)9 Collection (java.util.Collection)8 Stream (java.util.stream.Stream)8 Lists (com.google.common.collect.Lists)7