use of org.apache.commons.lang3.Range in project asterixdb by apache.
the class BTreeAccessMethod method createSecondaryToPrimaryPlan.
@Override
public ILogicalOperator createSecondaryToPrimaryPlan(Mutable<ILogicalExpression> conditionRef, OptimizableOperatorSubTree indexSubTree, OptimizableOperatorSubTree probeSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx, boolean retainInput, boolean retainNull, boolean requiresBroadcast, IOptimizationContext context) throws AlgebricksException {
Dataset dataset = indexSubTree.getDataset();
ARecordType recordType = indexSubTree.getRecordType();
ARecordType metaRecordType = indexSubTree.getMetaRecordType();
// we made sure indexSubTree has datasource scan
AbstractDataSourceOperator dataSourceOp = (AbstractDataSourceOperator) indexSubTree.getDataSourceRef().getValue();
List<Pair<Integer, Integer>> exprAndVarList = analysisCtx.getIndexExprsFromIndexExprsAndVars(chosenIndex);
int numSecondaryKeys = analysisCtx.getNumberOfMatchedKeys(chosenIndex);
// List of function expressions that will be replaced by the secondary-index search.
// These func exprs will be removed from the select condition at the very end of this method.
Set<ILogicalExpression> replacedFuncExprs = new HashSet<>();
// Info on high and low keys for the BTree search predicate.
ILogicalExpression[] lowKeyExprs = new ILogicalExpression[numSecondaryKeys];
ILogicalExpression[] highKeyExprs = new ILogicalExpression[numSecondaryKeys];
LimitType[] lowKeyLimits = new LimitType[numSecondaryKeys];
LimitType[] highKeyLimits = new LimitType[numSecondaryKeys];
boolean[] lowKeyInclusive = new boolean[numSecondaryKeys];
boolean[] highKeyInclusive = new boolean[numSecondaryKeys];
ILogicalExpression[] constantAtRuntimeExpressions = new ILogicalExpression[numSecondaryKeys];
LogicalVariable[] constAtRuntimeExprVars = new LogicalVariable[numSecondaryKeys];
/* TODO: For now we don't do any sophisticated analysis of the func exprs to come up with "the best" range
* predicate. If we can't figure out how to integrate a certain funcExpr into the current predicate,
* we just bail by setting this flag.*/
boolean couldntFigureOut = false;
boolean doneWithExprs = false;
boolean isEqCondition = false;
BitSet setLowKeys = new BitSet(numSecondaryKeys);
BitSet setHighKeys = new BitSet(numSecondaryKeys);
// Go through the func exprs listed as optimizable by the chosen index,
// and formulate a range predicate on the secondary-index keys.
// checks whether a type casting happened from a real (FLOAT, DOUBLE) value to an INT value
// since we have a round issues when dealing with LT(<) OR GT(>) operator.
boolean realTypeConvertedToIntegerType;
for (Pair<Integer, Integer> exprIndex : exprAndVarList) {
// Position of the field of matchedFuncExprs.get(exprIndex) in the chosen index's indexed exprs.
IOptimizableFuncExpr optFuncExpr = analysisCtx.getMatchedFuncExpr(exprIndex.first);
int keyPos = indexOf(optFuncExpr.getFieldName(0), chosenIndex.getKeyFieldNames());
if (keyPos < 0 && optFuncExpr.getNumLogicalVars() > 1) {
// If we are optimizing a join, the matching field may be the second field name.
keyPos = indexOf(optFuncExpr.getFieldName(1), chosenIndex.getKeyFieldNames());
}
if (keyPos < 0) {
throw CompilationException.create(ErrorCode.NO_INDEX_FIELD_NAME_FOR_GIVEN_FUNC_EXPR);
}
Pair<ILogicalExpression, Boolean> returnedSearchKeyExpr = AccessMethodUtils.createSearchKeyExpr(optFuncExpr, indexSubTree, probeSubTree);
ILogicalExpression searchKeyExpr = returnedSearchKeyExpr.first;
if (searchKeyExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
constantAtRuntimeExpressions[keyPos] = searchKeyExpr;
constAtRuntimeExprVars[keyPos] = context.newVar();
searchKeyExpr = new VariableReferenceExpression(constAtRuntimeExprVars[keyPos]);
}
realTypeConvertedToIntegerType = returnedSearchKeyExpr.second;
LimitType limit = getLimitType(optFuncExpr, probeSubTree);
//
if (realTypeConvertedToIntegerType) {
if (limit == LimitType.HIGH_EXCLUSIVE) {
limit = LimitType.HIGH_INCLUSIVE;
} else if (limit == LimitType.LOW_EXCLUSIVE) {
limit = LimitType.LOW_INCLUSIVE;
}
}
switch(limit) {
case EQUAL:
{
if (lowKeyLimits[keyPos] == null && highKeyLimits[keyPos] == null) {
lowKeyLimits[keyPos] = highKeyLimits[keyPos] = limit;
lowKeyInclusive[keyPos] = highKeyInclusive[keyPos] = true;
lowKeyExprs[keyPos] = highKeyExprs[keyPos] = searchKeyExpr;
setLowKeys.set(keyPos);
setHighKeys.set(keyPos);
isEqCondition = true;
} else {
// (once from analyzing each side of the join)
if (lowKeyLimits[keyPos] == limit && lowKeyInclusive[keyPos] == true && lowKeyExprs[keyPos].equals(searchKeyExpr) && highKeyLimits[keyPos] == limit && highKeyInclusive[keyPos] == true && highKeyExprs[keyPos].equals(searchKeyExpr)) {
isEqCondition = true;
break;
}
couldntFigureOut = true;
}
// If high and low keys are set, we exit for now.
if (setLowKeys.cardinality() == numSecondaryKeys && setHighKeys.cardinality() == numSecondaryKeys) {
doneWithExprs = true;
}
break;
}
case HIGH_EXCLUSIVE:
{
if (highKeyLimits[keyPos] == null || (highKeyLimits[keyPos] != null && highKeyInclusive[keyPos])) {
highKeyLimits[keyPos] = limit;
highKeyExprs[keyPos] = searchKeyExpr;
highKeyInclusive[keyPos] = false;
} else {
// (once from analyzing each side of the join)
if (highKeyLimits[keyPos] == limit && highKeyInclusive[keyPos] == false && highKeyExprs[keyPos].equals(searchKeyExpr)) {
break;
}
couldntFigureOut = true;
doneWithExprs = true;
}
break;
}
case HIGH_INCLUSIVE:
{
if (highKeyLimits[keyPos] == null) {
highKeyLimits[keyPos] = limit;
highKeyExprs[keyPos] = searchKeyExpr;
highKeyInclusive[keyPos] = true;
} else {
// (once from analyzing each side of the join)
if (highKeyLimits[keyPos] == limit && highKeyInclusive[keyPos] == true && highKeyExprs[keyPos].equals(searchKeyExpr)) {
break;
}
couldntFigureOut = true;
doneWithExprs = true;
}
break;
}
case LOW_EXCLUSIVE:
{
if (lowKeyLimits[keyPos] == null || (lowKeyLimits[keyPos] != null && lowKeyInclusive[keyPos])) {
lowKeyLimits[keyPos] = limit;
lowKeyExprs[keyPos] = searchKeyExpr;
lowKeyInclusive[keyPos] = false;
} else {
// (once from analyzing each side of the join)
if (lowKeyLimits[keyPos] == limit && lowKeyInclusive[keyPos] == false && lowKeyExprs[keyPos].equals(searchKeyExpr)) {
break;
}
couldntFigureOut = true;
doneWithExprs = true;
}
break;
}
case LOW_INCLUSIVE:
{
if (lowKeyLimits[keyPos] == null) {
lowKeyLimits[keyPos] = limit;
lowKeyExprs[keyPos] = searchKeyExpr;
lowKeyInclusive[keyPos] = true;
} else {
// (once from analyzing each side of the join)
if (lowKeyLimits[keyPos] == limit && lowKeyInclusive[keyPos] == true && lowKeyExprs[keyPos].equals(searchKeyExpr)) {
break;
}
couldntFigureOut = true;
doneWithExprs = true;
}
break;
}
default:
{
throw new IllegalStateException();
}
}
if (!couldntFigureOut) {
// Remember to remove this funcExpr later.
replacedFuncExprs.add(analysisCtx.getMatchedFuncExpr(exprIndex.first).getFuncExpr());
}
if (doneWithExprs) {
break;
}
}
if (couldntFigureOut) {
return null;
}
// If the select condition contains mixed open/closed intervals on multiple keys, then we make all intervals
// closed to obtain a superset of answers and leave the original selection in place.
boolean primaryIndexPostProccessingIsNeeded = false;
for (int i = 1; i < numSecondaryKeys; ++i) {
if (lowKeyInclusive[i] != lowKeyInclusive[0]) {
Arrays.fill(lowKeyInclusive, true);
primaryIndexPostProccessingIsNeeded = true;
break;
}
}
for (int i = 1; i < numSecondaryKeys; ++i) {
if (highKeyInclusive[i] != highKeyInclusive[0]) {
Arrays.fill(highKeyInclusive, true);
primaryIndexPostProccessingIsNeeded = true;
break;
}
}
// determine cases when prefix search could be applied
for (int i = 1; i < lowKeyExprs.length; i++) {
if (lowKeyLimits[0] == null && lowKeyLimits[i] != null || lowKeyLimits[0] != null && lowKeyLimits[i] == null || highKeyLimits[0] == null && highKeyLimits[i] != null || highKeyLimits[0] != null && highKeyLimits[i] == null) {
numSecondaryKeys--;
primaryIndexPostProccessingIsNeeded = true;
}
}
if (lowKeyLimits[0] == null) {
lowKeyInclusive[0] = true;
}
if (highKeyLimits[0] == null) {
highKeyInclusive[0] = true;
}
// Here we generate vars and funcs for assigning the secondary-index keys to be fed into the secondary-index
// search.
// List of variables for the assign.
ArrayList<LogicalVariable> keyVarList = new ArrayList<>();
// List of variables and expressions for the assign.
ArrayList<LogicalVariable> assignKeyVarList = new ArrayList<>();
ArrayList<Mutable<ILogicalExpression>> assignKeyExprList = new ArrayList<>();
int numLowKeys = createKeyVarsAndExprs(numSecondaryKeys, lowKeyLimits, lowKeyExprs, assignKeyVarList, assignKeyExprList, keyVarList, context, constantAtRuntimeExpressions, constAtRuntimeExprVars);
int numHighKeys = createKeyVarsAndExprs(numSecondaryKeys, highKeyLimits, highKeyExprs, assignKeyVarList, assignKeyExprList, keyVarList, context, constantAtRuntimeExpressions, constAtRuntimeExprVars);
BTreeJobGenParams jobGenParams = new BTreeJobGenParams(chosenIndex.getIndexName(), IndexType.BTREE, dataset.getDataverseName(), dataset.getDatasetName(), retainInput, requiresBroadcast);
jobGenParams.setLowKeyInclusive(lowKeyInclusive[0]);
jobGenParams.setHighKeyInclusive(highKeyInclusive[0]);
jobGenParams.setIsEqCondition(isEqCondition);
jobGenParams.setLowKeyVarList(keyVarList, 0, numLowKeys);
jobGenParams.setHighKeyVarList(keyVarList, numLowKeys, numHighKeys);
ILogicalOperator inputOp = null;
if (!assignKeyVarList.isEmpty()) {
// Assign operator that sets the constant secondary-index search-key fields if necessary.
AssignOperator assignConstantSearchKeys = new AssignOperator(assignKeyVarList, assignKeyExprList);
// Input to this assign is the EmptyTupleSource (which the dataSourceScan also must have had as input).
assignConstantSearchKeys.getInputs().add(new MutableObject<>(OperatorManipulationUtil.deepCopy(dataSourceOp.getInputs().get(0).getValue())));
assignConstantSearchKeys.setExecutionMode(dataSourceOp.getExecutionMode());
inputOp = assignConstantSearchKeys;
} else if (probeSubTree == null) {
//nonpure case
//Make sure that the nonpure function is unpartitioned
ILogicalOperator checkOp = dataSourceOp.getInputs().get(0).getValue();
while (checkOp.getExecutionMode() != ExecutionMode.UNPARTITIONED) {
if (checkOp.getInputs().size() == 1) {
checkOp = checkOp.getInputs().get(0).getValue();
} else {
return null;
}
}
inputOp = dataSourceOp.getInputs().get(0).getValue();
} else {
// All index search keys are variables.
inputOp = probeSubTree.getRoot();
}
ILogicalOperator secondaryIndexUnnestOp = AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType, metaRecordType, chosenIndex, inputOp, jobGenParams, context, false, retainInput, retainNull);
// Generate the rest of the upstream plan which feeds the search results into the primary index.
AbstractUnnestMapOperator primaryIndexUnnestOp = null;
boolean isPrimaryIndex = chosenIndex.isPrimaryIndex();
if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
// External dataset
UnnestMapOperator externalDataAccessOp = AccessMethodUtils.createExternalDataLookupUnnestMap(dataSourceOp, dataset, recordType, secondaryIndexUnnestOp, context, retainInput, retainNull);
indexSubTree.getDataSourceRef().setValue(externalDataAccessOp);
return externalDataAccessOp;
} else if (!isPrimaryIndex) {
primaryIndexUnnestOp = AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceOp, dataset, recordType, metaRecordType, secondaryIndexUnnestOp, context, true, retainInput, retainNull, false);
// Adds equivalence classes --- one equivalent class between a primary key
// variable and a record field-access expression.
EquivalenceClassUtils.addEquivalenceClassesForPrimaryIndexAccess(primaryIndexUnnestOp, dataSourceOp.getVariables(), recordType, metaRecordType, dataset, context);
} else {
List<Object> primaryIndexOutputTypes = new ArrayList<>();
AccessMethodUtils.appendPrimaryIndexTypes(dataset, recordType, metaRecordType, primaryIndexOutputTypes);
List<LogicalVariable> scanVariables = dataSourceOp.getVariables();
// If not, we create a new condition based on remaining ones.
if (!primaryIndexPostProccessingIsNeeded) {
List<Mutable<ILogicalExpression>> remainingFuncExprs = new ArrayList<>();
try {
getNewConditionExprs(conditionRef, replacedFuncExprs, remainingFuncExprs);
} catch (CompilationException e) {
return null;
}
// Generate new condition.
if (!remainingFuncExprs.isEmpty()) {
ILogicalExpression pulledCond = createSelectCondition(remainingFuncExprs);
conditionRef.setValue(pulledCond);
} else {
conditionRef.setValue(null);
}
}
// Checks whether LEFT_OUTER_UNNESTMAP operator is required.
boolean leftOuterUnnestMapRequired = false;
if (retainNull && retainInput) {
leftOuterUnnestMapRequired = true;
} else {
leftOuterUnnestMapRequired = false;
}
if (conditionRef.getValue() != null) {
// The job gen parameters are transferred to the actual job gen
// via the UnnestMapOperator's function arguments.
List<Mutable<ILogicalExpression>> primaryIndexFuncArgs = new ArrayList<>();
jobGenParams.writeToFuncArgs(primaryIndexFuncArgs);
// An index search is expressed as an unnest-map over an
// index-search function.
IFunctionInfo primaryIndexSearch = FunctionUtil.getFunctionInfo(BuiltinFunctions.INDEX_SEARCH);
UnnestingFunctionCallExpression primaryIndexSearchFunc = new UnnestingFunctionCallExpression(primaryIndexSearch, primaryIndexFuncArgs);
primaryIndexSearchFunc.setReturnsUniqueValues(true);
if (!leftOuterUnnestMapRequired) {
primaryIndexUnnestOp = new UnnestMapOperator(scanVariables, new MutableObject<ILogicalExpression>(primaryIndexSearchFunc), primaryIndexOutputTypes, retainInput);
} else {
primaryIndexUnnestOp = new LeftOuterUnnestMapOperator(scanVariables, new MutableObject<ILogicalExpression>(primaryIndexSearchFunc), primaryIndexOutputTypes, true);
}
} else {
if (!leftOuterUnnestMapRequired) {
primaryIndexUnnestOp = new UnnestMapOperator(scanVariables, ((UnnestMapOperator) secondaryIndexUnnestOp).getExpressionRef(), primaryIndexOutputTypes, retainInput);
} else {
primaryIndexUnnestOp = new LeftOuterUnnestMapOperator(scanVariables, ((LeftOuterUnnestMapOperator) secondaryIndexUnnestOp).getExpressionRef(), primaryIndexOutputTypes, true);
}
}
primaryIndexUnnestOp.getInputs().add(new MutableObject<>(inputOp));
// Adds equivalence classes --- one equivalent class between a primary key
// variable and a record field-access expression.
EquivalenceClassUtils.addEquivalenceClassesForPrimaryIndexAccess(primaryIndexUnnestOp, scanVariables, recordType, metaRecordType, dataset, context);
}
return primaryIndexUnnestOp;
}
use of org.apache.commons.lang3.Range in project gatk-protected by broadinstitute.
the class CoverageModelEMWorkspace method getViterbiAsNDArray.
/**
* Fetches the Viterbi copy ratio (or copy number) states as a target-sample matrix
*
* @return an {@link INDArray}
*/
protected INDArray getViterbiAsNDArray(final List<List<HiddenStateSegmentRecord<STATE, Target>>> segments) {
final INDArray res = Nd4j.create(numSamples, numTargets);
final TargetCollection<Target> targetCollection = new HashedListTargetCollection<>(processedTargetList);
for (int si = 0; si < numSamples; si++) {
final SexGenotypeData sampleSexGenotype = processedSampleSexGenotypeData.get(si);
/* start with all ref */
final double[] calls = IntStream.range(0, numTargets).mapToDouble(ti -> referenceStateFactory.apply(sampleSexGenotype, processedTargetList.get(ti)).getScalar()).toArray();
/* go through segments and mutate ref calls as necessary */
segments.get(si).forEach(seg -> {
final IndexRange range = targetCollection.indexRange(seg.getSegment());
final double copyRatio = seg.getSegment().getCall().getScalar();
for (int ti = range.from; ti < range.to; ti++) {
calls[ti] = copyRatio;
}
});
res.get(NDArrayIndex.point(si), NDArrayIndex.all()).assign(Nd4j.create(calls, new int[] { 1, numTargets }));
}
return res.transpose();
}
use of org.apache.commons.lang3.Range in project alf.io by alfio-event.
the class CheckInManager method extractStatus.
private TicketAndCheckInResult extractStatus(Optional<Event> maybeEvent, Optional<Ticket> maybeTicket, String ticketIdentifier, Optional<String> ticketCode) {
if (!maybeEvent.isPresent()) {
return new TicketAndCheckInResult(null, new DefaultCheckInResult(EVENT_NOT_FOUND, "Event not found"));
}
if (!maybeTicket.isPresent()) {
return new TicketAndCheckInResult(null, new DefaultCheckInResult(TICKET_NOT_FOUND, "Ticket with uuid " + ticketIdentifier + " not found"));
}
if (!ticketCode.filter(StringUtils::isNotEmpty).isPresent()) {
return new TicketAndCheckInResult(null, new DefaultCheckInResult(EMPTY_TICKET_CODE, "Missing ticket code"));
}
Ticket ticket = maybeTicket.get();
Event event = maybeEvent.get();
String code = ticketCode.get();
TicketCategory tc = ticketCategoryRepository.getById(ticket.getCategoryId());
ZonedDateTime now = ZonedDateTime.now(event.getZoneId());
if (!tc.hasValidCheckIn(now, event.getZoneId())) {
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd/MM/yyyy - hh:mm");
String from = tc.getValidCheckInFrom() == null ? ".." : formatter.format(tc.getValidCheckInFrom(event.getZoneId()));
String to = tc.getValidCheckInTo() == null ? ".." : formatter.format(tc.getValidCheckInTo(event.getZoneId()));
String formattedNow = formatter.format(now);
return new TicketAndCheckInResult(ticket, new DefaultCheckInResult(INVALID_TICKET_CATEGORY_CHECK_IN_DATE, String.format("Invalid check-in date: valid range for category %s is from %s to %s, current time is: %s", tc.getName(), from, to, formattedNow)));
}
log.trace("scanned code is {}", code);
log.trace("true code is {}", ticket.ticketCode(event.getPrivateKey()));
if (!code.equals(ticket.ticketCode(event.getPrivateKey()))) {
return new TicketAndCheckInResult(null, new DefaultCheckInResult(INVALID_TICKET_CODE, "Ticket qr code does not match"));
}
final TicketStatus ticketStatus = ticket.getStatus();
if (ticketStatus == TicketStatus.TO_BE_PAID) {
return new TicketAndCheckInResult(ticket, new OnSitePaymentResult(MUST_PAY, "Must pay for ticket", MonetaryUtil.centsToUnit(ticket.getFinalPriceCts()), event.getCurrency()));
}
if (ticketStatus == TicketStatus.CHECKED_IN) {
return new TicketAndCheckInResult(ticket, new DefaultCheckInResult(ALREADY_CHECK_IN, "Error: already checked in"));
}
if (ticket.getStatus() != TicketStatus.ACQUIRED) {
return new TicketAndCheckInResult(ticket, new DefaultCheckInResult(INVALID_TICKET_STATE, "Invalid ticket state, expected ACQUIRED state, received " + ticket.getStatus()));
}
return new TicketAndCheckInResult(ticket, new DefaultCheckInResult(OK_READY_TO_BE_CHECKED_IN, "Ready to be checked in"));
}
use of org.apache.commons.lang3.Range in project metron by apache.
the class WindowProcessorTest method testRepeatWithConflictingExclusionInclusion.
@Test
public void testRepeatWithConflictingExclusionInclusion() throws ParseException {
Window w = WindowProcessor.process("30 minute window every 24 hours from 7 days ago including saturdays excluding weekends");
Date now = new Date();
// avoid DST impacts if near Midnight
now.setHours(6);
List<Range<Long>> intervals = w.toIntervals(now.getTime());
Assert.assertEquals(0, intervals.size());
}
use of org.apache.commons.lang3.Range in project metron by apache.
the class WindowProcessorTest method testDateDaySpecifier.
@Test
public void testDateDaySpecifier() throws ParseException {
for (String text : new String[] { "30 minute window every 24 hours from 14 days ago including date:20171225:yyyyMMdd", "30 minute window every 24 hours from 14 days ago including date:2017-12-25:yyyy-MM-dd", "30 minute window every 24 hours from 14 days ago including date:2017/12/25" }) {
Window w = WindowProcessor.process(text);
SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm");
Date now = sdf.parse("2017/12/26 12:00");
List<Range<Long>> intervals = w.toIntervals(now.getTime());
Assert.assertEquals(1, intervals.size());
Date includedDate = new Date(intervals.get(0).getMinimum());
SimpleDateFormat equalityFormat = new SimpleDateFormat("yyyyMMdd");
Assert.assertEquals("20171225", equalityFormat.format(includedDate));
}
{
Window w = WindowProcessor.process("30 minute window every 24 hours from 14 days ago excluding date:2017/12/25");
SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm");
Date now = sdf.parse("2017/12/26 12:00");
List<Range<Long>> intervals = w.toIntervals(now.getTime());
Assert.assertEquals(13, intervals.size());
}
{
Window w = WindowProcessor.process("30 minute window every 24 hours from 14 days ago including date:2017/12/25, date:2017/12/24");
SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm");
Date now = sdf.parse("2017/12/26 12:00");
List<Range<Long>> intervals = w.toIntervals(now.getTime());
Assert.assertEquals(2, intervals.size());
{
Date includedDate = new Date(intervals.get(0).getMinimum());
SimpleDateFormat equalityFormat = new SimpleDateFormat("yyyyMMdd");
Assert.assertEquals("20171224", equalityFormat.format(includedDate));
}
{
Date includedDate = new Date(intervals.get(1).getMinimum());
SimpleDateFormat equalityFormat = new SimpleDateFormat("yyyyMMdd");
Assert.assertEquals("20171225", equalityFormat.format(includedDate));
}
}
}
Aggregations