Examples with Range com.amazonaws.athena.connector.lambda.domain.predicate.Range used on opensource projects

Example 31 with Range

use of com.amazonaws.athena.connector.lambda.domain.predicate.Range in project aws-athena-query-federation by awslabs.

the class MetricUtils method makeGetMetricDataRequest.

/**
 * Creates a Cloudwatch Metrics sample data request from the provided inputs
 *
 * @param readRecordsRequest The RecordReadRequest to make into a Cloudwatch Metrics Data request.
 * @return The Cloudwatch Metrics Data request that matches the requested read operation.
 */
protected static GetMetricDataRequest makeGetMetricDataRequest(ReadRecordsRequest readRecordsRequest) {
    Split split = readRecordsRequest.getSplit();
    String serializedMetricStats = split.getProperty(MetricStatSerDe.SERIALIZED_METRIC_STATS_FIELD_NAME);
    List<MetricStat> metricStats = MetricStatSerDe.deserialize(serializedMetricStats);
    GetMetricDataRequest dataRequest = new GetMetricDataRequest();
    com.amazonaws.services.cloudwatch.model.Metric metric = new com.amazonaws.services.cloudwatch.model.Metric();
    metric.setNamespace(split.getProperty(NAMESPACE_FIELD));
    metric.setMetricName(split.getProperty(METRIC_NAME_FIELD));
    List<MetricDataQuery> metricDataQueries = new ArrayList<>();
    int metricId = 1;
    for (MetricStat nextMetricStat : metricStats) {
        metricDataQueries.add(new MetricDataQuery().withMetricStat(nextMetricStat).withId("m" + metricId++));
    }
    dataRequest.withMetricDataQueries(metricDataQueries);
    ValueSet timeConstraint = readRecordsRequest.getConstraints().getSummary().get(TIMESTAMP_FIELD);
    if (timeConstraint instanceof SortedRangeSet && !timeConstraint.isNullAllowed()) {
        // SortedRangeSet is how >, <, between is represented which are easiest and most common when
        // searching logs so we attempt to push that down here as an optimization. SQL can represent complex
        // overlapping ranges which Cloudwatch can not support so this is not a replacement for applying
        // constraints using the ConstraintEvaluator.
        Range basicPredicate = ((SortedRangeSet) timeConstraint).getSpan();
        if (!basicPredicate.getLow().isNullValue()) {
            Long lowerBound = (Long) basicPredicate.getLow().getValue();
            // TODO: confirm timezone handling
            logger.info("makeGetMetricsRequest: with startTime " + (lowerBound * 1000) + " " + new Date(lowerBound * 1000));
            dataRequest.withStartTime(new Date(lowerBound * 1000));
        } else {
            // TODO: confirm timezone handling
            dataRequest.withStartTime(new Date(0));
        }
        if (!basicPredicate.getHigh().isNullValue()) {
            Long upperBound = (Long) basicPredicate.getHigh().getValue();
            // TODO: confirm timezone handling
            logger.info("makeGetMetricsRequest: with endTime " + (upperBound * 1000) + " " + new Date(upperBound * 1000));
            dataRequest.withEndTime(new Date(upperBound * 1000));
        } else {
            // TODO: confirm timezone handling
            dataRequest.withEndTime(new Date(System.currentTimeMillis()));
        }
    } else {
        // TODO: confirm timezone handling
        dataRequest.withStartTime(new Date(0));
        dataRequest.withEndTime(new Date(System.currentTimeMillis()));
    }
    return dataRequest;
}

Example 32 with Range

use of com.amazonaws.athena.connector.lambda.domain.predicate.Range in project aws-athena-query-federation by awslabs.

the class DataLakeRecordHandlerTest method getSingleValueSet.

private ValueSet getSingleValueSet(Object value) {
    Range range = Mockito.mock(Range.class, Mockito.RETURNS_DEEP_STUBS);
    Mockito.when(range.isSingleValue()).thenReturn(true);
    Mockito.when(range.getLow().getValue()).thenReturn(value);
    ValueSet valueSet = Mockito.mock(SortedRangeSet.class, Mockito.RETURNS_DEEP_STUBS);
    Mockito.when(valueSet.getRanges().getOrderedRanges()).thenReturn(Collections.singletonList(range));
    return valueSet;
}

Example 33 with Range

use of com.amazonaws.athena.connector.lambda.domain.predicate.Range in project aws-athena-query-federation by awslabs.

the class QueryUtils method makePredicate.

/**
 * Converts a single field constraint into a Document for use in a DocumentDB query.
 *
 * @param field The field for the given ValueSet constraint.
 * @param constraint The constraint to apply to the given field.
 * @return A Document describing the constraint for pushing down into DocumentDB.
 */
public static Document makePredicate(Field field, ValueSet constraint) {
    String name = field.getName();
    if (constraint.isNone()) {
        return documentOf(name, isNullPredicate());
    }
    if (constraint.isAll()) {
        return documentOf(name, isNotNullPredicate());
    }
    if (constraint.isNullAllowed()) {
        // TODO: support nulls mixed with discrete value constraints
        return null;
    }
    if (constraint instanceof EquatableValueSet) {
        Block block = ((EquatableValueSet) constraint).getValues();
        List<Object> singleValues = new ArrayList<>();
        FieldReader fieldReader = block.getFieldReaders().get(0);
        for (int i = 0; i < block.getRowCount(); i++) {
            Document nextEqVal = new Document();
            fieldReader.setPosition(i);
            Object value = fieldReader.readObject();
            nextEqVal.put(EQ_OP, convert(value));
            singleValues.add(singleValues);
        }
        return orPredicate(singleValues.stream().map(next -> new Document(name, next)).collect(toList()));
    }
    List<Object> singleValues = new ArrayList<>();
    List<Document> disjuncts = new ArrayList<>();
    for (Range range : constraint.getRanges().getOrderedRanges()) {
        if (range.isSingleValue()) {
            singleValues.add(convert(range.getSingleValue()));
        } else {
            Document rangeConjuncts = new Document();
            if (!range.getLow().isLowerUnbounded()) {
                switch(range.getLow().getBound()) {
                    case ABOVE:
                        rangeConjuncts.put(GT_OP, convert(range.getLow().getValue()));
                        break;
                    case EXACTLY:
                        rangeConjuncts.put(GTE_OP, convert(range.getLow().getValue()));
                        break;
                    case BELOW:
                        throw new IllegalArgumentException("Low Marker should never use BELOW bound: " + range);
                    default:
                        throw new AssertionError("Unhandled bound: " + range.getLow().getBound());
                }
            }
            if (!range.getHigh().isUpperUnbounded()) {
                switch(range.getHigh().getBound()) {
                    case ABOVE:
                        throw new IllegalArgumentException("High Marker should never use ABOVE bound: " + range);
                    case EXACTLY:
                        rangeConjuncts.put(LTE_OP, convert(range.getHigh().getValue()));
                        break;
                    case BELOW:
                        rangeConjuncts.put(LT_OP, convert(range.getHigh().getValue()));
                        break;
                    default:
                        throw new AssertionError("Unhandled bound: " + range.getHigh().getBound());
                }
            }
            // If rangeConjuncts is null, then the range was ALL, which should already have been checked for
            verify(!rangeConjuncts.isEmpty());
            disjuncts.add(rangeConjuncts);
        }
    }
    // Add back all of the possible single values either as an equality or an IN predicate
    if (singleValues.size() == 1) {
        disjuncts.add(documentOf(EQ_OP, singleValues.get(0)));
    } else if (singleValues.size() > 1) {
        disjuncts.add(documentOf(IN_OP, singleValues));
    }
    return orPredicate(disjuncts.stream().map(disjunct -> new Document(name, disjunct)).collect(toList()));
}

Example 34 with Range

use of com.amazonaws.athena.connector.lambda.domain.predicate.Range in project aws-athena-query-federation by awslabs.

the class ElasticsearchQueryUtils method getPredicateFromRange.

/**
 * Converts a range constraint into a predicate to use in an Elasticsearch query.
 * @param fieldName The name of the field for the given ValueSet constraint.
 * @param constraint The constraint to apply to the given field.
 * @return A string describing the constraint for pushing down into Elasticsearch.
 */
private static String getPredicateFromRange(String fieldName, ValueSet constraint) {
    List<String> singleValues = new ArrayList<>();
    List<String> disjuncts = new ArrayList<>();
    for (Range range : constraint.getRanges().getOrderedRanges()) {
        if (range.isSingleValue()) {
            String singleValue = range.getSingleValue().toString();
            if (range.getType() instanceof ArrowType.Date) {
                // Wrap a single date in quotes, e.g. my-birthday:("2000-11-11T06:57:44.123")
                singleValues.add("\"" + singleValue + "\"");
            } else {
                singleValues.add(singleValue);
            }
        } else {
            String rangeConjuncts;
            if (range.getLow().isLowerUnbounded()) {
                rangeConjuncts = LOWER_UNBOUNDED_RANGE;
            } else {
                switch(range.getLow().getBound()) {
                    case EXACTLY:
                        rangeConjuncts = LOWER_INCLUSIVE_RANGE + range.getLow().getValue().toString();
                        break;
                    case ABOVE:
                        rangeConjuncts = LOWER_EXCLUSIVE_RANGE + range.getLow().getValue().toString();
                        break;
                    case BELOW:
                        logger.warn("Low Marker should never use BELOW bound: " + range);
                        continue;
                    default:
                        logger.warn("Unhandled bound: " + range.getLow().getBound());
                        continue;
                }
            }
            rangeConjuncts += RANGE_OPER;
            if (range.getHigh().isUpperUnbounded()) {
                rangeConjuncts += UPPER_UNBOUNDED_RANGE;
            } else {
                switch(range.getHigh().getBound()) {
                    case EXACTLY:
                        rangeConjuncts += range.getHigh().getValue().toString() + UPPER_INCLUSIVE_RANGE;
                        break;
                    case BELOW:
                        rangeConjuncts += range.getHigh().getValue().toString() + UPPER_EXCLUSIVE_RANGE;
                        break;
                    case ABOVE:
                        logger.warn("High Marker should never use ABOVE bound: " + range);
                        continue;
                    default:
                        logger.warn("Unhandled bound: " + range.getHigh().getBound());
                        continue;
                }
            }
            disjuncts.add(rangeConjuncts);
        }
    }
    if (!singleValues.isEmpty()) {
        // value1 OR value2 OR value3...
        disjuncts.add(Strings.collectionToDelimitedString(singleValues, OR_OPER));
    }
    if (disjuncts.isEmpty()) {
        // There are no ranges stored.
        return EMPTY_PREDICATE;
    }
    // field:([value1 TO value2] OR value3 OR value4 OR value5...)
    return fieldName + ":(" + Strings.collectionToDelimitedString(disjuncts, OR_OPER) + ")";
}