Examples with MetricStat com.amazonaws.services.cloudwatch.model.MetricStat used on opensource projects

Example 1 with MetricStat

use of com.amazonaws.services.cloudwatch.model.MetricStat in project aws-athena-query-federation by awslabs.

the class MetricsRecordHandlerTest method mockMetricData.

private GetMetricDataResult mockMetricData(InvocationOnMock invocation, int numMetrics, int numSamples) {
    GetMetricDataRequest request = invocation.getArgumentAt(0, GetMetricDataRequest.class);
    /**
     * Confirm that all available criteria were pushed down into Cloudwatch Metrics
     */
    List<MetricDataQuery> queries = request.getMetricDataQueries();
    assertEquals(1, queries.size());
    MetricDataQuery query = queries.get(0);
    MetricStat stat = query.getMetricStat();
    assertEquals("m1", query.getId());
    assertNotNull(stat.getPeriod());
    assertNotNull(stat.getMetric());
    assertNotNull(stat.getStat());
    assertNotNull(stat.getMetric().getMetricName());
    assertNotNull(stat.getMetric().getNamespace());
    assertNotNull(stat.getMetric().getDimensions());
    assertEquals(1, stat.getMetric().getDimensions().size());
    String nextToken = (request.getNextToken() == null) ? "valid" : null;
    List<MetricDataResult> samples = new ArrayList<>();
    for (int i = 0; i < numMetrics; i++) {
        List<Double> values = new ArrayList<>();
        List<Date> timestamps = new ArrayList<>();
        for (double j = 0; j < numSamples; j++) {
            values.add(j);
            timestamps.add(new Date(System.currentTimeMillis() + (int) j));
        }
        samples.add(new MetricDataResult().withValues(values).withTimestamps(timestamps).withId("m1"));
    }
    return new GetMetricDataResult().withNextToken(nextToken).withMetricDataResults(samples);
}

Example 2 with MetricStat

use of com.amazonaws.services.cloudwatch.model.MetricStat in project aws-athena-query-federation by awslabs.

the class MetricsRecordHandlerTest method readMetricSamplesWithConstraint.

@Test
public void readMetricSamplesWithConstraint() throws Exception {
    logger.info("readMetricSamplesWithConstraint: enter");
    String namespace = "namespace";
    String metricName = "metricName";
    String statistic = "p90";
    String period = "60";
    String dimName = "dimName";
    String dimValue = "dimValue";
    List<Dimension> dimensions = Collections.singletonList(new Dimension().withName(dimName).withValue(dimValue));
    int numMetrics = 10;
    int numSamples = 10;
    AtomicLong numCalls = new AtomicLong(0);
    when(mockMetrics.getMetricData(any(GetMetricDataRequest.class))).thenAnswer((InvocationOnMock invocation) -> {
        numCalls.incrementAndGet();
        return mockMetricData(invocation, numMetrics, numSamples);
    });
    Map<String, ValueSet> constraintsMap = new HashMap<>();
    constraintsMap.put(NAMESPACE_FIELD, makeStringEquals(allocator, namespace));
    constraintsMap.put(STATISTIC_FIELD, makeStringEquals(allocator, statistic));
    constraintsMap.put(DIMENSION_NAME_FIELD, makeStringEquals(allocator, dimName));
    constraintsMap.put(DIMENSION_VALUE_FIELD, makeStringEquals(allocator, dimValue));
    S3SpillLocation spillLocation = S3SpillLocation.newBuilder().withBucket(UUID.randomUUID().toString()).withSplitId(UUID.randomUUID().toString()).withQueryId(UUID.randomUUID().toString()).withIsDirectory(true).build();
    List<MetricStat> metricStats = new ArrayList<>();
    metricStats.add(new MetricStat().withMetric(new Metric().withNamespace(namespace).withMetricName(metricName).withDimensions(dimensions)).withPeriod(60).withStat(statistic));
    Split split = Split.newBuilder(spillLocation, keyFactory.create()).add(MetricStatSerDe.SERIALIZED_METRIC_STATS_FIELD_NAME, MetricStatSerDe.serialize(metricStats)).add(METRIC_NAME_FIELD, metricName).add(NAMESPACE_FIELD, namespace).add(STATISTIC_FIELD, statistic).add(PERIOD_FIELD, period).build();
    ReadRecordsRequest request = new ReadRecordsRequest(identity, "catalog", "queryId-" + System.currentTimeMillis(), METRIC_SAMPLES_TABLE_NAME, METRIC_DATA_TABLE.getSchema(), split, new Constraints(constraintsMap), 100_000_000_000L, // 100GB don't expect this to spill
    100_000_000_000L);
    RecordResponse rawResponse = handler.doReadRecords(allocator, request);
    assertTrue(rawResponse instanceof ReadRecordsResponse);
    ReadRecordsResponse response = (ReadRecordsResponse) rawResponse;
    logger.info("readMetricSamplesWithConstraint: rows[{}]", response.getRecordCount());
    assertEquals(numCalls.get() * numMetrics * numSamples, response.getRecords().getRowCount());
    logger.info("readMetricSamplesWithConstraint: {}", BlockUtils.rowToString(response.getRecords(), 0));
    logger.info("readMetricSamplesWithConstraint: exit");
}

Example 3 with MetricStat

use of com.amazonaws.services.cloudwatch.model.MetricStat in project aws-athena-query-federation by awslabs.

the class MetricsMetadataHandler method doGetSplits.

/**
 * Each 'metric' in cloudwatch is uniquely identified by a quad of Namespace, List<Dimension>, MetricName, Statistic. If the
 * query is for the METRIC_TABLE we return a single split.  If the query is for actual metrics data, we start forming batches
 * of metrics now that will form the basis of GetMetricData requests during readSplits.
 *
 * @see MetadataHandler
 */
@Override
public GetSplitsResponse doGetSplits(BlockAllocator blockAllocator, GetSplitsRequest getSplitsRequest) throws Exception {
    validateTable(getSplitsRequest.getTableName());
    // Handle requests for the METRIC_TABLE which requires only 1 split to list available metrics.
    if (METRIC_TABLE_NAME.equals(getSplitsRequest.getTableName().getTableName())) {
        // The request is just for meta-data about what metrics exist.
        Split metricsSplit = Split.newBuilder(makeSpillLocation(getSplitsRequest), makeEncryptionKey()).build();
        return new GetSplitsResponse(getSplitsRequest.getCatalogName(), metricsSplit);
    }
    // handle generating splits for reading actual metrics data.
    try (ConstraintEvaluator constraintEvaluator = new ConstraintEvaluator(blockAllocator, METRIC_DATA_TABLE.getSchema(), getSplitsRequest.getConstraints())) {
        ListMetricsRequest listMetricsRequest = new ListMetricsRequest();
        MetricUtils.pushDownPredicate(getSplitsRequest.getConstraints(), listMetricsRequest);
        listMetricsRequest.setNextToken(getSplitsRequest.getContinuationToken());
        String period = getPeriodFromConstraint(getSplitsRequest.getConstraints());
        Set<Split> splits = new HashSet<>();
        ListMetricsResult result = invoker.invoke(() -> metrics.listMetrics(listMetricsRequest));
        List<MetricStat> metricStats = new ArrayList<>(100);
        for (Metric nextMetric : result.getMetrics()) {
            for (String nextStatistic : STATISTICS) {
                if (MetricUtils.applyMetricConstraints(constraintEvaluator, nextMetric, nextStatistic)) {
                    metricStats.add(new MetricStat().withMetric(new Metric().withNamespace(nextMetric.getNamespace()).withMetricName(nextMetric.getMetricName()).withDimensions(nextMetric.getDimensions())).withPeriod(Integer.valueOf(period)).withStat(nextStatistic));
                }
            }
        }
        if (CollectionUtils.isNullOrEmpty(metricStats)) {
            logger.info("No metric stats present after filtering predicates.");
            return new GetSplitsResponse(getSplitsRequest.getCatalogName(), splits, null);
        }
        List<List<MetricStat>> partitions = Lists.partition(metricStats, calculateSplitSize(metricStats.size()));
        for (List<MetricStat> partition : partitions) {
            String serializedMetricStats = MetricStatSerDe.serialize(partition);
            splits.add(Split.newBuilder(makeSpillLocation(getSplitsRequest), makeEncryptionKey()).add(MetricStatSerDe.SERIALIZED_METRIC_STATS_FIELD_NAME, serializedMetricStats).build());
        }
        String continuationToken = null;
        if (result.getNextToken() != null && !result.getNextToken().equalsIgnoreCase(listMetricsRequest.getNextToken())) {
            continuationToken = result.getNextToken();
        }
        return new GetSplitsResponse(getSplitsRequest.getCatalogName(), splits, continuationToken);
    }
}

Example 4 with MetricStat

use of com.amazonaws.services.cloudwatch.model.MetricStat in project aws-athena-query-federation by awslabs.

the class MetricsRecordHandler method readMetricSamplesWithConstraint.

/**
 * Handles retrieving the samples for a specific metric from Cloudwatch Metrics.
 */
private void readMetricSamplesWithConstraint(BlockSpiller blockSpiller, ReadRecordsRequest request, QueryStatusChecker queryStatusChecker) throws TimeoutException {
    GetMetricDataRequest dataRequest = MetricUtils.makeGetMetricDataRequest(request);
    Map<String, MetricDataQuery> queries = new HashMap<>();
    for (MetricDataQuery query : dataRequest.getMetricDataQueries()) {
        queries.put(query.getId(), query);
    }
    String prevToken;
    ValueSet dimensionNameConstraint = request.getConstraints().getSummary().get(DIMENSION_NAME_FIELD);
    ValueSet dimensionValueConstraint = request.getConstraints().getSummary().get(DIMENSION_VALUE_FIELD);
    do {
        prevToken = dataRequest.getNextToken();
        GetMetricDataResult result = invoker.invoke(() -> metrics.getMetricData(dataRequest));
        for (MetricDataResult nextMetric : result.getMetricDataResults()) {
            MetricStat metricStat = queries.get(nextMetric.getId()).getMetricStat();
            List<Date> timestamps = nextMetric.getTimestamps();
            List<Double> values = nextMetric.getValues();
            for (int i = 0; i < nextMetric.getValues().size(); i++) {
                int sampleNum = i;
                blockSpiller.writeRows((Block block, int row) -> {
                    /**
                     * Most constraints were already applied at split generation so we only need to apply
                     * a subset.
                     */
                    block.offerValue(METRIC_NAME_FIELD, row, metricStat.getMetric().getMetricName());
                    block.offerValue(NAMESPACE_FIELD, row, metricStat.getMetric().getNamespace());
                    block.offerValue(STATISTIC_FIELD, row, metricStat.getStat());
                    block.offerComplexValue(DIMENSIONS_FIELD, row, (Field field, Object val) -> {
                        if (field.getName().equals(DIMENSION_NAME_FIELD)) {
                            return ((Dimension) val).getName();
                        } else if (field.getName().equals(DIMENSION_VALUE_FIELD)) {
                            return ((Dimension) val).getValue();
                        }
                        throw new RuntimeException("Unexpected field " + field.getName());
                    }, metricStat.getMetric().getDimensions());
                    // This field is 'faked' in that we just use it as a convenient way to filter single dimensions. As such
                    // we always populate it with the value of the filter if the constraint passed and the filter was singleValue
                    String dimName = (dimensionNameConstraint == null || !dimensionNameConstraint.isSingleValue()) ? null : dimensionNameConstraint.getSingleValue().toString();
                    block.offerValue(DIMENSION_NAME_FIELD, row, dimName);
                    // This field is 'faked' in that we just use it as a convenient way to filter single dimensions. As such
                    // we always populate it with the value of the filter if the constraint passed and the filter was singleValue
                    String dimVal = (dimensionValueConstraint == null || !dimensionValueConstraint.isSingleValue()) ? null : dimensionValueConstraint.getSingleValue().toString();
                    block.offerValue(DIMENSION_VALUE_FIELD, row, dimVal);
                    block.offerValue(PERIOD_FIELD, row, metricStat.getPeriod());
                    boolean matches = true;
                    block.offerValue(VALUE_FIELD, row, values.get(sampleNum));
                    long timestamp = timestamps.get(sampleNum).getTime() / 1000;
                    block.offerValue(TIMESTAMP_FIELD, row, timestamp);
                    return matches ? 1 : 0;
                });
            }
        }
        dataRequest.setNextToken(result.getNextToken());
    } while (dataRequest.getNextToken() != null && !dataRequest.getNextToken().equalsIgnoreCase(prevToken) && queryStatusChecker.isQueryRunning());
}

Example 5 with MetricStat

use of com.amazonaws.services.cloudwatch.model.MetricStat in project aws-athena-query-federation by awslabs.

the class MetricStatSerDeTest method serializeTest.

@Test
public void serializeTest() {
    String schema = "schema";
    String table = "table";
    Integer period = 60;
    String statistic = "p90";
    String metricName = "metricName";
    String namespace = "namespace";
    List<Dimension> dimensions = new ArrayList<>();
    dimensions.add(new Dimension().withName("dim_name1").withValue("dim_value1"));
    dimensions.add(new Dimension().withName("dim_name2").withValue("dim_value2"));
    List<MetricStat> metricStats = new ArrayList<>();
    metricStats.add(new MetricStat().withMetric(new Metric().withNamespace(namespace).withMetricName(metricName).withDimensions(dimensions)).withPeriod(60).withStat(statistic));
    String actualSerialization = MetricStatSerDe.serialize(metricStats);
    logger.info("serializeTest: {}", actualSerialization);
    List<MetricStat> actual = MetricStatSerDe.deserialize(actualSerialization);
    assertEquals(EXPECTED_SERIALIZATION, actualSerialization);
    assertEquals(metricStats, actual);
}