Examples with FieldDetail co.cask.cdap.dq.FieldDetail used on opensource projects

Example 1 with FieldDetail

use of co.cask.cdap.dq.FieldDetail in project cdap by caskdata.

the class DataQualityAppTest method testTotals.

@Test
public void testTotals() throws Exception {
    Map<String, Set<String>> testMap = new HashMap<>();
    Set<String> testSet = new HashSet<>();
    testSet.add("DiscreteValuesHistogram");
    testMap.put("content_length", testSet);
    testMap.put("status", testSet);
    testMap.put("request_time", testSet);
    DataQualityApp.DataQualityConfig config = new DataQualityApp.DataQualityConfig(WORKFLOW_SCHEDULE_MINUTES, getStreamSource(), "histogram", testMap);
    ApplicationId appId = NamespaceId.DEFAULT.app("newApp3");
    AppRequest<DataQualityApp.DataQualityConfig> appRequest = new AppRequest<>(new ArtifactSummary(appArtifact.getArtifact(), appArtifact.getVersion()), config);
    ApplicationManager applicationManager = deployApplication(appId, appRequest);
    MapReduceManager mrManager = applicationManager.getMapReduceManager("FieldAggregator").start();
    mrManager.waitForRun(ProgramRunStatus.COMPLETED, 180, TimeUnit.SECONDS);
    Map<String, Integer> expectedMap = new HashMap<>();
    expectedMap.put("256", 3);
    /* Test for the aggregationsGetter handler */
    ServiceManager serviceManager = applicationManager.getServiceManager(DataQualityService.SERVICE_NAME).start();
    serviceManager.waitForStatus(true);
    URL url = new URL(serviceManager.getServiceURL(), "v1/sources/logStream/fields/content_length/aggregations/DiscreteValuesHistogram/totals");
    HttpResponse httpResponse = HttpRequests.execute(HttpRequest.get(url).build());
    Assert.assertEquals(HttpURLConnection.HTTP_OK, httpResponse.getResponseCode());
    String response = httpResponse.getResponseBodyAsString();
    Map<String, Integer> histogramMap = GSON.fromJson(response, TOKEN_TYPE_MAP_STRING_INTEGER);
    Assert.assertEquals(expectedMap, histogramMap);
    /* Test for the fieldsGetter handler */
    url = new URL(serviceManager.getServiceURL(), "v1/sources/logStream/fields");
    httpResponse = HttpRequests.execute(HttpRequest.get(url).build());
    Assert.assertEquals(HttpURLConnection.HTTP_OK, httpResponse.getResponseCode());
    response = httpResponse.getResponseBodyAsString();
    Set<FieldDetail> outputSet = GSON.fromJson(response, TOKEN_TYPE_SET_FIELD_DETAIL);
    Set<FieldDetail> expectedSet = new HashSet<>();
    AggregationTypeValue aggregationTypeValue = new AggregationTypeValue("DiscreteValuesHistogram", true);
    Set<AggregationTypeValue> aggregationTypeValuesList = Sets.newHashSet(aggregationTypeValue);
    expectedSet.add(new FieldDetail("content_length", aggregationTypeValuesList));
    expectedSet.add(new FieldDetail("request_time", aggregationTypeValuesList));
    expectedSet.add(new FieldDetail("status", aggregationTypeValuesList));
    Assert.assertEquals(expectedSet, outputSet);
    /* Test for the aggregationTypesGetter handler */
    url = new URL(serviceManager.getServiceURL(), "v1/sources/logStream/fields/content_length");
    httpResponse = HttpRequests.execute(HttpRequest.get(url).build());
    Assert.assertEquals(HttpURLConnection.HTTP_OK, httpResponse.getResponseCode());
    response = httpResponse.getResponseBodyAsString();
    List<AggregationTypeValue> expectedAggregationTypeValuesList = new ArrayList<>();
    List<AggregationTypeValue> outputAggregationTypeValuesList = GSON.fromJson(response, TOKEN_TYPE_LIST_AGGREGATION_TYPE_VALUES);
    expectedAggregationTypeValuesList.add(new AggregationTypeValue("DiscreteValuesHistogram", true));
    Assert.assertEquals(expectedAggregationTypeValuesList, outputAggregationTypeValuesList);
    serviceManager.stop();
    serviceManager.waitForRun(ProgramRunStatus.KILLED, 180, TimeUnit.SECONDS);
}

Example 2 with FieldDetail

use of co.cask.cdap.dq.FieldDetail in project cdap by caskdata.

the class FieldDetailTest method test.

@Test
public void test() {
    Set<AggregationTypeValue> aggregationTypeValuesSet1 = Sets.newHashSet(new AggregationTypeValue("a", true), new AggregationTypeValue("b", true));
    Set<AggregationTypeValue> aggregationTypeValuesSet2 = Sets.newHashSet(new AggregationTypeValue("a", true), new AggregationTypeValue("c", true));
    Set<AggregationTypeValue> aggregationTypeValuesSetExpected = Sets.newHashSet(new AggregationTypeValue("a", true), new AggregationTypeValue("b", true), new AggregationTypeValue("c", true));
    FieldDetail fieldDetail1 = new FieldDetail("name", aggregationTypeValuesSet1);
    FieldDetail fieldDetail2 = new FieldDetail("name", aggregationTypeValuesSet2);
    fieldDetail1.addAggregations(fieldDetail2.getAggregationTypeSet());
    FieldDetail expected = new FieldDetail("name", aggregationTypeValuesSetExpected);
    Assert.assertEquals(expected, fieldDetail1);
}