Examples with RangeFilter io.cdap.cdap.report.proto.RangeFilter used on opensource projects

Example 1 with RangeFilter

use of io.cdap.cdap.report.proto.RangeFilter in project cdap by caskdata.

the class ReportGenerationAppTest method testFilterSerialization.

@Test
public void testFilterSerialization() throws Exception {
    List<Filter> filters = ImmutableList.of(new ValueFilter<>(Constants.NAMESPACE, ImmutableSet.of("ns1", "ns2"), null), new RangeFilter<>(Constants.DURATION, new RangeFilter.Range<>(null, 500L)));
    long startSecs = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
    ReportGenerationRequest request = new ReportGenerationRequest("ns1_ns2_report", startSecs, startSecs + 30, new ArrayList<>(ReportField.FIELD_NAME_MAP.keySet()), ImmutableList.of(new Sort(Constants.DURATION, Sort.Order.DESCENDING)), filters);
    String serialized = GSON.toJson(request);
    Assert.assertNotNull(serialized);
    ReportGenerationRequest deserialized = GSON.fromJson(serialized, ReportGenerationRequest.class);
    Assert.assertEquals(request, deserialized);
}

Example 2 with RangeFilter

use of io.cdap.cdap.report.proto.RangeFilter in project cdap by caskdata.

the class ReportGenerationAppTest method testGenerateReport.

@Test
public void testGenerateReport() throws Exception {
    Map<String, String> runTimeArguments = new HashMap<>();
    // disable tms subscriber thread as the RunMetaFileSet avro files are written directly by the test case
    // if the tms subscriber thread is enabled, in order to find the latest message id to start fetching from,
    // we read the latest RunMetaFileSet avro file's content
    // whereas the derived message_id will be invalid in TMS as these runs aren't in TMS,
    // in order to avoid the exception we disable the tms subscriber thread for the test case
    runTimeArguments.put(Constants.DISABLE_TMS_SUBSCRIBER_THREAD, "true");
    Long currentTimeMillis = System.currentTimeMillis();
    DatasetId metaFileset = createAndInitializeDataset(NamespaceId.DEFAULT, currentTimeMillis);
    SparkManager sparkManager = deployAndStartReportingApplication(NamespaceId.DEFAULT, runTimeArguments);
    URL url = sparkManager.getServiceURL(1, TimeUnit.MINUTES);
    Assert.assertNotNull(url);
    URL reportURL = url.toURI().resolve("reports/").toURL();
    List<Filter> filters = ImmutableList.of(// white list filter
    new ValueFilter<>(Constants.NAMESPACE, ImmutableSet.of("ns1", "ns2"), null), new RangeFilter<>(Constants.DURATION, new RangeFilter.Range<>(null, 500L)), // black list filter
    new ValueFilter<>(Constants.ARTIFACT_NAME, null, ImmutableSet.of("cdap-data-streams", "cdap-data-pipeline")));
    long startSecs = TimeUnit.MILLISECONDS.toSeconds(currentTimeMillis);
    ReportGenerationRequest request = new ReportGenerationRequest("ns1_ns2_report", startSecs, startSecs + 30, new ArrayList<>(ReportField.FIELD_NAME_MAP.keySet()), ImmutableList.of(new Sort(Constants.DURATION, Sort.Order.DESCENDING)), filters);
    HttpURLConnection urlConn = (HttpURLConnection) reportURL.openConnection();
    urlConn.setDoOutput(true);
    urlConn.setRequestMethod("POST");
    urlConn.getOutputStream().write(GSON.toJson(request).getBytes(StandardCharsets.UTF_8));
    if (urlConn.getErrorStream() != null) {
        Assert.fail(Bytes.toString(ByteStreams.toByteArray(urlConn.getErrorStream())));
    }
    Assert.assertEquals(200, urlConn.getResponseCode());
    Map<String, String> reportIdMap = getResponseObject(urlConn, STRING_STRING_MAP);
    String reportId = reportIdMap.get("id");
    Assert.assertNotNull(reportId);
    URL reportIdURL = reportURL.toURI().resolve("info?report-id=" + reportId).toURL();
    validateReportSummary(reportIdURL, startSecs);
    // share the report to get the share id
    URL shareReportURL = reportURL.toURI().resolve(reportId + "/share").toURL();
    HttpURLConnection shareURLConnection = (HttpURLConnection) shareReportURL.openConnection();
    shareURLConnection.setRequestMethod("POST");
    ShareId shareId = getResponseObject(shareURLConnection, ShareId.class);
    // test if we are able to get summary and read the summary using share id
    URL shareIdSummaryURL = reportURL.toURI().resolve("info?share-id=" + shareId.getShareId()).toURL();
    validateReportSummary(shareIdSummaryURL, startSecs);
    // assert the number of report details is correct
    URL reportRunsURL = reportURL.toURI().resolve("download?report-id=" + reportId).toURL();
    validateReportContent(reportRunsURL);
    // test if we are able to download and read the report using share id
    URL shareIdRunsURL = reportURL.toURI().resolve("download?share-id=" + shareId.getShareId()).toURL();
    validateReportContent(shareIdRunsURL);
    // save the report with a new name and description
    URL reportSaveURL = reportURL.toURI().resolve(reportId + "/" + "save").toURL();
    urlConn = (HttpURLConnection) reportSaveURL.openConnection();
    urlConn.setDoOutput(true);
    urlConn.setRequestMethod("POST");
    urlConn.getOutputStream().write(GSON.toJson(new ReportSaveRequest("newName", "newDescription")).getBytes(StandardCharsets.UTF_8));
    if (urlConn.getErrorStream() != null) {
        Assert.fail(Bytes.toString(ByteStreams.toByteArray(urlConn.getErrorStream())));
    }
    Assert.assertEquals(200, urlConn.getResponseCode());
    // verify that the name and description of the report have been updated, and the expiry time is null
    ReportGenerationInfo reportGenerationInfo = getResponseObject(reportIdURL.openConnection(), REPORT_GEN_INFO_TYPE);
    Assert.assertEquals("newName", reportGenerationInfo.getName());
    Assert.assertEquals("newDescription", reportGenerationInfo.getDescription());
    Assert.assertNull(reportGenerationInfo.getExpiry());
    // save the report again should fail
    urlConn = (HttpURLConnection) reportSaveURL.openConnection();
    urlConn.setDoOutput(true);
    urlConn.setRequestMethod("POST");
    urlConn.getOutputStream().write(GSON.toJson(new ReportSaveRequest("anotherNewName", "anotherNewDescription")).getBytes(StandardCharsets.UTF_8));
    if (urlConn.getErrorStream() != null) {
        Assert.fail(Bytes.toString(ByteStreams.toByteArray(urlConn.getErrorStream())));
    }
    Assert.assertEquals(403, urlConn.getResponseCode());
    // delete the report
    URL reportDeleteURL = reportURL.toURI().resolve(reportId).toURL();
    urlConn = (HttpURLConnection) reportDeleteURL.openConnection();
    urlConn.setRequestMethod("DELETE");
    Assert.assertEquals(200, urlConn.getResponseCode());
    // getting status of a deleted report will get 404
    Assert.assertEquals(404, ((HttpURLConnection) reportIdURL.openConnection()).getResponseCode());
    // deleting a deleted report again will get 404
    urlConn = (HttpURLConnection) reportDeleteURL.openConnection();
    urlConn.setRequestMethod("DELETE");
    Assert.assertEquals(404, urlConn.getResponseCode());
    // test querying for time range before the start secs, to verify empty results contents
    validateEmptyReports(reportURL, startSecs - TimeUnit.HOURS.toSeconds(2), startSecs - 30, filters);
    // test querying for time range after the start secs, but with filter that doesnt match any records
    // to verify empty results contents
    List<Filter> filters2 = ImmutableList.of(new ValueFilter<>(Constants.NAMESPACE, ImmutableSet.of("ns1", "ns2"), null), // all the programs are run by user alice, user bob will match no records.
    new ValueFilter<>(Constants.USER, ImmutableSet.of(USER_BOB), null));
    validateEmptyReports(reportURL, startSecs, startSecs + 30, filters2);
    List<Filter> filters3 = ImmutableList.of(new ValueFilter<>(Constants.NAMESPACE, ImmutableSet.of("ns1", "ns2"), null), // all the programs have the same test artifact name, blacklisting that will provide empty results
    new ValueFilter<>(Constants.ARTIFACT_NAME, null, ImmutableSet.of(TEST_ARTIFACT_NAME)));
    validateEmptyReports(reportURL, startSecs, startSecs + 30, filters3);
    sparkManager.stop();
    sparkManager.waitForStopped(60, TimeUnit.SECONDS);
    deleteDatasetInstance(metaFileset);
}

Example 3 with RangeFilter

use of io.cdap.cdap.report.proto.RangeFilter in project cdap by caskdata.

the class ReportGenerationAppTest method testReportExpiration.

@Test
public void testReportExpiration() throws Exception {
    NamespaceId testNamespace = new NamespaceId("reporting");
    NamespaceMeta reportingNamespace = new NamespaceMeta.Builder().setName(testNamespace).setDescription("Reporting namespace used to test report expiry").build();
    getNamespaceAdmin().create(reportingNamespace);
    long currentTimeMillis = System.currentTimeMillis();
    DatasetId datasetId = createAndInitializeDataset(testNamespace, currentTimeMillis);
    Map<String, String> runTimeArguments = new HashMap<>();
    // set report expiration time to be 10 seconds
    runTimeArguments.put(Constants.Report.REPORT_EXPIRY_TIME_SECONDS, "1");
    runTimeArguments.put(Constants.DISABLE_TMS_SUBSCRIBER_THREAD, "true");
    SparkManager sparkManager = deployAndStartReportingApplication(testNamespace, runTimeArguments);
    URL url = sparkManager.getServiceURL(1, TimeUnit.MINUTES);
    Assert.assertNotNull(url);
    URL reportURL = url.toURI().resolve("reports/").toURL();
    List<Filter> filters = ImmutableList.of(new ValueFilter<>(Constants.NAMESPACE, ImmutableSet.of("ns1", "ns2"), null), new RangeFilter<>(Constants.DURATION, new RangeFilter.Range<>(null, 500L)));
    long startSecs = TimeUnit.MILLISECONDS.toSeconds(currentTimeMillis);
    ReportGenerationRequest request = new ReportGenerationRequest("ns1_ns2_report", startSecs, startSecs + 30, new ArrayList<>(ReportField.FIELD_NAME_MAP.keySet()), ImmutableList.of(new Sort(Constants.DURATION, Sort.Order.DESCENDING)), filters);
    HttpURLConnection urlConn = (HttpURLConnection) reportURL.openConnection();
    urlConn.setDoOutput(true);
    urlConn.setRequestMethod("POST");
    urlConn.getOutputStream().write(GSON.toJson(request).getBytes(StandardCharsets.UTF_8));
    if (urlConn.getErrorStream() != null) {
        Assert.fail(Bytes.toString(ByteStreams.toByteArray(urlConn.getErrorStream())));
    }
    Assert.assertEquals(200, urlConn.getResponseCode());
    Map<String, String> reportIdMap = getResponseObject(urlConn, STRING_STRING_MAP);
    String reportId = reportIdMap.get("id");
    Assert.assertNotNull(reportId);
    // perform get reports list for 60 seconds and make sure we get 0 reports due to expiry
    Tasks.waitFor(0, () -> {
        return getReportsList(url);
    }, 5, TimeUnit.MINUTES, 1, TimeUnit.SECONDS);
    sparkManager.stop();
    sparkManager.waitForStopped(2, TimeUnit.MINUTES);
    deleteDatasetInstance(datasetId);
    getNamespaceAdmin().delete(testNamespace);
}