Examples with BrowseEntity io.cdap.cdap.etl.api.connector.BrowseEntity used on opensource projects

Example 1 with BrowseEntity

use of io.cdap.cdap.etl.api.connector.BrowseEntity in project cdap by caskdata.

the class DataPipelineConnectionTest method addFilesInDirectory.

private List<BrowseEntity> addFilesInDirectory(File directory) throws IOException {
    List<BrowseEntity> entities = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        if (i % 2 == 0) {
            File folder = new File(directory, "file" + i);
            folder.mkdir();
            entities.add(BrowseEntity.builder(folder.getName(), folder.getCanonicalPath(), "directory").canSample(true).canBrowse(true).build());
            continue;
        }
        // generate file with 100 lines
        File file = new File(directory, "file" + i + ".txt");
        try (BufferedWriter writer = Files.newBufferedWriter(file.toPath())) {
            for (int j = 0; j < 100; j++) {
                writer.write(i + "");
                if (j != 99) {
                    writer.newLine();
                }
            }
        }
        entities.add(BrowseEntity.builder(file.getName(), file.getCanonicalPath(), "file").canSample(true).build());
    }
    return entities;
}

Example 2 with BrowseEntity

use of io.cdap.cdap.etl.api.connector.BrowseEntity in project cdap by caskdata.

the class DataPipelineConnectionTest method testBrowseSample.

@Test
public void testBrowseSample() throws Exception {
    File directory = TEMP_FOLDER.newFolder();
    List<BrowseEntity> entities = addFilesInDirectory(directory);
    String conn = "BrowseSample";
    addConnection(conn, new ConnectionCreationRequest("", new PluginInfo(FileConnector.NAME, Connector.PLUGIN_TYPE, null, Collections.emptyMap(), // in set up we add "-mocks" as the suffix for the artifact id
    new ArtifactSelectorConfig("system", APP_ARTIFACT_ID.getArtifact() + "-mocks", APP_ARTIFACT_ID.getVersion()))));
    // get all 10 results back
    BrowseDetail browseDetail = browseConnection(conn, directory.getCanonicalPath(), 10);
    BrowseDetail expected = BrowseDetail.builder().setTotalCount(10).setEntities(entities).build();
    Assert.assertEquals(expected, browseDetail);
    // only retrieve 5 back, count should still be 10
    browseDetail = browseConnection(conn, directory.getCanonicalPath(), 5);
    expected = BrowseDetail.builder().setTotalCount(10).setEntities(entities.subList(0, 5)).build();
    Assert.assertEquals(expected, browseDetail);
    // browse the created directory, should give empty result
    browseDetail = browseConnection(conn, entities.get(0).getPath(), 10);
    expected = BrowseDetail.builder().setTotalCount(0).build();
    Assert.assertEquals(expected, browseDetail);
    // browse the file, since it is not browsable, it should return itself
    browseDetail = browseConnection(conn, entities.get(1).getPath(), 10);
    expected = BrowseDetail.builder().setTotalCount(1).addEntity(entities.get(1)).build();
    Assert.assertEquals(expected, browseDetail);
    List<StructuredRecord> records = new ArrayList<>();
    Schema schema = Schema.recordOf("schema", Schema.Field.of("offset", Schema.of(Schema.Type.LONG)), Schema.Field.of("body", Schema.of(Schema.Type.STRING)));
    for (int i = 0; i < 100; i++) {
        records.add(StructuredRecord.builder(schema).set("offset", i * 2L).set("body", "1").build());
    }
    ArtifactSelectorConfig artifact = new ArtifactSelectorConfig("SYSTEM", APP_ARTIFACT_ID.getArtifact() + "-mocks", APP_ARTIFACT_ID.getVersion());
    Map<String, String> properties = ImmutableMap.of("path", entities.get(1).getPath(), "useConnection", "true", "connection", String.format("${conn(%s)}", conn));
    ConnectorDetail detail = new ConnectorDetail(ImmutableSet.of(new PluginDetail("file", "batchsource", properties, artifact, schema), new PluginDetail("file", "streamingsource", properties, artifact, schema)));
    SampleResponse expectedSample = new SampleResponse(detail, schema, records);
    // sample the file, the file has 100 lines, so 200 should retrieve all lines
    SampleResponse sampleResponse = sampleConnection(conn, entities.get(1).getPath(), 200);
    Assert.assertEquals(expectedSample, sampleResponse);
    // sample 100, should get all
    sampleResponse = sampleConnection(conn, entities.get(1).getPath(), 100);
    Assert.assertEquals(expectedSample, sampleResponse);
    // sample 50, should only get 50
    sampleResponse = sampleConnection(conn, entities.get(1).getPath(), 50);
    expectedSample = new SampleResponse(detail, schema, records.subList(0, 50));
    Assert.assertEquals(expectedSample, sampleResponse);
    deleteConnection(conn);
}

Example 3 with BrowseEntity

use of io.cdap.cdap.etl.api.connector.BrowseEntity in project cdap by caskdata.

the class DataPipelineConnectionTest method testConnectionMetrics.

@Test
public void testConnectionMetrics() throws Exception {
    File directory = TEMP_FOLDER.newFolder();
    List<BrowseEntity> entities = addFilesInDirectory(directory);
    ConnectionCreationRequest connRequest = new ConnectionCreationRequest("", new PluginInfo(FileConnector.NAME, Connector.PLUGIN_TYPE, null, Collections.emptyMap(), // in set up we add "-mocks" as the suffix for the artifact id
    new ArtifactSelectorConfig("system", APP_ARTIFACT_ID.getArtifact() + "-mocks", APP_ARTIFACT_ID.getVersion())));
    ConnectionCreationRequest dummyRequest = new ConnectionCreationRequest("", new PluginInfo("dummy", Connector.PLUGIN_TYPE, null, Collections.emptyMap(), // in set up we add "-mocks" as the suffix for the artifact id
    new ArtifactSelectorConfig("system", APP_ARTIFACT_ID.getArtifact() + "-mocks", APP_ARTIFACT_ID.getVersion())));
    Map<String, String> tagsFile = new HashMap<>(SERVICE_TAGS);
    tagsFile.put(Constants.Metrics.Tag.APP_ENTITY_TYPE, Constants.CONNECTION_SERVICE_NAME);
    tagsFile.put(Constants.Metrics.Tag.APP_ENTITY_TYPE_NAME, FileConnector.NAME);
    Map<String, String> tagsDummy = new HashMap<>(SERVICE_TAGS);
    tagsDummy.put(Constants.Metrics.Tag.APP_ENTITY_TYPE, Constants.CONNECTION_SERVICE_NAME);
    tagsDummy.put(Constants.Metrics.Tag.APP_ENTITY_TYPE_NAME, "dummy");
    // this is needed because studio service is running through the entire tests, so we need to ensure old
    // metrics emitted by other tests do not affect this one
    long existingMetricsTotal = getMetricsManager().getTotalMetric(SERVICE_TAGS, "user." + Constants.Metrics.Connection.CONNECTION_COUNT);
    long existingMetricsFile = getMetricsManager().getTotalMetric(tagsFile, "user." + Constants.Metrics.Connection.CONNECTION_COUNT);
    long existingMetricsDummy = getMetricsManager().getTotalMetric(tagsDummy, "user." + Constants.Metrics.Connection.CONNECTION_COUNT);
    // add 5 file connections, add 5 dummy connections without the artifact
    for (int i = 0; i < 5; i++) {
        addConnection("conn" + i, connRequest);
    }
    for (int i = 5; i < 10; i++) {
        addConnection("conn" + i, dummyRequest);
    }
    // validate 10 conns added, 5 for file, 5 for dummy
    validateMetrics(SERVICE_TAGS, Constants.Metrics.Connection.CONNECTION_COUNT, existingMetricsTotal, 10L);
    validateMetrics(tagsFile, Constants.Metrics.Connection.CONNECTION_COUNT, existingMetricsFile, 5L);
    validateMetrics(tagsDummy, Constants.Metrics.Connection.CONNECTION_COUNT, existingMetricsDummy, 5L);
    // add 5 more dummy connections
    for (int i = 10; i < 15; i++) {
        addConnection("conn" + i, dummyRequest);
    }
    // validate 15 conns added, 5 files, 10 dummy
    validateMetrics(SERVICE_TAGS, Constants.Metrics.Connection.CONNECTION_COUNT, existingMetricsTotal, 15L);
    validateMetrics(tagsFile, Constants.Metrics.Connection.CONNECTION_COUNT, existingMetricsFile, 5L);
    validateMetrics(tagsDummy, Constants.Metrics.Connection.CONNECTION_COUNT, existingMetricsDummy, 10L);
    // get old get metrics number
    existingMetricsTotal = getMetricsManager().getTotalMetric(SERVICE_TAGS, "user." + Constants.Metrics.Connection.CONNECTION_GET_COUNT);
    existingMetricsFile = getMetricsManager().getTotalMetric(tagsFile, "user." + Constants.Metrics.Connection.CONNECTION_GET_COUNT);
    existingMetricsDummy = getMetricsManager().getTotalMetric(tagsDummy, "user." + Constants.Metrics.Connection.CONNECTION_GET_COUNT);
    // get these 15 conns
    for (int i = 0; i < 15; i++) {
        getConnection("conn" + i);
    }
    // validate 15 get metrics for these connections, 5 for file, 10 for dummy
    validateMetrics(SERVICE_TAGS, Constants.Metrics.Connection.CONNECTION_GET_COUNT, existingMetricsTotal, 15L);
    validateMetrics(tagsFile, Constants.Metrics.Connection.CONNECTION_GET_COUNT, existingMetricsFile, 5L);
    validateMetrics(tagsDummy, Constants.Metrics.Connection.CONNECTION_GET_COUNT, existingMetricsDummy, 10L);
    // get old browse number
    existingMetricsTotal = getMetricsManager().getTotalMetric(SERVICE_TAGS, "user." + Constants.Metrics.Connection.CONNECTION_BROWSE_COUNT);
    existingMetricsFile = getMetricsManager().getTotalMetric(tagsFile, "user." + Constants.Metrics.Connection.CONNECTION_BROWSE_COUNT);
    // browse each file connection twice
    for (int i = 0; i < 5; i++) {
        browseConnection("conn" + i, directory.getCanonicalPath(), 10);
        browseConnection("conn" + i, directory.getCanonicalPath(), 10);
    }
    // validate 10 browse metrics are emitted for file
    validateMetrics(SERVICE_TAGS, Constants.Metrics.Connection.CONNECTION_BROWSE_COUNT, existingMetricsTotal, 10L);
    validateMetrics(tagsFile, Constants.Metrics.Connection.CONNECTION_BROWSE_COUNT, existingMetricsFile, 10L);
    existingMetricsTotal = getMetricsManager().getTotalMetric(SERVICE_TAGS, "user." + Constants.Metrics.Connection.CONNECTION_SAMPLE_COUNT);
    existingMetricsFile = getMetricsManager().getTotalMetric(tagsFile, "user." + Constants.Metrics.Connection.CONNECTION_SAMPLE_COUNT);
    long existingMetricsSpecTotal = getMetricsManager().getTotalMetric(SERVICE_TAGS, "user." + Constants.Metrics.Connection.CONNECTION_SPEC_COUNT);
    long existingMetricsSpecFile = getMetricsManager().getTotalMetric(tagsFile, "user." + Constants.Metrics.Connection.CONNECTION_SPEC_COUNT);
    // sample each file connection
    for (int i = 0; i < 5; i++) {
        sampleConnection("conn" + i, entities.get(1).getPath(), 10);
    }
    // validate 5 sample and spec metrics are emitted for file
    validateMetrics(SERVICE_TAGS, Constants.Metrics.Connection.CONNECTION_SAMPLE_COUNT, existingMetricsTotal, 5L);
    validateMetrics(tagsFile, Constants.Metrics.Connection.CONNECTION_SAMPLE_COUNT, existingMetricsFile, 5L);
    validateMetrics(SERVICE_TAGS, Constants.Metrics.Connection.CONNECTION_SPEC_COUNT, existingMetricsSpecTotal, 5L);
    validateMetrics(tagsFile, Constants.Metrics.Connection.CONNECTION_SPEC_COUNT, existingMetricsSpecFile, 5L);
    // get existing delete number
    existingMetricsTotal = getMetricsManager().getTotalMetric(SERVICE_TAGS, "user." + Constants.Metrics.Connection.CONNECTION_DELETED_COUNT);
    existingMetricsFile = getMetricsManager().getTotalMetric(tagsFile, "user." + Constants.Metrics.Connection.CONNECTION_DELETED_COUNT);
    existingMetricsDummy = getMetricsManager().getTotalMetric(tagsDummy, "user." + Constants.Metrics.Connection.CONNECTION_DELETED_COUNT);
    // delete all connections
    for (int i = 0; i < 15; i++) {
        deleteConnection("conn" + i);
    }
    // validate 15 delete metrics for these connections, 5 for file, 10 for dummy
    validateMetrics(SERVICE_TAGS, Constants.Metrics.Connection.CONNECTION_DELETED_COUNT, existingMetricsTotal, 15L);
    validateMetrics(tagsFile, Constants.Metrics.Connection.CONNECTION_DELETED_COUNT, existingMetricsFile, 5L);
    validateMetrics(tagsDummy, Constants.Metrics.Connection.CONNECTION_DELETED_COUNT, existingMetricsDummy, 10L);
}

Example 4 with BrowseEntity

use of io.cdap.cdap.etl.api.connector.BrowseEntity in project cdap by caskdata.

the class DataPipelineConnectionTest method testConnectionAuthorization.

@Test
public void testConnectionAuthorization() throws Exception {
    File directory = TEMP_FOLDER.newFolder();
    List<BrowseEntity> entities = addFilesInDirectory(directory);
    user = ALICE_NAME;
    // Check Alice can't list requests
    expectedCode = HttpURLConnection.HTTP_FORBIDDEN;
    listConnections(NamespaceId.DEFAULT.getNamespace());
    // Grant Alice nesessary privileges
    getAccessController().grant(Authorizable.fromEntityId(NamespaceId.DEFAULT), ALICE_PRINCIPAL, EnumSet.of(StandardPermission.GET));
    getAccessController().grant(Authorizable.fromEntityId(NamespaceId.DEFAULT, EntityType.SYSTEM_APP_ENTITY), ALICE_PRINCIPAL, EnumSet.of(StandardPermission.LIST));
    // Check Alice can list requests now
    expectedCode = HttpURLConnection.HTTP_OK;
    listConnections(NamespaceId.DEFAULT.getNamespace());
    // Check Bob still can't do it
    user = "bob";
    expectedCode = HttpURLConnection.HTTP_FORBIDDEN;
    listConnections(NamespaceId.DEFAULT.getNamespace());
    // Check Alice can't do it for another namespace
    user = ALICE_NAME;
    listConnections("testNamespace");
    // Check Alice still can't create, delete and use connections
    String conn = "test_connection";
    ConnectionCreationRequest creationRequest = new ConnectionCreationRequest("", new PluginInfo(FileConnector.NAME, Connector.PLUGIN_TYPE, null, Collections.emptyMap(), // in set up we add "-mocks" as the suffix for the artifact id
    new ArtifactSelectorConfig("system", APP_ARTIFACT_ID.getArtifact() + "-mocks", APP_ARTIFACT_ID.getVersion())));
    addConnection(conn, creationRequest);
    deleteConnection(conn);
    browseConnection(conn, directory.getCanonicalPath(), 10);
    sampleConnection(conn, entities.get(1).getPath(), 100);
    getConnectionSpec(conn, directory.getCanonicalPath(), null, null);
    // Grant Alice permissions to create connection
    ConnectionEntityId connectionEntityId = new ConnectionEntityId(NamespaceId.DEFAULT.getNamespace(), ConnectionId.getConnectionId(conn));
    getAccessController().grant(Authorizable.fromEntityId(connectionEntityId), ALICE_PRINCIPAL, EnumSet.of(StandardPermission.CREATE));
    expectedCode = HttpURLConnection.HTTP_OK;
    addConnection(conn, creationRequest);
    // Grant Alice permission to use connection
    getAccessController().grant(Authorizable.fromEntityId(connectionEntityId), ALICE_PRINCIPAL, EnumSet.of(StandardPermission.USE));
    browseConnection(conn, directory.getCanonicalPath(), 10);
    sampleConnection(conn, entities.get(1).getPath(), 100);
    getConnectionSpec(conn, directory.getCanonicalPath(), null, null);
    // but Alice still can't update or delete connection
    expectedCode = HttpURLConnection.HTTP_FORBIDDEN;
    addConnection(conn, creationRequest);
    deleteConnection(conn);
    // Grant Alice permission to delete connection
    getAccessController().grant(Authorizable.fromEntityId(connectionEntityId), ALICE_PRINCIPAL, EnumSet.of(StandardPermission.DELETE));
    expectedCode = HttpURLConnection.HTTP_OK;
    deleteConnection(conn);
}