Examples with SampleResponse io.cdap.cdap.etl.proto.connection.SampleResponse used on opensource projects

Example 1 with SampleResponse

use of io.cdap.cdap.etl.proto.connection.SampleResponse in project cdap by caskdata.

the class DataPipelineConnectionTest method testBrowseSample.

@Test
public void testBrowseSample() throws Exception {
    File directory = TEMP_FOLDER.newFolder();
    List<BrowseEntity> entities = addFilesInDirectory(directory);
    String conn = "BrowseSample";
    addConnection(conn, new ConnectionCreationRequest("", new PluginInfo(FileConnector.NAME, Connector.PLUGIN_TYPE, null, Collections.emptyMap(), // in set up we add "-mocks" as the suffix for the artifact id
    new ArtifactSelectorConfig("system", APP_ARTIFACT_ID.getArtifact() + "-mocks", APP_ARTIFACT_ID.getVersion()))));
    // get all 10 results back
    BrowseDetail browseDetail = browseConnection(conn, directory.getCanonicalPath(), 10);
    BrowseDetail expected = BrowseDetail.builder().setTotalCount(10).setEntities(entities).build();
    Assert.assertEquals(expected, browseDetail);
    // only retrieve 5 back, count should still be 10
    browseDetail = browseConnection(conn, directory.getCanonicalPath(), 5);
    expected = BrowseDetail.builder().setTotalCount(10).setEntities(entities.subList(0, 5)).build();
    Assert.assertEquals(expected, browseDetail);
    // browse the created directory, should give empty result
    browseDetail = browseConnection(conn, entities.get(0).getPath(), 10);
    expected = BrowseDetail.builder().setTotalCount(0).build();
    Assert.assertEquals(expected, browseDetail);
    // browse the file, since it is not browsable, it should return itself
    browseDetail = browseConnection(conn, entities.get(1).getPath(), 10);
    expected = BrowseDetail.builder().setTotalCount(1).addEntity(entities.get(1)).build();
    Assert.assertEquals(expected, browseDetail);
    List<StructuredRecord> records = new ArrayList<>();
    Schema schema = Schema.recordOf("schema", Schema.Field.of("offset", Schema.of(Schema.Type.LONG)), Schema.Field.of("body", Schema.of(Schema.Type.STRING)));
    for (int i = 0; i < 100; i++) {
        records.add(StructuredRecord.builder(schema).set("offset", i * 2L).set("body", "1").build());
    }
    ArtifactSelectorConfig artifact = new ArtifactSelectorConfig("SYSTEM", APP_ARTIFACT_ID.getArtifact() + "-mocks", APP_ARTIFACT_ID.getVersion());
    Map<String, String> properties = ImmutableMap.of("path", entities.get(1).getPath(), "useConnection", "true", "connection", String.format("${conn(%s)}", conn));
    ConnectorDetail detail = new ConnectorDetail(ImmutableSet.of(new PluginDetail("file", "batchsource", properties, artifact, schema), new PluginDetail("file", "streamingsource", properties, artifact, schema)));
    SampleResponse expectedSample = new SampleResponse(detail, schema, records);
    // sample the file, the file has 100 lines, so 200 should retrieve all lines
    SampleResponse sampleResponse = sampleConnection(conn, entities.get(1).getPath(), 200);
    Assert.assertEquals(expectedSample, sampleResponse);
    // sample 100, should get all
    sampleResponse = sampleConnection(conn, entities.get(1).getPath(), 100);
    Assert.assertEquals(expectedSample, sampleResponse);
    // sample 50, should only get 50
    sampleResponse = sampleConnection(conn, entities.get(1).getPath(), 50);
    expectedSample = new SampleResponse(detail, schema, records.subList(0, 50));
    Assert.assertEquals(expectedSample, sampleResponse);
    deleteConnection(conn);
}

Example 2 with SampleResponse

use of io.cdap.cdap.etl.proto.connection.SampleResponse in project cdap by caskdata.

the class ConnectionUtils method getSampleResponse.

/**
 * Return {@link SampleResponse} for the connector
 *
 * @throws IOException
 */
public static SampleResponse getSampleResponse(Connector connector, ConnectorContext connectorContext, SampleRequest sampleRequest, ConnectorDetail detail, ServicePluginConfigurer pluginConfigurer) throws IOException {
    if (connector instanceof DirectConnector) {
        DirectConnector directConnector = (DirectConnector) connector;
        List<StructuredRecord> sample = directConnector.sample(connectorContext, sampleRequest);
        return new SampleResponse(detail, sample.isEmpty() ? null : sample.get(0).getSchema(), sample);
    }
    if (connector instanceof BatchConnector) {
        LimitingConnector limitingConnector = new LimitingConnector((BatchConnector) connector, pluginConfigurer);
        List<StructuredRecord> sample = limitingConnector.sample(connectorContext, sampleRequest);
        return new SampleResponse(detail, sample.isEmpty() ? null : sample.get(0).getSchema(), sample);
    }
    throw new ConnectionBadRequestException("Connector is not supported. " + "The supported connector should be DirectConnector or BatchConnector.");
}

Example 3 with SampleResponse

use of io.cdap.cdap.etl.proto.connection.SampleResponse in project cdap by caskdata.

the class ConnectionHandler method sampleLocally.

private void sampleLocally(String namespace, String sampleRequestString, Connection conn, HttpServiceResponder responder) throws IOException {
    SampleRequest sampleRequest = GSON.fromJson(sampleRequestString, SampleRequest.class);
    ServicePluginConfigurer pluginConfigurer = getContext().createServicePluginConfigurer(namespace);
    ConnectorConfigurer connectorConfigurer = new DefaultConnectorConfigurer(pluginConfigurer);
    ConnectorContext connectorContext = new DefaultConnectorContext(new SimpleFailureCollector(), pluginConfigurer);
    PluginInfo plugin = conn.getPlugin();
    // use tracked selector to get exact plugin version that gets selected since the passed version can be null
    TrackedPluginSelector pluginSelector = new TrackedPluginSelector(new ArtifactSelectorProvider().getPluginSelector(plugin.getArtifact()));
    try (Connector connector = getConnector(pluginConfigurer, plugin, namespace, pluginSelector)) {
        connector.configure(connectorConfigurer);
        ConnectorSpecRequest specRequest = ConnectorSpecRequest.builder().setPath(sampleRequest.getPath()).setConnection(conn.getName()).setProperties(sampleRequest.getProperties()).build();
        ConnectorSpec spec = connector.generateSpec(connectorContext, specRequest);
        ConnectorDetail detail = ConnectionUtils.getConnectorDetail(pluginSelector.getSelectedArtifact(), spec);
        try {
            SampleResponse sampleResponse = ConnectionUtils.getSampleResponse(connector, connectorContext, sampleRequest, detail, pluginConfigurer);
            responder.sendString(GSON.toJson(sampleResponse));
        } catch (ConnectionBadRequestException e) {
            // should not happen
            responder.sendError(HttpURLConnection.HTTP_BAD_REQUEST, e.getMessage());
        }
    }
}

Example 4 with SampleResponse

use of io.cdap.cdap.etl.proto.connection.SampleResponse in project cdap by caskdata.

the class RemoteConnectionSampleTask method execute.

@Override
public String execute(SystemAppTaskContext systemAppContext, RemoteConnectionRequest request) throws Exception {
    String namespace = request.getNamespace();
    Connection connection = request.getConnection();
    // Plugin selector and configurer
    TrackedPluginSelector pluginSelector = new TrackedPluginSelector(new ArtifactSelectorProvider().getPluginSelector(connection.getPlugin().getArtifact()));
    ServicePluginConfigurer pluginConfigurer = systemAppContext.createServicePluginConfigurer(namespace);
    ConnectorContext connectorContext = ConnectionUtils.getConnectorContext(pluginConfigurer);
    SampleRequest sampleRequest = GSON.fromJson(request.getRequest(), SampleRequest.class);
    try (Connector connector = getConnector(systemAppContext, pluginConfigurer, connection.getPlugin(), namespace, pluginSelector)) {
        connector.configure(new DefaultConnectorConfigurer(pluginConfigurer));
        ConnectorSpecRequest specRequest = ConnectorSpecRequest.builder().setPath(sampleRequest.getPath()).setConnection(connection.getName()).setProperties(sampleRequest.getProperties()).build();
        ConnectorSpec spec = connector.generateSpec(connectorContext, specRequest);
        ConnectorDetail detail = ConnectionUtils.getConnectorDetail(pluginSelector.getSelectedArtifact(), spec);
        SampleResponse sampleResponse = ConnectionUtils.getSampleResponse(connector, connectorContext, sampleRequest, detail, pluginConfigurer);
        return GSON.toJson(sampleResponse);
    }
}

Example 5 with SampleResponse

use of io.cdap.cdap.etl.proto.connection.SampleResponse in project cdap by caskdata.

the class DataPipelineConnectionTest method sampleConnection.

private SampleResponse sampleConnection(String connection, String path, int limit) throws IOException {
    String url = URLEncoder.encode(String.format("v1/contexts/%s/connections/%s/sample", NamespaceId.DEFAULT.getNamespace(), connection), StandardCharsets.UTF_8.name());
    URL validatePipelineURL = serviceURI.resolve(String.format("%s?path=%s&limit=%s", url, path, limit)).toURL();
    HttpRequest.Builder request = HttpRequest.builder(HttpMethod.POST, validatePipelineURL).withBody(GSON.toJson(SampleRequest.builder(limit).setPath(path).build()));
    HttpResponse response = executeRequest(request);
    Assert.assertEquals("Wrong answer: " + response.getResponseBodyAsString(), expectedCode, response.getResponseCode());
    return expectedCode != HttpURLConnection.HTTP_OK ? null : GSON.fromJson(response.getResponseBodyAsString(), SampleResponse.class);
}