Examples with ArtifactSelectorProvider io.cdap.cdap.etl.common.ArtifactSelectorProvider used on opensource projects

Example 1 with ArtifactSelectorProvider

use of io.cdap.cdap.etl.common.ArtifactSelectorProvider in project cdap by caskdata.

the class RemoteConnectionSpecTask method execute.

@Override
public String execute(SystemAppTaskContext systemAppContext, RemoteConnectionRequest request) throws Exception {
    String namespace = request.getNamespace();
    Connection connection = request.getConnection();
    // Plugin selector and configurer
    TrackedPluginSelector pluginSelector = new TrackedPluginSelector(new ArtifactSelectorProvider().getPluginSelector(connection.getPlugin().getArtifact()));
    ServicePluginConfigurer pluginConfigurer = systemAppContext.createServicePluginConfigurer(namespace);
    ConnectorConfigurer connectorConfigurer = new DefaultConnectorConfigurer(pluginConfigurer);
    ConnectorContext connectorContext = ConnectionUtils.getConnectorContext(pluginConfigurer);
    SpecGenerationRequest specRequest = GSON.fromJson(request.getRequest(), SpecGenerationRequest.class);
    try (Connector connector = getConnector(systemAppContext, pluginConfigurer, connection.getPlugin(), namespace, pluginSelector)) {
        connector.configure(connectorConfigurer);
        ConnectorSpecRequest connectorSpecRequest = ConnectorSpecRequest.builder().setPath(specRequest.getPath()).setConnection(connection.getName()).setProperties(specRequest.getProperties()).build();
        ConnectorSpec spec = connector.generateSpec(connectorContext, connectorSpecRequest);
        ConnectorSpec newSpec = ConnectionUtils.filterSpecWithPluginNameAndType(spec, specRequest.getPluginName(), specRequest.getPluginType());
        return GSON.toJson(ConnectionUtils.getConnectorDetail(pluginSelector.getSelectedArtifact(), newSpec));
    }
}

Example 2 with ArtifactSelectorProvider

use of io.cdap.cdap.etl.common.ArtifactSelectorProvider in project cdap by caskdata.

the class RemoteConnectionTestTask method execute.

@Override
public String execute(SystemAppTaskContext systemAppContext, RemoteConnectionRequest request) throws Exception {
    String namespace = request.getNamespace();
    ConnectionCreationRequest connectionCreationRequest = GSON.fromJson(request.getRequest(), ConnectionCreationRequest.class);
    ServicePluginConfigurer pluginConfigurer = systemAppContext.createServicePluginConfigurer(namespace);
    ConnectorConfigurer connectorConfigurer = new DefaultConnectorConfigurer(pluginConfigurer);
    SimpleFailureCollector failureCollector = new SimpleFailureCollector();
    ConnectorContext connectorContext = new DefaultConnectorContext(failureCollector, pluginConfigurer);
    TrackedPluginSelector pluginSelector = new TrackedPluginSelector(new ArtifactSelectorProvider().getPluginSelector(connectionCreationRequest.getPlugin().getArtifact()));
    try (Connector connector = getConnector(systemAppContext, pluginConfigurer, connectionCreationRequest.getPlugin(), namespace, pluginSelector)) {
        connector.configure(connectorConfigurer);
        try {
            connector.test(connectorContext);
            failureCollector.getOrThrowException();
            return "";
        } catch (ValidationException e) {
            return GSON.toJson(e.getFailures());
        }
    }
}

Example 3 with ArtifactSelectorProvider

use of io.cdap.cdap.etl.common.ArtifactSelectorProvider in project cdap by caskdata.

the class PipelineSpecGenerator method configureStage.

/**
 * Configures a plugin and returns the spec for it.
 *
 * @param stageName the unique plugin id
 * @param etlPlugin user provided configuration for the plugin
 * @param pipelineConfigurer default pipeline configurer to configure the plugin
 * @return the spec for the plugin
 * @throws IllegalArgumentException if the plugin with same id is already deployed
 * @throws ValidationException if the plugin threw an exception during configuration
 */
public StageSpec.Builder configureStage(String stageName, ETLPlugin etlPlugin, DefaultPipelineConfigurer pipelineConfigurer) throws ValidationException {
    TrackedPluginSelector pluginSelector = new TrackedPluginSelector(new ArtifactSelectorProvider().getPluginSelector(etlPlugin.getArtifactConfig()));
    String type = etlPlugin.getType();
    String pluginName = etlPlugin.getName();
    DefaultStageConfigurer stageConfigurer = pipelineConfigurer.getStageConfigurer();
    FailureCollector collector = stageConfigurer.getFailureCollector();
    Object plugin = getPlugin(stageName, etlPlugin, pluginSelector, type, pluginName, collector);
    try {
        if (type.equals(BatchJoiner.PLUGIN_TYPE)) {
            MultiInputPipelineConfigurable multiPlugin = (MultiInputPipelineConfigurable) plugin;
            multiPlugin.configurePipeline(pipelineConfigurer);
            // to the BatchAutoJoiner while preserving backwards compatibility in the pipeline config.
            if (plugin instanceof AutoJoiner) {
                configureAutoJoiner(stageName, (AutoJoiner) plugin, stageConfigurer, collector);
            }
        } else if (type.equals(SplitterTransform.PLUGIN_TYPE)) {
            MultiOutputPipelineConfigurable multiOutputPlugin = (MultiOutputPipelineConfigurable) plugin;
            multiOutputPlugin.configurePipeline(pipelineConfigurer);
        } else if (!type.equals(Constants.SPARK_PROGRAM_PLUGIN_TYPE)) {
            PipelineConfigurable singlePlugin = (PipelineConfigurable) plugin;
            singlePlugin.configurePipeline(pipelineConfigurer);
            // evaluate macros and find out if there is connection used
            if ((sourcePluginTypes.contains(type) || BatchSink.PLUGIN_TYPE.equals(type)) && runtimeEvaluator == null) {
                pluginConfigurer.evaluateMacros(etlPlugin.getProperties(), connectionEvaluator, options);
            }
        }
    } catch (InvalidConfigPropertyException e) {
        collector.addFailure(e.getMessage(), String.format("Provide valid value for config property '%s'.", e.getProperty())).withConfigProperty(e.getProperty());
    } catch (InvalidStageException e) {
        if (e.getReasons().isEmpty()) {
            collector.addFailure(e.getMessage(), null);
        }
        for (InvalidStageException reason : e.getReasons()) {
            if (reason instanceof InvalidConfigPropertyException) {
                InvalidConfigPropertyException configException = (InvalidConfigPropertyException) reason;
                collector.addFailure(configException.getMessage(), String.format("Provide valid value for config property '%s'.", configException.getProperty())).withConfigProperty(configException.getProperty());
            } else {
                collector.addFailure(reason.getMessage(), null);
            }
        }
    } catch (ValidationException e) {
        throw e;
    } catch (NullPointerException e) {
        // handle the case where plugin throws null pointer exception, this is to avoid having 'null' as error message
        collector.addFailure(String.format("Null error occurred while configuring the stage %s.", stageName), null).withStacktrace(e.getStackTrace());
    } catch (ArrayIndexOutOfBoundsException e) {
        // handle the case where plugin throws index out of bounds exception,
        // this is to avoid having a number like '2', '8' etc as error message
        collector.addFailure(String.format("Index out of bounds error occurred while configuring the stage %s.", stageName), null).withStacktrace(e.getStackTrace());
    } catch (ConnectionBadRequestException e) {
        collector.addFailure(e.getMessage(), "Provide a valid connection name.");
    } catch (Exception e) {
        collector.addFailure(String.format("Error encountered while configuring the stage: '%s'", e.getMessage()), null).withStacktrace(e.getStackTrace());
    }
    // throw validation exception if there are any errors being carried by failure collector
    collector.getOrThrowException();
    PluginSpec pluginSpec = new PluginSpec(type, pluginName, etlPlugin.getProperties(), pluginSelector.getSelectedArtifact());
    StageSpec.Builder specBuilder = StageSpec.builder(stageName, pluginSpec).addInputSchemas(pipelineConfigurer.getStageConfigurer().getInputSchemas()).setErrorSchema(stageConfigurer.getErrorSchema());
    if (type.equals(SplitterTransform.PLUGIN_TYPE)) {
        specBuilder.setPortSchemas(stageConfigurer.getOutputPortSchemas());
    } else {
        specBuilder.setOutputSchema(stageConfigurer.getOutputSchema());
    }
    return specBuilder;
}

Example 4 with ArtifactSelectorProvider

use of io.cdap.cdap.etl.common.ArtifactSelectorProvider in project cdap by caskdata.

the class ConnectionHandler method sampleLocally.

private void sampleLocally(String namespace, String sampleRequestString, Connection conn, HttpServiceResponder responder) throws IOException {
    SampleRequest sampleRequest = GSON.fromJson(sampleRequestString, SampleRequest.class);
    ServicePluginConfigurer pluginConfigurer = getContext().createServicePluginConfigurer(namespace);
    ConnectorConfigurer connectorConfigurer = new DefaultConnectorConfigurer(pluginConfigurer);
    ConnectorContext connectorContext = new DefaultConnectorContext(new SimpleFailureCollector(), pluginConfigurer);
    PluginInfo plugin = conn.getPlugin();
    // use tracked selector to get exact plugin version that gets selected since the passed version can be null
    TrackedPluginSelector pluginSelector = new TrackedPluginSelector(new ArtifactSelectorProvider().getPluginSelector(plugin.getArtifact()));
    try (Connector connector = getConnector(pluginConfigurer, plugin, namespace, pluginSelector)) {
        connector.configure(connectorConfigurer);
        ConnectorSpecRequest specRequest = ConnectorSpecRequest.builder().setPath(sampleRequest.getPath()).setConnection(conn.getName()).setProperties(sampleRequest.getProperties()).build();
        ConnectorSpec spec = connector.generateSpec(connectorContext, specRequest);
        ConnectorDetail detail = ConnectionUtils.getConnectorDetail(pluginSelector.getSelectedArtifact(), spec);
        try {
            SampleResponse sampleResponse = ConnectionUtils.getSampleResponse(connector, connectorContext, sampleRequest, detail, pluginConfigurer);
            responder.sendString(GSON.toJson(sampleResponse));
        } catch (ConnectionBadRequestException e) {
            // should not happen
            responder.sendError(HttpURLConnection.HTTP_BAD_REQUEST, e.getMessage());
        }
    }
}

Example 5 with ArtifactSelectorProvider

use of io.cdap.cdap.etl.common.ArtifactSelectorProvider in project cdap by caskdata.

the class ConnectionHandler method testLocally.

private void testLocally(String namespace, ConnectionCreationRequest creationRequest, HttpServiceResponder responder) throws IOException {
    ServicePluginConfigurer pluginConfigurer = getContext().createServicePluginConfigurer(namespace);
    ConnectorConfigurer connectorConfigurer = new DefaultConnectorConfigurer(pluginConfigurer);
    SimpleFailureCollector failureCollector = new SimpleFailureCollector();
    ConnectorContext connectorContext = new DefaultConnectorContext(failureCollector, pluginConfigurer);
    TrackedPluginSelector pluginSelector = new TrackedPluginSelector(new ArtifactSelectorProvider().getPluginSelector(creationRequest.getPlugin().getArtifact()));
    try (Connector connector = getConnector(pluginConfigurer, creationRequest.getPlugin(), namespace, pluginSelector)) {
        connector.configure(connectorConfigurer);
        try {
            connector.test(connectorContext);
            failureCollector.getOrThrowException();
        } catch (ValidationException e) {
            responder.sendJson(e.getFailures());
            return;
        }
    }
    responder.sendStatus(HttpURLConnection.HTTP_OK);
}