Examples with ConnectorDefinition org.apache.pulsar.common.io.ConnectorDefinition used on opensource projects

Example 1 with ConnectorDefinition

use of org.apache.pulsar.common.io.ConnectorDefinition in project pulsar by apache.

the class ConnectorUtils method searchForConnectors.

public static TreeMap<String, Connector> searchForConnectors(String connectorsDirectory, String narExtractionDirectory) throws IOException {
    Path path = Paths.get(connectorsDirectory).toAbsolutePath();
    log.info("Searching for connectors in {}", path);
    TreeMap<String, Connector> connectors = new TreeMap<>();
    if (!path.toFile().exists()) {
        log.warn("Connectors archive directory not found");
        return connectors;
    }
    try (DirectoryStream<Path> stream = Files.newDirectoryStream(path, "*.nar")) {
        for (Path archive : stream) {
            try {
                NarClassLoader ncl = NarClassLoaderBuilder.builder().narFile(new File(archive.toString())).extractionDirectory(narExtractionDirectory).build();
                Connector.ConnectorBuilder connectorBuilder = Connector.builder();
                ConnectorDefinition cntDef = ConnectorUtils.getConnectorDefinition(ncl);
                log.info("Found connector {} from {}", cntDef, archive);
                connectorBuilder.archivePath(archive);
                if (!StringUtils.isEmpty(cntDef.getSourceClass())) {
                    if (!StringUtils.isEmpty(cntDef.getSourceConfigClass())) {
                        connectorBuilder.sourceConfigFieldDefinitions(ConnectorUtils.getConnectorConfigDefinition(ncl, cntDef.getSourceConfigClass()));
                    }
                }
                if (!StringUtils.isEmpty(cntDef.getSinkClass())) {
                    if (!StringUtils.isEmpty(cntDef.getSinkConfigClass())) {
                        connectorBuilder.sinkConfigFieldDefinitions(ConnectorUtils.getConnectorConfigDefinition(ncl, cntDef.getSinkConfigClass()));
                    }
                }
                connectorBuilder.classLoader(ncl);
                connectorBuilder.connectorDefinition(cntDef);
                connectors.put(cntDef.getName(), connectorBuilder.build());
            } catch (Throwable t) {
                log.warn("Failed to load connector from {}", archive, t);
            }
        }
        return connectors;
    }
}

Example 2 with ConnectorDefinition

use of org.apache.pulsar.common.io.ConnectorDefinition in project pulsar by apache.

the class SinkConfigUtils method validateSinkConfig.

public static void validateSinkConfig(SinkConfig sinkConfig, NarClassLoader narClassLoader) {
    if (sinkConfig.getRetainKeyOrdering() != null && sinkConfig.getRetainKeyOrdering() && sinkConfig.getProcessingGuarantees() != null && sinkConfig.getProcessingGuarantees() == FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE) {
        throw new IllegalArgumentException("When effectively once processing guarantee is specified, retain Key ordering cannot be set");
    }
    if (sinkConfig.getRetainKeyOrdering() != null && sinkConfig.getRetainKeyOrdering() && sinkConfig.getRetainOrdering() != null && sinkConfig.getRetainOrdering()) {
        throw new IllegalArgumentException("Only one of retain ordering or retain key ordering can be set");
    }
    try {
        ConnectorDefinition defn = ConnectorUtils.getConnectorDefinition(narClassLoader);
        if (defn.getSinkConfigClass() != null) {
            Class configClass = Class.forName(defn.getSinkConfigClass(), true, narClassLoader);
            validateSinkConfig(sinkConfig, configClass);
        }
    } catch (IOException e) {
        throw new IllegalArgumentException("Error validating sink config", e);
    } catch (ClassNotFoundException e) {
        throw new IllegalArgumentException("Could not find sink config class", e);
    }
}

Example 3 with ConnectorDefinition

use of org.apache.pulsar.common.io.ConnectorDefinition in project pulsar by apache.

the class SourceConfigUtils method validateSourceConfig.

public static void validateSourceConfig(SourceConfig sourceConfig, NarClassLoader narClassLoader) {
    try {
        ConnectorDefinition defn = ConnectorUtils.getConnectorDefinition(narClassLoader);
        if (defn.getSourceConfigClass() != null) {
            Class configClass = Class.forName(defn.getSourceConfigClass(), true, narClassLoader);
            validateSourceConfig(sourceConfig, configClass);
        }
    } catch (IOException e) {
        throw new IllegalArgumentException("Error validating source config", e);
    } catch (ClassNotFoundException e) {
        throw new IllegalArgumentException("Could not find source config class");
    }
}

Example 4 with ConnectorDefinition

use of org.apache.pulsar.common.io.ConnectorDefinition in project pulsar by apache.

the class SinksImpl method getSinkList.

@Override
public List<ConnectorDefinition> getSinkList() {
    List<ConnectorDefinition> connectorDefinitions = getListOfConnectors();
    List<ConnectorDefinition> retval = new ArrayList<>();
    for (ConnectorDefinition connectorDefinition : connectorDefinitions) {
        if (!org.apache.commons.lang.StringUtils.isEmpty(connectorDefinition.getSinkClass())) {
            retval.add(connectorDefinition);
        }
    }
    return retval;
}

Example 5 with ConnectorDefinition

use of org.apache.pulsar.common.io.ConnectorDefinition in project pulsar by apache.

the class SourcesImpl method getSourceList.

@Override
public List<ConnectorDefinition> getSourceList() {
    List<ConnectorDefinition> connectorDefinitions = getListOfConnectors();
    List<ConnectorDefinition> retval = new ArrayList<>();
    for (ConnectorDefinition connectorDefinition : connectorDefinitions) {
        if (!org.apache.commons.lang.StringUtils.isEmpty(connectorDefinition.getSourceClass())) {
            retval.add(connectorDefinition);
        }
    }
    return retval;
}