Examples with Connector org.apache.pulsar.functions.utils.io.Connector used on opensource projects

Example 1 with Connector

use of org.apache.pulsar.functions.utils.io.Connector in project pulsar by apache.

the class FunctionActioner method getBuiltinArchive.

private File getBuiltinArchive(FunctionDetails.Builder functionDetails) throws IOException, ClassNotFoundException {
    if (functionDetails.hasSource()) {
        SourceSpec sourceSpec = functionDetails.getSource();
        if (!StringUtils.isEmpty(sourceSpec.getBuiltin())) {
            Connector connector = connectorsManager.getConnector(sourceSpec.getBuiltin());
            File archive = connector.getArchivePath().toFile();
            String sourceClass = connector.getConnectorDefinition().getSourceClass();
            SourceSpec.Builder builder = SourceSpec.newBuilder(functionDetails.getSource());
            builder.setClassName(sourceClass);
            functionDetails.setSource(builder);
            fillSourceTypeClass(functionDetails, connector.getClassLoader(), sourceClass);
            return archive;
        }
    }
    if (functionDetails.hasSink()) {
        SinkSpec sinkSpec = functionDetails.getSink();
        if (!StringUtils.isEmpty(sinkSpec.getBuiltin())) {
            Connector connector = connectorsManager.getConnector(sinkSpec.getBuiltin());
            File archive = connector.getArchivePath().toFile();
            String sinkClass = connector.getConnectorDefinition().getSinkClass();
            SinkSpec.Builder builder = SinkSpec.newBuilder(functionDetails.getSink());
            builder.setClassName(sinkClass);
            functionDetails.setSink(builder);
            fillSinkTypeClass(functionDetails, connector.getClassLoader(), sinkClass);
            return archive;
        }
    }
    if (!StringUtils.isEmpty(functionDetails.getBuiltin())) {
        return functionsManager.getFunctionArchive(functionDetails.getBuiltin()).toFile();
    }
    throw new IOException("Could not find built in archive definition");
}

Example 2 with Connector

use of org.apache.pulsar.functions.utils.io.Connector in project pulsar by apache.

the class SinkApiV3ResourceTest method testRegisterSinkSuccessK8sWithUpload.

/*
    Externally managed runtime,
    uploadBuiltinSinksSources == true
    Make sure uploadFileToBookkeeper is called
    */
@Test
public void testRegisterSinkSuccessK8sWithUpload() throws Exception {
    final String injectedErrMsg = "uploadFileToBookkeeper triggered";
    mockedWorkerService.getWorkerConfig().setUploadBuiltinSinksSources(true);
    mockStatic(WorkerUtils.class, ctx -> {
        ctx.when(() -> WorkerUtils.uploadFileToBookkeeper(anyString(), any(File.class), any(Namespace.class))).thenThrow(new RuntimeException(injectedErrMsg));
    });
    NarClassLoader mockedClassLoader = mock(NarClassLoader.class);
    mockStatic(FunctionCommon.class, ctx -> {
        ctx.when(() -> FunctionCommon.getSinkType(any())).thenReturn(String.class);
        ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod();
        ctx.when(() -> FunctionCommon.isFunctionCodeBuiltin(any())).thenReturn(true);
        ctx.when(() -> FunctionCommon.isFunctionCodeBuiltin(any())).thenReturn(true);
        ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())).thenReturn(mockedClassLoader);
    });
    ConnectorsManager mockedConnManager = mock(ConnectorsManager.class);
    Connector connector = Connector.builder().classLoader(mockedClassLoader).build();
    when(mockedConnManager.getConnector("cassandra")).thenReturn(connector);
    when(mockedConnManager.getSinkArchive(any())).thenReturn(getPulsarIOCassandraNar().toPath());
    when(mockedWorkerService.getConnectorsManager()).thenReturn(mockedConnManager);
    when(mockedRuntimeFactory.externallyManaged()).thenReturn(true);
    when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(false);
    SinkConfig sinkConfig = createDefaultSinkConfig();
    sinkConfig.setArchive("builtin://cassandra");
    try (FileInputStream inputStream = new FileInputStream(getPulsarIOCassandraNar())) {
        try {
            resource.registerSink(tenant, namespace, sink, inputStream, mockedFormData, null, sinkConfig, null, null);
            Assert.fail();
        } catch (RuntimeException e) {
            Assert.assertEquals(e.getMessage(), injectedErrMsg);
        }
    }
}

Example 3 with Connector

use of org.apache.pulsar.functions.utils.io.Connector in project pulsar by apache.

the class SinkApiV3ResourceTest method testRegisterSinkSuccessK8sNoUpload.

/*
    Externally managed runtime,
    uploadBuiltinSinksSources == false
    Make sure uploadFileToBookkeeper is not called
    */
@Test
public void testRegisterSinkSuccessK8sNoUpload() throws Exception {
    mockedWorkerService.getWorkerConfig().setUploadBuiltinSinksSources(false);
    mockStatic(WorkerUtils.class, ctx -> {
        ctx.when(() -> WorkerUtils.uploadFileToBookkeeper(anyString(), any(File.class), any(Namespace.class))).thenThrow(new RuntimeException("uploadFileToBookkeeper triggered"));
    });
    NarClassLoader mockedClassLoader = mock(NarClassLoader.class);
    mockStatic(FunctionCommon.class, ctx -> {
        ctx.when(() -> FunctionCommon.getSinkType(any())).thenReturn(String.class);
        ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod();
        ctx.when(() -> FunctionCommon.isFunctionCodeBuiltin(any())).thenReturn(true);
        ctx.when(() -> FunctionCommon.isFunctionCodeBuiltin(any())).thenReturn(true);
        ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())).thenReturn(mockedClassLoader);
    });
    ConnectorsManager mockedConnManager = mock(ConnectorsManager.class);
    Connector connector = Connector.builder().classLoader(mockedClassLoader).build();
    when(mockedConnManager.getConnector("cassandra")).thenReturn(connector);
    when(mockedWorkerService.getConnectorsManager()).thenReturn(mockedConnManager);
    when(mockedRuntimeFactory.externallyManaged()).thenReturn(true);
    when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(false);
    SinkConfig sinkConfig = createDefaultSinkConfig();
    sinkConfig.setArchive("builtin://cassandra");
    try (FileInputStream inputStream = new FileInputStream(getPulsarIOCassandraNar())) {
        resource.registerSink(tenant, namespace, sink, inputStream, mockedFormData, null, sinkConfig, null, null);
    }
}

Example 4 with Connector

use of org.apache.pulsar.functions.utils.io.Connector in project pulsar by apache.

the class LocalRunner method isBuiltInSink.

private ClassLoader isBuiltInSink(String sinkType) throws IOException {
    // Validate the connector type from the locally available connectors
    TreeMap<String, Connector> connectors = getConnectors();
    String sink = sinkType.replaceFirst("^builtin://", "");
    Connector connector = connectors.get(sink);
    if (connector != null && connector.getConnectorDefinition().getSinkClass() != null) {
        // Sink type is a valid built-in connector type
        return connector.getClassLoader();
    } else {
        return null;
    }
}

Example 5 with Connector

use of org.apache.pulsar.functions.utils.io.Connector in project pulsar by yahoo.

the class SourcesImpl method validateUpdateRequestParams.

private Function.FunctionDetails validateUpdateRequestParams(final String tenant, final String namespace, final String sourceName, final SourceConfig sourceConfig, final File sourcePackageFile) {
    // The rest end points take precedence over whatever is there in sourceconfig
    sourceConfig.setTenant(tenant);
    sourceConfig.setNamespace(namespace);
    sourceConfig.setName(sourceName);
    org.apache.pulsar.common.functions.Utils.inferMissingArguments(sourceConfig);
    ClassLoader classLoader = null;
    // check if source is builtin and extract classloader
    if (!StringUtils.isEmpty(sourceConfig.getArchive())) {
        String archive = sourceConfig.getArchive();
        if (archive.startsWith(org.apache.pulsar.common.functions.Utils.BUILTIN)) {
            archive = archive.replaceFirst("^builtin://", "");
            Connector connector = worker().getConnectorsManager().getConnector(archive);
            // check if builtin connector exists
            if (connector == null) {
                throw new IllegalArgumentException("Built-in source is not available");
            }
            classLoader = connector.getClassLoader();
        }
    }
    boolean shouldCloseClassLoader = false;
    try {
        // if source is not builtin, attempt to extract classloader from package file if it exists
        if (classLoader == null && sourcePackageFile != null) {
            classLoader = getClassLoaderFromPackage(sourceConfig.getClassName(), sourcePackageFile, worker().getWorkerConfig().getNarExtractionDirectory());
            shouldCloseClassLoader = true;
        }
        if (classLoader == null) {
            throw new IllegalArgumentException("Source package is not provided");
        }
        SourceConfigUtils.ExtractedSourceDetails sourceDetails = SourceConfigUtils.validateAndExtractDetails(sourceConfig, classLoader, worker().getWorkerConfig().getValidateConnectorConfig());
        return SourceConfigUtils.convert(sourceConfig, sourceDetails);
    } finally {
        if (shouldCloseClassLoader) {
            ClassLoaderUtils.closeClassLoader(classLoader);
        }
    }
}