Examples with InstanceConfig org.apache.pulsar.functions.instance.InstanceConfig used on opensource projects

Example 1 with InstanceConfig

use of org.apache.pulsar.functions.instance.InstanceConfig in project incubator-pulsar by apache.

the class ProcessRuntimeTest method testJavaConstructor.

@Test
public void testJavaConstructor() {
    InstanceConfig config = createJavaInstanceConfig(FunctionConfig.Runtime.JAVA);
    ProcessRuntime container = factory.createContainer(config, userJarFile);
    List<String> args = container.getProcessArgs();
    assertEquals(args.size(), 41);
    args.remove(args.size() - 1);
    String expectedArgs = "java -cp " + javaInstanceJarFile + " -Dlog4j.configurationFile=java_instance_log4j2.yml " + "-Dpulsar.log.dir=" + logDirectory + " -Dpulsar.log.file=" + config.getFunctionConfig().getName() + " org.apache.pulsar.functions.runtime.JavaInstanceMain" + " --jar " + userJarFile + " --instance_id " + config.getInstanceId() + " --function_id " + config.getFunctionId() + " --function_version " + config.getFunctionVersion() + " --tenant " + config.getFunctionConfig().getTenant() + " --namespace " + config.getFunctionConfig().getNamespace() + " --name " + config.getFunctionConfig().getName() + " --function_classname " + config.getFunctionConfig().getClassName() + " --log_topic " + config.getFunctionConfig().getLogTopic() + " --input_topics " + TEST_NAME + "-input1," + TEST_NAME + "-input2" + " --auto_ack false" + " --output_topic " + config.getFunctionConfig().getOutput() + " --output_serde_classname " + config.getFunctionConfig().getOutputSerdeClassName() + " --processing_guarantees ATLEAST_ONCE" + " --pulsar_serviceurl " + pulsarServiceUrl + " --max_buffered_tuples 1024 --port";
    assertEquals(expectedArgs, String.join(" ", args));
}

Example 2 with InstanceConfig

use of org.apache.pulsar.functions.instance.InstanceConfig in project incubator-pulsar by apache.

the class JavaInstanceMain method start.

public void start() throws Exception {
    InstanceConfig instanceConfig = new InstanceConfig();
    instanceConfig.setFunctionId(functionId);
    instanceConfig.setFunctionVersion(functionVersion);
    instanceConfig.setInstanceId(instanceId);
    instanceConfig.setMaxBufferedTuples(maxBufferedTuples);
    FunctionConfig.Builder functionConfigBuilder = FunctionConfig.newBuilder();
    functionConfigBuilder.setTenant(tenant);
    functionConfigBuilder.setNamespace(namespace);
    functionConfigBuilder.setName(functionName);
    functionConfigBuilder.setClassName(className);
    if (defaultSerdeInputTopics != null) {
        String[] inputTopics = defaultSerdeInputTopics.split(",");
        for (String inputTopic : inputTopics) {
            functionConfigBuilder.addInputs(inputTopic);
        }
    }
    if (customSerdeInputTopics != null && customSerdeClassnames != null) {
        String[] inputTopics = customSerdeInputTopics.split(",");
        String[] inputSerdeClassNames = customSerdeClassnames.split(",");
        if (inputTopics.length != inputSerdeClassNames.length) {
            throw new RuntimeException("Error specifying inputs");
        }
        for (int i = 0; i < inputTopics.length; ++i) {
            functionConfigBuilder.putCustomSerdeInputs(inputTopics[i], inputSerdeClassNames[i]);
        }
    }
    if (outputSerdeClassName != null) {
        functionConfigBuilder.setOutputSerdeClassName(outputSerdeClassName);
    }
    if (outputTopicName != null) {
        functionConfigBuilder.setOutput(outputTopicName);
    }
    if (logTopic != null) {
        functionConfigBuilder.setLogTopic(logTopic);
    }
    functionConfigBuilder.setProcessingGuarantees(processingGuarantees);
    if (autoAck.equals("true")) {
        functionConfigBuilder.setAutoAck(true);
    } else {
        functionConfigBuilder.setAutoAck(false);
    }
    if (userConfig != null && !userConfig.isEmpty()) {
        Type type = new TypeToken<Map<String, String>>() {
        }.getType();
        Map<String, String> userConfigMap = new Gson().fromJson(userConfig, type);
        functionConfigBuilder.putAllUserConfig(userConfigMap);
    }
    FunctionConfig functionConfig = functionConfigBuilder.build();
    instanceConfig.setFunctionConfig(functionConfig);
    ThreadRuntimeFactory containerFactory = new ThreadRuntimeFactory("LocalRunnerThreadGroup", pulsarServiceUrl, stateStorageServiceUrl);
    RuntimeSpawner runtimeSpawner = new RuntimeSpawner(instanceConfig, jarFile, containerFactory, null);
    server = ServerBuilder.forPort(port).addService(new InstanceControlImpl(runtimeSpawner)).build().start();
    log.info("JaveInstance Server started, listening on " + port);
    java.lang.Runtime.getRuntime().addShutdownHook(new Thread() {

        @Override
        public void run() {
            // Use stderr here since the logger may have been reset by its JVM shutdown hook.
            try {
                server.shutdown();
                runtimeSpawner.close();
            } catch (Exception ex) {
                System.err.println(ex);
            }
        }
    });
    log.info("Starting runtimeSpawner");
    runtimeSpawner.start();
    runtimeSpawner.join();
    log.info("RuntimeSpawner quit, shutting down JavaInstance");
    server.shutdown();
}

Example 3 with InstanceConfig

use of org.apache.pulsar.functions.instance.InstanceConfig in project incubator-pulsar by apache.

the class ProcessRuntimeTest method testPythonConstructor.

@Test
public void testPythonConstructor() {
    InstanceConfig config = createJavaInstanceConfig(FunctionConfig.Runtime.PYTHON);
    ProcessRuntime container = factory.createContainer(config, userJarFile);
    List<String> args = container.getProcessArgs();
    assertEquals(args.size(), 40);
    args.remove(args.size() - 1);
    String expectedArgs = "python " + pythonInstanceFile + " --py " + userJarFile + " --logging_directory " + logDirectory + " --logging_file " + config.getFunctionConfig().getName() + " --instance_id " + config.getInstanceId() + " --function_id " + config.getFunctionId() + " --function_version " + config.getFunctionVersion() + " --tenant " + config.getFunctionConfig().getTenant() + " --namespace " + config.getFunctionConfig().getNamespace() + " --name " + config.getFunctionConfig().getName() + " --function_classname " + config.getFunctionConfig().getClassName() + " --log_topic " + config.getFunctionConfig().getLogTopic() + " --input_topics " + TEST_NAME + "-input1," + TEST_NAME + "-input2" + " --auto_ack false" + " --output_topic " + config.getFunctionConfig().getOutput() + " --output_serde_classname " + config.getFunctionConfig().getOutputSerdeClassName() + " --processing_guarantees ATLEAST_ONCE" + " --pulsar_serviceurl " + pulsarServiceUrl + " --max_buffered_tuples 1024 --port";
    assertEquals(expectedArgs, String.join(" ", args));
}

Example 4 with InstanceConfig

use of org.apache.pulsar.functions.instance.InstanceConfig in project incubator-pulsar by apache.

the class ProcessRuntimeTest method createJavaInstanceConfig.

InstanceConfig createJavaInstanceConfig(FunctionConfig.Runtime runtime) {
    InstanceConfig config = new InstanceConfig();
    config.setFunctionConfig(createFunctionConfig(runtime));
    config.setFunctionId(java.util.UUID.randomUUID().toString());
    config.setFunctionVersion("1.0");
    config.setInstanceId(java.util.UUID.randomUUID().toString());
    config.setMaxBufferedTuples(1024);
    return config;
}

Example 5 with InstanceConfig

use of org.apache.pulsar.functions.instance.InstanceConfig in project incubator-pulsar by apache.

the class FunctionActioner method startFunction.

private void startFunction(FunctionRuntimeInfo functionRuntimeInfo) throws Exception {
    Function.Instance instance = functionRuntimeInfo.getFunctionInstance();
    FunctionMetaData functionMetaData = instance.getFunctionMetaData();
    log.info("Starting function {} - {} ...", functionMetaData.getFunctionConfig().getName(), instance.getInstanceId());
    File pkgDir = new File(workerConfig.getDownloadDirectory(), getDownloadPackagePath(functionMetaData));
    pkgDir.mkdirs();
    int instanceId = functionRuntimeInfo.getFunctionInstance().getInstanceId();
    File pkgFile = new File(pkgDir, new File(FunctionConfigUtils.getDownloadFileName(functionMetaData.getFunctionConfig())).getName());
    if (!pkgFile.exists()) {
        // download only when the package file doesn't exist
        File tempPkgFile;
        while (true) {
            tempPkgFile = new File(pkgDir, pkgFile.getName() + "." + instanceId + "." + UUID.randomUUID().toString());
            if (!tempPkgFile.exists() && tempPkgFile.createNewFile()) {
                break;
            }
        }
        try {
            log.info("Function package file {} will be downloaded from {}", tempPkgFile, functionMetaData.getPackageLocation());
            Utils.downloadFromBookkeeper(dlogNamespace, new FileOutputStream(tempPkgFile), functionMetaData.getPackageLocation().getPackagePath());
            // this ensures one instance will successfully download the package.
            try {
                Files.createLink(Paths.get(pkgFile.toURI()), Paths.get(tempPkgFile.toURI()));
                log.info("Function package file is linked from {} to {}", tempPkgFile, pkgFile);
            } catch (FileAlreadyExistsException faee) {
                // file already exists
                log.warn("Function package has been downloaded from {} and saved at {}", functionMetaData.getPackageLocation(), pkgFile);
            }
        } finally {
            tempPkgFile.delete();
        }
    }
    InstanceConfig instanceConfig = new InstanceConfig();
    instanceConfig.setFunctionConfig(functionMetaData.getFunctionConfig());
    // TODO: set correct function id and version when features implemented
    instanceConfig.setFunctionId(UUID.randomUUID().toString());
    instanceConfig.setFunctionVersion(UUID.randomUUID().toString());
    instanceConfig.setInstanceId(String.valueOf(instanceId));
    instanceConfig.setMaxBufferedTuples(1024);
    RuntimeSpawner runtimeSpawner = new RuntimeSpawner(instanceConfig, pkgFile.getAbsolutePath(), runtimeFactory, workerConfig.getInstanceLivenessCheckFreqMs());
    functionRuntimeInfo.setRuntimeSpawner(runtimeSpawner);
    runtimeSpawner.start();
}