Examples with ThreadRuntimeFactory org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory used on opensource projects

Example 1 with ThreadRuntimeFactory

use of org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory in project pulsar by apache.

the class JavaInstanceStarter method start.

public void start(String[] args, ClassLoader functionInstanceClassLoader, ClassLoader rootClassLoader) throws Exception {
    Thread.currentThread().setContextClassLoader(functionInstanceClassLoader);
    JCommander jcommander = new JCommander(this);
    // parse args by JCommander
    jcommander.parse(args);
    InstanceConfig instanceConfig = new InstanceConfig();
    instanceConfig.setFunctionId(functionId);
    instanceConfig.setFunctionVersion(functionVersion);
    instanceConfig.setInstanceId(instanceId);
    instanceConfig.setMaxBufferedTuples(maxBufferedTuples);
    instanceConfig.setClusterName(clusterName);
    instanceConfig.setMaxPendingAsyncRequests(maxPendingAsyncRequests);
    instanceConfig.setExposePulsarAdminClientEnabled(exposePulsarAdminClientEnabled);
    Function.FunctionDetails.Builder functionDetailsBuilder = Function.FunctionDetails.newBuilder();
    if (functionDetailsJsonString.charAt(0) == '\'') {
        functionDetailsJsonString = functionDetailsJsonString.substring(1);
    }
    if (functionDetailsJsonString.charAt(functionDetailsJsonString.length() - 1) == '\'') {
        functionDetailsJsonString = functionDetailsJsonString.substring(0, functionDetailsJsonString.length() - 1);
    }
    JsonFormat.parser().merge(functionDetailsJsonString, functionDetailsBuilder);
    Function.FunctionDetails functionDetails = functionDetailsBuilder.build();
    instanceConfig.setFunctionDetails(functionDetails);
    instanceConfig.setPort(port);
    instanceConfig.setMetricsPort(metricsPort);
    Map<String, String> secretsProviderConfigMap = null;
    if (!StringUtils.isEmpty(secretsProviderConfig)) {
        if (secretsProviderConfig.charAt(0) == '\'') {
            secretsProviderConfig = secretsProviderConfig.substring(1);
        }
        if (secretsProviderConfig.charAt(secretsProviderConfig.length() - 1) == '\'') {
            secretsProviderConfig = secretsProviderConfig.substring(0, secretsProviderConfig.length() - 1);
        }
        Type type = new TypeToken<Map<String, String>>() {
        }.getType();
        secretsProviderConfigMap = new Gson().fromJson(secretsProviderConfig, type);
    }
    if (StringUtils.isEmpty(secretsProviderClassName)) {
        secretsProviderClassName = ClearTextSecretsProvider.class.getName();
    }
    SecretsProvider secretsProvider;
    try {
        secretsProvider = (SecretsProvider) Reflections.createInstance(secretsProviderClassName, functionInstanceClassLoader);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
    secretsProvider.init(secretsProviderConfigMap);
    // Collector Registry for prometheus metrics
    FunctionCollectorRegistry collectorRegistry = FunctionCollectorRegistry.getDefaultImplementation();
    RuntimeUtils.registerDefaultCollectors(collectorRegistry);
    containerFactory = new ThreadRuntimeFactory("LocalRunnerThreadGroup", pulsarServiceUrl, stateStorageImplClass, stateStorageServiceUrl, AuthenticationConfig.builder().clientAuthenticationPlugin(clientAuthenticationPlugin).clientAuthenticationParameters(clientAuthenticationParameters).useTls(isTrue(useTls)).tlsAllowInsecureConnection(isTrue(tlsAllowInsecureConnection)).tlsHostnameVerificationEnable(isTrue(tlsHostNameVerificationEnabled)).tlsTrustCertsFilePath(tlsTrustCertFilePath).build(), secretsProvider, collectorRegistry, narExtractionDirectory, rootClassLoader, exposePulsarAdminClientEnabled, webServiceUrl);
    runtimeSpawner = new RuntimeSpawner(instanceConfig, jarFile, // we really dont use this in thread container
    null, containerFactory, expectedHealthCheckInterval * 1000);
    server = ServerBuilder.forPort(port).addService(new InstanceControlImpl(runtimeSpawner)).build().start();
    log.info("JavaInstance Server started, listening on " + port);
    java.lang.Runtime.getRuntime().addShutdownHook(new Thread() {

        @Override
        public void run() {
            // Use stderr here since the logger may have been reset by its JVM shutdown hook.
            try {
                close();
            } catch (Exception ex) {
                System.err.println(ex);
            }
        }
    });
    log.info("Starting runtimeSpawner");
    runtimeSpawner.start();
    // starting metrics server
    log.info("Starting metrics server on port {}", metricsPort);
    metricsServer = new HTTPServer(new InetSocketAddress(metricsPort), collectorRegistry, true);
    if (expectedHealthCheckInterval > 0) {
        healthCheckTimer = InstanceCache.getInstanceCache().getScheduledExecutorService().scheduleAtFixedRate(() -> {
            try {
                if (System.currentTimeMillis() - lastHealthCheckTs > 3 * expectedHealthCheckInterval * 1000) {
                    log.info("Haven't received health check from spawner in a while. Stopping instance...");
                    close();
                }
            } catch (Exception e) {
                log.error("Error occurred when checking for latest health check", e);
            }
        }, expectedHealthCheckInterval * 1000, expectedHealthCheckInterval * 1000, TimeUnit.MILLISECONDS);
    }
    runtimeSpawner.join();
    log.info("RuntimeSpawner quit, shutting down JavaInstance");
    close();
}

Example 2 with ThreadRuntimeFactory

use of org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory in project pulsar by apache.

the class LocalRunner method startThreadedMode.

private void startThreadedMode(org.apache.pulsar.functions.proto.Function.FunctionDetails functionDetails, int parallelism, int instanceIdOffset, String serviceUrl, String stateStorageServiceUrl, AuthenticationConfig authConfig, String userCodeFile) throws Exception {
    if (metricsPortStart != null) {
        if (metricsPortStart < 0 || metricsPortStart > 65535) {
            throw new IllegalArgumentException("Metrics port need to be within the range of 0 and 65535");
        }
    }
    SecretsProvider secretsProvider;
    if (secretsProviderClassName != null) {
        secretsProvider = (SecretsProvider) Reflections.createInstance(secretsProviderClassName, ClassLoader.getSystemClassLoader());
        Map<String, String> config = null;
        if (secretsProviderConfig != null) {
            config = (Map<String, String>) new Gson().fromJson(secretsProviderConfig, Map.class);
        }
        secretsProvider.init(config);
    } else {
        secretsProvider = new ClearTextSecretsProvider();
    }
    boolean exposePulsarAdminClientEnabled = false;
    if (functionConfig != null && functionConfig.getExposePulsarAdminClientEnabled() != null) {
        exposePulsarAdminClientEnabled = functionConfig.getExposePulsarAdminClientEnabled();
    }
    // Collector Registry for prometheus metrics
    FunctionCollectorRegistry collectorRegistry = FunctionCollectorRegistry.getDefaultImplementation();
    RuntimeUtils.registerDefaultCollectors(collectorRegistry);
    ClassLoader originalClassLoader = Thread.currentThread().getContextClassLoader();
    try {
        if (userCodeClassLoader != null) {
            Thread.currentThread().setContextClassLoader(userCodeClassLoader);
        }
        runtimeFactory = new ThreadRuntimeFactory("LocalRunnerThreadGroup", serviceUrl, stateStorageImplClass, stateStorageServiceUrl, authConfig, secretsProvider, collectorRegistry, narExtractionDirectory, null, exposePulsarAdminClientEnabled, webServiceUrl);
    } finally {
        Thread.currentThread().setContextClassLoader(originalClassLoader);
    }
    for (int i = 0; i < parallelism; ++i) {
        InstanceConfig instanceConfig = new InstanceConfig();
        instanceConfig.setFunctionDetails(functionDetails);
        // TODO: correctly implement function version and id
        instanceConfig.setFunctionVersion(UUID.randomUUID().toString());
        instanceConfig.setFunctionId(UUID.randomUUID().toString());
        instanceConfig.setInstanceId(i + instanceIdOffset);
        instanceConfig.setMaxBufferedTuples(1024);
        if (metricsPortStart != null) {
            instanceConfig.setMetricsPort(metricsPortStart);
        }
        instanceConfig.setClusterName("local");
        if (functionConfig != null) {
            instanceConfig.setMaxPendingAsyncRequests(functionConfig.getMaxPendingAsyncRequests());
            if (functionConfig.getExposePulsarAdminClientEnabled() != null) {
                instanceConfig.setExposePulsarAdminClientEnabled(functionConfig.getExposePulsarAdminClientEnabled());
            }
        }
        RuntimeSpawner runtimeSpawner = new RuntimeSpawner(instanceConfig, userCodeFile, null, runtimeFactory, instanceLivenessCheck);
        spawners.add(runtimeSpawner);
        runtimeSpawner.start();
    }
    if (metricsPortStart != null) {
        // starting metrics server
        log.info("Starting metrics server on port {}", metricsPortStart);
        metricsServer = new HTTPServer(new InetSocketAddress(metricsPortStart), collectorRegistry, true);
    }
}

Example 3 with ThreadRuntimeFactory

use of org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory in project pulsar by apache.

the class SchedulerManagerTest method testHeartbeatFunction.

@Test
public void testHeartbeatFunction() throws Exception {
    List<Function.FunctionMetaData> functionMetaDataList = new LinkedList<>();
    final long version = 5;
    final String workerId1 = "host-workerId-1";
    final String workerId2 = "host-workerId-2";
    Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails(Function.FunctionDetails.newBuilder().setName(workerId1).setNamespace(SchedulerManager.HEARTBEAT_NAMESPACE).setTenant(SchedulerManager.HEARTBEAT_TENANT).setParallelism(1)).setVersion(version).build();
    Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionDetails(Function.FunctionDetails.newBuilder().setName(workerId2).setNamespace(SchedulerManager.HEARTBEAT_NAMESPACE).setTenant(SchedulerManager.HEARTBEAT_TENANT).setParallelism(1)).setVersion(version).build();
    functionMetaDataList.add(function1);
    functionMetaDataList.add(function2);
    doReturn(functionMetaDataList).when(functionMetaDataManager).getAllFunctionMetaData();
    ThreadRuntimeFactory factory = mock(ThreadRuntimeFactory.class);
    doReturn(factory).when(functionRuntimeManager).getRuntimeFactory();
    Map<String, Map<String, Function.Assignment>> currentAssignments = new HashMap<>();
    Map<String, Function.Assignment> assignmentEntry1 = new HashMap<>();
    currentAssignments.put("worker-1", assignmentEntry1);
    doReturn(currentAssignments).when(functionRuntimeManager).getCurrentAssignments();
    List<WorkerInfo> workerInfoList = new LinkedList<>();
    workerInfoList.add(WorkerInfo.of(workerId1, "workerHostname-1", 5000));
    workerInfoList.add(WorkerInfo.of(workerId2, "workerHostname-1", 6000));
    doReturn(workerInfoList).when(membershipManager).getCurrentMembership();
    // i am leader
    doReturn(true).when(leaderService).isLeader();
    callSchedule();
    List<Invocation> invocations = getMethodInvocationDetails(message, TypedMessageBuilder.class.getMethod("send"));
    Assert.assertEquals(invocations.size(), 2);
    invocations = getMethodInvocationDetails(message, TypedMessageBuilder.class.getMethod("value", Object.class));
    invocations.forEach(invocation -> {
        try {
            Assignment assignment = Assignment.parseFrom((byte[]) invocation.getRawArguments()[0]);
            String functionName = assignment.getInstance().getFunctionMetaData().getFunctionDetails().getName();
            String assignedWorkerId = assignment.getWorkerId();
            Assert.assertEquals(functionName, assignedWorkerId);
        } catch (InvalidProtocolBufferException e) {
            throw new RuntimeException(e);
        }
    });
}

Example 4 with ThreadRuntimeFactory

use of org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory in project pulsar by apache.

the class SchedulerManagerTest method testScalingUp.

@Test
public void testScalingUp() throws Exception {
    List<Function.FunctionMetaData> functionMetaDataList = new LinkedList<>();
    long version = 5;
    Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1").setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version).build();
    Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-2").setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version).build();
    functionMetaDataList.add(function1);
    functionMetaDataList.add(function2);
    doReturn(functionMetaDataList).when(functionMetaDataManager).getAllFunctionMetaData();
    ThreadRuntimeFactory factory = mock(ThreadRuntimeFactory.class);
    doReturn(factory).when(functionRuntimeManager).getRuntimeFactory();
    // set assignments
    Function.Assignment assignment1 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0).build()).build();
    Map<String, Map<String, Function.Assignment>> currentAssignments = new HashMap<>();
    Map<String, Function.Assignment> assignmentEntry1 = new HashMap<>();
    assignmentEntry1.put(FunctionCommon.getFullyQualifiedInstanceId(assignment1.getInstance()), assignment1);
    currentAssignments.put("worker-1", assignmentEntry1);
    doReturn(currentAssignments).when(functionRuntimeManager).getCurrentAssignments();
    // single node
    List<WorkerInfo> workerInfoList = new LinkedList<>();
    workerInfoList.add(WorkerInfo.of("worker-1", "workerHostname-1", 5000));
    doReturn(workerInfoList).when(membershipManager).getCurrentMembership();
    // i am leader
    doReturn(true).when(leaderService).isLeader();
    callSchedule();
    List<Invocation> invocations = getMethodInvocationDetails(message, TypedMessageBuilder.class.getMethod("send"));
    Assert.assertEquals(invocations.size(), 1);
    invocations = getMethodInvocationDetails(message, TypedMessageBuilder.class.getMethod("value", Object.class));
    byte[] send = (byte[]) invocations.get(0).getRawArguments()[0];
    Assignment assignments = Assignment.parseFrom(send);
    log.info("assignments: {}", assignments);
    Function.Assignment assignment2 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2).setInstanceId(0).build()).build();
    Assert.assertEquals(assignments, assignment2);
    // updating assignments
    currentAssignments.get("worker-1").put(FunctionCommon.getFullyQualifiedInstanceId(assignment2.getInstance()), assignment2);
    // scale up
    Function.FunctionMetaData function2Scaled = Function.FunctionMetaData.newBuilder().setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-2").setNamespace("namespace-1").setTenant("tenant-1").setParallelism(3)).setVersion(version).build();
    functionMetaDataList = new LinkedList<>();
    functionMetaDataList.add(function1);
    functionMetaDataList.add(function2Scaled);
    doReturn(functionMetaDataList).when(functionMetaDataManager).getAllFunctionMetaData();
    Function.Assignment assignment2Scaled1 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2Scaled).setInstanceId(0).build()).build();
    Function.Assignment assignment2Scaled2 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2Scaled).setInstanceId(1).build()).build();
    Function.Assignment assignment2Scaled3 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2Scaled).setInstanceId(2).build()).build();
    callSchedule();
    invocations = getMethodInvocationDetails(message, TypedMessageBuilder.class.getMethod("send"));
    Assert.assertEquals(invocations.size(), 4);
    invocations = getMethodInvocationDetails(message, TypedMessageBuilder.class.getMethod("value", Object.class));
    Set<Assignment> allAssignments = Sets.newHashSet();
    invocations.forEach(invocation -> {
        try {
            allAssignments.add(Assignment.parseFrom((byte[]) invocation.getRawArguments()[0]));
        } catch (InvalidProtocolBufferException e) {
            throw new RuntimeException(e);
        }
    });
    assertTrue(allAssignments.contains(assignment2Scaled1));
    assertTrue(allAssignments.contains(assignment2Scaled2));
    assertTrue(allAssignments.contains(assignment2Scaled3));
}

Example 5 with ThreadRuntimeFactory

use of org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory in project pulsar by apache.

the class SchedulerManagerTest method testNothingNewToSchedule.

@Test
public void testNothingNewToSchedule() throws Exception {
    List<Function.FunctionMetaData> functionMetaDataList = new LinkedList<>();
    long version = 5;
    Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionDetails(Function.FunctionDetails.newBuilder().setName("func-1").setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version).build();
    functionMetaDataList.add(function1);
    doReturn(functionMetaDataList).when(functionMetaDataManager).getAllFunctionMetaData();
    ThreadRuntimeFactory factory = mock(ThreadRuntimeFactory.class);
    doReturn(factory).when(functionRuntimeManager).getRuntimeFactory();
    // set assignments
    Function.Assignment assignment1 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0).build()).build();
    Map<String, Map<String, Function.Assignment>> currentAssignments = new HashMap<>();
    Map<String, Function.Assignment> assignmentEntry1 = new HashMap<>();
    assignmentEntry1.put(FunctionCommon.getFullyQualifiedInstanceId(assignment1.getInstance()), assignment1);
    currentAssignments.put("worker-1", assignmentEntry1);
    doReturn(currentAssignments).when(functionRuntimeManager).getCurrentAssignments();
    // single node
    List<WorkerInfo> workerInfoList = new LinkedList<>();
    workerInfoList.add(WorkerInfo.of("worker-1", "workerHostname-1", 5000));
    doReturn(workerInfoList).when(membershipManager).getCurrentMembership();
    // i am leader
    doReturn(true).when(leaderService).isLeader();
    callSchedule();
    List<Invocation> invocations = getMethodInvocationDetails(message, TypedMessageBuilder.class.getMethod("send"));
    Assert.assertEquals(invocations.size(), 0);
    verify(errorNotifier, times(0)).triggerError(any());
}