Examples with PulsarClient org.apache.pulsar.client.api.PulsarClient used on opensource projects

Example 1 with PulsarClient

use of org.apache.pulsar.client.api.PulsarClient in project incubator-pulsar by apache.

the class CmdProduce method run.

/**
 * Run the producer.
 *
 * @return 0 for success, < 0 otherwise
 * @throws Exception
 */
public int run() throws PulsarClientException {
    if (mainOptions.size() != 1)
        throw (new ParameterException("Please provide one and only one topic name."));
    if (this.numTimesProduce <= 0)
        throw (new ParameterException("Number of times need to be positive number."));
    if (messages.size() == 0 && messageFileNames.size() == 0)
        throw (new ParameterException("Please supply message content with either --messages or --files"));
    int totalMessages = (messages.size() + messageFileNames.size()) * numTimesProduce;
    if (totalMessages > MAX_MESSAGES) {
        String msg = "Attempting to send " + totalMessages + " messages. Please do not send more than " + MAX_MESSAGES + " messages";
        throw new ParameterException(msg);
    }
    String topic = this.mainOptions.get(0);
    int numMessagesSent = 0;
    int returnCode = 0;
    try {
        PulsarClient client = clientBuilder.build();
        Producer<byte[]> producer = client.newProducer().topic(topic).create();
        List<byte[]> messageBodies = generateMessageBodies(this.messages, this.messageFileNames);
        RateLimiter limiter = (this.publishRate > 0) ? RateLimiter.create(this.publishRate) : null;
        for (int i = 0; i < this.numTimesProduce; i++) {
            List<Message<byte[]>> messages = generateMessages(messageBodies);
            for (Message<byte[]> msg : messages) {
                if (limiter != null)
                    limiter.acquire();
                producer.send(msg);
                numMessagesSent++;
            }
        }
        client.close();
    } catch (Exception e) {
        LOG.error("Error while producing messages");
        LOG.error(e.getMessage(), e);
        returnCode = -1;
    } finally {
        LOG.info("{} messages successfully produced", numMessagesSent);
    }
    return returnCode;
}

Example 2 with PulsarClient

use of org.apache.pulsar.client.api.PulsarClient in project incubator-pulsar by apache.

the class SchedulerManagerTest method testUpdate.

@Test
public void testUpdate() throws Exception {
    List<Function.FunctionMetaData> functionMetaDataList = new LinkedList<>();
    long version = 5;
    Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath("/foo/bar1")).setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1").setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version).build();
    Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath("/foo/bar1")).setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-2").setNamespace("namespace-1").setTenant("tenant-1").setParallelism(3)).setVersion(version).build();
    functionMetaDataList.add(function1);
    functionMetaDataList.add(function2);
    doReturn(functionMetaDataList).when(functionMetaDataManager).getAllFunctionMetaData();
    // set assignments
    Function.Assignment assignment1 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0).build()).build();
    Map<String, Map<String, Function.Assignment>> currentAssignments = new HashMap<>();
    Map<String, Function.Assignment> assignmentEntry1 = new HashMap<>();
    assignmentEntry1.put(Utils.getFullyQualifiedInstanceId(assignment1.getInstance()), assignment1);
    currentAssignments.put("worker-1", assignmentEntry1);
    doReturn(currentAssignments).when(functionRuntimeManager).getCurrentAssignments();
    // set version
    doReturn(version).when(functionRuntimeManager).getCurrentAssignmentVersion();
    // single node
    List<MembershipManager.WorkerInfo> workerInfoList = new LinkedList<>();
    workerInfoList.add(MembershipManager.WorkerInfo.of("worker-1", "workerHostname-1", 5000));
    doReturn(workerInfoList).when(membershipManager).getCurrentMembership();
    // i am leader
    doReturn(true).when(membershipManager).isLeader();
    callSchedule();
    List<Invocation> invocations = getMethodInvocationDetails(producer, Producer.class.getMethod("sendAsync", Object.class));
    Assert.assertEquals(invocations.size(), 1);
    byte[] send = (byte[]) invocations.get(0).getRawArguments()[0];
    Request.AssignmentsUpdate assignmentsUpdate = Request.AssignmentsUpdate.parseFrom(send);
    log.info("assignmentsUpdate: {}", assignmentsUpdate);
    Function.Assignment assignment2_1 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2).setInstanceId(0).build()).build();
    Function.Assignment assignment2_2 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2).setInstanceId(1).build()).build();
    Function.Assignment assignment2_3 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2).setInstanceId(2).build()).build();
    Assert.assertEquals(assignmentsUpdate, Request.AssignmentsUpdate.newBuilder().setVersion(version + 1).addAssignments(assignment1).addAssignments(assignment2_1).addAssignments(assignment2_2).addAssignments(assignment2_3).build());
    // scale down
    PulsarClient pulsarClient = mock(PulsarClient.class);
    doReturn(producer).when(pulsarClient).createProducer(any(), any());
    Function.FunctionMetaData function2Updated = Function.FunctionMetaData.newBuilder().setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath("/foo/bar2")).setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-2").setNamespace("namespace-1").setTenant("tenant-1").setParallelism(3)).setVersion(version).build();
    functionMetaDataList = new LinkedList<>();
    functionMetaDataList.add(function1);
    functionMetaDataList.add(function2Updated);
    doReturn(functionMetaDataList).when(functionMetaDataManager).getAllFunctionMetaData();
    Function.Assignment assignment2Updated1 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2Updated).setInstanceId(0).build()).build();
    Function.Assignment assignment2Updated2 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2Updated).setInstanceId(1).build()).build();
    Function.Assignment assignment2Updated3 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2Updated).setInstanceId(2).build()).build();
    callSchedule();
    invocations = getMethodInvocationDetails(producer, Producer.class.getMethod("sendAsync", Object.class));
    Assert.assertEquals(invocations.size(), 2);
    send = (byte[]) invocations.get(1).getRawArguments()[0];
    assignmentsUpdate = Request.AssignmentsUpdate.parseFrom(send);
    Assert.assertEquals(assignmentsUpdate, Request.AssignmentsUpdate.newBuilder().setVersion(version + 1 + 1).addAssignments(assignment1).addAssignments(assignment2Updated1).addAssignments(assignment2Updated2).addAssignments(assignment2Updated3).build());
}

Example 3 with PulsarClient

use of org.apache.pulsar.client.api.PulsarClient in project incubator-pulsar by apache.

the class SchedulerManagerTest method testScalingDown.

@Test
public void testScalingDown() throws Exception {
    List<Function.FunctionMetaData> functionMetaDataList = new LinkedList<>();
    long version = 5;
    Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1").setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version).build();
    Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-2").setNamespace("namespace-1").setTenant("tenant-1").setParallelism(3)).setVersion(version).build();
    functionMetaDataList.add(function1);
    functionMetaDataList.add(function2);
    doReturn(functionMetaDataList).when(functionMetaDataManager).getAllFunctionMetaData();
    // set assignments
    Function.Assignment assignment1 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0).build()).build();
    Map<String, Map<String, Function.Assignment>> currentAssignments = new HashMap<>();
    Map<String, Function.Assignment> assignmentEntry1 = new HashMap<>();
    assignmentEntry1.put(Utils.getFullyQualifiedInstanceId(assignment1.getInstance()), assignment1);
    currentAssignments.put("worker-1", assignmentEntry1);
    doReturn(currentAssignments).when(functionRuntimeManager).getCurrentAssignments();
    // set version
    doReturn(version).when(functionRuntimeManager).getCurrentAssignmentVersion();
    // single node
    List<MembershipManager.WorkerInfo> workerInfoList = new LinkedList<>();
    workerInfoList.add(MembershipManager.WorkerInfo.of("worker-1", "workerHostname-1", 5000));
    doReturn(workerInfoList).when(membershipManager).getCurrentMembership();
    // i am leader
    doReturn(true).when(membershipManager).isLeader();
    callSchedule();
    List<Invocation> invocations = getMethodInvocationDetails(producer, Producer.class.getMethod("sendAsync", Object.class));
    Assert.assertEquals(invocations.size(), 1);
    byte[] send = (byte[]) invocations.get(0).getRawArguments()[0];
    Request.AssignmentsUpdate assignmentsUpdate = Request.AssignmentsUpdate.parseFrom(send);
    log.info("assignmentsUpdate: {}", assignmentsUpdate);
    Function.Assignment assignment2_1 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2).setInstanceId(0).build()).build();
    Function.Assignment assignment2_2 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2).setInstanceId(1).build()).build();
    Function.Assignment assignment2_3 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2).setInstanceId(2).build()).build();
    Assert.assertEquals(assignmentsUpdate, Request.AssignmentsUpdate.newBuilder().setVersion(version + 1).addAssignments(assignment1).addAssignments(assignment2_1).addAssignments(assignment2_2).addAssignments(assignment2_3).build());
    // scale down
    PulsarClient pulsarClient = mock(PulsarClient.class);
    doReturn(producer).when(pulsarClient).createProducer(any(), any());
    Function.FunctionMetaData function2Scaled = Function.FunctionMetaData.newBuilder().setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-2").setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version).build();
    functionMetaDataList = new LinkedList<>();
    functionMetaDataList.add(function1);
    functionMetaDataList.add(function2Scaled);
    doReturn(functionMetaDataList).when(functionMetaDataManager).getAllFunctionMetaData();
    Function.Assignment assignment2Scaled = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2Scaled).setInstanceId(0).build()).build();
    callSchedule();
    invocations = getMethodInvocationDetails(producer, Producer.class.getMethod("sendAsync", Object.class));
    Assert.assertEquals(invocations.size(), 2);
    send = (byte[]) invocations.get(1).getRawArguments()[0];
    assignmentsUpdate = Request.AssignmentsUpdate.parseFrom(send);
    Assert.assertEquals(assignmentsUpdate, Request.AssignmentsUpdate.newBuilder().setVersion(version + 1 + 1).addAssignments(assignment1).addAssignments(assignment2Scaled).build());
}

Example 4 with PulsarClient

use of org.apache.pulsar.client.api.PulsarClient in project incubator-pulsar by apache.

the class SchedulerManagerTest method testScalingUp.

@Test
public void testScalingUp() throws Exception {
    List<Function.FunctionMetaData> functionMetaDataList = new LinkedList<>();
    long version = 5;
    Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1").setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version).build();
    Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-2").setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version).build();
    functionMetaDataList.add(function1);
    functionMetaDataList.add(function2);
    doReturn(functionMetaDataList).when(functionMetaDataManager).getAllFunctionMetaData();
    // set assignments
    Function.Assignment assignment1 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0).build()).build();
    Map<String, Map<String, Function.Assignment>> currentAssignments = new HashMap<>();
    Map<String, Function.Assignment> assignmentEntry1 = new HashMap<>();
    assignmentEntry1.put(Utils.getFullyQualifiedInstanceId(assignment1.getInstance()), assignment1);
    currentAssignments.put("worker-1", assignmentEntry1);
    doReturn(currentAssignments).when(functionRuntimeManager).getCurrentAssignments();
    // set version
    doReturn(version).when(functionRuntimeManager).getCurrentAssignmentVersion();
    // single node
    List<MembershipManager.WorkerInfo> workerInfoList = new LinkedList<>();
    workerInfoList.add(MembershipManager.WorkerInfo.of("worker-1", "workerHostname-1", 5000));
    doReturn(workerInfoList).when(membershipManager).getCurrentMembership();
    // i am leader
    doReturn(true).when(membershipManager).isLeader();
    callSchedule();
    List<Invocation> invocations = getMethodInvocationDetails(producer, Producer.class.getMethod("sendAsync", Object.class));
    Assert.assertEquals(invocations.size(), 1);
    byte[] send = (byte[]) invocations.get(0).getRawArguments()[0];
    Request.AssignmentsUpdate assignmentsUpdate = Request.AssignmentsUpdate.parseFrom(send);
    log.info("assignmentsUpdate: {}", assignmentsUpdate);
    Function.Assignment assignment2 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2).setInstanceId(0).build()).build();
    Assert.assertEquals(assignmentsUpdate, Request.AssignmentsUpdate.newBuilder().setVersion(version + 1).addAssignments(assignment1).addAssignments(assignment2).build());
    // scale up
    PulsarClient pulsarClient = mock(PulsarClient.class);
    doReturn(producer).when(pulsarClient).createProducer(any(), any());
    Function.FunctionMetaData function2Scaled = Function.FunctionMetaData.newBuilder().setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-2").setNamespace("namespace-1").setTenant("tenant-1").setParallelism(3)).setVersion(version).build();
    functionMetaDataList = new LinkedList<>();
    functionMetaDataList.add(function1);
    functionMetaDataList.add(function2Scaled);
    doReturn(functionMetaDataList).when(functionMetaDataManager).getAllFunctionMetaData();
    Function.Assignment assignment2Scaled1 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2Scaled).setInstanceId(0).build()).build();
    Function.Assignment assignment2Scaled2 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2Scaled).setInstanceId(1).build()).build();
    Function.Assignment assignment2Scaled3 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2Scaled).setInstanceId(2).build()).build();
    callSchedule();
    invocations = getMethodInvocationDetails(producer, Producer.class.getMethod("sendAsync", Object.class));
    Assert.assertEquals(invocations.size(), 2);
    send = (byte[]) invocations.get(1).getRawArguments()[0];
    assignmentsUpdate = Request.AssignmentsUpdate.parseFrom(send);
    Assert.assertEquals(assignmentsUpdate, Request.AssignmentsUpdate.newBuilder().setVersion(version + 1 + 1).addAssignments(assignment1).addAssignments(assignment2Scaled1).addAssignments(assignment2Scaled2).addAssignments(assignment2Scaled3).build());
}

Example 5 with PulsarClient

use of org.apache.pulsar.client.api.PulsarClient in project incubator-pulsar by apache.

the class SchedulerManagerTest method setup.

@BeforeMethod
public void setup() throws PulsarClientException {
    WorkerConfig workerConfig = new WorkerConfig();
    workerConfig.setWorkerId("worker-1");
    workerConfig.setThreadContainerFactory(new WorkerConfig.ThreadContainerFactory().setThreadGroupName("test"));
    workerConfig.setPulsarServiceUrl("pulsar://localhost:6650");
    workerConfig.setStateStorageServiceUrl("foo");
    workerConfig.setFunctionAssignmentTopicName("assignments");
    workerConfig.setSchedulerClassName(RoundRobinScheduler.class.getName());
    workerConfig.setAssignmentWriteMaxRetries(0);
    producer = mock(Producer.class);
    completableFuture = spy(new CompletableFuture<>());
    completableFuture.complete(MessageId.earliest);
    byte[] bytes = any();
    when(producer.sendAsync(bytes)).thenReturn(completableFuture);
    PulsarClient pulsarClient = mock(PulsarClient.class);
    doReturn(producer).when(pulsarClient).createProducer(any(), any());
    schedulerManager = spy(new SchedulerManager(workerConfig, pulsarClient));
    functionRuntimeManager = mock(FunctionRuntimeManager.class);
    functionMetaDataManager = mock(FunctionMetaDataManager.class);
    membershipManager = mock(MembershipManager.class);
    schedulerManager.setFunctionMetaDataManager(functionMetaDataManager);
    schedulerManager.setFunctionRuntimeManager(functionRuntimeManager);
    schedulerManager.setMembershipManager(membershipManager);
}