Examples with Producer org.apache.pulsar.client.api.Producer used on opensource projects

Example 6 with Producer

use of org.apache.pulsar.client.api.Producer in project incubator-pulsar by apache.

the class SchedulerManagerTest method testScalingDown.

@Test
public void testScalingDown() throws Exception {
    List<Function.FunctionMetaData> functionMetaDataList = new LinkedList<>();
    long version = 5;
    Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1").setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version).build();
    Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-2").setNamespace("namespace-1").setTenant("tenant-1").setParallelism(3)).setVersion(version).build();
    functionMetaDataList.add(function1);
    functionMetaDataList.add(function2);
    doReturn(functionMetaDataList).when(functionMetaDataManager).getAllFunctionMetaData();
    // set assignments
    Function.Assignment assignment1 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0).build()).build();
    Map<String, Map<String, Function.Assignment>> currentAssignments = new HashMap<>();
    Map<String, Function.Assignment> assignmentEntry1 = new HashMap<>();
    assignmentEntry1.put(Utils.getFullyQualifiedInstanceId(assignment1.getInstance()), assignment1);
    currentAssignments.put("worker-1", assignmentEntry1);
    doReturn(currentAssignments).when(functionRuntimeManager).getCurrentAssignments();
    // set version
    doReturn(version).when(functionRuntimeManager).getCurrentAssignmentVersion();
    // single node
    List<MembershipManager.WorkerInfo> workerInfoList = new LinkedList<>();
    workerInfoList.add(MembershipManager.WorkerInfo.of("worker-1", "workerHostname-1", 5000));
    doReturn(workerInfoList).when(membershipManager).getCurrentMembership();
    // i am leader
    doReturn(true).when(membershipManager).isLeader();
    callSchedule();
    List<Invocation> invocations = getMethodInvocationDetails(producer, Producer.class.getMethod("sendAsync", Object.class));
    Assert.assertEquals(invocations.size(), 1);
    byte[] send = (byte[]) invocations.get(0).getRawArguments()[0];
    Request.AssignmentsUpdate assignmentsUpdate = Request.AssignmentsUpdate.parseFrom(send);
    log.info("assignmentsUpdate: {}", assignmentsUpdate);
    Function.Assignment assignment2_1 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2).setInstanceId(0).build()).build();
    Function.Assignment assignment2_2 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2).setInstanceId(1).build()).build();
    Function.Assignment assignment2_3 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2).setInstanceId(2).build()).build();
    Assert.assertEquals(assignmentsUpdate, Request.AssignmentsUpdate.newBuilder().setVersion(version + 1).addAssignments(assignment1).addAssignments(assignment2_1).addAssignments(assignment2_2).addAssignments(assignment2_3).build());
    // scale down
    PulsarClient pulsarClient = mock(PulsarClient.class);
    doReturn(producer).when(pulsarClient).createProducer(any(), any());
    Function.FunctionMetaData function2Scaled = Function.FunctionMetaData.newBuilder().setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-2").setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version).build();
    functionMetaDataList = new LinkedList<>();
    functionMetaDataList.add(function1);
    functionMetaDataList.add(function2Scaled);
    doReturn(functionMetaDataList).when(functionMetaDataManager).getAllFunctionMetaData();
    Function.Assignment assignment2Scaled = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2Scaled).setInstanceId(0).build()).build();
    callSchedule();
    invocations = getMethodInvocationDetails(producer, Producer.class.getMethod("sendAsync", Object.class));
    Assert.assertEquals(invocations.size(), 2);
    send = (byte[]) invocations.get(1).getRawArguments()[0];
    assignmentsUpdate = Request.AssignmentsUpdate.parseFrom(send);
    Assert.assertEquals(assignmentsUpdate, Request.AssignmentsUpdate.newBuilder().setVersion(version + 1 + 1).addAssignments(assignment1).addAssignments(assignment2Scaled).build());
}

Example 7 with Producer

use of org.apache.pulsar.client.api.Producer in project incubator-pulsar by apache.

the class SchedulerManagerTest method testScalingUp.

@Test
public void testScalingUp() throws Exception {
    List<Function.FunctionMetaData> functionMetaDataList = new LinkedList<>();
    long version = 5;
    Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1").setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version).build();
    Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-2").setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version).build();
    functionMetaDataList.add(function1);
    functionMetaDataList.add(function2);
    doReturn(functionMetaDataList).when(functionMetaDataManager).getAllFunctionMetaData();
    // set assignments
    Function.Assignment assignment1 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0).build()).build();
    Map<String, Map<String, Function.Assignment>> currentAssignments = new HashMap<>();
    Map<String, Function.Assignment> assignmentEntry1 = new HashMap<>();
    assignmentEntry1.put(Utils.getFullyQualifiedInstanceId(assignment1.getInstance()), assignment1);
    currentAssignments.put("worker-1", assignmentEntry1);
    doReturn(currentAssignments).when(functionRuntimeManager).getCurrentAssignments();
    // set version
    doReturn(version).when(functionRuntimeManager).getCurrentAssignmentVersion();
    // single node
    List<MembershipManager.WorkerInfo> workerInfoList = new LinkedList<>();
    workerInfoList.add(MembershipManager.WorkerInfo.of("worker-1", "workerHostname-1", 5000));
    doReturn(workerInfoList).when(membershipManager).getCurrentMembership();
    // i am leader
    doReturn(true).when(membershipManager).isLeader();
    callSchedule();
    List<Invocation> invocations = getMethodInvocationDetails(producer, Producer.class.getMethod("sendAsync", Object.class));
    Assert.assertEquals(invocations.size(), 1);
    byte[] send = (byte[]) invocations.get(0).getRawArguments()[0];
    Request.AssignmentsUpdate assignmentsUpdate = Request.AssignmentsUpdate.parseFrom(send);
    log.info("assignmentsUpdate: {}", assignmentsUpdate);
    Function.Assignment assignment2 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2).setInstanceId(0).build()).build();
    Assert.assertEquals(assignmentsUpdate, Request.AssignmentsUpdate.newBuilder().setVersion(version + 1).addAssignments(assignment1).addAssignments(assignment2).build());
    // scale up
    PulsarClient pulsarClient = mock(PulsarClient.class);
    doReturn(producer).when(pulsarClient).createProducer(any(), any());
    Function.FunctionMetaData function2Scaled = Function.FunctionMetaData.newBuilder().setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-2").setNamespace("namespace-1").setTenant("tenant-1").setParallelism(3)).setVersion(version).build();
    functionMetaDataList = new LinkedList<>();
    functionMetaDataList.add(function1);
    functionMetaDataList.add(function2Scaled);
    doReturn(functionMetaDataList).when(functionMetaDataManager).getAllFunctionMetaData();
    Function.Assignment assignment2Scaled1 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2Scaled).setInstanceId(0).build()).build();
    Function.Assignment assignment2Scaled2 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2Scaled).setInstanceId(1).build()).build();
    Function.Assignment assignment2Scaled3 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2Scaled).setInstanceId(2).build()).build();
    callSchedule();
    invocations = getMethodInvocationDetails(producer, Producer.class.getMethod("sendAsync", Object.class));
    Assert.assertEquals(invocations.size(), 2);
    send = (byte[]) invocations.get(1).getRawArguments()[0];
    assignmentsUpdate = Request.AssignmentsUpdate.parseFrom(send);
    Assert.assertEquals(assignmentsUpdate, Request.AssignmentsUpdate.newBuilder().setVersion(version + 1 + 1).addAssignments(assignment1).addAssignments(assignment2Scaled1).addAssignments(assignment2Scaled2).addAssignments(assignment2Scaled3).build());
}

Example 8 with Producer

use of org.apache.pulsar.client.api.Producer in project incubator-pulsar by apache.

the class SchedulerManagerTest method setup.

@BeforeMethod
public void setup() throws PulsarClientException {
    WorkerConfig workerConfig = new WorkerConfig();
    workerConfig.setWorkerId("worker-1");
    workerConfig.setThreadContainerFactory(new WorkerConfig.ThreadContainerFactory().setThreadGroupName("test"));
    workerConfig.setPulsarServiceUrl("pulsar://localhost:6650");
    workerConfig.setStateStorageServiceUrl("foo");
    workerConfig.setFunctionAssignmentTopicName("assignments");
    workerConfig.setSchedulerClassName(RoundRobinScheduler.class.getName());
    workerConfig.setAssignmentWriteMaxRetries(0);
    producer = mock(Producer.class);
    completableFuture = spy(new CompletableFuture<>());
    completableFuture.complete(MessageId.earliest);
    byte[] bytes = any();
    when(producer.sendAsync(bytes)).thenReturn(completableFuture);
    PulsarClient pulsarClient = mock(PulsarClient.class);
    doReturn(producer).when(pulsarClient).createProducer(any(), any());
    schedulerManager = spy(new SchedulerManager(workerConfig, pulsarClient));
    functionRuntimeManager = mock(FunctionRuntimeManager.class);
    functionMetaDataManager = mock(FunctionMetaDataManager.class);
    membershipManager = mock(MembershipManager.class);
    schedulerManager.setFunctionMetaDataManager(functionMetaDataManager);
    schedulerManager.setFunctionRuntimeManager(functionRuntimeManager);
    schedulerManager.setMembershipManager(membershipManager);
}

Example 9 with Producer

use of org.apache.pulsar.client.api.Producer in project incubator-pulsar by apache.

the class FunctionsImpl method triggerFunction.

@POST
@Path("/{tenant}/{namespace}/{functionName}/trigger")
@Consumes(MediaType.MULTIPART_FORM_DATA)
public Response triggerFunction(@PathParam("tenant") final String tenant, @PathParam("namespace") final String namespace, @PathParam("name") final String functionName, @FormDataParam("data") final String input, @FormDataParam("dataStream") final InputStream uploadedInputStream) {
    FunctionConfig functionConfig;
    // validate parameters
    try {
        validateTriggerRequestParams(tenant, namespace, functionName, input, uploadedInputStream);
    } catch (IllegalArgumentException e) {
        log.error("Invalid trigger function request @ /{}/{}/{}", tenant, namespace, functionName, e);
        return Response.status(Status.BAD_REQUEST).type(MediaType.APPLICATION_JSON).entity(new ErrorData(e.getMessage())).build();
    }
    FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager();
    if (!functionMetaDataManager.containsFunction(tenant, namespace, functionName)) {
        log.error("Function in getFunction does not exist @ /{}/{}/{}", tenant, namespace, functionName);
        return Response.status(Status.NOT_FOUND).type(MediaType.APPLICATION_JSON).entity(new ErrorData(String.format("Function %s doesn't exist", functionName))).build();
    }
    FunctionMetaData functionMetaData = functionMetaDataManager.getFunctionMetaData(tenant, namespace, functionName);
    String inputTopicToWrite;
    if (functionMetaData.getFunctionConfig().getInputsList().size() > 0) {
        inputTopicToWrite = functionMetaData.getFunctionConfig().getInputsList().get(0);
    } else {
        inputTopicToWrite = functionMetaData.getFunctionConfig().getCustomSerdeInputs().entrySet().iterator().next().getKey();
    }
    String outputTopic = functionMetaData.getFunctionConfig().getOutput();
    Reader reader = null;
    Producer producer = null;
    try {
        if (outputTopic != null && !outputTopic.isEmpty()) {
            reader = worker().getClient().newReader().topic(outputTopic).startMessageId(MessageId.latest).create();
        }
        producer = worker().getClient().newProducer().topic(inputTopicToWrite).create();
        byte[] targetArray;
        if (uploadedInputStream != null) {
            targetArray = new byte[uploadedInputStream.available()];
            uploadedInputStream.read(targetArray);
        } else {
            targetArray = input.getBytes();
        }
        MessageId msgId = producer.send(targetArray);
        if (reader == null) {
            return Response.status(Status.OK).build();
        }
        long curTime = System.currentTimeMillis();
        long maxTime = curTime + 1000;
        while (curTime < maxTime) {
            Message msg = reader.readNext(10000, TimeUnit.MILLISECONDS);
            if (msg == null)
                break;
            if (msg.getProperties().containsKey("__pfn_input_msg_id__") && msg.getProperties().containsKey("__pfn_input_topic__")) {
                MessageId newMsgId = MessageId.fromByteArray(Base64.getDecoder().decode((String) msg.getProperties().get("__pfn_input_msg_id__")));
                if (msgId.equals(newMsgId) && msg.getProperties().get("__pfn_input_topic__").equals(inputTopicToWrite)) {
                    return Response.status(Status.OK).entity(msg.getData()).build();
                }
            }
            curTime = System.currentTimeMillis();
        }
        return Response.status(Status.REQUEST_TIMEOUT).build();
    } catch (Exception e) {
        return Response.status(Status.INTERNAL_SERVER_ERROR).build();
    } finally {
        if (reader != null) {
            reader.closeAsync();
        }
        if (producer != null) {
            producer.closeAsync();
        }
    }
}

Example 10 with Producer

use of org.apache.pulsar.client.api.Producer in project incubator-pulsar by apache.

the class ProducerImpl method connectionOpened.

@Override
public void connectionOpened(final ClientCnx cnx) {
    // we set the cnx reference before registering the producer on the cnx, so if the cnx breaks before creating the
    // producer, it will try to grab a new cnx
    connectionHandler.setClientCnx(cnx);
    cnx.registerProducer(producerId, this);
    log.info("[{}] [{}] Creating producer on cnx {}", topic, producerName, cnx.ctx().channel());
    long requestId = client.newRequestId();
    cnx.sendRequestWithId(Commands.newProducer(topic, producerId, requestId, producerName, conf.isEncryptionEnabled(), metadata), requestId).thenAccept(pair -> {
        String producerName = pair.getLeft();
        long lastSequenceId = pair.getRight();
        // set the cnx pointer so that new messages will be sent immediately
        synchronized (ProducerImpl.this) {
            if (getState() == State.Closing || getState() == State.Closed) {
                // Producer was closed while reconnecting, close the connection to make sure the broker
                // drops the producer on its side
                cnx.removeProducer(producerId);
                cnx.channel().close();
                return;
            }
            resetBackoff();
            log.info("[{}] [{}] Created producer on cnx {}", topic, producerName, cnx.ctx().channel());
            connectionId = cnx.ctx().channel().toString();
            connectedSince = DateFormatter.now();
            if (this.producerName == null) {
                this.producerName = producerName;
            }
            if (this.lastSequenceIdPublished == -1 && conf.getInitialSequenceId() == null) {
                this.lastSequenceIdPublished = lastSequenceId;
                this.msgIdGenerator = lastSequenceId + 1;
            }
            if (!producerCreatedFuture.isDone() && isBatchMessagingEnabled()) {
                // schedule the first batch message task
                client.timer().newTimeout(batchMessageAndSendTask, conf.getBatchingMaxPublishDelayMicros(), TimeUnit.MICROSECONDS);
            }
            resendMessages(cnx);
        }
    }).exceptionally((e) -> {
        Throwable cause = e.getCause();
        cnx.removeProducer(producerId);
        if (getState() == State.Closing || getState() == State.Closed) {
            // Producer was closed while reconnecting, close the connection to make sure the broker
            // drops the producer on its side
            cnx.channel().close();
            return null;
        }
        log.error("[{}] [{}] Failed to create producer: {}", topic, producerName, cause.getMessage());
        if (cause instanceof PulsarClientException.ProducerBlockedQuotaExceededException) {
            synchronized (this) {
                log.warn("[{}] [{}] Topic backlog quota exceeded. Throwing Exception on producer.", topic, producerName);
                if (log.isDebugEnabled()) {
                    log.debug("[{}] [{}] Pending messages: {}", topic, producerName, pendingMessages.size());
                }
                PulsarClientException bqe = new PulsarClientException.ProducerBlockedQuotaExceededException("Could not send pending messages as backlog exceeded");
                failPendingMessages(cnx(), bqe);
            }
        } else if (cause instanceof PulsarClientException.ProducerBlockedQuotaExceededError) {
            log.warn("[{}] [{}] Producer is blocked on creation because backlog exceeded on topic.", producerName, topic);
        }
        if (cause instanceof PulsarClientException.TopicTerminatedException) {
            setState(State.Terminated);
            failPendingMessages(cnx(), (PulsarClientException) cause);
            producerCreatedFuture.completeExceptionally(cause);
            client.cleanupProducer(this);
        } else if (// 
        producerCreatedFuture.isDone() || (cause instanceof PulsarClientException && connectionHandler.isRetriableError((PulsarClientException) cause) && System.currentTimeMillis() < createProducerTimeout)) {
            // Either we had already created the producer once (producerCreatedFuture.isDone()) or we are
            // still within the initial timeout budget and we are dealing with a retriable error
            reconnectLater(cause);
        } else {
            setState(State.Failed);
            producerCreatedFuture.completeExceptionally(cause);
            client.cleanupProducer(this);
        }
        return null;
    });
}