Examples with ObjectWriter org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectWriter used on opensource projects

Example 6 with ObjectWriter

use of org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectWriter in project pulsar by yahoo.

the class PerformanceProducer method main.

public static void main(String[] args) throws Exception {
    final Arguments arguments = new Arguments();
    JCommander jc = new JCommander(arguments);
    jc.setProgramName("pulsar-perf-producer");
    try {
        jc.parse(args);
    } catch (ParameterException e) {
        System.out.println(e.getMessage());
        jc.usage();
        System.exit(-1);
    }
    if (arguments.help) {
        jc.usage();
        System.exit(-1);
    }
    if (arguments.destinations.size() != 1) {
        System.out.println("Only one topic name is allowed");
        jc.usage();
        System.exit(-1);
    }
    if (arguments.confFile != null) {
        Properties prop = new Properties(System.getProperties());
        prop.load(new FileInputStream(arguments.confFile));
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("brokerServiceUrl");
        }
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("webServiceUrl");
        }
        // fallback to previous-version serviceUrl property to maintain backward-compatibility
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("serviceUrl", "http://localhost:8080/");
        }
        if (arguments.authPluginClassName == null) {
            arguments.authPluginClassName = prop.getProperty("authPlugin", null);
        }
        if (arguments.authParams == null) {
            arguments.authParams = prop.getProperty("authParams", null);
        }
    }
    arguments.testTime = TimeUnit.SECONDS.toMillis(arguments.testTime);
    // Dump config variables
    ObjectMapper m = new ObjectMapper();
    ObjectWriter w = m.writerWithDefaultPrettyPrinter();
    log.info("Starting Pulsar perf producer with config: {}", w.writeValueAsString(arguments));
    // Read payload data from file if needed
    byte[] payloadData;
    if (arguments.payloadFilename != null) {
        payloadData = Files.readAllBytes(Paths.get(arguments.payloadFilename));
    } else {
        payloadData = new byte[arguments.msgSize];
    }
    // Now processing command line arguments
    String prefixTopicName = arguments.destinations.get(0);
    List<Future<Producer>> futures = Lists.newArrayList();
    EventLoopGroup eventLoopGroup;
    if (SystemUtils.IS_OS_LINUX) {
        eventLoopGroup = new EpollEventLoopGroup(Runtime.getRuntime().availableProcessors(), new DefaultThreadFactory("pulsar-perf-producer"));
    } else {
        eventLoopGroup = new NioEventLoopGroup(Runtime.getRuntime().availableProcessors(), new DefaultThreadFactory("pulsar-perf-producer"));
    }
    ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setConnectionsPerBroker(arguments.maxConnections);
    clientConf.setStatsInterval(arguments.statsIntervalSeconds, TimeUnit.SECONDS);
    if (isNotBlank(arguments.authPluginClassName)) {
        clientConf.setAuthentication(arguments.authPluginClassName, arguments.authParams);
    }
    PulsarClient client = new PulsarClientImpl(arguments.serviceURL, clientConf, eventLoopGroup);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setSendTimeout(0, TimeUnit.SECONDS);
    producerConf.setCompressionType(arguments.compression);
    // enable round robin message routing if it is a partitioned topic
    producerConf.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
    if (arguments.batchTime > 0) {
        producerConf.setBatchingMaxPublishDelay(arguments.batchTime, TimeUnit.MILLISECONDS);
        producerConf.setBatchingEnabled(true);
        producerConf.setMaxPendingMessages(arguments.msgRate);
    }
    for (int i = 0; i < arguments.numTopics; i++) {
        String topic = (arguments.numTopics == 1) ? prefixTopicName : String.format("%s-%d", prefixTopicName, i);
        log.info("Adding {} publishers on destination {}", arguments.numProducers, topic);
        for (int j = 0; j < arguments.numProducers; j++) {
            futures.add(client.createProducerAsync(topic, producerConf));
        }
    }
    final List<Producer> producers = Lists.newArrayListWithCapacity(futures.size());
    for (Future<Producer> future : futures) {
        producers.add(future.get());
    }
    log.info("Created {} producers", producers.size());
    Runtime.getRuntime().addShutdownHook(new Thread() {

        public void run() {
            printAggregatedStats();
        }
    });
    Collections.shuffle(producers);
    AtomicBoolean isDone = new AtomicBoolean();
    executor.submit(() -> {
        try {
            RateLimiter rateLimiter = RateLimiter.create(arguments.msgRate);
            long startTime = System.currentTimeMillis();
            // Send messages on all topics/producers
            long totalSent = 0;
            while (true) {
                for (Producer producer : producers) {
                    if (arguments.testTime > 0) {
                        if (System.currentTimeMillis() - startTime > arguments.testTime) {
                            log.info("------------------- DONE -----------------------");
                            printAggregatedStats();
                            isDone.set(true);
                            Thread.sleep(5000);
                            System.exit(0);
                        }
                    }
                    if (arguments.numMessages > 0) {
                        if (totalSent++ >= arguments.numMessages) {
                            log.info("------------------- DONE -----------------------");
                            printAggregatedStats();
                            isDone.set(true);
                            Thread.sleep(5000);
                            System.exit(0);
                        }
                    }
                    rateLimiter.acquire();
                    final long sendTime = System.nanoTime();
                    producer.sendAsync(payloadData).thenRun(() -> {
                        messagesSent.increment();
                        bytesSent.add(payloadData.length);
                        long latencyMicros = NANOSECONDS.toMicros(System.nanoTime() - sendTime);
                        recorder.recordValue(latencyMicros);
                        cumulativeRecorder.recordValue(latencyMicros);
                    }).exceptionally(ex -> {
                        log.warn("Write error on message", ex);
                        System.exit(-1);
                        return null;
                    });
                }
            }
        } catch (Throwable t) {
            log.error("Got error", t);
        }
    });
    // Print report stats
    long oldTime = System.nanoTime();
    Histogram reportHistogram = null;
    String statsFileName = "perf-producer-" + System.currentTimeMillis() + ".hgrm";
    log.info("Dumping latency stats to {}", statsFileName);
    PrintStream histogramLog = new PrintStream(new FileOutputStream(statsFileName), false);
    HistogramLogWriter histogramLogWriter = new HistogramLogWriter(histogramLog);
    // Some log header bits
    histogramLogWriter.outputLogFormatVersion();
    histogramLogWriter.outputLegend();
    while (true) {
        try {
            Thread.sleep(10000);
        } catch (InterruptedException e) {
            break;
        }
        if (isDone.get()) {
            break;
        }
        long now = System.nanoTime();
        double elapsed = (now - oldTime) / 1e9;
        double rate = messagesSent.sumThenReset() / elapsed;
        double throughput = bytesSent.sumThenReset() / elapsed / 1024 / 1024 * 8;
        reportHistogram = recorder.getIntervalHistogram(reportHistogram);
        log.info("Throughput produced: {}  msg/s --- {} Mbit/s --- Latency: mean: {} ms - med: {} - 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: {}", throughputFormat.format(rate), throughputFormat.format(throughput), dec.format(reportHistogram.getMean() / 1000.0), dec.format(reportHistogram.getValueAtPercentile(50) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(95) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.9) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.99) / 1000.0), dec.format(reportHistogram.getMaxValue() / 1000.0));
        histogramLogWriter.outputIntervalHistogram(reportHistogram);
        reportHistogram.reset();
        oldTime = now;
    }
    client.close();
}

Example 7 with ObjectWriter

use of org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectWriter in project spring-framework by spring-projects.

the class AbstractJackson2HttpMessageConverter method writeInternal.

@Override
protected void writeInternal(Object object, Type type, HttpOutputMessage outputMessage) throws IOException, HttpMessageNotWritableException {
    MediaType contentType = outputMessage.getHeaders().getContentType();
    JsonEncoding encoding = getJsonEncoding(contentType);
    JsonGenerator generator = this.objectMapper.getFactory().createGenerator(outputMessage.getBody(), encoding);
    try {
        writePrefix(generator, object);
        Class<?> serializationView = null;
        FilterProvider filters = null;
        Object value = object;
        JavaType javaType = null;
        if (object instanceof MappingJacksonValue) {
            MappingJacksonValue container = (MappingJacksonValue) object;
            value = container.getValue();
            serializationView = container.getSerializationView();
            filters = container.getFilters();
        }
        if (type != null && value != null && TypeUtils.isAssignable(type, value.getClass())) {
            javaType = getJavaType(type, null);
        }
        ObjectWriter objectWriter;
        if (serializationView != null) {
            objectWriter = this.objectMapper.writerWithView(serializationView);
        } else if (filters != null) {
            objectWriter = this.objectMapper.writer(filters);
        } else {
            objectWriter = this.objectMapper.writer();
        }
        if (javaType != null && javaType.isContainerType()) {
            objectWriter = objectWriter.forType(javaType);
        }
        SerializationConfig config = objectWriter.getConfig();
        if (contentType != null && contentType.isCompatibleWith(MediaType.TEXT_EVENT_STREAM) && config.isEnabled(SerializationFeature.INDENT_OUTPUT)) {
            objectWriter = objectWriter.with(this.ssePrettyPrinter);
        }
        objectWriter.writeValue(generator, value);
        writeSuffix(generator, object);
        generator.flush();
    } catch (JsonProcessingException ex) {
        throw new HttpMessageNotWritableException("Could not write content: " + ex.getMessage(), ex);
    }
}

Example 8 with ObjectWriter

use of org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectWriter in project torodb by torodb.

the class ConfigUtils method printYamlConfig.

public static <T> void printYamlConfig(T config, Console console) throws IOException, JsonGenerationException, JsonMappingException {
    ObjectMapper objectMapper = yamlMapper();
    ObjectWriter objectWriter = objectMapper.writer();
    printConfig(config, console, objectWriter);
}

Example 9 with ObjectWriter

use of org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectWriter in project ratpack by ratpack.

the class JsonRenderer method render.

@Override
public void render(Context ctx, JsonRender object) throws Exception {
    ObjectWriter writer = object.getObjectWriter();
    if (writer == null) {
        writer = ctx.maybeGet(ObjectWriter.class).orElseGet(() -> ctx.get(ObjectMapper.class).writer());
    }
    Class<?> viewClass = object.getViewClass();
    if (viewClass != null) {
        writer = writer.withView(viewClass);
    }
    ByteBuf buffer = ctx.get(ByteBufAllocator.class).buffer();
    OutputStream outputStream = new ByteBufOutputStream(buffer);
    try {
        writer.writeValue(outputStream, object.getObject());
    } catch (JsonProcessingException e) {
        buffer.release();
        ctx.error(e);
        return;
    }
    ctx.getResponse().contentTypeIfNotSet(HttpHeaderConstants.JSON).send(buffer);
}

Example 10 with ObjectWriter

use of org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectWriter in project pulsar by yahoo.

the class ConsumerStats method init.

private void init(ConsumerConfiguration conf) {
    ObjectMapper m = new ObjectMapper();
    m.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
    ObjectWriter w = m.writerWithDefaultPrettyPrinter();
    try {
        log.info("Starting Pulsar consumer perf with config: {}", w.writeValueAsString(conf));
        log.info("Pulsar client config: {}", w.writeValueAsString(pulsarClient.getConfiguration()));
    } catch (IOException e) {
        log.error("Failed to dump config info: {}", e);
    }
    stat = (timeout) -> {
        if (timeout.isCancelled()) {
            return;
        }
        try {
            long now = System.nanoTime();
            double elapsed = (now - oldTime) / 1e9;
            oldTime = now;
            long currentNumMsgsReceived = numMsgsReceived.sumThenReset();
            long currentNumBytesReceived = numBytesReceived.sumThenReset();
            long currentNumReceiveFailed = numReceiveFailed.sumThenReset();
            long currentNumAcksSent = numAcksSent.sumThenReset();
            long currentNumAcksFailed = numAcksFailed.sumThenReset();
            totalMsgsReceived.add(currentNumMsgsReceived);
            totalBytesReceived.add(currentNumBytesReceived);
            totalReceiveFailed.add(currentNumReceiveFailed);
            totalAcksSent.add(currentNumAcksSent);
            totalAcksFailed.add(currentNumAcksFailed);
            if ((currentNumMsgsReceived | currentNumBytesReceived | currentNumReceiveFailed | currentNumAcksSent | currentNumAcksFailed) != 0) {
                log.info("[{}] [{}] [{}] Prefetched messages: {} --- Consume throughput: {} msgs/s --- " + "Throughput received: {} msg/s --- {} Mbit/s --- " + "Ack sent rate: {} ack/s --- " + "Failed messages: {} --- " + "Failed acks: {}", consumer.getTopic(), consumer.getSubscription(), consumer.consumerName, consumer.incomingMessages.size(), throughputFormat.format(currentNumMsgsReceived / elapsed), throughputFormat.format(currentNumBytesReceived / elapsed * 8 / 1024 / 1024), throughputFormat.format(currentNumAcksSent / elapsed), currentNumReceiveFailed, currentNumAcksFailed);
            }
        } catch (Exception e) {
            log.error("[{}] [{}] [{}]: {}", consumer.getTopic(), consumer.subscription, consumer.consumerName, e.getMessage());
        } finally {
            // schedule the next stat info
            statTimeout = pulsarClient.timer().newTimeout(stat, statsIntervalSeconds, TimeUnit.SECONDS);
        }
    };
    oldTime = System.nanoTime();
    statTimeout = pulsarClient.timer().newTimeout(stat, statsIntervalSeconds, TimeUnit.SECONDS);
}