Examples with TChunk org.honu.thrift.TChunk used on opensource projects

Example 1 with TChunk

use of org.honu.thrift.TChunk in project Honu by jboulon.

the class ThriftCollectorLockFreeImpl method process.

public Result process(TChunk tChunk) throws TException {
    // Stop adding chunks if it's no running
    if (!isRunning) {
        Log.warn("Rejecting some incoming trafic!");
        Result result = new Result();
        result.setMessage("Shutting down");
        result.setResultCode(ResultCode.TRY_LATER);
        return result;
    }
    // If there's no log Events then return OK
    if (tChunk.getLogEventsSize() == 0) {
        Result result = new Result();
        result.setMessage("" + tChunk.getSeqId());
        result.setResultCode(ResultCode.OK);
        return result;
    }
    Tracer t = Tracer.startNewTracer("honu.server.processChunk");
    //this.counters.get(chunkCountField).incrementAndGet();
    ChunkBuilder cb = new ChunkBuilder();
    List<String> logEvents = tChunk.getLogEvents();
    for (String logEvent : logEvents) {
        cb.addRecord(logEvent.getBytes());
    }
    Chunk c = cb.getChunk();
    c.setApplication(tChunk.getApplication());
    c.setDataType(tChunk.getDataType());
    c.setSeqID(tChunk.getSeqId());
    c.setSource(tChunk.getSource());
    c.setTags(tChunk.getTags());
    if (isDebug) {
        System.out.println("\n\t ===============");
        System.out.println("tChunk.getApplication() :" + tChunk.getApplication());
        System.out.println("tChunk.getDataType() :" + tChunk.getDataType());
        System.out.println("tChunk.getSeqId() :" + tChunk.getSeqId());
        System.out.println("tChunk.getSource() :" + tChunk.getSource());
        System.out.println("tChunk.getStreamName() :" + tChunk.getStreamName());
        System.out.println("tChunk.getTags() :" + tChunk.getTags());
        System.out.println("c.getApplication() :" + c.getApplication());
        System.out.println("c.getDataType() :" + c.getDataType());
        System.out.println("c.getSeqID() :" + c.getSeqID());
        System.out.println("c.getSource() :" + c.getSource());
        System.out.println("c.getTags() :" + c.getTags());
        System.out.println("c.getData()" + new String(c.getData()));
    }
    boolean addResult = false;
    try {
        addResult = chunkQueue.offer(c, 2000, TimeUnit.MILLISECONDS);
    } catch (OutOfMemoryError ex) {
        ex.printStackTrace();
        DaemonWatcher.bailout(-1);
    } catch (Throwable e) {
        e.printStackTrace();
        addResult = false;
    }
    Result result = new Result();
    if (addResult) {
        try {
            Counter.increment("honu.server.chunkCount");
            Counter.increment("honu.server.logCount", logEvents.size());
            Counter.increment("honu.server." + tChunk.getApplication() + ".chunkCount");
            Counter.increment("honu.server." + tChunk.getApplication() + ".logCount", logEvents.size());
            (new Tracer("honu.server.chunkSize [messages, not msec]", logEvents.size())).logTracer();
            (new Tracer("honu.server." + tChunk.getApplication() + ".chunkSize [messages, not msec]", logEvents.size())).logTracer();
        } catch (Exception ignored) {
        }
        result.setMessage("" + tChunk.getSeqId());
        result.setResultCode(ResultCode.OK);
    } else {
        try {
            Counter.increment("honu.server.tryLater");
            Counter.increment("honu.server." + tChunk.getApplication() + ".tryLater");
        } catch (Exception ignored) {
        }
        result.setMessage("" + tChunk.getSeqId());
        result.setResultCode(ResultCode.TRY_LATER);
    }
    if (t != null) {
        t.stopAndLogTracer();
    }
    return result;
}

Example 2 with TChunk

use of org.honu.thrift.TChunk in project Honu by jboulon.

the class MessageConsumer method produceChunk.

private void produceChunk() throws InterruptedException {
    TChunk chunk = new TChunk();
    chunk.setSeqId(System.currentTimeMillis());
    chunk.setSource(source);
    chunk.setStreamName(streamname);
    chunk.setApplication(application);
    chunk.setDataType(dataType);
    chunk.setTags(tags);
    List<String> logEvents = new LinkedList<String>();
    chunk.setLogEvents(logEvents);
    int qSize = messages.size();
    if (qSize > maxQueueSize) {
        maxQueueSize = qSize;
    }
    int count = 0;
    // the messages list
    synchronized (wakeUpLink) {
        do {
            logEvents.add(messages.remove(0));
            count++;
        } while (!messages.isEmpty() && count < maxMessageCountPerChunk);
    }
    try {
        (new Tracer("honu.client.messageQueueSize [messages, not msec]", messages.size())).logTracer();
        (new Tracer("honu.client." + chunk.getApplication() + ".messageQueueSize [messages, not msec]", messages.size())).logTracer();
    } catch (Exception ignored) {
    }
    // a backup file
    if (chunkQueue.size() >= maxChunkQueueSize) {
        try {
            Counter.increment("honu.client.lostChunks");
            Counter.increment("honu.client.lostMessages", chunk.getLogEventsSize());
            Counter.increment("honu.client." + chunk.getApplication() + ".lostChunks");
            Counter.increment("honu.client." + chunk.getApplication() + ".lostMessages", chunk.getLogEventsSize());
        } catch (Exception ignored) {
        }
        kv.startMessage("HonuLostStats");
        kv.addKeyValue("lostChunk", 1);
        kv.addKeyValue("lostLines", chunk.getLogEventsSize());
        kv.addKeyValue("RecordType", chunk.getDataType());
        kv.addKeyValue("SeqId", chunk.getSeqId());
        kv.addKeyValue("period", statFrequency);
        log.error(kv.generateMessage());
        MessageManager.getInstance().updateLostDataStats(chunk);
    } else {
        chunkQueue.put(chunk);
    }
}

Example 3 with TChunk

use of org.honu.thrift.TChunk in project Honu by jboulon.

the class MessageSender method run.

public void run() {
    TChunk chunk = null;
    long timeOut = DEFAULT_POLL_TIMEOUT;
    long curr = 0l;
    while (running || (chunkQueue.size() != 0)) {
        try {
            curr = System.currentTimeMillis();
            if (shutDownNow()) {
                logApp.info("[==HONU==] Honu message sender [" + Thread.currentThread().getId() + "] ShutdownNow");
                break;
            }
            if (curr >= nextStatPeriod) {
                kv.startMessage("HonuSenderStats");
                kv.addKeyValue("chunkCount", chunkCount);
                kv.addKeyValue("lineCount", lineCount);
                kv.addKeyValue("exceptionCount", exceptionCount);
                kv.addKeyValue("period", statFrequency);
                log.info(kv.generateMessage());
                // Keep System.out for debug purpose
                if (log.isDebugEnabled()) {
                    System.out.println(Thread.currentThread().getId() + " - " + new java.util.Date() + " - Chunk sent:" + chunkCount + " - lines:" + lineCount + " - in: 1 min+" + (curr - nextStatPeriod) + " ms");
                }
                lineCount = 0;
                chunkCount = 0;
                exceptionCount = 0;
                nextStatPeriod = System.currentTimeMillis() + statFrequency;
            }
            chunk = chunkQueue.poll(timeOut, TimeUnit.MILLISECONDS);
            // increment sleep time up to maxPollTimeOut
            if (chunk == null) {
                if (timeOut < MAX_POLL_TIMEOUT) {
                    timeOut += DEFAULT_POLL_TIMEOUT;
                }
                continue;
            }
            timeOut = DEFAULT_POLL_TIMEOUT;
            Tracer t = Tracer.startNewTracer("honu.client.sendChunk");
            try {
                (new Tracer("honu.client.chunkQueueSize [chunks, not msec]", chunkQueue.size())).logTracer();
                (new Tracer("honu.client." + chunk.getApplication() + ".chunkQueueSize [chunks, not msec]", chunkQueue.size())).logTracer();
            } catch (Exception ignored) {
            }
            if (System.currentTimeMillis() > leaseTs) {
                logApp.info("Time for lease renewal");
                closeConnection();
            }
            sendChunk(chunk);
            if (t != null) {
                t.stopAndLogTracer();
            }
            chunkCount++;
            lineCount += chunk.getLogEventsSize();
            Thread.yield();
        } catch (Throwable e) {
            logApp.warn("Error in main loop", e);
        }
    }
    if (messageManager != null) {
        messageManager.senderShutdownCallback();
    }
    logApp.info("[==HONU==] Honu message sender [" + Thread.currentThread().getId() + "] shutdown completed, messageQueue:" + chunkQueue.size());
}