Examples with ChunkBuilder org.apache.hadoop.chukwa.ChunkBuilder used on opensource projects

Example 1 with ChunkBuilder

use of org.apache.hadoop.chukwa.ChunkBuilder in project Honu by jboulon.

the class CmdLineConverter method main.

/**
   * @param args
   * @throws ClassNotFoundException 
   */
@SuppressWarnings("unchecked")
public static void main(String[] args) throws ClassNotFoundException {
    if (args.length != 3) {
        System.out.println("java org.honu.inputtools.converter.CmdLineConverter <dataType> <codec> <outputFile>");
        System.out.println("codec: NONE , for uncompressed seqFile");
        System.out.println("codec: org.apache.hadoop.io.compress.GzipCodec , for GZIP compressed seqFile");
        System.out.println("codec: org.apache.hadoop.io.compress.LzoCodec , for LZO compressed seqFile");
        System.exit(-1);
    }
    String dataType = args[0];
    String codecClass = args[1];
    String outpFileName = args[2];
    if (codecClass.equalsIgnoreCase("none")) {
        codecClass = null;
    }
    int lineCount = 0;
    Path newOutputPath = null;
    try {
        BufferedReader in = new BufferedReader(new InputStreamReader(System.in));
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.getLocal(conf);
        newOutputPath = new Path(outpFileName);
        CompressionCodec codec = null;
        if (codecClass != null) {
            Class classDefinition = Class.forName(codecClass);
            codec = (CompressionCodec) ReflectionUtils.newInstance(classDefinition, conf);
        }
        FSDataOutputStream newOutputStr = fs.create(newOutputPath);
        SequenceFile.Writer seqFileWriter = null;
        if (codec != null) {
            seqFileWriter = SequenceFile.createWriter(conf, newOutputStr, ChukwaArchiveKey.class, ChunkImpl.class, SequenceFile.CompressionType.BLOCK, codec);
        } else {
            seqFileWriter = SequenceFile.createWriter(conf, newOutputStr, ChukwaArchiveKey.class, ChunkImpl.class, SequenceFile.CompressionType.NONE, codec);
        }
        String str = null;
        ChunkBuilder cb = null;
        do {
            str = in.readLine();
            if (str != null) {
                lineCount++;
                if (cb == null) {
                    cb = new ChunkBuilder();
                }
                cb.addRecord(str.getBytes());
                if (lineCount % 300 == 0) {
                    append(seqFileWriter, getChunk(cb, dataType));
                    cb = null;
                }
            }
        } while (str != null);
        if (cb != null) {
            append(seqFileWriter, getChunk(cb, dataType));
        }
        seqFileWriter.close();
        newOutputStr.close();
    } catch (Throwable e) {
        e.printStackTrace();
        System.exit(-1);
    }
    System.out.println(new java.util.Date() + ", CmdLineConverter [" + dataType + "] [" + newOutputPath + "], Total lineCount: " + lineCount);
    System.exit(0);
}

Example 2 with ChunkBuilder

use of org.apache.hadoop.chukwa.ChunkBuilder in project Honu by jboulon.

the class ThriftCollectorLockFreeImpl method process.

public Result process(TChunk tChunk) throws TException {
    // Stop adding chunks if it's no running
    if (!isRunning) {
        Log.warn("Rejecting some incoming trafic!");
        Result result = new Result();
        result.setMessage("Shutting down");
        result.setResultCode(ResultCode.TRY_LATER);
        return result;
    }
    // If there's no log Events then return OK
    if (tChunk.getLogEventsSize() == 0) {
        Result result = new Result();
        result.setMessage("" + tChunk.getSeqId());
        result.setResultCode(ResultCode.OK);
        return result;
    }
    Tracer t = Tracer.startNewTracer("honu.server.processChunk");
    //this.counters.get(chunkCountField).incrementAndGet();
    ChunkBuilder cb = new ChunkBuilder();
    List<String> logEvents = tChunk.getLogEvents();
    for (String logEvent : logEvents) {
        cb.addRecord(logEvent.getBytes());
    }
    Chunk c = cb.getChunk();
    c.setApplication(tChunk.getApplication());
    c.setDataType(tChunk.getDataType());
    c.setSeqID(tChunk.getSeqId());
    c.setSource(tChunk.getSource());
    c.setTags(tChunk.getTags());
    if (isDebug) {
        System.out.println("\n\t ===============");
        System.out.println("tChunk.getApplication() :" + tChunk.getApplication());
        System.out.println("tChunk.getDataType() :" + tChunk.getDataType());
        System.out.println("tChunk.getSeqId() :" + tChunk.getSeqId());
        System.out.println("tChunk.getSource() :" + tChunk.getSource());
        System.out.println("tChunk.getStreamName() :" + tChunk.getStreamName());
        System.out.println("tChunk.getTags() :" + tChunk.getTags());
        System.out.println("c.getApplication() :" + c.getApplication());
        System.out.println("c.getDataType() :" + c.getDataType());
        System.out.println("c.getSeqID() :" + c.getSeqID());
        System.out.println("c.getSource() :" + c.getSource());
        System.out.println("c.getTags() :" + c.getTags());
        System.out.println("c.getData()" + new String(c.getData()));
    }
    boolean addResult = false;
    try {
        addResult = chunkQueue.offer(c, 2000, TimeUnit.MILLISECONDS);
    } catch (OutOfMemoryError ex) {
        ex.printStackTrace();
        DaemonWatcher.bailout(-1);
    } catch (Throwable e) {
        e.printStackTrace();
        addResult = false;
    }
    Result result = new Result();
    if (addResult) {
        try {
            Counter.increment("honu.server.chunkCount");
            Counter.increment("honu.server.logCount", logEvents.size());
            Counter.increment("honu.server." + tChunk.getApplication() + ".chunkCount");
            Counter.increment("honu.server." + tChunk.getApplication() + ".logCount", logEvents.size());
            (new Tracer("honu.server.chunkSize [messages, not msec]", logEvents.size())).logTracer();
            (new Tracer("honu.server." + tChunk.getApplication() + ".chunkSize [messages, not msec]", logEvents.size())).logTracer();
        } catch (Exception ignored) {
        }
        result.setMessage("" + tChunk.getSeqId());
        result.setResultCode(ResultCode.OK);
    } else {
        try {
            Counter.increment("honu.server.tryLater");
            Counter.increment("honu.server." + tChunk.getApplication() + ".tryLater");
        } catch (Exception ignored) {
        }
        result.setMessage("" + tChunk.getSeqId());
        result.setResultCode(ResultCode.TRY_LATER);
    }
    if (t != null) {
        t.stopAndLogTracer();
    }
    return result;
}