Examples with TransformedRecord com.twitter.distributedlog.thrift.messaging.TransformedRecord used on opensource projects

Example 1 with TransformedRecord

use of com.twitter.distributedlog.thrift.messaging.TransformedRecord in project distributedlog by twitter.

the class StreamTransformer method main.

public static void main(String[] args) throws Exception {
    if (3 != args.length) {
        System.out.println(HELP);
        return;
    }
    String dlUriStr = args[0];
    final String srcStreamName = args[1];
    final String targetStreamName = args[2];
    URI uri = URI.create(dlUriStr);
    DistributedLogConfiguration conf = new DistributedLogConfiguration();
    // 16KB
    conf.setOutputBufferSize(16 * 1024);
    // 5ms
    conf.setPeriodicFlushFrequencyMilliSeconds(5);
    DistributedLogNamespace namespace = DistributedLogNamespaceBuilder.newBuilder().conf(conf).uri(uri).build();
    // open the dlm
    System.out.println("Opening log stream " + srcStreamName);
    DistributedLogManager srcDlm = namespace.openLog(srcStreamName);
    System.out.println("Opening log stream " + targetStreamName);
    DistributedLogManager targetDlm = namespace.openLog(targetStreamName);
    Transformer<byte[], byte[]> replicationTransformer = new IdenticalTransformer<byte[]>();
    LogRecordWithDLSN lastTargetRecord;
    DLSN srcDlsn;
    try {
        lastTargetRecord = targetDlm.getLastLogRecord();
        TransformedRecord lastTransformedRecord = new TransformedRecord();
        try {
            lastTransformedRecord.read(protocolFactory.getProtocol(new TIOStreamTransport(new ByteArrayInputStream(lastTargetRecord.getPayload()))));
            srcDlsn = DLSN.deserializeBytes(lastTransformedRecord.getSrcDlsn());
            System.out.println("Last transformed record is " + srcDlsn);
        } catch (TException e) {
            System.err.println("Error on reading last transformed record");
            e.printStackTrace(System.err);
            srcDlsn = DLSN.InitialDLSN;
        }
    } catch (LogNotFoundException lnfe) {
        srcDlsn = DLSN.InitialDLSN;
    } catch (LogEmptyException lee) {
        srcDlsn = DLSN.InitialDLSN;
    }
    AsyncLogWriter targetWriter = FutureUtils.result(targetDlm.openAsyncLogWriter());
    try {
        readLoop(srcDlm, srcDlsn, targetWriter, replicationTransformer);
    } finally {
        FutureUtils.result(targetWriter.asyncClose(), Duration.apply(5, TimeUnit.SECONDS));
        targetDlm.close();
        srcDlm.close();
        namespace.close();
    }
}

Example 2 with TransformedRecord

use of com.twitter.distributedlog.thrift.messaging.TransformedRecord in project distributedlog by twitter.

the class StreamTransformer method transform.

private static void transform(final AsyncLogWriter writer, LogRecordWithDLSN record, Transformer<byte[], byte[]> replicationTransformer, final CountDownLatch keepAliveLatch) throws Exception {
    DLSN srcDLSN = record.getDlsn();
    byte[] payload = record.getPayload();
    byte[] transformedPayload = replicationTransformer.transform(payload);
    TransformedRecord transformedRecord = new TransformedRecord(ByteBuffer.wrap(transformedPayload));
    transformedRecord.setSrcDlsn(srcDLSN.serializeBytes());
    ByteArrayOutputStream baos = new ByteArrayOutputStream(4096);
    transformedRecord.write(protocolFactory.getProtocol(new TIOStreamTransport(baos)));
    byte[] data = baos.toByteArray();
    writer.write(new LogRecord(record.getSequenceId(), data)).addEventListener(new FutureEventListener<DLSN>() {

        @Override
        public void onFailure(Throwable cause) {
            System.err.println("Encountered error on writing records to stream " + writer.getStreamName());
            cause.printStackTrace(System.err);
            keepAliveLatch.countDown();
        }

        @Override
        public void onSuccess(DLSN dlsn) {
            System.out.println("Write transformed record " + dlsn);
        }
    });
}