Examples with Kryo com.esotericsoftware.kryo.Kryo used on opensource projects

Example 86 with Kryo

use of com.esotericsoftware.kryo.Kryo in project ignite by apache.

the class GridMarshallerPerformanceTest method testKryo.

/**
 * @throws Exception If failed.
 */
public void testKryo() throws Exception {
    final Kryo kryo = new Kryo();
    final ByteArrayOutputStream out = new ByteArrayOutputStream();
    IgniteInClosure<TestObject> writer = new CI1<TestObject>() {

        @Override
        public void apply(TestObject obj) {
            out.reset();
            Output kryoOut = null;
            try {
                kryoOut = new Output(out);
                kryo.writeObject(kryoOut, obj);
            } finally {
                U.close(kryoOut, log);
            }
        }
    };
    IgniteOutClosure<TestObject> reader = new CO<TestObject>() {

        @Override
        public TestObject apply() {
            Input kryoIn = null;
            try {
                kryoIn = new Input(new ByteArrayInputStream(out.toByteArray()));
                return kryo.readObject(kryoIn, TestObject.class);
            } finally {
                U.close(kryoIn, log);
            }
        }
    };
    runTest("Kryo", writer, reader);
}

Example 87 with Kryo

use of com.esotericsoftware.kryo.Kryo in project apex-malhar by apache.

the class AbstractFileInputOperatorTest method testPartitioningStateTransferInterrupted.

/**
 * Test for testing dynamic partitioning.
 * - Create 4 file with 3 records each.
 * - Create a single partition, and read some records, populating pending files in operator.
 * - Split it in two operators
 * - Try to emit the remaining records.
 */
@Test
public void testPartitioningStateTransferInterrupted() throws Exception {
    LineByLineFileInputOperator oper = new LineByLineFileInputOperator();
    oper.getScanner().setFilePatternRegexp(".*partition([\\d]*)");
    oper.setDirectory(new File(testMeta.dir).getAbsolutePath());
    oper.setScanIntervalMillis(0);
    oper.setEmitBatchSize(2);
    LineByLineFileInputOperator initialState = new Kryo().copy(oper);
    // Create 4 files with 3 records each.
    Path path = new Path(new File(testMeta.dir).getAbsolutePath());
    FileContext.getLocalFSFileContext().delete(path, true);
    int file;
    for (file = 0; file < 4; file++) {
        FileUtils.write(new File(testMeta.dir, "partition00" + file), "a\nb\nc\n");
    }
    CollectorTestSink<String> queryResults = new CollectorTestSink<String>();
    @SuppressWarnings({ "unchecked", "rawtypes" }) CollectorTestSink<Object> sink = (CollectorTestSink) queryResults;
    oper.output.setSink(sink);
    int wid = 0;
    // Read some records
    oper.setup(testMeta.context);
    for (int i = 0; i < 5; i++) {
        oper.beginWindow(wid);
        oper.emitTuples();
        oper.endWindow();
        wid++;
    }
    Assert.assertEquals("Partial tuples read ", 6, sink.collectedTuples.size());
    Assert.assertEquals(1, initialState.getCurrentPartitions());
    initialState.setPartitionCount(2);
    StatsListener.Response rsp = initialState.processStats(null);
    Assert.assertEquals(true, rsp.repartitionRequired);
    // Create partitions of the operator.
    List<Partition<AbstractFileInputOperator<String>>> partitions = Lists.newArrayList();
    partitions.add(new DefaultPartition<AbstractFileInputOperator<String>>(oper));
    // incremental capacity controlled partitionCount property
    Collection<Partition<AbstractFileInputOperator<String>>> newPartitions = initialState.definePartitions(partitions, new PartitioningContextImpl(null, 0));
    Assert.assertEquals(2, newPartitions.size());
    Assert.assertEquals(1, initialState.getCurrentPartitions());
    Map<Integer, Partition<AbstractFileInputOperator<String>>> m = Maps.newHashMap();
    for (Partition<AbstractFileInputOperator<String>> p : newPartitions) {
        m.put(m.size(), p);
    }
    initialState.partitioned(m);
    Assert.assertEquals(2, initialState.getCurrentPartitions());
    /* Collect all operators in a list */
    List<AbstractFileInputOperator<String>> opers = Lists.newArrayList();
    for (Partition<AbstractFileInputOperator<String>> p : newPartitions) {
        LineByLineFileInputOperator oi = (LineByLineFileInputOperator) p.getPartitionedInstance();
        oi.setup(testMeta.context);
        oi.output.setSink(sink);
        opers.add(oi);
    }
    sink.clear();
    for (int i = 0; i < 10; i++) {
        for (AbstractFileInputOperator<String> o : opers) {
            o.beginWindow(wid);
            o.emitTuples();
            o.endWindow();
        }
        wid++;
    }
    Assert.assertEquals("Remaining tuples read ", 6, sink.collectedTuples.size());
}

Example 88 with Kryo

use of com.esotericsoftware.kryo.Kryo in project apex-malhar by apache.

the class AbstractFileInputOperatorTest method checkpoint.

/**
 * This method checkpoints the given operator.
 * @param oper The operator to checkpoint.
 * @param bos The ByteArrayOutputStream which saves the checkpoint data temporarily.
 * @return new operator.
 */
public static <T> T checkpoint(T oper, ByteArrayOutputStream bos) throws Exception {
    Kryo kryo = new Kryo();
    Output loutput = new Output(bos);
    kryo.writeObject(loutput, oper);
    loutput.close();
    Input lInput = new Input(bos.toByteArray());
    @SuppressWarnings("unchecked") T checkPointedOper = kryo.readObject(lInput, (Class<T>) oper.getClass());
    lInput.close();
    return checkPointedOper;
}

Example 89 with Kryo

use of com.esotericsoftware.kryo.Kryo in project apex-malhar by apache.

the class AbstractFileInputOperatorTest method restoreCheckPoint.

/**
 * Restores the checkpointed operator.
 * @param checkPointOper The checkpointed operator.
 * @param bos The ByteArrayOutputStream which saves the checkpoint data temporarily.
 */
@SuppressWarnings({ "unchecked", "rawtypes" })
public static <T> T restoreCheckPoint(T checkPointOper, ByteArrayOutputStream bos) throws Exception {
    Kryo kryo = new Kryo();
    Input lInput = new Input(bos.toByteArray());
    T oper = kryo.readObject(lInput, (Class<T>) checkPointOper.getClass());
    lInput.close();
    return oper;
}

Example 90 with Kryo

use of com.esotericsoftware.kryo.Kryo in project apex-malhar by apache.

the class FileEnrichmentTest method testEnrichmentOperatorFixedWidthFSLoader.

@Test
public void testEnrichmentOperatorFixedWidthFSLoader() throws IOException, InterruptedException {
    URL origUrl = this.getClass().getResource("/fixed-width-sample.txt");
    MapEnricher oper = new MapEnricher();
    FixedWidthFSLoader store = new FixedWidthFSLoader();
    store.setFieldDescription("Year:INTEGER:4,Make:STRING:5,Model:STRING:40,Description:STRING:40,Price:DOUBLE:8,Date:DATE:10:\"dd:mm:yyyy\"");
    store.setHasHeader(true);
    store.setPadding('_');
    store.setFileName(origUrl.toString());
    oper.setLookupFields(Arrays.asList("Year"));
    oper.setIncludeFields(Arrays.asList("Year", "Make", "Model", "Price", "Date"));
    oper.setStore(store);
    oper.setup(null);
    CollectorTestSink<Map<String, Object>> sink = new CollectorTestSink<>();
    @SuppressWarnings({ "unchecked", "rawtypes" }) CollectorTestSink<Object> tmp = (CollectorTestSink) sink;
    oper.output.setSink(tmp);
    oper.activate(null);
    oper.beginWindow(0);
    Map<String, Object> tuple = Maps.newHashMap();
    tuple.put("Year", 1997);
    Kryo kryo = new Kryo();
    oper.input.process(kryo.copy(tuple));
    oper.endWindow();
    oper.deactivate();
    oper.teardown();
    /* Number of tuple, emitted */
    Assert.assertEquals("Number of tuple emitted ", 1, sink.collectedTuples.size());
    Map<String, Object> emitted = sink.collectedTuples.iterator().next();
    /* The fields present in original event is kept as it is */
    Assert.assertEquals("Number of fields in emitted tuple", 5, emitted.size());
    Assert.assertEquals("Value of Year is 1997", tuple.get("Year"), emitted.get("Year"));
    /* Check if Make is added to the event */
    Assert.assertEquals("Make is part of tuple", true, emitted.containsKey("Make"));
    Assert.assertEquals("Value of Make", "Ford", emitted.get("Make"));
    /* Check if Model is added to the event */
    Assert.assertEquals("Model is part of tuple", true, emitted.containsKey("Model"));
    Assert.assertEquals("Value of Model", "E350", emitted.get("Model"));
    /* Check if Price is added to the event */
    Assert.assertEquals("Price is part of tuple", true, emitted.containsKey("Price"));
    Assert.assertEquals("Value of Price is 3000", 3000.0, emitted.get("Price"));
    Assert.assertTrue(emitted.get("Price") instanceof Double);
    /* Check if Date is added to the event */
    Assert.assertEquals("Date is part of tuple", true, emitted.containsKey("Date"));
    Date mfgDate = (Date) emitted.get("Date");
    Assert.assertEquals("value of day", 1, mfgDate.getDate());
    Assert.assertEquals("value of month", 0, mfgDate.getMonth());
    Assert.assertEquals("value of year", 2016, mfgDate.getYear() + 1900);
    Assert.assertTrue(emitted.get("Date") instanceof Date);
}