Examples with Kryo com.esotericsoftware.kryo.Kryo used on opensource projects

Example 91 with Kryo

use of com.esotericsoftware.kryo.Kryo in project apex-malhar by apache.

the class FileEnrichmentTest method testEnrichmentOperatorDelimitedFSLoader.

@Test
public void testEnrichmentOperatorDelimitedFSLoader() throws IOException, InterruptedException {
    URL origUrl = this.getClass().getResource("/productmapping-delim.txt");
    URL fileUrl = new URL(this.getClass().getResource("/").toString() + "productmapping-delim1.txt");
    FileUtils.deleteQuietly(new File(fileUrl.getPath()));
    FileUtils.copyFile(new File(origUrl.getPath()), new File(fileUrl.getPath()));
    MapEnricher oper = new MapEnricher();
    DelimitedFSLoader store = new DelimitedFSLoader();
    // store.setFieldDescription("productCategory:INTEGER,productId:INTEGER");
    store.setFileName(fileUrl.toString());
    store.setSchema("{\"separator\":\",\",\"fields\": [{\"name\": \"productCategory\",\"type\": \"Integer\"},{\"name\": \"productId\",\"type\": \"Integer\"},{\"name\": \"mfgDate\",\"type\": \"Date\",\"constraints\": {\"format\": \"dd/MM/yyyy\"}}]}");
    oper.setLookupFields(Arrays.asList("productId"));
    oper.setIncludeFields(Arrays.asList("productCategory", "mfgDate"));
    oper.setStore(store);
    oper.setup(null);
    CollectorTestSink<Map<String, Object>> sink = new CollectorTestSink<>();
    @SuppressWarnings({ "unchecked", "rawtypes" }) CollectorTestSink<Object> tmp = (CollectorTestSink) sink;
    oper.output.setSink(tmp);
    oper.activate(null);
    oper.beginWindow(0);
    Map<String, Object> tuple = Maps.newHashMap();
    tuple.put("productId", 3);
    tuple.put("channelId", 4);
    tuple.put("amount", 10.0);
    Kryo kryo = new Kryo();
    oper.input.process(kryo.copy(tuple));
    oper.endWindow();
    oper.deactivate();
    /* Number of tuple, emitted */
    Assert.assertEquals("Number of tuple emitted ", 1, sink.collectedTuples.size());
    Map<String, Object> emitted = sink.collectedTuples.iterator().next();
    /* The fields present in original event is kept as it is */
    Assert.assertEquals("Number of fields in emitted tuple", 5, emitted.size());
    Assert.assertEquals("value of productId is 3", tuple.get("productId"), emitted.get("productId"));
    Assert.assertEquals("value of channelId is 4", tuple.get("channelId"), emitted.get("channelId"));
    Assert.assertEquals("value of amount is 10.0", tuple.get("amount"), emitted.get("amount"));
    /* Check if productCategory is added to the event */
    Assert.assertEquals("productCategory is part of tuple", true, emitted.containsKey("productCategory"));
    Assert.assertEquals("value of product category is 1", 5, emitted.get("productCategory"));
    Assert.assertTrue(emitted.get("productCategory") instanceof Integer);
    /* Check if mfgDate is added to the event */
    Assert.assertEquals("mfgDate is part of tuple", true, emitted.containsKey("productCategory"));
    Date mfgDate = (Date) emitted.get("mfgDate");
    Assert.assertEquals("value of day", 1, mfgDate.getDate());
    Assert.assertEquals("value of month", 0, mfgDate.getMonth());
    Assert.assertEquals("value of year", 2016, mfgDate.getYear() + 1900);
    Assert.assertTrue(emitted.get("mfgDate") instanceof Date);
}

Example 92 with Kryo

use of com.esotericsoftware.kryo.Kryo in project apex-malhar by apache.

the class POJOTimeBasedJoinOperatorTest method testLeftOuterJoinOperator.

@Test
public void testLeftOuterJoinOperator() throws IOException, InterruptedException {
    Kryo kryo = new Kryo();
    POJOJoinOperator oper = new POJOJoinOperator();
    JoinStore store = new InMemoryStore(200, 200);
    oper.setLeftStore(kryo.copy(store));
    oper.setRightStore(kryo.copy(store));
    oper.setIncludeFields("ID,Name;OID,Amount");
    oper.setKeyFields("ID,CID");
    oper.outputClass = CustOrder.class;
    oper.setStrategy("left_outer_join");
    oper.setup(MapTimeBasedJoinOperator.context);
    CollectorTestSink<List<CustOrder>> sink = new CollectorTestSink<List<CustOrder>>();
    @SuppressWarnings({ "unchecked", "rawtypes" }) CollectorTestSink<Object> tmp = (CollectorTestSink) sink;
    oper.outputPort.setSink(tmp);
    oper.beginWindow(0);
    Customer tuple1 = new Customer(1, "Anil");
    oper.input1.process(tuple1);
    CountDownLatch latch = new CountDownLatch(1);
    Order order = new Order(102, 3, 300);
    oper.input2.process(order);
    Order order2 = new Order(103, 7, 300);
    oper.input2.process(order2);
    oper.endWindow();
    latch.await(500, TimeUnit.MILLISECONDS);
    oper.beginWindow(1);
    Order order3 = new Order(104, 5, 300);
    oper.input2.process(order3);
    Customer tuple2 = new Customer(5, "DT");
    oper.input1.process(tuple2);
    latch.await(500, TimeUnit.MILLISECONDS);
    oper.endWindow();
    latch.await(500, TimeUnit.MILLISECONDS);
    oper.beginWindow(2);
    oper.endWindow();
    latch.await(5000, TimeUnit.MILLISECONDS);
    /* Number of tuple, emitted */
    Assert.assertEquals("Number of tuple emitted ", 2, sink.collectedTuples.size());
    Iterator<List<CustOrder>> ite = sink.collectedTuples.iterator();
    List<CustOrder> emittedList = ite.next();
    CustOrder emitted = emittedList.get(0);
    Assert.assertEquals("value of ID :", tuple2.ID, emitted.ID);
    Assert.assertEquals("value of Name :", tuple2.Name, emitted.Name);
    Assert.assertEquals("value of OID: ", order3.OID, emitted.OID);
    Assert.assertEquals("value of Amount: ", order3.Amount, emitted.Amount);
    emittedList = ite.next();
    emitted = emittedList.get(0);
    Assert.assertEquals("Joined Tuple ", "{ID=1, Name='Anil', OID=0, Amount=0}", emitted.toString());
}

Example 93 with Kryo

use of com.esotericsoftware.kryo.Kryo in project apex-malhar by apache.

the class POJOTimeBasedJoinOperatorTest method testRightOuterJoinOperator.

@Test
public void testRightOuterJoinOperator() throws IOException, InterruptedException {
    Kryo kryo = new Kryo();
    POJOJoinOperator oper = new POJOJoinOperator();
    JoinStore store = new InMemoryStore(200, 200);
    oper.setLeftStore(kryo.copy(store));
    oper.setRightStore(kryo.copy(store));
    oper.setIncludeFields("ID,Name;OID,Amount");
    oper.setKeyFields("ID,CID");
    oper.outputClass = CustOrder.class;
    oper.setStrategy("right_outer_join");
    oper.setup(MapTimeBasedJoinOperator.context);
    CollectorTestSink<List<CustOrder>> sink = new CollectorTestSink<List<CustOrder>>();
    @SuppressWarnings({ "unchecked", "rawtypes" }) CollectorTestSink<Object> tmp = (CollectorTestSink) sink;
    oper.outputPort.setSink(tmp);
    oper.beginWindow(0);
    Customer tuple1 = new Customer(1, "Anil");
    oper.input1.process(tuple1);
    CountDownLatch latch = new CountDownLatch(1);
    Order order = new Order(102, 3, 300);
    oper.input2.process(order);
    Order order2 = new Order(103, 7, 300);
    oper.input2.process(order2);
    oper.endWindow();
    latch.await(500, TimeUnit.MILLISECONDS);
    oper.beginWindow(1);
    Order order3 = new Order(104, 5, 300);
    oper.input2.process(order3);
    Customer tuple2 = new Customer(5, "DT");
    oper.input1.process(tuple2);
    latch.await(500, TimeUnit.MILLISECONDS);
    oper.endWindow();
    latch.await(500, TimeUnit.MILLISECONDS);
    oper.beginWindow(2);
    oper.endWindow();
    latch.await(5000, TimeUnit.MILLISECONDS);
    /* Number of tuple, emitted */
    Assert.assertEquals("Number of tuple emitted ", 2, sink.collectedTuples.size());
    Iterator<List<CustOrder>> ite = sink.collectedTuples.iterator();
    List<CustOrder> emittedList = ite.next();
    CustOrder emitted = emittedList.get(0);
    Assert.assertEquals("value of ID :", tuple2.ID, emitted.ID);
    Assert.assertEquals("value of Name :", tuple2.Name, emitted.Name);
    Assert.assertEquals("value of OID: ", order3.OID, emitted.OID);
    Assert.assertEquals("value of Amount: ", order3.Amount, emitted.Amount);
    emittedList = ite.next();
    Assert.assertEquals("Joined Tuple ", "{ID=0, Name='null', OID=102, Amount=300}", emittedList.get(0).toString());
    Assert.assertEquals("Joined Tuple ", "{ID=0, Name='null', OID=103, Amount=300}", emittedList.get(1).toString());
}

Example 94 with Kryo

use of com.esotericsoftware.kryo.Kryo in project apex-malhar by apache.

the class KryoSerializableStreamCodec method readObject.

private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
    in.defaultReadObject();
    this.kryo = new Kryo();
    this.kryo.setClassLoader(Thread.currentThread().getContextClassLoader());
}