use of com.esotericsoftware.kryo.serializers.FieldSerializer in project apex-malhar by apache.
the class HiveMockTest method testHDFSHiveCheckpoint.
@Test
public void testHDFSHiveCheckpoint() throws SQLException, TException {
hiveInitializeDatabase(createStore(null));
HiveStore hiveStore = createStore(null);
hiveStore.setFilepath(testdir);
HiveOperator outputOperator = new HiveOperator();
HiveOperator newOp;
outputOperator.setStore(hiveStore);
ArrayList<String> hivePartitionColumns = new ArrayList<String>();
hivePartitionColumns.add("dt");
FSRollingTestImpl fsRolling = new FSRollingTestImpl();
hiveInitializeDatabase(createStore(null));
outputOperator.setHivePartitionColumns(hivePartitionColumns);
outputOperator.setTablename(tablename);
fsRolling.setFilePath(testdir);
short persmission = 511;
fsRolling.setFilePermission(persmission);
fsRolling.setAlwaysWriteToTmp(false);
fsRolling.setMaxLength(128);
AttributeMap.DefaultAttributeMap attributeMap = new AttributeMap.DefaultAttributeMap();
attributeMap.put(OperatorContext.PROCESSING_MODE, ProcessingMode.AT_LEAST_ONCE);
attributeMap.put(OperatorContext.ACTIVATION_WINDOW_ID, -1L);
attributeMap.put(DAG.APPLICATION_ID, APP_ID);
OperatorContext context = mockOperatorContext(OPERATOR_ID, attributeMap);
fsRolling.setup(context);
FilePartitionMapping mapping1 = new FilePartitionMapping();
FilePartitionMapping mapping2 = new FilePartitionMapping();
FilePartitionMapping mapping3 = new FilePartitionMapping();
outputOperator.setup(context);
mapping1.setFilename(APP_ID + "/" + OPERATOR_ID + "/" + "2014-12-10" + "/" + "0-transaction.out.part.0");
ArrayList<String> partitions1 = new ArrayList<String>();
partitions1.add("2014-12-10");
mapping1.setPartition(partitions1);
mapping2.setFilename(APP_ID + "/" + OPERATOR_ID + "/" + "2014-12-11" + "/" + "0-transaction.out.part.0");
ArrayList<String> partitions2 = new ArrayList<String>();
partitions2.add("2014-12-11");
mapping2.setPartition(partitions2);
ArrayList<String> partitions3 = new ArrayList<String>();
partitions3.add("2014-12-12");
mapping3.setFilename(APP_ID + "/" + OPERATOR_ID + "/" + "2014-12-12" + "/" + "0-transaction.out.part.0");
mapping3.setPartition(partitions3);
for (int wid = 0, total = 0; wid < NUM_WINDOWS; wid++) {
fsRolling.beginWindow(wid);
for (int tupleCounter = 0; tupleCounter < BLAST_SIZE && total < DATABASE_SIZE; tupleCounter++, total++) {
fsRolling.input.process("2014-12-1" + tupleCounter);
}
if (wid == 7) {
fsRolling.committed(wid - 1);
outputOperator.processTuple(mapping1);
outputOperator.processTuple(mapping2);
}
fsRolling.endWindow();
if ((wid == 6) || (wid == 9)) {
fsRolling.beforeCheckpoint(wid);
fsRolling.checkpointed(wid);
}
if (wid == 9) {
Kryo kryo = new Kryo();
FieldSerializer<HiveOperator> f1 = (FieldSerializer<HiveOperator>) kryo.getSerializer(HiveOperator.class);
FieldSerializer<HiveStore> f2 = (FieldSerializer<HiveStore>) kryo.getSerializer(HiveStore.class);
f1.setCopyTransient(false);
f2.setCopyTransient(false);
newOp = kryo.copy(outputOperator);
outputOperator.teardown();
newOp.setup(context);
newOp.beginWindow(7);
newOp.processTuple(mapping3);
newOp.endWindow();
newOp.teardown();
break;
}
}
hiveStore.connect();
client.execute("select * from " + tablename + " where dt='2014-12-10'");
List<String> recordsInDatePartition1 = client.fetchAll();
client.execute("select * from " + tablename + " where dt='2014-12-11'");
List<String> recordsInDatePartition2 = client.fetchAll();
client.execute("select * from " + tablename + " where dt='2014-12-12'");
List<String> recordsInDatePartition3 = client.fetchAll();
client.execute("drop table " + tablename);
hiveStore.disconnect();
Assert.assertEquals(7, recordsInDatePartition1.size());
for (int i = 0; i < recordsInDatePartition1.size(); i++) {
LOG.debug("records in first date partition are {}", recordsInDatePartition1.get(i));
/*An array containing partition and data is returned as a string record, hence we need to upcast it to an object first
and then downcast to a string in order to use in Assert.*/
Object record = recordsInDatePartition1.get(i);
Object[] records = (Object[]) record;
Assert.assertEquals("2014-12-10", records[1]);
}
Assert.assertEquals(7, recordsInDatePartition2.size());
for (int i = 0; i < recordsInDatePartition2.size(); i++) {
LOG.debug("records in second date partition are {}", recordsInDatePartition2.get(i));
Object record = recordsInDatePartition2.get(i);
Object[] records = (Object[]) record;
Assert.assertEquals("2014-12-11", records[1]);
}
Assert.assertEquals(10, recordsInDatePartition3.size());
for (int i = 0; i < recordsInDatePartition3.size(); i++) {
LOG.debug("records in second date partition are {}", recordsInDatePartition3.get(i));
Object record = recordsInDatePartition3.get(i);
Object[] records = (Object[]) record;
Assert.assertEquals("2014-12-12", records[1]);
}
}
use of com.esotericsoftware.kryo.serializers.FieldSerializer in project Dempsy by Dempsy.
the class TestDefaultSerializer method testChildClassSerializationWithRegistrationAndOptimization.
@Test
public void testChildClassSerializationWithRegistrationAndOptimization() throws Throwable {
KryoSerializer<Object> ser = new KryoSerializer<Object>(defaultMock3Optimizer);
JavaSerializer<Object> serJ = new JavaSerializer<Object>();
KryoSerializer<Object> serR = new KryoSerializer<Object>(defaultMock3Optimizer, new Registration(MockClass.class.getName(), 10));
KryoSerializer<Object> serRR = new KryoSerializer<Object>(defaultMock3Optimizer, new Registration(MockClass.class.getName(), 10), new Registration(Mock3.class.getName(), 11));
KryoSerializer<Object> serRROb = new KryoSerializer<Object>(defaultMock3Optimizer, new Registration(MockClass.class.getName()), new Registration(Mock3.class.getName()));
KryoSerializer<Object> serRRO = new KryoSerializer<Object>(new Registration(MockClass.class.getName(), 10), new Registration(Mock3.class.getName(), 11), new Registration(UUID.class.getName(), 12));
serRRO.setKryoOptimizer(new KryoOptimizer() {
@Override
public void preRegister(Kryo kryo) {
kryo.setRegistrationRequired(true);
@SuppressWarnings("unchecked") FieldSerializer<MockClass> mockClassSer = (FieldSerializer<MockClass>) kryo.getSerializer(MockClass.class);
mockClassSer.setFieldsCanBeNull(false);
@SuppressWarnings("unchecked") FieldSerializer<Mock2> mock2Ser = (FieldSerializer<Mock2>) kryo.getSerializer(MockClass.class);
mock2Ser.setFixedFieldTypes(true);
mock2Ser.setFieldsCanBeNull(false);
}
@Override
public void postRegister(Kryo kryo) {
com.esotericsoftware.kryo.Registration reg = kryo.getRegistration(UUID.class);
reg.setSerializer(uuidSerializer);
}
});
Mock2 o = new Mock3(1, new MockClass(2, "Hello"));
byte[] data = ser.serialize(o);
byte[] dataJ = serJ.serialize(o);
byte[] dataR = serR.serialize(o);
byte[] dataRR = serRR.serialize(o);
byte[] dataRROb = serRROb.serialize(o);
byte[] dataRRO = serRRO.serialize(o);
assertTrue(dataJ.length > data.length);
assertTrue(dataR.length < data.length);
assertTrue(dataRR.length < dataR.length);
assertTrue(dataRROb.length == dataRR.length);
assertTrue(dataRRO.length <= dataRR.length);
Mock2 o2 = (Mock2) ser.deserialize(data);
assertEquals(1, o2.getInt());
assertEquals(new MockClass(2, "Hello"), o2.getMockClass());
assertTrue(o2 instanceof Mock3);
assertEquals(1, ((Mock3) o2).myI);
assertEquals(o, serR.deserialize(dataR));
assertEquals(o, serRR.deserialize(dataRR));
assertEquals(o, serRRO.deserialize(dataRRO));
}
use of com.esotericsoftware.kryo.serializers.FieldSerializer in project hive by apache.
the class SerializationUtilities method removeField.
private static void removeField(Kryo kryo, Class type, String fieldName) {
FieldSerializer fld = new FieldSerializer(kryo, type);
fld.removeField(fieldName);
kryo.register(type, fld);
}
Aggregations