Examples with Tuple edu.iu.dsc.tws.api.comms.structs.Tuple used on opensource projects

Example 31 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class KeyedBatchIteratorLinkWrapper method persist.

@Override
public KeyedPersistedTSet<K, V> persist() {
    // handling checkpointing
    if (getTSetEnv().isCheckpointingEnabled()) {
        String persistVariableName = this.getId() + "-persisted";
        BatchChkPntEnvironment chkEnv = (BatchChkPntEnvironment) getTSetEnv();
        Boolean persisted = chkEnv.initVariable(persistVariableName, false);
        if (persisted) {
            // create a source function with the capability to read from disk
            DiskPartitionBackedSource<Tuple<K, V>> sourceFn = new DiskPartitionBackedSource<>(this.getId());
            // pass the source fn to the checkpointed tset (that would create a source tset from the
            // source function, the same way as a persisted tset. This preserves the order of tsets
            // that are being created in the checkpointed env)
            KeyedCheckpointedTSet<K, V> checkTSet = new KeyedCheckpointedTSet<>(getTSetEnv(), sourceFn, this.getTargetParallelism(), getSchema());
            // adding checkpointed tset to the graph, so that the IDs would not change
            addChildToGraph(checkTSet);
            // run only the checkpointed tset so that it would populate the inputs in the executor
            getTSetEnv().runOne(checkTSet);
            return checkTSet;
        } else {
            KeyedPersistedTSet<K, V> storable = this.doPersist();
            chkEnv.updateVariable(persistVariableName, true);
            chkEnv.commit();
            return storable;
        }
    }
    return doPersist();
}

Example 32 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class KeyedSerializerLargeTest method testBuildLargeListByteMessage.

@SuppressWarnings("Unchecked")
@Test
public void testBuildLargeListByteMessage() {
    int numBuffers = 32;
    int size = 1024000;
    for (int numObjects = 1; numObjects < 5000; numObjects++) {
        System.out.println("Starting test : " + numObjects);
        List<Object> data = new AggregatedObjects<>();
        for (int i = 0; i < numObjects; i++) {
            Object o = createKeyedData(100, MessageTypes.INTEGER_ARRAY, 1, MessageTypes.INTEGER);
            data.add(o);
        }
        InMessage inMessage = keyedListValueCase(numBuffers, size, data, MessageTypes.INTEGER_ARRAY, MessageTypes.INTEGER);
        List<Object> result = (List<Object>) inMessage.getDeserializedData();
        Assert.assertEquals(numObjects, result.size());
        for (int i = 0; i < result.size(); i++) {
            Tuple deserializedData = (Tuple) result.get(i);
            Tuple d = (Tuple) data.get(i);
            Assert.assertEquals(deserializedData.getKey(), d.getKey());
            Assert.assertArrayEquals((int[]) deserializedData.getValue(), (int[]) ((Tuple) d).getValue());
        }
    }
}

Example 33 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class AggregatorListTest method testAggregatedObject.

@Test
public void testAggregatedObject() {
    int numBuffers = 16;
    int size = 1000;
    List<Object> data = new ArrayList<>();
    for (int i = 0; i < 4; i++) {
        data.add("I " + i);
    }
    Tuple tuple = new Tuple(1, data);
    InMessage inMessage = keyedSingleValueCase(numBuffers, size, MessageTypes.OBJECT, MessageTypes.INTEGER, tuple);
    Tuple result = (Tuple) inMessage.getDeserializedData();
    Assert.assertEquals((int) result.getKey(), 1);
    List<Object> resultList = (List<Object>) tuple.getValue();
    Assert.assertEquals(resultList, data);
}

Example 34 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class KeyedSerializerTest method testBuildLargeListByteMessage.

@SuppressWarnings("Unchecked")
@Test
public void testBuildLargeListByteMessage() {
    int numBuffers = 32;
    int size = 1000;
    List<Object> data = new AggregatedObjects<>();
    for (int i = 0; i < 4; i++) {
        Object o = createKeyedData(800, MessageTypes.BYTE_ARRAY, MessageTypes.BYTE_ARRAY);
        data.add(o);
    }
    InMessage inMessage = keyedListValueCase(numBuffers, size, data, MessageTypes.BYTE_ARRAY, MessageTypes.BYTE_ARRAY);
    List<Object> result = (List<Object>) inMessage.getDeserializedData();
    for (int i = 0; i < result.size(); i++) {
        Tuple deserializedData = (Tuple) result.get(i);
        Tuple d = (Tuple) data.get(i);
        Assert.assertArrayEquals((byte[]) deserializedData.getKey(), (byte[]) ((Tuple) d).getKey());
        Assert.assertArrayEquals((byte[]) deserializedData.getValue(), (byte[]) ((Tuple) d).getValue());
    }
}

Example 35 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class KeyedSerializerTest method testBuildLargeLongMessage.

@Test
public void testBuildLargeLongMessage() {
    int numBuffers = 10;
    int size = 1000;
    MessageType type = MessageTypes.LONG_ARRAY;
    Object data = createKeyedData(800, type, MessageTypes.LONG);
    InMessage inMessage = keyedSingleValueCase(numBuffers, size, type, MessageTypes.LONG, data);
    Tuple deserializedData = (Tuple) inMessage.getDeserializedData();
    Assert.assertEquals((long) deserializedData.getKey(), (long) ((Tuple) data).getKey());
    Assert.assertArrayEquals((long[]) deserializedData.getValue(), (long[]) ((Tuple) data).getValue());
}