Examples with LMDBMemoryManager edu.iu.dsc.tws.data.memory.lmdb.LMDBMemoryManager used on opensource projects

Example 1 with LMDBMemoryManager

use of edu.iu.dsc.tws.data.memory.lmdb.LMDBMemoryManager in project twister2 by DSC-SPIDAL.

the class BasicMemoryManagerContainer method testPrimitivesLMDB.

/**
 * test primitives with LMDB memory manager
 */
public boolean testPrimitivesLMDB() {
    LOG.info("## Running LMDB primitives test ##");
    boolean allPassed = true;
    Path dataPath = new Path("/home/pulasthi/work/twister2/lmdbdatabase");
    MemoryManager memoryManager = new LMDBMemoryManager(dataPath);
    int opID = 1;
    OperationMemoryManager op = memoryManager.addOperation(opID, DataMessageType.INTEGER);
    // Test single integer operation
    ByteBuffer key = ByteBuffer.allocateDirect(4);
    ByteBuffer value = ByteBuffer.allocateDirect(4);
    key.putInt(1);
    int testInt = 1231212121;
    byte[] val = Ints.toByteArray(testInt);
    value.put(val);
    op.put(key, value);
    ByteBuffer results = op.get(key);
    int res = results.getInt();
    if (res != testInt) {
        allPassed = false;
    }
    if (allPassed) {
        System.out.println("Passed LMDB int test");
    }
    // test int array, put should replace the current value
    int[] testarray = { 234, 14123, 534, 6345 };
    value = ByteBuffer.allocateDirect(16);
    for (int i : testarray) {
        value.putInt(i);
    }
    op.put(key, value);
    results = op.get(key);
    for (int i : testarray) {
        if (i != results.getInt()) {
            allPassed = false;
        }
    }
    if (allPassed) {
        System.out.println("Passed LMDB int array test");
    }
    // get retuls with iterator
    Iterator<Object> iter = op.iterator();
    int[] dataset = (int[]) iter.next();
    for (int i = 0; i < dataset.length; i++) {
        if (dataset[i] != testarray[i]) {
            allPassed = false;
        }
    }
    if (allPassed) {
        System.out.println("Passed LMDB int array iterator test, number of values returned by" + "iterator : " + 1);
    }
    // iterator with more than 1 key will test that keys are sorted properly
    int[][] datamultiarray = { { 1, 11, 111, 1111 }, { 2, 22, 222, 2222 }, { 3, 33, 333, 3333 }, { 4, 44, 444, 4444 } };
    ByteBuffer value2 = ByteBuffer.allocateDirect(16);
    ByteBuffer value3 = ByteBuffer.allocateDirect(16);
    ByteBuffer value4 = ByteBuffer.allocateDirect(16);
    key.clear();
    value.clear();
    key.putInt(4);
    for (int i : datamultiarray[3]) {
        value.putInt(i);
    }
    op.put(key, value);
    key.clear();
    value.clear();
    key.putInt(1);
    for (int i : datamultiarray[0]) {
        value.putInt(i);
    }
    op.put(key, value);
    key.clear();
    value.clear();
    key.putInt(3);
    for (int i : datamultiarray[2]) {
        value.putInt(i);
    }
    op.put(key, value);
    key.clear();
    value.clear();
    key.putInt(2);
    for (int i : datamultiarray[1]) {
        value.putInt(i);
    }
    op.put(key, value);
    Iterator<Object> itermulti = op.iterator();
    int itercount = 0;
    itercount = 0;
    while (itermulti.hasNext()) {
        if (itercount > 3) {
            break;
        }
        dataset = (int[]) itermulti.next();
        for (int i = 0; i < 4; i++) {
            if (dataset[i] != datamultiarray[itercount][i]) {
                allPassed = false;
            }
        }
        itercount++;
    }
    if (allPassed) {
        System.out.println("Passed LMDB int multi array iterator test, number of values returned by" + "iterator : " + itercount);
    }
    // test append function
    key.clear();
    value.clear();
    key.putInt(6);
    for (int i : datamultiarray[0]) {
        value.putInt(i);
    }
    op.put(key, value);
    value.clear();
    for (int i : datamultiarray[1]) {
        value.putInt(i);
    }
    op.append(key, value);
    results = op.get(key);
    for (int i : datamultiarray[0]) {
        int itemp = results.getInt();
        if (i != itemp) {
            allPassed = false;
        }
    }
    for (int i : datamultiarray[1]) {
        int itemp = results.getInt();
        if (i != itemp) {
            allPassed = false;
        }
    }
    if (allPassed) {
        System.out.println("Passed LMDB int array append test");
    }
    return allPassed;
}

Example 2 with LMDBMemoryManager

use of edu.iu.dsc.tws.data.memory.lmdb.LMDBMemoryManager in project twister2 by DSC-SPIDAL.

the class MPIDataFlowOperation method setStoreBased.

public void setStoreBased(boolean storeBased) {
    isStoreBased = storeBased;
    if (isStoreBased) {
        // TODO : need to load this from config file, both the type of memory manager and the datapath
        // TODO : need to make the memory manager available globally
        opertionID = (int) System.currentTimeMillis();
        this.kryoSerializer = new KryoSerializer();
        // Path dataPath = new Path("/scratch/pulasthi/terasort/lmdbdatabase_" + this.executor);
        Path dataPath = new Path(MPIContext.networkStoragePath(config) + " /lmdbdatabase_" + this.executor);
        this.memoryManager = new LMDBMemoryManager(dataPath);
        if (!isKeyed) {
            this.operationMemoryManager = memoryManager.addOperation(opertionID, MessageTypeUtils.toDataMessageType(type));
        } else {
            this.operationMemoryManager = memoryManager.addOperation(opertionID, MessageTypeUtils.toDataMessageType(type), MessageTypeUtils.toDataMessageType(keyType));
        }
    }
}

Example 3 with LMDBMemoryManager

use of edu.iu.dsc.tws.data.memory.lmdb.LMDBMemoryManager in project twister2 by DSC-SPIDAL.

the class SimpleTaskQueueWithMM method init.

/**
 * Initialize the container
 */
public void init(Config cfg, int containerId, ResourcePlan plan) {
    LOG.log(Level.INFO, "Starting the example with container id: " + plan.getThisId());
    // Creates task an task executor instance to be used in this container
    taskExecutor = new TaskExecutorFixedThread();
    this.status = Status.INIT;
    // lets create the task plan
    TaskPlan taskPlan = Utils.createTaskPlan(cfg, plan);
    // first get the communication config file
    TWSNetwork network = new TWSNetwork(cfg, taskPlan);
    TWSCommunication channel = network.getDataFlowTWSCommunication();
    // we are sending messages from 0th task to 1st task
    Set<Integer> sources = new HashSet<>();
    sources.add(0);
    int dests = 1;
    Map<String, Object> newCfg = new HashMap<>();
    LOG.info("Setting up reduce dataflow operation");
    Path dataPath = new Path("/home/pulasthi/work/twister2/lmdbdatabase");
    MemoryManager memoryManager = new LMDBMemoryManager(dataPath);
    // this method calls the init method
    // I think this is wrong
    // TODO: Does the task genereate the communication or is it done by a controller for examples
    // the direct comm between task 0 and 1 is it done by the container or the the task
    // TODO: if the task creates the dataflowop does the task progress it or the executor
    // TODO : FOR NOW the dataflowop is created at container and sent to task
    LinkedQueue<Message> pongQueue = new LinkedQueue<Message>();
    taskExecutor.registerQueue(0, pongQueue);
    direct = channel.direct(newCfg, MessageType.OBJECT, 0, sources, dests, new PingPongReceive());
    taskExecutor.initCommunication(channel, direct);
    // Memory Manager
    if (containerId == 0) {
        byte[] val = Longs.toByteArray(1231212121213L);
        byte[] val2 = Longs.toByteArray(22222222L);
        ByteBuffer valbuf = ByteBuffer.allocateDirect(8192);
        memoryManager.put(0, "temp", valbuf);
    // memoryManager.put(0, "temp", val);
    // memoryManager.put(0, "temp", val2);
    // the map thread where data is produced
    // LOG.log(Level.INFO, "Starting map thread");
    // SourceTask<Object> mapTask = new MapWorker(0, direct);
    // mapTask.setMemoryManager(memoryManager);
    // taskExecutor.registerTask(mapTask);
    // taskExecutor.submitTask(0);
    // taskExecutor.progres();
    } else if (containerId == 1) {
        byte[] val3 = Longs.toByteArray(3333333L);
        ByteBuffer val3buf = ByteBuffer.wrap(val3);
        try {
            Thread.sleep(2000);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        ByteBuffer results = memoryManager.get(0, "temp");
        if (results.limit() == 8192) {
            System.out.println("Correct " + results.limit());
        }
        ByteBuffer valbuf2 = ByteBuffer.allocateDirect(16192);
        memoryManager.put(0, "temp", valbuf2);
        results = memoryManager.get(0, "temp");
        if (results.limit() == 16192) {
            System.out.println("Correct " + results.limit());
        }
        ByteBuffer results2 = memoryManager.get(0, "temp");
        ByteBuffer results3 = memoryManager.get(0, "temp");
        if (results2 == null) {
            System.out.println("Missing key is null");
        }
        if (results3.getLong() == 1231212121213L) {
            System.out.println("Long value is correct");
        }
        memoryManager.append(0, "temp", val3buf);
        ByteBuffer resultsappend = memoryManager.get(0, "temp");
        System.out.println("Long value 1 :" + resultsappend.getLong());
        System.out.println("Long value 1 :" + resultsappend.getLong());
    // ArrayList<Integer> inq = new ArrayList<>();
    // inq.add(0);
    // taskExecutor.setTaskMessageProcessLimit(10000);
    // SinkTask<Object> recTask = new RecieveWorker(1);
    // recTask.setMemoryManager(memoryManager);
    // taskExecutor.registerSinkTask(recTask, inq);
    // taskExecutor.progres();
    }
}

Example 4 with LMDBMemoryManager

use of edu.iu.dsc.tws.data.memory.lmdb.LMDBMemoryManager in project twister2 by DSC-SPIDAL.

the class LMDBTest method testPrimitives.

public boolean testPrimitives() {
    boolean allPassed = true;
    Path dataPath = new Path("/home/pulasthi/work/twister2/lmdbdatabase");
    MemoryManager memoryManager = new LMDBMemoryManager(dataPath);
    int opID = 1;
    memoryManager.addOperation(opID, DataMessageType.INTEGER);
    // Test single integer operation
    ByteBuffer key = ByteBuffer.allocateDirect(4);
    ByteBuffer value = ByteBuffer.allocateDirect(4);
    key.putInt(1);
    int testInt = 1231212121;
    byte[] val = Ints.toByteArray(testInt);
    value.put(val);
    memoryManager.put(0, key, value);
    ByteBuffer results = memoryManager.get(opID, key);
    int res = results.getInt();
    if (res == testInt) {
        System.out.println("true");
    } else {
        allPassed = false;
        System.out.println("false");
    }
    return true;
}