Examples with Twister2RuntimeException edu.iu.dsc.tws.api.exceptions.Twister2RuntimeException used on opensource projects

Example 51 with Twister2RuntimeException

use of edu.iu.dsc.tws.api.exceptions.Twister2RuntimeException in project twister2 by DSC-SPIDAL.

the class TWSUCXChannel method sendMessage.

@Override
public boolean sendMessage(int id, ChannelMessage message, ChannelListener callback) {
    AtomicInteger buffersLeft = new AtomicInteger(message.getBuffers().size());
    for (DataBuffer buffer : message.getBuffers()) {
        buffer.getByteBuffer().limit(buffer.getSize());
        buffer.getByteBuffer().position(0);
        int tag = this.workerId * tagWIdOffset + message.getHeader().getEdge();
        LOG.log(Level.FINE, () -> String.format("SENDING to %d[%d] : %s, TAG[%d]", id, message.getHeader().getEdge(), buffer.getByteBuffer(), tag));
        this.endpoints.get(id).sendTaggedNonBlocking(buffer.getByteBuffer(), tag, new UcxCallback() {

            @Override
            public void onSuccess(UcpRequest request) {
                pendingSendRequests.decrementAndGet();
                if (buffersLeft.decrementAndGet() == 0) {
                    callback.onSendComplete(id, message.getHeader().getEdge(), message);
                }
            }

            @Override
            public void onError(int ucsStatus, String errorMsg) {
                // This is a catastrophic failure
                LOG.severe("UCX send request failed to worker " + id + " with status " + ucsStatus + ". Error : " + errorMsg);
                throw new Twister2RuntimeException("Send request to worker : " + id + " failed. " + errorMsg);
            }
        });
        this.pendingSendRequests.incrementAndGet();
    }
    return true;
}

Example 52 with Twister2RuntimeException

use of edu.iu.dsc.tws.api.exceptions.Twister2RuntimeException in project twister2 by DSC-SPIDAL.

the class HashJoinUtils method join.

/**
 * Disk based inner join
 */
public static Iterator<JoinedTuple> join(ResettableIterator<Tuple<?, ?>> leftIt, ResettableIterator<Tuple<?, ?>> rightIt, CommunicationContext.JoinType joinType, MessageType keyType) {
    // choosing hashing and probing relations
    // if inner join:
    // hashing = left
    // probing = right
    // if left join:
    // hashing = right
    // probing = left
    // if right join:
    // hashing = left
    // probing = right
    final ResettableIterator<Tuple<?, ?>> hashingRelation = joinType.equals(CommunicationContext.JoinType.LEFT) ? rightIt : leftIt;
    final ResettableIterator<Tuple<?, ?>> probingRelation = joinType.equals(CommunicationContext.JoinType.LEFT) ? leftIt : rightIt;
    // set the memory limits based on the heap allocation
    final double lowerMemoryBound = Runtime.getRuntime().totalMemory() * 0.1;
    return new Iterator<JoinedTuple>() {

        private boolean hashingDone;

        private Map<Object, List> keyHash = new THashMap<>(keyType);

        // always keep the nextJoinTuple in memory. hasNext() will use this field
        private JoinedTuple nextJoinTuple;

        /**
         * This method will perform following actions in order
         * <ol>
         *   <li>Clear existing HashMap</li>
         *   <li>Create HashMap from the hashingRelation till it hit the memory limits</li>
         *   <li>Determine whether the hashingRelation is fully consumed</li>
         * </ol>
         */
        private void doHashing() {
            this.keyHash.clear();
            // building the hash, as long as memory permits
            while (Runtime.getRuntime().freeMemory() > lowerMemoryBound && hashingRelation.hasNext()) {
                Tuple<?, ?> nextLeft = hashingRelation.next();
                keyHash.computeIfAbsent(nextLeft.getKey(), k -> new ArrayList()).add(nextLeft.getValue());
            }
            // determine whether hashRelation is fully consumed
            hashingDone = !hashingRelation.hasNext();
            if (!hashingDone && this.keyHash.isEmpty()) {
                // problem!. We have cleared the old hash, yet there's no free memory available to proceed
                throw new Twister2RuntimeException("Couldn't progress due to memory limitations." + "Available free memory : " + Runtime.getRuntime().freeMemory() + ", Expected free memory : " + lowerMemoryBound);
            }
        }

        {
            // initially do hashing & probing
            doHashing();
            doProbing();
        }

        // when iterating over the right(probing) relation, current element
        // (which has been returned by next()) will be kept in memory since it should be combined
        // with all the tuples in leftListForCurrentKey. But this has to be done on demand, on next()
        // call of joined iterator.
        private Tuple<?, ?> currentProbingTuple;

        // list of tuples from left relation(hashing relation),
        // that matches with the currentRightTuple
        private List leftListForCurrentKey;

        // keeping the index of leftListForCurrentKey
        private int leftListIndex = 0;

        /**
         * This method should be guaranteed to create a {@link JoinedTuple}. If a tuple can't be
         * created, caller should determine that before calling this method.
         * Additionally, this method should clear everything if everything related to
         * currentRightTuple is processed.
         */
        private void progressProbing() {
            Object key = this.currentProbingTuple.getKey();
            // we have interchanged original iterators based on the join type.
            // that should be taken into consideration when creating the JoinedTuple
            Object left = joinType.equals(CommunicationContext.JoinType.LEFT) ? this.currentProbingTuple.getValue() : leftListForCurrentKey.get(leftListIndex);
            Object right = joinType.equals(CommunicationContext.JoinType.LEFT) ? leftListForCurrentKey.get(leftListIndex) : this.currentProbingTuple.getValue();
            this.nextJoinTuple = JoinedTuple.of(key, left, right);
            leftListIndex++;
            // if end of the list has reached, reset everything!
            if (leftListIndex == leftListForCurrentKey.size()) {
                currentProbingTuple = null;
                leftListForCurrentKey = null;
                leftListIndex = 0;
            }
        }

        /**
         * This method iterates through the right relation(probing relation).
         */
        private void doProbing() {
            // if there is a non null nextJoinTuple, no need of proceeding
            while (this.nextJoinTuple == null) {
                // hashed list and still in the middle of combining that list
                if (this.currentProbingTuple == null) {
                    if (probingRelation.hasNext()) {
                        this.currentProbingTuple = probingRelation.next();
                        this.leftListForCurrentKey = this.keyHash.get(currentProbingTuple.getKey());
                        if (this.leftListForCurrentKey == null) {
                            // handle left and right joins here
                            if (joinType.equals(CommunicationContext.JoinType.LEFT)) {
                                this.nextJoinTuple = JoinedTuple.of(currentProbingTuple.getKey(), currentProbingTuple.getValue(), null);
                            } else if (joinType.equals(CommunicationContext.JoinType.RIGHT)) {
                                this.nextJoinTuple = JoinedTuple.of(currentProbingTuple.getKey(), null, currentProbingTuple.getValue());
                            }
                            // any join : We are done with currentProbingTuple
                            this.currentProbingTuple = null;
                        } else {
                            progressProbing();
                        }
                    } else {
                        // right iterator has reached to an end for current HashMap.
                        if (!hashingDone) {
                            // clear current hash and reset the right iterator
                            doHashing();
                            probingRelation.reset();
                        } else {
                            // end of join operation. Yay!
                            break;
                        }
                    }
                } else {
                    progressProbing();
                }
            }
        }

        @Override
        public boolean hasNext() {
            return this.nextJoinTuple != null;
        }

        @Override
        public JoinedTuple next() {
            if (!hasNext()) {
                throw new Twister2RuntimeException("Join operation has reached to an end. " + "Use hasNext() to check the status.");
            }
            JoinedTuple currentJoinTuple = nextJoinTuple;
            nextJoinTuple = null;
            // create the next JoinTuple before returning
            doProbing();
            return currentJoinTuple;
        }
    };
}

Example 53 with Twister2RuntimeException

use of edu.iu.dsc.tws.api.exceptions.Twister2RuntimeException in project twister2 by DSC-SPIDAL.

the class LocalSubmitter method submitJob.

/**
 * This method can be used to run a {@link Twister2Job} with default configurations.
 * Additional configurations can be defined/overridden by passing the {@link Config} object.
 */
public static Twister2JobState submitJob(Twister2Job twister2Job, Config config) {
    Twister2JobState state = new Twister2JobState(false);
    if (!prepared) {
        prepare();
    }
    Config newConfig = overrideConfigs(config);
    CyclicBarrier cyclicBarrier = new CyclicBarrier(twister2Job.getNumberOfWorkers());
    List<Thread> threads = new ArrayList<>();
    for (int i = 0; i < twister2Job.getNumberOfWorkers(); i++) {
        Thread thread = startWorker(twister2Job, newConfig, i, cyclicBarrier);
        threads.add(thread);
    }
    for (Thread workerThred : threads) {
        try {
            workerThred.join();
        } catch (InterruptedException e) {
            failed = true;
            fault = e;
        }
    }
    if (failed) {
        LOG.log(Level.SEVERE, "Job failed unexpectedly", fault);
        state.setJobstate(DriverJobState.FAILED);
        if (fault instanceof Exception) {
            state.setCause((Exception) fault);
        } else {
            state.setCause(new Twister2RuntimeException(fault));
        }
    } else {
        state.setJobstate(DriverJobState.COMPLETED);
    }
    // reset the local state for next job
    failed = false;
    fault = null;
    return state;
}

Example 54 with Twister2RuntimeException

use of edu.iu.dsc.tws.api.exceptions.Twister2RuntimeException in project twister2 by DSC-SPIDAL.

the class LocalSubmitter method startWorker.

/**
 * This method starts a new worker instance on a separate thread.
 */
private static Thread startWorker(Twister2Job twister2Job, Config config, int workerId, CyclicBarrier cyclicBarrier) {
    Thread.UncaughtExceptionHandler hndler = (th, ex) -> {
        failed = true;
        fault = ex;
    };
    LocalClassLoader localClassLoader = new LocalClassLoader(LocalSubmitter.class.getClassLoader());
    localClassLoader.addJobClass(twister2Job.getWorkerClass());
    try {
        Object o = localClassLoader.loadClass(MockWorker.class.getName()).getConstructor(twister2Job.getClass(), config.getClass(), Integer.class, CyclicBarrier.class).newInstance(twister2Job, config, workerId, cyclicBarrier);
        Thread thread = new Thread((Runnable) o);
        thread.setName("worker-" + workerId);
        thread.setUncaughtExceptionHandler(hndler);
        thread.start();
        return thread;
    } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException e) {
        throw new Twister2RuntimeException("Failed to start the worker thread", e);
    }
}

Example 55 with Twister2RuntimeException

use of edu.iu.dsc.tws.api.exceptions.Twister2RuntimeException in project twister2 by DSC-SPIDAL.

the class LocalSubmitter method prepare.

/**
 * This method will create a mock config dir and call {@link LocalSubmitter#prepare(String)}
 */
private static LocalSubmitter prepare() {
    try {
        File tempDir = Files.createTempDirectory(UUID.randomUUID().toString()).toFile();
        // create standalone and common directory
        File commonConfig = new File(tempDir, "common");
        File standaloneConfig = new File(tempDir, "standalone");
        List<File> directories = new ArrayList<>();
        directories.add(commonConfig);
        directories.add(standaloneConfig);
        List<File> files = new ArrayList<>();
        for (String f : FILES_LIST) {
            files.add(new File(commonConfig, f));
            files.add(new File(standaloneConfig, f));
        }
        File loggerProperties = new File(commonConfig, "logger.properties");
        files.add(loggerProperties);
        directories.forEach(File::mkdir);
        for (File file : files) {
            file.createNewFile();
        }
        Files.write(loggerProperties.toPath(), FileConstants.getLoggerContent().getBytes());
        return prepare(tempDir.getAbsolutePath());
    } catch (IOException e) {
        throw new Twister2RuntimeException("Failed to create a mock config directory");
    }
}